# -*- coding: utf-8 -*-

"""
Description: To access the labels in .json file.
License_info: GPL
what to learn: os.remove; dump vs dumps are like the matlab's jsondecode
"""
import os
import json
from random import shuffle,randint
from keras.utils import to_categorical
from help_func.feature_transformation import *
from help_func.utilities import CLASS_NUM

def querylabel(ranges, form):
    assert (isinstance(ranges, tuple))
    assert (len(ranges) == 2)
    begining, ending = ranges
    assert (isinstance(begining, int))
    assert (isinstance(ending, int))
    # form = self.labeltable[filename]
    value = None
    for items in form:
        if begining >= items[0] and ending <= items[1]:
            value = items[2]
            break
    if value != None:
        return value
    else:
        print('[*ERROR] Sth wrong!')
        assert (0)
class dataset_operator():
    def __init__(self,labelpath=None,foldpath=None):
        self.labeltable = self.readlabels(labelpath)
        self.datatable = self.loaddatadirectory(foldpath)

    def readlabels(self,jpath=None):
        if jpath is None:
            jpath = os.path.join(os.getcwd(), 'indices', 'content.json')
        else:
            assert(os.path.exists(jpath))
            assert(os.path.isfile(jpath))
        with open(jpath) as json_labels:
            d = json.load(json_labels)
        lookuptable = {}
        for items in d:
            name = items[0]
            labels = []
            for units in items[1]['_TableRecords_']:
                unit = (units[0][0], units[0][1], units[1][0])
                labels.append(unit)
            lookuptable[name] = labels
        return lookuptable

    def loaddatadirectory(self,dpath=None):
        if dpath is None:
            dpath = os.path.join( os.getcwd(), 'data')
        assert (os.path.exists(dpath))
        assert (os.path.isdir(dpath))
        dname = os.listdir(dpath)
        dcpath = [os.path.join(dpath,i) for i in dname]
        datatable = tuple(zip(dname,dcpath))
        return datatable
        #thw output is a nested tuple.

#target: as soon as possible and use the generator+one-step+tb method to monitor other metrics
class buildTrainValtest():
    def __init__(self,datatable,labeltable):
        self.datatable = datatable
        self.labeltable = labeltable
        self.nclookupt = self.gen5foldsforOneRecording()
        print()
    # all about the baseline !!!!!!!!!!!!!
    # start from the end of the last work
    # isolated approaches
    # generator+one-step+tb method
    # oversample for imbalanced
    #
    #
    def gen5foldsforOneRecording(self,mode='fixed'):
        nclookupt = {}
        assert(mode in ('fixed','flexible'))
        if mode == 'fixed':
            for i in range(5):
                strr = 'fold_' +str(i)
                nclookupt[strr] = list(range( 12*i, 12*(i+1) ))
            return nclookupt
        else:
            a = list(range(60))
            shuffle(a)
            for i in range(5):
                strr = 'fold_' + str(i)
                nclookupt[strr] = a[12 * i: 12 * (i + 1)]
            return nclookupt

    def cutintopieces5folds(self,flag):
        assert(flag in range(5))
        print('It is about the non-cross-peopled 5folds cross validation method!')
        shift = 0.1
        testPieces = {}
        testPieces['Present'] = []
        testPieces['Absent'] = []
        testPieces['Weak'] = []
        testPieces['Uncertain'] = []
        trainPieces = {}
        trainPieces['Present'] = []
        trainPieces['Absent'] = []
        trainPieces['Weak'] = []
        trainPieces['Uncertain'] = []
        strr = 'fold_'+str(flag)
        testrange = self.nclookupt[strr]
        for name, path in self.datatable:
            wavsignal, fs = read_wav_data(path)
            image = SimpleMfccFeatures(wavsignal, fs, shift)
            pwidth = int(1 / shift)
            for i in range(60):
                piece = image[i * pwidth:(i + 1) * pwidth, :]
                piece = piece.reshape(piece.shape[0], piece.shape[1], 1)
                tag = querylabel((i, i + 1), self.labeltable[name])
                assert (tag in ('Present', 'Absent', 'Weak', 'Uncertain'))
                if i in testrange:
                    testPieces[tag].append((piece, tag))
                else:
                    trainPieces[tag].append((piece, tag))
        for i in ('Present', 'Absent', 'Weak', 'Uncertain'):
            shuffle(trainPieces[i])
            shuffle(testPieces[i])
        testcnts = self.setCounting(testPieces)
        #join together
        testoverall = []
        trainoverall = []
        for i in ('Present', 'Absent', 'Weak', 'Uncertain'):
            testoverall = testoverall + testPieces[i]
            trainoverall = trainoverall + trainPieces[i]
        assert(len(testoverall) == sum(testcnts))
        shuffle(testoverall)
        self.trainPieces = trainPieces
        self.trainList = trainoverall
        self.testList = testoverall

    def cutintopiecesLoo(self,flag):
        '''
        This function is just form the base of the training,validation and testing data
        :param flag:
        :param ddir:
        :return: training data plus labels, validation data plus labels and testing data plus labels
        '''
        assert(flag in range(60))
        print('It is about the non-cross-peopled leave-one-out validation method!')
        shift = 0.1  # At present, shift should keep equal to wlen otherwise the last pieces is shorter than else.
        testPieces = {}
        testPieces['Present'] = []
        testPieces['Absent'] = []
        testPieces['Weak'] = []
        testPieces['Uncertain'] = []
        trainPieces = {}
        trainPieces['Present'] = []
        trainPieces['Absent'] = []
        trainPieces['Weak'] = []
        trainPieces['Uncertain'] = []

        for name,path in self.datatable:
            wavsignal, fs = read_wav_data(path)
            image = SimpleMfccFeatures(wavsignal, fs, shift)
            pwidth = int(1 / shift)
            for i in range(60):
                piece = image[i * pwidth:(i + 1) * pwidth, :]
                piece = piece.reshape(piece.shape[0], piece.shape[1], 1)
                tag = querylabel((i, i + 1), self.labeltable[name])
                assert (tag in ('Present', 'Absent', 'Weak', 'Uncertain'))
                if i != flag:
                    trainPieces[tag].append((piece, tag))
                else:
                    testPieces[tag].append((piece, tag))
        for i in ('Present', 'Absent', 'Weak', 'Uncertain'):
            shuffle(trainPieces[i])
            shuffle(testPieces[i])
        testcnts = self.setCounting(testPieces)
        #join together
        testoverall = []
        trainoverall = []
        for i in ('Present', 'Absent', 'Weak', 'Uncertain'):
            testoverall = testoverall + testPieces[i]
            trainoverall = trainoverall + trainPieces[i]
        assert(len(testoverall) == sum(testcnts))
        shuffle(testoverall)
        self.trainPieces = trainPieces
        self.trainList = trainoverall
        self.testList = testoverall

    def getNonRepetitiveData(self,n_start, type='train'):
        assert(isinstance(n_start,int))
        assert(type in ('train','test'))
        dataChecked = self.testList if type == 'test' else self.trainList
        num = len(dataChecked)
        data_input, data_label = dataChecked[n_start%num]
        maplist = {'Present': 0, 'Absent': 1, 'Weak': 1, 'Uncertain': 1}
        data_label = maplist[data_label]
        return data_input, data_label

    def generatorSetting(self,batch_size=32):
        #1. get the counts of training data
        traincnts = self.setCounting(self.trainPieces)
        assigncnts = [ myround(i/float(sum(traincnts))*batch_size) for i in traincnts]
        if sum(assigncnts) == batch_size:
            pass
        else:
            assert(abs(sum(assigncnts)-batch_size)==1)
            if sum(assigncnts)>32:
                assigncnts[0] = assigncnts[0] - 1
            else:
                assigncnts[0] = assigncnts[0] + 1
        iteration_per_epoch =  min([aggrega//single for aggrega,single in zip(traincnts, assigncnts)])
        traincnts[1] = traincnts[1] + traincnts[2] + traincnts[3]
        del traincnts[2:4]
        classWeights = [sum(traincnts) / (CLASS_NUM * el) for el in traincnts]
        self.sampleoffset = assigncnts
        self.iteration_per_epoch = iteration_per_epoch
        self.classWeights = classWeights

    def getData(self, n_start):  # Due to the class weight, samples in every batch does not need to be class-equal.
        assert(isinstance(n_start,int))
        data_input = []
        data_label = []
        for tag,offset in zip(('Present', 'Absent', 'Weak', 'Uncertain'),self.sampleoffset):
            begining = n_start * offset
            temp = self.trainPieces[tag][begining:begining+offset]
            tempunzip = list(map(list,zip(*temp)))
            data_input = data_input + tempunzip[0]
            data_label = data_label + tempunzip[1]
        together = list(zip(data_input,data_label))
        shuffle(together)
        data_input,data_label = zip(*together)
        maplist = { 'Present':0, 'Absent':1, 'Weak':1, 'Uncertain':1 }
        data_label = [maplist[i] for i in data_label]
        return data_input, data_label

    def data_genetator(self): #This generator is only in charge of one single epoch
        for i in ('Present', 'Absent', 'Weak', 'Uncertain'):
            shuffle(self.trainPieces[i])
        while True:
            ran_num = randint(0, self.iteration_per_epoch-1)  # 获取一个随机数
            X,y = self.getData(n_start=ran_num)

            # from collections import Counter
            # a = list(Counter(y).items())

            X = np.array(X)
            y = np.array(y)
            yield X, to_categorical(y, num_classes=CLASS_NUM)  # 功能只是转成独热编码
        pass

    def setCounting(self,dataset):
        assert(isinstance(dataset,dict))
        assert(all([i in ('Present', 'Absent', 'Weak', 'Uncertain') for i in dataset.keys()]))
        return [len(dataset['Present']), len(dataset['Absent']), len(dataset['Weak']),len(dataset['Uncertain'])]

if __name__ == '__main__':
    ##test the first class
    # jpath = os.path.join(os.getcwd(), 'indices', 'content.json')
    # dpath = os.path.join(os.getcwd(), 'data')
    # opit = dataset_operator(jpath)
    # # print(opit.querylabel('huangjun-bowel08211.wav',(59,60)))
    # opit.loaddatadirectory(dpath)
    # opit.write2json()

    #test feature abstraction
    # path = os.path.join( os.getcwd(), 'data','f_dingcong-bowel08201.wav' )
    # wavsignal, fs = read_wav_data(path)

    #Test jointly

    opit = dataset_operator()
    shifter = buildTrainValtest(opit.datatable,opit.labeltable)
    shifter.cutintopieces(flag=3)
    # shifter.functionCheck()
    # print(shifter.setCounting(shifter.trainPieces))
    # print(shifter.getData())
    shifter.generatorSetting(batch_size=16)
    yielddata = shifter.data_genetator()
    for i in yielddata:
        qq,dd = i
        a=1


def histogram(args):
    print(args)
    for i in args:
        for j in range(i):
            print("*",end=" ")
        print("\n")
       # a=["*" for i in args for j in range(i)]
#a=[4,9,7]
n=list(map(int,input("Enter a list of Values").split(' ')))
histogram(n)


import os
import numpy as np
import matplotlib.pyplot as plt
import mne
from numpy.random import randn
from scipy import stats as stats
import pandas as pd
import seaborn as sbn

import mne

#Set folders and files
channelName = 'MEG1311'
dataDir = '/media/NAS/lpower/BetaSourceLocalization/restData/'+ channelName +'/'
subjectsDir = '/home/timb/camcan/subjects/'
stcPrefix = 'rest_DICS_stcGAvg-lh.stc'

stcFileName = os.path.join(dataDir, stcPrefix)
StatsFileName = os.path.join(dataDir, 'ROIdata/BF_rest_stats.csv')
outFileName = os.path.join(dataDir, 'DICS_rest_tstat')

plotOK = True

#################################
# Analysis starts here

# Note: see Automatic parcellation of human cortical gyri and sulci using standard anatomical nomenclature,
#
#	Labels of interest are (per hemisphere, indexed from 1, aparc.a2009s)
#		28: Postcentral gyrus
#       29: Precentral gyrus
#       15: Middle frontal gyrus
#       16: Superior frontal gyrus
#       26: Supramarginal gyrus
#       27: Superior parietal lobule
#
# labels list is not in the same indexed order, must find these regions by name
ROIs = [u'G_postcentral-lh', u'G_postcentral-rh',
			u'G_precentral-lh', u'G_precentral-rh',
			u'G_front_middle-lh', u'G_front_middle-rh',
                        u'G_front_sup-lh', u'G_front_sup-rh',
                        u'G_pariet_inf-Supramar-lh',u'G_pariet_inf-Supramar-rh',
			u'G_parietal_sup-lh', u'G_parietal_sup-rh']

#Read ROI time course for all conditions from numpy file
ROIData_df = pd.read_csv(StatsFileName)

labels = mne.read_labels_from_annot('fsaverage', parc='aparc.a2009s', subjects_dir=subjectsDir)
ROIlabels = []
for ROI in ROIs:
    print(ROI)
    b = [x for x in labels if x.name == ROI]
    ROIlabels.append(b[0])

#Make an array for STC data that has all zeroes. This will get filled with ROI data one label at a time
stcData = np.zeros((20484, 1))

#Pull t-stats out of ROI stats file
tstats = ROIData_df['t-stat'].tolist()
print(tstats)

#Loop over ROIs
for ctr in np.arange(len(ROIlabels)):
    thislabel = ROIlabels[ctr]
    if 'lh' in thislabel.name:
        theseVertices = thislabel.get_vertices_used()
    else:
        theseVertices = thislabel.get_vertices_used() + 10242
    thislabelValue = tstats[ctr]

    #Write the tstat value to each vertex in the region of interest
    for vertex in theseVertices:
        stcData[vertex] = thislabelValue

#Put this stc data into an fsaverage stc object
stc = mne.read_source_estimate(stcFileName)
stc.data = stcData
stc.save(outFileName)


# coding=utf-8
# author=UlionTse

from translators.apis import *

query_text1 = '季姬寂，集鸡，鸡即棘鸡。棘鸡饥叽，季姬及箕稷济鸡。👍👍👍'
query_text2 = """北国风光，千里冰封，万里雪飘。望长城内外，惟余莽莽；大河上下，顿失滔滔。山舞银蛇，原驰蜡象，欲与天公试比高。\n
须晴日，看红装素裹，分外妖娆。江山如此多娇，引无数英雄竞折腰。惜秦皇汉武，略输文采；唐宗宋祖，稍逊风骚。一代天骄，成吉思汗，只识弯弓射大雕。\n
俱往矣，数风流人物，还看今朝。\n
"""
query_text3 = 'All the past, a number of heroes, but also look at the present.'

html_text = """
<!DOCTYPE html>

<address>
Written by Donald Duck.

Visit us at:

Example.com

Box 564, Disneyland

USA
</address>

北国风光，千里冰封，万里雪飘。望长城内外，惟余莽莽；大河上下，顿失滔滔。山舞银蛇，原驰蜡象，欲与天公试比高。\n
须晴日，看红装素裹，分外妖娆。江山如此多娇，引无数英雄竞折腰。惜秦皇汉武，略输文采；唐宗宋祖，稍逊风骚。一代天骄，成吉思汗，只识弯弓射大雕。\n
俱往矣，数风流人物，还看今朝。\n

5%

123\t

hello world

import translators as ts

"""

@logger.catch
def test():
    for query_text in [query_text1,query_text2,query_text3]:
        print(alibaba(query_text))
        print(baidu(query_text))
        print(bing(query_text))
        print(deepl(query_text))
        print(google(query_text))
        print(sogou(query_text))
        print(tencent(query_text))
        print(yandex(query_text))
        print(youdao(query_text))

    for i in range(5):
        r = translate_html(html_text, to_language='zh', translator=google)
        print(i, r)

if __name__ == '__main__':
    test()


import logging

from pymongo import ReturnDocument
from telegram import ReplyKeyboardMarkup
from telegram import ReplyKeyboardRemove
from telegram import Update
from telegram.ext import CallbackContext
from telegram.ext import ConversationHandler

from thx_bot.commands import CHOOSING
from thx_bot.commands import TYPING_REPLY
from thx_bot.commands import user_data_to_str
from thx_bot.models.channels import Channel
from thx_bot.services.thx_api_client import get_asset_pool_info
from thx_bot.validators import only_chat_admin
from thx_bot.validators import only_if_channel_configured
from thx_bot.validators import only_in_private_chat

logger = logging.getLogger(__name__)

CONNECTION_SUCCESSFUL = "✨ ✨ ✨  Connection to THX is successful! You can now start using THX."
CONNECTION_FAILED = "⛔⛔⛔  Oops. Something is wrong with configuration. " \
                    "Please, check client id and secret"

OPTION_CLIENT_ID = "Client id"
OPTION_CLIENT_SECRET = "Client secret"
OPTION_POOL_ADDRESS = "Pool address"
REPLY_KEYBOARD = [
    [OPTION_CLIENT_ID, OPTION_CLIENT_SECRET, OPTION_POOL_ADDRESS],
    ['Done', 'Test Connection'],
]
REPLY_OPTION_TO_DB_KEY = {
    OPTION_CLIENT_ID.lower(): "client_id",
    OPTION_CLIENT_SECRET.lower(): "client_secret",
    OPTION_POOL_ADDRESS.lower(): "pool_address",
}
MARKUP = ReplyKeyboardMarkup(REPLY_KEYBOARD, one_time_keyboard=True)

@only_chat_admin
@only_in_private_chat
def start_setting_channel(update: Update, context: CallbackContext) -> int:
    reply_text = "🛠 Please, fill *ALL* items to start using THX API in your channel\.\n\n" \
                 "Hit *Done* button once you are done to test connection to THX API"
    update.message.reply_text(reply_text, parse_mode='MarkdownV2', reply_markup=MARKUP)

    return CHOOSING

@only_chat_admin
@only_in_private_chat
def regular_choice_channel(update: Update, context: CallbackContext) -> int:
    text = update.message.text.lower()
    # In case user typed incorrect answer
    if text not in REPLY_OPTION_TO_DB_KEY.keys():
        update.message.reply_text("Unknown choice. Please, click on inline buttons with choices")
        return ConversationHandler.END
    context.user_data['choice'] = text
    channel = Channel.collection.find_one_and_update(
        {'channel_id': context.user_data['channel_id']},
        {'$setOnInsert': {'channel_id': context.user_data['channel_id']}},
        upsert=True, return_document=ReturnDocument.AFTER,
    )
    if channel.get(REPLY_OPTION_TO_DB_KEY[text]):
        reply_text = (
            f"Your {text}? I already know the following about that: "
            f"{channel.get(REPLY_OPTION_TO_DB_KEY[text])}"
        )
    else:
        reply_text = f'Your {text}? Yes, please fill it!'
    update.message.reply_text(reply_text)

    return TYPING_REPLY

@only_chat_admin
@only_in_private_chat
def received_information_channel(update: Update, context: CallbackContext) -> int:
    text = update.message.text
    category = context.user_data['choice']
    context.user_data[category] = text.lower()
    del context.user_data['choice']

    channel = Channel.collection.find_one_and_update(
        {'channel_id': context.user_data['channel_id']},
        {'$set': {REPLY_OPTION_TO_DB_KEY[category]: text}},
        # Create new document in case there is none
        upsert=True,
        return_document=ReturnDocument.AFTER
    )
    update.message.reply_text(
        "Cool! Your configuration is:"
        f"{user_data_to_str(channel)}"
        "You can change client or secret any time.",
        reply_markup=MARKUP,
    )

    return CHOOSING

@only_chat_admin
@only_in_private_chat
def done_channel(update: Update, context: CallbackContext) -> int:
    if 'choice' in context.user_data:
        del context.user_data['choice']

    channel = Channel.collection.find_one({'channel_id': context.user_data['channel_id']})
    update.message.reply_text(
        f"Your configuration: {user_data_to_str(channel)}\nYou can change configuration of your"
        f" channel any time. Just hit /register_channel again!",
        reply_markup=ReplyKeyboardRemove(),
    )
    return ConversationHandler.END

@only_if_channel_configured
@only_chat_admin
@only_in_private_chat
def check_connection_channel(update: Update, context: CallbackContext) -> int:
    if 'choice' in context.user_data:
        del context.user_data['choice']
    channel = Channel.collection.find_one({'channel_id': context.user_data['channel_id']})
    status, __ = get_asset_pool_info(Channel(channel))
    if status == 200:
        update.message.reply_text(CONNECTION_SUCCESSFUL)
    else:
        update.message.reply_text(CONNECTION_FAILED)
    return ConversationHandler.END


import json
from botocore.vendored import requests
from python.shared import BeautifulSoup

def news(event, context):

    result = requests.get('https://www.ithome.com.tw/news?page=0')
    html = result.text
    soup = BeautifulSoup(html, 'html.parser')

    resp=[]

    posts = soup.find_all('div',class_='span4 channel-item')

    for item in posts:
        title = item.find('p', class_='title')
        url = title.a['href']
        title = title.a.text
        obj = item.find('a', class_='js-auto_break_title')
        resp.append({
            'title': title,
            'url': f'https://www.ithome.com.tw{url}',
            'description': item.find('div', class_='summary').text
        })

    posts = soup.find_all('div',class_='span4 channel-item no-margin-left')

    for item in posts:
        title = item.find('p', class_='title')
        url = title.a['href']
        title = title.a.text
        obj = item.find('a', class_='js-auto_break_title')
        resp.append({
            'title': title,
            'url': f'https://www.ithome.com.tw{url}',
            'description': item.find('div', class_='summary').text
        })

    response = {
        "statusCode": 200,
        "body": json.dumps(resp)
    }

    return response


###############################
# Index for exported data
#
# - handles Object -> Number conversions for FBX 7.3
# - system has room for 9999 of each object type.... should be enough
'''
	Numbering:
	# static:
	0 			= RootNode
	10 			= Document
	100 		= Pose

	# objects:
	10xxxx 		= Geometry
	11xxxx		= ShapeGeometry
	20xxxx 		= Mesh
	22xxxx		= Model::Null
	30xxxx 		= Mesh::LimbNode/Root
	31xxxx 		= NodeAttribute::LimbNode/Root
	32xxxx		= NodeAttribute::Null

	40xxxx 		= Material
	41xxxx 		= Texture
	42xxxx 		= Video

	# skins/shapes
	50xxxx 		= Skin
	51xxxx 		= Cluster
	60xxxx		= Shape
	61xxxx		= ShapeChannel

	#animations:
	800000		= AnimationStack
	810000		= AnimationLayer

	# animation curve data may need a bigger index
	# - thanks to every vector component curve needing its own unique id :(
	1000000		= AnimationCurveNode
	2000000		= AnimationCurve
'''

# index vars
index_fbxModels = []
index_fbxBones = []

index_fbxNulls = []

index_fbxMaterials = []
index_fbxTextures = []

# not sure about this
# - some files include the skeleton as a null object,
# but it doesn't seem to affect anything
#index_fbxSkeletons = []

index_fbxSkins = []
index_fbxClusters = []

index_fbxShapeGeom = []
index_fbxShapes = []
index_fbxShapeChannels = []

# animations
index_fbxAnimStacks = []
index_fbxAnimLayers = []
index_fbxAnimCurveNodes = []
index_fbxAnimCurves = []

# moved other exporter data since it was already global

fbx_meshes = []  # ob_meshes
fbx_bones = []  # ob_bones
fbx_nulls = []  # ob_nulls

fbx_poses = []  # pose_items

fbx_actions = []  # temp_actions
fbx_taggedactions = []  # tagged_actions

ob_anim_lists = []

# getters for id codes used to identify objects:
def get_fbx_GeomID(obname):
    for i in range(len(index_fbxModels)):
        if obname == index_fbxModels[i]:
            return 100000 + i
    return 0

def get_fbx_ShapeGeomID(obname):
    for i in range(len(index_fbxModels)):
        if obname == index_fbxModels[i]:
            return 110000 + i
    return 0

def get_fbx_MeshID(obname):
    for i in range(len(index_fbxModels)):
        if obname == index_fbxModels[i]:
            return 200000 + i
    return 0

def get_fbx_NullModelID(innull):
    for i in range(len(index_fbxNulls)):
        if innull == index_fbxNulls[i]:
            return 220000 + i
    return 0

def get_fbx_BoneID(inbone):
    for i in range(len(index_fbxBones)):
        if inbone == index_fbxBones[i]:
            return 300000 + i
    return 0

def get_fbx_BoneAttributeID(inbone):
    for i in range(len(index_fbxBones)):
        if inbone == index_fbxBones[i]:
            return 310000 + i
    return 0

def get_fbx_NullAttributeID(innull):
    for i in range(len(index_fbxNulls)):
        if innull == index_fbxNulls[i]:
            return 310000 + i
    return 0

def get_fbx_DeformerSkinID(instring):
    for i in range(len(index_fbxSkins)):
        if instring == index_fbxSkins[i]:
            return 500000 + i
    return 0

def get_fbx_DeformerClusterID(instring):
    for i in range(len(index_fbxClusters)):
        if instring == index_fbxClusters[i]:
            return 510000 + i
    return 0

def get_fbx_MaterialID(mat):
    for i in range(len(index_fbxMaterials)):
        if mat == index_fbxMaterials[i]:
            return 400000 + i
    return 0

def get_fbx_TextureID(texname):
    for i in range(len(index_fbxTextures)):
        if texname == index_fbxTextures[i]:
            return 410000 + i
    return 0

def get_fbx_VideoID(texname):
    for i in range(len(index_fbxTextures)):
        if texname == index_fbxTextures[i]:
            return 420000 + i
    return 0

def get_fbx_AnimStackID(animname):
    for i in range(len(index_fbxAnimStacks)):
        if animname == index_fbxAnimStacks[i]:
            return 800000 + i
    return 0

def get_fbx_AnimLayerID(animname):
    for i in range(len(index_fbxAnimStacks)):
        if animname == index_fbxAnimStacks[i]:
            return 810000 + i
    return 0

def get_fbx_AnimCurveNodeID(animname):
    for i in range(len(index_fbxAnimCurveNodes)):
        if animname == index_fbxAnimCurveNodes[i]:
            return 1000000 + i
    return 0

def get_fbx_AnimCurveID(animname):
    for i in range(len(index_fbxAnimCurves)):
        if animname == index_fbxAnimCurves[i]:
            return 2000000 + i
    return 0

def clear_fbxData():
    print("Exporter: Deleting temp files....")

    del index_fbxModels[:]
    del index_fbxBones[:]
    del index_fbxNulls[:]
    del index_fbxMaterials[:]
    del index_fbxTextures[:]
    del index_fbxSkins[:]
    del index_fbxClusters[:]
    del index_fbxShapeGeom[:]
    del index_fbxShapes[:]
    del index_fbxShapeChannels[:]
    del index_fbxAnimStacks[:]
    del index_fbxAnimCurveNodes[:]
    del index_fbxAnimCurves[:]

    del fbx_meshes[:]
    del fbx_bones[:]
    del fbx_nulls[:]
    del fbx_poses[:]
    del fbx_actions[:]
    del fbx_taggedactions[:]


# python3

import sys
#import threading

def compute_height(n, parents):

    # Build parent tree using dict
    #[4 -1 4 1 1]  --> {4: [0, 2], -1: [1], 1: [3, 4]}
    parent_tree = {}
    for i in range(n):
        if parents[i] in parent_tree:
            parent_tree[parents[i]].append(i)
        else:
            parent_tree[parents[i]] = [i]

    # Breadth First traversal
    # Queue
    if parent_tree :
        nodes = [(parent_tree[-1][0], 1)] # [(root_node, height)]
        height = 1

    while nodes:
        node = nodes.pop(0)
        if node[0] in parent_tree:
            for child in parent_tree[node[0]]:
                # use append instead of insert (~2 times slower)
                # append child node to queue
                nodes.append((child, node[1] + 1))
                # update height if needed
                height = max(height, node[1] + 1)

    return height

def main():
    '''
    Height of d-ary tree
    ex: n = 5 nodes
        parents = [4 -1 4 1 1], -1 is root node
    Good job! (Max time used: 1.79/3.00, max memory used: 31830016/536870912.)
    '''
    n = int(input())
    parents = list(map(int, input().split()))

#    with open('./Programming-Assignment-1/tree_height/tests/01', 'r') as infile:
#        lines = infile.readlines()
#    n = int(lines[0])
#    parents = list(map(int, lines[1].split()))

    print(compute_height(n, parents))

main()

# In Python, the default limit on recursion depth is rather low,
# so raise it here for this problem. Note that to take advantage
# of bigger stack, we have to launch the computation in a new thread.

#sys.setrecursionlimit(10**7)  # max depth of recursion
#threading.stack_size(536870912)#(2**27)   # new thread will get stack of such size
#threading.Thread(target=main).start()

# *** fonctionne mais trop lent ***
#    if parent_tree :
#        nodes = [(parent_tree[-1][0], 1)] # tuple (curr_node, curr_height)
#        height = 1
#
#    while nodes:
#        node = nodes.pop(0)
#        if node[0] in parent_tree:
#            #height += 1
#            for child in parent_tree[node[0]]:
#                nodes.insert(0, (child, node[1] + 1))
#                height = max(height, node[1] + 1)


import multiprocessing
import os
import unittest
from typing import List, Any
from unittest.mock import mock_open, patch

from nix_review.app import main

TEST_ROOT = os.path.dirname(os.path.realpath(__file__))
DEBUG = False

class IgnoreArgument:
    def __repr__(self) -> str:
        return "(ignored)"

def read_asset(asset: str) -> str:
    with open(os.path.join(TEST_ROOT, "assets", asset)) as f:
        return f.read()

class MockError(Exception):
    pass

class Mock:
    def __init__(self, test, arg_spec):
        self.test = test
        self.arg_spec_iterator = iter(arg_spec)
        self.expected_args = []
        self.ret = None

    def __iter__(self):
        return self

    def __call__(self, *args, **kwargs):
        (self.expected_args, self.ret) = next(self.arg_spec_iterator)
        if DEBUG:
            print(f"({self.expected_args}) -> {self.ret}")
        if self.expected_args is IgnoreArgument:
            return self.ret
        if len(args[0]) == len(self.expected_args):
            for (i, arg) in enumerate(self.expected_args):
                if arg is IgnoreArgument:
                    args[0][i] = IgnoreArgument
        if self.expected_args != args[0]:
            raise MockError(f"expected {self.expected_args}\n got {args[0]}")
        return self.ret

PKG_LIST = read_asset("package_list_after.txt").encode("utf-8")

def local_eval_cmds():
    return [
        (IgnoreArgument, mock_open(read_data=read_asset("github-pull-1.json"))()),
        (
            IgnoreArgument,
            mock_open(read_data=read_asset("github-pull-1-statuses.json"))(),
        ),
        (
            [
                "git",
                "fetch",
                "--force",
                "https://github.com/NixOS/nixpkgs",
                "master:refs/nix-review/0",
                "pull/1/head:refs/nix-review/1",
            ],
            0,
        ),
        (["git", "rev-parse", "--verify", "refs/nix-review/0"], b"hash1"),
        (["git", "rev-parse", "--verify", "refs/nix-review/1"], b"hash2"),
        (["git", "worktree", "add", "./.review/pr-1", "hash1"], 0),
        (
            [
                "nix-env",
                "-f",
                "./.review/pr-1",
                "-qaP",
                "--xml",
                "--out-path",
                "--show-trace",
            ],
            b"<items></items>",
        ),
        (["git", "merge", "--no-commit", "hash2"], 0),
        (
            [
                "nix-env",
                "-f",
                "./.review/pr-1",
                "-qaP",
                "--xml",
                "--out-path",
                "--show-trace",
                "--meta",
            ],
            PKG_LIST,
        ),
    ]

def borg_eval_cmds() -> List[Any]:
    return [
        (IgnoreArgument, mock_open(read_data=read_asset("github-pull-37200.json"))()),
        (
            IgnoreArgument,
            mock_open(read_data=read_asset("github-pull-37200-statuses.json"))(),
        ),
        (
            "https://gist.githubusercontent.com/GrahamcOfBorg/4c9ebc3e608308c6096202375b0dc902/raw/",
            read_asset("gist-37200.txt").encode("utf-8").split(b"\n"),
        ),
        (
            [
                "git",
                "fetch",
                "--force",
                "https://github.com/NixOS/nixpkgs",
                "master:refs/nix-review/0",
                "pull/37200/head:refs/nix-review/1",
            ],
            0,
        ),
        (["git", "rev-parse", "--verify", "refs/nix-review/0"], b"hash1"),
        (["git", "rev-parse", "--verify", "refs/nix-review/1"], b"hash2"),
        (["git", "worktree", "add", "./.review/pr-37200", "hash1"], 0),
        (["git", "merge", "--no-commit", "hash2"], 0),
        (["nix", "eval", "--raw", "nixpkgs.system"], b"x86_64-linux"),
    ]

build_cmds = [
    (
        ["nix", "eval", "--json", IgnoreArgument],
        # hack to make sure the path exists
        b'{"pong3d": {"exists": true, "broken": false, "path": "'
        + __file__.encode("utf8")
        + b'"}}',
    ),
    (
        [
            "nix-shell",
            "--no-out-link",
            "--keep-going",
            "--max-jobs",
            str(multiprocessing.cpu_count()),
            "--option",
            "build-use-sandbox",
            "true",
            "--run",
            "true",
            "--builders",
            "ssh://joerg@10.243.29.170 aarch64-linux",
            "-p",
            "pong3d",
        ],
        0,
    ),
    (["nix-shell", "-p", "pong3d"], 0),
    (["git", "worktree", "prune"], 0),
]

class TestStringMethods(unittest.TestCase):
    def setUp(self):
        os.chdir(os.path.join(TEST_ROOT, "assets/nixpkgs"))

    @patch("urllib.request.urlopen")
    @patch("subprocess.Popen")
    @patch("subprocess.check_call")
    @patch("subprocess.check_output")
    def test_pr_local_eval(
        self, mock_check_output, mock_check_call, mock_popen, mock_urlopen
    ):
        effects = Mock(self, local_eval_cmds() + build_cmds)
        mock_check_call.side_effect = effects
        mock_popen.stdout.side_effect = effects
        mock_check_output.side_effect = effects
        mock_urlopen.side_effect = effects

        main(
            "nix-review",
            [
                "--build-args",
                '--builders "ssh://joerg@10.243.29.170 aarch64-linux"',
                "pr",
                "1",
            ],
        )

    @patch("urllib.request.urlopen")
    @patch("subprocess.Popen")
    @patch("subprocess.check_call")
    @patch("subprocess.check_output")
    def test_pr_borg_eval(
        self, mock_check_output, mock_check_call, mock_popen, mock_urlopen
    ):
        effects = Mock(self, borg_eval_cmds() + build_cmds)
        mock_check_call.side_effect = effects
        mock_popen.stdout.side_effect = effects
        mock_check_output.side_effect = effects
        mock_urlopen.side_effect = effects

        main(
            "nix-review",
            [
                "--build-args",
                '--builders "ssh://joerg@10.243.29.170 aarch64-linux"',
                "pr",
                "37200",
            ],
        )

if __name__ == "__main__":
    unittest.main(failfast=True)


import re
import time
import random
import numpy as np
import pandas as pd
import requests
from bs4 import BeautifulSoup

# スクレイピングClassの実装
class Model():

    def __init__(self, name, root_url, target_url, target_save, target_params, target_contents):
        self.name = name
        self.root_url = root_url
        self.target_url = target_url
        self.target_save = target_save
        self.target_params = target_params
        self.target_contents = target_contents
        self.soup = object
        self.content_list = []

    def run(self):
        #Requestsを使って、webから取得
        res = requests.get(self.target_url)

        #要素を抽出
        self.soup = BeautifulSoup(res.content, 'lxml')

        #HTMLファイルとして保存したい場合はファイルオープンして保存
        with open('files/'+ self.name + '.html', mode='w', encoding = 'utf-8') as fw:
            fw.write(self.soup.prettify())

        #「target_contents」の項目を全て取得
        for target_content in self.target_contents:
            content = target_content['selector']

            # エレメント取得
            elems = self.soup.select(content)

            # 初回のみコンテンツを格納する配列を初期化
            if len(self.content_list) < len(elems):

                # Jsonオブジェクトを取得した要素数分確保
                self.content_list = [ {} for i in range(len(elems)) ]

            # 各要素を配列に格納
            self.setContents(elems, target_content['type'], target_content['elem'])

        #「params」の項目を各オブジェクトに付与する
        for i in range(len(self.target_params)):
            param = self.target_params[i]
            key = [key for key in param][0]

            for j in range(len(self.content_list)):
                self.content_list[j].update({ key:param[key] })

        return self.content_list;

    def setContents(self, elems, _type, prop):

        # Jsonオブジェクトに変換しリストに格納する
        for i in range(len(elems)):

            elem = elems[i].getText() if _type == "text" else self.replace(elems[i][_type])
            print(elem)

            self.content_list[i].update({ prop:elem })

    def replace(self, item):
        #相対urlを絶対urlに置換するパターン
        pattern = '^http(s)'

        if not re.match(pattern, item):

            if not re.match('^/', item):

                item = '/' + item
            item = self.root_url + item
        return item


# 集合的运算

# 在对集合做运算时，不会影响原来的集合，而是返回一个运算结果
s = {1, 2, 3, 4, 5}
s2 = {3, 4, 5, 6, 7}

# & 交集运算
# 输出：{3, 4, 5}
result = s & s2
print(result)

# | 并集运算
result = s | s2  # {1,2,3,4,5,6,7}
# 输出：{1,2,3,4,5,6,7}
print(result)

# - 差集
result = s - s2
# 输出：{1, 2}
print(result)

# ^ 异或集 获取只在一个集合中出现的元素
result = s ^ s2
# 输出：{1, 2, 6, 7}
print(result)

"""
<= 检查一个集合是否是另一个集合的子集
如果a集合中的元素全部都在b集合中出现，那么a集合就是b集合的子集，b集合是a集合超集
"""
a = {1, 2, 3}
b = {1, 2, 3, 4, 5}

result = a <= b
# 输出：True
print(result)
result = {1, 2, 3, 4, 5} <= {1, 2, 3}
# 输出：False
print(result)

"""
< 检查一个集合是否是另一个集合的真子集
如果超集b中含有子集a中所有元素，并且b中还有a中没有的元素，则b就是a的真超集，a是b的真子集
"""
result = {1, 2, 3} < {1, 2, 3}  # False

"""
>= 检查一个集合是否是另一个的超集
> 检查一个集合是否是另一个的真超集
"""
result = {1, 2, 3, 4, 5} >= {1, 2, 3}
# 输出：True
print(result)


import collections

def main():
    od = collections.OrderedDict()

    od["first added"] = 0
    od["Colleen"] = 29
    od["Hao"] = 28
    od["May"] = 52
    od["Qiang"] = 56
    od["last added"] = 99

    printOD(od)

    # popitem(), default pop the last added
    print(od.popitem(last=True))
    print(od.popitem(last=False))

    od.move_to_end("Hao")
    printOD(od)

    od.move_to_end("Hao", last=False)
    printOD(od)

    print(od.get("non exist"))

    first = next(iter(od))
    x = 0

def printOD(od: collections.OrderedDict):
    print("------------------------------")
    for k, v in od.items():
        print("{} : {}".format(k, v))

if __name__ == "__main__":
    main()


import stripe

from django.conf import settings

def set_stripe_api_key():
    stripe.api_key = settings.STRIPE_API_KEY_SECRET

def create_charge_for_order(order, token):
    assert order.payment_required()
    set_stripe_api_key()
    return stripe.Charge.create(
        amount=order.cost_pence_incl_vat(),
        currency='gbp',
        description=f'PyCon UK 2017 order {order.order_id}',
        statement_descriptor=f'PyCon UK 2017 {order.order_id}',
        source=token,
    )

def refund_charge(charge_id):
    set_stripe_api_key()
    stripe.Refund.create(charge=charge_id)


import torch,cv2,random,os,time
from PIL import Image
import torch.nn as nn
from torch.autograd import Variable
import numpy as np
import pickle as pkl
import argparse
import threading, queue
from torch.multiprocessing import Pool, Process, set_start_method
from darknet import Darknet
from imutils.video import WebcamVideoStream,FPS
# from camera import write
import win32com.client as wincl       #### Python's Text-to-speech (tts) engine for windows, multiprocessing
speak = wincl.Dispatch("SAPI.SpVoice")    #### This initiates the tts engine

torch.multiprocessing.set_start_method('spawn', force=True)
from torchvision import transforms as trans
##  Setting up torch for gpu utilization
if torch.cuda.is_available():
    torch.backends.cudnn.enabled = True
    torch.backends.cudnn.benchmark = True
    torch.backends.cudnn.deterministic = True
    torch.set_default_tensor_type('torch.cuda.FloatTensor')

def letterbox_image(img, inp_dim):
    '''resize image with unchanged aspect ratio using padding'''
    img_w, img_h = img.shape[1], img.shape[0]
    w, h = inp_dim
    new_w = int(img_w * min(w/img_w, h/img_h))
    new_h = int(img_h * min(w/img_w, h/img_h))
    resized_image = cv2.resize(img, (new_w,new_h), interpolation = cv2.INTER_CUBIC)
    canvas = np.full((inp_dim[1], inp_dim[0], 3), 128)
    canvas[(h-new_h)//2:(h-new_h)//2 + new_h,(w-new_w)//2:(w-new_w)//2 + new_w,  :] = resized_image

    return canvas

def load_classes(namesfile):
    fp = open(namesfile, "r")
    names = fp.read().split("\n")[:-1]
    return names

def prep_image(img, inp_dim):
    """
    Prepare image for inputting to the neural network.
    Returns a Variable
    """
    orig_im = img
    dim = orig_im.shape[1], orig_im.shape[0]
    img = (letterbox_image(orig_im, (inp_dim, inp_dim)))
    img_ = img[:, :, ::-1].transpose((2, 0, 1)).copy()
    img_ = torch.from_numpy(img_).float().div(255.0).unsqueeze(0)
    return img_, orig_im, dim

class ObjectDetection:
    def __init__(self, id):
        # self.cap = cv2.VideoCapture(id)
        # self.cap = WebcamVideoStream(src = id).start()
        self.cfgfile = "cfg/yolov4.cfg"
        self.weightsfile = "yolov4.weights"
        self.confidence = float(0.09)
        self.nms_thesh = float(0.2)
        self.num_classes = 80
        self.classes = load_classes('data/coco.names')
        self.colors = pkl.load(open("pallete", "rb"))
        self.model = Darknet(self.cfgfile)
        self.CUDA = torch.cuda.is_available()
        self.model.load_weights(self.weightsfile)
        self.width = 1280 #640#1280
        self.height = 720 #360#720
        self.id = id
        self.transform = trans.Compose([
                    trans.ToTensor(),
                    trans.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
                ])
        print("Loading network.....")
        if self.CUDA:
            self.model.cuda()
        print("Network successfully loaded")

        self.model.eval()

    def main(self):
        frame = cv2.imread(id)
        try:
            img, orig_im, dim = prep_image(frame, 160)

            im_dim = torch.FloatTensor(dim).repeat(1,2)
            if self.CUDA:                            #### If you have a gpu properly installed then it will run on the gpu
                im_dim = im_dim.cuda()
                img = img.cuda()
            frame = cv2.imread(self.id)

            output = self.model(img)

            from tool.utils import post_processing,plot_boxes_cv2
            bounding_boxes = post_processing(img,self.confidence, self.nms_thesh, output)
            frame = plot_boxes_cv2(frame, bounding_boxes[0], savename= None, class_names=self.classes, color = None, colors=self.colors)

        except:
            pass

        cv2.imwrite('IMG_0748-1.JPG',frame)
        torch.cuda.empty_cache()

if __name__ == "__main__":
    id = "IMG_0748.JPG"
    ObjectDetection(id).main()


import pygame, os

class screengenerator:
    def __init__(self):
        print('init')

    def make_window(self, width=1000, height=700, color=(0, 0, 0), image="null"):
        size = width, height
        screen = pygame.display.set_mode(size)
        screen.fill(color)

        if image != "null":
            b = image + ".bmp"
            img = pygame.image.load(os.path.join("images", b)).convert_alpha()
            screen.blit(img, (0, 0))

        return screen

    def make_miniwindow(self, width=600, height=300, color=(0, 0, 0)):
        size = width, height
        miniwindow = pygame.Surface(size)
        miniwindow.fill(color)
        return miniwindow

    def make_logwindow(self, width=325, height=625, color=(0, 0, 0)):
        size = width, height
        logwindow = pygame.Surface(size)
        logwindow.fill(color)
        return logwindow

    def make_textsurface(self, fontsize=20, message="no message passed into object", bg = (0, 0, 0), fg = (255, 255, 255), stdfont=False ):
        if stdfont:
            font = pygame.font.Font(None, fontsize)
        else:
            f = os.path.join("fonts", "oetmt.ttf")
            font = pygame.font.Font(f, fontsize)
        textline = font.render(message, 0, fg, bg)
        return textline


# -*- coding: utf-8 -*-

from odoo import models, fields, api
from odoo.exceptions import UserError, except_orm, MissingError, ValidationError
from datetime import datetime, timedelta, date
from dateutil.relativedelta import relativedelta

class PosOrder(models.Model):
    _inherit = 'pos.order'

    # them data cho bien tạo account_bank_statement_line
    def _prepare_bank_statement_line_payment_values(self, data):
        args = super(PosOrder, self)._prepare_bank_statement_line_payment_values(data)
        if self._context.get('x_bank_card_id', False):
            args['x_bank_card_id'] = self._context.get('x_bank_card_id', False)
        return args

    @api.multi
    def action_confirm_order(self):
        for line in self.statement_ids:
            if line.x_bank_card_id and line.journal_id.type == 'bank':
                self._get_move_bank_card(line)
        return super(PosOrder,self).action_confirm_order()

    def _get_move_bank_card(self, statement_id):
        for item in statement_id.x_bank_card_id.line_ids:
            move_lines = []
            amount = item.cost_rate /100 * statement_id.amount
            if amount > 0:
                credit_move_vals = {
                    'name': statement_id.name,
                    'account_id': statement_id.journal_id.default_credit_account_id.id,
                    'credit': amount,
                    'debit': 0.0,
                    'partner_id': statement_id.pos_statement_id.partner_id.id,
                }
                debit_move_vals = {
                    'name': item.description,
                    'account_id': item.account_id.id,
                    'credit': 0.0,
                    'debit': amount,
                    'partner_id': statement_id.pos_statement_id.partner_id.id,
                }
                move_lines.append((0, 0, debit_move_vals))
                move_lines.append((0, 0, credit_move_vals))
            else:
                credit_move_vals = {
                    'name': statement_id.name,
                    'account_id': statement_id.journal_id.default_credit_account_id.id,
                    'credit': 0.0,
                    'debit': abs(amount),
                    'partner_id': statement_id.pos_statement_id.partner_id.id,
                }
                debit_move_vals = {
                    'name': statement_id.name,
                    'account_id': item.account_id.id,
                    'credit': abs(amount),
                    'debit': 0.0,
                    'partner_id': statement_id.pos_statement_id.partner_id.id,
                }
                move_lines.append((0, 0, debit_move_vals))
                move_lines.append((0, 0, credit_move_vals))
            vals_account = {
                'date': fields.Datetime.now(),
                'ref': statement_id.pos_statement_id.name,
                'journal_id': statement_id.journal_id.id,
                'line_ids': move_lines,
                'company_id': statement_id.pos_statement_id.company_id.id,
                'check_additional_account_move': True,  # thêm bởi HoiHD: Đây là bút toán phát sinh
            }
            move_id = self.env['account.move'].create(vals_account)
            move_id.post()
        return True


import socketio
from logging_handler import LoggingHandler

from flask import Flask
app = Flask(__name__)

logger = LoggingHandler.getLogger('comm_server_')

# preparing socket client here

sio = socketio.Client(reconnection=True, reconnection_attempts=60, reconnection_delay=1)
class Staticc:
    connectivity_status = None

@sio.on('server_second')
def on_message(data):
    print('done ending the flow !' + data)

@sio.event
def connect():
    Staticc.connectivity_status = True
    print("comm connected!" + str(Staticc.connectivity_status))
    # sio.emit('client_first', 'hi')

@sio.event
def connect_error():
    print("The connection failed!")

@sio.event
def disconnect():
    Staticc.connectivity_status = False
    print("I'm disconnected!" + str(Staticc.connectivity_status))

# logic.. if could't not connect then no issue. will try it in future.

try:     # reconnecting
    sio.connect('http://d_server:7000')
    # sio.wait()
except Exception as e:
    logger.exception("error in connecting first" + e)

@app.route("/")
def index():
    return """
        Welcome to my website!

        Go to hello world
    """

@app.route("/hello")
def hello():
    return """
        Hello World!

        Back to index
    """

@app.route('/user-message/<userid>')
def user(userid):
    logger.debug(" i am processing here.." + userid)
    logger.error(" and i am done.")
    return "user id " + userid + " processed successfully"

@app.route('/bot-message/<botid>')
def bott(botid):
    logger.debug(" bot processed here .." + botid + str(Staticc.connectivity_status))
    if Staticc.connectivity_status == True:
        logger.debug(" connection is still open")
    else:
        logger.debug(" connection is closed now.. writing logic for reconnecting")

        try:     # reconnecting
            sio.connect('http://d_server:7000')
        except Exception as e:
            logger.exception("error in reconnecting" + str(e))

        logger.debug('simple connect statement, now sending data')

    sio.emit('bot_message_came', 'something detected' + botid)

    logger.error(" and bot is done.")
    return "bot id " + botid + " has been processed successfully by bot"

if __name__ == '__main__':
    # Will make the server available externally as well
    app.run(host='d_comm', port=8080)


# -*- coding: utf-8 -*-
'''
pytestsalt.utils
~~~~~~~~~~~~~~~~

Some pytest fixtures used in pytest-salt
'''

# Import Python libs
from __future__ import absolute_import
import os
import re
import sys
import json
import time
import errno
import atexit
import pprint
import signal
import socket
import logging
import subprocess
import threading
import weakref
from operator import itemgetter
from collections import namedtuple

# Import 3rd party libs
import pytest
import psutil
try:
    import setproctitle
    HAS_SETPROCTITLE = True
except ImportError:
    HAS_SETPROCTITLE = False

log = logging.getLogger(__name__)

def set_proc_title(title):
    if HAS_SETPROCTITLE is False:
        return
    setproctitle.setproctitle('[{}] - {}'.format(title, setproctitle.getproctitle()))

def get_unused_localhost_port():
    '''
    Return a random unused port on localhost
    '''
    usock = socket.socket(family=socket.AF_INET, type=socket.SOCK_STREAM)
    usock.bind(('127.0.0.1', 0))
    port = usock.getsockname()[1]
    usock.close()
    return port

def collect_child_processes(pid):
    '''
    Try to collect any started child processes of the provided pid
    '''
    # Let's get the child processes of the started subprocess
    try:
        parent = psutil.Process(pid)
        children = parent.children(recursive=True)
    except psutil.NoSuchProcess:
        children = []
    return children

def _get_cmdline(proc):
    # pylint: disable=protected-access
    try:
        return proc._cmdline
    except AttributeError:
        # Cache the cmdline since that will be inaccessible once the process is terminated
        # and we use it in log calls
        try:
            cmdline = proc.cmdline()
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            # OSX is more restrictive about the above information
            cmdline = None
        except OSError:
            # On Windows we've seen something like:
            #   File " c: ... \lib\site-packages\pytestsalt\utils\__init__.py", line 182, in terminate_process
            #     terminate_process_list(process_list, kill=slow_stop is False, slow_stop=slow_stop)
            #   File " c: ... \lib\site-packages\pytestsalt\utils\__init__.py", line 130, in terminate_process_list
            #     _terminate_process_list(process_list, kill=kill, slow_stop=slow_stop)
            #   File " c: ... \lib\site-packages\pytestsalt\utils\__init__.py", line 78, in _terminate_process_list
            #     cmdline = process.cmdline()
            #   File " c: ... \lib\site-packages\psutil\__init__.py", line 786, in cmdline
            #     return self._proc.cmdline()
            #   File " c: ... \lib\site-packages\psutil\_pswindows.py", line 667, in wrapper
            #     return fun(self, *args, **kwargs)
            #   File " c: ... \lib\site-packages\psutil\_pswindows.py", line 745, in cmdline
            #     ret = cext.proc_cmdline(self.pid, use_peb=True)
            #   OSError: [WinError 299] Only part of a ReadProcessMemory or WriteProcessMemory request was completed: 'originated from ReadProcessMemory(ProcessParameters)

            # Late import
            cmdline = None
        if not cmdline:
            try:
                cmdline = proc.as_dict()
            except psutil.NoSuchProcess:
                cmdline = '<could not be retrived; dead process: {}>'.format(proc)
            except (psutil.AccessDenied, OSError):
                cmdline = weakref.proxy(proc)
        proc._cmdline = cmdline
    return proc._cmdline
    # pylint: enable=protected-access

def _terminate_process_list(process_list, kill=False, slow_stop=False):
    log.info(
        'Terminating process list:\n%s',
        pprint.pformat([_get_cmdline(proc) for proc in process_list])
    )
    for process in process_list[:]:  # Iterate over copy of the list
        if not psutil.pid_exists(process.pid):
            process_list.remove(process)
            continue
        try:
            if not kill and process.status() == psutil.STATUS_ZOMBIE:
                # Zombie processes will exit once child processes also exit
                continue
            if kill:
                log.info('Killing process(%s): %s', process.pid, _get_cmdline(process))
                process.kill()
            else:
                log.info('Terminating process(%s): %s', process.pid, _get_cmdline(process))
                try:
                    if slow_stop:
                        # Allow coverage data to be written down to disk
                        process.send_signal(signal.SIGTERM)
                        try:
                            process.wait(2)
                        except psutil.TimeoutExpired:
                            if psutil.pid_exists(process.pid):
                                continue
                    else:
                        process.terminate()
                except OSError as exc:
                    if exc.errno not in (errno.ESRCH, errno.EACCES):
                        raise
            if not psutil.pid_exists(process.pid):
                process_list.remove(process)
        except psutil.NoSuchProcess:
            process_list.remove(process)

def terminate_process_list(process_list, kill=False, slow_stop=False):

    def on_process_terminated(proc):
        log.info('Process %s terminated with exit code: %s', getattr(proc, '_cmdline', proc), proc.returncode)

    # Try to terminate processes with the provided kill and slow_stop parameters
    log.info('Terminating process list. 1st step. kill: %s, slow stop: %s', kill, slow_stop)

    # Remove duplicates from the process list
    seen_pids = []
    start_count = len(process_list)
    for proc in process_list[:]:
        if proc.pid in seen_pids:
            process_list.remove(proc)
        seen_pids.append(proc.pid)
    end_count = len(process_list)
    if end_count < start_count:
        log.debug('Removed %d duplicates from the initial process list', start_count - end_count)

    _terminate_process_list(process_list, kill=kill, slow_stop=slow_stop)
    psutil.wait_procs(process_list, timeout=15, callback=on_process_terminated)

    if process_list:
        # If there's still processes to be terminated, retry and kill them if slow_stop is False
        log.info('Terminating process list. 2nd step. kill: %s, slow stop: %s', slow_stop is False, slow_stop)
        _terminate_process_list(process_list, kill=slow_stop is False, slow_stop=slow_stop)
        psutil.wait_procs(process_list, timeout=10, callback=on_process_terminated)

    if process_list:
        # If there's still processes to be terminated, just kill them, no slow stopping now
        log.info('Terminating process list. 3rd step. kill: True, slow stop: False')
        _terminate_process_list(process_list, kill=True, slow_stop=False)
        psutil.wait_procs(process_list, timeout=5, callback=on_process_terminated)

    if process_list:
        # In there's still processes to be terminated, log a warning about it
        log.warning('Some processes failed to properly terminate: %s', process_list)

def terminate_process(pid=None, process=None, children=None, kill_children=None, slow_stop=False):
    '''
    Try to terminate/kill the started processe
    '''
    children = children or []
    process_list = []

    if kill_children is None:
        # Always kill children if kill the parent process and kill_children was not set
        kill_children = True if slow_stop is False else kill_children

    if pid and not process:
        try:
            process = psutil.Process(pid)
            process_list.append(process)
        except psutil.NoSuchProcess:
            # Process is already gone
            process = None

    if kill_children:
        if process:
            children.extend(collect_child_processes(pid))
        if children:
            process_list.extend(children)

    if process_list:
        if process:
            log.info('Stopping process %s and respective children: %s', process, children)
        else:
            log.info('Terminating process list: %s', process_list)
        terminate_process_list(process_list, kill=slow_stop is False, slow_stop=slow_stop)

def start_daemon(request,
                 daemon_name=None,
                 daemon_id=None,
                 daemon_log_prefix=None,
                 daemon_cli_script_name=None,
                 daemon_config=None,
                 daemon_config_dir=None,
                 daemon_class=None,
                 bin_dir_path=None,
                 fail_hard=False,
                 start_timeout=10,
                 slow_stop=True,
                 environ=None,
                 cwd=None,
                 max_attempts=3,
                 **kwargs):
    '''
    Returns a running salt daemon
    '''
    if fail_hard:
        fail_method = pytest.fail
    else:
        fail_method = pytest.xfail
    log.info('[%s] Starting pytest %s(%s)', daemon_name, daemon_log_prefix, daemon_id)
    attempts = 0
    process = None
    while attempts <= max_attempts:  # pylint: disable=too-many-nested-blocks
        attempts += 1
        process = daemon_class(request,
                               daemon_config,
                               daemon_config_dir,
                               bin_dir_path,
                               daemon_log_prefix,
                               cli_script_name=daemon_cli_script_name,
                               slow_stop=slow_stop,
                               environ=environ,
                               cwd=cwd,
                               **kwargs)
        process.start()
        if process.is_alive():
            try:
                connectable = process.wait_until_running(timeout=start_timeout)
                if connectable is False:
                    connectable = process.wait_until_running(timeout=start_timeout/2)
                    if connectable is False:
                        process.terminate()
                        if attempts >= max_attempts:
                            fail_method(
                                'The pytest {}({}) has failed to confirm running status '
                                'after {} attempts'.format(daemon_name, daemon_id, attempts))
                        continue
            except Exception as exc:  # pylint: disable=broad-except
                log.exception('[%s] %s', daemon_log_prefix, exc, exc_info=True)
                terminate_process(process.pid, kill_children=True, slow_stop=slow_stop)
                if attempts >= max_attempts:
                    fail_method(str(exc))
                continue
            log.info(
                '[%s] The pytest %s(%s) is running and accepting commands '
                'after %d attempts',
                daemon_log_prefix,
                daemon_name,
                daemon_id,
                attempts
            )

            def stop_daemon():
                log.info('[%s] Stopping pytest %s(%s)', daemon_log_prefix, daemon_name, daemon_id)
                terminate_process(process.pid, kill_children=True, slow_stop=slow_stop)
                log.info('[%s] pytest %s(%s) stopped', daemon_log_prefix, daemon_name, daemon_id)

            request.addfinalizer(stop_daemon)
            break
        else:
            terminate_process(process.pid, kill_children=True, slow_stop=slow_stop)
            continue
    else:
        if process is not None:
            terminate_process(process.pid, kill_children=True, slow_stop=slow_stop)
        fail_method(
            'The pytest {}({}) has failed to start after {} attempts'.format(
                daemon_name,
                daemon_id,
                attempts-1
            )
        )
    return process

class SaltScriptBase(object):
    '''
    Base class for Salt CLI scripts
    '''

    cli_display_name = None

    def __init__(self,
                 request,
                 config,
                 config_dir,
                 bin_dir_path,
                 log_prefix,
                 cli_script_name=None,
                 slow_stop=False,
                 environ=None,
                 cwd=None):
        self.request = request
        self.config = config
        if not isinstance(config_dir, str):
            config_dir = config_dir.realpath().strpath
        self.config_dir = config_dir
        self.bin_dir_path = bin_dir_path
        self.log_prefix = log_prefix
        if cli_script_name is None:
            raise RuntimeError('Please provide a value for the cli_script_name keyword argument')
        self.cli_script_name = cli_script_name
        if self.cli_display_name is None:
            self.cli_display_name = '{}({})'.format(self.__class__.__name__,
                                                    self.cli_script_name)
        self.slow_stop = slow_stop
        self.environ = environ or os.environ.copy()
        self.cwd = cwd or os.getcwd()
        self._terminal = self._children = None

    def get_script_path(self, script_name):
        '''
        Returns the path to the script to run
        '''
        script_path = os.path.join(self.bin_dir_path, script_name)
        if not os.path.exists(script_path):
            pytest.fail('The CLI script {!r} does not exist'.format(script_path))
        return script_path

    def get_base_script_args(self):
        '''
        Returns any additional arguments to pass to the CLI script
        '''
        return ['-c', self.config_dir]

    def get_script_args(self):  # pylint: disable=no-self-use
        '''
        Returns any additional arguments to pass to the CLI script
        '''
        return []

    def init_terminal(self, cmdline, **kwargs):
        '''
        Instantiate a terminal with the passed cmdline and kwargs and return it.

        Additionaly, it sets a reference to it in self._terminal and also collects
        an initial listing of child processes which will be used when terminating the
        terminal
        '''
        # Late import
        import salt.utils.nb_popen as nb_popen
        self._terminal = nb_popen.NonBlockingPopen(cmdline, **kwargs)
        self._children = collect_child_processes(self._terminal.pid)
        atexit.register(self.terminate)
        return self._terminal

    def terminate(self):
        '''
        Terminate the started daemon
        '''
        if self._terminal is None:
            return
        # Lets log and kill any child processes which salt left behind
        if self._terminal.stdout:
            self._terminal.stdout.close()
        if self._terminal.stderr:
            self._terminal.stderr.close()
        terminate_process(pid=self._terminal.pid,
                          children=self._children,
                          kill_children=True,
                          slow_stop=self.slow_stop)

class SaltDaemonScriptBase(SaltScriptBase):
    '''
    Base class for Salt Daemon CLI scripts
    '''

    def __init__(self, *args, **kwargs):
        self._process_cli_output_in_thread = kwargs.pop('process_cli_output_in_thread', True)
        event_listener_config_dir = kwargs.pop('event_listener_config_dir', None)
        if event_listener_config_dir and not isinstance(event_listener_config_dir, str):
            event_listener_config_dir = event_listener_config_dir.realpath().strpath
        self.event_listener_config_dir = event_listener_config_dir
        super(SaltDaemonScriptBase, self).__init__(*args, **kwargs)
        self._running = threading.Event()
        self._connectable = threading.Event()

    def is_alive(self):
        '''
        Returns true if the process is alive
        '''
        return self._running.is_set()

    def get_check_ports(self):  # pylint: disable=no-self-use
        '''
        Return a list of ports to check against to ensure the daemon is running
        '''
        return []

    def get_check_events(self):  # pylint: disable=no-self-use
        '''
        Return a list of event tags to check against to ensure the daemon is running
        '''
        return []

    def get_salt_run_fixture(self):
        if self.request.scope == 'session':
            try:
                return self.request.getfixturevalue('session_salt_run')
            except AttributeError:
                return self.request.getfuncargvalue('session_salt_run')
        try:
            return self.request.getfixturevalue('salt_run')
        except AttributeError:
            return self.request.getfuncargvalue('salt_run')

    def start(self):
        '''
        Start the daemon subprocess
        '''
        # Late import
        log.info('[%s][%s] Starting DAEMON in CWD: %s', self.log_prefix, self.cli_display_name, self.cwd)
        proc_args = [
            self.get_script_path(self.cli_script_name)
        ] + self.get_base_script_args() + self.get_script_args()

        if sys.platform.startswith('win'):
            # Windows needs the python executable to come first
            proc_args.insert(0, sys.executable)

        log.info('[%s][%s] Running \'%s\'...', self.log_prefix, self.cli_display_name, ' '.join(proc_args))

        self.init_terminal(proc_args, env=self.environ, cwd=self.cwd)
        self._running.set()
        if self._process_cli_output_in_thread:
            process_output_thread = threading.Thread(target=self._process_output_in_thread)
            process_output_thread.daemon = True
            process_output_thread.start()
        return True

    def _process_output_in_thread(self):
        '''
        The actual, coroutine aware, start method
        '''
        try:
            while self._running.is_set() and self._terminal.poll() is None:
                # We're not actually interested in processing the output, just consume it
                if self._terminal.stdout is not None:
                    self._terminal.recv()
                if self._terminal.stderr is not None:
                    self._terminal.recv_err()
                time.sleep(0.125)
                if self._terminal.poll() is not None:
                    self._running.clear()
        except (SystemExit, KeyboardInterrupt):
            self._running.clear()
        finally:
            if self._terminal.stdout:
                self._terminal.stdout.close()
            if self._terminal.stderr:
                self._terminal.stderr.close()

    @property
    def pid(self):
        terminal = getattr(self, '_terminal', None)
        if not terminal:
            return
        return terminal.pid

    def terminate(self):
        '''
        Terminate the started daemon
        '''
        # Let's get the child processes of the started subprocess
        self._running.clear()
        self._connectable.clear()
        time.sleep(0.0125)
        super(SaltDaemonScriptBase, self).terminate()

    def wait_until_running(self, timeout=None):
        '''
        Blocking call to wait for the daemon to start listening
        '''
        if self._connectable.is_set():
            return True

        expire = time.time() + timeout
        check_ports = self.get_check_ports()
        if check_ports:
            log.info(
                '[%s][%s] Checking the following ports to assure running status: %s',
                self.log_prefix,
                self.cli_display_name,
                check_ports
            )
        check_events = self.get_check_events()
        if check_events:
            log.info(
                '[%s][%s] Checking the following event tags to assure running status: %s',
                self.log_prefix,
                self.cli_display_name,
                check_events
            )
        log.debug('Wait until running expire: %s  Timeout: %s  Current Time: %s', expire, timeout, time.time())
        with EventListener(self.event_listener_config_dir or self.config_dir, self.log_prefix) as event_listener:
            try:
                while True:
                    if self._running.is_set() is False:
                        # No longer running, break
                        log.warning('No longer running!')
                        break

                    if time.time() > expire:
                        # Timeout, break
                        log.warning('Wait until running expired at %s(was set to %s)', time.time(), expire)
                        break

                    if not check_ports and not check_events:
                        self._connectable.set()
                        break

                    if check_events:
                        for tag in event_listener.wait_for_events(check_events, timeout=timeout - 0.5):
                            check_events.remove(tag)

                    if not check_events:
                        stop_sending_events_file = self.config.get('pytest_stop_sending_events_file')
                        if stop_sending_events_file and os.path.exists(stop_sending_events_file):
                            log.info('Removing pytest_stop_sending_events_file: %s', stop_sending_events_file)
                            os.unlink(stop_sending_events_file)

                    for port in set(check_ports):
                        if isinstance(port, int):
                            log.debug('[%s][%s] Checking connectable status on port: %s',
                                      self.log_prefix,
                                      self.cli_display_name,
                                      port)
                            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                            conn = sock.connect_ex(('localhost', port))
                            try:
                                if conn == 0:
                                    log.debug('[%s][%s] Port %s is connectable!',
                                              self.log_prefix,
                                              self.cli_display_name,
                                              port)
                                    check_ports.remove(port)
                                    sock.shutdown(socket.SHUT_RDWR)
                            except socket.error:
                                continue
                            finally:
                                sock.close()
                                del sock
                    time.sleep(0.5)
            except KeyboardInterrupt:
                pass
        if self._connectable.is_set():
            log.info('[%s][%s] All ports checked. Running!', self.log_prefix, self.cli_display_name)
        return self._connectable.is_set()

class ShellResult(namedtuple('Result', ('exitcode', 'stdout', 'stderr', 'json'))):
    '''
    This class serves the purpose of having a common result class which will hold the
    data from the bigret backend(despite the backend being used).

    This will allow filtering by access permissions and/or object ownership.

    '''
    __slots__ = ()

    def __new__(cls, exitcode, stdout, stderr, _json):
        return super(ShellResult, cls).__new__(cls, exitcode, stdout, stderr, _json)

    # These are copied from the namedtuple verbose output in order to quiet down PyLint
    exitcode = property(itemgetter(0), doc='Alias for field number 0')
    stdout = property(itemgetter(1), doc='Alias for field number 1')
    stderr = property(itemgetter(2), doc='Alias for field number 2')
    json = property(itemgetter(3), doc='Alias for field number 3')

    def __eq__(self, other):
        '''
        Allow comparison against the parsed JSON or the output
        '''
        if self.json:
            return self.json == other
        return self.stdout == other

class SaltCliScriptBase(SaltScriptBase):
    '''
    Base class which runs Salt's non daemon CLI scripts
    '''

    DEFAULT_TIMEOUT = 25

    def __init__(self, *args, **kwargs):
        self.default_timeout = kwargs.pop('default_timeout', self.DEFAULT_TIMEOUT)
        super(SaltCliScriptBase, self).__init__(*args, **kwargs)

    def get_base_script_args(self):
        return SaltScriptBase.get_base_script_args(self) + ['--out=json']

    def get_minion_tgt(self, **kwargs):
        return kwargs.pop('minion_tgt', None)

    def run(self, *args, **kwargs):
        '''
        Run the given command synchronously
        '''
        # Late import
        import salt.ext.six as six
        timeout = kwargs.get('timeout', self.default_timeout)
        if 'fail_hard' in kwargs:
            # Explicit fail_hard passed
            fail_hard = kwargs.pop('fail_hard')
        else:
            # Get the value of the _salt_fail_hard fixture
            try:
                fail_hard = self.request.getfixturevalue('_salt_fail_hard')
            except AttributeError:
                fail_hard = self.request.getfuncargvalue('_salt_fail_hard')
        if fail_hard is True:
            fail_method = pytest.fail
        else:
            fail_method = pytest.xfail
        log.info('The fail hard setting for %s is: %s', self.cli_script_name, fail_hard)
        minion_tgt = self.get_minion_tgt(**kwargs)
        timeout_expire = time.time() + kwargs.pop('timeout', self.default_timeout)
        environ = self.environ.copy()
        environ['PYTEST_LOG_PREFIX'] = '[{}] '.format(self.log_prefix)
        environ['PYTHONUNBUFFERED'] = '1'
        proc_args = [
            self.get_script_path(self.cli_script_name)
        ] + self.get_base_script_args() + self.get_script_args()

        if sys.platform.startswith('win'):
            # Windows needs the python executable to come first
            proc_args.insert(0, sys.executable)

        if minion_tgt is not None:
            proc_args.append(minion_tgt)
        proc_args.extend(list(args))
        for key in kwargs:
            proc_args.append('{}={}'.format(key, kwargs[key]))

        log.info('[%s][%s] Running \'%s\' in CWD: %s ...',
                 self.log_prefix, self.cli_display_name, ' '.join(proc_args), self.cwd)

        terminal = self.init_terminal(proc_args,
                                      cwd=self.cwd,
                                      env=environ,
                                      stdout=subprocess.PIPE,
                                      stderr=subprocess.PIPE)

        # Consume the output
        stdout = six.b('')
        stderr = six.b('')

        try:
            while True:
                # We're not actually interested in processing the output, just consume it
                if terminal.stdout is not None:
                    try:
                        out = terminal.recv(4096)
                    except IOError:
                        out = six.b('')
                    if out:
                        stdout += out
                if terminal.stderr is not None:
                    try:
                        err = terminal.recv_err(4096)
                    except IOError:
                        err = ''
                    if err:
                        stderr += err
                if out is None and err is None:
                    break
                if timeout_expire < time.time():
                    self.terminate()
                    fail_method(
                        '[{}][{}] Failed to run: args: {!r}; kwargs: {!r}; Error: {}'.format(
                            self.log_prefix,
                            self.cli_display_name,
                            args,
                            kwargs,
                            '[{}][{}] Timed out after {} seconds!'.format(self.log_prefix,
                                                                          self.cli_display_name,
                                                                          timeout)
                        )
                    )
        except (SystemExit, KeyboardInterrupt):
            pass
        finally:
            self.terminate()

        if six.PY3:
            # pylint: disable=undefined-variable
            stdout = stdout.decode(__salt_system_encoding__)
            stderr = stderr.decode(__salt_system_encoding__)
            # pylint: enable=undefined-variable

        exitcode = terminal.returncode
        stdout, stderr, json_out = self.process_output(minion_tgt, stdout, stderr, cli_cmd=proc_args)
        return ShellResult(exitcode, stdout, stderr, json_out)

    def process_output(self, tgt, stdout, stderr, cli_cmd=None):
        if stdout:
            try:
                json_out = json.loads(stdout)
            except ValueError:
                log.debug('[%s][%s] Failed to load JSON from the following output:\n%r',
                          self.log_prefix,
                          self.cli_display_name,
                          stdout)
                json_out = None
        else:
            json_out = None
        return stdout, stderr, json_out

class EventListener(object):

    DEFAULT_TIMEOUT = 60

    def __init__(self, config_dir, log_prefix):
        # Late import
        self.config_dir = config_dir
        self.log_prefix = '[{}][PyTestEventListener]'.format(log_prefix)
        self._listener = None

    def wait_for_events(self, check_events, timeout=None):
        if timeout is None:
            timeout = self.DEFAULT_TIMEOUT
        log.info('%s waiting %s seconds for events: %s',
                 self.log_prefix,
                 timeout,
                 check_events)
        matched_events = set()
        events_to_match = set(check_events)
        events_processed = 0
        max_timeout = time.time() + timeout
        last_log = 0
        log_freq = 10
        while True:
            if not events_to_match:
                log.info('%s ALL EVENT TAGS FOUND!!!', self.log_prefix)
                return matched_events

            if time.time() > max_timeout:
                log.warning(
                    '%s Failed to find all of the required event tags(%s). '
                    'Total events processed: %s. Total events found: %s.',
                    self.log_prefix,
                    check_events,
                    events_processed,
                    len(matched_events)
                )
                return matched_events

            event = self._listener.get_event(full=True, auto_reconnect=True)
            if event is None:
                continue

            tag = event['tag']
            log.info('Got event: %s', event)
            if tag in events_to_match:
                matched_events.add(tag)
                events_to_match.remove(tag)

            events_processed += 1
            if time.time() - last_log > log_freq:
                log.debug('%s Events processed so far: %d',
                          self.log_prefix,
                          events_processed)
                last_log = time.time()

    def terminate(self):
        if self._listener is not None:
            listener = self._listener
            self._listener = None
            listener.destroy()

    def __enter__(self):
        if self._listener is None:
            # Late import
            import salt.config
            import salt.utils.event
            opts = salt.config.master_config(os.path.join(self.config_dir, 'master'))
            self._listener = salt.utils.event.get_event('master', opts=opts, listen=True)
            atexit.register(self.terminate)
        return self

    def __exit__(self, *args):
        self.terminate()

@pytest.mark.trylast
def pytest_configure(config):
    pytest.helpers.utils.register(get_unused_localhost_port)


# encoding: utf-8

from lupa import LuaRuntime, LuaError
import threading
import urllib2
import os

lua_modules = {}

def get_stored_modules():
    modules = sorted(x[:-4] for x in os.listdir('data/lua/'))
    return modules

def load_module(m):
    with open('data/lua/%s.lua' % m.replace('/','_')) as f:
        source = f.read()
    l = LuaModule()
    result = l.run(source)
    try:
        success, result = result
    except:
        success = result
        pass
    if not success:
        raise LuaError(result)
    else:
        lua_modules[m] = l

def unload_module(m):
    try:
        lua_modules[module].stop()
    except:
        pass
    del lua_modules[module]

def handle_message(irc, channel, origin, command, args):
    for n in lua_modules:
        lua_modules[n].event_message(irc, channel, origin, command, args)

def execute(l):
        try:
            lua = LuaModule()
            output = lua.run(l)
        except LuaError as e:
            raise
        else:
            try:
                output = list(output)
                success = output.pop(0)
            except:
                success = output
                output = []
            if success:
                return output
            else:
                raise LuaError(', '.join(output))

class LuaModule(object):
    def __init__(self):
        self.lua = LuaRuntime(register_eval=False, attribute_filter=self.filter)

        # A subset of the standard library
        self.env = self.lua.eval("""{
            assert=assert,
            error=error,
            ipairs=ipairs,
            next=next,
            pairs=pairs,
            pcall=pcall,
            select=select,
            tonumber=tonumber,
            tostring=tostring,
            type=type,
            unpack=unpack,
            _VERSION=_VERSION,
            xpcall=xpcall,
            coroutine=coroutine,
            string=string,
            table=table,
            math=math,
            os={
                clock=os.clock,
                date=os.date,
                difftime=os.difftime,
                time=os.time
            }
        }""")

        self.env.kb = self.lua.table()
        self.lua.globals()['kb_private'] = self.lua.table(
            webget=self.lua_webget,
            schedule_at=self.lua_schedule_at,
            schedule_cron=self.lua_schedule_cron
        )
        self.env.kb['webget'] = self.lua.eval('function(url, callback) kb_private.webget(url, callback) end')
        self.env.kb['schedule_at'] = self.lua.eval('function(when, user_data, callback) kb_private.schedule_at(when, user_data, callback) end')
        self.env.kb['schedule_cron'] = self.lua.eval('function (period, user_data, callback) kb_private.schedule_cron(period, user_data, callback) end')

        self.events = self.lua.table()
        self.env.events = self.events

        self.lua.globals().env = self.env

        self.run = self.lua.eval("""
        function (untrusted_code)
          if untrusted_code:byte(1) == 27 then return nil, "binary bytecode prohibited" end
          local untrusted_function, message = loadstring(untrusted_code)
          if not untrusted_function then return nil, message end
          setfenv(untrusted_function, env)
          return pcall(untrusted_function)
        end
        """)

        self.lua.execute("""
            counter = 0
            running = 1
            debug.sethook(function (event, line)
                counter = counter + 1
                if not running or counter > 10000 then
                    error("That's enough!")
                end
            end, "", 1)
        """)

    def filter(self, o, attribute, is_setting):
        raise AttributeError("forbidden: %s.%s" % (o, attribute))

    def irc_wrapper(self, irc):
        wrapper = self.lua.table()
        wrapper['_network'] = irc
        wrapper['_send_message'] = lambda channel, message: m('irc_helpers').message(irc, channel, message)
        wrapper.send_message = self.lua.eval("function (self, channel, message) self._send_message(channel, message) end")
        return wrapper

    def channel_wrapper(self, irc, channel):
        wrapper = self.lua.table()
        wrapper['irc'] = self.irc_wrapper(irc)
        wrapper['_send'] = lambda message: m('irc_helpers').message(irc, channel, message)
        wrapper['send'] = self.lua.eval("function (self, message) self._send(message) end")
        try:
            chan = m('chantrack').network(irc)[channel]
        except KeyError:
            wrapper['members'] = self.lua.table()
        else:
            wrapper['members'] = self.list_to_table([self.user_wrapper(x) for x in chan.users.values()])
            wrapper['topic'] = chan.topic
        wrapper['name'] = channel
        return wrapper

    def user_wrapper(self, user):
        if user is None:
            return None
        wrapper = self.lua.table(
            hostname=user.hostname,
            nick=user.nick,
            ident=user.ident,
            realname=getattr(user, 'realname', None),
            modes=getattr(user, 'modes', None),
            _send=lambda message: m('irc_helpers').message(irc, user.nick, message),
            send=self.lua.eval('function(self, message) self._send(message) end')
        )
        return wrapper

    def event_message(self, irc, channel, origin, command, args):
        if self.events.received_command:
            self.lua.globals().counter = 0
            self.events.received_command(self.channel_wrapper(irc, channel), self.user_wrapper(origin), command, self.list_to_table(args))

    def list_to_table(self, iter):
        return self.lua.table(*iter)

    def lua_webget(self, url, callback):
        def do_callback():
            try:
                f = urllib2.urlopen(url)
                result = f.read()
            except urllib2.URLError as e:
                callback(False, str(e))
            else:
                callback(True, result)
        threading.Thread(target=do_callback).start()

    def lua_schedule_at(self, when, user_data, callback):
        try:
            m('cron').add_at(when, callback, when, user_data)
        except ModuleNotLoaded:
            return False
        else:
            return True

    def lua_schedule_cron(self, period, user_data, callback):
        try:
            m('cron').add_cron(period, callback, user_data)
        except ModuleNotLoaded:
            return False
        else:
            return True

    def stop(self):
        self.lua.globals().running = 0


'''
Trading strategies based on explained cross-sectional returns
'''
import pandas as pd
import numpy as np
from pandas_datareader import data
from pandas import ExcelWriter
from pandas import ExcelFile
from sklearn.linear_model import LinearRegression

def pandasToExcel(df, name):
	wr = pd.ExcelWriter(name, engine = 'xlsxwriter')
	df.to_excel(wr, sheet_name = "data")
	wr.save()

'''
function HML
	Inputs: df (pandas), percentage (we usually assume 30%)
	Outputs: pandas dataframe with more data
	Formula: 0.5 * (small val + big val) - 0.5 * (small growth + big growth)
'''
def HML(df, percentage):
	df['LASTDATE'] = pd.to_datetime(df.LASTDATE)
	numStocks = int(round(percentage * 500))
	df['HML_VALUE'], df["HML_GROWTH"], df["HML"] = "", "", ""
	df['HML_VALUE'] = df.sort_values(['BMR'],ascending=False).groupby(['LASTDATE'])['RETURN'].transform(lambda x: sum(x.head(numStocks)) / numStocks)
	df['HML_GROWTH'] = df.sort_values(['BMR'],ascending=False).groupby(['LASTDATE'])['RETURN'].transform(lambda x: sum(x.tail(numStocks)) / numStocks)
	for idx, row in df.iterrows():
		df.at[idx, 'HML'] = 0.5 * row['HML_VALUE'] - 0.5 * row['HML_GROWTH']
	return df

'''
function SMB
	Inputs: df (pandas), percentage (we usually assume 30%)
	Outputs: pandas dataframe with more data
	Formula: 0.33 * (small val + small neutral + small growth) - 0.33 * (big val + big neutral + big growth)
'''
def SMB(df, percentage):
	numStocks = int(round(percentage * 500))
	df['SMB_SMALL'], df["SMB_LARGE"], df["SMB"] = "", "", ""
	df['SMB_SMALL'] = df.sort_values(['MARKET_CAP'],ascending=False).groupby(['LASTDATE'])['RETURN'].transform(lambda x: sum(x.head(numStocks)) / numStocks)
	df['SMB_LARGE'] = df.sort_values(['MARKET_CAP'],ascending=False).groupby(['LASTDATE'])['RETURN'].transform(lambda x: sum(x.tail(numStocks)) / numStocks)
	for idx, row in df.iterrows():
		df.at[idx, 'SMB'] = row['SMB_SMALL'] / 3 - row['SMB_LARGE'] / 3
	return df

'''
function stockreturns
	Inputs: s1, s2, s3, s4 -- steps on whether we need to continue
	Outputs: Excel file
'''
def dataProcessing(s1, s2, s3, s4):
	# This function only keeps the last of month data
	def removeDaily(data):
		return data[data['DATE'].dt.date == data['LASTDATE'].dt.date]
	# This function returns 2017 and 2018 data
	def trim(data):
		return data[data['YEAR'] <= 2016]
	# Step 1: We only keep the last day of month data (represent monthly)
	CP = None
	if s1:
		CP = pd.read_excel('CLOSING.xlsx', sheet_name='WRDS')
		CP = removeDaily(CP)
		CP = trim(CP)
		pandasToExcel(CP, "Step1.xlsx")
	else:
		CP = pd.read_excel('Step1.xlsx', sheet_name='data')

	# Step 2: Read from BVPS to determine BV ratio data
	if s2:
		missing = []
		BVPS = pd.read_excel('BVPS.xlsx', sheet_name='WRDS')
		#print(BVPS['KEY'], BVPS['YEAR'], CP['KEY'], CP['YEAR'])
		CP = pd.merge(CP, BVPS, how = "inner", on = ['KEY', 'YEAR'])
		CP['BMR'], CP['MARKET_CAP'] = "", ""
		for idx, row in CP.iterrows():
			if row['BVPS'] != None and row['BVPS'] != 0:
				CP.at[idx, 'BMR'] = row['BVPS'] / row['PRICE']
			else:
				missing.append((row['KEY'], row['YEAR']))
		for idx, row in CP.iterrows():
			CP.at[idx, 'MARKET_CAP'] = row['SHARE'] * row['PRICE']
		pandasToExcel(CP, "Step2.xlsx")
	else:
		CP = pd.read_excel('Step2.xlsx', sheet_name='data')

	if s3:
		# Step 3: Include HML
		CP = HML(CP, 0.3)
		# Step 4: Include SMB
		CP = SMB(CP, 0.4)
		pandasToExcel(CP, "Step3.xlsx")
	else:
		CP = pd.read_excel('Step3.xlsx', sheet_name='data')

	#Step 5: risk free rates
	if s4:
		TR = pd.read_excel('Treasury.xls', sheet_name='FRED')
		TR = TR[TR['DATE'].dt.date == TR['LASTDATE'].dt.date]
		CP = pd.merge(CP, TR, how = "left", on = ['LASTDATE'])

		#Step 6: market returns
		MRTN = pd.read_excel('Market_Returns.xlsx', sheet_name='WRDS')
		MRTN = MRTN[MRTN['DATE'].dt.date == MRTN['LASTDATE'].dt.date]
		CP = pd.merge(CP, MRTN, how = "left", on = ['LASTDATE'])
		CP['RM-RF'], CP["RETURN-RF"] = '', ''
		for idx, row in CP.iterrows():
			CP.at[idx,'RETURN-RF'] = row['RETURN'] - row['RF'] / 100
			CP.at[idx,'RM-RF'] = row['MK_RETURN'] - row['RF'] / 100
		CP = CP[['RETURN-RF','RM-RF', 'HML', 'SMB', 'LASTDATE']]
		pandasToExcel(CP, "Step4.xlsx")
		CP = CP[['RETURN-RF','RM-RF', 'HML', 'SMB']]
		pandasToExcel(CP, "final.xlsx")
	else:
		CP = pd.read_excel('final.xlsx', sheet_name='data')
	return CP.dropna()

def processing():
	data = dataProcessing(0, 0, 0, 1)
	k = data.values
	f = k[:, 1:]
	p = k[:, 0:1]
	reg = LinearRegression().fit(f, p)
	print(reg.coef_)
processing()

'''
NOTE: THE CODE BELOW IS AN OLDER-VERSION CODE
'''

'''
def download(ticker, start_date, end_date):
	pd = data.DataReader(ticker, 'yahoo', start_date, end_date)
	t = pd["Close"].values #temporary
	return np.reshape(t, (len(t), 1))

def factors(start_date, end_date):
	# Oil stocks: https://www.fool.com/investing/2018/09/06/the-5-biggest-oil-stocks-in-the-us.aspx
	F1 = data.DataReader("XOM", 'yahoo', start_date, end_date)["Close"].values
	F1 = np.reshape(F1, (len(F1), 1))
	F2 = data.DataReader("CVX", 'yahoo', start_date, end_date)["Close"].values
	F2 = np.reshape(F2, (len(F2), 1))
	v = np.hstack((F1, F2))
	F3 = data.DataReader("COP", 'yahoo', start_date, end_date)["Close"].values
	F3 = np.reshape(F3, (len(F3), 1))
	v = np.hstack((v, F3))
	F4 = data.DataReader("EOG", 'yahoo', start_date, end_date)["Close"].values
	F4 = np.reshape(F4, (len(F4), 1))
	v = np.hstack((v, F4))
	F5 = data.DataReader("OXY", 'yahoo', start_date, end_date)["Close"].values
	F5 = np.reshape(F5, (len(F5), 1))
	v = np.hstack((v, F5))

	# Aircraft providers
	F6 = data.DataReader("BA", 'yahoo', start_date, end_date)["Close"].values
	F6 = np.reshape(F6, (len(F6), 1))
	v = np.hstack((v, F6))
	F7 = data.DataReader("AB", 'yahoo', start_date, end_date)["Close"].values
	F7 = np.reshape(F7, (len(F7), 1))
	v = np.hstack((v, F7))
	return v

def loss(start_date, end_date, ticker, reg):
	f = factors(start_date, end_date)
	p = download(ticker, start_date, end_date)
	return reg.score(f, p)

def linearRegression():
	# American Airlines, Delta Airlines, Spirit Airlines, SouthWest Airlines
	ticker = ["AAL", "DAL", "SAVE", "LUV"]
	start, end = '2016-01-01', '2017-12-31'
	for tick in ticker:
		p = download(tick, start, end)
		f = factors(start, end)
		reg = LinearRegression().fit(f, p)
		#Testing error
		teststart, testend = '2018-01-01', '2018-12-31'
		print(tick, loss(teststart, testend, tick, reg))

def mainFn():
	linearRegression()

mainFn()
'''


#!/usr/bin/python

import sys
import os
sys.argv.append('-b')
import ROOT
from ROOT import gStyle
from ROOT import gROOT
from ROOT import TStyle

print(sys.argv[1])
ff = ROOT.TFile(sys.argv[1])
print(ff)
qq = sys.argv[1]
ww = qq.split("/")
zz = ww[-1]
#print(ff.GetListOfKeys())
#print(zz)
xxx = zz.split(".")
zzz = xxx[0]
#print(zzz)
typeX = int(sys.argv[2])
LogOn = int(sys.argv[3])

def plotdistributer(type,zz,zzz):
	makeplot(type,"LogOFF",zz,zzz)
	if type[4]:

		makeplot(type,"LogON",zz,zzz)

def makeplot(type,logtitle,zz,zzz):
	hist_title = type[0]
	h1 = ff.Get(hist_title)
	c1 = ROOT.TCanvas('c1','c1',120,100)
	if type[6] > 0:
		h1.GetXaxis().SetRange(type[5],type[6])
	h1.Draw("colz")
	if type[7] > 0:
		h1.SetAxisRange(0,type[7],"Y")
	if type[8] > 0:
		h2 = ff.Get(type[9])
		h2.SetLineColorAlpha(2,1)
		h2.Draw("SAME")
		h2.SetAxisRange(0,110000,"Y")
		logtitle += "_Double"

	#gStyle.SetOptStat(0)
	h1.SetTitle(type[1])
	h1.SetXTitle(type[2])
	h1.SetYTitle(type[3])
	#h1.SetLineWidth(5) #use this to make line width thicker

	if logtitle == "LogON":
		c1.SetLogz()
	c1.Draw()
	c1.Print("plots/{}/original_python_pdfs/{}_{}.pdf".format(zzz,logtitle,type[0]))

"""
for kk in ff.GetListOfKeys():
  obj = kk.ReadObj()
  print(obj.GetName())
  print(obj.__class__)
  print(obj.GetTitle())
  print(obj.GetNbinsX())
  print(obj.GetNbinsY())
  #print("value of x bin is: ")
  #print(obj.GetXaxis().FindBin(100))#obj.FindLastBinAbove()))
#h1 = ff.Get("5038_H_proton_DeltaBeta_momentum_S2")#.ProjectionY("cutg",0,40,"[cutg]")#5038_Hist_deltaB_psec1_layer1")#.ProjectionY("cutg",0,40,"[cutg]")
"""
#h1 = ff.Get("5039_Hist_deltaB_psec1_layer1")#.ProjectionY("cutg",0,40,"[cutg]")

#type1 = "output_file_histos_Hist_xB_Q2"

"""FORMAT: Hist name, title, xaxis, yaxis,logON/LogOFF,xmin,xmax,ymax,1 = enable double plots,second histo name"""
type1 = ("output_file_histos_Hist_heleproTheta",
	"Proton vs. Electron Theta Angle","Proton Angle","Electron Angle",
	1,0,0,0,0,0)
type2 = ("output_file_histos_Hist_heleproThetaDVMP",
	"Proton vs. Electron Theta Angle, DVPP Candidates","Proton Angle","Electron Angle",
	1,0,0,0,0,0)
type3 = ("output_file_histos_Hist_heleTheta",
	"Electron Angle (Theta), with Proton Coincidence","Electron Angle","Counts",
	0,0,0,0,0,0)
type4 = ("output_file_histos_Hist_hproTheta","Proton Angle (Theta)","Proton Angle","Counts",
	0,0,0,110000,0,0)
type5 = ("output_file_histos_Hist_LeptHadAngle",
	"Angle Between Lepton and Hadron Planes","Angle","Counts",
	0,0,90,0,0,0)

type6 = ("output_file_histos_Hist_hproThetaCD",
	"Proton Angle (Theta) in CD (Status > 4000)","Angle","Counts",
	0,0,0,300,0,0)

type7 = ("output_file_histos_Hist_hproThetaFD",
	"Proton Angle (Theta) in FD (2000 < Status < 4000)","Angle","Counts",
	0,0,0,300,0,0)

Dtype8 = ("output_file_histos_Hist_hproThetaFD",
	"Proton Angle (Theta) in FD and in CD","Angle","Counts",
	0,0,0,200000,1,"output_file_histos_Hist_hproThetaCD")

Dtype9 = ("output_file_histos_Hist_hproThetaFDaftercuts",
	"Proton Angle (Theta) in FD and in CD after Excl. Cuts","Angle","Counts",
	0,0,0,300,1,"output_file_histos_Hist_hproThetaCDaftercuts")

#plots = [type1,type9]
#plots = [type1,type2,type3,type4,type5,type6,type7,Dtype8]
plots = [type6,type7,Dtype8,Dtype9]

os.mkdir("plots/"+zzz)
os.mkdir("plots/"+zzz+"/original_python_pdfs")

#Full xb range:
#xbRange = ["0.00", "0.10", "0.20", "0.30","0.40", "0.50", "0.60", "0.70", "0.80"]
#Full q2 range:
#q2Range = ["0.0","0.5","1.0","1.5","2.0","2.5","3.0", "3.5","4.0", "4.5","5.0", "5.5","6.0", "6.5", "7.0","7.5","8.0", "8.5"]

"""
xbRange = ["0.10", "0.20", "0.30","0.40", "0.50", "0.60", "0.70", "0.80"]
q2Range = ["1.0","1.5","2.0","2.5","3.0", "3.5","4.0", "4.5","5.0", "5.5","6.0"]
tRange = ["0.09","0.15","0.2","0.3","0.4","0.6","1.0","1.5","2","5"]

for kk in ff.GetListOfKeys():
  obj = kk.ReadObj()
  title = obj.GetName()
  if "Ultra_Phi" in title:
	if obj.GetEntries()>10 and obj.GetMaximum()>10:
	  histTitle = title
	  type9 = (title,histTitle,"Phi","Counts",0,0,0,0,0,0)
	  print(obj.GetEntries())
	  plotdistributer(type9,zz,zzz)
"""

"""
for k in range(0,len(tRange)-1):
	for j in range(0,len(q2Range)-1) :
		for i in range(0,len(xbRange)-1):
			title = "output_file_histos_Hist_Ultra_Phi{} < xB < {}_ {} < q2 < {} {} < t < {}".format(xbRange[i],xbRange[i+1],q2Range[j],q2Range[j+1],tRange[k],tRange[k+1])
			histTitle = "Counts vs. Phi, {} < xB < {}_ {} < q2 < {} {} < t < {}".format(xbRange[i],xbRange[i+1],q2Range[j],q2Range[j+1],tRange[k],tRange[k+1])
			type9 = (title,histTitle,"Phi","Counts",0,0,0,0,0,0)
			print(title)
			plotdistributer(type9,zz,zzz)
"""
"""FORMAT: Hist name, title, xaxis, yaxis,logON/LogOFF,xmin,xmax,ymax,1 = enable double plots,second histo name"""

"""
for j in range(0,len(q2Range)-1) :
	for i in range(0,len(xbRange)-1):
		title = "output_file_histos_Hist_beta_T{} < xB < {}_ {} < q2 < {}".format(xbRange[i],xbRange[i+1],q2Range[j],q2Range[j+1])
		histTitle = "Counts vs. t, {} < xB < {}_ {} < q2 < {}".format(xbRange[i],xbRange[i+1],q2Range[j],q2Range[j+1])
		type9 = (title,histTitle,"t (GeV^2)","Counts",0,0,25,1000,0,0)
		print(title)
		plotdistributer(type9,zz,zzz)
"""
for type in plots:
	plotdistributer(type,zz,zzz)