from opentrons import protocol_api
from opentrons_functions.magbeads import (
    remove_supernatant, bead_wash, transfer_elute)
from opentrons_functions.transfer import add_buffer

metadata = {'apiLevel': '2.5',
            'author': 'Jon Sanders'}

# Set to `True` to perform a short run, with brief pauses and only
# one column of samples
test_run = True

if test_run:
    pause_bind = 5*60
    pause_mag = 10*60
    pause_dry = 5*60
    pause_elute = 5*60

    # Limit columns
    cols = ['A8', 'A9']
else:
    pause_bind = 5*60
    pause_mag = 10*60
    pause_dry = 5*60
    pause_elute = 5*60

    # Limit columns
    cols = ['A1', 'A2', 'A3', 'A4', 'A5', 'A6',
            'A7', 'A8', 'A9', 'A10', 'A11', 'A12']

# bead aspiration flow rate
bead_flow = .25

# wash mix mutliplier
wash_mix = 5

tb1_cols = ['A1', 'A2']

blt_col = 'A3'

tsb_col = 'A4'

i5_col = 'A5'

# PCR MM columns
pcr_cols = ['A6', 'A7']

# Wash 1 (TWB) columns
twb_cols = ['A1', 'A2']

twb_fill = 12000

h2o_col = 'A3'

beads_col = 'A4'

# EtOH columns
eth_cols = ['A5', 'A6', 'A7']

eth_fill = 12000

def run(protocol: protocol_api.ProtocolContext):

    # ### HackFlex Illumina-compatible library prep protocol

    # ### Deck

    # 1. samples; libraries
    # 2. reagent reservoir
    # 3. reagent strip tubes
    # 4. 300 tips (wash); 200f tips (elute)
    # 5. 10f tips (samples)
    # 6. i7 primers
    # 7. 20 tips (reagents)
    # 8. 300 tips (reagents)
    # 9. 10f tips (primers)
    # 10. mag module
    # 11. waste
    # 12. trash

    # define custom labware for strip tubes block
    # reagent strip tubes:
    # 1: TB1 200 µL
    # 2: TB1 200 µL
    # 3: BLT 150 µL
    # 4: TSB 150 µL
    # 5: i5 primers 150 µL
    # 6: PCR MM 200 µL
    # 7: PCR MM 200 µL

    # buffer reservoirs:
    # 1: TWB (10 mL)
    # 2: TWB (10 mL)
    # 3: H2O (8 mL)
    # 4: beads (6 mL)
    # 5: 80% EtOH
    # 6: 80% EtOH
    # 7: 80% EtOH

    # ### Setup

    # define deck positions and labware

    # define hardware modules
    magblock = protocol.load_module('magnetic module gen2', 10)
    magblock.disengage()

    # tips
    tiprack_samples = protocol.load_labware('opentrons_96_filtertiprack_10ul',
                                            5)
    tiprack_buffers = protocol.load_labware('opentrons_96_tiprack_300ul',
                                            8)
    tiprack_wash = protocol.load_labware('opentrons_96_tiprack_300ul',
                                         4)
    tiprack_primers = protocol.load_labware('opentrons_96_filtertiprack_10ul',
                                            9)
    tiprack_reagents = protocol.load_labware('opentrons_96_tiprack_20ul',
                                             7)

    # reagents
    # should be new custom labware with strip tubes
    reagents = protocol.load_labware('tubeblockvwrpcrstriptube_96_wellplate_250ul',
                                     3, 'reagents')
    buffers = protocol.load_labware('nest_12_reservoir_15ml',
                                    2, 'wash buffers')
    waste = protocol.load_labware('nest_1_reservoir_195ml',
                                  11, 'liquid waste')

    # plates
    samples = protocol.load_labware('biorad_96_wellplate_200ul_pcr',
                                   1, 'samples')
    i7_primers = protocol.load_labware('biorad_96_wellplate_200ul_pcr',
                                       6, 'i7 primers')

    # load plate on magdeck
    # mag_plate = magblock.load_labware('vwr_96_wellplate_1000ul')
    mag_plate = magblock.load_labware('nest_96_wellplate_100ul_pcr_full_skirt')

    # initialize pipettes
    pipette_left = protocol.load_instrument('p300_multi',
                                            'left',
                                            tip_racks=[tiprack_buffers])
    pipette_right = protocol.load_instrument('p10_multi',
                                            'right',
                                            tip_racks=[tiprack_reagents])

    # TB1 wells
    tb1_wells = [reagents[x] for x in tb1_cols]

    # TWB wash wells
    twb_wells = [buffers[x] for x in twb_cols]

    # PCR MM wells
    pcr_wells = [reagents[x] for x in pcr_cols]

    # EtOH wells
    eth_wells = [buffers[x] for x in eth_cols]

    # DNA plate

    # Step 1: Tagmentation
    # Diluted BLT: 1 mL; 120 (150 µL) per tip
    # TB1: 2.4 mL; 300 (350 µL) per tip

    # add TB1.
    # buffer tips 1

    tb1_wells, tb1_remaining = add_buffer(pipette_left,
                                          tb1_wells,
                                          mag_plate,
                                          cols,
                                          25,
                                          200,
                                          tip=None,
                                          tip_vol=300,
                                          remaining=None,
                                          drop_tip=True,
                                          dead_vol=10)

    # add BLT
    # reagent tips 2

    # mix BLT first
    pipette_right.pick_up_tip()
    pipette_right.mix(10,
                      10,
                      reagents[blt_col])
    pipette_right.transfer(10,
                           reagents[blt_col],
                           [mag_plate[x] for x in cols],
                           mix_before=(2, 10),
                           new_tip='never')
    pipette_right.drop_tip()

    # add sample

    for col in cols:
        pipette_right.pick_up_tip(tiprack_samples[col])
        pipette_right.transfer(10,
                               samples[col],
                               mag_plate[col],
                               mix_after=(10, 10),
                               new_tip='never',
                               trash=False)
        pipette_right.drop_tip()

    # Prompt user to remove plate and run on thermocycler
    protocol.pause('Remove plate from magblock, seal, and run '
                   'program TAG on thermocycler. Then spin down, unseal, '
                   'and return to magblock.')

    # Step 2: Stop reaction
    # TSB: 1 mL; 120 (150 µL) per tip

    # add TSB to each sample.
    # Prompt user to remove plate and run on thermocycler

    ### Is this step going to cross-contaminate? Seems wasteful to take.
    ### new tip for each sample. z = -1 meant to help.

    # reagent tips 2
    pipette_right.transfer(10,
                           reagents[tsb_col],
                           [mag_plate[x].top(z=-1) for x in cols],
                           touch_tip=False,
                           new_tip='once')

    protocol.pause('Remove plate from magblock, seal, vortex, spin, and run '
                   'program PTC on thermocycler. Then spin down, unseal, '
                   'and return to magblock.')

    # Step 3: Cleanup
    # TWB: 20 mL; 1200 (1500 µL) per tip

    # Magnet wash 2X

    # bind for specified length of time

    protocol.comment('Binding beads to magnet.')
    magblock.engage()

    protocol.delay(seconds=pause_mag)

    # ### Do first wash: 100 µL TWB
    # buffer tips 2

    # ### Do first wash
    protocol.comment('Doing wash #1.')
    twb_remaining, twb_wells = bead_wash(
                                         # global arguments
                                         protocol,
                                         magblock,
                                         pipette_left,
                                         mag_plate,
                                         cols,
                                         # super arguments
                                         waste['A1'],
                                         tiprack_wash,
                                         # wash buffer arguments
                                         twb_wells,
                                         twb_fill/8,
                                         # mix arguments
                                         tiprack_wash,
                                         # optional arguments,
                                         resuspend_beads=False,
                                         super_vol=70,
                                         rate=bead_flow,
                                         wash_vol=100,
                                         drop_super_tip=False,
                                         touch_wash_tip=True,
                                         mix_n=wash_mix,
                                         mix_vol=90,
                                         wash_tip_vol=300,
                                         super_tip_vol=200,
                                         remaining=None,
                                         pause_s=pause_mag)

    # ### Do second wash: 100 µL TWB
    # buffer tips 3
    protocol.comment('Doing wash #2.')

    twb_remaining, twb_wells = bead_wash(
                                         # global arguments
                                         protocol,
                                         magblock,
                                         pipette_left,
                                         mag_plate,
                                         cols,
                                         # super arguments
                                         waste['A1'],
                                         tiprack_wash,
                                         # wash buffer arguments
                                         twb_wells,
                                         twb_fill/8,
                                         # mix arguments
                                         tiprack_wash,
                                         # optional arguments,
                                         resuspend_beads=False,
                                         super_vol=120,
                                         rate=bead_flow,
                                         wash_vol=100,
                                         drop_super_tip=False,
                                         touch_wash_tip=True,
                                         mix_n=wash_mix,
                                         mix_vol=90,
                                         wash_tip_vol=300,
                                         super_tip_vol=200,
                                         remaining=twb_remaining,
                                         pause_s=pause_mag)

    # remove supernatant
    remove_supernatant(pipette_left,
                       mag_plate,
                       cols,
                       tiprack_wash,
                       waste['A1'],
                       super_vol=120,
                       rate=bead_flow,
                       bottom_offset=.8,
                       drop_tip=False)

    magblock.disengage()
    # remove supernatant
    # Step 3: amplification
    # MM: 3 mL; 350 (400 µL) per tip
    # buffer tips 4

    # Prompt user to unseal i7 PCR plate

    protocol.pause('Now, unseal i7 PCR primer plate, and uncap PCR master'
                   ' mix and i5 primer strip tubes.')

    pcr_wells, pcr_remaining = add_buffer(pipette_left,
                                          pcr_wells,
                                          mag_plate,
                                          cols,
                                          30,
                                          200,
                                          tip=None,
                                          tip_vol=300,
                                          remaining=None,
                                          drop_tip=True,
                                          dead_vol=10)

    # plate: primers i5
    # reagent tips 3
    pipette_right.transfer(10,
                           reagents[i5_col],
                           [mag_plate[x].top(z=-1) for x in cols],
                           touch_tip=True,
                           new_tip='once')

    # plate: primers i7
    for col in cols:
        pipette_right.pick_up_tip(tiprack_primers[col])
        pipette_right.transfer(10,
                               i7_primers[col],
                               mag_plate[col],
                               touch_tip=True,
                               new_tip='never',
                               trash=False)
        pipette_right.drop_tip()

    # Prompt user to remove plate and run on thermocycler

    protocol.pause('Remove plate from magblock, seal, vortex, spin and run '
                   'amplification program on thermocycler. Click continue when'
                   ' done.')

    protocol.delay(seconds=2)
    # Step 4: Size selection
    # H2O: 72 µL per sample; 7.2 mL; 600 per tip
    # EtOH: 400 µL per sample; 5000 per tip
    # Beads: 6 mL; 720 µL per tip

    # Prompt user to remove plate and run on thermocycler

    protocol.pause('Replace tips in position {0}'
                   ' with a fresh box'.format(tiprack_wash.parent))

    protocol.delay(seconds=2)

    protocol.pause('Remove sample plate from position {0}, seal, and store. '
                   'Place a new, clean, 96-well Nest PCR plate in position'
                   ' {0}. Remove library plate from PCR machine and spin at '
                   '280 xg for one minute. Then unseal and return to'
                   ' magblock.'.format(samples.parent))

    protocol.comment('Binding beads to magnet.')
    magblock.engage()

    protocol.delay(seconds=pause_mag)

    # Add buffers for large-cut size selection to new plate
    protocol.comment('Preparing large-cut bead conditions in new plate.')

    # add 40 µL H2O
    # buffer tips 5
    pipette_left.distribute(40,
                            buffers[h2o_col],
                            [samples[x].top(z=-1) for x in cols],
                            touch_tip=True,
                            disposal_volume=10,
                            new_tip='once')

    # Add 45 µL SPRI beads
    # buffer tips 6
    pipette_left.pick_up_tip()
    pipette_left.mix(10, 200, buffers[beads_col])
    pipette_left.distribute(45,
                            buffers[beads_col],
                            [samples[x].top(z=-1) for x in cols],
                            mix_before=(2,40),
                            touch_tip=True,
                            disposal_volume=10,
                            new_tip='never')
    pipette_left.drop_tip()

    # Transfer 45 µL PCR supernatant to new plate
    transfer_elute(pipette_left,
                   mag_plate,
                   samples,
                   cols,
                   tiprack_wash,
                   45,
                   z_offset=0.5,
                   x_offset=1,
                   rate=0.25,
                   drop_tip=False,
                   mix_n=7,
                   mix_vol=100)

    protocol.pause('Remove and discard plate from mag block. '
                   'Move plate in position {0} to mag block, and replace '
                   'with a new, clean 96-well Nest PCR plate.'.format(
                    samples.parent))

    protocol.comment('Binding beads to magnet.')
    magblock.engage()
    protocol.delay(seconds=pause_mag)

    # Add buffers for small-cut size selection to new plate
    # Add 20 µL SPRI beads
    # buffer tips 7
    pipette_left.pick_up_tip()
    pipette_left.mix(10, 100, buffers[beads_col])
    pipette_left.distribute(20,
                            buffers[beads_col],
                            [samples[x] for x in cols],
                            mix_before=(2,15),
                            touch_tip=True,
                            new_tip='never')
    pipette_left.drop_tip()

    # Transfer 115 µL large-cut supernatant to new plate

    transfer_elute(pipette_left,
                   mag_plate,
                   samples,
                   cols,
                   tiprack_wash,
                   115,
                   z_offset=0.5,
                   x_offset=1,
                   rate=0.25,
                   drop_tip=False,
                   mix_n=7,
                   mix_vol=100)

    protocol.pause('Remove and discard plate from mag block. '
                   'Move plate in position {0} to mag block, and replace '
                   'with a new, clean 96-well **BioRad** PCR plate. '
                   'Also, fill Ethanol wells at this stage.'.format(
                    samples.parent))

    protocol.comment('Binding beads to magnet.')
    magblock.engage()
    protocol.delay(seconds=pause_mag)

    # ### Do first wash: 150 µL EtOH
    # buffer tips 8
    protocol.comment('Doing wash #1.')
    eth_remaining, eth_wells = bead_wash(# global arguments
                                         protocol,
                                         magblock,
                                         pipette_left,
                                         mag_plate,
                                         cols,
                                         # super arguments
                                         waste['A1'],
                                         tiprack_wash,
                                         # wash buffer arguments,
                                         eth_wells,
                                         eth_fill/8,
                                         # mix arguments
                                         tiprack_wash,
                                         # optional arguments
                                         resuspend_beads=False,
                                         wash_vol=150,
                                         super_vol=125,
                                         drop_super_tip=False,
                                         mix_n=0,
                                         mix_vol=140,
                                         remaining=None,
                                         pause_s=pause_mag)

    # ### Do first wash: 150 µL EtOH
    # buffer tips 9
    protocol.comment('Doing wash #2.')
    eth_remaining, eth_wells = bead_wash(# global arguments
                                         protocol,
                                         magblock,
                                         pipette_left,
                                         mag_plate,
                                         cols,
                                         # super arguments
                                         waste['A1'],
                                         tiprack_wash,
                                         # wash buffer arguments,
                                         eth_wells,
                                         eth_fill/8,
                                         # mix arguments
                                         tiprack_wash,
                                         # optional arguments
                                         resuspend_beads=False,
                                         wash_vol=150,
                                         super_vol=170,
                                         drop_super_tip=False,
                                         mix_n=0,
                                         mix_vol=140,
                                         remaining=eth_remaining,
                                         pause_s=pause_mag)

    # ### Dry
    protocol.comment('Removing wash and drying beads.')

    # This should:
    # - pick up tip in position 8
    # - pick up supernatant from magplate
    # - dispense in waste, position 11
    # - repeat
    # - trash tip
    # - leave magnet engaged

    # remove supernatant

    remove_supernatant(pipette_left,
                       mag_plate,
                       cols,
                       tiprack_wash,
                       waste['A1'],
                       super_vol=170,
                       rate=bead_flow,
                       bottom_offset=.3,
                       drop_tip=True)

    # dry

    protocol.delay(seconds=pause_dry)

    protocol.pause('Replace empty tiprack in position {0} with new rack of '
                   '200 µL filter tips.'.format(tiprack_wash.parent))

    # ### Elute
    protocol.comment('Eluting DNA from beads.')

    # This should:
    # - disengage magnet
    # - pick up tip from position 6
    # - pick up reagents from column 2 of position 9
    # - dispense into magplate
    # - mix 10 times
    # - blow out, touch tip
    # - return tip to position 6
    # - wait (5 seconds)
    # - engage magnet
    # - wait (5 seconds)
    # - pick up tip from position 6
    # - aspirate from magplate
    # - dispense to position 3
    # - trash tip

    # transfer elution buffer to mag plate

    magblock.disengage()

    # add elution buffer and mix
    for col in cols:
        pipette_left.pick_up_tip(tiprack_wash.wells_by_name()[col])
        pipette_left.aspirate(35, buffers[h2o_col], rate=1)
        pipette_left.dispense(35, mag_plate[col].bottom(z=1))
        pipette_left.mix(10, 29, mag_plate[col].bottom(z=1))
        pipette_left.blow_out(mag_plate[col].top())
        pipette_left.touch_tip()
        # we'll use these same tips for final transfer
        pipette_left.return_tip()

    protocol.delay(seconds=pause_elute)

    # bind to magnet
    protocol.comment('Binding beads to magnet.')

    magblock.engage()

    protocol.delay(seconds=pause_mag)

    protocol.comment('Transferring eluted DNA to final plate.')

    transfer_elute(pipette_left,
                   mag_plate,
                   samples,
                   cols,
                   tiprack_wash,
                   30,
                   z_offset=1.5,
                   x_offset=1,
                   rate=0.1,
                   drop_tip=True)

    magblock.disengage()


#-*- coding:utf-8 -*-
import sys
import os
import math
import logging
import cPickle
import time
import re

class myTools(object):
    @staticmethod
    def mkdir(path, trash=False, clear=True):
        '''
        创建文件夹，
        :param trash: True， 表示，如果存在该文件夹，1、将该文件夹重命名为 .Trash 文件夹 2、在建立该文件夹
        '''
        if not os.path.exists(path):
            os.mkdir(path)
        elif trash:
            os.system('''cp -rf {0}  .Trash '''.format(path))
            os.system('''rm -rf {0}  '''.format(path))
            os.mkdir(path)
        if clear:
            try:
                os.system('''rm -rf .Trash  ''')
            except:
                pass
    @staticmethod
    def cp(from_path, to_path, is_dir=False):
        '''
        复制文件
        :param from_path: 原文件
        :param to_path: 复制后的文件
        '''
        if is_dir:
            os.system(''' cp -rf {0} {1} ;'''.format(from_path, to_path))
        else:
            os.system(''' cp {0} {1} ;'''.format(from_path, to_path))

class Process(object):
    def __init__(self, split_data=None, target=None, args=(), kwargs={}, logger='', dis_n=10, save_path='', name='', keep_dis=False, dis_files=[], is_class=False, broadcast=[], run_py=''):
        self._target = target.__name__          # 目标函数
        self._split_data = split_data           # 拆分数据
        self._args = args                       # 函数参数
        self._kwargs = kwargs                   # 函数参数
        self._dis_n = dis_n                     # 进程数
        self._name = name if name != '' else 'dis_run'
        self._is_class = is_class
        # 设置路径
        self._root_path = os.getcwd()               # 运行脚本的路径
        self._save_path = save_path + os.sep + self._name if save_path != '' else self._root_path + os.sep + self._name
        self._keep_dis = keep_dis                   # 保留中间文件
        # self._dis_py = sys.argv[0].split('.')[0]
        self._broadcast = broadcast                 # 广播变量
        self._dis_run_py = 'multi_dis.py'
        self._dis_py = sys.argv[0].split('.')[0] if run_py == '' else run_py   # 调用的文件
        self._dis_data = 'split_data.pkl'
        self._dis_res = 'split_result.pkl'
        self.results = None
        self._file_path = os.path.dirname(os.path.abspath(__file__))            # multi 的路径
        self._dis_files = [self._file_path] + dis_files     # 广播文件
        if logger == '':
            self._setLogger()
        else:
            self._getLogger(logger)

    def _logger(mon_str):
        def wrapper_func(func):
            def wrapper_args(self, *args, **kwargs):
                self.logger.info('{0} ...'.format(mon_str))
                result = func(self, *args, **kwargs)
                self.logger.info('{0} Finish !'.format(mon_str))
                return result
            return wrapper_args
        return wrapper_func

    def _getLogger(self, logger):
        '''
        取得log日志
        '''
        self.logger = logger

    def _setLogger(self):
        '''
        设置log日志
        '''
        logging.basicConfig(level=logging.INFO,
                            format='%(asctime)s (%(filename)s) [line:%(lineno)d] [ %(levelname)s ] %(message)s',
                            datefmt='%a, %d %b %Y %H:%M:%S',
                            # filename='ModelSolve.log',
                            # filemode='w'
                            )
        self.logger = logging.getLogger("SubProcess")

    def _monitor(self, dis_n, dis_file):
        '''
        监控所有进程信息
        '''
        def findProcess(dis_path):
            '''
            监控单个进程运行状态
            '''
            dis_log = dis_path + os.sep + 'run_log.log'
            dis_pro = os.popen(''' tail -n 1 {0} '''.format(dis_log)).readlines()
            if len(dis_pro) > 0:
                dis_pro_str = dis_pro[0]
                dis_pro_str_re = re.findall('(\(.*?/.*?\))', dis_pro_str)
                log_str = dis_pro_str_re[0] if len(dis_pro_str_re) > 0 else '( --/-- )'
            else:
                log_str = '( 0/0 )'
            return log_str

        def findError(file_name):
            content = ''.join(os.popen(''' cat {0} '''.format(file_name)).readlines())
            re_find = re.findall(r'(Error|Traceback|Exception)', content)
            if len(re_find) > 0:
                return [True, content]
            return [False, '-1']

        check_dis = ['dis_{0}'.format(i) for i in range(dis_n)]
        check_dis_len = dis_n + 1
        check_path = self._save_path
        check_file = check_path + os.sep + 'result_monitor.log'
        last_dis_len = 0
        while (check_dis_len > last_dis_len):
            time.sleep(5)               # 每 5 秒显示一次进程状态
            check_content = os.popen(''' cat {0} '''.format(check_file)).readlines()
            last_dis_len = len(check_content)
            if last_dis_len > 1:
                complete_dis = map(lambda x: x.split('\t')[0].replace(' ', ''), check_content[1:])
                rest_dis = list(set(check_dis) - set(complete_dis))
                rest_dis_str_list = map(lambda x: x + findProcess(check_path + os.sep + x), rest_dis)
                rest_dis_len = len(rest_dis_str_list)
                rest_dis_str = ', '.join(rest_dis_str_list)
            else:
                rest_dis = check_dis
                rest_dis_str_list = map(lambda x: x + findProcess(check_path + os.sep + x), rest_dis)
                rest_dis_len = dis_n
                rest_dis_str = ', '.join(rest_dis_str_list)
            for check_error in rest_dis:
                check_i = check_error.split('_')[1]
                check_error_path = check_path + os.sep + check_error + os.sep + 'nohup_multi_dis_{0}.log'.format(check_i)
                is_error, content = findError(check_error_path)
                if is_error:
                    self._killSubProcess()
                    raise Exception(content)
            self.logger.info('[ ({0}) Dis is Running ] : {1}'.format(rest_dis_len, rest_dis_str))

    @_logger('Kill Sub Process')
    def _killSubProcess(self):
        os.chdir(self._save_path)
        os.system(''' bash  {0}'''.format(self._save_path + os.sep + 'process_monitor.sh'))

    @_logger('Make Monitor')
    def _mkMonitor(self):
        '''
        创建监控子进程文件
        '''
        os.system(''' echo '-------------------------------' >  {0} '''.format(self._save_path + os.sep + 'result_monitor.log'))
        os.system(''' echo '#!/bin/bash' >  {0}'''.format(self._save_path + os.sep + 'process_monitor.sh'))
        return 0

    @_logger('Split the Data')
    def _splitData(self):
        '''
        拆分数据
        '''
        all_data_len = len(self._split_data)
        self.split_step = int(math.ceil(all_data_len / self._dis_n))
        return 0

    @_logger('Distribute the Data')
    def _disRun(self, run_mode=0):
        '''
        建立分布文件。启动多个进程。
        run_mode:   0 - 拆分数据 + 分布式运行
                    1 - 拆分数据
                    2 - 分布式运行
        '''
        if run_mode != 2:
            myTools.mkdir(self._save_path, trash=True)
        self._mkMonitor()
        for i in xrange(self._dis_n):
            sub_path = self._save_path + os.sep + 'dis_{0}'.format(i)
            if run_mode != 2:
                myTools.mkdir(sub_path)
                tmp_split = [[self._split_data[i*self.split_step: (i+1)*self.split_step]] + self._broadcast,
                             # self._target,
                             self._args,
                             self._kwargs,
                             self._is_class]
                with open(sub_path + os.sep + self._dis_data, 'wb') as f:
                    cPickle.dump(tmp_split, f)
            # myTools.cp(self._root_path + os.sep + '{0}.py'.format(self._dis_py), sub_path + '/.')
            myTools.cp(self._file_path + os.sep + self._dis_run_py, sub_path + '/.')
            myTools.cp(self._file_path + os.sep + '__init__.py', sub_path + '/.')
            for each in self._dis_files:
                is_dir = os.path.isdir(each)
                myTools.cp(each, sub_path + '/.', is_dir=is_dir)
            if run_mode != 1:
                self._disRunPython(sub_path, 'dis_{0}'.format(i))
                self.logger.info('Run the Sub Process [ {0} ]'.format(i))
        return 0

    def _disRunPython(self, sub_path, mon_num):
        '''
        启动python文件处理
        '''
        os.chdir(sub_path)
        os.system(''' nohup python {0} {1} {2} {3} {4} {5} >  nohup_multi_{5}.log  2>&1 &'''.format(
                      self._dis_run_py, self._dis_py, self._target, self._dis_data, self._dis_res, mon_num))
        os.chdir(self._root_path)

    @_logger('Read Dis Files')
    def _readDisFiles(self):
        '''
        读取分布的结果文件
        '''
        dis_res = []
        for each in os.listdir(self._save_path):
            if os.path.isdir(self._save_path + os.sep + each):
                this_dis_path = self._save_path + os.sep + each + os.sep + self._dis_res
                with open(this_dis_path, 'rb') as f:
                    this_dis_data = cPickle.load(f)
                    dis_res.extend(this_dis_data)
        self.results = dis_res
        return self.results

    @_logger('Clean Dis Files')
    def _cleanDisFiles(self):
        os.system(''' rm -rf {0} '''.format(self._save_path))
        return 0

    def start(self, dis_n=-1):
        self._dis_n = dis_n if dis_n > 0 else self._dis_n
        self._splitData()                           # 2. 拆分数据
        self._disRun()                              # 3. 启动多个进程
        self._monitor(self._dis_n, self._name)      # 启动进程监控
        self._readDisFiles()                        # 4. 读取分布式数据
        if not self._keep_dis:
            self._cleanDisFiles()
        return self.results

    def split(self):
        self._splitData()
        self._disRun(run_mode=1)
        return 0

    def run(self):
        self._disRun(run_mode=2)
        self._monitor(self._dis_n, self._name)      # 启动进程监控
        self._readDisFiles()                        # 4. 读取分布式数据
        if not self._keep_dis:
            self._cleanDisFiles()
        return self.results


import requests
import lxml.html
from datetime import datetime
from time import sleep

class Crawler(object):
    def __init__(self,
                 base_url='https://www.csie.ntu.edu.tw/news/',
                 rel_url='news.php?class=101'):
        self.base_url = base_url
        self.rel_url = rel_url

    def crawl(self, start_date, end_date,
              date_thres=datetime(2012, 1, 1)):
        """Main crawl API

        1. Note that you need to sleep 0.1 seconds for any request.
        2. It is welcome to modify TA's template.
        """
        if end_date < date_thres:
            end_date = date_thres
        contents = list()
        page_num = 0
        while True:
            rets, last_date = self.crawl_page(
                start_date, end_date, page=f'&no={page_num}')
            page_num += 10
            if rets:
                contents += rets
            if last_date < start_date:
                break
        return contents

    def crawl_page(self, start_date, end_date, page=''):
        """Parse ten rows of the given page

        Parameters:
            start_date (datetime): the start date (included)
            end_date (datetime): the end date (included)
            page (str): the relative url specified page num

        Returns:
            content (list): a list of date, title, and content
            last_date (datetime): the smallest date in the page
        """
        html = requests.get(
            self.base_url + self.rel_url + page,
            headers={'Accept-Language':
                     'zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-CN;q=0.6'}
        )
        doc = lxml.html.fromstring(html.content)
        sleep(0.1)
        # TODO: parse the response and get dates, titles and relative url with etree
        contents = list()
        table = doc.xpath('.//table[@id="RSS_Table_page_news_1"]')[0]
        body = table.xpath('tbody')[0]
        total_news = body.xpath('.//tr')
        rel_urls = []
        dates = []
        titles = []

        for news in total_news:
            date = news[0].text
            date = datetime.strptime(date, "%Y-%m-%d")
            last_date = date
            rel_url_path = news.xpath('.//a')[0]
            title = rel_url_path.text
            title = title.replace('"','""')
            title = title.split()
            title = ','.join(title)
            rel_url = rel_url_path.get('href')
            if last_date >= start_date and last_date <= end_date:
                dates.append(date)
                titles.append(title)
                rel_urls.append(rel_url)

        i = 0
        for rel_url in rel_urls:
            # TODO: 1. concatenate relative url to full url
            #       2. for each url call self.crawl_content
            #          to crawl the content
            #       3. append the date, title and content to
            #          contents

            full_url = self.base_url + rel_url
            contents.append(dates[i])
            contents.append(titles[i])
            contents.append(self.crawl_content(full_url))
            i += 1
        return contents, last_date

    def crawl_content(self, url):
        """Crawl the content of given url

        For example, if the url is
        https://www.csie.ntu.edu.tw/news/news.php?Sn=15216
        then you are to crawl contents of
        ``Title : 我與DeepMind的A.I.研究之路, My A.I. Journey with DeepMind Date : 2019-12-27 2:20pm-3:30pm Location : R103, CSIE Speaker : 黃士傑博士, DeepMind Hosted by : Prof. Shou-De Lin Abstract: 我將與同學們分享，我博士班研究到加入DeepMind所參與的projects (AlphaGo, AlphaStar與AlphaZero)，以及從我個人與DeepMind的視角對未來AI發展的展望。 Biography: 黃士傑, Aja Huang 台灣人，國立臺灣師範大學資訊工程研究所博士，現為DeepMind Staff Research Scientist。``
        """
        html = requests.get(url)
        doc = lxml.html.fromstring(html.content)
        all_editor_content = doc.xpath('.//div[@class="editor content"]')
        content = ''
        for editor_content in all_editor_content:
           content += ','.join(map(str.strip, editor_content.xpath(".//text()")))
        content = content.replace('"','""')
        content = content.split()
        content = ','.join(content)
        return content
        raise NotImplementedError


import gym
from agent.mountain_car import MountainCarAgent
import pickle

experiment_path = '../results/mountain-car-experiment-1'
episode_count = 10
max_steps = 200
discount = 0.99
max_plan_steps = 1000
plan_tol = 1e-5

env = gym.make('MountainCar-v0')
agent = MountainCarAgent(env.action_space, env.observation_space, discount)

env.monitor.start(experiment_path)
for i in range(episode_count):
    observation = env.reset()
    for j in range(max_steps):
        if i == 0:
            action = env.action_space.sample()
        else:
            action = agent.act(observation)
        result, _, done, _ = env.step(action)
        agent.add_transition(observation, action, result)
        observation = result
        if done:
            break
    agent.model()
    agent.plan(max_plan_steps, plan_tol)
env.monitor.close()

with open(experiment_path + '/agent.pickle', 'wb') as file:
    pickle.dump(agent, file, pickle.HIGHEST_PROTOCOL)


import matplotlib.pyplot as plt
import scipy.special as special
import numpy as np

def step_fun(point, a, b, deltaX, N, x, K, m, sigma):
    # массив для хранения количества xi, попавших в интервал
    sum = []
    # массив выборочных вероятностей
    F = [0]
    # массив для хранения координат по оси Х
    fx = [a]
    for q in range(K):
        sum.append(0)
        # правая граница интервала
        b = a + (q + 1) * deltaX
        fx.append(b)
        for i in range(len(x)):
            # если xi попадает в интервал, то добавляем в массив сумм
            if a <= x[i] < b:
                sum[q] += 1
        F.append(sum[q] / N)
    # строим ступенчатую диаграмму и показываем графики
    plt.step(fx, F)
    if point == 0:
        plt.title("Равномерное распределение")
        fx_even(a, b, x)
    elif point == 1:
        plt.title("Нормальное распределение")
        fx_gauss(x, m, sigma)
    elif point == 2:
        plt.title("Распределение Рэлея")
        fx_rayleigh(x, sigma)
    plt.show()

# функция равномерного распределения
def fx_even(a, b, fx):
    fy = []
    for i in range(len(fx)):
        fy.append(((fx[i] - a) / (b - a)))
    plt.plot(fx, fy, color='r')

# функция гаусовского распределения
def fx_gauss(fx, m, sigma):
    fy = []
    for i in range(len(fx)):
        fy.append((1 / 2) * (1 + special.erf((fx[i] - m) / (np.sqrt(2 * sigma**2)))))
    plt.plot(fx, fy, color='r')

# функция рэлеевского распределения
def fx_rayleigh(fx, sigma):
    fy = []
    for i in range(len(fx)):
        fy.append(1 - np.exp(-1 * (fx[i] ** 2) / (2 * sigma ** 2)))
    plt.plot(fx, fy, color='r')


from LipNet import LipNet
from LipNet_Generator import DataGenerator
from datetime import datetime
from keras.callbacks import CSVLogger, Callback
from keras import backend as K
from keras.optimizers import Adam
import os
import sys

class LearningRateChanger(Callback):

    def __init__(self, kind=1, verbose=1):
        super(Callback, self).__init__()
        self.verbose = verbose
        self.best_val_score = -1.
        self.kind = kind
        self.wait = 0
        self.second_start = 11
        self.second_end = 11
        self.third_start = 12
        self.patience = 10
        if self.kind == 2:
            self.second_start = 10
            self.second_end = 18
            self.third_start = 19
            self.patience = 10

    def on_epoch_end(self, epoch, logs=None):
        current_val_score = logs.get('val_acc')
        self.best_val_score = max(self.best_val_score, current_val_score)

        if self.second_start <= epoch <= self.second_end:
            new_learning_rate = 0.0003
            K.set_value(self.model.optimizer.lr, new_learning_rate)
        elif epoch >= self.third_start:
            new_learning_rate = 0.0001
            K.set_value(self.model.optimizer.lr, new_learning_rate)
            if current_val_score > self.best_val_score:
                self.best_val_score = current_val_score
                self.wait = 0
            else:
                self.wait += 1
                if self.wait >= self.patience:
                    if self.verbose > 0:
                        print('\nEpoch {}: early stopping'.format(epoch))
                    self.model.stop_training = True

if __name__ == '__main__':

    outman = 'hpk'
    group = 6  # 1: open apps  2: preferences  3: wechat  4: edit text  5: notification
    lr_type = 1

    if len(sys.argv) == 4:
        print(str(sys.argv))
        group = int(sys.argv[1])
        outman = sys.argv[2]
        lr_type = int(sys.argv[3])

    gestureIDs = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    if group == 1:
        gestureIDs = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    elif group == 2:
        gestureIDs = [11, 12, 13, 14, 15, 16, 17, 42, 43]
    elif group == 3:
        gestureIDs = [19, 20, 22, 23, 24, 25, 26]
    elif group == 4:
        gestureIDs = [27, 28, 29, 30, 31, 33, 34, 35]
    elif group == 5:
        gestureIDs = [36, 37, 38, 39, 40, 41]
    elif group == 6:
        gestureIDs = [11, 12, 13, 14, 15, 16, 17, 42, 43, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    elif group == 7:
        gestureIDs = [11, 12, 13, 14, 15, 16, 17, 42, 43, 19, 20, 22, 23, 24, 25, 26]
    elif group == 8:
        gestureIDs = [11, 12, 13, 14, 15, 16, 17, 42, 43, 27, 28, 29, 30, 31, 33, 34, 35]
    elif group == 9:
        gestureIDs = [11, 12, 13, 14, 15, 16, 17, 42, 43, 36, 37]

    ttFolder = '../resource/new_training_testing_list/group_' + str(group) + '/'
    training_filename = ttFolder + 'training_list_' + outman + '_out.txt'
    with open(training_filename) as f:
        list_training_IDs = f.readlines()
        list_training_IDs = [x.strip() for x in list_training_IDs]

    validation_filename = ttFolder + 'testing_list_' + outman + '_out.txt'
    with open(validation_filename) as f:
        list_validation_IDs = f.readlines()
        list_validation_IDs = [x.strip() for x in list_validation_IDs]

    batch_size = 32
    epochs = 45
    class_n = len(gestureIDs)

    training_generator = DataGenerator(class_n=class_n, frames_n=70, img_h=80, img_w=100, img_c=3,
                                       batch_size=batch_size, shuffle=True, gestureIDs=gestureIDs).generate(list_training_IDs)
    validation_generator = DataGenerator(class_n=class_n, frames_n=70, img_h=80, img_w=100, img_c=3,
                                         batch_size=batch_size, shuffle=True, gestureIDs=gestureIDs).generate(list_validation_IDs)

    # create model
    lipnet = LipNet(frames_n=70, img_h=80, img_w=100, img_c=3, output_n=class_n)
    # lipnet.summary()

    adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
    lipnet.model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])

    outputFolder = '../resource/new_model/group_' + str(group) + '/'
    if not os.path.exists(outputFolder):
        os.mkdir(outputFolder)

    now = datetime.now().strftime("%Y-%m-%d-%H")
    csv_logger = CSVLogger(outputFolder + outman + '_out_' + now + '_log.csv', separator=',', append=False)

    lr_changer = LearningRateChanger(kind=lr_type, verbose=1)

    lipnet.model.fit_generator(generator=training_generator,
                               steps_per_epoch=len(list_training_IDs)/batch_size,
                               epochs=epochs,
                               verbose=1,
                               callbacks=[csv_logger, lr_changer],
                               validation_data=validation_generator,
                               validation_steps=len(list_validation_IDs)/batch_size)

    lipnet.model.save(outputFolder + outman + '_out_' + now + '_model.h5')


# -*- coding: utf-8 -*-
import requests, json
from requests.exceptions import ConnectionError
import tokens

#  Метод для корректной обработки строк в кодировке UTF-8 как в Python 3, так и в Python 2
import sys

if sys.version_info < (3,):
    def u(x):
        try:
            return x.encode("utf8")
        except UnicodeDecodeError:
            return x
else:
    def u(x):
        if type(x) == type(b''):
            return x.decode('utf8')
        else:
            return x

# --- Входные данные ---
#  Адрес сервиса Keywords для отправки JSON-запросов (регистрозависимый)

CampaignsURL = 'https://api-sandbox.direct.yandex.com/json/v5/keywords'
#CampaignsURL = 'https://api.direct.yandex.com/json/v5/keywords'

# --- Подготовка, выполнение и обработка запроса ---
#  Создание HTTP-заголовков запроса
headers = {"Authorization": "Bearer " + tokens.token,  # OAuth-токен. Использование слова Bearer обязательно
           "Accept-Language": "ru",  # Язык ответных сообщений
           }

LimitedBy = 10000

# Создание тела запроса
body = {"method": "get",  # Используемый метод.
        "params": {"SelectionCriteria": {
                        "CampaignIds":[""],
                        },
                   "FieldNames": [
                        "Id", "Keyword", "AdGroupId", "CampaignId",# "Status", "State", "Bid",
                        ],  # Имена параметров, которые требуется получить.
                    "Page": {
                       "Limit": (long),
                        "Offset": (LimitedBy)
                    }
                  },
        }

# Кодирование тела запроса в JSON
jsonBody = json.dumps(body, ensure_ascii=False).encode('utf8')

# Выполнение запроса
try:
    result = requests.post(CampaignsURL, jsonBody, headers=headers)

    # Отладочная информация
    # print("Заголовки запроса: {}".format(result.request.headers))
    # print("Запрос: {}".format(u(result.request.body)))
    # print("Заголовки ответа: {}".format(result.headers))
    # print("Ответ: {}".format(u(result.text)))
    # print("\n")

    # Обработка запроса
    if result.status_code != 200 or result.json().get("error", False):
        print("Произошла ошибка при обращении к серверу API Директа.")
        print("Код ошибки: {}".format(result.json()["error"]["error_code"]))
        print("Описание ошибки: {}".format(u(result.json()["error"]["error_detail"])))
        print("RequestId: {}".format(result.headers.get("RequestId", False)))
    else:
#        print("RequestId: {}".format(result.headers.get("RequestId", False)))
#        print("Информация о баллах: {}".format(result.headers.get("Units", False)))

# Put all keywords in 1 list
        kw_list = []
        for campaign in result.json()["result"]["Keywords"]:
            kw_list.append(campaign['Keyword'])

# Вывод списка
#        print("Amount: ", len(kw_list))
        print(kw_list)

        print("{}, {}, {}".format(
                campaign['Keyword'],
                campaign["CampaignId"],
                campaign['AdGroupId'],
#               campaign['Status'],
#               campaign['State'],
#               campaign['Bid'],
                ))

        if result.json()['result'].get('LimitedBy', False):
            # Если ответ содержит параметр LimitedBy, значит,  были получены не все доступные объекты.
            # В этом случае следует выполнить дополнительные запросы для получения всех объектов.
            # Подробное описание постраничной выборки - https://tech.yandex.ru/direct/doc/dg/best-practice/get-docpage/#page
            print ("Limit {}".format(result.json()['result']['LimitedBy']))
            print("Получены не все доступные объекты.")

# Обработка ошибки, если не удалось соединиться с сервером API Директа
except ConnectionError:
    # В данном случае мы рекомендуем повторить запрос позднее
    print("Произошла ошибка соединения с сервером API.")

# Если возникла какая-либо другая ошибка
except:
    # В данном случае мы рекомендуем проанализировать действия приложения
    print("Произошла непредвиденная ошибка.")


from bs4 import BeautifulSoup as bs
import requests, re
import lib
import shutil

class users:
     def __init__(self, cookies):
           try: shutil.rmtree("lib/__pycache__")
           except: pass
           self.cookies = cookies

     def getGrup(self, next=None):
           use = [[[],[]]]
           url = "https://m.facebook.com"
           req = requests.get(f"{url}{next}", cookies={'cookie': self.cookies})
           nama = parser = bs(req.text.encode("utf-8"), "html.parser").title.text
           print(f"               \x1b[30;1m: \x1b[32;1m{nama}")
           while True:
                grup = requests.get(f"{url}{next}", cookies={'cookie': self.cookies})
                parser = bs(grup.text.encode("utf-8"), "html.parser")
                if parser.find_all("a", class_="bd") != []:
                           div = parser.find_all("a", class_="bd")
                else:
                           div = parser.find_all("a", class_="bf")
                for i in div:
                     try:
                         id = re.findall('/?id=(.*?)&', i['href'])
                         if len(id) == 0:
                              id = re.findall('/(.*?)\\?ref=', i['href'])
                              if len(id) == 0:
                                    id = re.findall('/\\?uid=(.*?)&', i['href'])
                     except TypeError:
                          id = []
                     if len(id) == 1:
                             use[0][0].append(id[0])
                             use[0][1].append(i.text)

                if 'Lihat Selengkapnya' in str(parser):
                      next = parser.find('a', string='Lihat Selengkapnya')['href']
                else:
                      break

           print(f"\x1b[32;1m(\x1b[36;1m*\x1b[32;1m) \x1b[36;1mTOTAL USER \x1b[30;1m: \x1b[32;1m{len(use[0][1])}\n\x1b[36;1m─────────────────────────\n")
           lib.setting.seting(use, self.cookies)

     def getUser(self, count, next=None):
           use = [[[],[]]]
           status = False
           url = "https://m.facebook.com"
           while True:
               if "https://m.facebook.com/search/people/" in str(next):
                        teman = requests.get(f"{next}", cookies={'cookie': self.cookies})
                        parser = bs(teman.text.encode("utf-8"), "html.parser")
                        div = parser.find_all('a')
               else:
                        teman = requests.get(f"{url}{next}", cookies={'cookie': self.cookies})
                        parser = bs(teman.text.encode("utf-8"), "html.parser")
                        div = parser.find_all(style='vertical-align: middle')
               for i in div:
                   if i.find("a") == None:
                         x = i.find("div")
                         if x == None:
                              continue
                         else:
                              try:
                                   id = re.findall('\\?id=(.*?)&', str(i))
                                   if len(id) == 0:
                                         id = re.findall('/(.*?)\\?refid=', str(i))
                              except TypeError:
                                   id = []
                   else:
                        x = i.find("a")
                        try:
                             id = re.findall('/?id=(.*?)&', x['href'])
                             if len(id) == 0:
                                  id = re.findall('/(.*?)\\?fref=', x['href'])
                                  if len(id) == 0:
                                       id = re.findall('/\\?uid=(.*?)&', x['href'])
                        except TypeError:
                          id = []
                   if len(id) == 1:
                         use[0][0].append(id[0])
                         use[0][1].append(x.text)
                   if len(use[0][0]) >= count:
                         status = True
                         break

               if status == False:
                    if 'Lihat Teman Lain' in str(parser):
                            next = parser.find('a', string='Lihat Teman Lain')['href']
                    elif 'Lihat selengkapnya' in str(parser):
                            next = parser.find('a', string='Lihat selengkapnya')['href']
                    elif 'Lihat Hasil Selanjutnya' in str(parser):
                            next = parser.find('a', string='Lihat Hasil Selanjutnya')['href']
                    else:
                            break
               else:
                    break

           print(f"\x1b[32;1m(\x1b[36;1m*\x1b[32;1m) \x1b[36;1mTOTAL USER \x1b[30;1m: \x1b[32;1m{len(use[0][1])}\n\x1b[36;1m─────────────────────────\n")
           lib.setting.seting(use, self.cookies)


import numpy as np
import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec
import sys
from scipy import stats
import statistics
import random
import copy
import math
import time

# Structure to store all the data needed per population member
class PopulationMember:
    def __init__(self, airplane):
        self.airplane = airplane
        self.cost = 0
        self.invert_cost = 0
        self.probability = 0

routes = np.array([0, 1, 2, 3, 4])
revenue_lost_per_passenger_turned_away = np.array([13, 20, 7, 7, 15])

passenger_demand_per_route = np.array([800, 900, 700, 650, 380])

number_of_aircraft_available_per_type = [10, 19, 25, 16]

aircraft_capacity_per_spot = np.matrix([[16, 10, 30, 23], [16, 10, 30, 23],
                                        [16, 10, 30, 23], [16, 10, 30, 23],
                                        [16, 10, 30, 23]])

operational_cost_per_spot = np.matrix([[12, 20, 30, 19], [2, 34, 10, 20],
                                       [43, 63, 40, 12], [32, 10, 6, 34],
                                       [20, 30, 10, 87]])

number_of_aircraft_per_spot = np.matrix([[0, 0, 0, 0], [0, 0, 0, 0],
                                         [0, 0, 0, 0], [0, 0, 0, 0],
                                         [0, 0, 0, 0]])

# Function to generate 1 column of a population member
def generate_column(row_nb, aircraft_type_count):
    column = np.zeros((row_nb))
    local_total = 0
    for index in range(0, row_nb):
        # pick a random number between 0 and the aircraft count
        val = random.randint(0, aircraft_type_count - local_total)
        local_total += val
        # if we are at the end but the total is not yet reached
        if aircraft_type_count != local_total and index == row_nb - 1:
            column[index] = aircraft_type_count - local_total
        # else, we add the random value to the airplane matrix
        else:
            column[index] = val
    return column

# Function to generate a population member
def generate_population_member():
    # get the dimension (row*column) of the number_of_aircraft_per_spot matrix)
    row_nb, column_nb = number_of_aircraft_per_spot.shape
    distribution = np.zeros((row_nb, column_nb))
    for index in range(0, column_nb):
        # get the aircraft count per type
        aircraft_type_count = number_of_aircraft_available_per_type[index]
        # generate the corresponding column and assign
        distribution[:, index] = generate_column(row_nb, aircraft_type_count)
    return distribution

# Function to generate the initial population
def generate_first_population(pop_count):
    all_population = []
    for index in range(0, pop_count):
        # generate a random distribution
        distribution = generate_population_member()
        all_population.append(PopulationMember(distribution))
    return all_population

# Function to computes the total revenu lost of one route
def get_revenu_lost_per_route(airplanes):
    revenu_lost = 0
    for index in routes:
        revenu_lost += max(((passenger_demand_per_route[index] - np.multiply(
            airplanes[index], aircraft_capacity_per_spot[index]).sum()) *
                            revenue_lost_per_passenger_turned_away[index]), 0)
    return revenu_lost

# The cost function
def cost_function(member):
    total_operating_cost = np.multiply(operational_cost_per_spot,
                                       member.airplane).sum()
    total_revenue_lost = get_revenu_lost_per_route(member.airplane)
    total_lost = total_operating_cost + total_revenue_lost
    return total_lost

# Function to compute the cost of all the population
def compute_costs(population):
    for index in range(0, len(population)):
        cost = cost_function(population[index])
        population[index].cost = cost
        population[index].invert_cost = 1 / cost
    return population

# Function to compute the picking probability according to the cost
def compute_probabilities(all_population):
    sum_cost = sum(p.invert_cost for p in all_population)
    for index in range(0, len(all_population)):
        probability = math.exp(
            math.exp((1 / all_population[index].cost) / sum_cost))
        all_population[index].probability = probability
    return all_population

# Function to pick the two best individual among the population
def pick_parents_elitist(all_population):
    parents = sorted(all_population, key=lambda k: k.cost)
    return [parents[0], parents[1]]

# Function to pick the two best individual among the population
# according to their probability
def pick_parents_proba(all_population):
    all_proba = list(map(lambda x: x.probability, all_population))
    return random.choices(all_population, weights=all_proba, k=2)

# Cross function
def cross(parent1, parent2, cross_threshold):
    # Copy both parents
    children = [
        PopulationMember(copy.deepcopy(parent1.airplane)),
        PopulationMember(copy.deepcopy(parent2.airplane))
    ]
    row_nb, column_nb = number_of_aircraft_per_spot.shape
    for index in range(0, column_nb):
        np.random.seed(10)
        rand = random.uniform(0, 1)
        # If the random is lower than the threshold, the two
        # column are switched
        if rand <= cross_threshold:
            children[0].airplane[:, index] = copy.deepcopy(
                parent2.airplane[:, index])
            children[1].airplane[:, index] = copy.deepcopy(
                parent1.airplane[:, index])
    return children[0], children[1]

# Mutation function by recreating the whole column
def mutation_generate_column(child, mutation_threshold):
    row_nb, column_nb = number_of_aircraft_per_spot.shape
    for index in range(0, column_nb):
        np.random.seed(10)
        rand = random.uniform(0, 1)
        if rand <= mutation_threshold:
            aircraft_type_count = number_of_aircraft_available_per_type[index]
            child.airplane[:, index] = generate_column(row_nb,
                                                       aircraft_type_count)
    return child

# Mutation function by switching two values of the same column
def mutation_switch_one_element(child, mutation_threshold):
    row_nb, column_nb = number_of_aircraft_per_spot.shape
    child_copy = copy.deepcopy(child)
    for index in range(0, column_nb):
        np.random.seed(10)
        rand = random.uniform(0, 1)
        if rand <= mutation_threshold:
            index1 = random.randint(0, row_nb - 1)
            index2 = random.randint(0, row_nb - 1)
            while index1 == index2:
                index2 = random.randint(0, row_nb - 1)
            tmp = child_copy.airplane[index1, index]
            child_copy.airplane[index1, index] = child_copy.airplane[index2,
                                                                     index]
            child_copy.airplane[index2, index] = tmp
    return child_copy

# Function to regenerate a new population
def generate_new_population(all_population, cross_threshold,
                            mutation_threshold):
    new_pop = []
    for index in range(0, len(all_population)):
        parents = pick_parents_elitist(all_population)
        parent1 = copy.deepcopy(parents[0])
        parent2 = copy.deepcopy(parents[1])
        child1, child2 = cross(parent1, parent2, cross_threshold)
        np.random.seed(10)
        rand_cross = random.uniform(0, 1)
        rand_mutation = random.uniform(0, 1)
        if rand_cross >= 0.5:
            picked_child = child1
        else:
            picked_child = child2
        if rand_mutation >= 0.5:
            picked_child = mutation_switch_one_element(picked_child,
                                                       mutation_threshold)
        new_pop.append(copy.deepcopy(picked_child))
    return new_pop

def generate_plot(generation, all_mins, initial_population_count,
                  cross_threshold, mutation_threshold):
    results = list(map(lambda x: x.cost, all_mins))
    print(all_mins[generation - 1].airplane)
    print(all_mins[generation - 1].cost)
    plt.plot(results)
    title = 'Total costs : ' + str(
        all_mins[generation - 1].cost) + '$' + ' | Generation : ' + str(
            generation) + ', Initial population : ' + str(
                initial_population_count)
    plt.xlabel(title)
    plt.savefig('../outputs/airplanes_g' + str(generation) + '_i' +
                str(initial_population_count) + '_cT' + str(cross_threshold) +
                '_mT' + str(mutation_threshold) + '.png')

def draw_table(mean_of_mins, std_of_mins, times_of_mins, row_labels,
               col_labels, title):

    fig = plt.figure(figsize=(12, 4))
    gs = GridSpec(nrows=2, ncols=2)

    ax1 = fig.add_subplot(gs[:, 1])
    ax2 = fig.add_subplot(gs[1, 0])
    ax3 = fig.add_subplot(gs[0, 0])

    plt.set_cmap('Spectral')
    im = ax1.imshow(mean_of_mins, aspect='auto')
    im2 = ax2.imshow(mean_of_mins, aspect='auto')
    im3 = ax3.imshow(mean_of_mins, aspect='auto')

    ax1.set_title("Mean of cost function call to reach 19200")
    ax2.set_title("Standard deviation")
    ax3.set_title("Time (sec.)")

    # We want to show all ticks...
    ax1.set_xticks(np.arange(len(col_labels)))
    ax1.set_yticks(np.arange(len(row_labels)))
    # We want to show all ticks...
    ax2.set_xticks(np.arange(len(col_labels)))
    ax2.set_yticks(np.arange(len(row_labels)))
    # We want to show all ticks...
    ax3.set_yticks(np.arange(len(row_labels)))
    # ... and label them with the respective list entries
    ax1.set_xticklabels(list(map(lambda x: "p=" + str(x), col_labels)))
    ax1.set_yticklabels(list(map(lambda x: "g=" + str(x), row_labels)))
    # ... and label them with the respective list entries
    ax2.set_xticklabels(list(map(lambda x: "p=" + str(x), col_labels)))
    ax2.set_yticklabels(list(map(lambda x: "g=" + str(x), row_labels)))
    # ... and label them with the respective list entries
    ax3.set_yticklabels(list(map(lambda x: "g=" + str(x), row_labels)))

    # Rotate the tick labels and set their alignment.
    plt.setp(ax1.get_xticklabels(),
             rotation=45,
             ha="right",
             rotation_mode="anchor")
    # Rotate the tick labels and set their alignment.
    plt.setp(ax2.get_xticklabels(),
             rotation=45,
             ha="right",
             rotation_mode="anchor")
    # Rotate the tick labels and set their alignment.
    plt.setp(ax3.get_xticklabels(), visible=False)

    # Loop over data dimensions and create text annotations.
    for i in range(len(row_labels)):
        for j in range(len(col_labels)):
            text = ax1.text(j,
                            i,
                            mean_of_mins[i][j],
                            ha="center",
                            va="center",
                            color="black")
            text2 = ax2.text(j,
                             i,
                             std_of_mins[i][j],
                             ha="center",
                             va="center",
                             color="black")
            text3 = ax3.text(j,
                             i,
                             times_of_mins[i][j],
                             ha="center",
                             va="center",
                             color="black")
    plt.suptitle(title)
    print('matplotlib-table_' + str(title))
    plt.savefig('../outputs/matplotlib-table_' + str(title) + '.png',
                bbox_inches='tight',
                pad_inches=0.05,
                dpi=400)

def main():
    mins_of_mins = []
    std_of_mins = []
    times_of_mins = []
    calls_of_mins = []
    calls_of_mins_std = []
    generations = [160]
    init_pops = [80]
    reach = 19200
    cross_threshold = float(sys.argv[1])
    mutation_threshold = float(sys.argv[2])
    runs = 2
    # For each generations
    for gen in range(0, len(generations)):
        generation = generations[gen]
        all_mins_mean = []
        all_mins_std = []
        all_mins_times = []
        all_calls_of_pop = []
        all_calls_of_pop_std = []
        # For each population
        for init_p in range(0, len(init_pops)):
            initial_population_count = init_pops[init_p]
            start = time.time()
            mins_of_pop = []
            calls_of_pop = []
            # Run n times
            for n in range(0, runs):
                all_mins_of_the_current_gen = []
                # initial population
                all_population_of_the_current_gen = generate_first_population(
                    initial_population_count)
                compute_cost_call = 0
                print('Time :', n, 'and generation :', generation, 'for pop :',
                      initial_population_count)
                # From 0 to the size of the generation
                for gen_2 in range(0, generation):
                    # Compute the cost of all the population
                    all_population_of_the_current_gen = compute_costs(
                        all_population_of_the_current_gen)
                    # Get the minimum cost and add it into the list
                    min_cost = min(all_population_of_the_current_gen,
                                   key=lambda x: x.cost)
                    all_mins_of_the_current_gen.append(min_cost)
                    if (min_cost.cost >= reach):
                        compute_cost_call += 1
                    # Compute probabilities
                    all_population_of_the_current_gen = compute_probabilities(
                        all_population_of_the_current_gen)
                    # Pick parents
                    all_population_of_the_current_gen = generate_new_population(
                        all_population_of_the_current_gen, cross_threshold,
                        mutation_threshold)
                # Pick the last min (the best we get) and put it
                # into the min of mins
                mins_of_pop.append(all_mins_of_the_current_gen[generation -
                                                               1].cost)
                # Pick the number of time the cost function have
                # been called to reach the REACH value
                calls_of_pop.append(compute_cost_call)
            # After 10 tried, only keep the mean of all the mins
            print("###############")
            all_mins_mean.append(round(statistics.mean(mins_of_pop)))
            all_mins_std.append(round(statistics.stdev(mins_of_pop), 2))
            all_calls_of_pop.append(round(statistics.mean(calls_of_pop), 2))
            all_calls_of_pop_std.append(
                round(statistics.stdev(calls_of_pop), 2))
            all_mins_times.append(round((time.time() - start) / runs, 2))
        # Put the array of mins into the array containing all array of mins
        mins_of_mins.append(all_mins_mean)
        std_of_mins.append(all_mins_std)
        times_of_mins.append(all_mins_times)
        calls_of_mins.append(all_calls_of_pop)
        calls_of_mins_std.append(all_calls_of_pop_std)
        #generate_plot(generation, mins_of_pop, initial_population_count,
        #              cross_threshold, mutation_threshold)
    draw_table(mins_of_mins,
               std_of_mins,
               times_of_mins,
               row_labels=generations,
               col_labels=init_pops,
               title='cT=' + str(cross_threshold) + ' & mT=' +
               str(mutation_threshold) + " (" + str(runs) + " runs)")

main()


#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.

import unittest
import os
import json
from subprocess import call
import cairis.core.BorgFactory
from cairis.core.Borg import Borg
from cairis.core.EnvironmentParameters import EnvironmentParameters
from cairis.core.AssetParameters import AssetParameters
from cairis.core.AssetEnvironmentProperties import AssetEnvironmentProperties
from cairis.core.GoalParameters import GoalParameters
from cairis.core.GoalEnvironmentProperties import GoalEnvironmentProperties
from cairis.core.ARM import DatabaseProxyException

class GoalTest(unittest.TestCase):

  def setUp(self):
    call([os.environ['CAIRIS_SRC'] + "/test/initdb.sh"])
    cairis.core.BorgFactory.initialise()
    f = open(os.environ['CAIRIS_SRC'] + '/test/goals.json')
    d = json.load(f)
    f.close()
    self.ienvs = d['environments']
    self.iep1 = EnvironmentParameters(self.ienvs[0]["theName"],self.ienvs[0]["theShortCode"],self.ienvs[0]["theDescription"])
    b = Borg()
    b.dbProxy.addEnvironment(self.iep1)
    self.oenvs = b.dbProxy.getEnvironments()

    self.iassets = d['assets']
    self.iaeps1 = [AssetEnvironmentProperties(self.iassets[0]["theEnvironmentProperties"][0][0],self.iassets[0]["theEnvironmentProperties"][0][1],self.iassets[0]["theEnvironmentProperties"][0][2])]
    self.iap1 = AssetParameters(self.iassets[0]["theName"],self.iassets[0]["theShortCode"],self.iassets[0]["theDescription"],self.iassets[0]["theSignificance"],self.iassets[0]["theType"],"0","N/A",[],[],self.iaeps1)
    b.dbProxy.addAsset(self.iap1)
    self.oap = b.dbProxy.getAssets()

    self.iGoals = d['goals']

  def testGoal(self):
    b = Borg()
    igep1 = GoalEnvironmentProperties(self.iGoals[0]["theEnvironmentProperties"][0],self.iGoals[0]["theEnvironmentProperties"][1],self.iGoals[0]["theEnvironmentProperties"][2],self.iGoals[0]["theEnvironmentProperties"][3],self.iGoals[0]["theEnvironmentProperties"][4],self.iGoals[0]["theEnvironmentProperties"][5],self.iGoals[0]["theEnvironmentProperties"][6],[],[],[self.iGoals[0]["theEnvironmentProperties"][7]],[])
    igp1 = GoalParameters(self.iGoals[0]["theName"],self.iGoals[0]["theOriginator"],[],[igep1])
    b.dbProxy.addGoal(igp1)
    oGoals = b.dbProxy.getGoals()
    og = oGoals[self.iGoals[0]["theName"]]
    self.assertEqual(igp1.name(), og.name())
    self.assertEqual(igp1.originator(), og.originator())

    b.dbProxy.deleteGoal(og.id())

  def tearDown(self):
    b = Borg()

    b.dbProxy.deleteAsset(self.oap[self.iap1.name()].id())
    b.dbProxy.deleteEnvironment(self.oenvs[self.iep1.name()].id())
    b.dbProxy.close()
    call([os.environ['CAIRIS_SRC'] + "/test/dropdb.sh"])

if __name__ == '__main__':
  unittest.main()


import mido
from mido import Message
from mido import MidiFile
from pprint import pprint
import mingus.core.chords as chords
import random
from os import system
import time

outport = mido.open_output()

input1 = mido.get_input_names()[0]
for inp in mido.get_input_names():
    if inp.find('Keystation')>-1:
        input1 = inp

"""
Application:

1. random generate chord and type(ex C, dominant)
2. translate with mingus
3. show on screen
4. wait until correctly pressed(or just one chance and show succes/failure)

New features:
ok- say chord name and correct/failed

ok- consider the chord once n notes are playe(n is the number of notes of the current chord), then say right or wrong

ok- wait until no more notes pressed

- read midi and play it

ok- read midi withotu playing it but stoping until piano plays the current note

- deal with chords

- deal with chord names annotated on midi files (only do this)

"""

degrees = ['C','D','E','F','G','A','B']

root_notes = [
    'C','B#',
    'C#','Db',
    'D',
    'D#','Eb',
    'E','Fb',
    'F','E#',
    'F#','Gb',
    'G',
    'G#','Ab',
    'A','Bbb',
    'A#','Bb',
    'B','Cb']

note = [
    ('C','B#','Dbb'),
    ('C#','B##','Db'),
    ('D','C##','Ebb'),
    ('D#','Eb'),
    ('E','D##','Fb'),
    ('F','E#','Gbb'),
    ('F#','E##','Gb'),
    ('G','F##','Abb'),
    ('G#','Ab'),
    ('A','G##','Bbb'),
    ('A#','Bb'),
    ('B','A##','Cb')]

notes = { i:note[i%12]  for i in range(128) }

chord_types = {
'major': [4,3],
'minor': [3,4],
'': [4,3,4],
'7':[4,3,3],
'm7':[3,4,3],
'7b5':[3,3,4],
'dim7':[3,3,3]
}

def find_real_note(note_name):

    if isinstance(note_name,str):
        for n in note:
            for synonime in n:
                if synonime == note_name:
                    return n
    elif isinstance(note_name,tuple):
        """
            Stupid assumption, if it is a tuple, it'sa real note already
        """
        return note_name

    return None

def find_note_position(real_note):

    if isinstance(real_note,str):
        real_note = find_real_note(real_note)

    return note.index(real_note)

def choose_random_chord():

    root = root_notes[random.randint(0,len(root_notes)-1)]

    chord_types_keys = list(chord_types.keys())
    chord_type = chord_types[chord_types_keys[random.randint(0,len(chord_types_keys)-1)]]

    final_chord = [root,]

    # find position of root
    current_note = root
    #final_chord.append(find_real_note(current_note))
    posi = find_note_position(current_note)

    for interval in chord_type:
        # add semitones to find next note
        posi = (posi + interval) % len(note)

        # update curernt note
        current_note = note[posi]
        final_chord.append(current_note)

    return final_chord

def parse_chord(a_chord):
    """
    PENDING:
    - compute semitones between notes of the chord
    - according to the semitones between the previous note
    and the previous note first name(ex C,D#)
    choose the current note (ex if C -> E, if B -> D#

    establish some priority:
    1. first I,III,V,VII
    2. then II,IV,VI
    """

    #print(a_chord)

    root = a_chord[0][0]

    foundi = degrees.index(root[0])
    ld = len(degrees)
    the_degrees = [
                    degrees[foundi],
                    degrees[(foundi+2)%ld],
                    degrees[(foundi+4)%ld],
                    degrees[(foundi+6)%ld],
                    ]
    #print(the_degrees)

    parsed_chord = [a_chord[0]]

    for i in range(1,len(a_chord)):
        #print("match",the_degrees[i]," in",a_chord[i])
        for synonym in a_chord[i]:
            if synonym.lower()[0] == the_degrees[i].lower():
                parsed_chord.append(synonym)
                break

    return chords.determine(parsed_chord), parsed_chord

def match_chord(pressed_notes, thechord):

    # convert to real chord
    thechord2 = [ thechord[i] if i>0 else find_real_note(thechord[i]) for i in range(len(thechord)) ]

    #print("real chord", thechord2)

    # sort the pressed notes by note value ascending
    pressed_notes.sort(key=lambda x: x[0])
    pressed_notes2 = [t[1] for t in pressed_notes]
    #print("presset notes", pressed_notes2)

    match = True
    for i in range(len(thechord2)):

        #print(i, len(pressed_notes2), i+1, pressed_notes2[i], thechord2[i])
        if len(pressed_notes2)<i+1:
            match = False
            break
        elif pressed_notes2[i] != thechord2[i]:
            match = False
            break

    return match

def say_chord(chordname, stop=True, restart=True):
    print(chordname)
    if isinstance(chordname, str):
        my_say(chordname.replace('b',' flat ').replace('#',' sharp '), stop=stop, restart=restart)
    elif isinstance(chordname[0],str):
        my_say(chordname[0].replace('b',' flat ').replace('#',' sharp '), stop=stop, restart=restart)
    else:
        my_say(chordname, stop=stop, restart=restart)

def my_say(text, stop=True, restart=True):
    if stop:
        system('./config.sh stop')

    system('say '+text)
    waiting = (len(text) / 7 )
    time.sleep(1+waiting)
    if restart:
        system('./config.sh start')

# my_say(" First test")
# my_say(" This is my very first test. You should see config.sh appear later than before")
# my_say(" And now it should appear much faster")

def wait_for_chord(inport, thechord, chord_name, parsed_chord):
    #print(parsed_chord)
    print(chord_name)
    if len(chord_name)>0: say_chord(chord_name[0])

    pressed_notes = []
    times_up = False
    correct_answer = False
    for msg in inport:
        if 'note' not in dir(msg):
            continue

        if msg.is_meta:
            continue

        current_note = notes[msg.note]
        if msg.velocity > 0 :
            pressed_notes.append((msg.note,current_note))
        else:
            foundi = pressed_notes.index((msg.note,current_note))
            if foundi > -1:
                pressed_notes.pop(foundi)

        if len(pressed_notes) == len(thechord):
            #time.sleep(2)
            times_up = True

            print(pressed_notes, thechord)
            if match_chord(pressed_notes, thechord):
                correct_answer = True

        if times_up:
            if len(pressed_notes)==0:
                if correct_answer:
                    print("Correct!", parsed_chord )
                    my_say('Correct', restart=False)
                else:
                    print("Failed", parsed_chord )
                    my_say('Failed', restart=False)

                say_chord(' '.join(parsed_chord), stop=False)
                break

def chord_training():

    with mido.open_input(input1) as inport:
        while(True):
            thechord = choose_random_chord()
            chord_name, parsed_chord = parse_chord(thechord)
            wait_for_chord(inport,thechord,chord_name, parsed_chord)

def wait_until_correct_input(msg):
    with mido.open_input(input1) as inport:
        thechord = [(notes[msg.note])]
        print(thechord)
        chord_name, parsed_chord = notes[msg.note][0],[notes[msg.note][0]]
        wait_for_chord(inport, thechord,chord_name, parsed_chord)

#chord_training()

print("output:")
outputs = mido.get_output_names()
selected_output = [ out for out in outputs if out.find('Midi Through')>-1 ]
print(selected_output[0])
port = mido.open_output(selected_output[0])

mid = MidiFile('../muss_2.mid')
for msg in mid.play():
    #print(msg.time)
    #time.sleep(msg.time)
    if not msg.is_meta and 'note' in dir(msg):
        #print(dir(msg))
        print(msg)
        #port.send(msg)
        wait_until_correct_input(msg)


import json
import uuid
from unittest import TestCase

from mock import patch, create_autospec

from walkoff.case.database import CaseDatabase
from walkoff.case.logger import CaseLogger
from walkoff.case.subscription import SubscriptionCache, Subscription
from walkoff.events import WalkoffEvent, EventType

class TestCaseLogger(TestCase):

    def setUp(self):
        self.logger = self.get_basic_case_logger()

    @staticmethod
    def get_case_logger(subscriptions):
        repo = create_autospec(CaseDatabase)
        return CaseLogger(repo, subscriptions=subscriptions)

    @staticmethod
    def get_basic_case_logger():
        subscription_cache = SubscriptionCache()
        return TestCaseLogger.get_case_logger(subscription_cache)

    @staticmethod
    def assert_mock_called_once_with(mock, *args):
        mock.assert_called_once()
        mock.assert_called_with(*args)

    @patch.object(SubscriptionCache, 'add_subscriptions')
    def test_add_subscriptions(self, mock_add_subs):
        subs = [Subscription('id', ['e1'])]
        self.logger.add_subscriptions('case1', subs)
        self.assert_mock_called_once_with(mock_add_subs, 'case1', subs)

    @patch.object(SubscriptionCache, 'update_subscriptions')
    def test_update_subscriptions(self, mock_update_subs):
        subs = [Subscription('id', ['e1'])]
        self.logger.update_subscriptions('case1', subs)
        self.assert_mock_called_once_with(mock_update_subs, 'case1', subs)

    @patch.object(SubscriptionCache, 'delete_case')
    def test_delete_case(self, mock_delete_case):
        self.logger.delete_case('case1')
        self.assert_mock_called_once_with(mock_delete_case, 'case1')

    @patch.object(SubscriptionCache, 'clear')
    def test_clear_subscriptions(self, mock_clear):
        self.logger.clear_subscriptions()
        self.assert_mock_called_once_with(mock_clear)

    def test_format_data_with_none(self):
        self.assertEqual(CaseLogger._format_data(None), '')

    def test_format_data_with_string(self):
        string = 'something'
        self.assertEqual(CaseLogger._format_data(string), string)

    def test_format_data_with_jsonable_dict(self):
        data = {'a': 'something', 'b': 5}
        expected = json.dumps(data)
        self.assertEqual(CaseLogger._format_data(data), expected)

    def test_format_data_with_unjsonable_dict(self):
        class A: pass

        data = {'a': 'something', 'b': A()}
        expected = str(data)
        self.assertEqual(CaseLogger._format_data(data), expected)

    def test_create_event_entry(self):
        event = WalkoffEvent.WorkflowExecutionStart
        uid = uuid.uuid4()
        data = {'a': 'something', 'b': 5}
        expected_data = json.dumps(data)
        event_entry = CaseLogger._create_event_entry(event, uid, data)
        self.assertEqual(event_entry.type, EventType.workflow.name)
        self.assertEqual(event_entry.originator, uid)
        self.assertEqual(event_entry.message, event.value.message)
        self.assertEqual(event_entry.data, expected_data)

    @patch.object(SubscriptionCache, 'get_cases_subscribed')
    @patch.object(CaseDatabase, 'add_event')
    def test_log_unloggable_event(self, mock_repo_add_event, mock_get_cases_subscribed):
        event = WalkoffEvent.SendMessage
        self.logger.log(event, uuid.uuid4())
        mock_repo_add_event.assert_not_called()
        mock_get_cases_subscribed.assert_not_called()

    @patch.object(SubscriptionCache, 'get_cases_subscribed', return_value=set())
    @patch.object(CaseDatabase, 'add_event')
    def test_log_no_cases(self, mock_repo_add_event, mock_get_cases_subscribed):
        event = WalkoffEvent.WorkflowExecutionStart
        uid = uuid.uuid4()
        self.logger.log(event, uid)
        mock_repo_add_event.assert_not_called()
        self.assert_mock_called_once_with(mock_get_cases_subscribed, str(uid), event.signal_name)

    @patch.object(SubscriptionCache, 'get_cases_subscribed', return_value={1, 2})
    @patch.object(CaseDatabase, 'add_event')
    def test_log(self, mock_repo_add_event, mock_get_cases_subscribed):
        event = WalkoffEvent.WorkflowExecutionStart
        uid = uuid.uuid4()
        self.logger.log(event, uid)
        self.assert_mock_called_once_with(mock_get_cases_subscribed, str(uid), event.signal_name)


from core.utils import parse_args, setup_determinism
from core.utils import build_loss_func, build_optim
from core.utils import build_scheduler, load_checkpoint
from core.config import get_cfg_defaults
from core.dataset import build_dataloader
from core.model import build_model, train_loop, valid_model, test_model
from core.model import valid_model_macro

from torch.utils.tensorboard import SummaryWriter
from torch.cuda.amp import GradScaler
import torch.nn as nn

import os

import torch.multiprocessing

torch.multiprocessing.set_sharing_strategy("file_system")

# SET UP GLOBAL VARIABLE
scaler = GradScaler()

def main(cfg, args):
    # Setup logger
    sum_writer = SummaryWriter(f"test2")

    # Declare variables
    best_metric = 0
    start_epoch = 0
    mode = args.mode

    # Setup folder
    if not os.path.isdir(cfg.DIRS.WEIGHTS):
        os.mkdir(cfg.DIRS.WEIGHTS)

    # Load Data
    trainloader = build_dataloader(cfg, mode="train")
    validloader = build_dataloader(cfg, mode="valid")
    testloader = build_dataloader(cfg, mode="test")

    # Define model/loss/optimizer/Scheduler
    model = build_model(cfg)
    # model = nn.DataParallel(model)
    loss = build_loss_func(cfg)
    optimizer = build_optim(cfg, model)
    scheduler = build_scheduler(args, len(trainloader), cfg)
    # Load model checkpoint
    model, start_epoch, best_metric = load_checkpoint(args, model)

    if cfg.SYSTEM.GPU:
        model = model.cuda()

    # Run Script
    if mode == "train":
        for epoch in range(start_epoch, cfg.TRAIN.EPOCHES):
            train_loss = train_loop(
                cfg,
                epoch,
                model,
                trainloader,
                loss,
                scheduler,
                optimizer,
                scaler,
                sum_writer,
            )
            best_metric = valid_model(
                cfg,
                mode,
                epoch,
                model,
                validloader,
                loss,
                sum_writer,
                best_metric=best_metric,
            )

    elif mode == "valid":
        valid_model(
            cfg, mode, 0, model, validloader, loss, sum_writer, best_metric=best_metric
        )
    elif mode == "test":
        test_model(cfg, mode, model, testloader, loss)

if __name__ == "__main__":
    # Set up Variable
    seed = 10

    args = parse_args()
    cfg = get_cfg_defaults()

    if args.config != "":
        cfg.merge_from_file(args.config)

    # Set seed for reproducible result
    setup_determinism(seed)

    main(cfg, args)


import numpy as np
from util import shape_assert

from filterpy.kalman import KalmanFilter

class LowPassIIRBase(object):
    """
    represents a generic 1-corner butterworth filter. use the great website
    below to generate constants for subclasses:

    https://www-users.cs.york.ac.uk/~fisher/mkfilter/trad.html

    in practice a low pass filter does not cut it for smoothing out our skeletal
    joint positions, it just spreads out the noise over a larger spectrum. see
    kalman filter code for next attempt.
    """
    NZEROS=2
    NPOLES=2
    def __init__(self,shape):
        self.shape = shape
        self.xv = np.zeros((self.NZEROS+1,)+tuple(shape),np.float32)
        self.yv = np.zeros((self.NPOLES+1,)+tuple(shape),np.float32)

    def filter(self,sample):
        shape_assert(sample,self.shape)
        self.xv[0:2,...] = self.xv[1:3,...]
        self.xv[2,...] = sample / self.GAIN
        self.yv[0:2,...] = self.yv[1:3,...]
        self.yv[2,...] = (self.xv[0,...] + self.xv[2,...]) + 2 * self.xv[1,...] \
                         + (self.MAGIC_1 * self.yv[0,...]) + (self.MAGIC_2 *
                                                              self.yv[1,...])
        return self.yv[2,...].copy()

    def __call__(self,sample):
        return self.filter(sample)

class LowPass8Hz(LowPassIIRBase):
    """
    30 hz sample frequency, 8 Hz corner frequency lowpass
    """
    GAIN=3.084091054e+00
    MAGIC_1 = -0.1742373434
    MAGIC_2 = -0.1227412250

class LowPass5Hz(LowPassIIRBase):
    GAIN=6.449489743e+00
    MAGIC_1= -0.2404082058
    MAGIC_2= 0.6202041029

#########################################################################
# wonderful: https://www-users.cs.york.ac.uk/~fisher/mkfilter/trad.html #
#                                                                       #
# static float xv[NZEROS+1], yv[NPOLES+1];                              #
#                                                                       #
# static void filterloop()                                              #
#   { for (;;)                                                          #
#       { xv[0] = xv[1]; xv[1] = xv[2];                                 #
#         xv[2] = next_input_value / GAIN;                              #
#         yv[0] = yv[1]; yv[1] = yv[2];                                 #
#         yv[2] =   (xv[0] + xv[2]) + 2 * xv[1]                         #
#                      + ( MAGIC_1 * yv[0]) + ( MAGIC_2 * yv[1]);       #
#         next_output_value = yv[2];                                    #
#       }                                                               #
#   }                                                                   #
#########################################################################

class Kalman2DPoint(object):
    """
    simplest type of Kalman filtering: treat each skeleton joint as an independent
    point, and use constant-acceleration kalman filtering on it.

    use a heuristic method for converting neural-net joint position confidence to
    position variance.

    solve for estimated 2D joint location based on N-way replicated Kalman filter.
    """

    def __init__(self,process_variance=0.001):
        self.kf = KalmanFilter(dim_x = 6, #x,y,x_dot,y_dot,x_accel,y_accel
                               dim_z = 2) #x,y
        #engineering: define dt to be 1 frame, simplifying all coefficients
        A = 0.5 #coeff to get position update from const acceleration
        self.kf.F = np.float32([
            [1,0,1,0,A,0], #y is updated by prev y + yv * dt + 0.5 ya * dt^2
            [0,1,0,1,0,A], #x is updated by prev x +   xv * dt + 0.5 xa dt^2
            [0,0,1,0,1,0], #yv is prev yv + ya * dt
            [0,0,0,1,0,1], #xv is prev xv + xa * dt
            [0,0,0,0,1,0], #ya is prev ya
            [0,0,0,0,0,1]]) #xa is prev xa

        self.kf.H = np.float32([
            [1,0,0,0,0,0],
            [0,1,0,0,0,0]]) #only x and y are observable

        #initialize x to nonsense value:
        self.kf.x = np.zeros((6,),dtype=np.float32)

        #initialize P to very high uncertainty (variance)
        self.kf.P *=10000

        #set the process noise variance. this is the noise introduced
        #by flaws in our model of constant-accel point motion, not by
        #flaws in measurement. The measurement noise is characterized
        #by R, later on.

        Q_a = np.float32([
            [0,0,0,0,0,0],
            [0,0,0,0,0,0],
            [0,0,0,0,0,0],
            [0,0,0,0,0,0],
            [0,0,0,0,1,0],
            [0,0,0,0,0,1]]) * process_variance
        #the process variance is modeled as a random acceleration applied
        #to points at each time step.

        self.kf.Q = np.matmul(np.matmul(self.kf.F,Q_a),self.kf.F.T)

    def apply_filter(self,point,confidence):
        self.kf.predict()
        variance = self.var_from_conf(confidence)
        self.kf.R = np.float32([[variance,0],
                                [0,variance]])
        self.kf.update(point)
        out_point = self.kf.x[:2].copy()
        #clip
        out_point[out_point>1]=1
        out_point[out_point<0]=0
        return out_point

    def var_from_conf(self,conf):
        """
        hueristic for getting measurement variance from neural-net confidence.
        the confidence is a number between 0 and 1. The x and y values are also
        between 0 and 1, because they're normalized to the image dimensions.

        intuitively:

        confidence of zero is highest possible variance: the point can be anywhere.
        set variance to 1.

        confidence of 1 is the lowest possible variance. but we should not trust the
        neural net to be 100% right when it's 100% confident. so discount by factor

        """
        conf = conf * 0.99 #discount

        #var =  (1 - conf)**2 #if we interpret confidence as a normalized distance...
        #var = var**2
        epsilon = 1e-6
        var = 1/(conf+epsilon) - 1.0
        return var

    def __call__(self,point,confidence):
        return self.apply_filter(point,confidence)

class Pointwise2DSkelKF(object):
    """
    use the 2D kalman filter on a whole skeleton
    """
    CONF_MEMORY=10
    def __init__(self):
        self.ptf = [Kalman2DPoint(process_variance = self.get_pv(i))
                    for i in range(17)]#todo: smarter process var
        self.conf_history = np.zeros((17,self.CONF_MEMORY),dtype=np.float32)
        #based on skeletal joint index

    def apply_filter(self,skeleton,confidence):
        shape_assert(skeleton,[17,2])
        shape_assert(confidence,[17])
        self.conf_history[:,:self.CONF_MEMORY-1] = \
                self.conf_history[:,1:]
        self.conf_history[:,-1] = confidence
        avg_conf = np.mean(self.conf_history,axis=1)
        return np.float32([
            k(pt,c) for (k,pt,c) in zip(self.ptf,
                                        skeleton,
                                        confidence)
            ]),avg_conf
    def __call__(self,skeleton,confidence):
        return self.apply_filter(skeleton,confidence)

    def get_pv(self,joint_idx):
        import human_foo
        h = human_foo.Part
        if joint_idx in (h.LShoulder,h.RShoulder,h.LHip,h.RHip):
            #parts of body that tend to change direction less
            return 0.0001
        else:
            return 0.001


"""REST_API_Project URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.urls import include, path
from rest_framework import routers
from REST_API_App import views

router = routers.DefaultRouter()
router.register(r'shops', views.ShopsViewSet)
router.register(r'category', views.CategoryViewSet)
router.register(r'product', views.ProductViewSet)
router.register(r'media', views.MediaViewSet)

urlpatterns = [
    path('', include(router.urls)),
    # path(r'^api/(?P<id>\w{0,50})/$', views.FetchAPIData.as_view(), name='FetchAPIData'),
    path('api/product', views.FetchAPIData.as_view(), name='FetchAPIData'),
    path('api-auth/', include('rest_framework.urls', namespace='rest_framework'))

]