import time
import datetime
import pytz
import numpy as np
import pandas as pd
import zmq
import threading
import json
# IMPORTING MY MADE UP FUNCTIONS"
from functions import stimtypes, get_psd, get_erd, get_binary_decision, client_save_data, \
    stimulation_decision, compute_ccorr, update_Wmatrix, do_Wmatrix_operations, set_timers, keep_current_stim_data, \
    update_timers, update_cooldowns, server_save_data, server_save_report, reset_vars  # THESE ARE MY MADE UP FUNCTIONS
###########################################################################
#for alg

cooldown = [1*60,1*60,1*60] #no stims for first minute of use
cooldown_time = 15*60  ###########!!!!!!!!!!!!!!!!!!!!!!!!!!!pls change this to 30 * 60 if not testing
saving = True
stim_threshold = 0.51
reward_value = 0.05

tbuffer = 20 #longer buffer of tbuffer for tDCS stims (see phosphene comment)
MOTORt = 2*60 + tbuffer
COGNITIVEt = 5*60 + tbuffer
RELAXt = 5*60 + tbuffer
PHOSPHENEt = (10 + 10) #it takes 5 seconds, there's a 10 sec buffer to let the person react. is tACS so gets fast reaction

timers_set, timer1, timer2, timer3, timer4, phos_row, phos_sender, motor_row, motor_sender, cog_row, cog_sender, relax_row, relax_sender, i, check1, check2, check3, check4, baselinedone, loop, boop, reward = reset_vars()
ccorr_at_laststimtime = 0
current_stim_time = 0
binary_matrix_stim = []
users = ['user1', 'user2', 'user3']
stims = ['motor', 'cognitive', 'relax', 'phosphene']
reportcols = ['stimulation time and date', 'Wmatrix', 'reward', 'sender', 'final binary stim matrix',
              'ccorr at stimulation time', 'ccorr after stim period']
report = pd.DataFrame(columns=reportcols)
avgccorraggregated = pd.DataFrame(columns=['avgccorr'])
finalstimvalue = []
counter = 0

# initialise/load W matrix
# there should be n rows for n inputs, m columns for m stims
#this is what the initial Wmatrix is. make sure the file loaded is this matrix if first day
#to create a new save:
# Wmatrix = np.array([
#     [0.5, 0.61, 0.5, 0.5],
#     [0.5, 0.6, 0.5, 0.5],
#     [0.62, 0.5, 0.5, 0.5],
#     [0.5, 0.5, 0.5, 0.59],
#     [0.5, 0.5, 0.5, 0.5],
#     [0.5, 0.5, 0.5, 0.5],
#     [0.5, 0.5, 0.5, 0.5]
# ])
#np.save('FILENAME.npy', Wmatrix)

Wmatrix = np.load('Wmatrix.npy')
#################################################################################################################

context = zmq.Context()
socket = context.socket(zmq.REP)
socket.bind("tcp://*:8080")
FORMAT = 'utf-8'

oneuser = False
usesynthetic = False
msg = []
user_inputs = ['undefined','undefined','undefined']
candidates = ['none','none','none']
connected = False

def kaylas_angry():
    global init
    global user_inputs
    global usesynthetic
    global candidates
    global connected

    global cooldown
    global cooldown_time
    global saving
    global stim_threshold
    global reward_value
    global tbuffer
    global MOTORt
    global COGNITIVEt
    global RELAXt
    global PHOSPHENEt
    global users
    global stims
    global reportcols
    global report
    global avgccorraggregated
    global finalstimvalue
    global Wmatrix

    global timers_set
    global timer1
    global timer2
    global timer3
    global timer4
    global phos_row
    global phos_sender
    global motor_row
    global motor_sender
    global cog_row
    global cog_sender
    global relax_row
    global relax_sender
    global ccorr_at_laststimtime
    global current_stim_time
    global binary_matrix_stim
    global i
    global check1
    global check2
    global check3
    global check4
    global baselinedone
    global loop
    global boop
    global reward

    print('kayla is angry')
    # if saving:
    #     current_time = datetime.datetime.now(pytz.timezone('Australia/Sydney'))
    #     avgccorraggregated = server_save_data(current_time, avgccorr, avgccorraggregated)

    ################################## ALGORITHM CONT FROM CLIENT
    binary_matrix, BxW, candcolumnindex, candvalues, candidates, finalstimvalue, senderindex, senders \
        = do_Wmatrix_operations(np.asarray(user_inputs[0][0]),
                                np.asarray(user_inputs[1][0]),
                                np.asarray(user_inputs[2][0]),
                                Wmatrix,
                                nameofstims=stims,
                                nameofusers=users)
    finalstimvalue = stimulation_decision(finalstimvalue,
                                          stim_threshold,
                                          users,
                                          candidates,
                                          senders,
                                          cooldown)

    avgccorr = compute_ccorr(np.asarray(user_inputs[0][1]),
                             np.asarray(user_inputs[1][1]),
                             np.asarray(user_inputs[2][1]))
    print('current avgccorr: ' + str(avgccorr))
    if (any(x == 1 for x in finalstimvalue) or timers_set):  # continue if there's actually a stim candidate, timers have been set, or one of our timers from a previous stim is finished,
        if not timers_set:
            current_stim_time = datetime.datetime.now(
                pytz.timezone('Australia/Sydney'))  # time stims were sent
            print('stims ready to send')#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
            # add cooldown of 30 mins to users being stimulated
            for bean in [0, 1, 2]:
                cooldown[bean] += cooldown_time
            print('ccorr b4: ' + str(avgccorr))
            timers_set, timer1, timer2, timer3, timer4, phos_row, phos_sender, motor_row, motor_sender, cog_row, cog_sender, relax_row, relax_sender, binary_matrix_stim, ccorr_at_laststimtime \
                = set_timers(candidates, senderindex, binary_matrix, PHOSPHENEt, MOTORt, COGNITIVEt, RELAXt,
                             timer1, timer2, timer3, timer4, phos_row, phos_sender, motor_row, motor_sender,
                             cog_row, cog_sender, relax_row, relax_sender, avgccorr)
        if timers_set and (
                check1 or check2 or check3 or check4):  # written like this in case there are multiple checks going at once, the keep function checks which timer is up and keeps the variable info for that specific stim
            BxW_colref, BxW_rowref, sender, timer1, timer2, timer3, timer4, check1, check2, check3, check4 = \
                keep_current_stim_data(timer1, timer2, timer3, timer4, check1, check2, check3, check4,
                                       phos_row, phos_sender, motor_row, motor_sender, cog_row, cog_sender, relax_row,
                                       relax_sender)
            # computing circular correlation coefficient

            stim_ccorr = compute_ccorr(np.asarray(user_inputs[0][1]),
                                       np.asarray(user_inputs[1][1]),
                                       np.asarray(user_inputs[2][1]))
            print('ccorr after: ' + str(stim_ccorr))
            ccorr_ers = (100 * (
                    stim_ccorr - ccorr_at_laststimtime) / ccorr_at_laststimtime)  # ers not erd because subtracting before from after
            if ccorr_ers > 0.5:
                reward = True
            else:
                reward = False
            print('ccorr_erd is ' + str(ccorr_ers))
            # if ccorr increased, successful state-stim associations should get rewarded
            Wmatrix = update_Wmatrix(reward,
                                     Wmatrix,
                                     BxW_rowref,
                                     BxW_colref,
                                     sender,
                                     binary_matrix_stim)
            print(Wmatrix)

            if saving:
                np.save('Wmatrix.npy', Wmatrix)
                report = server_save_report(report, current_stim_time, Wmatrix, reward, users[sender],
                                            binary_matrix_stim, ccorr_at_laststimtime, stim_ccorr)
        if timer1 == timer2 == timer3 == timer4 == -5:
            print('ccorrperiod donezo')
            # is there anything else to reset???????????????????????????????????????
            timers_set, timer1, timer2, timer3, timer4, phos_row, phos_sender, motor_row, motor_sender, cog_row, cog_sender, relax_row, relax_sender, i, check1, check2, check3, check4, baselinedone, loop, boop, reward = reset_vars()
            print("IT FUCKING WORKED")
    user1 = False
    user2 = False
    user3 = False

    timer1, timer2, timer3, timer4, check1, check2, check3, check4 = update_timers(timers_set, timer1,
                                                                                timer2,
                                                                                timer3, timer4, check1,
                                                                                check2, check3, check4)
    cooldown = update_cooldowns(cooldown)
    print(cooldown)

##########################################################################END ALGORITHM

connected = True
while connected:
    ind = 0 #so script doesnt break if nothing comes in
    #######  RECEIVE MESSAGE HERE  as "msg" #################
    message = socket.recv()
    print(len(message))
    msg = json.loads(message)

    if msg["Username"] == "Buffy":
        user_inputs[1] = [msg['Binaries'][0], msg['eeg']]
        ind = 1
        print(user_inputs[ind][0])
        print('buffy in')

    if msg["Username"] == "Willow":
        user_inputs[2] = [msg['Binaries'][0], msg['eeg']]
        ind = 2
        print(user_inputs[ind][0])
        print('willow in')

    if msg["Username"] == "Spike":
        user_inputs[0] = [msg['Binaries'][0], msg['eeg']]
        ind = 0
        print(user_inputs[ind][0])
        print('spike in')

    if candidates[ind] != 'none' and candidates[ind] != 'undefined':
        socket.send(json.dumps(candidates).encode(FORMAT))
        print('sending ' + str(candidates))
        candidates[ind] = 'none'
    else:
        socket.send(json.dumps("no stims atm").encode(FORMAT))
        print('sent no stims')

    if ind == 0 and (candidates[0] == 'none' and candidates[1] == 'none' and candidates[2] == 'none'): #one person runs the algorithm
        if usesynthetic or oneuser:
            user_inputs[1] = user_inputs[0].copy()
            user_inputs[2] = user_inputs[0].copy()
        if not (user_inputs[0] == 'undefined' or user_inputs[1] == 'undefined' or user_inputs[2] == 'undefined'): #at startup waiting for input from all users before starting algorithm
            algo = threading.Thread(target=kaylas_angry)  #note that only one user needs to run the algo for everyone
            algo.start()
            print(user_inputs)

    ########## send stims to users


from gurobipy import *
from utils import get_cap_cost, load_timeseries, get_nodal_inputs
from results_processing import node_results_retrieval, system_ts_sum, process_results
import numpy as np
import pandas as pd
import os
import datetime

def create_operation_model(args, nodes_capacity_results, scenario_name, config, lan_tlnd_out, cap_solving_time):
    print("operation model building and solving")
    print("--------####################------------")
    T = args.num_hour_ope
    trange = range(T)
    solar_region = lan_tlnd_out.DistName.lower()
    dome_load_hourly_kw, solar_po_hourly, rain_rate_daily_mm_m2 = load_timeseries(args, solar_region, mod_level="ope")
    # Extract nodal load inputs
    nodal_load_input = get_nodal_inputs(args, lan_tlnd_out)
    if config == 1:
        num_nodes = len(nodal_load_input)
    elif config == 2:
        num_nodes = 1

    # retrieve the capacity model results; Fixed solar and battery capacities, and limit the diesel capacities.
    solar_cap_model = {}
    diesel_cap_model = {}
    battery_la_cap_kwh_model = {}
    battery_la_cap_kw_model = {}
    battery_li_cap_kwh_model = {}
    battery_li_cap_kw_model = {}
    for i in range(num_nodes):
        solar_cap_model[i] = round(nodes_capacity_results.solar_cap_kw[i],2)
        diesel_cap_model[i] = round(nodes_capacity_results.diesel_cap_kw[i],2)
        battery_la_cap_kwh_model[i] = round(nodes_capacity_results.batt_la_energy_cap_kwh[i],3)
        battery_la_cap_kw_model[i]  = round(nodes_capacity_results.batt_la_power_cap_kw[i],3)
        battery_li_cap_kwh_model[i] = round(nodes_capacity_results.batt_li_energy_cap_kwh[i],3)
        battery_li_cap_kw_model[i]  = round(nodes_capacity_results.batt_li_power_cap_kw[i],3)
        print('node',i,'solar:',solar_cap_model[i],'diesel',diesel_cap_model[i],'batt_la',battery_la_cap_kwh_model[i])

    # Retrieve capital prices for solar, battery, and diesel generators
    solar_cap_cost, solar_single_cap_cost, battery_la_cap_cost_kwh, battery_li_cap_cost_kwh, \
    battery_inverter_cap_cost_kw, diesel_cap_cost_kw = get_cap_cost(args, args.num_year_ope)

    # initialize results table for storing the nodal generator capacities from operation model
    nodes_results = pd.DataFrame()
    # specify the time series outputs array
    ts_results = np.zeros((T,11,num_nodes))

    # set up the nodal loop. Each iteration will calculate the capacities of each node.
    for i in range(num_nodes):
        m = Model("operation_model_node_" + str(i))
        print('operation model node ' + str(i) + ' building and solving')

        if config == 1:
            irrigation_area_m2 = nodal_load_input["irrigation_area_ha"][i] * 1e4
            dome_load = dome_load_hourly_kw * nodal_load_input["domestic_load_customers_no"][i]
            com_power = np.array([nodal_load_input["com_power_kw"][i]])
            com_peak_hours = np.array([nodal_load_input["com_wk_hours_per_day"][i]])
        elif config == 2:
            irrigation_area_m2 = np.sum(nodal_load_input["irrigation_area_ha"]) * 1e4
            dome_load = dome_load_hourly_kw * np.sum(nodal_load_input["domestic_load_customers_no"])
            com_power_full = np.array(nodal_load_input["com_power_kw"])
            com_peak_hours_full = np.array(nodal_load_input["com_wk_hours_per_day"])
            no_com_true = com_power_full * com_peak_hours_full
            com_power = com_power_full[no_com_true > 0]
            com_peak_hours = com_peak_hours_full[no_com_true > 0]
        else:
            print("Error - wrong config")

        # Initialize capacity variables / bind the capacity from the capacity model
        solar_cap = m.addVar(name='solar_cap')
        m.setPWLObj(solar_cap, args.solar_pw_cap_kw, solar_cap_cost)
        diesel_cap = m.addVar(obj=diesel_cap_cost_kw, name='diesel_cap')
        diesel_binary = m.addVar(name='diesel_cap_binary', vtype=GRB.BINARY)
        battery_la_cap_kwh = m.addVar(obj=battery_la_cap_cost_kwh, name = 'batt_la_energy_cap')
        battery_la_cap_kw  = m.addVar(obj=battery_inverter_cap_cost_kw, name = 'batt_la_power_cap')
        battery_li_cap_kwh = m.addVar(obj=battery_li_cap_cost_kwh, name = 'batt_li_energy_cap')
        battery_li_cap_kw  = m.addVar(obj=battery_inverter_cap_cost_kw, name = 'batt_li_power_cap')

        # constrains from the capacity model
        if args.diesel_ava: # fix the battery capacity
            m.addConstr(solar_cap == solar_cap_model[i])
            m.addConstr(diesel_cap >= diesel_cap_model[i])             # give a flexibility for diesel
            m.addConstr(diesel_cap - args.diesel_min_cap * diesel_binary >= 0)
            m.addConstr(diesel_cap * (1 - diesel_binary) == 0)
            m.addConstr(battery_la_cap_kwh == battery_la_cap_kwh_model[i])
            m.addConstr(battery_la_cap_kw  == battery_la_cap_kw_model[i])
            m.addConstr(battery_li_cap_kwh == battery_li_cap_kwh_model[i])
            m.addConstr(battery_li_cap_kw  == battery_li_cap_kw_model[i])
        else: # give flexibility for battery
            m.addConstr(solar_cap >= solar_cap_model[i])
            m.addConstr(diesel_cap == diesel_cap_model[i])
            if args.battery_la_ava:
                m.addConstr(battery_la_cap_kwh >= battery_la_cap_kwh_model[i])
                m.addConstr(battery_la_cap_kw >= battery_la_cap_kw_model[i])
                m.addConstr(battery_la_cap_kwh * (1 - args.battery_la_min_soc) * float(args.battery_la_p2e_ratio_range[0]) <= battery_la_cap_kw)
                m.addConstr(battery_la_cap_kwh * (1 - args.battery_la_min_soc) * float(args.battery_la_p2e_ratio_range[1]) >= battery_la_cap_kw)
            else:
                m.addConstr(battery_la_cap_kwh == 0)
            if args.battery_li_ava:
                m.addConstr(battery_li_cap_kwh >= battery_li_cap_kwh_model[i])
                m.addConstr(battery_li_cap_kw >= battery_li_cap_kw_model[i])
                m.addConstr(battery_li_cap_kwh * (1 - args.battery_li_min_soc) * float(args.battery_li_p2e_ratio_range[0]) <= battery_li_cap_kw)
                m.addConstr(battery_li_cap_kwh * (1 - args.battery_li_min_soc) * float(args.battery_li_p2e_ratio_range[1]) >= battery_li_cap_kw)
            else:
                m.addConstr(battery_li_cap_kwh == 0)

        # Initialize time-series variables
        irrigation_load = m.addVars(trange, obj=args.irrigation_nominal_cost, name = 'irrigation_load')
        irrigation_binary = m.addVars(trange, vtype=GRB.BINARY, name = "irrigation_binary")
        com_load = m.addVars(trange, obj=args.com_nominal_cost, name = 'commercial_load')

        solar_util      = m.addVars(trange, name = 'solar_util')

        battery_la_charge     = m.addVars(trange, obj = args.nominal_charge_discharge_cost_kwh,name= 'batt_la_charge')
        battery_la_discharge  = m.addVars(trange, obj = args.nominal_charge_discharge_cost_kwh,name= 'batt_la_discharge')
        battery_la_level      = m.addVars(trange, name='batt_la_level')
        battery_li_charge     = m.addVars(trange, obj = args.nominal_charge_discharge_cost_kwh,name= 'batt_li_charge')
        battery_li_discharge  = m.addVars(trange, obj = args.nominal_charge_discharge_cost_kwh,name= 'batt_li_discharge')
        battery_li_level      = m.addVars(trange, name='batt_li_level')

        diesel_kwh_fuel_cost = args.diesel_cost_liter * args.liter_per_kwh / args.diesel_eff
        diesel_gen = m.addVars(trange, obj=diesel_kwh_fuel_cost, name="diesel_gen")
        m.update()

        # Add time-series Constraints
        for j in trange:
            # solar and diesel generation constraint
            m.addConstr(diesel_gen[j] <= diesel_cap)
            # round the number to reduce the minimum decimals
            m.addConstr(solar_util[j] <= solar_cap * round(solar_po_hourly[j], 4))

            # Energy Balance
            m.addConstr(solar_util[j] + diesel_gen[j] - battery_la_charge[j] + battery_la_discharge[j] - \
                        battery_li_charge[j] + battery_li_discharge[j] == \
                        dome_load[j] + irrigation_load[j] + com_load[j])

            # Battery operation constraints
            m.addConstr(args.battery_la_eff * battery_la_charge[j] - battery_la_cap_kw <= 0)
            m.addConstr(battery_la_discharge[j] / args.battery_la_eff - battery_la_cap_kw <= 0)
            m.addConstr(battery_la_level[j] - battery_la_cap_kwh <= 0)
            m.addConstr(battery_la_level[j] - battery_la_cap_kwh * args.battery_la_min_soc >=0)

            m.addConstr(args.battery_li_eff * battery_li_charge[j] - battery_li_cap_kw <= 0)
            m.addConstr(battery_li_discharge[j] / args.battery_li_eff - battery_li_cap_kw <= 0)
            m.addConstr(battery_li_level[j] - battery_li_cap_kwh <= 0)
            m.addConstr(battery_li_level[j] - battery_li_cap_kwh * args.battery_li_min_soc >=0)

            ## Battery control
            if j == 0:
                m.addConstr(battery_la_discharge[j] / args.battery_la_eff - args.battery_la_eff * battery_la_charge[j] ==
                            battery_la_level[T - 1] - battery_la_level[j])
                m.addConstr(battery_li_discharge[j] / args.battery_li_eff - args.battery_li_eff * battery_li_charge[j] ==
                            battery_li_level[T - 1] - battery_li_level[j])
            else:
                m.addConstr(battery_la_discharge[j] / args.battery_la_eff - args.battery_la_eff * battery_la_charge[j] ==
                            battery_la_level[j - 1] - battery_la_level[j])
                m.addConstr(battery_li_discharge[j] / args.battery_li_eff - args.battery_li_eff * battery_li_charge[j] ==
                            battery_li_level[j - 1] - battery_li_level[j])

            # make irrigation operation more reasonable by adding lowest power: 1kW
            m.addConstr(irrigation_load[j] >= float(args.irrigation_minimum_power) * irrigation_binary[j])
            m.addConstr(irrigation_load[j] <= float(args.irrigation_maximum_power) * irrigation_binary[j])
        m.update()

        # Irrigation + Rain Rate Constraints:
        #   1. create water storage in soil
        #   2. constrains on irrigation
        day_range = range(int(T/24))
        ground_water_level_mm = m.addVars(day_range, obj=args.nominal_water_level, name='ground_water_level_mm')
        ground_water_charge_mm = m.addVars(day_range, name='ground_water_charge_mm')
        ground_water_discharge_mm = m.addVars(day_range, obj=args.nominal_water_discharge, name='ground_water_discharge_mm')
        m.update()

        if irrigation_area_m2 > 0:
            m.addConstr(ground_water_level_mm[args.ope_model_1_season_start] == 0)
            m.addConstr(ground_water_level_mm[args.ope_model_2_season_start] == 0)
            for d in list(range(args.ope_model_1_season_start, args.ope_model_1_season_end+1)) + \
                     list(range(args.ope_model_2_season_start, args.ope_model_2_season_end+1)):
                # limit the hours of irrigation.
                m.addConstr(quicksum(irrigation_load[k] for k in [x+d*24 for x in args.no_irrigation_hours]) == 0)

                irrigation_daily_mm = quicksum(irrigation_load[k] for k in range((d*24), ((d+1)*24))) / \
                                         args.irrigation_kwh_p_kg / irrigation_area_m2
                m.addConstr(rain_rate_daily_mm_m2[d] + irrigation_daily_mm + ground_water_discharge_mm[d] >=
                            args.water_demand_kg_m2_day + ground_water_charge_mm[d])
                m.addConstr(ground_water_level_mm[d+1] == ground_water_level_mm[d] +
                            ground_water_charge_mm[d] - ground_water_discharge_mm[d])
                m.addConstr(ground_water_level_mm[d+1] <= (args.water_account_days-1) * args.water_demand_kg_m2_day)
                m.addConstr(ground_water_discharge_mm[d] <= args.water_demand_kg_m2_day)

            for d in list(range(args.ope_model_3_season_start, args.ope_model_3_season_end+1)):
                irrigation_daily_mm = quicksum(irrigation_load[k] for k in range((d*24), ((d+1)*24))) / \
                                      args.irrigation_kwh_p_kg / irrigation_area_m2
                m.addConstr(irrigation_daily_mm == 0)
                m.addConstr(ground_water_level_mm[d] == 0)
                m.addConstr(ground_water_charge_mm[d] == 0)
                m.addConstr(ground_water_discharge_mm[d] == 0)

        else:
            for d in day_range:
                irrigation_daily_mm = quicksum(irrigation_load[k] for k in range((d*24), ((d+1)*24)))
                m.addConstr(irrigation_daily_mm == 0)
                m.addConstr(ground_water_level_mm[d] == 0)
                m.addConstr(ground_water_charge_mm[d] == 0)
                m.addConstr(ground_water_discharge_mm[d] == 0)
        m.update()

        # Commercial load constraint / initialize variables for each commercial load
        for m_num in range(len(com_power)):
            # binary variable indicating the machine turn on or off
            com_load_binary = m.addVars(trange, vtype=GRB.BINARY, name = f"com_load_binary_{m_num}")

            for d in list(range(args.ope_model_1_season_start, args.ope_model_1_season_end + 1)) + \
                     list(range(args.ope_model_2_season_start, args.ope_model_2_season_end + 1)):
                com_hours_daily = quicksum(com_load_binary[k] for k in range((d * 24), ((d + 1) * 24)))
                m.addConstr(com_hours_daily == com_peak_hours[m_num])
                com_closed_hours_sum = quicksum(com_load_binary[k] for k in [x + d * 24 for x in args.no_com_hours])
                m.addConstr(com_closed_hours_sum == 0)

            for d in list(range(args.ope_model_3_season_start, args.ope_model_3_season_end + 1)):
                com_hours_daily = quicksum(com_load_binary[k] for k in range((d * 24), ((d + 1) * 24)))
                m.addConstr(com_hours_daily == com_peak_hours[m_num])
                com_closed_hours_sum = quicksum(com_load_binary[k] for k in [x + d * 24 for x in args.no_com_hours])
                m.addConstr(com_closed_hours_sum == 0)
        m.update()

        # retrieve all com_load_binary, put into a dictionary
        com_load_binary_dict = {}
        for m_num in range(len(com_power)):
            for j in trange:
                com_load_binary_dict[m_num, j] = m.getVarByName(f'com_load_binary_{m_num}[{j}]')
        # sum up the total commercial load
        for j in trange:
            m.addConstr(com_load[j] == quicksum(com_load_binary_dict[m_num, j] * com_power[m_num]
                                                for m_num in range(len(com_power))))
        m.update()

        # Set model solver parameters
        m.setParam("FeasibilityTol", args.feasibility_tol)
        m.setParam("OptimalityTol",  args.optimality_tol)
        m.setParam("Method",         args.solver_method)
        m.setParam("TimeLimit", args.operation_model_time_limit)
        m.setParam("OutputFlag", 1)
        # Solve the model
        ope_start_solving_time = datetime.datetime.now()
        m.optimize()
        ope_solving_time = datetime.datetime.now() - ope_start_solving_time

        ### ------------------------- Results Output ------------------------- ###
        # Process the model solution
        single_node_results, single_node_ts_results = node_results_retrieval(args, m, i, T, nodal_load_input, config, solar_region)
        nodes_results = pd.concat([nodes_results, single_node_results])
        nodes_results = nodes_results.reset_index(drop=True)
        ts_results[:,:,i] = single_node_ts_results

        # daily rain / irrigation time series
        # irrigation_daily_ts_results = get_irrigation_ts(args, m)

    # save results / get final processed results
    scenario_dir = scenario_name
    if not os.path.exists(os.path.join(args.results_dir, scenario_dir)):
        os.makedirs(os.path.join(args.results_dir, scenario_dir))
    system_ts_results = system_ts_sum(ts_results)
    #irrigation_daily_ts_results.round(decimals=3).to_csv(os.path.join(args.results_dir, scenario_dir, 'irrigation_ts.csv'))

    processed_results = process_results(args, nodes_results, system_ts_results, nodes_capacity_results,
                                        config, lan_tlnd_out, cap_solving_time, ope_solving_time)

    nodes_results.round(decimals=3).to_csv(os.path.join(args.results_dir, scenario_dir, 'raw_results.csv'))
    system_ts_results.round(decimals=3).to_csv(os.path.join(args.results_dir, scenario_name, 'ts_results.csv'))
    processed_results.round(decimals=3).to_csv(os.path.join(args.results_dir, scenario_name, 'processed_results.csv'))

    return None


from typing import List

class Solution:
    def removeDuplicates(self, nums: List[int]) -> int:
        k = 1
        n = len(nums)
        for i in range(1,n):
            if nums[i-1]<nums[i]:
                k+=1
                nums[k-1] = nums[i]
        for i in range(k,n):
            nums[i] = "_"
        return k,nums
        # return k
obj = Solution()
print(obj.removeDuplicates(nums = [0,0,1,1,1,2,2,3,3,4]))

LeetCode/Remove Duplicates from Sorted Array.py


import numpy as np
import sympy as sp
from logger import Logger
from scipy.optimize import minimize
from util import to_np_mat, to_np_vec, sigmoid, rotate, gaussian
from simulation import SimulationReporter
from delay import DelayArray

# ------------------------------------------------------------------------
# PID
# ------------------------------------------------------------------------

class PD:
    """Proportional Derivative Controller.

    a = ad + Kp (xd - x) + Kd (ud - u)

    """

    def __init__(self, Kp, Kd):
        """
        Parameters
        ----------

        Kp: proportional gain

        Kd: derivative gain

        """

        self.Kp = Kp
        self.Kd = Kd
        self.t = 0

    def compute(self, x, u, xd, ud, ad):
        """
        Parameters
        ----------

        x: current position
        u: current velocity
        xd: desired position
        ud: desired velocity
        ad: desired acceleration

        Returns
        -------

        the requred acceleration

        """

        return ad + self.Kp * (xd - x) + self.Kd * (ud - u)

class PID:
    """
    An implementation of Proportional Integral Derivative controller.

    e = xd - x
    u = Kp e + Ki \int_0^t e d\tau + Kd de/dt

    """

    def __init__(self, Kp, Ki, Kd):
        """
        Parameters
        ----------

        Kp: proportional gain
        Ki: integral gain
        Kd: derivative gain

        """

        self.Kp = Kp
        self.Ki = Ki
        self.Kd = Kd
        self.t = 0
        self.error_sum = 0

    def compute(self, t, x, u, xd, ud):
        """
        Parameters
        ----------

        x: current position
        u: current velocity
        xd: desired position
        ud: desired velocity

        Returns
        -------

        the requred acceleration

        Note: TODO if the numerical integration goes backward in time then this
        may cause instabilities

        """

        dt = np.abs(self.t - t)

        error = xd - x
        error_d = ud - u
        self.error_sum = self.error_sum + error * dt
        self.t = t

        return self.Kp * error + self.Kd * error_d + self.Ki * self.error_sum

# ------------------------------------------------------------------------
# JointSpaceController
# ------------------------------------------------------------------------

class JointSpaceController:
    """Simple joint space tracking controller.

    """

    def __init__(self, model):
        """ Constructor.

        Parameters
        ----------

        model: reference to the model

        """
        self.logger = Logger('JointSpaceController')
        self.model = model
        self.pid = PID(10, 2, 2)
        self.reporter = SimulationReporter(model)

    def controller(self, x, t, x0):
        """Default simple joint space PD controller.

        Parameters
        ----------

        x: state
        t: time
        x0: initial state

        """
        # self.logger.debug('Time: ' + str(t))
        n = self.model.nd
        q = np.array(x[:n])
        u = np.array(x[n:])
        qd = np.array(self.target(x, t, x0))
        ud = np.zeros((n))
        tau = np.array(self.pid.compute(t, q, u, qd, ud))

        # record
        self.reporter.t.append(t)
        self.reporter.q.append(q)
        self.reporter.qd.append(qd)
        self.reporter.tau.append(tau)
        # self.fs_reporter.record(t, q, u, tau.reshape((n, 1)))

        return tau

    def target(self, x, t, x0):
        """Default joint space target function.

        Parameters
        ----------

        x: state
        t: time
        x0: initial state

        """
        return [np.deg2rad(130), np.deg2rad(60), np.deg2rad(95)]

# ------------------------------------------------------------------------
# TaskSpaceController
# ------------------------------------------------------------------------

class TaskSpaceController:
    """A task space controller that moves the task goal in a direction.

    """

    def __init__(self, model, task, angle=0, evaluate_muscle_forces=False):
        """Constructor.

        Parameters
        ----------

        model: Model
            a reference to the model

        task: TaskSpace
            a reference to TaskSpace

        angle: rad (default=0)
            direction to move the task

        evaluate_muscle_forces: Boolean (default=False)
            compute muscle forces that satisfy the task constraint

        """
        self.logger = Logger('TaskSpaceController')
        self.model = model
        self.task = task
        self.angle = angle
        self.evaluate_muscle_forces = evaluate_muscle_forces
        self.pd = PD(50, 5)
        self.reporter = SimulationReporter(model)

    def controller(self, x, t, x0):
        """Controller function.

        Parameters
        ----------

        x: state
        t: time
        x0: initial state

        """
        self.logger.debug('Time: ' + str(t))
        n = self.model.nd
        q = np.array(x[:n])
        u = np.array(x[n:])
        pose = self.model.model_parameters(q=q, u=u)

        # task variables
        xc = to_np_mat(self.task.x(pose))
        uc = to_np_mat(self.task.u(pose, u))
        xd, ud, ad = self.target(x, t, x0)
        ad = sp.Matrix(self.pd.compute(xc, uc, xd, ud, ad))

        # forces
        tau, ft = self.task.calculate_force(ad, pose)
        ft = to_np_vec(ft)
        tau = to_np_vec(tau)

        # solve static optimization
        fm = None
        if self.evaluate_muscle_forces:
            m = self.model.md
            R = to_np_mat(self.model.R.subs(pose))
            RT = R.transpose()

            def objective(x):
                return np.sum(x**2)

            def inequality_constraint(x):
                return np.array(tau + RT * (x.reshape(-1, 1))).reshape(-1,)

            x0 = np.zeros(m)
            bounds = tuple([(0, self.model.Fmax[i, i]) for i in range(0, m)])
            constraints = ({'type': 'ineq', 'fun': inequality_constraint})
            sol = minimize(objective, x0,  method='SLSQP',
                           bounds=bounds,
                           constraints=constraints)
            if sol.success is False:
                raise Exception('Static optimization failed at: ' + t)

            fm = sol.x.reshape(-1,)

        # record
        self.reporter.t.append(t)
        self.reporter.q.append(q)
        self.reporter.u.append(u)
        self.reporter.x.append(np.array(xc).reshape(-1,))
        self.reporter.xd.append(np.array(xd).reshape(-1,))
        self.reporter.tau.append(tau)
        self.reporter.ft.append(ft)
        self.reporter.fm.append(fm)
        # self.fs_reporter.record(t, q, u, tau)

        return tau

    def target(self, x, t, x0):
        """ A directed sigmoid function target.

        Parameters
        ----------

        x: state
        t: time
        x0: initial state

        Returns
        -------

        (x, u, a)

        """
        pose0 = self.model.model_parameters(q=x0[:self.model.nd])
        xt0 = self.task.x(pose0)

        t0 = 1
        A = 0.3
        B = 4
        o = np.asmatrix([[0], [0]])
        xd, ud, ad = sigmoid(t, t0, A, B)

        xd = rotate(o, np.asmatrix([[xd], [0]]), self.angle)
        ud = rotate(o, np.asmatrix([[ud], [0]]), self.angle)
        ad = rotate(o, np.asmatrix([[ad], [0]]), self.angle)

        return (np.asmatrix(xt0 + xd),
                np.asmatrix(ud),
                np.asmatrix(ad))

# ------------------------------------------------------------------------
# MuscleSpaceControllerJS
# ------------------------------------------------------------------------

class MuscleSpaceControllerJS:
    """This controller uses the muscle space EoM to driving a model from a
    reference pose to a desired pose in joint space.

    """

    def __init__(self, model, musclespace):
        """Constructor.

        Parameters
        ----------

        model: a reference to the model
        musclespace: a reference to MuscleSpace

        """
        self.logger = Logger('MsucleSpaceControllerJS')
        self.model = model
        self.musclespace = musclespace
        self.pid = PID(10, 0, 10)
        self.reporter = SimulationReporter(model)

    def controller(self, x, t, x0):
        """Default simple joint space PD controller.

        Parameters
        ----------

        x: state
        t: time
        x0: initial state

        """
        self.logger.debug('Time: ' + str(t))
        n = self.model.nd
        m = self.model.md
        q = np.array(x[:n])
        u = np.array(x[n:])
        qd = np.array(self.target(x, t, x0))
        ud = np.zeros((n))
        qddot = np.array(self.pid.compute(t, q, u, qd, ud))

        # evalutate desired lmdd_d
        pose = self.model.model_parameters(q=q, u=u)
        RDotQDot = self.model.RDotQDot.subs(pose)
        RQDDot = self.model.R.subs(pose) * sp.Matrix(qddot)
        lmdd_d = sp.Matrix(RDotQDot + RQDDot)
        lmd = to_np_vec(self.model.lmd.subs(pose))
        lmd_d = to_np_vec(self.model.R.subs(pose) * sp.Matrix(u))

        tau = self.musclespace.calculate_force(lmdd_d, pose)[0]
        tau = to_np_vec(tau)

        # store record
        self.reporter.t.append(t)
        self.reporter.q.append(q)
        self.reporter.u.append(u)
        self.reporter.qd.append(qd)
        self.reporter.lmd.append(lmd)
        self.reporter.lmd_d.append(lmd_d)
        self.reporter.tau.append(tau)

        return tau

    def target(self, x, t, x0):
        """Default joint space target function.

        Parameters
        ----------

        x: state
        t: time
        x0: initial state

        """
        return [np.deg2rad(130), np.deg2rad(60), np.deg2rad(95)]

# ------------------------------------------------------------------------
# PosturalMuscleSpaceController
# ------------------------------------------------------------------------

class PosturalMuscleSpaceController:
    """This is a posture controller in muscle space. The system is disturbed and a
    muscle length controller is responsible for restoring it to its initial
    pose.

    """

    def __init__(self, model, musclespace, kp, kd, delay, a, t0, sigma, gamma):
        """ Constructor.

        Parameters
        ----------

        model: a reference to ToyModel

        muscle: a reference to TaskSpace

        kp: proportional gain

        kd: derivative gain

        delay: the delay of the reflex loops

        a: Gaussian aptitude (disturbance)

        t0: time of application (disturbance)

        sigma: outspread (disturbance)

        gamma: direction of disturbance

        """
        self.logger = Logger('PosturalMsucleSpaceController')
        self.model = model
        self.musclespace = musclespace
        self.pd = PD(kp, kd)  # (10, 10) -> full controller, (0, 10) -> reflex
        self.delay = delay
        self.a = a
        self.t0 = t0
        self.sigma = sigma
        self.gamma = gamma
        self.reporter = SimulationReporter(model)
        self.__initialize_delay_components()
        self.__calculate_task()

    def __calculate_task(self):
        """Calculate model's end effector Jacobian transpose.

        """
        xt = sp.Matrix(self.model.ee)
        Jt = xt.jacobian(self.model.Q())
        self.JtT = Jt.transpose()

    def __add_in_distrurbance(self, t, tau, pose):
        """Adds distrubance to end effector.

        """
        angle = self.gamma
        o = np.asmatrix([[0], [0]])
        fd = np.asmatrix([[gaussian(t, self.a, self.t0, self.sigma)], [0]])
        fd = rotate(o, fd, angle)
        return tau + self.JtT.subs(pose) * fd

    def __initialize_delay_components(self):
        """Initializes the delay components with the default muscle length (state(0))
        and zero length velocities.

        """
        n = self.model.nd
        m = self.model.md

        state0 = self.model.state0
        q = state0[:n]
        u = state0[n:]
        pose = self.model.model_parameters(q=q, u=u)
        self.lm0 = self.model.lm.subs(pose)
        self.lm0 = to_np_vec(self.lm0)

        delay = np.full(m, self.delay)

        self.lm_del = DelayArray(m, delay, self.lm0)
        self.lmd_del = DelayArray(m, delay, np.zeros(m))

    def controller(self, x, t, x0):
        """ Controller.

        Parameters
        ----------

        x: state
        t: time
        x0: initial state

        """
        self.logger.debug('Time: ' + str(t))
        m = self.model.md
        n = self.model.nd
        q = np.array(x[:n])
        u = np.array(x[n:])

        # compute current muscle length and derivative
        pose = self.model.model_parameters(q=q, u=u)
        lm = self.model.lm.subs(pose)
        lm = to_np_vec(lm)
        lmd = self.model.lmd.subs(pose)
        lmd = to_np_vec(lmd)

        # compute target
        lm_des = self.target(x, t, x0)
        lmd_des = np.zeros(m)
        lmdd_des = np.zeros(m)

        # update delayed values and get current (must update first)
        self.lm_del.add(t, lm)
        self.lmd_del.add(t, lmd)

        lm_del = np.array(self.lm_del.get_delayed())
        lmd_del = np.array(self.lmd_del.get_delayed())

        lmdd_des = sp.Matrix(self.pd.compute(
            lm_del, lmd_del, lm_des, lmd_des, lmdd_des))

        tau, fm = self.musclespace.calculate_force(lmdd_des, pose)
        tau = self.__add_in_distrurbance(t, tau, pose)
        tau = to_np_vec(tau)

        # record
        self.reporter.t.append(t)
        self.reporter.q.append(q)
        self.reporter.u.append(u)
        self.reporter.lm.append(lm)
        self.reporter.lm_d.append(lm_des)
        self.reporter.fm.append(fm)
        self.reporter.tau.append(tau)

        return tau

    def target(self, x, t, x0):
        """Default muscle space target function.

        Parameters
        ----------

        x: state
        t: time
        x0: initial state

        """
        return self.lm0


import argparse
import torch
import time
import imageio
import numpy as np
import matplotlib.pyplot as plt
import math
from copy import deepcopy
from pathlib import Path
from torch.autograd import Variable
from utils.make_env import make_env
from algorithms.maddpg import MADDPG

def run(config):
    global torch_actions
    model_path = (Path('./models') / config.env_id / config.model_name /
                  ('run%i' % config.run_num))
    if config.incremental is not None:
        model_path = model_path / 'incremental' / ('model_ep%i.pt' %
                                                   config.incremental)
    else:
        model_path = model_path / 'model.pt'

    if config.save_gifs:
        gif_path = model_path.parent / 'gifs'
        gif_path.mkdir(exist_ok=True)

    maddpg = MADDPG.init_from_save(model_path)
    env = make_env(config.env_id, discrete_action=maddpg.discrete_action)
    maddpg.prep_rollouts(device='cpu')
    ifi = 1 / config.fps  # inter-frame interval

    # count all tests
    good_hit = 0
    bad_hit = 0
    out_of_episode_length = 0

    if config.n_episodes == 1:
        e_pos = [[] for i in range(1)]
        p_pos = [[] for i in range(2)]
        e_vel = [[] for i in range(1)]
        p_vel = [[] for i in range(2)]
        e_reward = [[] for i in range(1)]
        p_reward = [[] for i in range(2)]
    ts = [[],[]]
    for ep_i in range(config.n_episodes):
        print("Episode %i of %i" % (ep_i + 1, config.n_episodes))
        obs = env.reset()
        if config.save_gifs:
            frames = []
            frames.append(env.render('rgb_array')[0])
        env.render('human')

        for t_i in range(config.episode_length):
            calc_start = time.time()
            # rearrange observations to be per agent, and convert to torch Variable
            torch_obs = [Variable(torch.Tensor(obs[i]).view(1, -1),
                                  requires_grad=False)
                         for i in range(maddpg.nagents)]
            # get actions as torch Variables
            torch_actions = maddpg.step(torch_obs, explore=False)
            # convert actions to numpy arrays
            actions = [ac.data.numpy().flatten() for ac in torch_actions]
            # print(actions) # 三个智能体模型给出的动作决策，整体模型的输出控制量，(action[i][0] + 1.5 )*2为控制量速度v, action[i][1]*pi/2为控制量omega
            obs, rewards, dones, infos = env.step(actions)
            # print(obs) # 三个智能体的观测结果，整体模型的输入控制量，每个智能体均为[自身速度v,theta + 自身角速度omega + 与基地的相对距离x,y + 其他智能体的相对距离和他们的速度x,y,v,theta]

            if config.n_episodes == 1:
                for i in range(infos["n_adv"]):
                    e_pos[i].append(deepcopy(infos["adversary"][0][i]))
                    e_vel[i].append(deepcopy(infos["adversary"][1][i]))
                    e_reward[i].append(deepcopy(infos["adversary"][2][i]))
                for i in range(infos["n_good"]):
                    p_pos[i].append(deepcopy(infos["goodagent"][0][i]))
                    p_vel[i].append(deepcopy(infos["goodagent"][1][i]))
                    p_reward[i].append(deepcopy(infos["goodagent"][2][i]))

            if config.save_gifs:
                frames.append(env.render('rgb_array')[0])
            if config.n_episodes == 1:
                calc_end = time.time()
                elapsed = calc_end - calc_start
                if elapsed < ifi:
                    time.sleep(ifi - elapsed)
                env.render('human')

            # judge hit(break) condition
            if -1 in dones:
                bad_hit += 1
                ts[1].append(t_i)
                print("protection was hit by adversary!")
                break
            elif 1 in dones:
                good_hit += 1
                print("good agents hit adversary!")
                ts[0].append(t_i)
                break
            elif t_i == config.episode_length-1:
                out_of_episode_length += 1
                print("out of episode_length!")
        if config.save_gifs:
            gif_num = 0
            while (gif_path / ('%i_%i.gif' % (gif_num, ep_i))).exists():
                gif_num += 1
            imageio.mimsave(str(gif_path / ('%i_%i.gif' % (gif_num, ep_i))),
                            frames, duration=ifi)
    print("-------------------------------------------------------------------")
    # print("Episodes: %d, good hit: %d, bad hit: %d, good hit percent: %.3f, good ave step:  %.1f, bad ave step:  %.1f"
    #       %(config.n_episodes, good_hit, bad_hit, good_hit*1.0/config.n_episodes, np.mean(np.array(ts[0])),np.mean(np.array(ts[1]))))
    print("Episodes: %d, good hit: %d, bad hit: %d, good hit percent: %.3f"
          %(config.n_episodes, good_hit, bad_hit, good_hit*1.0/config.n_episodes ))

    # from msvcrt import getch
    # getch()
    #
    if config.n_episodes == 1:
        plotpics(e_pos,p_pos,e_vel,p_vel,e_reward,p_reward)

    env.close()

def plotpics(e_pos,p_pos,e_vel,p_vel,e_reward,p_reward):
    e_pos_arr = np.array(e_pos)
    p_pos_arr = np.array(p_pos)
    e_vel_arr = np.array(e_vel)
    p_vel_arr = np.array(p_vel)
    e_reward_arr = np.array(e_reward)
    p_reward_arr = np.array(p_reward)

    # i is e_nums; j is p_nums
    dis_arr = np.array([[[np.sqrt(np.sum(np.square(p_pos_arr[j][k] - e_pos_arr[i][k])))
                          for k in range(len(e_pos_arr[0]))]
                         for j in range(2)]
                        for i in range(1)])

    dis_theta_arr = np.array([[[math.atan2((e_pos_arr[i][k] - p_pos_arr[j][k])[1],
                                           (e_pos_arr[i][k] - p_pos_arr[j][k])[0]) * 180/math.pi
                                for k in range(len(e_pos_arr[0]))]
                               for j in range(2)]
                              for i in range(1)])

    lips = len(e_pos_arr[0])
    step_list = list(range(lips))

    plt.figure(1) # draw move trace
    for i in range(1): # plot pos of e
        plt.plot(e_pos_arr[i][0:lips,0],e_pos_arr[i][0:lips,1],label=('e_'+str(i)))
    for i in range(2): # plot pos of p
        plt.plot(p_pos_arr[i][0:lips,0],p_pos_arr[i][0:lips,1],label=('p_'+str(i)))
    plt.legend()
    plt.xlabel('x')
    plt.ylabel('y')
    # plt.legend(['p','e'])

    plt.figure(2) # draw distance between e & p
    for i in range(1):
        for j in range(2):
            plt.plot(step_list, dis_arr[i][j],label=("p"+str(j)+"_e"+str(i)))
    # plt.plot(step_list,state_array[0:len(step_list),0])
    plt.legend()
    plt.xlabel('step')
    plt.ylabel('dis')

    plt.figure(3) # draw thetas
    line_shape = ['-','--','-.']
    for i in range(1):
        # plt.plot(step_list, np.array([math.atan2(e_vel_arr[i][m][1],e_vel_arr[i][m][0])*180/math.pi for m in step_list]), ls='-', label=("e"+str(i)+"-vel-theta"))
        for j in range(2):
            if i==0:
                plt.plot(step_list, np.array([p_vel_arr[j][m][1]*180/math.pi for m in step_list]), ls = line_shape[j], label=("p"+str(j)+"-vel-theta"))
            plt.plot(step_list, dis_theta_arr[i][j], ls = line_shape[j], label=("p"+str(j)+"_e"+str(i)+"_dis_theta"))
    plt.legend()
    plt.xlabel('step')
    plt.ylabel('theta')

    plt.figure(4) # darw v of e & p
    for i in range(1): # plot vel of e
        plt.plot(step_list, np.array([e_vel_arr[i][m][0] for m in step_list]), label=('v_e_'+str(i)))
    for i in range(2): # plot vel of p
        plt.plot(step_list, np.array([p_vel_arr[i][m][0] for m in step_list]), label=('v_p_'+str(i)))
    plt.legend()
    plt.xlabel('step')
    plt.ylabel('v')

    plt.figure(5)
    for i in range(1): # plot reward of e
        plt.plot(step_list[0:lips-1], np.array([e_reward_arr[i][m] for m in step_list][0:lips-1]), label=('reward_e_'+str(i)))
    for i in range(2): # plot reward of p
        plt.plot(step_list[0:lips-1], np.array([p_reward_arr[i][m] for m in step_list][0:lips-1]), label=('reward_p_'+str(i)))
    plt.legend()
    plt.xlabel('step')
    plt.ylabel('reward')
    plt.show()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("env_id", help="Name of environment") # Name of environment
    parser.add_argument("model_name",
                        help="Name of model") # Name of model
    parser.add_argument("run_num", default=1, type=int) # run_x
    parser.add_argument("--save_gifs", action="store_true",
                        help="Saves gif of each episode into model directory") # Saves gif of each episode into model directory
    parser.add_argument("--incremental", default=None, type=int,
                        help="Load incremental policy from given episode " +
                             "rather than final policy") # 如果采用某一次中间策略而不是最终策略
    parser.add_argument("--n_episodes", default=100, type=int) # 测试轮数
    parser.add_argument("--episode_length", default=800, type=int) #每轮测试步长
    parser.add_argument("--fps", default=60, type=int) # 帧数

    config = parser.parse_args()

    run(config)


from typing import Tuple

import pandas as pd
import zarr
from anndata import AnnData

from zarr_table.writer import (
    ATTR_COL_ORD,
    ATTR_INDEX,
    GROUP_OBS,
    GROUP_TABLE,
    GROUP_VAR,
    GROUP_X,
)

def table_to_dataframe(group: zarr.Group) -> pd.DataFrame:
    """
    Converts a Zarr table (X and obs) to a Pandas dataframe.

    Args:
        group: The Zarr table group container

    Returns:
        A dataframe with point coordinates and annotations
    """
    assert group.basename == GROUP_TABLE
    x, obs, var = _table_to_x_obs_var(group)
    return pd.concat([x, obs], axis=1)

def table_to_anndata(group: zarr.Group) -> AnnData:
    """
    Converts a Zarr table (X, obs and var) to an AnnData object.

    Args:
        group: The Zarr table group container

    Returns:
        An AnnData object
    """
    assert group.basename == GROUP_TABLE
    x, obs, var = _table_to_x_obs_var(group)
    return AnnData(X=x, obs=obs, var=var)

def _group_to_dataframe(group: zarr.Group):
    dct = {}
    columns = group.attrs[ATTR_COL_ORD]
    for col in columns:
        dct[col] = group[col]
    df = pd.DataFrame(dct)
    df.set_index(group.attrs[ATTR_INDEX], inplace=True)
    return df

def _table_to_x_obs_var(
    group: zarr.Group,
) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]:
    assert group.basename == GROUP_TABLE
    x = None
    obs = None
    var = None
    if GROUP_X in group:
        x = pd.DataFrame(group[GROUP_X])
    if GROUP_OBS in group:
        obs = _group_to_dataframe(group[GROUP_OBS])
        if GROUP_X in group:
            x.index = obs.index
    if GROUP_VAR in group:
        var = _group_to_dataframe(group[GROUP_VAR])
        if GROUP_X in group:
            x.columns = var.index
    return x, obs, var


from mock import patch, Mock
import uuid
from .test_base import (
    client,
    create_test_database,
    database_test_session,
)
from app.plants.models import Plant
from app.tasks.tasks import Tasks
import datetime
from freezegun import freeze_time

@freeze_time("2013-04-09")
class TestApp:
    def _insert_test_plant(self, session, plant: dict = {}):
        key = uuid.uuid4()
        data = {
            "name": "Test plant",
            "description": "Test Description",
            "water_every": 3,
            "journaling_key": key,
            "alive": True,
            "last_watering": datetime.datetime.now(),
        }
        data.update(plant)
        db_plant = Plant(**data)
        session.add(db_plant)
        session.commit()
        return db_plant

    @patch("app.notifications.notifications.Notifications.send")
    def test_sending_watering_reminders(
        self, m_send_notification, database_test_session
    ):
        dry_plant = self._insert_test_plant(database_test_session, {"water_every": 4})
        another_dry_plant = self._insert_test_plant(
            database_test_session, {"water_every": 3}
        )
        # Dead plant
        self._insert_test_plant(
            database_test_session, {"water_every": 12, "alive": False}
        )
        # Not a dry plant
        self._insert_test_plant(database_test_session, {"water_every": 5})
        with freeze_time("2013-04-13"):
            result = Tasks.send_watering_reminders(database_test_session)
        m_send_notification.assert_any_call(
            "en", "There are 2 plants that need to be watered"
        )
        m_send_notification.assert_any_call("es", "Hay 2 plantas que necesitan regarse")
        assert len(result) == 2
        assert set(result) == set([dry_plant, another_dry_plant])


"""
online.py

goes though each word on a word list and downloads the mp3 from forvo
"""
import asyncio
import aiohttp
from bs4 import BeautifulSoup

import re
import base64
from io import BytesIO

from pydub import AudioSegment

from post_processing import audio_post_processing

HDR = {
    'User-Agent': '''Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) \
    Chrome/23.0.1271.64 Safari/537.11''',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
    'Accept-Encoding': 'none',
    'Accept-Language': 'en-US,en;q=0.8',
    'Connection': 'keep-alive'
}

def get_mp3_from_word(word):
    """downloads the mp3 for a single word

    Args:
        word (str): a single word
    """
    sound = asyncio.run(get_mp3(word))
    return sound

async def get_mp3(word):
    """
    Downloads the mp3 file of a word
    """
    # Download from webpage
    print("Downloading mp3 file")

    async with aiohttp.ClientSession(headers=HDR) as session:
        await __request_mp3_from_forvo(session, word)
    return word

def get_mp3s_from_wordlist(words):
    """downloads mp3 from wordlist

    Args:
        words (list): a list of words in a sentence
    """
    list_of_words = asyncio.run(get_mp3s(words))

    for word in list_of_words:
        word = audio_post_processing(word)

    return list_of_words

async def get_mp3s(words):
    """
    Downloads each mp3 file for a wordlist using asyncio
    """
    # Download from webpage
    print("Downloading mp3 files")

    async with aiohttp.ClientSession(headers=HDR) as session:
        tasks = []
        for word in words:
            tasks.append(asyncio.ensure_future(__request_mp3_from_forvo(session, word)))
        forvo_data = await asyncio.gather(*tasks)

    return forvo_data

async def __parse_forvo(page):
    """
    parses html for the mp3 id
    """
    index = BeautifulSoup(page, 'html.parser')
    # find span with class play  icon-size-l
    if index.find('article', {"class" : re.compile('search_words empty')}):
        print("Failed to find word on Forvo")
        return 1

    div = index.find('div', {"id" : re.compile('play_\d+')})

    # print(span)
    # get onclick method
    onclick = div.get('onclick')
    # remove play( and ) from onclick method
    onclick = onclick[5:-1]

    # get second element
    play_id = onclick.split(",")[1]
    # get id of audio track
    play_id = base64.b64decode(play_id).decode('utf-8')

    return play_id

async def __request_mp3_from_forvo(session, word):
    """
    requests forvo for the mp3 id and then downloads it
    """
    async with session.get("https://forvo.com/search/" + word +"/en/") as response:
        forvo_data = await response.text()
        # parse webpage
        play_id =  await __parse_forvo(forvo_data)

    # Download mp3 from forvo
    async with session.get("https://forvo.com/mp3/" + play_id) as response:
        return AudioSegment.from_mp3(BytesIO(await response.read()))


import streamlit as st
import requests
from streamlit_lottie import st_lottie

# Find more emojis here: https://www.webfx.com/tools/emoji-cheat-sheet/
st.set_page_config(page_title="My Webage", page_icon=":tada:",
                   layout="wide")

def load_lottie_url(url):
    r = requests.get(url)
    if r.status_code != 200:
        return None
    return r.json()

def local_css(file_name):
    with open(file_name) as f:
        st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html=True)
local_css("style.css")

contact_form = """
        <form action="https://formsubmit.co/3b395cb5994b120cbd88920acb514e10" method="POST">
         <input type="hidden" name="_captcha" value="false">
         <input type="text" name="name" placeholder="Your name" required>
         <input type="email" name="email" placeholder="Your email" required>
         <textarea name="message" placeholder="Your message here" required></textarea>
         <button class="css-1x8cf1d edgvbvh10" type="submit">Send</button>
        </form>

        """

# --- LOAD ASSETS ---
lottie_an = load_lottie_url("https://assets2.lottiefiles.com/packages/lf20_b4hn3bqt.json")

# ---BODY---
with st.container():

    left_col, right_col = st.columns(2)
    with left_col:
        st.subheader("Hi, I am Katya Klimenkova :wave:")
        st.title("Looking for a specialist/assistant position in Minsk or remote")
        st.write("I'm passionate about finding ways to use Appscript and Python to be more efficient in my job")
        st.write("[My LinkedIn ->](https://www.linkedin.com/in/katya-klimenkova-628700224/)")

    with right_col:
        st_lottie(lottie_an, height=300, key="coding")

st.write("---")
with st.container():
    lc, cc,  rc = st.columns((3, 1, 3))
    with lc:
        st.header("About Me")
        st.write("#")
        st.write(
            """
            I'm an **easy-going** person, motivated
            by **a friendly atmosphere** in a team, where one
            can share their ideas and reach
            agreements in a rational way. I'm always **eager to learn**
            new things and tools.
            """
            )
        st.write(
            """
            I'm more of an **action-oriented** worker – an
            implementer, so if you are looking for one, I'm
            the best fit for your team.
            """
            )

        st.write("#")

        with open("KatyaKlimenkova_CV.pdf", 'rb') as f:
            pdf = f.read()

        st.download_button(
            "Download my CV",
            data=pdf,
            file_name="KatyaKlimenkova_CV.pdf",
            mime='application/octet-stream'
        )

    with rc:
        st.header("Contact Me")
        st.markdown(contact_form, unsafe_allow_html=True)


from __future__ import division, print_function
from braviz.utilities import set_pyqt_api_2
set_pyqt_api_2()

import PyQt4.QtGui as QtGui
import PyQt4.QtCore as QtCore

from braviz.interaction.qt_guis.parallel_coordinates import Ui_parallel_coordinates
from braviz.interaction.qt_models import VarListModel
from braviz.interaction.qt_dialogs import SelectOneVariableWithFilter
from braviz.interaction.sample_select import SampleLoadDialog
from braviz.readAndFilter.config_file import get_config

import braviz.readAndFilter.tabular_data as braviz_tab_data

import logging

__author__ = 'da.angulo39'

class ParallelCoordinatesApp(QtGui.QMainWindow):

    def __init__(self, scenario, broadcast, receive):

        super(ParallelCoordinatesApp, self).__init__()
        self.ui = None
        self.url = None
        self.broadcast = broadcast
        self.receive = receive

        config = get_config(__file__)
        def_vars = config.get_default_variables()
        def_var_codes = map(def_vars.get, ("nom2", "ratio1", "ratio2"))
        def_var_codes = map(braviz_tab_data.get_var_idx, def_var_codes)
        def_var_codes = filter(lambda x: x is not None, def_var_codes)
        self.cathegorical_var = braviz_tab_data.get_var_idx(def_vars["nom1"])

        self.attributes = def_var_codes
        self.sample_id = None
        self.vars_model = VarListModel(checkeable=True)
        self.vars_model.select_items(self.attributes)

        self.generate_url()
        self.setup_ui()

    def setup_ui(self):
        self.ui = Ui_parallel_coordinates()
        self.ui.setupUi(self)

        self.ui.vars_list.setModel(self.vars_model)
        self.vars_model.CheckedChanged.connect(self.vars_changed)
        self.ui.search_box.returnPressed.connect(self.filter_list)

        self.ui.cathegory_combo.addItem(
            braviz_tab_data.get_var_name(self.cathegorical_var))
        self.ui.cathegory_combo.insertSeparator(
            self.ui.cathegory_combo.count())
        self.ui.cathegory_combo.addItem("<Select cathegory>")
        self.ui.cathegory_combo.activated.connect(self.change_cathegory)
        self.ui.cathegory_combo.setCurrentIndex(0)
        self.ui.actionSelect_Sample.triggered.connect(self.set_sample)
        self.ui.clear_button.clicked.connect(self.clear_selection)

        web_view = self.ui.webView
        set=web_view.settings()
        set.setAttribute(set.DeveloperExtrasEnabled,True)

        self.refresh_web_view()

    def refresh_web_view(self):
        self.ui.webView.load(QtCore.QUrl(self.url))
        link = 'open in web browser: %s' % (
            self.url, self.url)
        self.ui.url_label.setText(link)

    def generate_url(self):
        all_vars = [self.cathegorical_var] + self.attributes
        str_vars = ",".join(map(str, all_vars))
        url = "http://127.0.0.1:8100/parallel?vars=%s" % str_vars
        if self.sample_id is not None:
            url += "&sample=%s" % self.sample_id
        self.url = url
        return url

    def vars_changed(self):
        self.attributes = [
            braviz_tab_data.get_var_idx(v) for v in sorted(self.vars_model.checked_set)]
        self.generate_url()
        self.refresh_web_view()

    def filter_list(self):
        mask = "%%%s%%" % self.ui.search_box.text()
        self.vars_model.update_list(mask)

    def resizeEvent(self, *args, **kwargs):
        # super(ParallelCoordinatesApp,self).resizeEvent(*args,**kwargs)
        self.refresh_web_view()

    def change_cathegory(self):
        if self.ui.cathegory_combo.currentIndex() == self.ui.cathegory_combo.count() - 1:
            # print "dispatching dialog"
            params = {}
            dialog = SelectOneVariableWithFilter(
                params, accept_real=False, accept_nominal=True)
            selection = dialog.exec_()
            logger = logging.getLogger(__name__)
            logger.info("Cathegories selection %s", params)
            if selection > 0:
                var = params["selected_outcome"]
                self.ui.cathegory_combo.insertItem(0, var)
                self.ui.cathegory_combo.setCurrentIndex(0)
            else:
                return
        else:
            var = self.ui.cathegory_combo.itemText(
                self.ui.cathegory_combo.currentIndex())
        self.cathegorical_var = braviz_tab_data.get_var_idx(var)
        self.generate_url()
        self.refresh_web_view()

    def set_sample(self):
        dialog = SampleLoadDialog()
        res = dialog.exec_()
        if res == dialog.Accepted:
            new_sample = dialog.current_sample_idx
            self.sample_id = new_sample
        self.generate_url()
        self.refresh_web_view()

    def clear_selection(self):
        self.vars_model.clear_selection()

if __name__ == "__main__":
    import sys

    scenario = None
    broadcast = None
    receive = None
    if len(sys.argv) > 1:
        scenario = sys.argv[1]
    if len(sys.argv) > 3:
        broadcast = sys.argv[2]
        receive = sys.argv[3]

    app = QtGui.QApplication([])
    main_window = ParallelCoordinatesApp(scenario, broadcast, receive)
    main_window.show()
    app.exec_()


from __future__ import division
import math
from datetime import datetime
import json
import pickle  # do not remove this
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.feature_selection import chi2, SelectKBest
from sklearn.tree import DecisionTreeClassifier
from sklearn.pipeline import Pipeline
from sklearn.base import TransformerMixin, BaseEstimator
from sklearn.metrics import confusion_matrix, classification_report, accuracy_score

data = pd.read_csv('data\kick_train.csv', index_col='Unnamed: 0')
data.drop(['staff_pick', 'backers_count'], axis=1, inplace=True)
data['state'] = [1 if x == 'successful' else 0 for x in data['state']]
#################################################################
xtrain, xtest, ytrain, ytest = train_test_split(data.drop('state', axis=1), data['state'],
                                                test_size=0.3, stratify=data['state'], random_state=456)
print(sum(ytrain) / ytrain.shape[0])
print(sum(ytest) / ytest.shape[0])

#################################################################

def unix_date_diff(df, col1, col2):
    out_list = []
    for x, y in zip(df[col1], df[col2]):
        z = math.floor(abs(x / 86400 - y / 86400))
        out_list.append(z)
    return out_list

def one_hot_encoding(df, val_dict):
    model_df = pd.DataFrame()
    for col in df.columns:
        dummy_df = pd.get_dummies(df[col].astype('category', categories=val_dict[col]), prefix=col)
        model_df = pd.concat([model_df, dummy_df], axis=1)
    return model_df

def categories_ohe(df):
    rcat_df = df[new_cat_features]
    cat_dict = {}
    for col in rcat_df.columns:
        rcategories = rcat_df[col].unique()
        cat_dict[col] = rcategories
    return cat_dict

def preprocess_data(df):
    df.loc[:, "profile_name"] = df["profile"].apply(lambda x: json.loads(x).get("name", None))
    df.loc[:, "profile_name"] = df["profile_name"].apply(lambda x: 1 if x == 'null' else 0)
    df.loc[:, "profile_blurb"] = df["profile"].apply(lambda x: json.loads(x).get("blurb", None))
    df.loc[:, "profile_blurb"] = df["profile_blurb"].apply(lambda x: 1 if x == 'null' else 0)
    df.loc[:, "link"] = df["profile"].apply(lambda x: json.loads(x).get("link_url", None))
    df.loc[:, "link"] = df["link"].apply(lambda x: 1 if x == 'null' else 0)
    df.loc[:, "should_show_feature_image_section"] = df["profile"].apply(lambda x: json.loads(x).get(
        "should_show_feature_image_section", None))
    df.loc[:, "show_feature_image"] = df["profile"].apply(lambda x: json.loads(x).get("show_feature_image", None))
    df.loc[:, "new_id"] = df["category"].apply(lambda x: json.loads(x).get("id", None))
    df.loc[:, "position"] = df["category"].apply(lambda x: json.loads(x).get("position", None))
    df.loc[:, "colour"] = df["category"].apply(lambda x: json.loads(x).get("color", None))
    df.loc[:, "new_category"] = df["category"].apply(lambda x: json.loads(x).get("parent_id", None))
    df.loc[:, "sub_category"] = df["category"].apply(lambda x: json.loads(x).get("name", None))
    df.loc[:, "launched_month"] = df["launched_at"].apply(lambda x: (datetime.fromtimestamp(
        int(x))).strftime('%Y-%m-%d %H:%M:%S'))
    df.loc[:, "launched_month"] = [(x.date()).month for x in pd.to_datetime(df["launched_month"])]
    df.loc[:, "created_month"] = df["created_at"].apply(lambda x: (datetime.fromtimestamp(
        int(x))).strftime('%Y-%m-%d %H:%M:%S'))
    df.loc[:, "created_month"] = [(x.date()).month for x in pd.to_datetime(df["created_month"])]
    df.loc[:, "deadline_month"] = df["deadline"].apply(lambda x: (datetime.fromtimestamp(
        int(x))).strftime('%Y-%m-%d %H:%M:%S'))
    df.loc[:, "deadline_month"] = [(x.date()).month for x in pd.to_datetime(df["deadline_month"])]
    df.loc[:, "launched_wk"] = df["launched_at"].apply(lambda x: (datetime.fromtimestamp(
        int(x))).strftime('%Y-%m-%d %H:%M:%S'))
    df.loc[:, "launched_wk"] = [(x.date()).weekday() for x in pd.to_datetime(df["launched_wk"])]
    df.loc[:, "created_wk"] = df["created_at"].apply(lambda x: (datetime.fromtimestamp(
        int(x))).strftime('%Y-%m-%d %H:%M:%S'))
    df.loc[:, "created_wk"] = [(x.date()).weekday() for x in pd.to_datetime(df["created_wk"])]
    df.loc[:, "deadline_wk"] = df["deadline"].apply(lambda x: (datetime.fromtimestamp(
        int(x))).strftime('%Y-%m-%d %H:%M:%S'))
    df.loc[:, "deadline_wk"] = [(x.date()).weekday() for x in pd.to_datetime(df["deadline_wk"])]
    df.loc[:, 'new_goal'] = df['goal'] * df['static_usd_rate']
    df.loc[:, "desc_word_count"] = df["blurb"].apply(lambda x: len(str(x).split()))
    df.loc[:, 'Create_Launch'] = unix_date_diff(df, 'launched_at', 'created_at')
    df.loc[:, 'Create_Deadline'] = unix_date_diff(df, 'deadline', 'created_at')
    df.loc[:, 'Launch_Deadline'] = unix_date_diff(df, 'deadline', 'launched_at')
    return df

def cat_transform(df, cat_dict):
    for cat in cat_dict.keys():
        new_dict_v = cat_dict[cat]
        df.loc[:, cat] = [new_dict_v[i] for i in df[cat]]
        return df

class Num(TransformerMixin, BaseEstimator):
    def fit(self, df):
        return df

    def transform(self, df):
        rev_num_df = df[new_num_features]
        rev_cat_df = df[new_cat_features]
        rev_cat_df = cat_transform(rev_cat_df, new_cat_dict)
        mod_df = one_hot_encoding(rev_cat_df, categories)
        return (pd.concat([rev_num_df, mod_df], axis=1)).values

class New_Feat(TransformerMixin, BaseEstimator):
    def fit(self, df):
        return df

    def transform(self, df):
        df = preprocess_data(df)
        return df[features]

proc_xtrain = preprocess_data(xtrain)
cols = proc_xtrain.columns
#################################################################
new_feat = list(cols[-21:])
features = ['disable_communication', 'country']
features.extend(new_feat)
features.extend(['static_usd_rate'])
print(len(features))
print(features)
#################################################################
cat_features = features[:-6]
num_features = features[-6:]
print(cat_features)
print(num_features)
#################################################################
new_num_features = []
for num in num_features:
    num_df = pd.concat([proc_xtrain[[num]], ytrain], axis=1)
    num_df1 = num_df[num_df['state'] == 1]
    suc_tot = np.percentile(num_df1[num], 50)
    num_df2 = num_df[num_df['state'] == 0]
    unsuc_tot = np.percentile(num_df2[num], 50)
    if suc_tot != unsuc_tot:
        new_num_features.append(num)
    print('Median {} of success to non-success is {:.2f}, {:.2f}'.format(num, suc_tot, unsuc_tot))
print(new_num_features)
#################################################################
for num in new_num_features:
    sns.distplot(proc_xtrain[[num]])
    plt.title(num)
    # plt.show()
#################################################################
new_cat_features = []
new_cat_dict = {}
for cat in cat_features:
    cat_df = proc_xtrain[cat]
    cat_df = cat_df.value_counts() / cat_df.shape[0]
    cat_df = cat_df.to_frame()
    cat_df = cat_df.reset_index()
    #     cat_df.loc[:, 'Cat'] = ['others' if x < 0.025 else y for x, y in
    #                             zip(cat_df[cat], cat_df[cat_df.columns[0]])]
    cat_df.loc[:, 'Cat'] = [y for y in cat_df[cat_df.columns[0]]]
    cat_df2 = cat_df.groupby('Cat')[cat].sum()
    cat_df2 = cat_df2.reset_index()
    # if min(cat_df2[cat]) > 0.025 and max(cat_df2[cat]) < 1:
    if max(cat_df2[cat]) < 1:
        new_cat_features.append(cat)
        new_cat_dict[cat] = cat_df.set_index('index')['Cat'].to_dict()
print('New category features are {}'.format(new_cat_features))
print('New category dict is {}'.format(new_cat_dict))
#################################################################
new_train_dataf = proc_xtrain[new_cat_features]
for cat in new_cat_dict.keys():
    new_dict = new_cat_dict[cat]
    new_train_dataf.loc[:, cat] = [new_dict[i] for i in new_train_dataf[cat]]
categories = categories_ohe(new_train_dataf)
print('New categories are {}'.format(categories))
#################################################################
pipeline1 = Pipeline(steps=[
    ('new_feature', New_Feat()),
    ('feature_preprocessing', Num())]
)
pipeline2 = Pipeline(steps=[
    ('best_feat', SelectKBest(score_func=chi2)),
    ('dtc', DecisionTreeClassifier())
]
)
#################################################################
xtrain2 = pipeline1.transform(xtrain)
xtest2 = pipeline1.transform(xtest)
print(xtrain2.shape)
print(xtest2.shape)
################################################################
chi_list = list(chi2(xtrain2, ytrain.values))
new_df = pd.DataFrame(data=chi_list)
chi_df = round(new_df.transpose(), 2)
chi = chi_df[chi_df[1] < 0.05].shape[0]
print(chi)
#################################################################
params = {'dtc__max_depth': np.arange(7, 15, 1),
          'dtc__criterion': ['gini', 'entropy'],
          'dtc__class_weight': [None, 'balanced'],
          'best_feat__k': [chi]
          }
#################################################################
gcv = GridSearchCV(pipeline2, params, cv=5)
xtrain2 = pipeline1.transform(xtrain)
gcv.fit(xtrain2, ytrain.values)
print(gcv.best_estimator_)
xtest2 = pipeline1.transform(xtest)
ypred2 = gcv.predict(xtest2)
print(classification_report(ytest, ypred2))
print(confusion_matrix(ytest, ypred2))
print(accuracy_score(ytest, ypred2))


from elasticsearch import Elasticsearch, helpers
import os
from nltk.tokenize import word_tokenize
from nltk.stem import PorterStemmer
import re
import pandas as pd

ps = PorterStemmer()
x =5
docidlist = []
textlist = []
es = Elasticsearch("http://localhost:9200")
print(es)
count = 0
path =  "ap89_collection"
count = 0
textcheck = False
textvar = ''
gopar = True

stoptxt = open('my_stoplist.txt', "r")
data2 = stoptxt.read()
# data_into_list = data2.replace('\n', ' ').split(".")
# print(data)
# print(len(data))
# print(data_into_list)
print(len(data2))
stoptxt.close()
filecount = 1
readingcount=1
for i in os.listdir(path):
	print("reading file: %d",readingcount)
	readingcount = readingcount+1
	# print(i)
	data = []
	file = "ap89_collection\\"+i
	for line in open(file,encoding = "utf8", errors = 'ignore'):
		# data += [str(line)]
		line2 = str(line).lower()
		newsent = ''
	# print(data)

		# if i == "ap890101":
		if gopar == True:
			if '<doc>' in line2:
				count =count+1
			if '<docno>' in line2:
				docid = line2.replace("<docno>",'')
				docid = docid.replace("</docno>",'')
				docid = docid.replace(" ",'')
				docidlist.append(docid)
				# print(docid)
			if count ==1:
				lower = str(line).lower()
				lower1 = lower.split('<doc>')
				# print(lower1[0])

			# print(count)

			if textcheck == True:
				textvar += line2
			if '<text>' in line2:
				textcheck = True
				textvar += line2
			if '</text>' in line2:
			# 	textvar = textvar.replace("<text>",'')
			# 	textvar = textvar.replace("</text>",'')
			# 	textlist.append(textvar)
				textcheck = False
			# 	print(textvar)
			# 	textvar = ''

			if '</doc>' in line2:
				textvar = textvar.replace("<text>",'')
				textvar = textvar.replace("</text>",'')
				words = word_tokenize(textvar)
				newsent = ''
				for w in words:
					anew = ps.stem(w)
					newsent = newsent +" "+anew

				newsent = newsent.replace("u.s.",'usa')
				newsent = newsent.replace(",",'')
				newsent = newsent.replace("!",'')
				newsent = newsent.replace(". ",' ')
				newsent = newsent.replace("?",'')
				newsent = newsent.replace("{",'')
				newsent = newsent.replace("}",'')
				newsent = newsent.replace("_",'')
				newsent = newsent.replace(";",'')
				newsent = newsent.replace(":",'')
				newsent= newsent.replace('"', '')
				newsent = newsent.replace("'s",'')
				newsent = newsent.replace("'re",'')
				newsent = newsent.replace("'m",'')
				newsent = newsent.replace("'ll",'')
				newsent = newsent.replace("(",'')
				newsent = newsent.replace(")",'')
				newsent = newsent.replace("'",'')
				newsent = newsent.replace("  ",' ')

				textlist.append(newsent)
				# print(newsent)
				textcheck = False
				# print(newsent)
				textvar = ''

			lower=str(line).lower()

			lower2 = lower.split("<doc>")
			test = re.findall(r"<doc>", lower2[0])
			# print(test)
			if '<doc>' in lower and count ==0:
				p=7
				# print(lower)
				# count =count +1
				# print("heres doc")
			# else:
			# 	print(lower)

print(len(docidlist))
print(len(textlist))
print(count)
# upload = helpers.bulk(client,data)
# print(len(data))
# ELASTIC_PASSWORD = "<password>"

df = pd.DataFrame(
	{'DOC_ID':docidlist,
	'text' : textlist

	})

# mappings = {
#     "mappings": {
#         "properties": {
#             "ID": {
#                 "type": "text","analyzer": "standard" # formerly "string"
#             },
#             "TEXT": {
#                 "type": "text","analyzer": "standard"
#             }
#         }
#     }
# }

# client.indices.create(index="movies", mappings=mappings)
# client.indices.create(index='ap_files', mappings = mappings)
# client.indices.create(index='ap89_data3', mappings = mappings)

#****************************************************************
body = {
    "settings" : {
        "number_of_shards": 1,
        "number_of_replicas": 1,
        "analysis": {
            "filter": {
                "english_stop": {
                    "type": "stop",
                    "stopwords": data2
                }
            },
            "analyzer": {
                "stopped": {
                    "type": "custom",
                    "tokenizer": "standard",
                    "filter": [
                        "lowercase",
                        "english_stop"
                    ]
                }
            }
      }
    },
    "mappings": {
        "properties": {
            "text": {
                "type": "text",
                "fielddata": True,
                "analyzer": "stopped",
                # "analyzer": "standard",
                "index_options": "positions"
                # "store": True,
				# "term_vector": "with_positions_offsets_payloads"
            }
        }
    }
}

# body = {
#     "mappings": {
#         "properties": {
#             "text": {
#                 "type": "text",
#                 # "fielddata": True,
#                 # "analyzer": "stopped",
#                 "analyzer": "standard",
#                 # "index_options": "positions",
#                 "store": True,
# 				"term_vector": "with_positions_offsets_payloads"
#             }
#         }
#     }
# }

es.indices.create(index="ap89_data5", body=body)
# es.indices.create(index="apfiletest1", body=body)

for i, row in df.iterrows():
	print("loading file:%d",filecount)
	doc = {
		"text": row["text"]
	}
	filecount =filecount+1

# print(df)
	print(str(df['DOC_ID'].iloc[i]))
	# print(df['DOC_ID'].iloc[i])
	# es.index(index ="apfiletest1", id = df['DOC_ID'].iloc[i],document =doc)
	# es.index(index ="ap89_data4", id = str(df['DOC_ID'].iloc[i]),document =doc)
	es.index(index ="ap89_data5", id = str(df['DOC_ID'].iloc[i]),document =doc)

#**********************************************************************


#!/bin/env python
import cv2
import numpy as np
import sys

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("pass in file.")
        exit(1)
    print(f"Streaming file {sys.argv[1]}")
    cap = cv2.VideoCapture(sys.argv[1])

    if (cap.isOpened() == False):
        print("Cant open file")
        exit(1)

    while (cap.isOpened()):
        ret, frame = cap.read()
        if ret == True:
            cv2.imshow('Frame', frame)

            if cv2.waitKey(25) & 0xFF == ord('q'):
                break
        else:
            break

    cap.release()
    cv2.destroyAllWindows()


from bandit.cli.main import _get_profile, _init_extensions
from bandit.core.config import BanditConfig
from bandit.core.constants import RANKING
from bandit.core.manager import BanditManager

from prospector.message import Location, Message
from prospector.tools.base import ToolBase

class BanditTool(ToolBase):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.manager = None
        self.profile = None
        self.config_file = None
        self.agg_type = "file"
        self.severity = 0
        self.confidence = 0

    def configure(self, prospector_config, _):
        options = prospector_config.tool_options("bandit")

        if "profile" in options:
            self.profile = options["profile"]

        if "config" in options:
            self.config_file = options["config"]

        if "severity" in options:
            self.severity = options["severity"]
            if not 0 <= self.severity <= 2:
                raise ValueError(f"severity {self.severity!r} must be between 0 and 2")

        if "confidence" in options:
            self.confidence = options["confidence"]
            if not 0 <= self.confidence <= 2:
                raise ValueError(f"confidence {self.confidence!r} must be between 0 and 2")

        b_conf = BanditConfig(config_file=self.config_file)
        profile = _get_profile(b_conf, self.profile, self.config_file)
        extension_mgr = _init_extensions()
        extension_mgr.validate_profile(profile)

        self.manager = BanditManager(b_conf, None, profile=profile)

    def run(self, found_files):
        self.manager.files_list = sorted(found_files.files)
        self.manager.exclude_files = []

        if not self.manager.b_ts.tests:
            raise ValueError("No test will run for bandit")

        self.manager.run_tests()
        results = self.manager.get_issue_list(sev_level=RANKING[self.severity], conf_level=RANKING[self.confidence])
        messages = []
        for result in results:
            loc = Location(result.fname, None, "", int(result.lineno), 0)
            msg = Message("bandit", result.test_id, loc, result.text)
            messages.append(msg)
        return messages


import PyInstaller.__main__
import platform
import shutil
import os
import zipfile
import subprocess
import json
import sys

def gversion(path):
    with open(path) as f:
        data = json.load(f)

    if "{git-commit-hash}" in data["package"]["version"]:
        lv = subprocess.check_output(["git", "describe", "--always"]).strip()

        return data["package"]["version"].replace(
            "{git-commit-hash}", lv.decode()
        )
    return data["package"]["version"]

def package(version, source, destination, profile):
    with open(source) as f:
        data = json.load(f)

    data["package"]["profile"] = profile
    data["package"]["version"] = version
    with open(destination, "w") as f:
        json.dump(data, f)

def zipdir(path, ziph):
    # ziph is zipfile handle
    for root, dirs, files in os.walk(path):
        for file in files:
            ziph.write(os.path.join(root, file))

print(
    "Building Lynx for",
    platform.system(),
    "estimated build time: ~2 min, Continue? [Y]/[n]",
    end=": ",
)

exit = input()
if str(exit) == "n":
    sys.exit()

version = gversion("lynx/lynx.json")
zips = "Lynx " + str(platform.system()) + " " + version + ".zip"
fname = "Lynx"
os.mkdir(fname)

PyInstaller.__main__.run(
    ["lynx/main.py", "--noconsole", "--onefile", "-ilynx-profile/themes/icons/logo.ico", "--hidden-import=PyQt5.sip"]
)

# Rename Executable
if platform.system() == "Linux":
    shutil.move("dist/main", "./" + fname + "/Lynx")
if platform.system() == "Windows":
    shutil.move("dist/main.exe", "./" + fname + "/Lynx.exe")

# Copy Files
shutil.copytree("lynx-profile", "./" + fname + "/lynx-profile")

# Write Package Info
package(version, "lynx/lynx.json", "./" + fname + "/lynx.json", "lynx-profile/")

# Create Zip
zipf = zipfile.ZipFile(zips, "w", zipfile.ZIP_DEFLATED, compresslevel=1)
zipdir(fname, zipf)
zipf.close()

# Remove Temporary Directories
shutil.rmtree(fname)
shutil.rmtree("dist")
shutil.rmtree("build")
os.remove("main.spec")


from Bio import ExPASy
from Bio import SwissProt
import re

def fetch_protein_sequence(access_id):
    # Fetch protein sequence using UniProt access ID
    handle = ExPASy.get_sprot_raw(access_id)
    record = SwissProt.read(handle)
    return record.sequence

def find_nglycosylation_motif_locations(sequence):
    # Search for N-glycosylation motif locations in the protein sequence
    motif = re.compile(r'N[^P][ST][^P]')
    locations = [(match.start() + 1) for match in motif.finditer(sequence)]
    return locations

def main(file_path):
    with open(file_path, 'r') as file:
        access_ids = [line.strip() for line in file]

    for access_id in access_ids:
        try:
            sequence = fetch_protein_sequence(access_id)
            locations = find_nglycosylation_motif_locations(sequence)
            if locations:
                print(access_id)
                print(' '.join(map(str, locations)))
        except Exception as e:
            print(f"{access_id}: Error fetching sequence. {e}")

if __name__ == "__main__":
    input_file = "access_ids.txt"  # Replace with the path to your input file with UniProt access IDs
    main(input_file)

Python - Rosalind bioinformatics/glycosylation motif.py


import argparse
import json
import sys
import os
from tqdm import tqdm

from .v5_client import LuminosoClient
from .v5_constants import URL_BASE

DESCRIPTION = 'Download documents from a Luminoso project via the command line.'
DOCS_PER_BATCH = 1000

# The fields we want for "concise" or "expanded" downloads
CONCISE_FIELDS = ['title', 'text', 'metadata']
EXPANDED_FIELDS = CONCISE_FIELDS + ['terms', 'fragments', 'vector']

def _sanitize_filename(filename):
    """
    Get a filename that lacks the / character (so it doesn't express a path by
    accident) and also lacks spaces (just for tab-completion convenience).
    """
    return filename.replace('/', '_').replace(' ', '_')

def iterate_docs(client, expanded=False, progress=False):
    """
    Yield each document in a Luminoso project in turn. Requires a client whose
    URL points to a project.

    If expanded=True, it will include additional fields that Luminoso added in
    its analysis, such as 'terms' and 'vector'.

    Otherwise, it will contain only the fields necessary to reconstruct the
    document: 'title', 'text', and 'metadata'.

    Shows a progress bar if progress=True.
    """
    # Get total number of docs from the project record
    num_docs = client.get()['document_count']
    progress_bar = None
    try:
        if progress:
            progress_bar = tqdm(desc='Downloading documents', total=num_docs)

        for offset in range(0, num_docs, DOCS_PER_BATCH):
            response = client.get(
                'docs', offset=offset, limit=DOCS_PER_BATCH,
                fields=EXPANDED_FIELDS if expanded else CONCISE_FIELDS
            )
            docs = response['result']
            for doc in docs:
                if progress:
                    progress_bar.update()
                yield doc

    finally:
        if progress:
            progress_bar.close()

def download_docs(client, output_filename=None, expanded=False):
    """
    Given a LuminosoClient pointing to a project and a filename to write to,
    retrieve all its documents in batches, and write them to a JSON lines
    (.jsons) file with one document per line.
    """
    if output_filename is None:
        # Find a default filename to download to, based on the project name.
        projname = _sanitize_filename(client.get()['name'])
        output_filename = '{}.jsons'.format(projname)

        # If the file already exists, add .1, .2, ..., after the project name
        # to unobtrusively get a unique filename.
        counter = 0
        while os.access(output_filename, os.F_OK):
            counter += 1
            output_filename = '{}.{}.jsons'.format(projname, counter)

        print('Downloading project to {!r}'.format(output_filename))

    with open(output_filename, 'w', encoding='utf-8') as out:
        for doc in iterate_docs(client, expanded=expanded, progress=True):
            print(json.dumps(doc, ensure_ascii=False), file=out)

def _main(argv):
    """
    Handle arguments for the 'lumi-download' command.
    """
    parser = argparse.ArgumentParser(
        description=DESCRIPTION,
        formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument(
        '-b',
        '--base-url',
        default=URL_BASE,
        help='API root url, default: %s' % URL_BASE,
    )
    parser.add_argument('-f', '--token-file',
                        help='file where an API token was saved')
    parser.add_argument(
        '-e', '--expanded',
        help="Include Luminoso's analysis of each document, such as terms and"
             ' document vectors',
        action='store_true',
    )
    parser.add_argument(
        'project_id', help='The ID of the project in the Daylight API'
    )
    parser.add_argument(
        'output_file', nargs='?', default=None,
        help='The JSON lines (.jsons) file to write to'
    )
    args = parser.parse_args(argv)

    client = LuminosoClient.connect(
        url=args.base_url, token_file=args.token_file,
        user_agent_suffix='lumi-download'
    )
    proj_client = client.client_for_path('projects/{}'.format(args.project_id))
    download_docs(proj_client, args.output_file, args.expanded)

def main():
    """
    The setuptools entry point.
    """
    _main(sys.argv[1:])


import time
from utils import open_file

dir_path = __file__.split("/")
day = int(dir_path[-1][3:-3])
year = int(dir_path[-2])
import networkx as nx

TESTSTRING = """shiny gold bags contain 2 dark red bags.
dark red bags contain 2 dark orange bags.
dark orange bags contain 2 dark yellow bags.
dark yellow bags contain 2 dark green bags.
dark green bags contain 2 dark blue bags.
dark blue bags contain 2 dark violet bags.
dark violet bags contain no other bags."""

STRING = """light red bags contain 1 bright white bag, 2 muted yellow bags.
dark orange bags contain 3 bright white bags, 4 muted yellow bags.
bright white bags contain 1 shiny gold bag.
muted yellow bags contain 2 shiny gold bags, 9 faded blue bags.
shiny gold bags contain 1 dark olive bag, 2 vibrant plum bags.
dark olive bags contain 3 faded blue bags, 4 dotted black bags.
vibrant plum bags contain 5 faded blue bags, 6 dotted black bags.
faded blue bags contain no other bags.
dotted black bags contain no other bags."""

def parse_input():
    inputs = open_file(day, year)
    # inputs= TESTSTRING.split('\n')
    g = nx.DiGraph()
    for i in inputs:
        outer, inner = (
            i.strip()
            .replace("bags", "")
            .replace("bag", "")
            .replace(".", "")
            .strip()
            .split("contain")
        )
        outer = outer.strip()
        if "no other" in inner:
            g.add_node(outer)
            continue
        inner = inner.strip()
        inner_list = inner.split(",")
        for j in inner_list:
            if j.strip()[0].isdigit():
                real_inner_list = j.strip().split(" ")
                real_inner = real_inner_list[1] + " " + real_inner_list[2]
                weight = int(real_inner_list[0])
            else:
                real_inner = j
                weight = 1
            g.add_edge(outer, real_inner, weight=weight)
    return g

def run1(bags):
    return len(nx.ancestors(bags, "shiny gold"))

def run2(bags, u="shiny gold"):
    inner_bags = bags.successors(u)
    count = 0
    for v in inner_bags:
        count += bags.edges[u, v]["weight"] * (run2(bags, v) + 1)
    return count

if __name__ == "__main__":
    a = time.time()
    inputs = parse_input()
    f = run1(inputs)
    g = run2(inputs)
    print(f"Runtime: {round((time.time() - a)*1000,3)}ms")
    print(f"Part 1: {f}")
    print(f"Part 2: {g}")


#!/usr/bin/python3
from flask import Flask, make_response, request, render_template, redirect, url_for

app = Flask(__name__)

# entry point for our users
# renders a template that asks for their name
# index.html points to /setcookie
@app.route("/index")
@app.route("/")
def index():
    return render_template("index.html")

# set the cookie and send it back to the user
@app.route("/setcookie", methods = ["POST", "GET"])
def setcookie():
    try:
        if request.method == "POST":
            user_first = request.form["first"]
            user_last = request.form["last"]

        # Note that cookies are set on response objects.
        # Since you normally just return strings
        # Flask will convert them into response objects for you
            resp = make_response(render_template("readcookie.html"))
        # add a cookie to our response object
                   #cookievar #value
            resp.set_cookie("userID_f", user_first)
            resp.set_cookie("userID_l", user_last)

        # return our response object includes our cookie
            return resp
    except:
        return redirect(url_for("index"))

    if request.method == "GET":
        return redirect(url_for("getcookie"))

# check users cookie for their name
@app.route("/getcookie")
def getcookie():
    try:
        print("****I AM HERE*****")
        f_name = request.cookies.get("userID_f")
        l_name = request.cookies.get("userID_l")
        return '
welcome '+f_name+' '+l_name+'
'
    except:
        return redirect(url_for("index"))

if __name__ == "__main__":
    app.run(port=5006)