print(''' ----------------------------------------------------
NK search (Version 1.0)

----------------------------------------------------

By Ron Tidhar & Tim Ott

----------------------------------------------------''')

'''
Given a set of NK landscapes (see NK_Model_basic.py),
this script applies local search, decision weaving search (Ott & Eisenhardt,2016), and chunky search (Baumann & Siggelkow, 2013) strategies
Outputting the results with errors presented as one standard deviation
'''

# *** IMPORTS ***

import numpy as np
import itertools
from time import time
import os
import sys
import scipy.stats as st
import random

def main(inputFile, runs, thresh, chunky, ss_prob):
    try:
        NK = np.load(inputFile)
    except IndexError:
        print("ERROR: Please input a valie NK landscape (.npy file)")
        print("You can use NK_Model_basic.py to generate a set of landscapes")
        sys.exit(1)
    except FileNotFoundError:
        print("ERROR: Please input a valie NK landscape (.npy file)")
        print("You can use NK_Model_basic.py to generate a set of landscapes")
        sys.exit(1)

    '''
    NK has dimensions (i, 2**N, N*2+2), where
      i     = number of landscapes
      2**N  = the number of possible decision sets
      N*2+2 = the value of each point on each landscape.
                - the first N columns are for the combinations of N decision variables (DV)
                    (hence we have 2**N rows for the total number of possible combinations)
                - the second N columns are for the contribution values of each DV
                - the next valuer is for the total fit (avg of N contributions)
                - the last one is to find out whether it is the local peak (0 or 1)
    '''
    chunk_flags = {1:"chunky", 2:"incremental", 3:"integrated"}

    runs = runs
    threshold = thresh
    chunk_key = chunky
    ss_prob = ss_prob

    filename = str(inputFile)
    print("Simulating "+filename+"...", end=" ")
    P = int(filename[filename.find("P_")+2])
    D = int(filename[filename.find("D_")+2])
    Kw = float(filename[filename.find("within_")+7:filename.find("b")-3])
    Kb = float(filename[filename.find("between_")+8:filename.find("i_")-1])
    Domain_decision_set = domain_dec_set(D)
    i = NK.shape[0]
    N = int(np.log2(NK.shape[1]))
    Power_key = powerkey(N)

    num_local_steps = []
    num_local_scans = []
    final_local_fitness = []
    norm_local_fitness = []
    final_local_diff = []
    nonglobal_local_fit = []
    local_global_count = 0

    num_dw_steps = [[] for i in range(P)]
    num_dw_steps_per_domain = [[] for i in range(P)]
    num_dw_scans = [[] for i in range(P)]
    final_dw_fitness = [[] for i in range(P)]
    norm_dw_fitness = [[] for i in range(P)]
    final_dw_diff = [[] for i in range(P)]
    nonglobal_dw_fit = [[] for i in range(P)]
    dw_global_count = [0 for i in range(P)]
    dw_array = [[float('nan'),float('nan'),float('nan'),[]] for i in range(3)]

    num_chunky_steps = []
    num_chunky_scans = []
    final_chunky_fitness = []
    norm_chunky_fitness = []
    final_chunky_diff = []
    nonglobal_chunky_fit = []
    chunky_global_count = 0

    for land in range(i):
        max_fit = max_fitness(NK[land], N, Power_key)
        for _ in range(runs):
            start = random_start(N)
            start_fit = fitness(start, N, NK[land], Power_key)
            norm_start_fit = start_fit/max_fit

            (local_steps, local_scans, local_fit) = local_search(start, N, NK[land], Power_key)
            for p in range(P):
                (dw_steps, dw_scans, dw_fit, dw_steps_per_domain)= decision_weaving_threshold(start, N, P, D, NK[land], Power_key, Domain_decision_set, threshold, ss_prob, p)
                dw_array[p][0] = dw_steps
                dw_array[p][1] = dw_scans
                dw_array[p][2] = dw_fit
                dw_array[p][3] = dw_steps_per_domain
            (chunky_steps, chunky_scans, chunky_fit) = chunky_search(start, N, NK[land], Power_key, chunk_flags[chunk_key])

            num_local_steps.append(local_steps)
            num_local_scans.append(local_scans)
            final_local_fitness.append(local_fit)
            norm_local_fit = local_fit / max_fit
            norm_local_fitness.append(norm_local_fit)
            final_local_diff.append(norm_local_fit - norm_start_fit)
            if local_fit == max_fit:
                local_global_count += 1
            else:
                nonglobal_local_fit.append(norm_local_fit)

            for p in range(P):
                num_dw_steps[p].append(dw_array[p][0])
                num_dw_steps_per_domain[p].append(dw_array[p][3])
                num_dw_scans[p].append(dw_array[p][1])
                final_dw_fitness[p].append(dw_array[p][2])
                norm_dw_fit = (dw_array[p][2])/max_fit
                norm_dw_fitness[p].append(norm_dw_fit)
                final_dw_diff[p].append(norm_dw_fit - norm_start_fit)
                if dw_array[p][2] == max_fit:
                    dw_global_count[p] += 1
                else:
                    nonglobal_dw_fit[p].append(norm_dw_fit)

            num_chunky_steps.append(chunky_steps)
            num_chunky_scans.append(chunky_scans)
            final_chunky_fitness.append(chunky_fit)
            norm_chunky_fit = chunky_fit/max_fit
            norm_chunky_fitness.append(norm_chunky_fit)
            final_chunky_diff.append(norm_chunky_fit - norm_start_fit)
            # to recreate the table that B&S have need other variables
            if chunky_fit == max_fit:
                chunky_global_count += 1
            else:
                nonglobal_chunky_fit.append(norm_chunky_fit)

    print("Finished!")
    print("Local Search results for " + str(i) + " landscapes:")
    print_fitness(final_local_fitness, norm_local_fitness, final_local_diff)
    print_num_steps(num_local_steps)
    print_num_scans(num_local_scans)
    #B&S stats
    print_fitness_peak((local_global_count/(i*runs)), nonglobal_local_fit)

    dw_out = []
    for p in range(P):
        print("Decision Weaving results for " + str(i) + " landscapes starting on domain: " + str(p))
        print_fitness(final_dw_fitness[p], norm_dw_fitness[p], final_dw_diff[p])
        print_num_steps(num_dw_steps[p])
        print_num_domain_steps(num_dw_steps_per_domain[p])
        print_num_scans(num_dw_scans[p])
        #B&S stats
        print_fitness_peak((dw_global_count[p]/(i*runs)), nonglobal_dw_fit[p])

    print("Chunky Search results for " + str(i) + " landscapes:")
    print_fitness(final_chunky_fitness, norm_chunky_fitness, final_chunky_diff)
    print_num_steps(num_chunky_steps)
    print_num_scans(num_chunky_scans)
    # B&S stats
    print_fitness_peak((chunky_global_count/(i*runs)), nonglobal_chunky_fit)

    #Really what we want to be doing is returning the info so that it can be output later.
    """result_dict = {'P' : P, 'D' : D, 'Kw' : Kw, 'Kb' : Kb, 'threshold' : threshold, 'ss_prob' : ss_prob, 'local_result' : {}, 'dw_result' : {}, 'chunky_result' : {}}
    result_dict['local_result'] = {'norm_fit' : np.mean(norm_local_fitness), 'fit_conf' : conf_int_hw(norm_local_fitness), 'avg_impr' : np.mean(final_local_diff), 'impr_conf' : conf_int_hw(final_local_diff), 'pct_global' : (local_global_count/(i*runs)), 'nonglobal_fit' : np.mean(nonglobal_local_fit), 'nonglobal_conf' : conf_int_hw(nonglobal_local_fit), 'avg_steps' : np.mean(num_local_steps), 'step_conf' : conf_int_hw(num_local_steps), 'avg_scans' : np.mean(num_local_scans), 'scan_conf' : conf_int_hw(num_local_scans)}
    result_dict['dw_result'] = {'norm_fit' : np.mean(norm_dw_fitness), 'fit_conf' : conf_int_hw(norm_dw_fitness), 'avg_impr' : np.mean(final_dw_diff), 'impr_conf' : conf_int_hw(final_dw_diff), 'pct_global' : (dw_global_count/(i*runs)), 'nonglobal_fit' : np.mean(nonglobal_dw_fit), 'nonglobal_conf' : conf_int_hw(nonglobal_dw_fit), 'avg_steps' : np.mean(num_dw_steps), 'step_conf' : conf_int_hw(num_dw_steps), 'avg_scans' : np.mean(num_dw_scans), 'scan_conf' : conf_int_hw(num_dw_scans)}
    result_dict['chunky_result'] = {'norm_fit' : np.mean(norm_chunky_fitness), 'fit_conf' : conf_int_hw(norm_chunky_fitness), 'avg_impr' : np.mean(final_chunky_diff), 'impr_conf' : conf_int_hw(final_chunky_diff), 'pct_global' : (chunky_global_count/(i*runs)), 'nonglobal_fit' : np.mean(nonglobal_chunky_fit), 'nonglobal_conf' : conf_int_hw(nonglobal_chunky_fit), 'avg_steps' : np.mean(num_chunky_steps), 'step_conf' : conf_int_hw(num_chunky_steps), 'avg_scans' : np.mean(num_chunky_scans), 'scan_conf' : conf_int_hw(num_chunky_scans)}"""

    return(norm_dw_fitness)

# FUNCTIONS

def powerkey(N):
    '''
    Used to find the location on the landscape for a given combination
    of the decision variables. Maps a decision string to the correct row in the NK_land matrix.
    Returns a vector with the powers of two: (2^(N-1), ..., 1)
    '''
    Power_key = np.power(2, np.arange(N - 1, -1, -1))
    return(Power_key)

def calc_fit(N, NK_land, inter_m, Current_position, Power_key):
    '''
    Takes landscape and a combination and returns a vector of fitness
    values (contribution value for each of the N decision variables)
    '''
    Fit_vector = np.zeros(N)
    for ad1 in np.arange(N):
        Fit_vector[ad1] = NK_land[np.sum(Current_position * inter_m[ad1] * Power_key), ad1]
    return(Fit_vector)

def random_start(N):
    return np.random.randint(2, size=N)

def local_search(dec, N, NK, Power_key):
    new_dec = dec.copy()
    new_dec[0] = abs(dec[0] - 1)
    stepped = 0
    scanned = 1 #you scan your random spot before you start moving

    while(True):
        stepped += 1
        (new_dec, scans) = local_step(N, NK, dec, Power_key)
        scanned += scans
        if (all(new_dec == dec)):
            dec = new_dec
            break
        else:
            dec = new_dec
    return stepped, scanned, fitness(dec, N, NK, Power_key)

def local_step(N, NK, Current_position, Power_key):
    '''
    Local search strategy operates by changing only one decision
    to reach a better position in the solution space.
    Thus, for a given NK instance and decision set,
    we randomly iterate through neighbours, and go to the first, better option
    (In keeping with Levinthal, 1997, we don't assume the agent goes to the
    highest-valued neighbour)
    '''
    scans = 0

    # first make sure we're not at a local peak (if yes, we're done)
    if not local_max(Current_position, N, NK, Power_key):
        Indexes = np.arange(N)
        np.random.shuffle(Indexes)

        Current_fit = fitness(Current_position, N, NK, Power_key)
        New_position = Current_position.copy()

        for new_dec in Indexes:  # check for the neighbourhood
            scans += 1
            New_position[new_dec] = abs(New_position[new_dec] - 1)

            if (fitness(New_position, N, NK, Power_key) > Current_fit):
                # We have found a better position
                return (New_position, scans)
            # We didn't find a better position => change decision back
            New_position[new_dec] = abs(New_position[new_dec] - 1)

    # If we're here, we must be at a local optimum
    return (Current_position, scans)

def decision_weaving(Current_position, N, P, D, NK, Power_key, Domain_decision_set):
    New_position = Current_position.copy()
    New_position[0] = abs(Current_position[0] - 1)
    unvisited_policies = np.arange(P)
    stepped = 0
    scanned = 0

    np.random.shuffle(unvisited_policies)
    for policy in unvisited_policies:
        (New_position, steps, scans) = search_domain(N, P, D, NK, Current_position, policy, Power_key, Domain_decision_set)
        stepped += steps
        scanned += scans

        Current_position = New_position.copy()

    return stepped, scanned, fitness(Current_position, N, NK, Power_key)

def search_domain(N, P, D, NK, Current_position, policy, Power_key, Domain_decision_set):
    New_position = Current_position.copy()
    random.shuffle(Domain_decision_set)
    steps = 0
    scans = 0

    for pp in Domain_decision_set:
    # check for other decision sets within domain (policy can change)
        if not (all(pp == Current_position[policy*D:(policy+1)*D])):
            scans += 1
            New_position = Current_position.copy()
            New_position[policy*D:(policy+1)*D] = pp

            if (fitness(New_position, N, NK, Power_key) > fitness(Current_position, N, NK, Power_key)):
                # We have found a better position
                steps += 1
                Current_position = New_position.copy()

            New_position = stepping_stone(N, P, D, NK, Current_position, policy, Power_key)
            #scans += 1

            if not (all(Current_position == New_position)):
                steps += 1
                Current_position = New_position.copy()
            #if (steps >= 3):
            #    break

    return Current_position, steps, scans

def decision_weaving_threshold(Current_position, N, P, D, NK, Power_key, Domain_decision_set, threshold, ss_prob, p):
    New_position = Current_position.copy()
    #why?
    New_position[0] = abs(Current_position[0] - 1)
    unvisited_policies = np.arange(P)
    stepped = 0
    scanned = 1 #you scan the position you start in
    steps_per_domain = []

    #Need to set the first domain to p, and shuffle the rest
    temp_policies = np.delete(unvisited_policies, p)
    np.random.shuffle(temp_policies)
    unvisited_policies[0]=p
    unvisited_policies[1:]=temp_policies

    for policy in unvisited_policies:
        (New_position, steps, scans) = search_domain_threshold(N, P, D, NK, Current_position, policy, Power_key, Domain_decision_set, threshold, ss_prob)
        stepped += steps
        scanned += scans
        steps_per_domain.append(steps)

        Current_position = New_position.copy()

    return stepped, scanned, fitness(Current_position, N, NK, Power_key), steps_per_domain

def search_domain_threshold(N, P, D, NK, Current_position, policy, Power_key, Domain_decision_set, learning_threshold, ss_prob):
    New_position = Current_position.copy()
    random.shuffle(Domain_decision_set)
    focus_steps = 0
    focus_scans = 0
    ss_steps = 0

    for pp in Domain_decision_set:
    # check for other decision sets within domain (policy can change)
        if not (all(pp == Current_position[policy*D:(policy+1)*D])):
            focus_scans += 1
            New_position = Current_position.copy()
            New_position[policy*D:(policy+1)*D] = pp

            if (fitness(New_position, N, NK, Power_key) > fitness(Current_position, N, NK, Power_key)):
                # We have found a better position
                focus_steps += 1
                Current_position = New_position.copy()

            #Randomly look for a stepping stone, not always
            if (np.random.binomial(1,ss_prob)):
                #Stepping stone block
                New_position = stepping_stone(N, P, D, NK, Current_position, policy, Power_key)
                if not (all(Current_position == New_position)):
                    ss_steps += 1
                    Current_position = New_position.copy()

            # Want to switch domains if we have made more improvements in this domain than the threshold
            if (focus_steps >= learning_threshold):
                break

    return Current_position, focus_steps+ss_steps, focus_scans

def stepping_stone(N, P, D, NK, Current_position, policy, Power_key):
    #This is the first iteration of stepping stones, where we randomly look at a new position in any domain
    Current_fit = fitness(Current_position, N, NK, Power_key)
    New_position = Current_position.copy()

    stepping_stones = np.delete(range(P*D), list(range(policy*D, (policy+1)*D)))
    np.random.shuffle(stepping_stones)
    dec = stepping_stones[0]

    New_position[dec] = abs(New_position[dec] - 1)

    if (fitness(New_position, N, NK, Power_key) > Current_fit):
        # We have found a better position
        return New_position
    else:
        return Current_position

def stepping_stone2(N, P, D, NK, Current_position, policy, Power_key):
    #This is the first iteration of stepping stones, where we choose the best overall performance of the 4 possible choice sets in a given domain
    Current_fit = fitness(Current_position, N, NK, Power_key)
    temp_position = New_position = Current_position.copy()
    max_fit =0

    #Create a list of the domains that are not currently in focus
    ss_domains = np.delete(np.arange(P),policy)
    #randomly pick a domain
    np.random.shuffle(ss_domains)
    ss_domain = ss_domains[0]
    #loop through chosen domain and look for best possible 1 decision change that can be made
    for dec in list(range(ss_domain*D, (ss_domain+1)*D)):
        temp_position[dec] = abs(New_position[dec] - 1)
        if (fitness(temp_position, N, NK, Power_key) > max_fit):
            max_fit = fitness(temp_position, N, NK, Power_key)
            ss_dec = dec
    # after finding max of the possible SS, set new position to that and return
    New_position[ss_dec] = abs(New_position[ss_dec] - 1)
    #return New_position
    if (fitness(New_position, N, NK, Power_key) > Current_fit):
        # We have found a better position
        return New_position
    else:
        return Current_position

def chunky_search(Current_position, N, NK, Power_key, flag="chunky"):
    # 1. set chunk order: chunky = {5+1+...+1}, incremental = {1+...+1}, integrated = {9}
    # 2. local search first chunk
    # 3. converge
    # 4. expand chunk
    # 5. repeat until chunk = N (i.e., integrated - this is just local search)

    if (flag == "chunky"):
        chunks = [int((N+1)/2)] + [1 for i in range(N - int((N+1)/2))]
    elif (flag == "incremental"):
        chunks = [1 for i in range(N)]
    elif (flag == "integrated"):
        # chunks = [N] (this local search)
        return local_search(Current_position, N, NK, Power_key)
    else:
        print("ERROR: Please provide a correct flag to indicate chunky search type")

    stepped = 0
    scanned = 1 #you evaluate your initial starting position

    for j in range(len(chunks)):
        New_position = Current_position.copy()
        New_position[0] = abs(Current_position[0] - 1)
        chunk_ind = np.sum(chunks[:j+1])

        while(True):
            stepped += 1
            (New_position, scans) = chunk_local_step(N, NK, Current_position, chunk_ind, Power_key)
            scanned += scans
            if (all(New_position == Current_position)):
                Current_position = New_position
                break
            else:
                Current_position = New_position

    # At this point, we have an integrated search domain, which is simply local search
    (integrated_step, integrated_scan, fitness) = local_search(Current_position, N, NK, Power_key)
    stepped += integrated_step
    scanned += integrated_scan

    return stepped, scanned, fitness

def chunk_local_step(N, NK, Current_position, chunk_ind, Power_key):
    scans = 0

    Indexes = np.arange(chunk_ind)
    np.random.shuffle(Indexes)

    Current_fit = chunk_fitness(Current_position, chunk_ind, N, NK, Power_key)
    New_position = Current_position.copy()

    for new_dec in Indexes:  # check for the neighbourhood
        scans += 1
        New_position[new_dec] = abs(New_position[new_dec] - 1)

        if (chunk_fitness(New_position, chunk_ind, N, NK, Power_key) > Current_fit):
            # We have found a better position
            return (New_position, scans)
        # We didn't find a better position => change decision back
        New_position[new_dec] = abs(New_position[new_dec] - 1)

    # If we're here, we must be at a local (parochial) optimum
    return (Current_position, scans)

def current_policy(P, D, decision):
    policy = []
    for pp in range(P):
        pol = 0
        for dec in range(D):
            pol += decision[pp*D + dec]

        policy.append(int(int(pol) > int(D/2))) # Policy is defined by majority decisions

    return policy

def fitness(position, N, NK, Power_key):
    return NK[np.sum(position*Power_key), 2*N]

def local_max(position, N, NK, Power_key):
    return NK[np.sum(position*Power_key), 2*N+1]

def max_fitness(NK, N, Power_key):
    #Need to change this to utilize local_max column in landscape matrix
    max_fit=0
    for dec in itertools.product(range(2), repeat=N):
        curr_fit = fitness(dec, N, NK, Power_key)
        if curr_fit > max_fit:
            max_fit = curr_fit
    return max_fit

def chunk_fitness(position, num_dec_in_chunk, N, NK, Power_key):
    return np.mean(NK[np.sum(position*Power_key), N:(N+num_dec_in_chunk)])

def print_num_steps(steps, indent = "     "):
    print_str = indent + "Average # of steps  = "
    print_str += conf_interval(steps)

    print(print_str)

def print_num_domain_steps(steps, indent = "     "):
    l =np.asarray(steps).T.tolist()
    for index, item in enumerate(l):
        print_str = indent + "Average # of steps in domain " + str(index+1) + " = "
        print_str += conf_interval(item)

        print(print_str)

def print_num_scans(steps, indent = "     "):
    print_str = indent + "Average # of scans  = "
    print_str += conf_interval(steps)

    print(print_str)

def print_fitness(fitness, norm_fitness, diff, indent = "     "):
    #print_str = indent + "Average fitness     = "
    #print_str += conf_interval(fitness)

    #print(print_str)

    print_str = indent + "Normalized fitness     = "
    print_str += conf_interval(norm_fitness)

    print(print_str)

    print_str = indent + "Average improvement = "
    print_str += conf_interval(diff)

    print(print_str)

def print_fitness_peak(percent_global, norm_nonglobal_fitness, indent = "     "):
    print_str = indent + "% teams at global peak     = "
    print_str += str(percent_global*100)

    print(print_str)

    print_str = indent + "Nonglobal fitness     = "
    print_str += conf_interval(norm_nonglobal_fitness)

    print(print_str)

def conf_interval(values, delta=0.05):
    z = st.norm.ppf(1-delta/2)
    point_est = np.mean(values)
    hw = z * np.sqrt(np.var(values, ddof=1) / len(values))
    c_low = point_est - hw
    c_high = point_est + hw
    return "%.4f with %.f%% Confidence Interval [%.4f" %(point_est, (1-delta)*100, c_low) +  ", %.4f" %c_high + "]"

def conf_int_hw(values, delta=0.05):
    z = st.norm.ppf(1-delta/2)
    point_est = np.mean(values)
    hw = z * np.sqrt(np.var(values, ddof=1) / len(values))
    #c_low = point_est - hw
    #c_high = point_est + hw
    return hw

def domain_dec_set(D):
    dec_set = []
    for dec in itertools.product(range(2), repeat=D):
        dec_set.append(dec)

    random.shuffle(dec_set)
    return dec_set

if __name__ == '__main__':
    main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5])


glob_dict = {}
def estimate(pc):
    print_num = 0
    if len(pc) != 0:
       if pc[0] == '-':
            print_num += 1
            pc = pc.lstrip('-')
    while True:
        pc = pc.lstrip('+')
        if len(pc) == 0:
            break
        print_num += 2
        pc = pc.lstrip('-')
    return print_num

def all_happy(pancakes):
    return sum((1 for char in pancakes if char == "+")) == len(pancakes)

def min_flips(pancakes):
    if all_happy(pancakes):
        return 0
    plus = False
    done = False
    if pancakes[0] == '+':
        plus = True
    new_pc = []
    for char in pancakes:
        if done:
            new_pc.append(char)
            continue
        if plus:
            if char == '+':
                new_pc.append('-')
            else:
                done = True
                new_pc.append('-')
        else:
            if char == '-':
                new_pc.append('+')
            else:
                done = True
                new_pc.append('+')
    return 1 + min_flips(''.join(new_pc))

def flip(pancakes , index):
    flip_me = pancakes[:index+1]
    remaining = pancakes[index+1:]
    flip_me = list(flip_me[::-1])
    for i in range(len(flip_me)):
        if flip_me[i] == '+':
          flip_me[i] = '-'
        else:
            flip_me[i] = '+'
    flip_me.extend(list(remaining))
    ret = ''.join(flip_me)
    return ret

def main():
    cases = int(raw_input())
    for i in range(1,cases+1):
        pancakes = raw_input()
        est = estimate(pancakes)
        brut = min_flips(pancakes)
        if brut != est:
            print (pancakes + "est : " + str(est) + " brut : " + str(brut))
        print("CASE #" + str(i) + ": " + str(est))

main()


from selenium.webdriver import ActionChains
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
import logging

import Utils.ScreenControlDuringExecution
from EFields import EFields, Multiselect
from Utils import Utils, ScreenControlDuringExecution
from CATLogger import SOLog
import time

# #####################################################################################################################
class MedicationsPrescriptionTTO59(EFields.FieldDriver):

    def __init__(self, driver, field_type, field_label,
                 page_value,
                 user_value,
                 beh,
                 run_speed,
                 print_log_setting,
                 page_row_count,
                 user_row_count,
                 field_id_from_label
                 ):
        EFields.FieldDriver.__init__(self, driver, field_type, field_label, run_speed, print_log_setting,
                                     )
        self.page_value = ''  # always start with no page data when instantiating
        self.user_value = user_value
        self.beh = beh
        self.field_id_from_label = super().get_field_id_from_field_section()
        self.page_row_count = 0
        self.user_row_count = 0

    def proxy_content_exists(self):
        try:
            proxy_div = self.get_proxy_div_element()
            proxy_div_inner = proxy_div.get_attribute('innerHTML')
            if proxy_div_inner == '':
                return False
            else:
                return True
        except:
            return False  # replaced None with False to simplify later tests for True

    # -------------------- page table-level methods ------------------------
    def meds_table_tbody_xpath(self):
        if self.proxy_content_exists():
            meds_table_tbody_xpath = self.get_proxy_div_xpath() + '''/table/tbody'''
            return meds_table_tbody_xpath
        else:
            return None

    def meds_table_rows_xpath(self):
        if self.proxy_content_exists():
            meds_table_rows_xpath = self.get_proxy_div_xpath() + '''/table/tbody/tr'''
            return meds_table_rows_xpath
        else:
            return None

    def get_page_meds_row_elements(self):
        if self.proxy_content_exists():
            page_meds_rows_elements = self.driver.find_elements_by_xpath(self.meds_table_rows_xpath())
            return page_meds_rows_elements
        else:
            return None

    def count_page_meds_rows(self):
        if self.proxy_content_exists():
            # page_meds_rows_elements = self.driver.find_elements_by_xpath(self.meds_table_rows_xpath())
            count_page_meds_rows = len(self.get_page_meds_row_elements())
            self.page_row_count = count_page_meds_rows
            # print(count_page_meds_rows)
            return count_page_meds_rows
        else:
            return 0

    def get_page_meds_rows_text(self):
        row_elements = self.get_page_meds_row_elements()
        rows_text = []
        for row in row_elements:
            page_row_text = row.text
            rows_text.append(page_row_text)
        return rows_text

    def get_each_page_meds_row_xpath(self):
        row_elements = self.get_page_meds_row_elements()
        rows_xpaths = []
        for i in range(1,len(row_elements)):
            page_row_xpath = self.meds_table_rows_xpath() + '[' + str(i) + ']'
            rows_xpaths.append(page_row_xpath)
        return rows_xpaths

    def proxy_content_is_navigable(self):
        if not self.proxy_content_exists() is None and self.proxy_content_exists():
            table_flag = 1
        else:
            table_flag = 0
        return table_flag

    # -------------------- page row-level methods ------------------------
    def this_meds_row_xpath(self, row_scrape_count):
        this_row_xpath = self.meds_table_rows_xpath() + '[' + str(row_scrape_count) + ']'
        return this_row_xpath

    def this_meds_row_cells_xpath(self, row_scrape_count):
        cells_this_row_xpath = self.this_meds_row_xpath(row_scrape_count) + '/td'
        return cells_this_row_xpath

    def this_meds_row_cells_elements(self, row_scrape_count):
        cells_elements = self.driver.find_elements_by_xpath(self.this_meds_row_cells_xpath(row_scrape_count))
        return cells_elements

    def this_meds_row_text(self, row_scrape_count):
        cells_elements = self.this_meds_row_cells_elements(row_scrape_count)
        cells_text = [cell.text for cell in cells_elements]
        return cells_text

    def this_meds_row_delimited_text(self, row_scrape_count):
        cells_elements = self.this_meds_row_cells_elements(row_scrape_count)
        row_delimited_text = (''.join('^' + e.text for e in cells_elements))[1:]
        return row_delimited_text

    def row_is_pre_pack(self, row_scrape_count):
        pre_pack_cell_xpath = self.this_meds_row_xpath(row_scrape_count) + '/td[7]'
        pre_pack_cell_element = self.driver.find_element_by_xpath(pre_pack_cell_xpath)
        pre_pack_cell_text = pre_pack_cell_element.text
        if pre_pack_cell_text == 'pre-pack':
            return True
        else:
            return False

    # --------------------- page table-level derived from row-level ------------------------
    def all_meds_rows_delimited_text(self):
        page_table_delimited_text = []
        for i in range(self.page_row_count):
            page_table_delimited_text.append(self.this_meds_row_delimited_text(i+1))
        return page_table_delimited_text

    # -------------------- user table-level methods ------------------------
    # ##### the user data read in by the CAXLoader is a single string separated by hats at cells and pipes at row ends.
    def user_meds_rows_split(self):
        user_row_values_list = self.user_value.split('|')
        return user_row_values_list

    def user_meds_rows_count(self):
        user_row_values_list = self.user_meds_rows_split()
        if not isinstance(user_row_values_list, list):
            user_row_values_list = [user_row_values_list]
        self.user_row_count = len(user_row_values_list)
        return len(user_row_values_list)

    # -------------------- user row-level methods ------------------------
    @staticmethod
    def user_meds_row_is_pre_pack(delimited_row):
        # find text between hats #5 and #6 (hat = ^)
        hat_5_pos = Utils.findnth(delimited_row, '^', 5)
        hat_6_pos = Utils.findnth(delimited_row, '^', 6)
        test_string = delimited_row[hat_5_pos+1:hat_6_pos]
        if test_string == 'pre-pack':
            return True
        else:
            return False

    @staticmethod
    def user_meds_row_includes_dispensed(delimited_row):
        # find text between hats #6 and #7 (hat = ^)
        hat_6_pos = Utils.findnth(delimited_row, '^', 6)
        hat_7_pos = Utils.findnth(delimited_row, '^', 7)
        test_string = delimited_row[hat_6_pos+1:hat_7_pos]
        if test_string == '':
            return False
        else:
            return True

    @staticmethod
    def strip_dispensed_col_from_user_meds_row(delimited_row):
        # find text between hats #6 and #7 (hat = ^)
        hat_6_pos = Utils.findnth(delimited_row, '^', 6)
        hat_7_pos = Utils.findnth(delimited_row, '^', 7)
        return delimited_row[:hat_6_pos+1]

class MedsUITable(object):

    def __init__(self, driver):
        self.driver = driver
        self.ui_table_xpath = self.ui_meds_table_xpath()
        self.ui_table_visible = self.ui_table_is_visible()
        self.ui_table_rendered_rows = self.ui_table_rendered_data_entry_rows_count()
        self.ui_table_dispensed_col_visible = self.ui_table_dispensed_col_is_visible()

    @staticmethod
    def ui_meds_table_xpath():
        ui_table_xpath = '''//table[@id="medsTable"]/tbody'''
        return ui_table_xpath

    def ui_table_is_visible(self):
        try:
            header_row_element = self.driver.find_element_by_xpath(self.ui_table_xpath + '''/tr[1]''')
            if header_row_element.is_displayed():
                return True
            else:
                return False
        except:
            return None

    def get_ui_meds_table_data_entry_row_elements(self):
        if self.ui_table_visible:
            row_elements = self.driver.find_elements_by_xpath(self.ui_table_xpath + '''/tr''')
            data_entry_row_elements = []
            for row_element in row_elements:
                if row_element.text[:22] == 'Medication/Formulation':
                    pass
                else:
                    data_entry_row_elements.append(row_element)
            return data_entry_row_elements
        else:
            return []

    def ui_table_rendered_data_entry_rows_count(self):
        if self.ui_table_visible:  # and self.get_ui_meds_table_data_entry_row_elements() is not None:
            return len(self.get_ui_meds_table_data_entry_row_elements())
        else:
            return 0

    def ui_table_dispensed_col_is_visible(self):
        ui_table_header_element = self.driver.find_element_by_xpath(self.ui_table_xpath + '''/tr[1]''')
        ui_table_header_text = ui_table_header_element.text
        if 'Dispensed' in ui_table_header_text:
            return True
        else:
            return False

    def save_and_close(self):
        done_button_xpath = '''//button[.="Save & Close Medications"]'''
        done_button_element = self.driver.find_element_by_xpath(done_button_xpath)
        ScreenControlDuringExecution.scroll_to(self.driver, 'xpath', done_button_xpath)
        done_button_element.click()

class MedsUIRows(MedsUITable):

    def __init__(self, driver):

        MedsUITable.__init__(self, driver)
        # self.driver = super().driver
        self.ui_table_xpath = super().ui_meds_table_xpath()
        # self.ui_table_visible = super().ui_table_is_visible()
        self.ui_table_rendered_rows = super().ui_table_rendered_data_entry_rows_count()
        self.ui_table_dispensed_col_is_visible = super().ui_table_dispensed_col_is_visible()

    @staticmethod
    def button_flag(row_diff):
        if row_diff <= 0:
            # no spare rows
            # def add a new row and last used row = page row count
            # button on last used row - enter on +1
            button_flag = 1

            # continue adding as you go
        elif row_diff == 1:
            # no need to button - enter on row #3
            button_flag = 0

            # next round,
        elif row_diff == 2:
            # no need to button - enter on row #2
            button_flag = 0
        elif row_diff == 3:
            # no need to button - enter on row #1
            button_flag = 0
        else:
            # error
            print('ERR')
            button_flag = 0
        return button_flag

    def ui_this_row_xpath(self, this_row_index):
        ui_this_row_xpath = self.ui_table_xpath + '''/tr[''' + str(this_row_index) + ''']'''
        return ui_this_row_xpath

    def ui_this_row_button_xpath(self, this_row_index):
        ui_this_row_button_xpath = self.ui_table_xpath + \
            '''/tr[''' + str(this_row_index) + ''']/td[9]/span[.="+"]'''
        return ui_this_row_button_xpath

    def add_new_row(self, button_row):
        row_button_xpath = self.ui_this_row_button_xpath(button_row)
        add_new_row_button_element = self.driver.find_element_by_xpath(row_button_xpath)
        add_new_row_button_element.click()

    def fill_row_from_line_value(self, data_entry_line_number, line_value):
        # print('filling line: [{x}]'.format(x=data_entry_line_number))
        cell_values_list = line_value.split('^')
        cell_number = 0  # initialise
        for cell_value_to_enter in cell_values_list:
            cell_number += 1
            if cell_number >= 8:
                break  # avoid 8th cell as it's pharmacy only and not shown
            cell_index = str(cell_number)
            self.ui_cell_enter_data(str(data_entry_line_number), str(cell_index), cell_value_to_enter)

    # ----------------- ui-only cell methods --------------------------------
    def ui_cell_enter_data(self, row_index, cell_index, cell_data_to_enter):
        if cell_data_to_enter == '':
            pass
        else:
            row_xpath = self.ui_table_xpath + '''/tr[''' + str(int(row_index)) + ''']'''
            # print(row_xpath)
            cell_xpath = row_xpath + '''/td[''' + str(cell_index) + ''']'''
            # print('cell xpath: {x}'.format(x=cell_xpath))
            cell_element = self.driver.find_element_by_xpath(cell_xpath)
            actions = ActionChains(self.driver).move_to_element(cell_element).click(cell_element).send_keys(
                cell_data_to_enter)
            actions.perform()

class CTCAE(Multiselect.MultiSelect):
    def __init__(self,
                 driver,
                 field_type,
                 field_label,
                 page_value,
                 user_value,
                 beh,
                 run_speed,
                 print_log_setting,
                 extra_box
                 ):
        Multiselect.MultiSelect.__init__(self, driver, field_type, field_label, '',
                                         user_value, beh, run_speed, print_log_setting,
                                         extra_box)
        self.page_value = ''  # always start with no page data when instantiating
        self.user_value = user_value
        self.beh = beh
        self.extra_box = extra_box
        self.element_to_click = None
        self.set_element_to_click()
        self.toxicities_to_enter = None
        self.set_ctcae_toxicities()
        self.row_head_xpath = None
        self.cell_xpath = None
        self.cell_to_click = None
        self.ttox_div = None

    def set_ctcae_toxicities(self):
        field_value_as_list = self.user_value.split("\n")
        toxicities = []
        for value in field_value_as_list:
            toxicity = value.strip().split(', grade ')

            stub = toxicity[0][:5]

            if stub == '':
                # ignore trailing lines
                toxicity.insert(0, False)
            else:
                # all other lines are good
                toxicity.insert(0, True)
            print(toxicity[1] + ' ' + toxicity[2])
            toxicities.append(toxicity)
        self.toxicities_to_enter = toxicities

    def set_element_to_click(self):
        self.element_to_click = self.driver.find_element_by_id(self.field_id_from_label+'btn')

    def set_ctcae_ui_div(self):
        self.ttox_div = self.driver.find_element_by_id('TToxdiv'+self.field_id_from_label)

    def set_ctcae_cell(self, tox):
        self.row_head_xpath = '''//td[@class="TToxrowhead"][contains(.,"''' + tox[1] + '''")]'''
        # print('row_head_xpath: {path}'.format(path=self.row_head_xpath))
        self.cell_xpath = self.row_head_xpath + '''/following-sibling::td[''' + tox[2] + ''']'''
        # print('cell_xpath: {path}'.format(path=self.cell_xpath))
        try:
            self.cell_to_click = self.driver.find_element_by_xpath(self.cell_xpath)
        except NoSuchElementException:
            id_chapter_chooser = 'SystemOrderClass' + self.field_id_from_label
            id_search_box = 'SearchBox' + self.field_id_from_label
            elem = self.driver.find_element_by_id(id_chapter_chooser)  # select all chapter search
            elem.send_keys('All')
            elem = self.driver.find_element_by_id(id_search_box)  # use the search box
            elem.clear()
            elem.send_keys(tox[1])
            self.cell_to_click = self.driver.find_element_by_xpath(self.cell_xpath)

    def ctcae_tox_is_selected(self):
        if self.cell_to_click.get_attribute('title')[:15] == 'Click to select':
            return False
        else:
            return True

    def click_done(self):
        done_xpath = '''//button[.="Save & Close"]'''
        # print('done_xpath: {path}'.format(path=done_xpath))
        done_button = self.driver.find_element_by_xpath(done_xpath)
        done_button.click()

# #####################################################################################################################
def s_treatment_toxicity_ctcae_4_22(driver, run_speed, field_id, field_order, field_info, field_name, field_value,
                                   # grid_select, grid_store, val_flag, number_columns, grid_show_header,
                                   print_log_setting=1, behaviour="""""", cat_logger=None):
    """This takes a field data specification for a standard table grid field, logs the action and enters the data"""
    # logging.basicConfig(filename='example.log', filemode='w',
    #                     level=logging.INFO)  # write over existing log in this file
    SOLog.pretty_log_field('start', field_order, field_id, field_name, field_info,
                           print_log_setting)
    SOLog.pretty_log_data_value(field_value, print_log_setting)

    # instantiate a multiselect object
    ms = CTCAE(driver, 22, field_name, '', field_value, behaviour, run_speed, print_log_setting, extra_box=None)
    if ms.is_field_displayed():
        ScreenControlDuringExecution.scroll_to(driver, field_name, ms.get_label_xpath())
        if ms.is_field_read_only():
            print('yet to cater for read-only CTCAE')
            pass
        else:
            if field_value:
                ms.element_to_click.click()
                time.sleep(2)
                ms.set_ctcae_ui_div()
                for tox in ms.toxicities_to_enter:
                    ms.set_ctcae_cell(tox)
                    if ms.ctcae_tox_is_selected():
                        # skip this one - already selected
                        pass
                    else:
                        ms.cell_to_click.click()

                ms.click_done()

            # except:
            #     raise NoSuchElementException("Could not time element or done button in time.")
        ScreenControlDuringExecution.set_speed(driver, run_speed, field_value)
        # SOLog.CATLogging.save_screenshot_for_field(driver, form_name, field_order, field_name, field_id, field_info)
        print('cat time: {x}'.format(x=cat_logger.cat_time))
        logging.info('{ord} - finished processing'.format(ord=field_order + "_" + field_id))


import fabio
import h5py
import os
import glob
from PyQt4 import QtCore, QtGui
import cPickle

def load_pickle(file_name, faster_but_using_more_memory=True):
  """
  Wraps cPickle.load.

  Parameters
  ----------
  file_name : str
  faster_but_using_more_memory : bool, optional
      Optionally read the entirety of a file into memory before converting it
      into a python object.

  Returns
  -------
  object
  """
  if (faster_but_using_more_memory):
    return cPickle.loads(open(file_name, "rb").read())
  return cPickle.load(open(file_name, "rb"))

class ConstructingTree(QtCore.QThread):

    def __init__(self, tree):
        QtCore.QThread.__init__(self, tree)
        self.tree = tree

class ImageFactory(object):

    def __init__(self, tree, parent):
        self.h5_fn = None
        self.h5 = None
        self.tree = tree
        self.parent = parent
        self.filename = None
        self.types = ( 'edf',
                       'h5',
                       'cbf',
                       'img',
                       'sfrm',
                       'dm3',
                       'xml',
                       'kccd',
                       'msk',
                       'spr',
                       'tif',
                       'mccd',
                       'mar3450',
                       'pickle',
                       )
        self.filename = None
        self.h5_dic = {}
        self.filenames_dic = {}

    def openframe(self, fn, path='/data', index=None):
        self.filename = fn
        ext = os.path.splitext(fn)[1]
        if 'h5' in ext:
            return self.openh5(fn,index,path)
        elif 'pickle' in ext:
            return self.openpickle(fn)
        else:
            return self.openimg(fn)

    def openh5(self,fn, index=None, path='/data'):
        if fn != self.h5_fn:
            self.h5_fn = fn
            if self.h5 is not None: self.h5.close()
            self.h5 = h5py.File(fn)
        for dataset in  self.h5_dic[fn]:
                if path == dataset[0]:
                    if index is None:
                      data = self.h5[path][:]
                      return None,data.T
                    else:
                        data = self.h5[path][index,::]
                        return None, data.T

    def openimg(self,fn):
        img = fabio.open(fn)
        return img.header, img.data.T

    def openpickle(self,fn):

        img = load_pickle(self.filename)
        # This code snippet helps to remove the border of each tile of the CSPAD (lots of overload)
        #
        #data = img['DATA'].as_numpy_array()
        #mask = np.where(data > 0, 0, 1)
        #s = ndimage.generate_binary_structure(2, 1)
        #self.dset = data*np.logical_not(ndimage.binary_dilation(mask, structure =s ))
        data = img['DATA'].as_numpy_array()
        return None, data.T

    def getImages(self):
        path = QtGui.QFileDialog.getExistingDirectory(
                       self.parent,
                       "Select a folder",
                       '/Users/nico/PycharmProjects/npg_qt')
        if path:
            self.tree.clear()
            os.chdir(path)
            self.run()

    def run(self):
            files = []
            for f in self.types:
                files.extend(glob.glob('*.%s' % f))
            # TODO: Do not perform visitor_func for every data file !!
            # TODO: extract metadata from master
            # TODO: Remove paths from tree name - in progress
            masters = [f for f in files if 'master.h5' in f]
            for master in sorted(masters):
                item = QtGui.QTreeWidgetItem(self.tree)
                #s =
                item.setText(0, master)
                files.remove(master)
                root = master.strip('master.h5')
                datas = [f for f in files if root in f]

                for data in sorted(datas):
                    self.filename = data
                    self.h5_dic[self.filename] = []
                    with h5py.File(data, 'r') as f:

                        f.visititems(self.visitor_func)
                    self.construct_tree(data, item)
                    files.remove(data)

            for f in files:
                if not f.endswith('.h5'):
                    item = QtGui.QTreeWidgetItem(self.tree)
                    item.setText(0, f)
                    item.setText(1, "1")
                else:
                    with h5py.File(f, 'r') as h5:
                        self.filename = f
                        self.h5_dic[self.filename] =[]
                        h5.visititems(self.visitor_func)
                    self.construct_tree(f, self.tree)

    def append_object(self,obj):
        if os.path.isfile(obj):
            self.append_file(obj)
        elif os.path.isdir(obj):
            self.tree.clear()
            os.chdir(obj)
            self.run()

    def append_file(self, fn):
        filename, file_extension = os.path.splitext(fn)
        if file_extension.strip('.') in self.types:
            self.add_tree(fn)

    def add_tree(self,f):
        if not f.endswith('.h5'):
                    item = QtGui.QTreeWidgetItem(self.tree)
                    item.setText(0, f)
                    item.setText(1, "1")
        else:
            with h5py.File(f, 'r') as h5:
                self.filename = f
                self.h5_dic[self.filename] =[]
                h5.visititems(self.visitor_func)
            self.construct_tree(f, self.tree)

    def construct_tree(self, f, parent):

            item = QtGui.QTreeWidgetItem(parent)
            item.setText(0,f)
            if len(self.h5_dic[f]) == 1:
            #for dataset in self.h5_dic[f]:
                path , shape = self.h5_dic[f][0]
                if shape[0] == 1:
                    item.setText(1,"1")
                else:
                    item.setText(1, str(shape[0]))
                    for i in range(shape[0]):
                        item1 = QtGui.QTreeWidgetItem(item)
                        item1.setText(0, '%s %8i'%(path,i))
            else:
                count = 0
                for dataset in self.h5_dic[f]:
                    path , shape = dataset
                    item1 = QtGui.QTreeWidgetItem(item)
                    item1.setText(0,path)
                    if shape[0] == 1:
                        item1.setText(1,"1")
                    else:
                        item1.setText(1, str(shape[0]))
                        for i in range(min(100,shape[0])):
                            item2 = QtGui.QTreeWidgetItem(item1)
                            item2.setText(0, '%s %8i'%(path,i))
                    count += shape[0]
                    item.setText(1, str(count))

    def visitor_func(self, name, node):

            if isinstance(node, h5py.Dataset):
                if len(node.shape) == 2 and node.size > 512*512:
                    t = (1, node.shape[0], node.shape[1])
                    #try: self.h5_dic[self.filename].append((node.name, t))
                    #except KeyError: self.h5_dic[self.filename] = [(node.name, t)]
                    self.h5_dic[self.filename].append((node.name, t))
                if len(node.shape) == 3 and node.shape[1] * node.shape[2] > 512*512:
                    #print self.filename, node.name, node.shape
                    #try: self.h5_dic[self.filename].append((node.name, node.shape))
                    #except KeyError: self.h5_dic[self.filename] = [(node.name, node.shape)]
                    #self.h5_dic[self.filename] = (node.name, node.shape)
                    self.h5_dic[self.filename].append((node.name, node.shape))

            else:
                pass


import datetime as dt

def parse_tibia_datetime(datetime_str):
    """Parses date and time from the format used in Tibia.com

    Parameters
    -------------
    datetime_str: str
        The date and time as represented in Tibia.com

    Returns
    -----------
    :class:`datetime.datetime`
        The represented datetime, in UTC.
    """
    datetime_str = datetime_str.replace(",", "").replace(" ", " ")
    # Extracting timezone
    tz = datetime_str[-4:].strip()

    # Convert time string to time object
    # Removing timezone cause CEST and CET are not supported
    t = dt.datetime.strptime(datetime_str[:-4].strip(), "%b %d %Y %H:%M:%S")

    # Getting the offset
    if tz == "CET":
        utc_offset = 1
    elif tz == "CEST":
        utc_offset = 2
    else:
        utc_offset = 1
    # Add/subtract hours to get the real time
    t = t - dt.timedelta(hours=utc_offset)
    return t.replace(tzinfo=dt.timezone.utc)

def parse_tibia_date(date_str):
    """Parses a date from the format used in Tibia.com

    Parameters
    -----------
    date_str: str
        The date as represented in Tibia.com

    Returns
    -----------
    :class:`datetime.date`
        The represended date."""
    t = dt.datetime.strptime(date_str.strip(), "%b %d %Y")
    return t.date()


from os import listdir,mkdir
from os.path import join, isfile

import cv2

#put your ur video in the inputs file
onlyfiles = [join(".\Inputs", f) for f in listdir(".\Inputs") if isfile(join(".\Inputs", f))]

# for i in range(1545,1560):
#     dir = ".\Outputs" + "\\" + str(i)
#     mkdir(dir)

for i in range(len(onlyfiles)):
    x = onlyfiles[i]
    vid = cv2.VideoCapture(x)
    if not vid.isOpened():
        raise IOError("Couldn't open webcam or video")
    video_FourCC    = int(vid.get(cv2.CAP_PROP_FOURCC))
    video_fps       = vid.get(cv2.CAP_PROP_FPS)
    video_size      = (int(vid.get(cv2.CAP_PROP_FRAME_WIDTH)),
                        int(vid.get(cv2.CAP_PROP_FRAME_HEIGHT)))
    isOutput = True
    dir = ".\Outputs" + "\out_" + str(i)
    mkdir(dir)
    j = 0
    while True:
        return_value, frame = vid.read()
        if return_value == False:
            break;

        cv2.imwrite(".\Outputs" + "\out_" + str(i)+"\\"+str(j)+".jpg", frame);

        j+=1
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break


from numpy import linalg as LA
import numpy as np

# Put any initialization logic here.  The context object will be passed to
# the other methods in your algorithm.
def initialize(context):
    context.stocks = [sid(2518), sid(5692), sid(16841), sid(19831), sid(5061),
                      sid(3735), sid(25317), sid(6984), sid(1900), sid(13905)]
    # context.stocks = []
    context.max_notional = 1000000
    context.previous_prices = None
    context.day = None

    schedule_function(execute, date_rules.every_day())

def execute(context, data):
    if context.previous_prices == None:
        context.previous_prices = np.array([data.current(stock, 'price') for stock in context.stocks])
        return

    if context.day == None:
        context.day = data.current(context.stocks[0], 'last_traded').day
        return

    if data.current(context.stocks[0], 'last_traded').day == get_datetime().day:
        return

    # calculate and shift change from previous days price
    current_prices = np.array([data.current(stock, 'price') for stock in context.stocks])

    pct_change = current_prices / context.previous_prices - 1

    gains = pct_change * (pct_change > 0)
    loss = pct_change * (pct_change <= 0)
    # normalize

    norm_gains = gains / LA.norm(gains)
    norm_loss = loss / LA.norm(loss)

    free_cash = context.portfolio.cash * 0.9

    for i in range(len(norm_gains)):
        stock = context.stocks[i]
        sell_amount = round(-1 * context.portfolio.positions[stock].amount * (norm_gains[i] ** 2))

        if norm_gains[i] > 0 and abs(sell_amount) > 0:
            if norm_gains[i] ** 2 > 0.33:
                order(stock, sell_amount)

    for i in range(len(norm_loss)):
        stock = context.stocks[i]

        buy_amount = round((free_cash / data.current(stock, 'price')) * (norm_loss[i] ** 2))
        if norm_loss[i] < 0 and abs(buy_amount) > 0:
            if norm_loss[i] ** 2 > 0.33:
                order(stock, buy_amount)

    context.previous_prices = current_prices
    context.day = data.current(context.stocks[0], 'last_traded').day


# -*- coding: utf-8 -*-
"""
Examples of plots and calculations using the tmm package.
"""

#make division of integers work as expected
from __future__ import division

from tmm import *
from numpy import pi, linspace, inf, array
from scipy.interpolate import interp1d
import matplotlib.pyplot as plt

# "5 * degree" is 5 degrees expressed in radians
# "1.2 / degree" is 1.2 radians expressed in degrees
degree = pi/180

def sample1():
    """
    Here's a thin non-absorbing layer, on top of a thick absorbing layer, with
    air on both sides. Plotting reflected intensity versus wavenumber, at two
    different incident angles.
    """
    # list of layer thicknesses in nm
    d_list = [inf,100,300,inf]
    # list of refractive indices
    n_list = [1,2.2,3.3+0.3j,1]
    # list of wavenumbers to plot in nm^-1
    ks=linspace(0.0001,.01,num=400)
    # initialize lists of y-values to plot
    Rnorm=[]
    R45=[]
    for k in ks:
		# For normal incidence, s and p polarizations are identical.
		# I arbitrarily decided to use 's'.
        Rnorm.append(coh_tmm('s',n_list, d_list, 0, 1/k)['R'])
        R45.append(unpolarized_RT(n_list, d_list, 45*degree, 1/k)['R'])
    kcm = ks * 1e7 #ks in cm^-1 rather than nm^-1
    plt.figure()
    plt.plot(kcm,Rnorm,'blue',kcm,R45,'purple')
    plt.xlabel('k (cm$^{-1}$)')
    plt.ylabel('Fraction reflected')
    plt.title('Reflection of unpolarized light at 0$^\circ$ incidence (blue), '
                '45$^\circ$ (purple)')

def sample2():
    """
    Here's the transmitted intensity versus wavelength through a single-layer
    film which has some complicated wavelength-dependent index of refraction.
    (I made these numbers up, but in real life they could be read out of a
    graph / table published in the literature.) Air is on both sides of the
    film, and the light is normally incident.
    """
    #index of refraction of my material: wavelength in nm versus index.
    material_nk_data = array([[200, 2.1+0.1j],
                              [300, 2.4+0.3j],
                              [400, 2.3+0.4j],
                              [500, 2.2+0.4j],
                              [750, 2.2+0.5j]])
    material_nk_fn = interp1d(material_nk_data[:,0].real,
                              material_nk_data[:,1], kind='quadratic')
    d_list = [inf,300,inf] #in nm
    lambda_list = linspace(200,750,400) #in nm
    T_list = []
    for lambda_vac in lambda_list:
        n_list = [1, material_nk_fn(lambda_vac), 1]
        T_list.append(coh_tmm('s',n_list,d_list,0,lambda_vac)['T'])
    plt.figure()
    plt.plot(lambda_list,T_list)
    plt.xlabel('Wavelength (nm)')
    plt.ylabel('Fraction of power transmitted')
    plt.title('Transmission at normal incidence')

def sample3():
    """
    Here is a calculation of the psi and Delta parameters measured in
    ellipsometry. This reproduces Fig. 1.14 in Handbook of Ellipsometry by
    Tompkins, 2005.
    """
    n_list=[1,1.46,3.87+0.02j]
    ds=linspace(0,1000,num=100) #in nm
    psis=[]
    Deltas=[]
    for d in ds:
        e_data=ellips(n_list, [inf,d,inf], 70*degree, 633) #in nm
        psis.append(e_data['psi']/degree) # angle in degrees
        Deltas.append(e_data['Delta']/degree) # angle in degrees
    plt.figure()
    plt.plot(ds,psis,ds,Deltas)
    plt.xlabel('SiO2 thickness (nm)')
    plt.ylabel('Ellipsometric angles (degrees)')
    plt.title('Ellipsometric parameters for air/SiO2/Si, varying '
            'SiO2 thickness.\n'
            '@ 70$^\circ$, 633nm. '
            'Should agree with Handbook of Ellipsometry Fig. 1.14')

def sample4():
    d_list = [inf, 100, 300, inf] #in nm
    n_list = [1, 2.2+0.2j, 3.3+0.3j, 1]
    th_0=pi/4
    lam_vac=400
    pol='p'
    coh_tmm_data = coh_tmm(pol,n_list,d_list,th_0,lam_vac)

    ds = linspace(0,400,num=1000) #position in structure
    poyn=[]
    absor=[]
    for d in ds:
        layer, d_in_layer = find_in_structure_with_inf(d_list,d)
        data=position_resolved(layer,d_in_layer,coh_tmm_data)
        poyn.append(data['poyn'])
        absor.append(data['absor'])
    # convert data to numpy arrays for easy scaling in the plot
    poyn = array(poyn)
    absor = array(absor)
    plt.figure()
    plt.plot(ds,poyn,'blue',ds,200*absor,'purple')
    plt.xlabel('depth (nm)')
    plt.ylabel('AU')
    plt.title('Local absorption (purple), Poynting vector (blue)')


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        ('extuser', '0011_auto_20160519_1659'),
    ]

    operations = [
        migrations.AddField(
            model_name='extuser',
            name='refsave',
            field=models.ForeignKey(blank=True, to=settings.AUTH_USER_MODEL, null=True, related_name='referals_save', verbose_name='Вышестоящий реф. отвязанный'),
        ),
    ]


"""
:mod:`dic_utils` contains several utility functions used when analyzing DIC data, e.g. determining the step size,
going from pixel to millimeter coordinates, and determining deformations.

"""
import numpy as np
import warnings

__all__ = ["get_step", "point_to_indices", "get_initial_position", "get_displacement", "point_to_position"]

def get_step(dic_data):
    """
    Returns the step size of the DIC data

    Parameters
    ----------
    dic_data : dict
        Dictionary containing the DIC data.

    Returns
    -------
    int
        Step size.
    """
    return dic_data["x"][0, 1] - dic_data["x"][0, 0]

def point_to_indices(dic_data, pt):
    """
    Transforms ``(x, y)`` in pixel coordinates into the corresponding ``(row, col)`` to access the closest data point
    in the specified DIC data.

    Parameters
    ----------
    dic_data : dict
        Dictionary containing the DIC data.
    pt : (x, y)
        Two-dimensional coordinates of the pixel in global space.

    Returns
    -------
    (row, col) : (int, int)
        The row and column in ``dic_data`` that corresponds to the given pixel point.
    """
    step = get_step(dic_data)
    keys = ("y", "x")
    indices = [None, None]

    for i, key in enumerate(keys):
        min_key = '{}_min'.format(key)
        if min_key in dic_data:
            px_min = dic_data[min_key]
        else:
            px_min = dic_data[key].min()
        px = pt[(i + 1) % 2]
        indices[i] = int(round((px - px_min) / step))
    return indices

def get_initial_position(dic_data, row, col):
    """
    Retrieves the initial position (in mm if available, otherwise in pixels) held at the specified row and column.

    Parameters
    ----------
    dic_data : dict
        Dictionary containing the DIC data.
    row : int
        Row in the DIC data to access.
    col : int
        Column in the DIC data to access.

    Returns
    -------
    ``numpy.ndarray``
        Initial position ``(x, y, z)``.
    """
    try:
        return np.array([dic_data["X"][row, col], dic_data["Y"][row, col], dic_data["Z"][row, col]])
    except KeyError:
        warnings.warn("Position data in millimeters not provided. Falling back to position in pixels.")
        return np.array([dic_data["x"][row, col], dic_data["y"][row, col], dic_data["z"][row, col]])

def get_displacement(dic_data, row, col):
    """
    Retrieves the displacement (in mm if available, otherwise in pixels) held at the specified row and column.

    Parameters
    ----------
    dic_data : dict
        Dictionary containing the DIC data.
    row : int
        Row in the DIC data to access.
    col : int
        Column in the DIC data to access.

    Returns
    -------
    ``numpy.ndarray``
        Displacements ``(u, v, w)``.
    """
    try:
        return np.array([dic_data["U"][row, col], dic_data["V"][row, col], dic_data["W"][row, col]])
    except KeyError:
        warnings.warn("Displacement data in millimeters not provided. Falling back to displacement in pixels.")
        return np.array([dic_data["u"][row, col], dic_data["v"][row, col], dic_data["w"][row, col]])

def point_to_position(dic_data, pt, add_displacement=True):
    """
    Transforms a point in pixel space into its displaced coordinates (in mm if available, otherwise in pixels).

    Parameters
    ----------
    dic_data : dict
        Dictionary containing the DIC data.
    pt : (x, y)
        Two-dimensional coordinates of the pixel in global space.
    add_displacement : bool, optional
        Whether to add deformation to the undeformed position. Default is ``True``.

    Returns
    -------
    ``numpy.ndarray``
        ``(x, y, z)`` position of the point.
    """
    row, col = point_to_indices(dic_data, pt)
    pos = get_initial_position(dic_data, row, col)
    if add_displacement:
        pos += get_displacement(dic_data, row, col)
    return pos


from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import sent_tokenize, word_tokenize, RegexpTokenizer
import gensim.models.word2vec as w2v
from keras.preprocessing.text import Tokenizer, text_to_word_sequence
from keras.preprocessing import sequence
import multiprocessing
import pandas as pd
import numpy as np
import os

def tokenize_document(document):
    stop_words = set(stopwords.words('english'))
    lemmatizer = WordNetLemmatizer()
    tokenizer_ = RegexpTokenizer('[a-zA-Z]+')

    words = []
    for sentence in sent_tokenize(document):
        tokens = [lemmatizer.lemmatize(t.lower()) for t in tokenizer_.tokenize(sentence)\
                  if t.lower() not in stop_words]
        words += tokens

    words_ = str()
    for word in words:
        words_ = words_ + " " + word
    return words_

def tokenize_all_documents(df):
    df.text = df.text.map(tokenize_document)
    return df

def fit_encoder(list_of_documents_in_str,top_words = 20000):
    encoder = Tokenizer(top_words)
    encoder.fit_on_texts(list_of_documents_in_str)
    return encoder

def encode_a_tokenized_document(tokenized_document,encoder_):
        return encoder_.texts_to_sequences([tokenized_document])[0]

def encode_all_documents(df,encoder):
    df.text = df.text.apply(encode_a_tokenized_document, args = (encoder,))
    return df

def prepare_for_word_2_vec(sentences):
    sent_for_w2v = []
    for sentence in sentences:
        sentence = sentence.split(" ")[1:]
        sent_for_w2v.append(sentence)
    return sent_for_w2v

def load_dataset():
    df = pd.DataFrame()
    summ = 0
    classes = os.listdir("data/")

    print("Loading:")
    for class_ in classes:
        current_class_directory = "data/{}/".format(class_)
        print(current_class_directory)

        for name in sorted(os.listdir(current_class_directory)):
            path = os.path.join(current_class_directory, name)

            current_text = open(path)
            summ += 1
            df.loc[summ,"text"] = current_text.read()
            df.loc[summ,"class"] = class_

    df["class_meaning"] = df["class"]
    df["class"].replace({"business":0,\
                             "entertainment":1,\
                             "politics":2,\
                             "sport":3,\
                             "tech":4},inplace=True)

    return df

def predict_category(text,model,encoder,max_sequence_length):
    text = tokenize_document(text)
    text = encode_a_tokenized_document(text,encoder)
    text = sequence.pad_sequences([text], maxlen=max_sequence_length)

    categories= {0:"business",
                 1:"entertainment",
                 2:"politics",
                 3:"sport",
                 4:"tech"}

    confidence = (model.predict_proba(text,verbose=0)).max()*100
    category = categories[model.predict_classes(text,verbose=0)[0]]
    print("category: {}".format(category))
    print("confidence: {0:.2f}%".format(confidence))


# -*- encoding: utf-8 -*-

# Project :zs
# Time  :2019/7/25 上午8:58
# BY    :FormatFa

report = [

    # 图表元素
    {
        # 基本信息,类型chart图表，text ，文字,table表格
        'type':'chart',
        # 位置
        'index':0,
        'widht':400,
        'height':400,
        # 数据结构
        'recordid':1,
        'groupfield':'sex_name,source_provinces',
        # 分组类型，默认，
        'aggfield':'count_student_name',
        #排序字段 , count_student_name-asc,
        'orderBy':'null',
        #限制返回条数
        'limit':'-1',
        # 过滤数据,字段名+条件+值组成 ，条件有 like,>,<,==,>=,<=
        'filter':'sex_name-like-男',

        # 图表配置,原本有options就加载原本的option
        # 自定义图表类型
        'chartType':'',
        'option':'',
        # 是否可以更新数据,自定义图表和内置的图表,custom,internal,自定义和内置的，
        'dataType':'custom',
        # 文字配置
        'text':'',
        # 表格配置

    },
    {
'type':'chart',
        text:ff
    }

]


from uuid import uuid4

import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from pytest_mock import MockFixture
from requests import Session

from models import Movie

signed_url = "signed_url"

create_private_link_success = (
    True,
    {
        "uuid": str(uuid4()),
        "title": "Title",
        "filename": "filename",
        "premium": True,
    },
    signed_url,
    200,
    {"url": signed_url},
)

create_private_link_trial = (
    False,
    {
        "uuid": str(uuid4()),
        "title": "Title",
        "filename": "filename",
        "premium": True,
    },
    signed_url,
    403,
    {"detail": "Missing subscription"},
)

create_private_link_missing_file = (
    False,
    {"uuid": str(uuid4()), "title": "Title", "filename": None, "premium": True},
    signed_url,
    404,
    {"detail": "No existing video file for movie"},
)

@pytest.mark.parametrize(
    "premium_user,movie_data,signed_url,expected_status,expected_resp",
    [
        create_private_link_success,
        create_private_link_trial,
        create_private_link_missing_file,
    ],
)
def test_create_private_link(
    premium_user: bool,
    movie_data: dict,
    signed_url: str,
    expected_status: int,
    expected_resp: dict,
    mocker: MockFixture,
    app: FastAPI,
):
    client: Session = TestClient(app)
    headers = {"Authorization": "Bearer token"}

    movie = Movie(**movie_data)

    mocker.patch("api.v1.is_premium_user", return_value=premium_user)
    mocker.patch("api.v1.get_movie_by_id", return_value=movie)
    mocker.patch("api.v1.get_signed_url", return_value=signed_url)

    movie_uid = str(uuid4())
    response = client.post(
        app.url_path_for("create_private_link"), json={"id": movie_uid}, headers=headers
    )

    assert response.status_code == expected_status
    assert response.json() == expected_resp


# Add import from parent directory possible
import sys
import pandas as pd
import numpy
import matplotlib.pyplot as plt
from lib.DataOperations import *
from numpy.core.defchararray import lower
from lib.ReportSignals import *
from lib.indicator import indicator

# Creates  object

def CreateBollinger(prices, n=20, k=2):
    return Bollinger(prices, n, k)

# Bollinger object which creates Bollinger data

class Bollinger:

    def __init__(self, prices, n=20, k=2):
        self.n = n
        self.k = k
        self.consolidationLvl = 15  # percent
        self.variabilityLvl = 50  # percent
        self.mavg, self.upperBand, self.lowerBand = self.InitBollinger(
            prices, self.n, self.k)
        self.std = self.upperBand - self.lowerBand
        self.absStd = (self.std * 100) / self.std.max()

        # Signals
        fromBottom, fromTop = FindIntersections(self.upperBand, prices)
        self.sell = fromBottom
        fromBottom, fromTop = FindIntersections(self.lowerBand, prices)
        self.buy = fromTop
        self.consolidation = CreateSubsetByValues(
            self.absStd, 0, self.consolidationLvl)
        self.variability = CreateSubsetByValues(
            self.absStd, self.variabilityLvl, 100)

    # Set Bollinger indicator
    def InitBollinger(self, prices, n=20, k=2):
        mavg = prices.rolling(window=n, min_periods=1).mean()
        std = prices.rolling(window=n, min_periods=1).std()
        upperBand = mavg + (std * 2)
        lowerBand = mavg - (std * 2)
        return mavg, upperBand, lowerBand

    # Export indicator signals to report
    def ExportSignals(self, reportSignals):
        reportSignals.AddDataframeSignals(self.buy, 'Bollinger', 'buy')
        reportSignals.AddDataframeSignals(self.sell, 'Bollinger', 'sell')

    # Plot method
    def Plot(self):
        # Get index values for the X axis for facebook DataFrame
        x_axis = self.upperBand.index.get_level_values(0)

        # Plot shaded 21 Day Bollinger Band for Facebook
        plt.fill_between(x_axis, self.upperBand,
                         self.lowerBand, color='#BBBBBB')
        plt.plot(self.upperBand.index, self.upperBand, '--',
                 linewidth=1.0, color='#940006', label='Sell band')
        plt.plot(self.lowerBand.index, self.lowerBand, '--',
                 linewidth=1.0, color='#169400', label='Buy band')
        plt.plot(self.mavg.index, self.mavg, '--', linewidth=1.0,
                 color='#0000FF', label=('MA %s days' % self.n))

        # Signals plottting
        if (self.buy is not None and self.buy.size):
            plt.plot(self.buy.index, self.buy, 'o', color='#000000', ms=8)
            plt.plot(self.buy.index, self.buy, 'o',
                     label='Horiz. Buy', color='#00FF00')
        if (self.sell is not None and self.sell.size):
            plt.plot(self.sell.index, self.sell, 'o', color='#000000', ms=8)
            plt.plot(self.sell.index, self.sell, 'o',
                     label='Horiz. Sell', color='#FF0000')

    # Plot method
    def PlotAbsDeviation(self):
        plt.plot(self.absStd.index, self.absStd, linewidth=1.0,
                 color='#333333', label='Bol.AbsDeviation')
        plt.ylim(top=100, bottom=0)
        if (self.consolidation is not None and self.consolidation.size):
            plt.plot(self.consolidation.index, self.consolidation,
                     'o', label='Consolidation', color='cyan')
        if (self.variability is not None and self.variability.size):
            plt.plot(self.variability.index, self.variability,
                     'o', label='Variability', color='magenta')


# -*- coding:utf-8 -*-

''''''

'''

Linu环境下创建新进程：

    os模块中fork()函数:

        *   使用os模块中fork()函数可以创建新进程

        *   fork()调用一次返回两次，因为操作系统自动把当前进程（称为父进程）复制了一份（称为子进程），
        然后，分别在父进程和子进程内返回

        *   子进程永远返回0，父进程返回子进程的ID

        *   python程序一开始运行的时候，系统会生成一个进程(父进程)，在程序代码中调用fork()函数创建一个新进程
        ，新进程会是原来进程的子进程，原进程称为父进程，参考：http://www.01happy.com/python-fork-create-process/

        *   使用fork()创建子进程后，系统会先给子进程分配资源（如：存储数据和代码的空间），
        然后把父进程的所有资源都复制到子进程中，只有少数值与父进程的值不同（相当于克隆了一个自己，
        然后作为子进程）。

        *   创建进程成功后，系统会出现两个几乎完全相同的两个进程，这两个进程执行没有固定的先后顺序，
        那个进程先执行要看系统的调度算法

    getpid() getppid():

        * 每个进程都有一个标识符（pid：progress ID）

        *   getpid()获取子进程pid

        *   子进程使用getppid()获取父进程pid

进程有独立的内存空间，不能共享全局变量：

    *   如下面代码中的 num变量

    *   子进程中输出的结果是11，父进程中输出的结果是依然是10

    *   子进程中对num做了修改，并不影响父进程中的num

'''

import os
import time

#创建子进程之前声明的变量
num = 10

fid = os.fork()

if fid < 0:
    info = "进程创建失败... fid = %f" % fid
    print(info)

elif fid == 0:
    time.sleep(3) # 休眠3秒
    num = num + 1
    info = "子进程... fid = %d， pid = %d, ppid = %d " % (fid,os.getpid(),os.getppid(), num)
    print(info)

else:
    time.sleep(5) # 休眠5秒
    info = "父进程... fid = %d， pid = %d, ppid = %d, num = %d" % (fid, os.getpid(), os.getppid(),num)
    print(info)