import time
import json
from matplotlib import pyplot as plt
import math
from MetaMeta.class1.DataHandler import DataHandler
from MetaMeta.class1.FileGenerator import FileGenerator
from MetaMeta.class1.TSPAlgorithms import TSPAlgorithms
import numpy as np

def complexity_data_maker():
    generator = FileGenerator()
    generator.rm_dataset_directory()

    start = 10
    end = 120
    step = 2
    for i in range(start, end, step):
        generator.create_symmetric_dataset("GANGI", i)
    data = {"two_opt": [], "k_random": [], "repetitive_closest_neighbour": [], "two_opt_and_repetitive_neighbour": []}
    timestamp = time.time()
    for i in range(start, end, step):
        if i in [x * end // 10 for x in range(10)]:
            print(f"Done i={i}. It took {time.time() - timestamp} from previous")
            timestamp = time.time()
        _data = DataHandler(f"datasets/GANGI{i}/GANGI{i}.tsp")
        algos = TSPAlgorithms(_data.get_edge_weights(), _data.dimension())
        map_algos = {"two_opt": algos.two_opt_invert, "k_random": algos.k_random,
                     "repetitive_closest_neighbour": algos.repetitive_closest_neighbour,
                     "two_opt_and_repetitive_neighbour": algos.two_opt_with_repetitive_closest_neighbour}
        for key in list(map_algos.keys()):
            mean = 0
            amount = 0
            for _ in range(20):
                time_start = time.time()
                map_algos[key]()
                time_duration = time.time() - time_start
                mean += time_duration
                amount += 1
            data[key].append((i, mean / amount))
    with open(f"data/{filename}", 'w+') as file:
        file.write(json.dumps(data))

def two_opt_with_repetitive_tester():
    generator = FileGenerator()
    generator.rm_dataset_directory()

    start = 10
    end = 120
    step = 1
    for i in range(start, end, step):
        generator.create_symmetric_dataset("GANGI", i)
    data = {"two_opt_and_repetitive_neighbour": []}
    timestamp = time.time()
    for i in range(start, end, step):
        if i in [x * end // 10 for x in range(10)]:
            print(f"Done i={i}. It took {time.time() - timestamp} from previous")
            timestamp = time.time()
        generator.rm_dataset_directory()
        generator.create_symmetric_dataset("GANGI", i)
        _data = DataHandler(f"datasets/GANGI{i}/GANGI{i}.tsp")
        algos = TSPAlgorithms(_data.get_edge_weights(), _data.dimension())
        map_algos = {"two_opt_and_repetitive_neighbour": algos.two_opt_with_repetitive_closest_neighbour}
        for key in list(map_algos.keys()):
            mean = 0
            amount = 0
            for _ in range(30):
                time_start = time.time()
                map_algos[key]()
                time_duration = time.time() - time_start
                mean += time_duration
                amount += 1
                generator.rm_dataset_directory()
                generator.create_symmetric_dataset("GANGI", i)
                _data = DataHandler(f"datasets/GANGI{i}/GANGI{i}.tsp")
                algos = TSPAlgorithms(_data.get_edge_weights(), _data.dimension())
                map_algos = {"two_opt_and_repetitive_neighbour": algos.two_opt_with_repetitive_closest_neighbour}
            data[key].append((i, mean / amount))
    with open(f"data/{filename}", 'w+') as file:
        file.write(json.dumps(data))

def two_opt_with_repetitive_visualize():
    with open(f"data/{filename}", 'r') as file:
        data = json.loads(file.read())
    Xs = [x[0] for x in data['two_opt_and_repetitive_neighbour']]
    Ys = [x[1] for x in data['two_opt_and_repetitive_neighbour']]
    plt.plot(Xs, Ys)
    plt.suptitle("Two opt seeded by Repetitive Closest neighbour")
    plt.tight_layout()
    plt.show()

def complexity_visualize():
    with open(f"../data/complexity_data_maker_end=120_rep=20_json", "r") as file:
        data = json.loads(file.read())
    with open(f"../data/repetitive_complexity_data_maker_end=120_rep=20_json", 'r') as file:
        data2 = json.loads(file.read())
    data['two_opt_and_repetitive_neighbour'] = data2['two_opt_and_repetitive_neighbour']
    fig, axs = plt.subplots(2, 2, figsize=(10, 10))
    keys = list(data.keys())
    axes_for_keys = dict(zip(keys, [axs[0][0], axs[0][1], axs[1][0], axs[1][1]]))
    titles = {"two_opt": "Two Opt Invert",
              "k_random": "k_random for k=1000",
              "repetitive_closest_neighbour": "Repetitive Closest Neighbour",
              "two_opt_and_repetitive_neighbour": "Two opt seeded by Repetitive Closest neighbour"}
    for key in keys:
        Xs = [x[0] for x in data[key]]
        Ys = [x[1] for x in data[key]]
        axes_for_keys[key].plot(Xs, Ys, label=key, color="blue",linewidth='5')
        axes_for_keys[key].set_xlabel("N - dimension of the file")
        axes_for_keys[key].set_ylabel("Time[s]")
        axes_for_keys[key].title.set_text(titles[key])
        if key == "two_opt":
            function = np.zeros(len(Xs))
            function2 = np.zeros(len(Xs))
            function3 = np.zeros(len(Xs))
            for i in range(len(Xs)):
                function[i] = Xs[i]**3 / (math.log(Xs[i]) * 15000)
                function3[i] = Xs[i]**3 / 50000
            axes_for_keys[key].plot(Xs, function, label="y=x^3 / (log(x)*1.5*10^4)", color="red")
            axes_for_keys[key].plot(Xs, function3, label="y=x^3 / (5*10^4)", color="green")
        elif key == "k_random":
            function = np.zeros(len(Xs))
            for i in range(len(Xs)):
                function[i] = Xs[i] / 5600
            axes_for_keys[key].plot(Xs, function, label="y=x/(5.6*10^3)", color="red")
        elif key == "repetitive_closest_neighbour":
            function = np.zeros(len(Xs))
            for i in range(len(Xs)):
                function[i] = Xs[i] **3 / 25000000
            axes_for_keys[key].plot(Xs, function, label="y=x^3/(2.5*10^7)", color="red")
        elif key == "two_opt_and_repetitive_neighbour":
            function = np.zeros(len(Xs))
            for i in range(len(Xs)):
                function[i] = Xs[i] **3 / 500000
            axes_for_keys[key].plot(Xs, function, label="y=x^3 / (5*10^5)", color="red")
        axes_for_keys[key].legend()
    plt.tight_layout()
    plt.show()

def measure_k_random_effectiveness(n=50):
    data = []
    generator = FileGenerator()
    generator.rm_dataset_directory()

if __name__ == "__main__":
    filename = "repetitive_complexity_data_maker_end=120_rep=20_json"
    #two_opt_with_repetitive_tester()
    #two_opt_with_repetitive_visualize()
    # complexity_data_maker()
    complexity_visualize()


import torch
import torch.nn as nn

class dnn_classifier(nn.Module):

    def __init__(self, input_dim, num_cls, hid_dims):
        super(dnn_classifier, self).__init__()
        dims = [input_dim] + hid_dims + [num_cls]
        layers = nn.ModuleList()
        for n_layer, (in_dim, out_dim) in enumerate(
            zip(dims, dims[1:])):
            layers.append(nn.Linear(in_dim, out_dim))
            if out_dim != num_cls:
                layers.append(nn.BatchNorm1d(out_dim))
                layers.append(nn.ReLU())
        self.model = nn.Sequential(*layers)

    def forward(self, x):
        return self.model(x)


from collections import defaultdict

def apply_mask(value, mask):
    result = list(value)
    for i in range(len(value)):
        if mask[i] == '0':
            result[i] = '0'
        elif mask[i] == '1':
            result[i] = '1'

    return "".join(result)

if __name__ == "__main__":
    with open("input.txt", 'r') as input_file:
        instructions = [instruction.split(" = ") for instruction in input_file.read().split("\n")]

    MEMORY = defaultdict(lambda: 0)
    MASK = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"

    for instruction in instructions:
        if instruction[0] == 'mask':
            MASK = instruction[1]
        else:
            memory_address = instruction[0].strip("mem[]")

            initial_value = f"{int(instruction[1]):036b}"
            masked_value = apply_mask(initial_value, MASK)

            MEMORY[memory_address] = int(masked_value, 2)

    print(MEMORY)

    print(f"Sum of all values in memory: {sum(MEMORY.values())}")


# put table element from https://wiki.teamfortress.com/wiki/Casual_Mode as html

from bs4 import BeautifulSoup

soup = BeautifulSoup(html, 'html.parser')  # If this line causes an error, run 'pip install html5lib' or install html5lib

stuff = {}  # a list to store the extracted objects

table = soup.find('table', attrs={'style': 'border: 1px solid black; border-collapse: collapse; width: 480px',
                                  'class': 'wikitable mw-collapsible'})

for row in table.find_all('tr', style=True):  # Iterate over each <tr> tag with the 'style' attribute
    td_elements = row.find_all('td')  # Find all <td> tags within the current row

    # Extract the URL of the image from the first <td> tag
    image_url = td_elements[0].find('img')['src']

    # Extract the XP owned from the second <td> tag
    level = td_elements[1].get_text(strip=True)\

    xp = td_elements[2].get_text(strip=True)

    # Create an object and append it to the main list
    obj = {'badge':"https://wiki.teamfortress.com/" + image_url, 'xpRequired': xp}
    stuff[level] = obj

# Print the extracted objects
with open('scraped.txt', 'w') as f:
    f.write(str(stuff))


from tkinter import *
from PIL import ImageTk, Image #importing modules for images

root = Tk()
root.title('Icons and Images')
root.iconbitmap('icon python.ico')

my_img = ImageTk.PhotoImage(Image.open("tkinter-image.jpg"))
lblMyImage = Label(image=my_img)
lblMyImage.pack()

#root.quit just quit the program
btnExit = Button(root, text='Exit program', command=root.quit, padx=15, pady=15)
btnExit.pack()

root = mainloop()

#005 - Icons and images/iconsandimages.py


import Bio
from Bio import SeqIO
from Bio.SeqUtils.CodonUsage import CodonAdaptationIndex
import os
import numpy as np

def clean_CDS(input_file, output_file=False):
    """
    Accepts fasta file containing coding sequences.\n
    Returns or writes file containing only coding sequences
    that are divisible by 3, do not contain ambiguous nucleotides, and begin with ATG.
    """
    records = []
    n_dropped = 0
    for record in SeqIO.parse(input_file, "fasta"):
        if len(record.seq)%3 != 0:
            n_dropped += 1
            pass
        elif "N" in str(record.seq):
            n_dropped += 1
            pass
        elif str(record.seq[0:3]) != "ATG":
            n_dropped += 1
            pass
        else:
            records.append(record)
    print(f"{n_dropped} records were removed. {len(records)} were maintained.")
    if output_file:
        SeqIO.write(records, output_file, "fasta")
        print(f"cleaned CDS file saved as {output_file}")
    else:
        return records

def get_pref_codons(input_file, output_file=False):
    """
    Accepts coding sequences file (should be cleaned first to remove unusual sequences). \n
    Returns or saves file with codon adaptation index (CAI) scores.
    """
    index = CodonAdaptationIndex()
    index.generate_index(input_file)

    # Values of 1 is preferred codon. Otherwise, relative to preffered.
    preferred_codons_dict = index.index

    if output_file:
        import csv
        with open(output_file, 'w') as f:
            for key in preferred_codons_dict.keys():
                f.write("%s,%s\n"%(key,preferred_codons_dict[key]))
        print(f"preferred codons file saved as {output_file}")
    else:
        return preferred_codons_dict

def codon_optimize(input_file, preferred_codons_dict, output_file=False, gene_id=None, gene_description=None):
    index = CodonAdaptationIndex()
    index.set_cai_index(preferred_codons_dict)

    select_rec = SeqIO.read(f"{input_file}", "fasta")
    seq = str(select_rec.seq)
    print(f"Initial CAI: {index.cai_for_gene(seq)}")

    locs = np.where(np.array(list(preferred_codons_dict.values())) == 1.0)[0]
    pref = {}
    for i in locs:
        key = list(preferred_codons_dict.keys())[int(i)]
        val = Bio.Seq.translate(key)
        pref[val] = key

    optimized= ''
    for i in range(0,len(seq),3):
        try:
            codon = seq[i:i+3]
            aa = Bio.Seq.translate(codon)
            pref_codon = pref[aa]
            optimized += pref_codon
        except:
            print(f"Ignoring incomplete codon beginning at nucleotide {i}")

    print(f"Codon Optimized CAI: {index.cai_for_gene(optimized)}")
    opt_record = Bio.SeqRecord.SeqRecord(Bio.Seq.Seq(optimized))
    if output_file:
        opt_record.id = str(gene_id)
        opt_record.description = str(gene_description)
        if output_file == True:
            SeqIO.write(f"codon-optimized_{output_file}", "fasta")
        else:
            SeqIO.write(output_file, "fasta")
    else:
        return opt_record

if __name__ == "__main__":
    import sys
    clean_CDS(str(sys.argv[1]))


import boto3
from aws_xray_sdk.core import xray_recorder
from aws_xray_sdk.core import patch_all
from botocore import config
import logging
import os
import json
from MediaInsightsEngineLambdaHelper import Status as awsmie
from MediaInsightsEngineLambdaHelper import MediaInsightsOperationHelper
from MediaInsightsEngineLambdaHelper import MasExecutionError

patch_all()

# Setup logging
# Logging Configuration
# extra = {'requestid': app.current_request.context}
# fmt = '[%(levelname)s] [%(funcName)s] - %(message)s'
# logger = logging.getLogger('LUCIFER')
# logging.basicConfig(format=fmt)
# root = logging.getLogger()
# if root.handlers:
#     for handler in root.handlers:
#         root.removeHandler(handler)
# logging.basicConfig(format=fmt)
logger = logging.getLogger('boto3')
logger.setLevel(logging.INFO)

WORKFLOW_TABLE_NAME = os.environ["WORKFLOW_TABLE_NAME"]
STAGE_TABLE_NAME = os.environ["STAGE_TABLE_NAME"]
OPERATION_TABLE_NAME = os.environ["OPERATION_TABLE_NAME"]
WORKFLOW_EXECUTION_TABLE_NAME = os.environ["WORKFLOW_EXECUTION_TABLE_NAME"]
STAGE_EXECUTION_QUEUE_URL = os.environ["STAGE_EXECUTION_QUEUE_URL"]

ATT_NAME_WORKFLOW_NAME = '#workflow_name'
ATT_NAME_WORKFLOW_STATUS = '#workflow_status'
ATT_VALUE_WORKFLOW_STATUS = ':workflow_status'
ATT_VALUE_CURRENT_STAGE = ':current_stage'

if "WORKFLOW_SCHEDULER_LAMBDA_ARN" in os.environ:
    WORKFLOW_SCHEDULER_LAMBDA_ARN = os.environ["WORKFLOW_SCHEDULER_LAMBDA_ARN"]
else:
    WORKFLOW_SCHEDULER_LAMBDA_ARN = ""

if "SYSTEM_TABLE_NAME" in os.environ:
    SYSTEM_TABLE_NAME = os.environ["SYSTEM_TABLE_NAME"]
else:
    ""

if "ShortUUID" in os.environ:
    ShortUUID = os.environ["ShortUUID"]

if "DEFAULT_MAX_CONCURRENT_WORKFLOWS" in os.environ:
    DEFAULT_MAX_CONCURRENT_WORKFLOWS = int(os.environ["DEFAULT_MAX_CONCURRENT_WORKFLOWS"])
else:
    DEFAULT_MAX_CONCURRENT_WORKFLOWS = 10

mie_config = json.loads(os.environ['botoConfig'])
config = config.Config(**mie_config)

# DynamoDB
DYNAMO_CLIENT = boto3.resource("dynamodb", config=config)

# Step Functions
SFN_CLIENT = boto3.client('stepfunctions', config=config)

# Simple Queue Service
SQS_CLIENT = boto3.client('sqs', config=config)

# Lambda
LAMBDA_CLIENT = boto3.client("lambda", config=config)

def log_workflow_execution(workflow_execution):
    logger.info("workflow_execution: {}".format(
        json.dumps(workflow_execution)))

def list_workflow_executions_by_status(status):
    table = DYNAMO_CLIENT.Table(WORKFLOW_EXECUTION_TABLE_NAME)
    projection_expression = "Id, AssetId, CurrentStage, StateMachineExecutionArn, {}, Workflow.{}".format(
        ATT_NAME_WORKFLOW_STATUS, ATT_NAME_WORKFLOW_NAME)

    response = table.query(
        IndexName='WorkflowExecutionStatus',
        ExpressionAttributeNames={
            ATT_NAME_WORKFLOW_STATUS: "Status",
            ATT_NAME_WORKFLOW_NAME: "Name"
        },
        ExpressionAttributeValues={
            ATT_VALUE_WORKFLOW_STATUS: status
        },
        KeyConditionExpression='{} = {}'.format(ATT_NAME_WORKFLOW_STATUS, ATT_VALUE_WORKFLOW_STATUS),
        ProjectionExpression=projection_expression
    )

    workflow_executions = response['Items']
    while 'LastEvaluatedKey' in response:
        response = table.query(
            IndexName='WorkflowExecutionStatus',
            ExpressionAttributeNames={
                ATT_NAME_WORKFLOW_STATUS: "Status",
                ATT_NAME_WORKFLOW_NAME: "Name"
            },
            ExpressionAttributeValues={
                ATT_VALUE_WORKFLOW_STATUS: status
            },
            KeyConditionExpression='{} = {}'.format(ATT_NAME_WORKFLOW_STATUS, ATT_VALUE_WORKFLOW_STATUS),
            ProjectionExpression=projection_expression,
            ExclusiveStartKey=response['LastEvaluatedKey']
        )
        workflow_executions.extend(response['Items'])

    return workflow_executions

def workflow_scheduler_lambda(event, _context):

    execution_table = DYNAMO_CLIENT.Table(WORKFLOW_EXECUTION_TABLE_NAME)
    arn = ""
    workflow_execution = {}
    max_concurrent_workflows = DEFAULT_MAX_CONCURRENT_WORKFLOWS

    try:
        logger.info(json.dumps(event))

        # Get the MaxConcurrent configruation parameter, if it is not set, use the default
        system_table = DYNAMO_CLIENT.Table(SYSTEM_TABLE_NAME)

        # Check if any configuration has been added yet
        response = system_table.get_item(
            Key={
                'Name': 'MaxConcurrentWorkflows'
            },
            ConsistentRead=True)

        if "Item" in response:
            max_concurrent_workflows = int(response["Item"]["Value"])
            logger.info("Got MaxConcurrentWorkflows = {}".format(response["Item"]["Value"]))

        # Check if there are slots to run a workflow
        # TODO - we really need consistent read here.  Index query is not consistent read
        started_workflows = list_workflow_executions_by_status(awsmie.WORKFLOW_STATUS_STARTED)
        num_started_workflows = len(started_workflows)

        if num_started_workflows >= max_concurrent_workflows:
            logger.info("MaxConcurrentWorkflows has been reached {}/{} - nothing to do".format(num_started_workflows, max_concurrent_workflows))

        # We can only read 10 messages at a time from the queue.  Loop reading from the queue until
        # it is empty or we are out of slots
        while (num_started_workflows < max_concurrent_workflows):

            capacity = min(int(max_concurrent_workflows - num_started_workflows), 10)

            logger.info("MaxConcurrentWorkflows has not been reached {}/{} - check if a workflow is available to run".format(num_started_workflows, max_concurrent_workflows))

            # Check if there are workflows waiting to run on the STAGE_EXECUTION_QUEUE_URL
            messages = SQS_CLIENT.receive_message(
                QueueUrl=STAGE_EXECUTION_QUEUE_URL,
                MaxNumberOfMessages=capacity
            )
            if 'Messages' not in messages:  # when the queue is exhausted, the response dict contains no 'Messages' key
                logger.info('Queue is empty')
                break

            for message in messages['Messages']:  # 'Messages' is a list
                # process the messages
                logger.info(message['Body'])
                # next, we delete the message from the queue so no one else will process it again,
                # once it is in our hands it is going run or fail, no reprocessing
                # TODO - we may want to delay deleting the message until complete_stage is called on the
                # final stage so we can detect hung workflows and time them out.  For now, do the simple thing.
                SQS_CLIENT.delete_message(QueueUrl=STAGE_EXECUTION_QUEUE_URL, ReceiptHandle=message['ReceiptHandle'])

                workflow_execution = json.loads(message['Body'])
                queued_workflow_status = workflow_execution['Status']
                workflow_execution['Status'] = awsmie.WORKFLOW_STATUS_STARTED

                # Update the workflow status now to indicate we have taken it off the queue
                update_workflow_execution_status(workflow_execution["Id"], awsmie.WORKFLOW_STATUS_STARTED, "")

                # Resumed workflows state machines are already executing since they just paused
                # to wait for some external action
                if queued_workflow_status != awsmie.WORKFLOW_STATUS_RESUMED:
                    # Kick off the state machine for the workflow
                    response = SFN_CLIENT.start_execution(
                        stateMachineArn=workflow_execution["Workflow"]["StateMachineArn"],
                        name=workflow_execution["Workflow"]["Name"] + workflow_execution["Id"],
                        input=json.dumps(workflow_execution["Workflow"]["Stages"][workflow_execution["CurrentStage"]])
                    )

                    workflow_execution["StateMachineExecutionArn"] = response["executionArn"]

                    # Update the workflow with the state machine id
                    response = execution_table.update_item(
                        Key={
                            'Id': workflow_execution["Id"]
                        },
                        UpdateExpression='SET StateMachineExecutionArn = :arn',
                        ExpressionAttributeValues={
                            ':arn': response["executionArn"]
                        }
                    )

            started_workflows = list_workflow_executions_by_status(awsmie.WORKFLOW_STATUS_STARTED)
            num_started_workflows = len(started_workflows)

    except Exception as e:

        logger.info("Exception in scheduler {}".format(e))
        if "Id" in workflow_execution:

            update_workflow_execution_status(workflow_execution["Id"], awsmie.WORKFLOW_STATUS_ERROR, "Exception in workflow_scheduler_lambda {}".format(e))
        raise

    return arn

def filter_operation_lambda(event, _context):
    '''
    event is
    - Operation input
    - Operation configuration

    returns:
    Operation output
    - Operation status "Skipped" if operation should be skipped
    '''
    logger.info(json.dumps(event))

    operation_object = MediaInsightsOperationHelper(event)

    if operation_object.configuration["MediaType"] != "MetadataOnly" and operation_object.configuration["MediaType"] not in operation_object.input["Media"]:

        operation_object.update_workflow_status(awsmie.OPERATION_STATUS_SKIPPED)

    elif not operation_object.configuration["Enabled"]:

        operation_object.update_workflow_status(awsmie.OPERATION_STATUS_SKIPPED)

    else:

        operation_object.update_workflow_status(awsmie.OPERATION_STATUS_STARTED)

    return operation_object.return_output_object()

def start_wait_operation_lambda(event, _context):
    '''
    Pause a workflow to wait for external processing

    event is
    - Operation input
    - Operation configuration

    returns:
    Operation output

    '''
    logger.info(json.dumps(event))

    operator_object = MediaInsightsOperationHelper(event)

    try:
        update_workflow_execution_status(operator_object.workflow_execution_id, awsmie.WORKFLOW_STATUS_WAITING, "")
    except Exception as e:
        operator_object.update_workflow_status("Error")
        operator_object.add_workflow_metadata(WaitError="Unable to set workflow status to Waiting {e}".format(e=str(e)))
        raise MasExecutionError(operator_object.return_output_object())

    return operator_object.return_output_object()

def check_wait_operation_lambda(event, _context):
    '''
    Check if a workflow is still in a Waiting state.

    event is
    - Operation input
    - Operation configuration

    returns:
    Operation output

    '''
    logger.info(json.dumps(event))

    operator_object = MediaInsightsOperationHelper(event)
    execution_table = DYNAMO_CLIENT.Table(WORKFLOW_EXECUTION_TABLE_NAME)

    response = execution_table.get_item(
            Key={
                'Id': operator_object.workflow_execution_id
            },
            ConsistentRead=True)

    if "Item" in response:
        workflow_execution = response["Item"]
    else:
        workflow_execution = None
        # raise ChaliceViewError(
        operator_object.update_workflow_status("Error")
        operator_object.add_workflow_metadata(WaitError="Unable to find Waiting workflow execution {}".format(
            operator_object.workflow_execution_id))
        raise MasExecutionError(operator_object.return_output_object())

    log_workflow_execution(workflow_execution)

    if workflow_execution["Status"] == awsmie.WORKFLOW_STATUS_WAITING:
        operator_object.update_workflow_status("Executing")
        return operator_object.return_output_object()
    elif workflow_execution["Status"] == awsmie.WORKFLOW_STATUS_STARTED:
        operator_object.update_workflow_status("Complete")
    else:
        operator_object.update_workflow_status("Error")
        operator_object.add_workflow_metadata(WaitError="Unexpected workflow execution status {}".format(
            workflow_execution["Status"]))
        raise MasExecutionError(operator_object.return_output_object())

    return operator_object.return_output_object()
#

def complete_stage_execution_lambda(event, _context):
    '''
    event is a stage execution object
    '''
    logger.info(json.dumps(event))
    return complete_stage_execution(event["Name"], event["Status"], event["Outputs"], event["WorkflowExecutionId"])

def complete_stage_execution(stage_name, status, outputs, workflow_execution_id):

    def format_error_message(error):
        return "Exception while rolling up stage status {}".format(error)

    try:

        execution_table = DYNAMO_CLIENT.Table(WORKFLOW_EXECUTION_TABLE_NAME)
        # lookup the workflow
        response = execution_table.get_item(
            Key={
                'Id': workflow_execution_id
            },
            ConsistentRead=True)

        workflow_execution = response.get("Item")
        if "Item" not in response:
            raise ValueError(
                "Exception: workflow execution id '%s' not found" % workflow_execution_id)

        log_workflow_execution(workflow_execution)

        # Get the workflow stages
        stages = workflow_execution["Workflow"]["Stages"]

        # Roll-up the results of the stage execution.  If anything fails here, we will fail the
        # stage, but still attempt to update the workflow execution the stage belongs to
        try:
            # Roll up operation status
            # # if any operation did not complete successfully, the stage has failed
            opstatus = awsmie.STAGE_STATUS_COMPLETE
            for operation in (operation for operation in outputs
                              if operation["Status"] not in [awsmie.OPERATION_STATUS_COMPLETE, awsmie.OPERATION_STATUS_SKIPPED]):
                opstatus = awsmie.STAGE_STATUS_ERROR

            # don't overwrite an error
            if status != awsmie.STAGE_STATUS_ERROR:
                status = opstatus

            logger.info("Stage status: {}".format(status))

            stages[stage_name]["Outputs"] = outputs

            workflow_execution["Globals"].setdefault("MetaData", {})

            # Roll up operation media and metadata outputs from this stage and add them to
            # the global workflow metadata:
            #
            #     1. mediaType and metatdata output keys must be unique withina stage - if
            #        non-unique keys are found across operations within a stage, then the
            #        stage execution will fail.
            #     2. if a stage has a duplicates a mediaType or metadata output key from the globals,
            #        then the global value is replaced by the stage output value

            # Roll up media
            stage_output_media_type_keys = []
            for operation, media_type in ((operation, media_type)
                                          for operation in outputs
                                          for media_type in operation.get("Media", {}).keys()):
                # replace media with trasformed or created media from this stage
                logger.info(media_type)
                if media_type in stage_output_media_type_keys:
                    raise ValueError(
                        "Duplicate mediaType '%s' found in operation output media.  mediaType keys must be unique within a stage." % media_type)

                workflow_execution["Globals"]["Media"][media_type] = operation["Media"][media_type]
                stage_output_media_type_keys.append(media_type)

            # Roll up metadata
            for operation, key in ((operation, key)
                                   for operation in outputs
                                   for key in operation.get("MetaData", {}).keys()):
                logger.info(key)
                workflow_execution["Globals"]["MetaData"][key] = operation["MetaData"][key]

            stages[stage_name]["Status"] = status

        # The status roll up failed.  Handle the error and fall through to update the workflow status
        except Exception as e:

            message = format_error_message(e)
            logger.info(message)
            update_workflow_execution_status(workflow_execution["Id"], awsmie.WORKFLOW_STATUS_ERROR, message)
            status = awsmie.STAGE_STATUS_ERROR

            raise ValueError("Error rolling up stage status: %s" % e)

        logger.info("Updating the workflow status in dynamodb: current stage {}, globals {}".format(stages[stage_name], workflow_execution["Globals"]))
        # Save the new stage and workflow status
        response = execution_table.update_item(
            Key={
                'Id': workflow_execution_id
            },
            UpdateExpression='SET Workflow.Stages.#stage = :stage, Globals = :globals',
            ExpressionAttributeNames={
                '#stage': stage_name
            },
            ExpressionAttributeValues={
                # ':stage': json.dumps(stage)
                ':stage': stages[stage_name],
                ':globals': workflow_execution["Globals"]
                # ':step_function_arn': step_function_execution_arn
            }
        )

        # Start the next stage for execution
        # TODO - try always completing stage
        # status == awsmie.STAGE_STATUS_COMPLETE:
        workflow_execution = start_next_stage_execution(
            stage_name, workflow_execution)

        if status == awsmie.STAGE_STATUS_ERROR:
            raise ValueError("Stage {} encountered and error during execution, aborting the workflow".format(stage_name))

    except Exception as e:
        logger.info("Exception {}".format(e))

        # Need a try/catch here? Try to save the status
        message = format_error_message(e)
        execution_table.put_item(Item=workflow_execution)
        update_workflow_execution_status(workflow_execution["Id"], awsmie.WORKFLOW_STATUS_ERROR, message)

        raise ValueError(
            "Exception: '%s'" % e)

    # If it is not the end of the workflow, pass the next stage out to be consumed by the next stage.
    stages["End"] = {}
    return stages[workflow_execution["CurrentStage"]]

def start_next_stage_execution(stage_name, workflow_execution):

    try:
        logger.info("START NEXT STAGE: stage_name {}".format(stage_name))

        execution_table = DYNAMO_CLIENT.Table(WORKFLOW_EXECUTION_TABLE_NAME)

        current_stage = stage_name

        message = "stage completed with status {}".format(workflow_execution["Workflow"]["Stages"][current_stage]["Status"])

        if "End" in workflow_execution["Workflow"]["Stages"][current_stage]:

            if workflow_execution["Workflow"]["Stages"][current_stage]["End"]:
                workflow_execution["CurrentStage"] = "End"
                workflow_execution["Status"] = workflow_execution["Workflow"]["Stages"][current_stage]["Status"]

            # Save the new stage
            execution_table.update_item(
                Key={
                    'Id': workflow_execution["Id"]
                },
                UpdateExpression='SET CurrentStage = :current_stage',
                ExpressionAttributeValues={
                    ATT_VALUE_CURRENT_STAGE: "End"
                }
            )

            update_workflow_execution_status(workflow_execution["Id"], workflow_execution["Status"], message)

        elif workflow_execution["Workflow"]["Stages"][current_stage]["Status"] == "Error":
            workflow_execution["Status"] = workflow_execution["Workflow"]["Stages"][current_stage]["Status"]

            # Save the new stage and workflow status
            execution_table.update_item(
                Key={
                    'Id': workflow_execution["Id"]
                },
                UpdateExpression='SET CurrentStage = :current_stage',
                ExpressionAttributeValues={
                    ATT_VALUE_CURRENT_STAGE: "End"
                }
            )

            update_workflow_execution_status(workflow_execution["Id"], workflow_execution["Status"], message)

        elif "Next" in workflow_execution["Workflow"]["Stages"][current_stage]:
            current_stage = workflow_execution["CurrentStage"] = workflow_execution[
                "Workflow"]["Stages"][current_stage]["Next"]

            workflow_execution["Workflow"]["Stages"][current_stage]["Input"] = workflow_execution["Globals"]
            workflow_execution["Workflow"]["Stages"][current_stage]["Status"] = awsmie.STAGE_STATUS_STARTED

            try:
                logger.info(
                    "Updating workflow with new current stage")
                logger.info(json.dumps(workflow_execution))

                workitem = {
                    "WorkflowExecutionId": workflow_execution["Id"],
                    "stage": workflow_execution["Workflow"]["Stages"][current_stage]
                }

                # IMPORTANT: update the workflow_execution before queueing the work item...the
                # queued workitem must match the current stage when we start stage execution.
                # execution_table.put_item(Item=workflow_execution)
                execution_table.update_item(
                    Key={
                        'Id': workflow_execution["Id"]
                    },
                    UpdateExpression='SET Workflow.Stages.#stage = :stage, CurrentStage = :current_stage',
                    ExpressionAttributeNames={
                        '#stage': current_stage,
                    },
                    ExpressionAttributeValues={
                        ':stage': workflow_execution["Workflow"]["Stages"][current_stage],
                        ATT_VALUE_CURRENT_STAGE: current_stage

                    }
                )

                update_workflow_execution_status(workflow_execution["Id"], workflow_execution["Status"], message)
                logger.info(json.dumps(workitem))

            except Exception as e:
                message = "Exception queuing work item: {} ".format(e)

                workflow_execution["Status"] = awsmie.WORKFLOW_STATUS_ERROR

                logger.info(message)

                update_workflow_execution_status(workflow_execution["Id"], workflow_execution["Status"], message)

                # raise ChaliceViewError(
                raise ValueError(message)

    except Exception as e:
        workflow_execution["Status"] = awsmie.WORKFLOW_STATUS_ERROR
        message = "Exception {}".format(e)
        logger.info(message)
        update_workflow_execution_status(workflow_execution["Id"], workflow_execution["Status"], message)
        # raise ChaliceViewError(
        raise ValueError(
            "Exception: '%s'" % e)

    log_workflow_execution(workflow_execution)
    return workflow_execution

def update_workflow_execution_status(id, status, message):
    """
    Get the workflow execution by id from dyanamo and assign to this object
    :param id: The id of the workflow execution
    :param status: The new status of the workflow execution

    """
    logger.info("Update workflow execution {} set status = {}".format(id, status)) #nosec
    execution_table = DYNAMO_CLIENT.Table(WORKFLOW_EXECUTION_TABLE_NAME)

    if status == awsmie.WORKFLOW_STATUS_ERROR:
        execution_table.update_item(
            Key={
                'Id': id
            },
            UpdateExpression='SET {} = {}, Message = :message'.format(
                ATT_NAME_WORKFLOW_STATUS, ATT_VALUE_WORKFLOW_STATUS),
            ExpressionAttributeNames={
                ATT_NAME_WORKFLOW_STATUS: "Status"
            },
            ExpressionAttributeValues={
                ATT_VALUE_WORKFLOW_STATUS: status,
                ':message': message

            }
        )
    else:
        execution_table.update_item(
            Key={
                'Id': id
            },
            UpdateExpression='SET {} = {}'.format(
                ATT_NAME_WORKFLOW_STATUS, ATT_VALUE_WORKFLOW_STATUS),
            ExpressionAttributeNames={
                ATT_NAME_WORKFLOW_STATUS: "Status"
            },
            ExpressionAttributeValues={
                ATT_VALUE_WORKFLOW_STATUS: status
            }
        )

    if status in [awsmie.WORKFLOW_STATUS_QUEUED, awsmie.WORKFLOW_STATUS_COMPLETE, awsmie.WORKFLOW_STATUS_ERROR]:
        # Trigger the workflow_scheduler
        LAMBDA_CLIENT.invoke(
            FunctionName=WORKFLOW_SCHEDULER_LAMBDA_ARN,
            InvocationType='Event'
        )

# Find all of the execution error events for a state machine execution
def get_execution_errors(arn):

    try:

        executions = []
        paginator = SFN_CLIENT.get_paginator('get_execution_history')

        # return all the history
        page_iterator = paginator.paginate(executionArn=arn,
                maxResults= 20,
                reverseOrder=True)

        # return only the history with exceptions
        for page in page_iterator:
            for event in page["events"]:
              if (any(sub in event["type"] for sub in ['Failed', 'Aborted', 'TimedOut'])):
                  logger.info("Found error event: {}".format(event))
                  executions.append(event)

        return executions

    except:
        logger.error("Unable to retrieve execution history for state machine termination")
        raise

# Find the earliest error message in the executions and retrieve additonal info if available
def parse_execution_error(arn, executions, status):

    message = "Caught Step Function Execution Status Change event for execution: "+ arn +", status:"+ status

    # return the cause info from the most recent failure, if any
    for execution in executions:
      for key, value in execution.items():
        if (any(sub in key for sub in ['Failed', 'Aborted', 'TimedOut'])):
          if "cause" in value:
              message = message+", cause: "+value["cause"]
              break

    return message

# This lambda is invoked for Step Functions Execution Status Change events
# that contain the status [ERROR, ABORTED, TIME_OUT].  Failures that occur
# within the steps of a workflow state machine are handled by the
# OperotorFailed lambda function, but if the state machine service throws
# an exception that causes the state machine to terminate immediately,
# it can't be handled within the state machine because the execution
# is haulted.
#
# Info on events handled here:
#     https://docs.aws.amazon.com/step-functions/latest/dg/cw-events.html
#
# Propagate the error and cause to the workflow control plane to close off
# the workflow execution
def workflow_error_handler_lambda(event, _context):

  try:

    logger.info("workflow_error_handler_lambda: {}".format(json.dumps(event)))

    if not ShortUUID:
        raise LookupError('ShortUUID is not set in lambda environment.')

    logger.info("Process step function error event for stack with ShortUUID {}".format(ShortUUID))

    if "detail" not in event:
        raise LookupError('event.detail is missing.')
    if "name" not in event["detail"]:
        raise LookupError('name is missing in event.detail.')
    if "status" not in event["detail"]:
        raise LookupError('status is missing in event.detail.')
    if "executionArn" not in event["detail"]:
        raise LookupError('executionArn is missing in event.detail.')

    # only process events for state machines that are part of this stack
    if not event["detail"]["stateMachineArn"].find(ShortUUID):
        logger.info("Event not processed: This event is not from the stack with ShortUUID {}".format(ShortUUID))
        return {}

    executions = get_execution_errors(event["detail"]["executionArn"])

    message = parse_execution_error(event["detail"]["executionArn"], executions, event["detail"]["status"])

    state_machine_execution = event["detail"]["executionArn"]

    # Check the currently active workflows for this execution arn and set the
    # status to error if found.
    started_workflows = list_workflow_executions_by_status(awsmie.WORKFLOW_STATUS_STARTED)
    for workflow in started_workflows:
        if workflow["StateMachineExecutionArn"] == event["detail"]["executionArn"]:
            update_workflow_execution_status(workflow["Id"], awsmie.WORKFLOW_STATUS_ERROR, message)

    response = {
      "stateMachineExecution": state_machine_execution,
      "errorMessage": message
    }

    logger.info("workflow_error_handler_lambda caught error: {}".format(json.dumps(response)))

    return response
  except Exception as e:
    logger.error("Unable to handle workflow step function error: {}".format(e))
    raise e


#Face and eye detection,
#also see: http://docs.opencv.org/trunk/doc/py_tutorials/py_objdetect/py_face_detection/py_face_detection.html
#Training:
#http://docs.opencv.org/doc/user_guide/ug_traincascade.html
#Gerald Schuller, Dec 2014

import numpy as np
import cv2

face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
eye_cascade = cv2.CascadeClassifier('haarcascade_eye.xml')

cap = cv2.VideoCapture(0)

while(True):
    [retval, frame] = cap.read()
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    faces = face_cascade.detectMultiScale(gray, 1.3, 5)
    print(faces)
    for (x,y,w,h) in faces:
       cv2.rectangle(gray,(x,y),(x+w,y+h),255,2)
       roi_gray = gray[y:y+h, x:x+w]

       eyes = eye_cascade.detectMultiScale(roi_gray)

       for (ex,ey,ew,eh) in eyes:
          cv2.rectangle(gray,(x+ex,y+ey),(x+ex+ew,y+ey+eh),255,2)

    cv2.imshow('Face and Eye Detection',gray)
    #Keep window open until key 'q' is pressed:
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()


import argparse
from datasets import load_dataset
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

def main(args):
    colname = ['Ground Truth', 'SFT', 'PPO']
    df = pd.read_csv(args.file_path)
    score_ar = np.array(df)[:,1:]
    score_ar = np.concatenate(np.hsplit(score_ar, args.n_question), axis = 0)

    avg_score = score_ar.mean(0).astype(float).round(1)
    print(avg_score)

    ratio = {}
    for i in range(2,len(colname)):
        odds_l = []
        for j in range(2):
            odds = (score_ar[:, i] > score_ar[:, j]).astype(float).mean().round(2)
            odds_l.append(odds)
        ratio[colname[i]] = odds_l
    print(ratio)

    if args.plot:
        barWidth = 0.35
        plt.bar(colname, avg_score, width=barWidth)
        plt.yticks(np.arange(0, 3.1, 1))
        plt.xlabel('Avg Score( Max 3 )', fontsize = 15)
        plt.ylabel('Score', fontsize = 15)
        plt.savefig('plot/ppo_avg_score.png',dpi=400)
        plt.close()

        barWidth = 0.2
        br1 = np.arange(len(colname[:2]))
        p1 = plt.bar(br1, ratio[colname[2]], label=colname[:2], width=barWidth)
        plt.axhline(y = 0.5, color = 'r', linestyle = '--')
        plt.xticks([r for r in range(len(colname[:2]))], colname[:2])
        plt.yticks(np.arange(0, 1.1, 0.2))
        plt.xlabel('Human Preference', fontsize = 15)
        plt.ylabel('Odds', fontsize = 15)
        plt.legend((p1[0],), ('PPO',))

        plt.savefig('plot/ppo_preference.png',dpi=400)
        plt.close()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--file_path', type=str, default='human_preference_ppo.csv')
    parser.add_argument('--n_question', type=int, default=3)
    parser.add_argument('--plot', type=bool, default=True)
    args = parser.parse_args()
    main(args)


#Imports
import pandas as pd
import numpy as np
import os
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.colors import ListedColormap
from sklearn.linear_model import LinearRegression
import geopandas as gpd
import datetime
import scipy
from src.config import configurations
from src.dimensionality_reduction import compute_covariance, visualize_manifold_method, choose_dimension
from src.dimensionality_reduction import viz_cluster_map, viz_LLE, viz_Isomap, viz_SE, viz_DM
from src.utils import define_paths, load_data, convert_to_python_dict, save_obj, load_obj, find_closest_time_series
from src.utils import plot_time_series, find_cos_similarity, intersection, relabel_clusters
from rpy2.robjects.packages import importr
import rpy2.robjects as ro
from rpy2.robjects import r, pandas2ri
from rpy2.robjects.conversion import localconverter
from sklearn.manifold import Isomap, SpectralEmbedding, LocallyLinearEmbedding, trustworthiness
from sklearn.manifold import trustworthiness
from copy import copy
from kneed import DataGenerator, KneeLocator
import pickle
from matplotlib import gridspec
import matplotlib.dates as mdates
from matplotlib.dates import MO, TU, WE, TH, FR, SA, SU
import scipy
from datetime import datetime
from sklearn.metrics.pairwise import cosine_similarity

plt.rcParams['figure.dpi'] = 600

def GMM_clustering_R(X_method_df, method, default_cluster_num = None):
    """Function to check BIC and perform GMM clustering on embedded dataset"""
    #First, import r packages and fix random seed:
    base = importr('base')
    mclust = importr('mclust')
    ro.r('set.seed(0)')

    #Now, check BIC and make a plot
    num_components_to_try = pd.Series(np.arange(1,12)) #try up to 12 components
    with localconverter(ro.default_converter + pandas2ri.converter):
        ro.r('set.seed(0)')
        BIC_method = mclust.mclustBIC(X_method_df, G = num_components_to_try)

    model_names = ['EII', 'VII', 'EEI', 'VEI', 'EVI', 'VVI', 'EEE', 'EVE', 'VEE', 'VVE', 'EEV', 'VEV', 'EVV', 'VVV']
    sns.set(style="darkgrid")
#     sns.set_palette("tab10")
    BIC_method_df = pd.DataFrame(BIC_method, columns = model_names)
    BIC_method_df = BIC_method_df.dropna(axis=1) #drop parametrizations with NaNs
#     plt.figure()
    BIC_method_df.plot(marker = 'o')
    plt.title('GMM BIC on ' + method.__name__)

    #Now, find the knee point of the optimal BIC plot (the best GMM parametrization)
    best_parametrization = BIC_method_df.columns[BIC_method_df.max().argmax()]
    kneedle = KneeLocator(num_components_to_try, BIC_method_df[best_parametrization], S=1, curve='concave', direction='increasing', interp_method='polynomial')
#     plt.figure()
    kneedle.plot_knee()
    plt.title('GMM BIC on ' + method.__name__ + ': Knee Point')
    plt.xlabel('num_GMM_components')
    plt.ylabel('')
    print('Elbow point: {} components with BIC {}'.format(kneedle.knee, kneedle.knee_y))

    #Pick the best number of GMM components:
    best_num_components = kneedle.knee-1
    if default_cluster_num is not None:
        best_num_components = default_cluster_num-1
    with localconverter(ro.default_converter + pandas2ri.converter):
        ro.r('set.seed(0)')
        mc = mclust.Mclust(X_method_df, G = pd.Series([num_components_to_try[best_num_components]]))
        print(base.summary(mc))
        print('Uncertainty quantiles:', np.quantile(mc[15], [0, 0.25, 0.5, 0.75, 1]))
        mc_dict = convert_to_python_dict(mc)
        method_model_name = mc_dict['modelName']
        print(method_model_name)
        param = mc_dict['parameters']
        method_means = np.array(convert_to_python_dict(param)['mean'])
        method_uncertainty = np.array(mc_dict['uncertainty'])
        method_z = np.array(convert_to_python_dict(mc)['z'])
        method_clusters = np.array(convert_to_python_dict(mc)['classification'])
        method_means = pd.DataFrame(method_means, columns = ['V'+str(i+1) for i in range(method_means.shape[1])])
    return method_clusters, method_means, method_z, method_uncertainty

def create_avg_df(SE_clusters, index_X, covid_):
    SE_clusters_block_indexed = pd.Series(SE_clusters, index = index_X)
    df_CI = covid_.dropna()#[SE_clusters_block_indexed['x']==4]
    df_CI['cluster'] = SE_clusters_block_indexed.astype('int')
    df_CI['block'] = df_CI.index
    dates = df_CI.columns.drop(['block', 'cluster'])
    avg_per_clust = df_CI.groupby('cluster').mean()[dates]#[dates].plot()
    return avg_per_clust

def index_with_blocks_and_save(STATE, emb_df_optimal_D, emb_2D, emb_3D, clusters, z, uncertainty, index_X, emb_method):
    df_optimal_D = emb_df_optimal_D.set_index(index_X)
    df_2D = pd.DataFrame(emb_2D, columns = ['Mode ' + str(i) for i in range(emb_2D.shape[1])], index = index_X)
    df_3D = pd.DataFrame(emb_3D, columns = ['Mode ' + str(i) for i in range(emb_3D.shape[1])], index = index_X)
    df_clusters = pd.Series(clusters, index = index_X)
    df_z = z.set_index(index_X)
    df_uncertainty = pd.Series(uncertainty, index = index_X)

    #Save
    path_to_processed = os.path.join(configurations['DATA_PATH'], 'processed')
    df_optimal_D.to_csv(os.path.join(path_to_processed, emb_method.__name__ + str(df_optimal_D.shape[1]) + 'D_' + STATE + '.csv'))
    df_2D.to_csv(os.path.join(path_to_processed, emb_method.__name__ + str(df_2D.shape[1]) + 'D_' + STATE + '.csv'))
    df_3D.to_csv(os.path.join(path_to_processed, emb_method.__name__ + str(df_3D.shape[1]) + 'D_' + STATE + '.csv'))
    df_clusters.to_csv(os.path.join(path_to_processed, 'labels_' + STATE + '.csv'))
    df_z.to_csv(os.path.join(path_to_processed, 'z_' + STATE + '.csv'))
    df_uncertainty.to_csv(os.path.join(path_to_processed, 'uncertainty_' + STATE + '.csv'))
    return df_optimal_D, df_2D, df_3D, df_clusters, df_z, df_uncertainty

def add_state_to_fig(state, fig, spec, row, NUM_STATES, X, reordered_SE_clusters, index_X, reordered_avg_per_clust, load_path = None, save_path = None, separate = False, two_cols = False, configurations = None,
                     dm = False, dm_bandwidth = 'fixed'):
    SHAPE_PATH, FIGURE_PATH, RAW_DATA_PATH, INCOME_POPULATION_PATH = define_paths(state)
    almost_sequential_pal = configurations['clustering_palette']#['#1f78b4','#a6cee3','#fdbf6f','#ff7f00', '#cc78bc']
    sns.set_palette(almost_sequential_pal)
    NUM_COLS = 3
    if two_cols:
        NUM_COLS = 2
    SEPARATE_PATH = os.path.abspath(os.path.join(os.getcwd(), '..', 'reports', 'figures', 'Separate_Summary'))
    # First column
    if separate:
        fig = plt.figure(figsize = (5*7/3, 5*(6.25 - 0.77)/4))
        ax = fig.add_subplot(111, projection='3d')
    else:
        ax = fig.add_subplot(spec[NUM_COLS*row], projection = '3d')#NUM_STATES, NUM_COLS, NUM_COLS*row + 1, projection = '3d')
    ax.set_anchor('E')
    if two_cols:
        cbar = None
    else:
        cbar = 'clustering'
    if not dm: #If not diffusion maps
        _, X_3D_SE = viz_SE(X, reordered_SE_clusters, filename = None, alpha = 0.5, cbar = cbar, ax = ax, load_path = load_path, save_path = save_path)
    else:
        _, X_3D_SE = viz_DM(X, reordered_SE_clusters, filename=None, alpha=0.5, cbar=cbar, ax=ax, load_path=load_path,
                        save_path=save_path, dm_bandwidth = dm_bandwidth)
    if state == 'wa':
        if not dm:
            ax.view_init(30, 200)
        if dm:
            if dm_bandwidth == 'variable':
                ax.view_init(130, 60)#30,0
            else:
                ax.view_init(30,0)
    if state == 'ga':
        ax.view_init(30, 80)
    if state == 'tx':
        ax.view_init(30, 245)
    if state == 'ca':
        ax.view_init(30, 80)
    if state == 'demo':
        ax.view_init(30, 200)

    ax.set_xlim(np.array([np.min(X_3D_SE[:,0]), np.max(X_3D_SE[:,0])]))
    ax.set_ylim(np.array([np.min(X_3D_SE[:,1]), np.max(X_3D_SE[:,1])]))
    ax.set_zlim(np.array([np.min(X_3D_SE[:,2]), np.max(X_3D_SE[:,2])]))
    ax.set_xlabel('')
    ax.set_ylabel('')
    ax.set_zlabel('')
    plt.axis('tight')
    ax.w_xaxis.set_pane_color((1.0, 1.0, 1.0, 1.0))
    ax.w_yaxis.set_pane_color((1.0, 1.0, 1.0, 1.0))
    ax.w_zaxis.set_pane_color((1.0, 1.0, 1.0, 1.0))
    ax.set_xticklabels([])
    ax.set_yticklabels([])
    ax.set_zticklabels([])#,rotation=-15, va='center', ha='right')
    ax.set_xticks([])
    ax.set_yticks([])
    ax.set_zticks([])
    ax.xaxis._axinfo['juggled'] = (0,0,1)
#     ax.dist = 5.5
    # ax.yaxis._axinfo['juggled'] = (1,1,1)
    # ax.zaxis._axinfo['juggled'] = (1,0,0)
    ax.grid(False)
    ax.set_title('')
#     ax.set_ylabel("Median Population", fontname="Arial", fontsize=12)

    if separate:
        fig.savefig(os.path.join(FIGURE_PATH, state + '_embedding_col_1.png'), bbox_inches = 'tight',  pad_inches=0, dpi = 900)

    # Second column
    if separate:
        fig = plt.figure(figsize = (5*7/3, 5*(6.25 - 0.77)/4))
        ax = fig.add_subplot(111)
    else:
        ax = fig.add_subplot(spec[NUM_COLS*row + 1])#NUM_STATES, NUM_COLS, NUM_COLS*row + 2)
        ax.set_aspect('equal')
#     ax.set_anchor('C')
    if state == 'wa':
        ax.margins(0.2, 0.2)

    if two_cols:
        cmap = None
    else:
        cmap = 'clust'

    viz_cluster_map(reordered_SE_clusters, index_X, filename = None, cmap = cmap, ax = ax, title = state, state = state)
    ax.set_title('')
    plt.axis('tight')

    if separate:
        fig.savefig(os.path.join(FIGURE_PATH, state + '_map_col_2.png'), bbox_inches = 'tight',  pad_inches=0, dpi = 900)

    #Third column
    if not two_cols:
        if separate:
            fig = plt.figure(figsize = (5*7/3, 5*(6.25 - 0.77)/4))
            ax = fig.add_subplot(111)
        else:
            ax = fig.add_subplot(spec[NUM_COLS*row + 2])#NUM_STATES, NUM_COLS, NUM_COLS*row + 3)
        plot_df = reordered_avg_per_clust.T
        plot_df.index = [pd.to_datetime(date, format='%Y-%m-%d').date() for date in plot_df.index]

        almost_sequential_pal = ['#1f78b4','#a6cee3','#fdbf6f','#ff7f00', '#cc78bc']
        friendly_cmap = ListedColormap(sns.color_palette(almost_sequential_pal, len(np.unique(reordered_SE_clusters))).as_hex())

        plot_df.plot(ax = ax, ylim = [0.1, 0.7], legend = False, cmap = friendly_cmap)
        ax.xaxis.set_major_locator(mdates.MonthLocator(interval=1))
        ax.xaxis.set_major_formatter(mdates.DateFormatter('%m/%d'))

        ax.xaxis.set_minor_locator(mdates.WeekdayLocator(byweekday=MO, interval=1))
        ax.tick_params(which='minor', length=0, color='r')
        ax.tick_params(axis = 'both', length=10, width=3, which='major')
        ax.axvline(datetime(2020, 6, 1), color = 'k', alpha = .3, ls='--', lw =.5)

        ax.spines['right'].set_visible(False)
        ax.spines['top'].set_visible(False)
        # Only show ticks on the left and bottom spines
        ax.yaxis.set_ticks_position('left')
        ax.xaxis.set_ticks_position('bottom')
#         ax.xticks(fontsize=11*5)
#         ax.yticks(fontsize=11*5)
        ax.tick_params(axis='both', which='major', labelsize=6*5)
        ax.grid(True, 'minor', 'x', ls='--', lw=.5, c='k', alpha=.3)
        ax.set_xlim([datetime(2020, 2, 19), datetime(2020, 6, 21)])
        if separate:
            fig.savefig(os.path.join(FIGURE_PATH, state + '_time_series_col_3.png'), bbox_inches = 'tight',  pad_inches=0, dpi = 900)

def analysis(STATE, method, method_kwargs, hyperparams_to_test, fig, spec, row, precomputed = False, separate = False, two_cols = False, NUM_STATES = 1, configurations = None, default_cluster_num=5, dm = False, dm_bandwidth = 'fixed'):
    #First, define appropriate paths
    SHAPE_PATH, FIGURE_PATH, RAW_DATA_PATH, INCOME_POPULATION_PATH = define_paths(STATE)

    #Load the data
    covid_, X, index_X, columns_X = load_data(RAW_DATA_PATH)

    #Do dim red
    print('##################D-RED#################')
    emb_method = method
    if not precomputed:
        errors_results, embeddings_results, trustws_results = choose_dimension(X, emb_method, hyperparams_to_test, **method_kwargs)

        save_obj(embeddings_results, STATE + '_embeddings_results' + method.__name__)
        save_obj(errors_results, STATE + '_errors_results' + method.__name__)
        save_obj(trustws_results, STATE + '_trustws_result' + method.__name__)
    if precomputed:
        embeddings_results = load_obj(STATE + '_embeddings_results' + method.__name__)
        errors_results = load_obj(STATE + '_errors_results' + method.__name__)
        trustws_results = load_obj(STATE + '_trustws_result' + method.__name__)

    if (len(hyperparams_to_test['n_components']) > 1) and (errors_results['n_components'][0] is not None):
        plt.plot(hyperparams_to_test['n_components'], errors_results['n_components'])

    if (len(hyperparams_to_test['n_components']) > 1):
        kneedle = KneeLocator(hyperparams_to_test['n_components'], np.array(trustws_results['n_components']), S=1, curve='concave', direction='increasing', interp_method='polynomial', online = False)
        kneedle.plot_knee()
        plt.title(emb_method.__name__  + ' trustworthiness')
        plt.xlabel('n_components')
        plt.ylabel('trustworhiness')
        kneedle.knee, kneedle.knee_y

    #Save the dataframe with optimal dim
    if (len(hyperparams_to_test['n_components']) > 1):
        good_dim = int(np.squeeze(np.where(hyperparams_to_test['n_components'] == kneedle.knee)))
    else:
        good_dim = 0
    X_method = embeddings_results['n_components'][good_dim] #pick the best (knee point) n_components
    X_method_df = pd.DataFrame(X_method, columns = ['Mode {}'.format(i) for i in range(X_method.shape[1])])#, index = index_X)
    X_method_df.to_csv(os.path.join(configurations['DATA_PATH'], 'interim', method.__name__ + str(X_method.shape[1]) + 'D_' + STATE + '.csv'))
    print('Saving optimal embedding. Method: ', method.__name__, 'shape: ', X_method_df.shape)

    print('##################INITIAL VIZ#################')
    #Find the 2D and 3D embeddings and continuous colors based on that
    filename_initial = os.path.join(FIGURE_PATH, 'initial_' + method.__name__)
    if method.__name__ == 'Isomap':
        viz = viz_Isomap
    if method.__name__ == 'SpectralEmbedding':
        viz = viz_SE
    if method.__name__ == 'LocallyLinearEmbedding':
        viz = viz_LLE
    if method.__name__ == 'dm_wrapper':
        viz = viz_DM

    if precomputed:
        load_path = os.path.join('obj', STATE)
        save_path = None
    else:
        load_path = None
        save_path = os.path.join('obj', STATE)
    X_2D_emb, X_3D_emb = viz(X, colors = None, filename = filename_initial, alpha = 0.5, load_path = load_path, save_path = save_path)
    cos_colors = find_cos_similarity(X_2D_emb)
    #Color the manifold continuously
    filename_initial_colored = os.path.join(FIGURE_PATH, 'initial_'+ method.__name__ + '_colored')
    X_2D_emb, X_3D_emb = viz(X, colors = cos_colors, filename = filename_initial_colored, cbar = None, alpha = 0.5, load_path = load_path, save_path = save_path)

    print('##################GMM CLUSTERING#################')
    #Import R for clustering
    base = importr('base')
    mclust = importr('mclust')
    ro.r('set.seed(1)')

    dontprecomputeclusters = not precomputed
#     if not precomputed:
    if dontprecomputeclusters:
        clusters, means, z, uncertainty = GMM_clustering_R(X_method_df, method, default_cluster_num=default_cluster_num) #could change this to 5 to be consistent across states to auto-id clust #
        clusters_block_indexed = pd.Series(clusters, index = index_X)

        avg_per_clust = create_avg_df(clusters, index_X, covid_)

        reordered_clusters, reordered_means, reordered_z, reordered_uncertainty = relabel_clusters(clusters.astype('int'), avg_per_clust, means, z, uncertainty)
        reordered_avg_per_clust = create_avg_df(reordered_clusters, index_X, covid_)
        #Save
        np.save(os.path.join('obj', STATE + '_reordered_clusters.npy'), reordered_clusters,)
        reordered_means.to_csv(os.path.join('obj', STATE + '_reordered_means.csv'))
        reordered_z.to_csv(os.path.join('obj', STATE + '_reordered_z.csv'))
        np.save(os.path.join('obj', STATE + '_reordered_uncertainty.npy'),reordered_uncertainty)

        reordered_avg_per_clust.to_csv(os.path.join('obj', STATE + '_reordered_avg_per_clust.csv'))

#     if precomputed:
    if not dontprecomputeclusters:
        reordered_clusters = np.load(os.path.join('obj', STATE + '_reordered_clusters.npy'))
        reordered_means = pd.read_csv(os.path.join('obj', STATE + '_reordered_means.csv'), index_col = 0)
        reordered_z = pd.read_csv(os.path.join('obj', STATE + '_reordered_z.csv'), index_col = 0)
        reordered_uncertainty = np.load(os.path.join('obj', STATE + '_reordered_uncertainty.npy'))
        reordered_avg_per_clust = pd.read_csv(os.path.join('obj', STATE + '_reordered_avg_per_clust.csv'), index_col = 0)

    #Save the data for Dennis (for only this method)
    index_with_blocks_and_save(STATE, X_method_df, X_2D_emb, X_3D_emb, reordered_clusters, reordered_z, reordered_uncertainty, index_X, emb_method)

    N_TIMESERIES = 5
    closest_to_mean_samples, closest_to_mean_block_ids = find_closest_time_series(X_method_df, reordered_means, covid_, index_X, n = N_TIMESERIES)

    print('##################FINAL VIZ#################')
    sns.set(style="whitegrid")
    if two_cols:
        reordered_clusters = cos_colors #Change colors
    add_state_to_fig(STATE, fig, spec, row, NUM_STATES, X, reordered_clusters, index_X, reordered_avg_per_clust, load_path = load_path, save_path = save_path, separate = separate, two_cols = two_cols, configurations = configurations, dm = dm, dm_bandwidth = dm_bandwidth)
    return reordered_clusters
if __name__ == "__main__":
    pass


from django import forms
from django.contrib.admin.forms import AdminAuthenticationForm
from django.core.exceptions import ValidationError

class DateFilterForm(forms.Form):
    device_eui = forms.CharField(label='EUI', max_length = 50)
    datetime_ini = forms.DateTimeField(label='Data inicial')
    datetime_fin = forms.DateTimeField(label='Data final')

class WhiteTariffForm(forms.Form):
    # filtro relativo ao gráfico de consumo medido (gráfico 2)
    datetime_ref = forms.DateTimeField(label='Data')
    # filtro relativo à data inicial e à data final (navbar à esquerda)
    datetime_ini = forms.DateTimeField(label='Data inicial')
    datetime_fin = forms.DateTimeField(label='Data final')
    # filtro relativo à data inicial e à data final (campos ocultos)
    datetime_ini_hidden = forms.DateTimeField(label='Inicial')
    datetime_fin_hidden = forms.DateTimeField(label='Final')

class IcaroAdminAuthenticationForm(AdminAuthenticationForm):
    def confirm_login_allowed(self, user):
        # elimina a confirmação de is_staff rodando o mesmo código de AuthenticationForm
        if not user.is_active:
            raise ValidationError(
                self.error_messages['inactive'],
                code='inactive',
            )

class NewUserForm(forms.Form):
    username = forms.CharField(label='username', max_length = 50)
    first_name = forms.CharField(label='first_name', max_length=50)
    last_name = forms.CharField(label='last_name', max_length=50)
    email = forms.CharField(label='email', max_length=50)


import scrapy
import re

class CathoJobSpider(scrapy.Spider):
    name = "geekhunter"

    def start_requests(self):
        urls = [
            "https://www.geekhunter.com.br/vagas",
        ]

        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        page = 1
        for job in response.css("ul li article article div div h2"):

            yield {
                "url": job.css("a::attr(href)").get(),
                "title": job.css("a::text").get(),
                "job_id": int(re.findall(r"\d+", job.css("a::attr(href)").get())[0]),
            }
            page += 1
            next_page = "&page={}".format(page)

            response.urljoin(next_page)
            yield response.follow(next_page, callback=self.parse)
            # next_page = response.urljoin(next_page)
            # print("PAgina: {page}")
            # yield scrapy.Request(next_page, callback=self.parse)


from datetime import date, datetime

from django.conf import settings
from django.core.mail import EmailMultiAlternatives
from django.db.models import Q

from rest_framework import generics, status
from rest_framework.permissions import IsAuthenticated, AllowAny
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework_jwt.authentication import JSONWebTokenAuthentication

from users.models import User

from .models import (
    UserPolicy,
    Insurance,
    DocumentsRequest,
    InsuranceRequest,
    DomiRequestInsurer,
    Metadata,
    MetadataChoices,
    Answer,
)
from .serializers import (
    UserPolicySerializer,
    InsuranceSerializer,
    RequestSerializer,
    InsuranceDetailSerializer,
    InsuranceRequestSerializer
)

class UserPolicyDetail(generics.ListAPIView):
    serializer_class = UserPolicySerializer
    permission_classes = (IsAuthenticated,)
    authentication_classes = (JSONWebTokenAuthentication,)

    def get_queryset(self):
        queryset = UserPolicy.objects.filter(
            Q(client=self.request.user) |
            Q(insurance_request__client=self.request.user)
        )
        if queryset.exists():
            return queryset
        return ''

class InsuranceList(generics.ListAPIView):
    serializer_class = InsuranceSerializer
    permission_classes = (AllowAny,)

    def get_queryset(self):
        return Insurance.objects.filter(is_active=True).order_by('id')

class InsuranceDetail(generics.RetrieveAPIView):
    serializer_class = InsuranceDetailSerializer
    permission_classes = (AllowAny,)
    queryset = Insurance.objects.all()

class RequestViewSet(APIView):
    serializer_class = RequestSerializer
    permission_classes = (IsAuthenticated,)
    authentication_classes = (JSONWebTokenAuthentication,)

    def post(self, request, format=None):

        property1 = request.data['property1']
        property2 = request.data['property2']

        request_date = date.today()
        username = request.user
        name = request.data['name']
        adviser_code = request.data['adviser_code']
        fisico = request.data['fisico']
        optionId = request.data['optionId']
        price = request.data['price']

        user = User.objects.get(username=username)
        insurance = Insurance.objects.filter(name=name).first()
        # broker = User.objects.get(username='apptitud')

        try:
            request_obj = InsuranceRequest(
                status=InsuranceRequest.PENDING,
                request_date=request_date,
                client=user,
                insurance=insurance,
                adviser_code=adviser_code,
                price=price
            )
            request_obj.save()

            document_request = DocumentsRequest.objects.create(
                insurance_request=request_obj,
                property_card_front=property1,
                property_card_back=property2
            )
            document_request.save()

            try:
                oldSoat = request.data['oldSoat']
                document_request.old_soat = oldSoat
                document_request.save()
            except:
                oldSoat = None

            sendMail(username, property1, property2, oldSoat, optionId, fisico)
            return Response({'detail': 'Solicitud creada exitosamente'}, status=status.HTTP_201_CREATED)

        except Exception as e:
            print("EXception")
            print(e)
            return Response({'detail': 'Ha ocurrido un error al crear la solicitud'}, status=status.HTTP_400_BAD_REQUEST)

class Requests(generics.ListAPIView):
    """Lista de solicitudes de los usuarios
    """

    model = InsuranceRequest
    serializer_class = InsuranceRequestSerializer

    def get_queryset(self):
        return InsuranceRequest.objects.filter(client=self.request.user)

class RequestDetail(generics.RetrieveAPIView):
    """Detalle de una solicitud
    """

    model = InsuranceRequest
    serializer_class = InsuranceRequestSerializer
    queryset = InsuranceRequest.objects.all()

class UploadPaymentProof(APIView):
    serializer_class = RequestSerializer
    permission_classes = (IsAuthenticated,)
    authentication_classes = (JSONWebTokenAuthentication,)

    def post(self, request, format=None):

        paymentProof = request.data['payment']
        request_code = request.data['request_code']

        try:
            request_obj = InsuranceRequest.objects.get(request_code=request_code)
        except InsuranceRequest.DoesNotExists:
            request_obj = None

        if request_obj:
            request_obj.payment_proof = paymentProof
            request_obj.status = InsuranceRequest.PROCESS
            request_obj.save()
            sendPaymentMail(request_obj.client.username, request_obj, paymentProof)

            return Response({
                'detail': 'Recibo de pago enviado exitosamente'
            }, status=status.HTTP_200_OK)

        return Response({
                'detail': 'Ha ocurrido un error al recibir tu recibo de pago'
            }, status=status.HTTP_400_BAD_REQUEST)

class DomiRequestInsuranceCreate(APIView):
    """ Crea los datos de recogida del dinero cuando van a pagar en efectivo
    """

    permission_classes = (AllowAny,)

    def post(self, request, format=None):

        request_id = request.data['request_id']
        address = request.data['address']
        pickup_date = request.data['pickup_date']
        pickup_time = request.data['pickup_time']

        request_insurance = InsuranceRequest.objects.filter(id=request_id).first()

        try:
            domi = DomiRequestInsurer(
                request_insurance = request_insurance,
                address=address,
                pickup_date=pickup_date,
                pickup_time=pickup_time,
            )
            domi.save()

            # sendMail(username, property1, property2, oldSoat, optionId, fisico)
            return Response({'detail':'Solicitud creada exitosamente'}, status=status.HTTP_201_CREATED)

        except Exception as e:
            print ("EXception")
            print (e)
            return Response({'detail':'Ha ocurrido un error al crear la solicitud'}, status=status.HTTP_400_BAD_REQUEST)

class AnswerCreate(APIView):
    """Api para crear las respuestas de un formulario de un paciente
    """

    permission_classes = (IsAuthenticated,)
    authentication_classes = (JSONWebTokenAuthentication,)

    def post(self, request):
        data = request.data.copy()
        insurance = data[0]
        data.pop(0)
        insurance = Insurance.objects.get(pk=insurance['insurance'])
        user = request.user
        insurance_request = InsuranceRequest(
            insurance=insurance,
            client=user,
            request_date=datetime.now()
        )
        insurance_request.save()

        for answer in data:
            metadata = Metadata.objects.get(pk=answer['question'])
            if metadata.field_type != Metadata.SELECT:
                instance = Answer.objects.create(
                    insurance_request=insurance_request,
                    field=metadata,
                    value=answer['value']
                )
            elif metadata.field_type == Metadata.IMAGE:
                instance = Answer.objects.create(
                    insurance_request=insurance_request,
                    field=metadata,
                    image=answer['value']
                )
            else:
                choice = MetadataChoices.objects.get(pk=answer['value'])
                instance = Answer.objects.create(
                    insurance_request=insurance_request,
                    field=metadata,
                    value=str(choice.value),
                    choice=answer['value']
                )

        response = {'request_code': insurance_request.request_code}

        return Response(response, status=status.HTTP_201_CREATED)

def sendMail(username, doc1, doc2, oldSoat, optionId, fisico):
    subject, from_email = 'Solicitud de seguro', settings.EMAIL_USER
    text_content = 'Acabas de recibir una nueva solicitud de seguro del usuario {}. El codigo del seguro es {}. Este usuario ha \
    elegido la option de enviar el SOAT en fisico como {}. '.format(str(username), optionId, fisico)
    msg = EmailMultiAlternatives(subject, text_content, from_email, ['yo@diegonaranjo.co', 'alukas@segurosquality.com', 'soporte@segurosquality.com'])
    msg.attach('propiedad1.jpeg', doc1.read(), 'image/jpeg')
    msg.attach('propiedad2.jpeg', doc2.read(), 'image/jpeg')
    if oldSoat:
        msg.attach('Soat-antetior.jpeg', oldSoat.read(), 'image/jpeg')
    msg.send()

def sendPaymentMail(username, request_obj, img):
    subject, from_email = 'Recibo de pago SOAT', settings.EMAIL_USER
    text_content = 'Acabas de recibir el recibo de pago del SOAT con codigo {} del usuario {}. '.format(request_obj.request_code, request_obj.client.username)
    msg = EmailMultiAlternatives(subject, text_content, from_email, ['yo@diegonaranjo.co', 'alukas@segurosquality.com', 'soporte@segurosquality.com'])
    msg.attach('ReciboPago.jpeg', img.read(), 'image/jpeg')
    msg.send()


import requests, re, pprint, json
import numpy as np
from bs4 import BeautifulSoup

with open("ZCL.html") as zip_code:
    zip_code_dat = BeautifulSoup(zip_code, "html.parser") #opens static file
    zip_code_dat_parsed = zip_code_dat.find_all(class_="inner_table")[0] #pulls table with zipcodes
    zip_code_dat_parsed = zip_code_dat_parsed.find_all("td")[5:] #finds tr (table row) tags. ignores first 5 b/c they hold useless info.
    zip_code_txt = [c.text for c in zip_code_dat_parsed] #turns trs into text
    #print(len(zip_code_txt))
    zip_code_txt = [zip_code_txt[x:x+4] for x in range(0, len(zip_code_txt), 4)] #creates a list of lists. each list is split in to [zip code, city, county, random text]
    #print(zip_code_txt)
    #print(zip_code_text_list)
    city_to_county = {}
    counrty_to_city = {}
    for line in zip_code_txt: #iterates thru list
        #print(line)
        city = line[1]
        county = line[2]
        #print(city)
        #print(county)
        #print(city, county)

    #there's two dictionaries being created here. one is for use in another folder (city_to_county), the other is to check for cities that
    #exist in 2+ counties. this is to deal with a double-counting error in 'Opioid_parser.py'

        if county not in city_to_county: #if not in dictionary, add to dictionary
            city_to_county[county] = []
            city_to_county[county].append(city)
        else: #if already in dictionary, append
            if city not in city_to_county[county]:
                city_to_county[county].append(city)

        if city not in counrty_to_city: #if not in dictionary, add to dictionary
            counrty_to_city[city] = []
            counrty_to_city[city].append(county)
        else: #if already in dictionary, append
            if county not in counrty_to_city[city]:
                counrty_to_city[city].append(county)
    #pprint.pprint(city_to_county)
    #pprint.pprint(counrty_to_city)
    json.dump(city_to_county, open("zip_codes.json", "w")) #write to json file


import logging
import os
from typing import Optional

from cookiecutter.main import cookiecutter

import seedfarmer.errors
from seedfarmer import config

_logger: logging.Logger = logging.getLogger(__name__)

def create_module_dir(
    module_name: str, group_name: Optional[str], module_type: Optional[str], template_url: Optional[str]
) -> None:
    """Initializes a directory for a new module.

    Creates a new directory that contains files that will aid in setting up a development environment

    Parameters
    ----------
    group_name : Optional[str],
        Nmae of the group where the module will reside. If group is a nested dir, use `/` as a delimiter
    module_name : str
        Name of the module. The initialization will include project files pulled from the template_url
    module_type: Optional[str]
        They type of code the module deploys with, adding more boilerplate code
        -- only cdkv2 is supported here
    template_url : Optional[List[str]]
        A URL, for example a Github repo, that is or contains templating for the initialization
    """
    module_root = os.path.join(config.OPS_ROOT, "modules")
    module_path = os.path.join(module_root, module_name)
    output_dir = module_root

    if group_name:
        module_path = os.path.join(module_root, group_name, module_name)
        output_dir = os.path.join(module_root, group_name)

        if not os.path.exists(output_dir):
            _logger.info("Creating group dir: %s", output_dir)
            os.makedirs(output_dir)

    if os.path.exists(module_path):
        raise seedfarmer.errors.InvalidPathError(f"The module {module_name} already exists under {output_dir}.")

    if template_url == "https://github.com/awslabs/seed-farmer.git":
        checkout_branch = "init-module-cdkv2" if module_type == "cdkv2" else "init-module"

    _logger.info("New module will be created in the following dir: %s", output_dir)
    cookiecutter(
        template=template_url,
        checkout=checkout_branch,
        no_input=True,
        extra_context={"project_name": module_name, "module_name": module_name},
        output_dir=output_dir,
    )

def create_project(template_url: Optional[str]) -> None:
    """Initializes a new project directory.

    Creates a new directory that contains files that will aid in setting up a development environment

    Parameters
    ----------
    project_name : str
        Name of the project. The initialization will include project files pulled from the template_url
    template_url : Optional[List[str]]
        A URL, for example a Github repo, that is or contains templating for the initialization
    """

    checkout_branch = "init-project" if template_url == "https://github.com/awslabs/seed-farmer.git" else None
    _logger.info(" New project will be created in the following dir: %s", os.path.join(config.OPS_ROOT, config.PROJECT))
    cookiecutter(
        template=template_url,
        checkout=checkout_branch,
        no_input=True,
        extra_context={"project_name": config.PROJECT},
        output_dir=config.OPS_ROOT,
    )
    os.replace(
        os.path.join(config.OPS_ROOT, config.CONFIG_FILE),
        os.path.join(config.OPS_ROOT, config.PROJECT, config.CONFIG_FILE),
    )


import pandas as pd
import numpy as np
from pathlib import Path
import os
from tqdm import tqdm

def cal_joint_angles(file=None, destination_path=None, encoder_type='CNN'):
    """
    Calculate the joint angles between body parts and returns a dataframe of distances
    for only one animal

    Expects to read h5 files. Doesn't work for csv files

    Parameters
    ----------
    file: the path to file with the tracked points.
    destination_path: the path to save tracked points.
    encoder_type: either SAE, VAE or ANT
    """

    destination_name = Path(file).stem
    destination_name = destination_name[:destination_name.find(encoder_type)]

    name = 'animal_1'
    if destination_path is None:
        destination_path = str(Path(file).parent)
        destination_file = f"{destination_path}/{destination_name}{name}_joint_angles.h5"
    else:
        destination_file = f"{destination_path}/{destination_name}{name}_joint_angles.h5"

    if os.path.exists(destination_file):
        return

    print(file)
    # Read file
    with pd.HDFStore(file) as df:
        animal_key = df.keys()[0][1:]
        h5 = df[animal_key]

    # Get scorer
    scorer = h5.columns.get_level_values('scorer').unique().item()

    # Get list of individuals
    individuals = h5.columns.get_level_values('individuals').unique()

    center = 'midBody'

    vector1_bodyparts = ['Nose', 'midHip']

    vector2_bodyparts = ['leftEar', 'betweenEars', 'rightEar', 'rightMidWaist',
                         'leftMidWaist', 'leftHip', 'rightHip', 'tailStart']

    animals_angles = pd.DataFrame()
    for it, ind in enumerate(individuals):

        ind_h5 = h5[scorer][ind]

        for vec1 in vector1_bodyparts:
            vector1 = ind_h5[center].sub(ind_h5[vec1])
            vector1 = vector1 / np.linalg.norm(vector1, axis=1)[:, np.newaxis]
            for vec2 in vector2_bodyparts:
                vector2 = ind_h5[center].sub(ind_h5[vec2])
                vector2 = vector2 / np.linalg.norm(vector2, axis=1)[:, np.newaxis]
                angle_data = np.zeros((vector2.shape[0], 1))
                for t in tqdm(range(vector2.shape[0])):
                    v1 = vector1.iloc[t]
                    v2 = vector2.iloc[t]
                    angle = np.arccos(np.dot(v1, v2)) * 180 / np.pi
                    angle_data[t] = angle
                animals_angles[f'{vec1}_{vec2}'] = angle_data.flatten()

        name = f'animal_{it + 1}'
        if destination_path is None:
            destination_path = file.rsplit('/', 1)[0]
            destination_file = f"{destination_path}/{destination_name}{name}_joint_angles.h5"
        else:
            destination_file = f"{destination_path}/{destination_name}{name}_joint_angles.h5"

        print(destination_file)
        animals_angles.fillna(method='bfill', inplace=True)
        animals_angles.to_hdf(destination_file, animal_key)

    return animals_angles


from django.urls import path, include
from . import views
from django.conf import settings
from django.conf.urls.static import static

app_name = "framesharings"

urlpatterns = [
    path('frame',views.frame,name='frame'),
    path('framedetail/<int:id>',views.framedetail,name='framedetail'),
    path('framecreate', views.framecreate,name='framecreate'),
    path('frameupdate/<int:id>', views.frameupdate,name='frameupdate'),
    path('framedelete/<int:id>',views.framedelete,name='framedelete'),
    path('like_ajax/', views.like_ajax, name = 'like_ajax'),
    path('save_ajax/', views.save_ajax, name = 'save_ajax'),
] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) + static(settings.MEDIA_URL, document_root = settings.MEDIA_ROOT)