# spot_price_model.py
#
# Contributing authors: Daniel Clark, Cameron Craddock, 2015

'''
This module contains functions which return the total duration, cost,
expected failure time, expected wait time, and probability of failure
for a job submission to an AWS EC2 SPOT cluster
'''

# Calculate costs with the EBS model
def calc_s3_model_costs(run_time, wait_time, node_cost, first_iter_time,
                         num_jobs, num_nodes, jobs_per, av_zone,
                         in_gb, out_gb, up_rate, down_rate):
    '''
    Function to take results from the simulate_market function and
    calculate total costs and runtimes with data transfer and storage
    included for the S3-based storage model

    Parameters
    ----------
    run_time : float
        the total number of seconds all of the nodes were up running
    wait_time : float
        the total number of seconds spent waiting for the spot price
        to come down below bid
    node_cost : float
        the per-node running, or instance, cost
    first_iter_time : float
        the number of seconds the first job iteration took to complete;
        this is used to model when outputs begin downloading from EC2
    num_jobs : integer
        total number of jobs to run to complete job submission
    num_nodes : integer
        the number of nodes that the cluster uses to run job submission
    jobs_per : integer
        the number of jobs to run per node
    av_zone : string
        the AWS EC2 availability zone (sub-region) to get spot history
        from
    in_gb : float
        the total amount of input data for a particular job (in GB)
    out_gb : float
        the total amount of output data from a particular job (in GB)
    up_rate : float
        the average upload rate to transfer data to EC2 (in Mb/s)

    Returns
    -------
    total_cost : float
        the total amount of dollars the job submission cost
    instance_cost : float
        cost for running the instances, includes master and all slave
        nodes
    storage_cost : float
        cost associated with data storage
    xfer_cost : float
        cost associated with data transfer (out only as in is free)
    total_time : float
        the total amount of seconds for the entire job submission to
        complete
    run_time : float
        returns the input parameter run_time for convenience
    wait_time : float
        returns the input parameter wait_time for convenience
    xfer_up_time : float
        the amount of time it took to transfer the input data up to
        the master node (seconds)
    xfer_down_time : float
        the amount of time it took to transfer all of the output data
        from the master node (seconds)
    '''

    # Import packages
    import numpy as np

    # Init variables
    cpac_ami_gb = 30
    secs_per_avg_month = (365/12.0)*24*3600
    num_iter = np.ceil(num_jobs/float((jobs_per*num_nodes)))
    # Upload speed from instance to S3 (mbps/8 bits/1000 MB in 1 GB)
    upl_to_s3_gbps = 100/8.0/1000.0

    # Get total execution time as sum of running and waiting times
    exec_time = run_time + wait_time

    ### Master runtime + storage time (EBS) + xfer time ###
    up_gb_per_sec = up_rate/8.0/1000.0
    xfer_up_time = num_jobs*(in_gb/up_gb_per_sec)

    # Get the number of jobs ran through n-1 iterations
    num_jobs_n1 = ((num_iter-1)*num_nodes*jobs_per)
    # Calculate how long it takes to transfer up all of the output data
    # *This is modeled as happening as the jobs finish during the full run
    # Sequential uploads at full bandwidth
    # (could be simultaneous uploads at 1/jobs_per bandwidth - same upl time)
    s3_upl_time_n1 = num_jobs_n1*(out_gb/upl_to_s3_gbps)
    exec_time_n1 = exec_time - first_iter_time
    residual_jobs = num_jobs - num_jobs_n1

    # End of upload of master node
    # 1) Transfer data up
    # 2) First iteration of running
    # 3) Rest of iteration runtimes, or all but last iteration's upload times..
    # ....whichever is greater
    # 4) Last iteration's (residual) upload times
    master_up_time = xfer_up_time + \
                     first_iter_time + \
                     np.max([exec_time_n1, s3_upl_time_n1]) + \
                     residual_jobs*(out_gb/upl_to_s3_gbps)

    # Get total transfer up time
    s3_upl_time = s3_upl_time_n1 + residual_jobs*(out_gb/upl_to_s3_gbps)

    ### Get EBS storage costs ###
    ebs_ssd = get_ec2_costs(av_zone, 'ssd')
    # EBS should only need to hold per-iteration jobs (rm complete as they go)
    ebs_nfs_gb = in_gb*num_jobs + num_nodes*jobs_per*out_gb
    # Get GB-months
    master_gb_months = (ebs_nfs_gb+cpac_ami_gb)*\
                       (3600.0*np.ceil(master_up_time/3600.0)/secs_per_avg_month)
    nodes_gb_months = num_nodes*cpac_ami_gb*\
                      (3600.0*np.ceil(run_time/3600.0)/secs_per_avg_month)
    ebs_storage_cost = ebs_ssd*(master_gb_months + nodes_gb_months)

    ### Get S3 storage/xfer/requests costs ###
    # Return pricing for each storage, transfer, and requests
    # Assuming out_gb stored on S3 for month, up to 1TB/month price
    # S3 storage
    stor_gb_month = get_s3_costs(av_zone, 'stor')
    down_gb_per_sec = down_rate/8.0/1000.0
    secs_to_download_s3 = (num_jobs*out_gb)/down_gb_per_sec
    s3_storage_cost = stor_gb_month*\
                      (secs_to_download_s3/secs_per_avg_month)*\
                      (num_jobs*out_gb)
    # S3 download requests
    # How many input/output files get generated per job
    # Assume ~2 for input
    in_ratio = 2
    # Assume ~50 for outupt
    out_ratio = 50
    req_prices = get_s3_costs(av_zone, 'req')
    s3_req_cost = req_prices['get']*((out_ratio*num_jobs)/10000.0)
    # S3 download transfer
    xfer_per_gb = get_s3_costs(av_zone, 'xfer')
    s3_xfer_cost = xfer_per_gb*(num_jobs*out_gb)

    # Sum of storage, transfer, and requests
    s3_cost = s3_storage_cost + s3_req_cost + s3_xfer_cost

    ### Get computation costs ###
    # Add in master node costs - asssumed to be on-demand, t2.small
    master_on_demand = get_ec2_costs(av_zone, 'master')
    master_cost = master_on_demand*np.ceil(master_up_time/3600.0)
    # Get cumulative cost for running N nodes per iteration
    nodes_cost = node_cost*num_nodes
    # Sum master and slave nodes for total computation cost
    instance_cost = master_cost + nodes_cost

    ### Data transfer in costs are free ###

    ### Total cost ###
    total_cost = instance_cost + ebs_storage_cost + s3_cost
    ### Total time ###
    total_time = master_up_time + secs_to_download_s3

    # Return data frame entries
    return total_cost, instance_cost, ebs_storage_cost, s3_cost, \
           s3_storage_cost, s3_req_cost, s3_xfer_cost, \
           total_time, run_time, wait_time, \
           xfer_up_time, s3_upl_time, secs_to_download_s3

# Calculate costs with the EBS model
def calc_ebs_model_costs(run_time, wait_time, node_cost, first_iter_time,
                         num_jobs, num_nodes, jobs_per, av_zone,
                         in_gb, out_gb, out_gb_dl, up_rate, down_rate):
    '''
    Function to take results from the simulate_market function and
    calculate total costs and runtimes with data transfer and storage
    included for the EBS/EC2 storage model

    Parameters
    ----------
    run_time : float
        the total number of seconds all of the nodes were up running
    wait_time : float
        the total number of seconds spent waiting for the spot price
        to come down below bid
    node_cost : float
        the per-node running, or instance, cost
    first_iter_time : float
        the number of seconds the first job iteration took to complete;
        this is used to model when outputs begin downloading from EC2
    num_jobs : integer
        total number of jobs to run to complete job submission
    num_nodes : integer
        the number of nodes that the cluster uses to run job submission
    jobs_per : integer
        the number of jobs to run per node
    av_zone : string
        the AWS EC2 availability zone (sub-region) to get spot history
        from
    in_gb : float
        the total amount of input data for a particular job (in GB)
    out_gb : float
        the total amount of output data from a particular job (in GB)
    out_gb_dl : float
        the total amount of output data to download from EC2 (in GB)
    up_rate : float
        the average upload rate to transfer data to EC2 (in Mb/s)
    down_rate : float
        the average download rate to transfer data from EC2 (in Mb/s)

    Returns
    -------
    total_cost : float
        the total amount of dollars the job submission cost
    instance_cost : float
        cost for running the instances, includes master and all slave
        nodes
    storage_cost : float
        cost associated with data storage
    xfer_cost : float
        cost associated with data transfer (out only as in is free)
    total_time : float
        the total amount of seconds for the entire job submission to
        complete
    run_time : float
        returns the input parameter run_time for convenience
    wait_time : float
        returns the input parameter wait_time for convenience
    xfer_up_time : float
        the amount of time it took to transfer the input data up to
        the master node (seconds)
    xfer_down_time : float
        the amount of time it took to transfer all of the output data
        from the master node (seconds)
    '''

    # Import packages
    import numpy as np

    # Init variables
    cpac_ami_gb = 30
    secs_per_avg_month = (365/12.0)*24*3600
    num_iter = np.ceil(num_jobs/float((jobs_per*num_nodes)))

    # Get total execution time as sum of running and waiting times
    exec_time = run_time + wait_time

    ### Master runtime + storage time (EBS) + xfer time ###
    up_gb_per_sec = up_rate/8.0/1000.0
    down_gb_per_sec = down_rate/8.0/1000.0
    xfer_up_time = num_jobs*(in_gb/up_gb_per_sec)

    # Get the number of jobs ran through n-1 iterations
    num_jobs_n1 = ((num_iter-1)*num_nodes*jobs_per)
    # Calculate how long it takes to transfer down all of the jobs
    # *This is modeled as happening as the jobs finish during the full run
    xfer_down_time_n1 = num_jobs_n1*(out_gb_dl/down_gb_per_sec)
    exec_time_n1 = exec_time - first_iter_time
    residual_jobs = num_jobs - num_jobs_n1

    # End of download of master node
    master_up_time = xfer_up_time + \
                     first_iter_time + \
                     np.max([exec_time_n1, xfer_down_time_n1]) + \
                     residual_jobs*(out_gb_dl/down_gb_per_sec)

    # Get total transfer down time
    xfer_down_time = xfer_down_time_n1 + residual_jobs*(out_gb_dl/down_gb_per_sec)

    ### Get EBS storage costs ###
    ebs_ssd = get_ec2_costs(av_zone, 'ssd')
    ebs_nfs_gb = num_jobs*(in_gb+out_gb)

    # Get GB-months
    master_gb_months = (ebs_nfs_gb+cpac_ami_gb)*\
            (3600.0*np.ceil(master_up_time/3600.0)/secs_per_avg_month)
    nodes_gb_months = num_nodes*cpac_ami_gb*\
            (3600.0*np.ceil(run_time/3600.0)/secs_per_avg_month)
    storage_cost = ebs_ssd*(master_gb_months + nodes_gb_months)

    ### Get computation costs ###
    # Add in master node costs - asssumed to be on-demand, t2.small
    master_on_demand = get_ec2_costs(av_zone, 'master')
    master_cost = master_on_demand*np.ceil(master_up_time/3600.0)
    # Get cumulative cost for running N nodes per iteration
    nodes_cost = node_cost*num_nodes
    # Sum master and slave nodes for total computation cost
    instance_cost = master_cost + nodes_cost

    ### Data transfer costs ###
    ec2_xfer_out = get_ec2_costs(av_zone, 'xfer')
    xfer_cost = ec2_xfer_out*(num_jobs*out_gb_dl)

    ### Total cost ###
    total_cost = instance_cost + storage_cost + xfer_cost
    ### Total time ###
    total_time = master_up_time

    # Return data frame entries
    return total_cost, instance_cost, storage_cost, xfer_cost, \
           total_time, run_time, wait_time, \
           xfer_up_time, xfer_down_time

# Calculate cost over interval
def calculate_cost(start_time, uptime_seconds, interp_history,
                   interrupted=False):
    '''
    Function to calculate the runtime spot cost associated with an
    instance's spot history

    Parameters
    ----------
    start_time : datetime.datetime object
        start time of the spot history to calculate from
    uptime_seconds : float
        the number of seconds that the instance was running for
    interp_history : pandas.Series object
        the interpolated (second-resolution) spot history series,
        where the index is a timestamp and the values are prices
    interrupted : boolean (optional), default=False
        indicator of whether the instance was interrupted before
        terminating or not

    Returns
    -------
    total_cost : float
        the total amount of $ that the instance cost
    '''

    # Import packages
    import numpy as np
    import pandas as pd
    import datetime

    # Init variables
    pay_periods = np.ceil(uptime_seconds/3600.0)
    end_time = start_time + datetime.timedelta(seconds=uptime_seconds)
    hour_seq = pd.date_range(start_time, periods=pay_periods, freq='H')
    hourly_series = interp_history[hour_seq]

    # Sum up all but last hour price if interrupted
    total_cost = hourly_series[:-1].sum()

    # If the user ran residual time without interrupt after last hour
    if not interrupted:
        total_cost += hourly_series[-1]

    # Return the total cost
    return total_cost

# Lookup tables for pricing for EBS
def get_ec2_costs(av_zone, cost_type):
    '''
    Function to retrieve costs associated with using EC2

    Data transfer into EC2 from the internet is free (all regions)

    Region name - Location mapping
    ------------------------------
    us-east-1 - N. Virginia
    us-west-1 - N. California
    us-west-2 - Oregon
    eu-west-1 - Ireland
    eu-central-1 - Frankfurt
    ap-southeast-1 - Singapore
    ap-southeast-2 - Sydney
    ap-northeast-1 - Tokyo
    sa-east-1 - Sao Paulo

    References
    ----------
    EC2 pricing: http://aws.amazon.com/ec2/pricing/
    EBS pricing: http://aws.amazon.com/ebs/pricing/

    Parameters
    ----------
    av_zone : string
        the availability zone to get the pricing info for
    cost_type : string
        the type of cost to extract, supported types include:
        'ssd' - ssd EC2 EBS storage
        'mag' - magnetic EC2 EBS storage
        'xfer' - download from EC2 transfer costs
        'master' - t2.small hourly on-demand cost

    Returns
    -------
    ec2_cost : float
        the $ amount per unit of the cost type of interest
    '''

    # Init variables
    region = av_zone[:-1]

    # EBS general purpose storage
    ebs_gen_purp = {'us-east-1' : 0.1,
                    'us-west-1' : 0.12,
                    'us-west-2' : 0.1,
                    'eu-west-1' : 0.11,
                    'eu-central-1' : 0.119,
                    'ap-southeast-1' : 0.12,
                    'ap-southeast-2' : 0.12,
                    'ap-northeast-1' : 0.12,
                    'sa-east-1' : 0.19}

    # EBS magnetic storage (plus same price per million I/O requests)
    ebs_mag = {'us-east-1' : 0.05,
               'us-west-1' : 0.08,
               'us-west-2' : 0.05,
               'eu-west-1' : 0.055,
               'eu-central-1' : 0.059,
               'ap-southeast-1' : 0.08,
               'ap-southeast-2' : 0.08,
               'ap-northeast-1' : 0.08,
               'sa-east-1' : 0.12}

    # Get costs for downloading data from EC2 (up to 10TB/month), in $/GB
    ec2_xfer_out = {'us-east-1' : 0.09,
                    'us-west-1' : 0.09,
                    'us-west-2' : 0.09,
                    'eu-west-1' : 0.09,
                    'eu-central-1' : 0.09,
                    'ap-southeast-1' : 0.12,
                    'ap-southeast-2' : 0.14,
                    'ap-northeast-1' : 0.14,
                    'sa-east-1' : 0.25}

    # Get $/hour costs of running t2.small master node
    ec2_t2_small = {'us-east-1' : 0.026,
                    'us-west-1' : 0.034,
                    'us-west-2' : 0.026,
                    'eu-west-1' : 0.028,
                    'eu-central-1' : 0.030,
                    'ap-southeast-1' : 0.040,
                    'ap-southeast-2' : 0.040,
                    'ap-northeast-1' : 0.040,
                    'sa-east-1' : 0.054}

    # Select costs type
    if cost_type == 'ssd':
        ec2_cost = ebs_gen_purp[region]
    elif cost_type == 'mag':
        ec2_cost = ebs_mag[region]
    elif cost_type == 'xfer':
        ec2_cost = ec2_xfer_out[region]
    elif cost_type == 'master':
        ec2_cost = ec2_t2_small[region]
    else:
        err_msg = 'cost_type argument does not support %s' % cost_type
        raise Exception(err_msg)

    # Return the ec2 cost
    return ec2_cost

# Lookup tables for pricing for EBS
def get_s3_costs(av_zone, cost_type):
    '''
    Data transfer to S3 from anywhere is free (all regions)
    Data transfer from S3 to EC2 in same region is free (all regions)

    Parameters
    ----------
    av_zone : string
        the availability zone to get the pricing info for
    cost_type : string
        the type of cost to extract, supported types include:
        'stor' - S3 standard storage per-GB-month
        'xfer' - download from S3 transfer costs
        'req' - bucket key requests cost

    Returns
    -------
    s3_price : float
        the $ amount per unit of the cost type of interest

    References
    ----------
    S3 pricing: http://aws.amazon.com/s3/pricing/
    '''

    # Init variables
    region = av_zone[:-1]

    # S3 standard storage (up to 1TB/month), units of $/GB-month
    s3_stor = {'us-east-1' : 0.03,
               'us-west-1' : 0.033,
               'us-west-2' : 0.03,
               'eu-west-1' : 0.03,
               'eu-central-1' : 0.0324,
               'ap-southeast-1' : 0.03,
               'ap-southeast-2' : 0.033,
               'ap-northeast-1' : 0.033,
               'sa-east-1' : 0.0408}

    # Get costs for downloading data from S3 (up to 10TB/month), in $/GB
    s3_xfer_out = {'us-east-1' : 0.09,
                   'us-west-1' : 0.09,
                   'us-west-2' : 0.09,
                   'eu-west-1' : 0.09,
                   'eu-central-1' : 0.09,
                   'ap-southeast-1' : 0.12,
                   'ap-southeast-2' : 0.14,
                   'ap-northeast-1' : 0.14,
                   'sa-east-1' : 0.25}

    # Request pricing (put vs get on S3)
    # Put - $/1,000 reqs (upload)
    # Get - $/10,000 reqs (download)
    s3_reqs = {'us-east-1' : {'put' : 0.005, 'get' : 0.004},
               'us-west-1' : {'put' : 0.0055, 'get' : 0.0044},
               'us-west-2' : {'put' : 0.005, 'get' : 0.004},
               'eu-west-1' : {'put' : 0.005, 'get' : 0.004},
               'eu-central-1' : {'put' : 0.0054, 'get' : 0.0043},
               'ap-southeast-1' : {'put' : 0.005, 'get' : 0.004},
               'ap-southeast-2' : {'put' : 0.0055, 'get' : 0.0044},
               'ap-northeast-1' : {'put' : 0.0047, 'get' : 0.0037},
               'sa-east-1' : {'put' : 0.007, 'get' : 0.0056}}

    # Select costs type
    if cost_type == 'stor':
        s3_cost = s3_stor[region]
    elif cost_type == 'xfer':
        s3_cost = s3_xfer_out[region]
    elif cost_type == 'req':
        s3_cost = s3_reqs[region]
    else:
        err_msg = 'cost_type argument does not support %s' % cost_type
        raise Exception(err_msg)

    # Return the ec2 cost
    return s3_cost

# Find how often a number of jobs fails and its total cost
def simulate_market(start_time, spot_history, interp_history,
                    proc_time, num_iter, bid_price):
    '''
    Function to find the total execution time, cost, and number of interrupts
    for a given job submission and bid price

    Parameters
    ----------
    start_time : pandas.tslib.Timestamp object
        the time to start the simulation from
    spot_history : pandas.core.series.Series object
        timeseries of spot prices recorded from AWS
    interp_history : pandas.core.series.Series object
        interpolated spot price history to one second resolution
    proc_time : float
         the time to process one job iteration (in seconds)
    num_iter : integer
        the number of job iterations or waves to run
    bid_price : float
        the spot bid price in dollars per hour

    Returns
    -------
    total_runtime : float
        the total number of seconds all of the nodes were up running
    total_wait : float
        the total number of seconds spent waiting for the spot price
        to come down below bid
    total_cost : float
        the per-node running, or instance, cost
    num_interrupts : integer
        the number of times the job submission was interrupted
    first_iter_time : float
        the number of seconds the first job iteration took to complete;
        this is used to model when outputs begin downloading from EC2
    '''

    # Import packages
    import datetime
    import numpy as np

    # Init variables
    total_runtime = 0
    total_wait = 0
    total_cost = 0
    num_interrupts = 0

    # Get first spot history start time
    # Note: np.argmax returns first occurence of True
    start_idx = np.argmax(spot_history.index >= start_time)
    spot_history_start = spot_history.index[start_idx]

    # Init remaining rumtime
    remaining_runtime = proc_time*num_iter

    # Init 1st iteration time
    first_iter_flg = False
    first_iter_time = 0

    # While there is time left running
    while remaining_runtime > 0:
        # Get only current spot history
        curr_spot_history = spot_history[spot_history_start:]

        # Get instance-boot-up price (per hour price at start time)
        start_price = interp_history[start_time]

        # If start price is greater than bid, interrupt immediately
        if start_price >= bid_price:
            uptime_seconds = 0
            interrupt_time = start_time
        # Otherwise, start instances
        else:
            # Find interrupts
            interrupt_condition = curr_spot_history >= bid_price

            # Find timestamp where first interrupt occured
            if np.any(interrupt_condition):
                interrupt_time = min(curr_spot_history.index[interrupt_condition])
            else:
                interrupt_time = spot_history.index[-1]

            # Calculate total up-and-running time
            uptime = interrupt_time - start_time
            uptime_seconds = uptime.total_seconds()

        # See if job completed
        if uptime_seconds > remaining_runtime:

            # Add remaining runtime to execution time
            total_runtime += remaining_runtime

            # Add remaining runtime costs
            total_cost += calculate_cost(start_time, remaining_runtime, interp_history)

            # Clear remaining run time
            remaining_runtime = 0

        # Job suspended until price returns below bid
        else:
            # Increment it as an interrupt if we were running before hand
            if uptime_seconds > 0:
                num_interrupts += 1

            # Add up time to execution time
            total_runtime += uptime_seconds

            # Add to cost
            total_cost += calculate_cost(start_time, uptime_seconds,
                                         interp_history, interrupted=True)

            # Subtract uptime from remaining runtime
            # Add back remainder of time that was interrupted (need to re-do)
            remaining_runtime = (remaining_runtime-uptime_seconds) + \
                                (uptime_seconds % proc_time)

            # Find next time the history dips below the bid price
            curr_spot_history = spot_history[interrupt_time:]
            start_condition = curr_spot_history < bid_price
            start_times = curr_spot_history.index[start_condition]

            # If we've run out of processing time
            if len(start_times) == 0 or \
               start_times[0] == spot_history.index[-1]:
                err_msg = 'Job submission could not complete due to too many ' \
                          'interrupts or starting too recently'
                raise Exception(err_msg)

            # Get the next time we can start
            spot_history_start = min(start_times)
            # and set as the next spot time
            start_time = spot_history_start

            # And increment wait time by (next start)-(this interrupt)
            total_wait += (start_time - interrupt_time).total_seconds()

        # Check to see if we're setting first iter
        if not first_iter_flg:
            # If we were up for at least one amount of processing time
            if total_runtime >= proc_time:
                first_iter_time = proc_time + total_wait
                first_iter_flag = True

    # Return results
    return total_runtime, total_wait, total_cost, num_interrupts, first_iter_time

# Return a time series from csv data frame
def spothistory_from_dataframe(csv_file, instance_type, product, av_zone):
    '''
    Function to return a time and price series from a csv dataframe

    Parameters
    ----------
    csv_file : string
        file path to dataframe csv file
    instance_type : string
        the type of instance to gather spot history for
    product : string
        the type of OS product to gather spot history for
    av_zone : string
        the availability zone to get the pricing info for

    Returns
    -------
    spot_history : pandas.Series
        time series of spot history prices indexed by timestamp
    '''

    # Import packages
    import dateutil.parser
    import pandas as pd

    # Init variables

    # Load data frame
    print 'Loading dataframe %s...' % csv_file
    data_frame = pd.DataFrame.from_csv(csv_file)

    # Get only entries we care about
    df_bool = (data_frame['Instance type'] == instance_type) & \
              (data_frame['Product'] == product) & \
              (data_frame['Availability zone'] == av_zone)
    df_subset = data_frame[df_bool]

    # Get spot histories from data frame with str timestamps
    spot_history = df_subset.set_index('Timestamp')['Spot price']
    spot_history = spot_history.sort_index()

    # Get new histories with datetime timestamps
    datetimes = [dateutil.parser.parse(ts) for ts in spot_history.index]
    spot_history = pd.Series(spot_history.values, datetimes)

    # Return time series
    return spot_history

# Main routine
def main(sim_dir, proc_time, num_jobs, jobs_per, in_gb, out_gb, out_gb_dl,
         up_rate, down_rate, bid_ratio, instance_type, av_zone, product,
         csv_file=None):
    '''
    Function to calculate spot instance run statistics based on job
    submission parameters; this function will save the statistics and
    specific spot history in csv dataframes to execution directory

    Parameters
    ----------
    sim_dir : string
        base directory where to create the availability zone folders
        for storing the simulation results
    proc_time : float
        the number of minutes a single job of interest takes to run
    num_jobs : integer
        total number of jobs to run to complete job submission
    jobs_per : integer
        the number of jobs to run per node
    in_gb : float
        the total amount of input data for a particular job (in GB)
    out_gb : float
        the total amount of output data from a particular job (in GB)
    out_gb_dl : float
        the total amount of output data to download from EC2 (in GB)
    up_rate : float
        the average upload rate to transfer data to EC2 (in Mb/s)
    down_rate : float
        the average download rate to transfer data from EC2 (in Mb/s)
    bid_ratio : float
        the ratio to average spot history price to set the bid price to
    instance_type : string
        type of instance to run the jobs on and to get spot history for
    av_zone : string
        the AWS EC2 availability zone (sub-region) to get spot history
        from
    product : string
        the type of operating system product to get spot history for
    csv_file : string (optional), default is None
        the filepath to a csv dataframe to get spot history from;
        if not specified, the function will just get the most recent 90
        days worth of spot price history

    Returns
    -------
    spot_history : pd.DataFrame object
        in addition to saving this as './spot_history.csv' the
        dataframe can also be returned as an object in memory
    stat_df : pd.DataFrame object
        in addition to saving this as './<info>_stats.csv' the
        dataframe can also be returned as an object in memory
    '''

    # Import packages
    import dateutil
    import logging
    import numpy as np
    import os
    import pandas as pd
    import yaml

    # Import local packages
    import utils
    from record_spot_price import return_spot_history

    # Init variables
    proc_time *= 60.0
    num_nodes = min(np.ceil(float(num_jobs)/jobs_per), 20)

    # Init simulation market results dataframe
    sim_df_cols = ['start_time', 'spot_hist_csv', 'proc_time', 'num_datasets',
                   'jobs_per_node', 'num_jobs_iter', 'bid_ratio', 'bid_price',
                   'median_history', 'mean_history', 'stdev_history',
                   'compute_time', 'wait_time', 'per_node_cost',
                   'num_interrupts', 'first_iter_time']
    sim_df = pd.DataFrame(columns=sim_df_cols)

    # Init full run stats data frame
    stat_df_cols = ['Total cost', 'Instance cost', 'Storage cost', 'Tranfer cost',
                    'Total time', 'Run time', 'Wait time',
                    'Upload time', 'Download time']
    stat_df = pd.DataFrame(columns=stat_df_cols)

    # Set up logger
    base_dir = os.path.join(sim_dir, av_zone)
    if not os.path.exists(base_dir):
        try:
            os.makedirs(base_dir)
        except OSError as exc:
            print 'Found av zone directory %s, continuing...' % av_zone
    log_path = os.path.join(base_dir, '%s_%d-jobs_%.3f-bid.log' % \
                            (instance_type, num_jobs, bid_ratio))
    stat_log = utils.setup_logger('stat_log', log_path, logging.INFO, to_screen=True)

    # Check to see if simulation was already run (sim csv file exists)
    sim_csv = os.path.join(base_dir, '%s_%d-jobs_%.3f-bid_sim.csv' % \
                           (instance_type, num_jobs, bid_ratio))
    if os.path.exists(sim_csv):
        stat_log.info('Simulation file %s already exists, skipping...' % sim_csv)
        return

    # Calculate number of iterations given run configuration
    # Round up and assume that we're waiting for all jobs to finish
    # before terminating nodes
    num_iter = np.ceil(num_jobs/float((jobs_per*num_nodes)))
    stat_log.info('With %d jobs, %d nodes, and %d jobs running per node...\n' \
                  'job iterations: %d' % (num_jobs, num_nodes, jobs_per, num_iter))

    # Get spot price history, if we're getting it from a csv dataframe
    if csv_file:
        # Parse dataframe to form history
        spot_history = spothistory_from_dataframe(csv_file, instance_type,
                                                  product, av_zone)
        # Get rid of any duplicated timestamps
        spot_history = spot_history.groupby(spot_history.index).first()

    # Otherwise, just grab latest 90 days
    else:
        sh_list = return_spot_history(None, instance_type, product, av_zone)

        # Convert history into just timepoints and prices list of tuples
        timestamps = [dateutil.parser.parse(sh.timestamp) for sh in sh_list]
        prices = [sh.price for sh in sh_list]

        # Use pandas timeseries and sort in oldest -> newest
        spot_history = pd.Series(prices, timestamps)
        spot_history = spot_history.sort_index()

        # Write spot history to disk
        sh_csv = os.path.join(os.getcwd(), 'spot_history.csv')
        spot_history.to_csv(sh_csv)

    # Get interpolated times per second (forward fill)
    interp_seq = pd.date_range(spot_history.index[0], spot_history.index[-1],
                               freq='S')
    interp_history = spot_history.reindex(interp_seq)
    interp_history = interp_history.fillna(method='ffill')

    # Init simulation time series
    sim_seq = pd.date_range(interp_seq[0], interp_seq[-1], freq='20T')
    sim_series = interp_history[sim_seq]

    # Init loop variables
    sim_idx = 0
    sim_length = len(sim_series)
    beg_time = spot_history.index[0]
    end_time = spot_history.index[-1]
    time_needed = num_iter*(proc_time)

    # Get bid price
    spot_history_avg = interp_history.mean()
    bid_price = bid_ratio*spot_history_avg
    stat_log.info('Spot history average is $%.3f, bid ratio of %.3fx sets ' \
                  'bid to $%.3f' % (spot_history_avg, bid_ratio, bid_price))

    # Iterate through the interpolated timeseries
    for start_time, start_price in sim_series.iteritems():
        # First see if there's enough time to run jobs
        time_window = (end_time-start_time).total_seconds()
        if time_needed > time_window:
            stat_log.info('Total runtime exceeds time window, ending simulation...')

        # Simulate running job and get stats from that start time
        try:
            run_time, wait_time, pernode_cost, num_interrupts, first_iter_time = \
                    simulate_market(start_time, spot_history, interp_history,
                                    proc_time, num_iter, bid_price)
        except Exception as exc:
            stat_log.info('Could not run full simulation because of:\n%s' % exc)
            continue

        # Write simulate market output to dataframe
        sim_df.loc[sim_idx] = [start_time, csv_file, proc_time, num_jobs,
                               jobs_per, num_iter, bid_ratio, bid_price,
                               np.mean(spot_history), np.median(spot_history),
                               np.std(spot_history), run_time, wait_time,
                               pernode_cost, num_interrupts, first_iter_time]

        # Get complete time and costs from spot market simulation parameters
        total_cost, instance_cost, stor_cost, xfer_cost, \
        total_time, run_time, wait_time, \
        xfer_up_time, xfer_down_time = \
                calc_ebs_model_costs(run_time, wait_time, pernode_cost,
                                     first_iter_time, num_jobs, num_nodes,
                                     jobs_per, av_zone, in_gb, out_gb,
                                     out_gb_dl, up_rate, down_rate)

        # Add to output dataframe
        stat_df.loc[sim_idx] = [total_cost, instance_cost, stor_cost, xfer_cost,
                                total_time/60.0, run_time/60.0, wait_time/60.0,
                                xfer_up_time/60.0, xfer_down_time/60.0]

        # Print stats
        stat_log.info('Total cost: $%.3f' % total_cost)
        stat_log.info('Total time (minutes): %.3f' % (total_time/60.0))
        stat_log.info('run time (minutes): %.3f' % (run_time/60.0))
        stat_log.info('per-node cost: $%.3f' % pernode_cost)
        stat_log.info('number of interrupts: %d' % num_interrupts)
        stat_log.info('wait time (minutes): %.3f' % (wait_time/60.0))

        sim_idx += 1
        utils.print_loop_status(sim_idx, sim_length)

    # Add configuration parameters to dataframe
    sim_df['av_zone'] = av_zone
    sim_df['in_gb'] = in_gb
    sim_df['out_gb'] = out_gb
    sim_df['out_gb_dl'] = out_gb_dl
    sim_df['up_rate'] = up_rate
    sim_df['down_rate'] = down_rate

    # Write simulation dataframe to disk
    sim_df.to_csv(sim_csv)

    # Write stats dataframe to disk
    stat_csv = os.path.join(base_dir, '%s_%d-jobs_%.3f-bid_stats.csv' % \
                           (instance_type, num_jobs, bid_ratio))
    stat_df.to_csv(stat_csv)

    # Write parameters yaml to disk
    params_yml = os.path.join(base_dir, '%s_%d-jobs_%.3f-bid_params.yml' % \
                              (instance_type, num_jobs, bid_ratio))

    params = {'proc_time' : proc_time,
              'num_jobs' : num_jobs,
              'jobs_per' : jobs_per,
              'in_gb' : in_gb,
              'out_gb' : out_gb,
              'out_gb_dl' : out_gb_dl,
              'up_rate' : up_rate,
              'down_rate' : down_rate,
              'bid_ratio' : bid_ratio,
              'instance_type' : instance_type,
              'av_zone' : av_zone,
              'product' : product,
              'csv_file' : csv_file}

    with open(params_yml, 'w') as y_file:
        y_file.write(yaml.dump(params))

    # Give simulation-wide statistics
    interrupt_avg = sim_df['num_interrupts'].mean()
    time_avg = stat_df['Total time'].mean()
    cost_avg = stat_df['Total cost'].mean()

    # Print simulation statistics
    stat_log.info('\n' + 72*'-')
    stat_log.info('Submission of %d job iterations, ' \
                  'each takes %.3f mins to run:' % (num_iter, proc_time/60.0))
    stat_log.info('Average spot history price for %s in %s\n' \
                  'between %s and %s is: $%.3f' % \
                  (instance_type, av_zone, beg_time, end_time, spot_history_avg))
    stat_log.info('Spot ratio of %.3fx the average price set bid to $%.3f' % \
                  (bid_ratio, bid_price))
    stat_log.info('Average total time (mins): %f' % time_avg)
    stat_log.info('Average total cost: $%.3f' % cost_avg)
    stat_log.info('Average number of interruptions: %.3f' % interrupt_avg)
    stat_log.info(72*'-' + '\n')

    # Return dataframes
    return spot_history, sim_df, stat_df

# Make executable
if __name__ == '__main__':

    # Import packages
    import argparse

    # Init argparser
    parser = argparse.ArgumentParser(description=__doc__)

    # Required arguments
    parser.add_argument('-t', '--proc_time', nargs=1, required=True,
                        type=float, help='Processing time for one job to complete '\
                             'successfully (in minutes)')
    parser.add_argument('-j', '--num_jobs', nargs=1, required=True, type=int,
                        help='Total number of jobs to run in AWS')
    parser.add_argument('-per', '--jobs_per', nargs=1, required=True, type=int,
                        help='Number of jobs to run per node')
    parser.add_argument('-ig', '--in_gb', nargs=1, required=True, type=float,
                        help='Input size per job in GB to upload to EC2')
    parser.add_argument('-og', '--out_gb', nargs=1, required=True, type=float,
                        help='Output size per job in GB to store in EC2 EBS')
    parser.add_argument('-od', '--out_gb_dl', nargs=1, required=True, type=float,
                        help='Output size per job in GB to download from EC2')
    parser.add_argument('-ur', '--up_rate', nargs=1, required=True, type=float,
                        help='Upload rate in Mb/sec')
    parser.add_argument('-dr', '--down_rate', nargs=1, required=True, type=float,
                        help='Download rate in Mb/sec')
    parser.add_argument('-b', '--bid_ratio', nargs=1, required=True,
                        type=float, help='Bid ratio to average spot price')
    parser.add_argument('-i', '--instance_type', nargs=1, required=True,
                        type=str, help='Instance type to run the jobs on')

    # Optional arguments
    parser.add_argument('-z', '--av_zone', nargs=1, required=False, type=str,
                        help='Specify availability zone of interest; ' \
                             'default is \'us-east-1b\'')
    parser.add_argument('-p', '--product', nargs=1, required=False, type=str,
                        help='Specify product of interest; ' \
                             'default is \'Linux/Unix\'')
    parser.add_argument('-c', '--csv_file', nargs=1, required=False, type=str,
                        help='Specify csv dataframe to parse histories')

    # Parse arguments
    args = parser.parse_args()

    # Init variables
    # Pipeline config params
    proc_time = args.proc_time[0]
    num_jobs = args.num_jobs[0]
    # Cluster config params
    jobs_per = args.jobs_per[0]
    instance_type = args.instance_type[0]
    # Data in/out to store EBS
    in_gb = args.in_gb[0]
    out_gb = args.out_gb[0]
    # Data transfer
    out_gb_dl = args.out_gb_dl[0]
    up_rate = args.up_rate[0]
    down_rate = args.down_rate[0]
    # Bid ratio
    bid_ratio = args.bid_ratio[0]

    # Try and init optional arguments
    try:
        av_zone = args.av_zone[0]
    except TypeError as exc:
        av_zone = 'us-east-1b'
        print 'No availability zone argument found, using %s...' % av_zone
    try:
        product = args.product[0]
    except TypeError as exc:
        product = 'Linux/UNIX'
        print 'No product argument found, using %s...' % product
    try:
        csv_file = args.csv_file[0]
    except TypeError as exc:
        csv_file = None
        print 'No csv dataframe specified, only using latest history...'

    # Call main routine
    main(proc_time, num_jobs, jobs_per, in_gb, out_gb, out_gb_dl,
         up_rate, down_rate, bid_ratio, instance_type, av_zone, product,
         csv_file)


import numpy as np
from imutils.video import VideoStream
import io
import zlib
import requests
from requests.exceptions import HTTPError

import base64

import dhash
from PIL import Image
import time
import datetime
import json
from project.objCountByTimer import ObjCountByTimer
from multiprocessing import Process
from multiprocessing import Queue
import logging
import cv2

# initialize the list of class labels MobileNet SSD was trained to
# detect, then generate a set of bounding box colors for each class
CLASSES = ["background", "aeroplane", "bicycle", "bird", "boat",
           "bottle", "bus", "car", "cat", "chair", "cow", "diningtable",
           "dog", "horse", "motorbike", "person", "pottedplant", "sheep",
           "sofa", "train", "tvmonitor"]
LOOKED1 = {"person": [], "car": []}

subject_of_interes = ["person", "car"]
DNN_TARGET_MYRIAD = False

HASH_DELTA = 43  # bigger number  more precise object's count
DIMENSION_X = 300
DIMENSION_Y = 300
piCameraResolution = (640, 480)  # (1024,768) #(640,480)  #(1920,1080) #(1080,720) # (1296,972)
piCameraRate = 16
NUMBER_OF_THREADS = 1

logger = logging.getLogger('logger')

logger.setLevel(logging.INFO)

class Detection:
    def __init__(self, classify_server, confidence, model, video):
        self.confidence = confidence
        self.model = model
        self.video_url = video['url'] + '?' if '?' not in  video['url'] else '&' + 'stream=full&needlength'
        self.hashes = {}
        self.topic_label = 'no data'
        self.net = self.video_s = None
        self.cam = video['id']
        self.classify_server = classify_server

        for i in range(NUMBER_OF_THREADS):
            p_get_frame = Process(target=self.classify)

                                  #,output_queue))
            p_get_frame.daemon = False
            p_get_frame.start()
            logger.info("-------- Process was just started for video: {} --------".format(video))
            time.sleep(0.1 + 0.69/NUMBER_OF_THREADS)

    def classify(self):#, output_queue):
        if self.video_s is None:
            self.video_s = self.init_video_stream()
        while True:
            try:
                frame = self.read_video_stream(self.video_s)
                if frame is None: return
            except:
                print('Exception during reading stream by URL:{0}'.format(self.video_url))
                return
            result = call_classifier(self.classify_server, frame, self.cam, self.confidence, self.model)
            if(result is not None and 'rectangles' in result and len(result['rectangles'])>0):
                logger.info("result: {}".format(result))
            """           if 'rectangles' in result :
                                 # Draw rectangles

                                for rec in result['rectangles']:
                                    x = rec.get('startX') - 25
                                    y = rec.get('startY') - 25
                                    cv2.rectangle(frame, (x,y), (rec.get('endX') + 25, rec.get('endY') + 25), (255, 0, 0), 1)
                                    cv2.putText(frame, rec.get('text') , (x, y), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2)
                            cv2.putText(frame, result.get('topic_label', 'None')  , (10, 23), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2)
                            #output_queue.put(frame)
                            #output_queue.put_nowait(frame)
            """

    def init_video_stream(self):
        if 'picam' == self.video_url:
            video_s = VideoStream(usePiCamera=True, resolution=piCameraResolution, framerate=piCameraRate).start()
            time.sleep(2.0)

        else:
            # grab the frame from the threaded video stream
            video_s = cv2.VideoCapture(self.video_url)
        return video_s

    def read_video_stream(self, video_s):
        # print("Read video stream .. " + self.video_url)
        if 'picam' == self.video_url:
            frame = video_s.read()
        else:
            flag, frame = video_s.read()
            if not flag:
                video_s = cv2.VideoCapture(self.video_url)
                flag, frame = video_s.read()
                return frame
        return frame

def call_classifier(classify_server, frame, cam, confidence, model):
    _,data = cv2.imencode('.jpg', frame) # frame.tolist() #  , _ , _ = compress_nparr(frame)
    parameters = {'cam': cam, 'confidence': confidence, 'model': model}
    data = {'params': parameters, 'array': base64.b64encode(data).decode('utf-8')}
    jsonResponse = None
   # logger.info("------------ call_classifier just called for cam: {} -------".format(cam))
    try:
        response = requests.post(url=classify_server,
                            json=data)
                            #headers={'Content-Type': 'text/json'})
        response.raise_for_status()
        # access JSOn content
        jsonResponse = response.json()
        #print("Entire JSON response")
        #print(jsonResponse)

    except HTTPError as http_err:
        print('HTTP error occurred: {0}'.format(http_err))
    except Exception as err:
        print('Other error occurred: {0}')#.format(err))
    #logger.debug("call_classifier jsonResponse for cam: {} {}".format(cam,jsonResponse ))
    return jsonResponse

def compress_nparr(nparr):
    """
    Returns the given numpy array as compressed bytestring,
    the uncompressed and the compressed byte size.
    """
    bytestream = io.BytesIO()
    np.save(bytestream, nparr)
    uncompressed = bytestream.getvalue()
    compressed = zlib.compress(uncompressed)
    return compressed, len(uncompressed), len(compressed)

def uncompress_nparr(bytestring):
    """  Uncompress as bytestring into numpy array  """
    return np.load(io.BytesIO(zlib.decompress(bytestring)))

    #resp, _, _ = compress_nparr(data10)
    #return Response(response=resp, status=200,
    #                mimetype="application/octet_stream")


# Licensed under a 3-clause BSD style license - see LICENSE.rst
"""Model an astronomical observation for spectroscopic simulations.

An observation is usually initialized from a configuration used to create
a simulator and then accessible via its ``observation`` attribute, for example:

    >>> import specsim.simulator
    >>> simulator = specsim.simulator.Simulator('test')  # doctest: +IGNORE_OUTPUT
    >>> print(simulator.observation.exposure_time)
    1000.0 s

"""
from __future__ import print_function, division

import numpy as np

import astropy.units as u
import astropy.coordinates

import specsim.transform

class Observation(object):
    """Model the parameters describing a single spectroscopic observation.

    The following parameters can be changed after an Observation object has
    been created: exposure_time, exposure_start, pointing.

    Parameters
    ----------
    location : astropy.coordinates.EarthLocation
        Observatory location on the surface of the earth.
    exposure_time : astropy.units.Quantity
        Open shutter exposure time for this observation.
    exposure_start : astropy.time.Time
        Time when the shutter opens and the exposure starts.
    pointing : astropy.coordinates.SkyCoord
        Sky position where the telescope boresight is pointing during the
        observation.
    wavelength : astropy.units.Quantity
        Array of wavelength bin centers where the simulated spectrum is
        calculated, with units.
    pressure : astropy.units.Quantity
        Used to create an :func:`observing model
        <specsim.transform.create_observing_model>`.
    temperature : astropy.units.Quantity
        Used to create an :func:`observing model
        <specsim.transform.create_observing_model>`.
    relative_humidity : astropy.units.Quantity
        Used to create an :func:`observing model
        <specsim.transform.create_observing_model>`.
    """
    def __init__(self, location, exposure_time, exposure_start, pointing,
                 wavelength, pressure, temperature, relative_humidity):
        self.central_wavelength = 0.5 * (wavelength[0] + wavelength[-1])
        self._exposure_time = exposure_time
        self._location = location
        self._exposure_start = exposure_start
        self._pointing = pointing
        self._temperature = temperature
        self._pressure = pressure
        self._relative_humidity = relative_humidity
        self._update_model()

    def _update_model(self):
        """Update observing model. Triggered internally.
        """
        # Initialize an observing model at the middle of the exposure and
        # at the central wavelength of the simulation, i.e., ignore temporal
        # and chromatic variations (for now).
        exposure_midpoint = self.exposure_start + 0.5 * self.exposure_time
        self.observing_model = specsim.transform.create_observing_model(
            self.location, exposure_midpoint, self.central_wavelength,
            self.temperature, self.pressure, self.relative_humidity)
        # Calculate the boresight angles (fixed, since we do not consider
        # temporal or chromatic effects yet).
        self.boresight_altaz = specsim.transform.sky_to_altaz(
            self.pointing, self.observing_model)

    @property
    def location(self):
        """astropy.coordinates.EarthLocation: Observatory location.
        """
        return self._location

    @property
    def exposure_time(self):
        """astropy.units.Quantity: Exposure time for this observation.
        """
        return self._exposure_time

    @exposure_time.setter
    def exposure_time(self, value):
        try:
            self._exposure_time = value.to(u.s)
            self._update_model()
        except (AttributeError, u.UnitConversionError):
            raise ValueError('Invalid units for exposure_time.')

    @property
    def exposure_start(self):
        """astropy.time.Time: Time when the shutter opens.
        """
        return self._exposure_start

    @exposure_start.setter
    def exposure_start(self, value):
        self._exposure_start = value
        self._update_model()

    @property
    def pointing(self):
        """astropy.coordinates.SkyCoord: Telescope boresight pointing.
        """
        return self._pointing

    @pointing.setter
    def pointing(self, value):
        self._pointing = value
        self._update_model()

    @property
    def temperature(self):
        """astropy.units.Quantity: Air temperature at observatory.

        Used for atmospheric refraction model.
        """
        return self._temperature

    @property
    def pressure(self):
        """astropy.units.Quantity: Atmospheric pressure at observatory.

        Used for atmospheric refraction model.
        """
        return self._pressure

    @property
    def relative_humidity(self):
        """float: Relative humidity at observatory.

        Used for atmospheric refraction model.
        """
        return self._relative_humidity

    def locate_on_focal_plane(self, sky_position, instrument):
        """Transform a sky position into focal-plane coordinates.

        The input sky position can refer to one or more objects.

        Parameters
        ----------
        sky_position : astropy.coordinates.SkyCoord
            Position of one or more objects on the sky.
        instrument : specsim.instrument.Instrument
            Description of the instrument being simulated.

        Returns
        -------
        tuple
            Tuple (x, y) of focal-plane coordinates given as astropy
            quantities with length units.
        """
        altaz = specsim.transform.sky_to_altaz(
            sky_position, self.observing_model)
        # Calculate field angles relative to the boresight.
        x, y = specsim.transform.altaz_to_focalplane(
            altaz.alt, altaz.az,
            self.boresight_altaz.alt, self.boresight_altaz.az)
        # Convert field angles to focal-plane coordinates.
        angle = np.sqrt(x ** 2 + y ** 2)
        scale = np.zeros(angle.shape) * u.mm / u.deg
        nonzero = angle > 0
        try:
            scale[nonzero] = (
                instrument.field_angle_to_radius(angle[nonzero]) /
                angle[nonzero])
        except TypeError:
            if nonzero:
                scale = instrument.field_angle_to_radius(angle) / angle
        x = (x * scale).to(u.mm)
        y = (y * scale).to(u.mm)
        return x, y

def initialize(config):
    """Initialize the observation from configuration parameters.

    Parameters
    ----------
    config : :class:`specsim.config.Configuration`
        The configuration parameters to use.

    Returns
    -------
    Observation
        An initialized observation.
    """
    node = config.observation
    constants = config.get_constants(
        config.observation,
        ['exposure_time', 'temperature', 'relative_humidity'],
        optional_names=['pressure'])
    pressure = constants.get('pressure', None)
    location = specsim.transform.observatories[node.observatory]
    pointing = config.get_sky(node.pointing)
    exposure_start = config.get_timestamp(node.exposure_start)
    adjust_ha = getattr(node.exposure_start, 'adjust_to_hour_angle', None)
    if adjust_ha is not None:
        nominal_start = exposure_start
        point_radec = pointing.transform_to('icrs')
        hour_angle = astropy.coordinates.Angle(adjust_ha)
        try:
            lon = location.lon
        except AttributeError:
            # Required for astropy < 2.0
            lon = location.longitude
        exposure_start = specsim.transform.adjust_time_to_hour_angle(
            nominal_start, point_radec.ra, hour_angle, lon)
        # Put the requested HA at the middle of the exposure.
        exposure_start -= 0.5 * constants['exposure_time']

    obs = Observation(
        location, constants['exposure_time'], exposure_start, pointing,
        config.wavelength, pressure, constants['temperature'],
        constants['relative_humidity'])

    if config.verbose:
        print('Observatory located at (lat, lon, elev) = ',
              '({0:.1f}, {1:.1f}, {2:.1f}).'
              .format(*obs.location.to_geodetic()))
        point = obs.pointing.transform_to('icrs')
        print('Observing field center (ra, dec) = ({0}, {1}).'.format(
            point.ra, point.dec))
        print('Exposure start MJD {0:.3f}, duration {1}.'.format(
            obs.exposure_start.mjd, obs.exposure_time))
        if adjust_ha is not None:
            dt = exposure_start - nominal_start
            print('Adjusted by {0:+.3f} for HA {1}.'
                  .format(dt.to(u.hour), hour_angle))
        cond = obs.observing_model
        print('Conditions: pressure {0:.1f}, temperature {1:.1f}, RH {2}.'
              .format(cond.pressure, cond.temperature, cond.relative_humidity))
        altaz = obs.boresight_altaz
        print('Boresight (alt, az) = ({0:.1f}, {1:.1f}).'
              .format(altaz.alt, altaz.az))

    return obs


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
SimEVLA.py: This module simulates the slewing motion of the EVLA for use in
determining optimized schedules. Based largely on the equivalent code created
by NRAO SSS for use in the Observation Planning Tool. Small modifications have
been implemented to make it more scriptable.
"""

__author__ = "Seth Bruzewski"
__email__ = "bruzewskis@unm.edu"
__license__ = "GPL"

# Built in Modules
import logging

# Third Party Modules
import numpy as np
from astropy.coordinates import SkyCoord, Angle, EarthLocation
import astropy.units as u

class MotionSimulator(object):
    '''
    A pythonic class simulator of the EVLA slewing system, for implementation
    in scheduling codes.
    '''

    # Generate a logger
    __logger = logging.getLogger(__name__)

    # Motion Ranges
    __MIN_AZ = Angle(-85.0, unit='deg')
    __MAX_AZ = Angle(445.0, unit='deg')
    __MIN_EL = Angle(8.0, unit='deg')
    __MAX_EL = Angle(125.0, unit='deg')

    # Velocity and acceleration rates
    __VEL_AZ = 40.0/60 * u.Unit('deg/s')
    __VEL_EL = 20.0/60 * u.Unit('deg/s')
    __ACC_AZ = 1000 * u.Unit('deg/s^2')
    __ACC_EL = 1000 * u.Unit('deg/s^2')

    # Setup and Settling Time
    __MIN_SETUP_TIME = 0 * u.s
    __SETTLING_TIME = 7.0 * u.s

    # Default currentAz/El to midpoint of their ranges.
    __currentAz = Angle(225.0, unit='deg')
    __currentEl = Angle(35.0, unit='deg')

    # These variables keep track of what the new Az/El will be set to during
    # the moveTime calculation
    __newAz = None
    __newEl = None

    # Keep track of error codes generated during moveTo
    __errors = []

    def __init__(self, startaz=None, startel=None, wrap=None):
        '''
        Creates a EvlaTelescopeMotionSimulator with an initial position of
        `startAz` and `startEl` that is on wrap `wrap`.

        Args
        =======
        startaz (Angle) - Azimuth to initialize the telescope with as an
            Astropy Angle object. If not provided, telescope is initialized
            to default Azimuth, which is 225 degrees.
        startel (Angle) - Elevation to initialize the telescope with as an
            Astropy Angle object. If not provided, telescope is initialized
            to default Elevation, which is 35 degrees.
        wrap (string) - Preferred wrap of the telescope, may be CLOCKWISE,
            COUNTERCLOCKWISE, or NO_PREFERENCE

        Returns
        =======
        None

        Raises
        =======
        TypeError - Raised by class functions setCurrentAntennaAzimuth or
            setCurrentAntennaElevation when inputs are not the correct type
        ValueError - Raised by class functions setCurrentAntennaAzimuth or
            setCurrentAntennaElevation when inputs are outside of range
        '''

        self.setCurrentAntennaAzimuth(startaz, wrap)
        self.setCurrentAntennaElevation(startel)

    def moveTo(self, azEnd, elEnd, wrap=None, goOverTheTop=False):
        '''
        Returns the estimated move time from our current saved position to
        `az`, `el`.  For the very first calculation, a starting position
        due south at 65 degrees elevation is assumed.  The method takes into
        account antenna wrap by calculating 4 different move times and
        returning the smallest valid move time.  The 4 cases are as follows:

        1) The position we're moving to is on the COUNTERCLOCKWISE (left)
        wrap. We simple rotate the antenna to that position and see how long
        it takes. This is a valid option only if `wrap` is not CLOCKWISE
        and `goOverTheTop` is false.

        2) The position is on the CLOCKWISE (right) wrap.  Again, we calculate
        how long it will take to rotate the antenna to that position on that
        wrap. This is a valid option only if {@code wrap} is not
        COUNTERCLOCKWISE and `goOverTheTop` is false.

        3) We rotate the antenna 180 degrees and then go over the top to get
        to our target position. (i.e. we're subtracting 180 degrees from the
        AZ in case 1 above.)  This is only valid if `goOverTheTop' is
        true and the elevation is greater than 55 degrees (180 - MAX_EL).

        4) The same as 3 above, but we add 180 to the az instead of
        subtracting.

        This method also clears and refills the list of errors encountered
        while attempting this move.

        Args
        =======
        azEnd (Angle) - Target azimuth to slew to as an Astropy Angle. Will
            be checked against telescope motion ranges.
        elEnd (Angle) - Target elevation to slew to as an Astropy Angle. Will
            be checked against telescope motion ranges.
        wrap (string) - Preferred wrap of the telescope, may be CLOCKWISE,
            COUNTERCLOCKWISE, or NO_PREFERENCE
        goOverTheTop (bool) - Whether or not to allow the telescope to
            traverse in elevation beyond the zenith. For standard observing
            this is not advised.

        Returns
        =======
        tmin (Quantity) - The total slew time expressed as an Astropy Quantity
            object with units of time.

        Raises
        =======
        None
        '''

        # Check type of inputs
        if not isinstance(azEnd, Angle) or not isinstance(elEnd, Angle):
            self.__errors.append('IMPROPER_INPUT_TYPE')
            self.__logger.error('azEnd or elEnd are not Astropy Angles')
            return np.nan

        # Start up stuff
        azStart = self.__currentAz.copy()
        elStart = self.__currentEl.copy()

        # Keeping track of things
        self.__newAz = None
        self.__newEl = None
        self.__errors = []

        # Initialize min to something large
        tmin = 10 * u.day

        # Initialize wrap to a default if it was None
        w = 'NO_PREFERENCE' if wrap is None else wrap

        toAz = azStart
        toAzEnd = azEnd

        # Put the toAz on the left or COUNTERCLOCKWISE wrap.
        if toAz > self.__MIN_AZ + 360*u.deg:
            toAz -= 360*u.deg

        # Make sure the end position is on the same wrap!  This is checked
        # separately from the above because we aren't guaranteed that just
        # because we need to subtract 360 from the start pos that we need to
        # do so to the end as well.  This is mainly a concern for scans that
        # cross the 0d/360d boundary.
        if toAzEnd > self.__MIN_AZ + 360*u.deg:
            toAzEnd -= 360*u.deg

        toEl = elStart
        toElEnd = elEnd

        # Debug stuff
        coords = [toAz,toEl,toAzEnd,toElEnd]
        logstr = "Moving : ({0:.2f},{1:.2f}) --> ({2:.2f},{3:.2f})"
        self.__logger.debug(logstr.format(*[a.deg for a in coords]))

        # Whereas Longitude is always gaurunteed to be in bounds of our
        # motion, Latitude (elevation) is not, so we have to check that here
        # before going any further.  We only check the min, because the max a
        # Latitude can hold is 90 and our MAX_EL is 125 or so.
        if toEl < self.__MIN_EL:
            self.__errors.append('ELEVATION_OUT_OF_RANGE')
            toEl = self.__MIN_EL.copy()

        # We need to do the same check for the elevation END position too!
        if toElEnd < self.__MIN_EL:
            self.__errors.append('SCAN_END_ELEVATION_OUT_OF_RANGE')
            toElEnd = self.__MIN_EL.copy()

        # This checks to make sure that the if the user selected a CLOCKWISE
        # preference, but the source is in the region where the wraps do NOT
        # overlap, we still get a valid calculation.  The reason this comes
        # up is because for much of the logic below we're considering a wrap
        # to be a full 360 degrees when it is only actually 265.  So, a
        # source could be in the CLOCKWISE wrap (180 - 445 degrees) but be
        # missed in the logic below if it's position is less than 275 degrees.
        # (which is within the range (-85, -85 + 360))
        if w=='CLOCKWISE' and toAz > 85*u.deg:
            w = 'NO_PREFERENCE'

        # tmp variables for calculations
        tmpAz = toAz.copy()
        tmpAzEnd = toAzEnd.copy()
        tmpEl = toEl.copy()
        tmpElEnd = toElEnd.copy()

        # Counterclockwise or No Pref
        if w!='CLOCKWISE':
            # Case 3:
            if goOverTheTop:
                print('case3')
                tmpAz = toAz - 180*u.deg
                tmpAzEnd = toAzEnd - 180*u.deg
                tmpEl = 180*u.deg - toEl
                tmpElEnd = 180*u.deg - toElEnd

                if (tmpAz >= self.__MIN_AZ and tmpEl <= self.__MAX_EL):
                    tmin, updateBool = self.__updateMin(tmin,tmpAzEnd,tmpElEnd)
                    if updateBool:
                        self.__newAz = tmpAzEnd
                        self.__newEl = tmpElEnd

                        time_str = tmin.to('s').value
                        case_str = "Case 3 min = {0:.2f} s"
                        self.__logger.debug(case_str.format(time_str))

            # Case 1:
            else:
                tmin, updateBool = self.__updateMin(tmin, tmpAzEnd, tmpElEnd)
                if updateBool:
                    self.__newAz = tmpAzEnd
                    self.__newEl = tmpElEnd

                    time_str = tmin.to('s').value
                    case_str = "Case 1 min = {0:.2f} s"
                    self.__logger.debug(case_str.format(time_str))

        # Clockwise or No Pref
        if w!='COUNTERCLOCKWISE':
            # Case 4
            if goOverTheTop:
                print('case4')
                tmpAz = toAz + 180*u.deg
                tmpEl = 180*u.deg - toEl
                tmpAzEnd = toAzEnd + 180.0*u.deg
                tmpElEnd = 180*u.deg - toElEnd

                if tmpAz <= self.__MAX_AZ and tmpEl <= self.__MAX_EL:
                    tmin, updateBool = self.__updateMin(tmin,tmpAzEnd,tmpElEnd)
                    if updateBool:
                        self.__newAz = tmpAzEnd
                        self.__newEl = tmpElEnd

                        time_str = tmin.to('s').value
                        case_str = "Case 4 min = {0:.2f} s"
                        self.__logger.debug(case_str.format(time_str))

            # Case 2
            else:
                tmpAz = toAz + 360*u.deg
                tmpAzEnd = toAzEnd + 360*u.deg
                tmpEl = toEl.copy()
                tmpElEnd = toElEnd.copy()
                if tmpAz < self.__MAX_AZ:
                    tmin, updateBool = self.__updateMin(tmin,tmpAzEnd,tmpElEnd)
                    if updateBool:
                        self.__newAz = tmpAzEnd
                        self.__newEl = tmpElEnd

                        time_str = tmin.to('s').value
                        case_str = "Case 2 min = {0:.2f} s"
                        self.__logger.debug(case_str.format(time_str))

        if self.__newAz is None or self.__newEl is None:
            self.__errors.append('MOVE_NOT_POSSIBLE')
        else:
            # Azimuth checks
            if self.__MIN_AZ > self.__newAz:
                self.__logger.error("newAz < min: " + self.__newAz.to_string())
                self.__errors.append('SCAN_END_AZIMUTH_OUT_OF_RANGE')
                self.__currentAz = self.__MIN_AZ.copy()
            elif self.__MAX_AZ < self.__newAz:
                self.__logger.error("newAz > max: " + self.__newAz.to_string())
                self.__errors.append('SCAN_END_AZIMUTH_OUT_OF_RANGE')
                self.__currentAz = self.__MAX_AZ.copy()
            else:
                self.__currentAz = self.__newAz

            # Elevation checks
            if self.__MIN_EL > self.__newEl:
                self.__logger.error("newEl < min: " + self.__newEl.to_string())
                self.__errors.append('SCAN_END_ELEVATION_OUT_OF_RANGE')
                self.__currentEl = self.__MIN_EL.copy()
            elif self.__MAX_EL < self.__newEl:
                self.__logger.error("newEl > max: " + self.__newEl.to_string())
                self.__errors.append('SCAN_END_ELEVATION_OUT_OF_RANGE')
                self.__currentEl = self.__MAX_EL.copy()
            else:
                self.__currentEl = self.__newEl

            # Add any settling time necessary.
            tmin += self.__SETTLING_TIME

            # If the slew took less time than the min. time it takes to
            # prepare the antenna for observing, use that min. instead.
            if tmin < self.__MIN_SETUP_TIME:
                tmin = self.__MIN_SETUP_TIME.copy()

        # Decompose units just to be safe
        tmin = tmin.decompose()

        return tmin

    def getErrors(self):
        '''
        Returns a list of any errrors that may have occured during the lastest
        call to the moveTo method. These errors are stored as simple strings
        for the user to parse or use as they see fit.

        Args
        =======
        None

        Returns
        =======
        self.__errors.copy() (list) - List of errors generated by moveTo

        Raises
        =======
        None
        '''

        return self.__errors.copy()

    def getCurrentAntennaWrap(self):
        '''
        Returns which AntennaWrap this simulator is currently on using the
        following rules.  If the currentAz is greater than 275 degrees (MIN_AZ
        + 360) then we're in the CLOCKWISE (right) wrap.  Otherwise we're in
        the COUNTERCLOCKWISE (left) wrap.  Note: This could be changed to split
        the range in half: greater than 180 degrees is CLOCKWISE, the rest is
        COUNTERCLOCKWISE.  As it is now, you are only in the CLOCKWISE wrap if
        you are in the overlapping region.

        Args
        =======
        None

        Returns
        =======
        wrap (string) - The current antenna wrap expressed as a string.

        Raises
        =======
        None
        '''

        if self.__currentAz > 180*u.deg:
            return 'CLOCKWISE' #AntennaWrap.CLOCKWISE
        else:
            return 'COUNTERCLOCKWISE' #AntennaWrap.COUNTERCLOCKWISE

    def getCurrentAntennaAzimuth(self):
        '''
        Returns a copy of the current antenna azimuth. This value should be
        in the range -85 to 445 degrees.

        Args
        =======
        None

        Returns
        =======
        self.__currentAz.copy() (Angle) - The current antenna azimuth
            expressed as an Astropy Angle object.

        Raises
        =======
        None
        '''

        return self.__currentAz.copy()

    def setCurrentAntennaAzimuth(self, a, w=None):
        '''
        Sets the current Antenna Az to a clone of Angle equivalent to
        `a` at wrap `w` if `a` is within range.

        Args
        =======
        None

        Returns
        =======
        self.__currentAz.copy() (Angle) - The current antenna azimuth
            expressed as an Astropy Angle object.

        Raises
        =======
        TypeError - Raised when inputs are not the correct type
        ValueError - Raised when inputs are outside of range
        '''

        # Check input type
        if not a is None:
            if not isinstance(a, Angle):
                raise TypeError('Initializing Azimuth is not Astropy Angle')

        # Check that input is in range
        if w is None:
            if a is not None and self.__MIN_AZ < a < self.__MAX_AZ:
                self.__currentAz = a.copy()
            elif a is None:
                pass
            else:
                raise ValueError('Invalid Antenna Azimuth: '+str(a.deg))
        else:
            self.setCurrentAntennaAzimuth(self.toAntennaAzimuth(a, w))

    def getCurrentAntennaElevation(self):
        '''
        Returns a copy of the current antenna elevation. This value should be
        in the range 8 to 125 degrees.

        Args
        =======
        None

        Returns
        =======
        self.__currentEl.copy() (Angle) - The current antenna elevation
            expressed as an Astropy Angle object.

        Raises
        =======
        None
        '''

        return self.__currentEl.copy()

    def setCurrentAntennaElevation(self, a):
        '''
        Sets the current Antenna El to a clone of `a` if a is within
        range.

        Args
        =======
        None

        Returns
        =======
        self.__currentEl.copy() (Angle) - The current antenna azimuth
            expressed as an Astropy Angle object.

        Raises
        =======
        TypeError - Raised when inputs are not the correct type
        ValueError - Raised when inputs are outside of range
        '''

        # Check input type
        if not a is None:
            if not isinstance(a, Angle):
                raise TypeError('Initializing Elevation is not Astropy Angle')

        # Check that input is in range
        if a is not None and self.__MIN_EL < a < self.__MAX_EL:
            self.__currentEl = a.copy()
        elif a is None:
            pass
        else:
            raise ValueError('Invalid Antenna Elevation: '+str(a.deg))

    def toAntennaAzimuth(self, az, w):
        '''
        Returns an Angle in degrees between -85 and 445 degrees that
        represents `az` at AntennaWrap `w`.

        Args
        =======
        az (Angle) - Initial azimuth to be converted
        w (string) - Current wrap of the antenna

        Returns
        =======
        antennaAz (Angle) - Proper azimuth inside the antenna wrap

        Raises
        =======
        None
        '''

        antennaAz = None

        if az is not None:
            # az is 0 - 360 degrees
            if w=='CLOCKWISE':
                # 180 to 445 degrees
                if az < self.__MAX_AZ-360*u.deg:
                    az += 360*u.deg
            elif w=='COUNTERCLOCKWISE':
                # -85 to 180 degrees
                if az > self.__MIN_AZ+360*u.deg:
                    az -= 360*u.deg

            antennaAz = az

        return antennaAz

    def getAzimuthMinimum(self):
        '''
        Returns the minimum azimuth value for EVLA antenna pointings.

        Args
        =======
        None

        Returns
        =======
        self.__MIN_AZ.copy (Angle) - Telescope minimum azmiuth

        Raises
        =======
        None
        '''

        return self.__MIN_AZ.copy()

    def getAzimuthMaximum(self):
        '''
        Returns the maximum azimuth value for EVLA antenna pointings.

        Args
        =======
        None

        Returns
        =======
        self.__MAX_AZ.copy (Angle) - Telescope maximum azmiuth

        Raises
        =======
        None
        '''
        return self.__MAX_AZ.copy()

    def getAzimuthDefault(self):
        '''
        Returns the default azimuth value for EVLA antenna pointings.

        Args
        =======
        None

        Returns
        =======
        DEF_AZ (Angle) - Telescope default azimuth

        Raises
        =======
        None
        '''
        return Angle(225, unit='deg')

    def getElevationMinimum(self):
        '''
        Returns the minimum elevation value for EVLA antenna pointings.

        Args
        =======
        None

        Returns
        =======
        self.__MIN_EL.copy (Angle) - Telescope minimum elevation

        Raises
        =======
        None
        '''
        return self.__MIN_EL.copy()

    def getElevationMaximum(self):
        '''
        Returns the maximum elevation value for EVLA antenna pointings.

        Args
        =======
        None

        Returns
        =======
        self.__MAX_AZ.copy (Angle) - Telescope maximum elevation

        Raises
        =======
        None
        '''
        return self.__MIN_EL.copy()

    def getElevationDefault(self):
        '''
        Returns the default elevation value for EVLA antenna pointings.

        Args
        =======
        None

        Returns
        =======
        DEF_EL (Angle) - Telescope default elevation

        Raises
        =======
        None
        '''
        return Angle(35, unit='deg')

    def __updateMin(self, currentMin, az, el):
        '''
        This method updates the passed in `currentMin` variable with
        the minimum value of 'tmin' in the moveTo function, the time it takes
        to move from currentAz to az, and the time it takes to move from
        currentEl to el. If the currentMin is changed, we return true.

        Args
        =======
        currentMin (Quantity) - The current slew time minimum expressed as an
            Astropy Quantity object with units of time.
        az (Angle) - Target azimuth to slew to.
        el (Angle) - Target elevation to slew to.

        Returns
        =======
        newCurrentMin (Quantity) - The new current slew time minimum expressed
            as an Astropy Quantity object with units of time.
        update (bool) - Whether or not `tmin` will have been changed

        Raises
        =======
        None
        '''
        taz = self.__calcAzMoveTime(self.__currentAz, az)
        tel = self.__calcElMoveTime(self.__currentEl, el)

        # t is the larger of taz and tel
        t = max([taz,tel])

        if currentMin > t:
            return t, True
        else:
            return currentMin, False

    def __calcAzMoveTime(self, f, t):
        '''
        Calculates the move time from angle `f` to angle `t` in Azimuth.

        Args
        =======
        f (Angle) - Initial azimuth angle
        t (Angle) - Final azimuth angle

        Returns
        =======
        telSlewAz (Quantity) - Slew time, accounting for factors like the
            telescope slew acceleration, expressed as an Astropy Quantity
            with units of time.

        Raises
        =======
        None
        '''
        azd = abs(f-t)

        # Time it takes to reach full speed
        # Accounts for both acceleration & decceleration.
        timeAccAz = 2 * self.__VEL_AZ / self.__ACC_AZ

        # Distance it takes to reach full speed
        distAccAz = (self.__VEL_AZ * self.__VEL_AZ) / self.__ACC_AZ

        # If the antenna never reaches full speed, use this equation.
        if azd < distAccAz:
            telSlewAz = 2 * np.sqrt(azd / self.__ACC_AZ)
        # otherwise, use this equation.
        else:
            telSlewAz = timeAccAz + (azd - distAccAz) / self.__VEL_AZ

        return telSlewAz

    def __calcElMoveTime(self, f, t):
        '''
        Calculates the move time from angle `f` to angle `t` in Elevation.

        Args
        =======
        f (Angle) - Initial elevation angle
        t (Angle) - Final elevation angle

        Returns
        =======
        telSlewEl (Quantity) - Slew time, accounting for factors like the
            telescope slew acceleration, expressed as an Astropy Quantity
            with units of time.

        Raises
        =======
        None
        '''
        eld = abs(f-t)

        # Time it takes to reach full speed
        # Accounts for both acceleration & decceleration.
        timeAccEl = 2 * self.__VEL_EL / self.__ACC_EL

        # Distance it takes to reach full speed
        distAccEl = (self.__VEL_EL * self.__VEL_EL) / self.__ACC_EL

        if eld < distAccEl:
            # If the antenna never reaches full speed, use this equation.
            telSlewEl = 2 * np.sqrt(eld / self.__ACC_EL)
        else:
            # otherwise, use this equation.
            telSlewEl = timeAccEl + (eld - distAccEl) / self.__VEL_EL

        return telSlewEl

def EquitorialToHorizontal(ra, dec, lst):
    '''
    This function converts Equitorial coordinates (RA/DEC) to a horizontal
    coordinate system local to the VLA (AZ/EL). Ideally utilizes astropy
    Angles, but will convert where possible. Also requires a specific time
    given in LST. Note that Azimuth here is measured clockwise from the North.

    Args
    =======
    ra (various) - Either an Astropy Angle class, or a string/number in
        hourangle units. The latter will be converted to an Astropy Angle.
    dec (various) - Either an Astropy Angle class, or a string/number in
        degree units. The latter will be converted to an Astropy Angle.
    lst (string) - The local sidereal time at the VLA to be converted for,
        ideally in format HH:MM:SS.SS. This will be converted into hourangle.

    Returns
    =======
    Az (Angle) - The resulting Azimuth generated from the inputs. Measured
        clockwise from due North.
    Alt (Angle) - The resulting Altitude/Elevation generated from the inputs.
        Measured from the horizon.

    Raises
    =======
    UnitTypeError - Raised if the inputs cannot be coerced into Astropy Angles
    '''

    # Check type of inputs for safety
    if not isinstance(ra, Angle):
        ra = Angle(ra, unit=u.hourangle)
    if not isinstance(dec, Angle):
        dec = Angle(dec, unit=u.deg)
    if not isinstance(lst, Angle):
        dec = Angle(lst, unit=u.hourangle)

    # Let's just assume we're at the VLA
    vla = EarthLocation.of_site('VLA')
    ha = lst - ra
    lat = vla.lat

    # Do Trigonometry
    cosAlt_sinAz = -np.cos(dec)*np.sin(ha)
    cosAlt_cosAz = np.sin(dec)*np.cos(lat) - np.cos(dec)*np.cos(ha)*np.sin(lat)
    sinAlt = np.sin(dec)*np.sin(lat) + np.cos(dec)*np.cos(ha)*np.cos(lat)

    Az = Angle(np.arctan2(cosAlt_sinAz,cosAlt_cosAz)).wrap_at('360d')
    Alt = Angle(np.arcsin(sinAlt))

    return Az, Alt

def PointToPointTime(start_sc, end_sc, lst):
    '''
    Calculate the time to slew between two coordinates on the sky. Note that
    this function will not check if the slew is possible, instead allowing
    MotionSimulator to throw errors where it likes. This is largely useful
    for diagnostic checking.

    Args
    =======
    start_sc (SkyCoord) - The coordinates of the first point on the sky, where
        the telescope will begin its slew. Must be an Astropy SkyCoord object
    end_sc (SkyCoord) - The coordinates of the second point on the sky, where
        the telescope will end its slew. Must be an Astropy SkyCoord object
    lst (string) - The local sidereal time at the VLA to be converted for,
        ideally in format HH:MM:SS.SS. This will be converted into hourangle.

    Returns
    =======
    slewtime (Quantity) - The time it takes to slew between the input points.
        This is returned as an Astropy Quantity with units of time, that way it
        can be easily worked with.

    Raises
    =======
    TypeError - Raised when start_sc or end_sc are not SkyCoords
    '''
    # Check type of inputs
    if not isinstance(start_sc, SkyCoord):
        raise TypeError('param \'start_sc\' is not a Astropy SkyCoord')
    if not isinstance(end_sc, SkyCoord):
        raise TypeError('param \'end_sc\' is not a Astropy SkyCoord')

    # Basic logging
    mlogger = logging.getLogger(__name__)

    # Translate Start Point
    start_az, start_el = EquitorialToHorizontal(start_sc.ra,start_sc.dec,lst)

    # Translate End Point
    end_az, end_el = EquitorialToHorizontal(end_sc.ra,end_sc.dec,lst)

    # Nice Debug
    coo_fmt = '(ra:{0:.2f},dec:{1:.2f}) --> (az:{2:.2f},el:{3:.2f})'
    coo1 = [start_sc.ra.deg, start_sc.dec.deg, start_az.deg, start_el.deg]
    coo2 = [end_sc.ra.deg, end_sc.dec.deg, end_az.deg, end_el.deg]
    mlogger.debug('Converting : '+coo_fmt.format(*coo1))
    mlogger.debug('Converting : '+coo_fmt.format(*coo2))

    # Generate Simulator
    sim = MotionSimulator(start_az, start_el)
    slewtime = sim.moveTo(end_az, end_el)
    mlogger.debug('Took {0:.2f} seconds to slew'.format(slewtime.value))

    # Errors
    err = sim.getErrors()
    mlogger.debug('Found {:d} errors'.format(len(err)))
    for e in err:
        mlogger.error(e)

    return slewtime

if __name__=='__main__':
    '''
    Psuedomain function for simple tests to give some idea how this works

    Args
    =======
    None

    Returns
    =======
    None

    Raises
    =======
    None
    '''

    # Test 1
    print('Test 1:')
    s1 = SkyCoord(21.915*u.hourangle, -30*u.deg)
    s2 = SkyCoord(21.915*u.hourangle, 30*u.deg)
    sc2str = lambda sc : sc.to_string('hmsdms')
    print('Slewing from {0:s} to {1:s}'.format(sc2str(s1), sc2str(s2)))

    lst0 = Angle('21:54:55', unit='hourangle')
    st = PointToPointTime(s1, s2, lst0)
    st_sec = st.to('s').value
    st_dst = s1.separation(s2).deg
    fmt_str = 'Took {0:.2f} seconds to slew {1:.2f} degrees'
    print(fmt_str.format(st_sec, st_dst))

    # Test 2
    print('\nTest 2:')
    sim = MotionSimulator()
    totaltime = 0*u.s
    for _ in range(10):
        new_az = Angle(np.random.uniform(0,360) * u.deg)
        new_el = Angle(np.random.uniform(10,90) * u.deg)
        slewt = sim.moveTo(new_az, new_el)
        totaltime += slewt

    t2_st = totaltime.to('s').value
    print('Slewed to 10 points over {0:.2f} seconds'.format(t2_st))

import numpy,import astropy,from astropy


import os
import sys
import math
import numpy as np
from fvcore.nn import c2_xavier_fill

BASE_DIR = os.path.dirname(
    os.path.dirname(os.path.dirname(os.path.dirname(
        os.path.abspath(__file__)))))
sys.path.append(BASE_DIR)

from public.path import pretrained_models_path

from public.detection.models.backbone import ResNetBackbone
from public.detection.models.head import FCOSClsRegCntHead
from public.detection.models.anchor import FCOSPositions

import torch
import torch.nn as nn
import torch.nn.functional as F

__all__ = [
    'resnet18_yolof',
    'resnet34_yolof',
    'resnet50_yolof',
    'resnet101_yolof',
    'resnet152_yolof',
]

model_urls = {
    'resnet18_yolof':
    '/home/jovyan/data-vol-polefs-1/pretrained_models/resnet/resnet18-epoch100-acc70.316.pth',
    'resnet34_yolof':
    'empty',
    'resnet50_yolof':
    '/home/jovyan/data-vol-polefs-1/pretrained_models/resnet/resnet50-epoch100-acc76.512.pth',
    'resnet101_yolof':
    '/home/jovyan/data-vol-polefs-1/pretrained_models/resnet/resnet101-epoch100-acc77.724.pth',
    'resnet152_yolof':
    'empty',
}

class Bottleneck(nn.Module):

    def __init__(self,
                 in_channels: int = 512,
                 mid_channels: int = 128,
                 dilation: int = 1):
        super(Bottleneck, self).__init__()
        self.conv1 = nn.Sequential(
            nn.Conv2d(in_channels, mid_channels, kernel_size=1, padding=0),
            nn.BatchNorm2d(mid_channels),
            nn.ReLU(inplace=True)
        )
        self.conv2 = nn.Sequential(
            nn.Conv2d(mid_channels, mid_channels,
                      kernel_size=3, padding=dilation, dilation=dilation),
            nn.BatchNorm2d(mid_channels),
            nn.ReLU(inplace=True)
        )
        self.conv3 = nn.Sequential(
            nn.Conv2d(mid_channels, in_channels, kernel_size=1, padding=0),
            nn.BatchNorm2d(in_channels),
            nn.ReLU(inplace=True)
        )

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        identity = x
        out = self.conv1(x)
        out = self.conv2(out)
        out = self.conv3(out)
        out = out + identity
        return out

class DilatedEncoder(nn.Module):
    """
    Dilated Encoder for YOLOF.
    This module contains two types of components:
        - the original FPN lateral convolution layer and fpn convolution layer,
          which are 1x1 conv + 3x3 conv
        - the dilated residual block
    """

    def __init__(self, in_channels):
        super(DilatedEncoder, self).__init__()
        # fmt: off
        self.in_channels = in_channels
        self.encoder_channels = 512
        self.block_mid_channels = 128
        self.num_residual_blocks = 4
        self.block_dilations = [2, 4, 6, 8]
        # fmt: on

        # init
        self._init_layers()
        self._init_weight()

    def _init_layers(self):
        self.lateral_conv = nn.Conv2d(self.in_channels,
                                      self.encoder_channels,
                                      kernel_size=1)
        self.lateral_norm = nn.BatchNorm2d(self.encoder_channels)
        self.fpn_conv = nn.Conv2d(self.encoder_channels,
                                  self.encoder_channels,
                                  kernel_size=3,
                                  padding=1)
        self.fpn_norm = nn.BatchNorm2d(self.encoder_channels)
        encoder_blocks = []
        for i in range(self.num_residual_blocks):
            dilation = self.block_dilations[i]
            encoder_blocks.append(
                Bottleneck(
                    self.encoder_channels,
                    self.block_mid_channels,
                    dilation=dilation
                )
            )
        self.dilated_encoder_blocks = nn.Sequential(*encoder_blocks)

    def _init_weight(self):
        c2_xavier_fill(self.lateral_conv)
        c2_xavier_fill(self.fpn_conv)
        for m in [self.lateral_norm, self.fpn_norm]:
            nn.init.constant_(m.weight, 1)
            nn.init.constant_(m.bias, 0)
        for m in self.dilated_encoder_blocks.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.normal_(m.weight, mean=0, std=0.01)
                if hasattr(m, 'bias') and m.bias is not None:
                    nn.init.constant_(m.bias, 0)

            if isinstance(m, (nn.GroupNorm, nn.BatchNorm2d, nn.SyncBatchNorm)):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

    def forward(self, feature: torch.Tensor):
        out = self.lateral_norm(self.lateral_conv(feature))
        out = self.fpn_norm(self.fpn_conv(out))
        out = self.dilated_encoder_blocks(out)
        return out

class YOLOF(nn.Module):
    def __init__(self, resnet_type, num_classes=80, use_gn=False, planes=512):
        super(YOLOF, self).__init__()
        self.backbone = ResNetBackbone(resnet_type=resnet_type)
        expand_ratio = {
            "resnet18": 1,
            "resnet34": 1,
            "resnet50": 4,
            "resnet101": 4,
            "resnet152": 4
        }
        C5_inplanes = int(512 * expand_ratio[resnet_type])
        self.dila_encoder = DilatedEncoder(C5_inplanes)

        self.num_classes = num_classes
        self.planes = planes

        self.clsregcnt_head = FCOSClsRegCntHead(self.planes,
                                                self.num_classes,
                                                num_layers=4,
                                                prior=0.01,
                                                use_gn=use_gn,
                                                cnt_on_reg=True)

        self.strides = torch.tensor([16], dtype=torch.float)
        self.positions = FCOSPositions(self.strides)

        self.scales = nn.Parameter(
            torch.tensor([1.], dtype=torch.float32))
        self.trans = nn.ConvTranspose2d(in_channels=C5_inplanes, out_channels=int(C5_inplanes/2), kernel_size=4, stride=2, padding=1, bias=False)
        self.c4_out = nn.Conv2d(C5_inplanes, C5_inplanes, 1)

    def forward(self, inputs):
        self.batch_size, _, _, _ = inputs.shape
        device = inputs.device
        [C3, C4, C5] = self.backbone(inputs)

        del inputs

        C5_up = self.trans(C5)
        C5 = self.c4_out(torch.cat((C4, C5_up), 1))

        features = [self.dila_encoder(C5)]

        del C3, C4, C5

        self.fpn_feature_sizes = []
        cls_heads, reg_heads, center_heads = [], [], []
        for feature, scale in zip(features, self.scales):
            self.fpn_feature_sizes.append([feature.shape[3], feature.shape[2]])

            cls_outs, reg_outs, center_outs = self.clsregcnt_head(feature)

            # [N,num_classes,H,W] -> [N,H,W,num_classes]
            cls_outs = cls_outs.permute(0, 2, 3, 1).contiguous()
            cls_heads.append(cls_outs)
            # [N,4,H,W] -> [N,H,W,4]
            reg_outs = reg_outs.permute(0, 2, 3, 1).contiguous()
            reg_outs = reg_outs * torch.exp(scale)
            reg_heads.append(reg_outs)
            # [N,1,H,W] -> [N,H,W,1]
            center_outs = center_outs.permute(0, 2, 3, 1).contiguous()
            center_heads.append(center_outs)

        del features

        self.fpn_feature_sizes = torch.tensor(
            self.fpn_feature_sizes).to(device)

        batch_positions = self.positions(self.batch_size,
                                         self.fpn_feature_sizes)

        # if input size:[B,3,640,640]
        # features shape:[[B, 256, 80, 80],[B, 256, 40, 40],[B, 256, 20, 20],[B, 256, 10, 10],[B, 256, 5, 5]]
        # cls_heads shape:[[B, 80, 80, 80],[B, 40, 40, 80],[B, 20, 20, 80],[B, 10, 10, 80],[B, 5, 5, 80]]
        # reg_heads shape:[[B, 80, 80, 4],[B, 40, 40, 4],[B, 20, 20, 4],[B, 10, 10, 4],[B, 5, 5, 4]]
        # center_heads shape:[[B, 80, 80, 1],[B, 40, 40, 1],[B, 20, 20, 1],[B, 10, 10, 1],[B, 5, 5, 1]]
        # batch_positions shape:[[B, 80, 80, 2],[B, 40, 40, 2],[B, 20, 20, 2],[B, 10, 10, 2],[B, 5, 5, 2]]

        return cls_heads, reg_heads, center_heads, batch_positions

def _yolof(arch, pretrained, **kwargs):
    model = YOLOF(arch, **kwargs)
    # only load state_dict()
    if pretrained:
        pretrained_models = torch.load(model_urls[arch + "_yolof"],
                                       map_location=torch.device('cpu'))
        # del pretrained_models['cls_head.cls_head.8.weight']
        # del pretrained_models['cls_head.cls_head.8.bias']

        # only load state_dict()
        model.load_state_dict(pretrained_models, strict=False)

    return model

def resnet18_yolof(pretrained=False, **kwargs):
    return _yolof('resnet18', pretrained, **kwargs)

def resnet34_yolof(pretrained=False, **kwargs):
    return _yolof('resnet34', pretrained, **kwargs)

def resnet50_yolof(pretrained=False, **kwargs):
    return _yolof('resnet50', pretrained, **kwargs)

def resnet101_yolof(pretrained=False, **kwargs):
    return _yolof('resnet101', pretrained, **kwargs)

def resnet152_yolof(pretrained=False, **kwargs):
    return _yolof('resnet152', pretrained, **kwargs)

if __name__ == '__main__':
    net = YOLOF(resnet_type="resnet18")
    image_h, image_w = 512, 512
    cls_heads, reg_heads, center_heads, batch_positions = net(
        torch.autograd.Variable(torch.randn(3, 3, image_h, image_w)))
    annotations = torch.FloatTensor([[[113, 120, 183, 255, 5],
                                      [13, 45, 175, 210, 2]],
                                     [[11, 18, 223, 225, 1],
                                      [-1, -1, -1, -1, -1]],
                                     [[-1, -1, -1, -1, -1],
                                      [-1, -1, -1, -1, -1]]])

    print("1111", cls_heads[0].shape, reg_heads[0].shape,
          center_heads[0].shape, batch_positions[0].shape)
    print(batch_positions)


'''
@Author danielvallejo237
'''

import fasttext
import fasttext.util
import numpy as np
from scipy.spatial import distance
import re
import unidecode

#fasttext.util.download_model('es', if_exists='ignore')
#ft=fasttext.load_model('cc.en.300.bin')
#fasttext.util.reduce_model(ft, 100)
ft = fasttext.load_model('cc.es.100.bin')

def return_significant_vector(string,FastText=ft):
    splitted=string.split()
    sigs=np.zeros(100)
    for s in splitted:
        sigs=sigs+FastText.get_word_vector(s)
    sigs=sigs/len(splitted)
    return sigs

def cossine_distance(v1,v2):
    return distance.cosine(v1,v2)

def similaridad_frase(s1,s2,FT=ft):
    v1=return_significant_vector(s1,FastText=FT)
    v2=return_significant_vector(s2,FastText=FT)
    return cossine_distance(v1,v2),distance.euclidean(v1,v2)

saludos=["buenas tardes","buenos dias","hola","buenas noches","hola que tal","hola buenas tardes","Hola Garambubot", "¡Hola, buenos días!" , "¡Hola, buenas tardes!" , "¡Hola, buenas noches!" , "¿Cómo estás?" , "¿Qué tal?" ,  "¿Qué pasa?" , "¡Hola, buenas!", "¿Cómo vas?" , "Hola, ¿qué tal?", "Hola, qué tal estás?" ,  "¿Qué hay?" , "¡Buenas!" ]
despedidas=["adios","hasta luego","hasta pronto","nos vemos","chao","bye","cuidate","hasta ahora","¡Adiós, hasta luego!" , "¡Adiós, buenos días!" , "¡Adiós, buenas tardes!" , "¡Adiós, buenas noches!" , "Saludos"]

def IntGen(frase,greet,desp,fst=ft):
    gscores=[]
    efscr=[]
    for g in greet:
        a,b=similaridad_frase(frase,g,FT=fst)
        gscores.append(a)
        efscr.append(b)
    mg=min(gscores)
    dg=min(efscr)
    dscores=[]
    defscr=[]
    for g in desp:
        a,b=similaridad_frase(frase,g,FT=fst)
        dscores.append(a)
        defscr.append(b)
    md=min(dscores)
    dmd=min(defscr)
    gs=[mg,md]
    gs1=[dg,dmd]
    if min(gs)<0.4:
        if 1+gs.index(min(gs)) == 1+gs1.index(min(gs1)):
            return 1+gs.index(min(gs))
        else:
            return 3
    else:
        return 3

def hasHashTag(string):
    return len(re.findall(r"#[\w]+",string))>0

def IntEspecificas(frase,greet=saludos,desp=despedidas,fst=ft):
    frase=unidecode.unidecode(frase.lower())
    intencion=IntGen(frase,greet,desp,fst)
    if intencion==3:
        if hasHashTag(frase):
            intencion=4
    if hasHashTag(frase):
        intencion=4
    return intencion

if __name__=="__main__":
    pass