# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import ruamel.yaml.comments
import six
import warnings

import orquesta.expressions.base

from orquestaconvert.expressions import jinja
from orquestaconvert.expressions import yaql as yql
from orquestaconvert.utils import type_utils

class ExpressionConverter(object):

    @classmethod
    def convert(cls, expr, **kwargs):
        if (isinstance(expr, type_utils.dict_types)):
            return cls.convert_dict(expr, **kwargs)
        elif isinstance(expr, list):
            return cls.convert_list(expr, **kwargs)
        elif isinstance(expr, six.string_types):
            return cls.convert_string(expr, **kwargs)
        elif isinstance(expr, bool):
            return expr
        elif expr is None:
            # Note: The only point to explicitly converting None is to skip
            #       emitting the warning. Otherwise we would remove this elif
            #       and let the function return the input. Also, ruamel doesn't
            #       explicitly serialize null to None, it just leaves the key
            #       blank (in other words, it "implicitly" serializes it).
            #       Example:
            #
            #           next:
            #             - when: ...
            #               publish:
            #                 - continue:
            #               ...
            #
            #       See this link for more information:
            #       https://bitbucket.org/ruamel/yaml/issues/169/roundtripdumper-dumps-null-values
            return None
        elif isinstance(expr, int):
            return expr
        else:
            warnings.warn("Could not recognize expression '{}'; results may not "
                          "be accurate.".format(expr), SyntaxWarning)
            return expr

    @classmethod
    def expression_type(cls, expr):
        for name, evaluator in six.iteritems(orquesta.expressions.base.get_evaluators()):
            if evaluator.has_expressions(str(expr)):
                return name
        return None

    @classmethod
    def get_converter(cls, expr):
        expr_type = cls.expression_type(expr)
        if expr_type == 'jinja':
            return jinja.JinjaExpressionConverter
        elif expr_type == 'yaql':
            return yql.YaqlExpressionConverter
        return None

    @classmethod
    def unwrap_expression(cls, expr):
        converter = cls.get_converter(expr)
        if converter:
            return converter.unwrap_expression(expr)
        # this isn't a Jinja or YAQL expression, so return the raw string
        return expr

    @classmethod
    def convert_string(cls, expr, **kwargs):
        # - task('xxx').result -> result()
        #    if 'xxx' != current task name, error
        # - _. -> ctx().
        # - $. -> ctx().
        # - st2kv. -> st2kv('xxx')
        # others?
        converter = cls.get_converter(expr)
        if converter:
            return converter.convert_string(expr, **kwargs)
        # this isn't a Jinja or YAQL expression, so return the raw string
        return expr

    @classmethod
    def convert_dict(cls, expr_dict, **kwargs):
        converted = ruamel.yaml.comments.CommentedMap()
        for k, expr in six.iteritems(expr_dict):
            converted[cls.convert(k, **kwargs)] = cls.convert(expr, **kwargs)
        return converted

    @classmethod
    def convert_list(cls, expr_list, **kwargs):
        return [cls.convert(expr, **kwargs) for expr in expr_list]


#! /usr/bin/python

import json
import requests

url = 'http://127.0.0.1:8080/wm/onos/datagrid/add/intents/json'
headers = {'Content-type': 'application/json', 'Accept': 'application/json'}

for i in range(6,16):
	srcMac = '00:00:00:00:00:'+ str(hex(i)[2:]).zfill(2)
	dstMac = '00:00:00:00:00:'+ str(hex(i+10)[2:])
	srcSwitch = '00:00:00:00:00:00:10:'+ str(i).zfill(2)
	dstSwitch = '00:00:00:00:00:00:20:'+ str(i+25)
	srcPort = 1
	dstPort = 1

	intent = [{'intent_id': '%d' %(i),'intent_type':'shortest_intent_type','intent_op':'add','srcSwitch':srcSwitch,'srcPort':srcPort,'srcMac':srcMac,'dstSwitch':dstSwitch,'dstPort':dstPort,'dstMac':dstMac}]
	r = requests.post(url, data=json.dumps(intent, sort_keys=True), headers = headers)
	#print json.dumps(intent, sort_keys = True)

	intent = [{'intent_id': '%d' %(i+10),'intent_type':'shortest_intent_type','intent_op':'add','srcSwitch':dstSwitch,'srcPort':dstPort,'srcMac':dstMac,'dstSwitch':srcSwitch,'dstPort':srcPort,'dstMac':srcMac}]
	#print json.dumps(intent, sort_keys = True)
	r = requests.post(url, data=json.dumps(intent, sort_keys=True), headers = headers)


#!/usr/bin/env python

from __future__ import (
    absolute_import,
    division,
    print_function,
    unicode_literals,
)

import ast
import collections

Flake8Result = collections.namedtuple(
    'Flake8Result',
    ['lineno', 'col_offset', 'message']
)

class BaseLinter(ast.NodeVisitor):
    def __init__(self, *args, **kwargs):
        self.results = []

        super(BaseLinter, self).__init__(*args, **kwargs)

    def get_results(self):

        return self.results


import numpy as np
import cv2

def get_edge(img, edge_level=1):
    """
    Calculate edges on the image.
    Thanks to https://qiita.com/supersaiakujin/items/494cc16836738b5394c8

    Args:
        img: numpy array in HWC format.
        edge_level:
            0: thin edge line, patchy, scratchy.
            1: thick and sometimes doubly.

    Returns:
        2D uint8 image, non-zero elements are the edge pixels.
        > img[get_edge(img) > 0] = 0 # Put black edge pixels on the image.
    """
    org_shape = img.shape

    # preprocess: blur image to suppress small noise edges
    blur_kernel = np.array([1,1,1, 1,3,1, 1,1,1]) / 11
    img = cv2.filter2D(img, 256, blur_kernel)

    # preprocess: sharpen edges
    k = -1.
    sharpningKernel8 = np.array([k, k, k, k, 9.0, k, k, k, k])
    img = cv2.filter2D(img, 256, sharpningKernel8)

    # extract edges by image pyramid
    L = 2
    tmp = img.copy()
    edges = [cv2.Canny(tmp.astype(np.uint8),100,200 )]
    for idx in range(L-1):
        tmp = cv2.pyrDown(tmp)
        edges.append(cv2.Canny(tmp.astype(np.uint8),100,200 ))

    # recover size (edges[1] is currently half of org_img)
    edge = edges[0] if edge_level == 0 else cv2.resize(edges[1], org_shape[:2])

    return edge

def simplify_by_kmeans(img, K=10, quantize_color=True):
    """
    Simplify image by using kmeans clustering colors.

    Args:
        img: Input image in HWC format.
        K: Number of clusters (= colors).
        quantize_color: resulting colors are quantized.

    Returns:
        RGB numpy image array [HWC]

    Notes:
        .
    """
    # dilation & erusion to connect small piecies all together
    kernel = np.ones((10,10), np.uint8)
    img = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)

    # ensure shape is 3d
    reshaped_from_2d = False
    if len(img.shape) == 2:
        img = img[..., np.newaxis]
        reshaped_from_2d = True

    # apply kmeans()
    Z = img.reshape((-1, img.shape[-1]))
    Z = np.float32(Z)
    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 10, 1.0)
    ret, label, center=cv2.kmeans(Z, K, None, criteria, 10, cv2.KMEANS_RANDOM_CENTERS)

    # now convert back into uint8, and make original image
    center = np.uint8(center)
    res = center[label.flatten()]
    img = res.reshape((img.shape))

    # quantize colors
    if quantize_color:
        # Assign quantized grayscale colors to unique colors,
        # darker grayscale color is assigned to darker unique color.
        # This is done naturally by the sorted array `uniqs` below.
        # np.unique() returns sorted array.
        uniqs = np.unique(img.reshape(-1, img.shape[-1]), axis=0)
        N = len(uniqs)
        color_map = {(255 * i) // N: uc for i, uc in enumerate(uniqs)}
        for color_code, uc in color_map.items():
            img[np.logical_and.reduce(img == uc, axis = -1), :] = color_code

    if reshaped_from_2d:
        img = img[:, :, 0]

    return img

def abstract_image_array(img, K=10, grayscale=False, add_edge=True, quantize_color=True):
    """Make abstract version of image from array.
    This will:
        - Downgrade to K colors, resulting image will have only K colors.
        - (option) Convert to grayscale.
        - Edges extracted from original image will be overlayed on top of resulting image.
        - Quantize color value.

    Returns:
        abstract version of original image.
    """

    org_img = img.copy()

    # apply grayscale to make clustering easier
    if grayscale:
        img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)

    # apply clustering
    if K is not None:
        img = simplify_by_kmeans(img, K=K, quantize_color=quantize_color)

    # emphasize edge
    if add_edge:
        edge = get_edge(org_img)
        img[edge > 0] = 255

    # convert back to color image
    if grayscale:
        img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)

    return img

def abstract_image(img_file, K=10, grayscale=False, add_edge=True,
                   quantize_color=True, resize=None):
    """Make abstract version of image from file.

    Returns:
        orginal image: image loaded from file as is.
        abstract image: converted one.
    """

    img = cv2.imread(str(img_file))
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    if resize is not None:
        img = cv2.resize(img, resize)

    return img, abstract_image_array(img, K=K, grayscale=grayscale,
                                     add_edge=add_edge, quantize_color=quantize_color)


import sqlite3

class SQLite:

	def __init__(self,db_path):
		self.sql = sqlite3.connect(db_path)
		self.cur = self.sql.cursor()

	def init(self):
		try:
			self.cur.execute("""CREATE TABLE user_msg (
					email       VARCHAR(30) ,
					from_mail   VARCHAR(30) ,
					message     varchar(600),
					ip_from     varchar(20) ,
					datetime_    varchar(30)
				);
				""")
		except:
			pass

	def inserMessage(self,to,from_,mesg,ip,datetime):
		self.cur.execute(
				"""INSERT INTO
				user_msg (`email`,`from_mail`,`message`,`ip_from`,`datetime_`)
				VALUES (?,?,?,?,?)
				""",(to,from_,mesg,ip,datetime)
			);self.sql.commit()

	def selectByUser(self):
		self.cur.execute("SELECT * FROM user_msg ");
		return self.cur.fetchall()


import responder
from starlette.websockets import WebSocket, WebSocketDisconnect
from typing import Dict
import asyncio
from threading import Thread, Lock
from queue import Queue, Empty, Full

class WebSocketServer(object):
    def __init__(self, *, address: str = '0.0.0.0', port: int = 8080, location: str = '/'):
        self.address = address
        self.port = port
        self.location = location

        self.clients: Dict[str, WebSocket] = {}
        self.locker = Lock()

        self.elo = asyncio.get_event_loop()  # Eventloop Object
        self.bcPutTimeoutSecs = 5
        self.bcGetTimeoutSecs = 5
        self.bcCounter = 0
        self.bcQueue = Queue()  # Queue for message
        self.bcThread = Thread(target=self._bcWatcher, daemon=True)
        self.bcThread.start()

        self.api = responder.API()
        self.api.add_route(route=self.location, endpoint=self.wsSession, websocket=True)

        self.api.run(address=self.address, port=self.port)

    async def wsSession(self, ws: WebSocket):
        await ws.accept()
        key = ws.headers['sec-websocket-key']
        with self.locker:
            if key not in self.clients.keys():
                self.clients[key] = ws
        while True:
            try:
                message = await ws.receive_text()
            except (WebSocketDisconnect,) as e:
                break
            else:
                for k, v in self.clients.items():
                    if k != key:
                        await v.send_text(data=message)  # no exception ???

        with self.locker:
            if key in self.clients.keys():
                del self.clients[key]
        await ws.close()

    async def _onAir(self, *, message: str) -> None:
        with self.locker:
            for k, v in self.clients.items():
                await v.send_text(data=message)

    def _bcWatcher(self):
        while True:
            try:
                message = self.bcQueue.get(timeout=self.bcGetTimeoutSecs)
            except (Empty,) as e:
                self.broadCast(message='Queue is empty!')  # for debug
            else:
                self.elo.run_until_complete(future=self._onAir(message=message))
                self.bcCounter += 1

    # ---------------------------------------------------------------------------------
    def broadCast(self, *, message: str):
        try:
            self.bcQueue.put(item=message, timeout=self.bcPutTimeoutSecs)
        except (Full,) as e:
            print(e)
            pass
    # ---------------------------------------------------------------------------------

if __name__ == '__main__':
    def main():
        S = WebSocketServer()
        pass

    main()


#!/usr/bin/env python3

from __future__ import print_function

import datetime
import functools
import os
import socket
import sys
import time

import packet

project = os.environ["PACKET_PROJECT_ID"]
token = os.environ["PACKET_API_TOKEN"]
hostname = os.environ["DEVICE_HOSTNAME"]
ipxe_script_url = os.environ["DEVICE_IPXE_SCRIPT_URL"]
plan = os.environ["DEVICE_PLAN"]
facility = os.environ["DEVICE_FACILITY"]

timeout = 600

if sys.version_info[0] == 2:
    oldprint = print

    def myprint(*args, **kwargs):
        flush = kwargs.pop("flush", False)
        f = kwargs.get("file", sys.stdout)

        oldprint(*args, **kwargs)

        if flush:
            f.flush()

    print = functools.update_wrapper(myprint, print)

print = functools.partial(print, hostname + ":", flush=True)

class Device:
    def __init__(self, manager):
        self.m = manager
        self.dev = None
        self.id = None
        self.ip = None

    def __until(self, func):
        """
        Runs `func` in a loop until timeout or func returns Falsy. Will handle
        504 errors in a way that makes sense. Stop is indicated as Falsy so that
        funcs do not have to explicitly return anything, Python's implicit None
        will break the loop.
        """
        end = time.time() + timeout
        while end > time.time():
            try:
                if not func():
                    return
            except packet.baseapi.Error as e:
                if "504" in str(e):
                    print("504")
                    time.sleep(30)
                    continue
                print(e)
                sys.exit(1)

            time.sleep(10)

        print("timed out")
        sys.exit(1)

    def create(self, project, hostname, plan, facility, ipxe_script_url):
        def __create():
            tt = datetime.datetime.utcnow() + datetime.timedelta(hours=2)
            d = self.m.create_device(
                project,
                hostname,
                plan,
                facility,
                "custom_ipxe",
                ipxe_script_url=ipxe_script_url,
                spot_instance=True,
                spot_price_max=20,
                termination_time=tt.isoformat(),
            )

            self.dev = d
            self.id = d.id

        self.__until(__create)

    def refresh(self):
        def __refresh():
            d = self.m.get_device(self.id)
            self.dev = d
            if d.ip_addresses and not self.ip:
                ips = [
                    ip
                    for ip in d.ip_addresses
                    if ip["address_family"] == 4 and ip["public"]
                ]
                if ips:
                    self.ip = ips[0]["address"]

        self.__until(__refresh)

    def state(self):
        self.refresh()
        return self.dev.state

    def wait_for_ssh(self):
        def __wait_for_ssh():
            if self.state() != "active":
                print(self, "state is not active")
                return

            s = socket.socket()
            s.settimeout(10)
            try:
                s.connect((self.ip, 22))
                msg = s.recv(100).decode()
            except socket.error:
                return True

            if "ssh" in msg.lower():
                print(self, msg.strip())
                return

            print(self, "not ssh greeting")

        self.__until(__wait_for_ssh)

    def delete(self):
        def __delete():
            self.dev.delete()

        self.__until(__delete)

    def __repr__(self):
        state = None
        if self.dev:
            state = self.dev.state
        return "{} {} {}".format(self.id, state, self.ip)

def main():
    print("instantiating manager")
    m = packet.Manager(token)
    print("instantiating device")
    d = Device(m)
    print("creating device")
    d.create(project, hostname, plan, facility, ipxe_script_url)
    start = time.time()
    print(d)

    state = None
    while True:
        state, oldstate = d.state(), state
        if state != oldstate:
            print(d)
        if state == "active":
            break
        time.sleep(30)

    d.wait_for_ssh()
    print(d, "time-to-active", time.time() - start)
    print(d, "deleting")
    d.delete()
    print(d, "test-time", time.time() - start)

if __name__ == "__main__":
    try:
        main()
    except Exception as e:
        print(hostname + ":", e)


import numpy as np
import pandas as pd
import glob
from sys import platform

#importing KERAS
import keras
from keras.layers import Embedding, Input, Flatten, Dense, Activation
from keras.models import Model, Sequential
from keras.optimizers import SGD
from keras.utils import to_categorical
from keras.wrappers.scikit_learn import KerasClassifier

from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve
from sklearn.metrics import auc

import matplotlib
from matplotlib import pyplot as plt

#=========================data import and preprocessing
def load_data(dnnType,varlist):

    if platform == "linux" or platform == "linux2":
        pathwithwildcard = '../csv/2017/forBDTtraining/*/*.csv'
    elif platform == "darwin":
        pathwithwildcard = '../csv/2017/forBDTtraining/*/*.csv'
    elif platform == "win32":
        pathwithwildcard = '..\\csv\\2017\\forBDTtraining\\*\\*.csv'

    paths = glob.glob(pathwithwildcard)
    df1_signal_dict = {}
    df1_TT_dict = {}
    df1_TTV_dict = {}

    for path in paths:
        if 'ttHToNonbb' in path:
            df1_signal_dict[path] = pd.read_csv(path)
    for path in paths:
        if not (('ttHToNonbb' in path) or ('PSweights' in path)):
            if 'TTTo' in path:
                df1_TT_dict[path] = pd.read_csv(path)
            elif 'TTW' in path or 'TTZ' in path:
                df1_TTV_dict[path] = pd.read_csv(path)
            else:
                continue

    tmp_sig = []
    for keys in df1_signal_dict.keys():
        tmp_sig.append(df1_signal_dict[keys])
    tmp_TT = []
    for keys in df1_TT_dict.keys():
        tmp_TT.append(df1_TT_dict[keys])
    tmp_TTV = []
    for keys in df1_TTV_dict.keys():
        tmp_TTV.append(df1_TTV_dict[keys])
    df1_signal = pd.concat(tmp_sig)
    df1_TT = pd.concat(tmp_TT)
    df1_TTV = pd.concat(tmp_TTV)

    print (df1_signal.shape, df1_TT.shape, df1_TTV.shape)

    df1_signal = df1_signal[varlist]
    df1_signal_target = [0]*df1_signal.shape[0]
    df1_TT = df1_TT[varlist]
    df1_TT_target = [1]*df1_TT.shape[0]
    df1_TTV = df1_TTV[varlist]
    df1_TTV_target = [1]*df1_TTV.shape[0]

    tmp0 = []
    if dnnType == 'TT':
        tmp0 = [df1_signal,df1_TT]
        target = df1_signal_target+df1_TT_target
    if dnnType == 'TTV':
        tmp0 = [df1_signal,df1_TTV]
        target = df1_signal_target+df1_TTV_target
    if dnnType == 'sum':
        tmp0 = [df1_signal,df1_TT,df1_TTV]
        target = df1_signal_target+df1_TT_target+df1_TTV_target

    df1 = pd.concat(tmp0)
    return df1, target

#===========================data training

def train_pyKeras(df1, target, descrip, num_epochs, sizeofepoch):
    df1_train, df1_test, df1_train_target, df1_test_target = train_test_split(df1, target, test_size=0.5)

    df1_train_target_cat = to_categorical(df1_train_target)
    df1_test_target_cat = to_categorical(df1_test_target)

    print ('(#events,#vars)[train, test]=', df1_train_target_cat.shape, df1_train.shape, df1_test.shape, df1_test_target_cat.shape)

    print (df1_train.columns.values)

    #===============================defining keras model
    model = Sequential()

    for iterator in range(0,len(descrip)):
        if iterator == 0:
            model.add(Dense(descrip[iterator],activation='relu',input_dim=len(varlist)))
            continue
        elif iterator == len(descrip)-1:
            model.add(Dense(descrip[iterator],activation='softmax'))
            break
        else:
            model.add(Dense(descrip[iterator],activation='relu'))

    model.summary()

    model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['binary_accuracy'])

    model.fit(df1_train,df1_train_target_cat,
                 batch_size=sizeofepoch,
                 epochs=num_epochs,
                 validation_data=(df1_test,df1_test_target_cat),
                 verbose = 1
                 )

    #===================================roc curve
    fpr_keras, tpr_keras, thresholds_keras = roc_curve(df1_test_target_cat[:,0],model.predict(df1_test)[:,0])
    fpr_keras_train, tpr_keras_train, thresholds_keras_train = roc_curve(df1_train_target_cat[:,0],model.predict(df1_train)[:,0])
    auc_keras = auc(fpr_keras, tpr_keras)
    auc_keras_train = auc(fpr_keras_train, tpr_keras_train)
    plt.figure(1)
    plt.plot(fpr_keras, tpr_keras, label='pyKeras_test (area = {:.3f})'.format(auc_keras))
    plt.plot(fpr_keras_train, tpr_keras_train, label='pyKeras_train (area = {:.3f})'.format(auc_keras_train))
    plt.xlabel('False positive rate')
    plt.ylabel('True positive rate')
    plt.title('ROC curve')
    plt.legend(loc='best')
    #plt.savefig('pyKeras_batch_size'+str(sizeofepoch)+'_epochs'+str(num_epochs)+'__'+str(len(varlist))+'vars'+'__depth'+str(len(descrip))+'.pdf')
    plt.show()
    return model, plt

#========================variable selection
varlist=[
    #'AK12_lead_mass',
    #'AK12_lead_pt',
    #'DR_AK12_tau',
    'avg_dr_jet',
    'b1_loose_pt',
    'b1_pt',
    'b2_loose_pt',
    'b2_pt',
    #'detabb',
    #'detabb_loose',
    'dr_lep1_tau_os',
    'dr_leps',
    'drbb',
    'drbb_loose',
    'htmiss',
    'lep1_conePt',
    'lep1_eta',
    'lep1_fake_prob',
    'lep1_pt',
    'lep2_conePt',
    'lep2_eta',
    'lep2_genLepPt',
    'lep2_pt',
    'mT_lep1',
    'mT_lep2',
    'mTauTauVis',
    'max_lep_eta',
    'mbb',
    'mbb_loose',
    'minDR_AK12_L',
    'minDR_AK12_lep',
    'minDR_HTTv2_L',
    'minDR_HTTv2_lep',
    'minDR_HTTv2_tau',
    'mindr_lep1_jet',
    'mindr_lep2_jet',
    'mindr_tau_jet',
    'ptbb',
    'ptbb_loose',
    'ptmiss',
    'tau_eta',
    'tau_mva',
    'tau_pt',
    'N_jetAK12',
    'nBJetLoose',
    'nJet',
    'nMuon'
]

dnnType = 'TT'
descrip = [100,10,2]
num_epochs = 25
sizeofepoch = 100
filename = dnnType+'pyKeras_batch_size'+str(sizeofepoch)+'_epochs'+str(num_epochs)+'__'+str(len(varlist))+'vars'+'__depth'+str(len(descrip))+'.pdf'

data_from_dataloader = load_data(dnnType,varlist)
hist = train_pyKeras(data_from_dataloader[0], data_from_dataloader[1], descrip, num_epochs, sizeofepoch )
hist[1].savefig(filename)