import logging
import os
import sys
import time
import unittest

import pytest
from appium import webdriver as mobile_browser
from pom.pages import (
    AdminPage,
    CCCommunityPage,
    ChallengePage,
    CreateChallengePage,
    CreateIdeaPage,
    DiscoverPage,
    EditCommunityPage,
    EditProfilePage,
    GoogleSSOPage,
    IdeaPage,
    IdeaspaceCreationPage,
    LoginPage,
    NavBarActions,
    OAuth2AppListPage,
    OAuth2AuthPage,
    OAuth2CreateAppPage,
    OAuth2EditAppPage,
    PortalPage,
    ReviewBoard,
    WazokuAdminPage,
    WazokuAdminCreateDemoSitePage,
    YammerIntegrationPage,
)
from pom.pages.mobile import SideBarActions
from selenium import webdriver as desktop_browser
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from settings import (
    ADMIN_EMAIL,
    ADMIN_PASSWORD,
    CHROMEDRIVER,
    DOMAIN,
    SCREENSHOT_DIR,
)

logging.basicConfig(filename='log.txt', filemode='w', level=logging.INFO)

ci = pytest.mark.skipif(
    not pytest.config.getoption("--ci"),
    reason="an integration test with 3rd party providers"
)
mobile = pytest.mark.skipif(
    not pytest.config.getoption("--mobile"),
    reason="Tests using the Appium Suite"
)

class Actions(object):

    def default_login(self):
        self.loginpage.navigate()
        self.save_screenshot('loginpage')
        self.loginpage.enter_an_email(ADMIN_EMAIL)
        self.loginpage.enter_a_password(ADMIN_PASSWORD)
        self.loginpage.click_login()

    def save_screenshot(self, name, prefix=''):
        self.driver.screenshot_count += 1
        name = u'{}{:02d}_{}.png'.format(prefix, self.driver.screenshot_count, name)

        path = os.path.join(SCREENSHOT_DIR, name)
        self.driver.save_screenshot(path)

class SpotlightSeleniumTests(unittest.TestCase, Actions):
    domain = DOMAIN

    def is_mobile(self):
        return pytest.config.getoption("--mobile")

    def setUp(self):
        # Always use '.copy()' on the DesiredCapabilities object
        # to avoid the side effects of altering the Global class instance
        capabilities = DesiredCapabilities.CHROME.copy()
        capabilities['handlesAlerts'] = True
        capabilities['nativeEvents'] = True
        capabilities['acceptSslCerts'] = True
        capabilities['takesScreenshot'] = True
        capabilities['cssSelectorsEnabled'] = True
        capabilities['javascriptEnabled'] = True
        if self.is_mobile():
            self.driver = desktop_browser.Remote(desired_capabilities=capabilities)
        else:
            self.driver = desktop_browser.Chrome(
                CHROMEDRIVER, desired_capabilities=capabilities)
        self.driver.screenshot_count = 0

        self.loginpage = LoginPage(self.driver, self.domain)
        self.google_sso_page = GoogleSSOPage(self.driver, self.domain)
        self.navbar = NavBarActions(self.driver, self.domain)
        self.portalpage = PortalPage(self.driver, self.domain)
        self.createchallengepage = CreateChallengePage(self.driver, self.domain)
        self.challengepage = ChallengePage(self.driver, self.domain)
        self.createideapage = CreateIdeaPage(self.driver, self.domain)
        self.ideapage = IdeaPage(self.driver, self.domain)
        self.discoverpage = DiscoverPage(self.driver, self.domain)
        self.adminpage = AdminPage(self.driver, self.domain)
        self.reviewboard = ReviewBoard(self.driver, self.domain)
        self.oauth2_app_list_page = OAuth2AppListPage(self.driver, self.domain)
        self.oauth2_app_create_page = OAuth2CreateAppPage(self.driver, self.domain)
        self.oauth2_auth_page = OAuth2AuthPage(self.driver, self.domain)
        self.oauth2_edit_app_page = OAuth2EditAppPage(self.driver, self.domain)
        self.yammer_integration_page = YammerIntegrationPage(self.driver)
        self.edit_profile_page = EditProfilePage(self.driver, self.domain)
        self.editcommunity = EditCommunityPage(self.driver, self.domain)
        self.wazoku_admin_page = WazokuAdminPage(self.driver, self.domain)
        self.create_demo_site_page = WazokuAdminCreateDemoSitePage(
            self.driver, self.domain)
        self.cccpage = CCCommunityPage(self.driver, self.domain)
        self.ideaspace_creation_page = IdeaspaceCreationPage(self.driver, self.domain)

        """
        This is overriding the maximise() in base __init__
        If we need to change the screen size to something custom
        then there shouldn't be an issue with using the set_screen_size()
        """
        # self.set_screen_size(*DESKTOP)

    def set_screen_size(self, w, h):
        self.driver.set_window_size(w, h)
        self.navbar.wait_until(
            lambda driver: driver.get_window_size() == {u'width': w, u'height': h},
        )
        # Give the browser a little time to redraw itself
        time.sleep(0.5)

    def delete_challenge(self):
        self.challengepage.open_sidebar()
        self.challengepage.delete_challenge()

    def tearDown(self):
        try:
            if sys.exc_info()[0] is not None:
                self.save_screenshot(u'fail')
        finally:
            self.driver.quit()

@mobile
class SpotlightAppiumTests(unittest.TestCase, Actions):
    domain = DOMAIN

    def setUp(self):
        desired_caps = {}
        desired_caps['platformName'] = 'Android'
        desired_caps['platformVersion'] = '5.11'

        # For now this is hardcoded...
        desired_caps['deviceName'] = 'Galaxy_S6'

        # Browser == Native and Chrome == Chrome
        desired_caps['browserName'] = 'Chrome'

        # Set it to an hour
        desired_caps['newCommandTimeout'] = 3600  # seconds
        desired_caps['noReset'] = True

        self.driver = mobile_browser.Remote('http://localhost:4723/wd/hub', desired_caps)
        self.driver.screenshot_count = 0

        self.loginpage = LoginPage(self.driver, self.domain)
        self.sidebar = SideBarActions(self.driver, self.domain)
        self.portalpage = PortalPage(self.driver, self.domain)
        self.challengepage = ChallengePage(self.driver, self.domain)
        self.createideapage = CreateIdeaPage(self.driver, self.domain)
        self.ideapage = IdeaPage(self.driver, self.domain)
        self.discoverpage = DiscoverPage(self.driver, self.domain)
        self.edit_profile_page = EditProfilePage(self.driver, self.domain)
        self.cccpage = CCCommunityPage(self.driver, self.domain)

    def tearDown(self):
        try:
            if sys.exc_info()[0] is not None:
                self.save_screenshot(u'fail')
        finally:
            self.driver.quit()

@ci
# Should let the class inherit Actions so that it has access to save_screenshot()
class SpotlightSeleniumIntegrationTests(SpotlightSeleniumTests, Actions):
    pass


import tagui as t
import datetime
import pandas as pd
import os
import s3_function

try:
    t.init()
    t.wait(5)
    #read page
    t.url("http://bank.jrj.com.cn/bankpro/")
    t.wait(5)
    #locate items
    item_list = t.read(element_identifier='//div[@class = "qutop mt"]').split()

    #txt
    today = datetime.datetime.today()
    today = str(today.date())
    file_name = today+'product_daily.txt'
    with open(file_name,'w',encoding='utf-8') as f:
        f.writelines([item+'\n' for item in item_list])
    #excel
    # daily_on_sale = int(item_list[1][:-11])
    # monthly_on_sale =int(item_list[2][:-13])
    # monthly_expire = int(item_list[3][:-13])
    #
    # pd.DataFrame({'今日在售银行理财产品':int(daily_on_sale),'本月累计发售银行理财产品':int(monthly_on_sale),'本月累计到期银行理财产品':int(monthly_expire)},index=[0]).to_csv(today+'product_daily.csv',index = False, encoding = 'utf-8')
    s3_function.upload_to_aws_s3(file_name,'s3qingdao',file_name)
    os.remove(file_name)
    t.close()
except:
    with open('FAIL_product_daily' + str((datetime.datetime.today()).date()) + '.txt', 'w', encoding='utf-8') as f:
        f.write("今日理财产品日报任务启动失败")
    s3_function.upload_to_aws_s3('FAIL_product_daily' + str((datetime.datetime.today()).date()) + '.txt', 's3qingdao',
                                 'FAIL_product_daily' + str((datetime.datetime.today()).date()) + '.txt')
    os.remove('FAIL_product_daily' + str((datetime.datetime.today()).date()) + '.txt')


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

import logging
import argparse
from collections import deque

import torch
import numpy as np

from .agent import Agent
from .util import load_environment, UnityEnvironmentWrapper, get_state
from .util import STACK_SIZE, show_agent
from .train import reset_deque

def evaldqn(env, checkpointfn='checkpoint.pth'):
    """Function that uses Deep Q Networks to learn environments.

    Parameters
    ----------
        env: Environment
            execution environment
        checkpointfn: str
            Name of the file to load network parameters
    """
    brain_name = env.brain_names[0]
    brain = env.brains[brain_name]
    env_info = env.reset(train_mode=True)[brain_name]

    action_size = brain.vector_action_space_size
    state = env_info.vector_observations[0]
    state_size = len(state)

    if state_size == 0:
        use_visual = True
        initial_state = get_state(env_info, use_visual)
        state_size = list(initial_state.shape)
        state_size.insert(2, STACK_SIZE)
        state_size = tuple(state_size)

    agent = Agent.load(checkpointfn, use_visual=True)

    state_deque = reset_deque(initial_state)
    env_info = env.reset(train_mode=False)[brain_name]
    state = get_state(env_info, use_visual)
    state_deque.append(state)

    score = 0
    first = True
    while True:
        state = np.stack(state_deque, axis=-1) \
                .squeeze(axis=0).transpose(0, -1, 1, 2)

        action = agent.act(state)
        env_info = env.step(action)[brain_name]

        next_state = get_state(env_info, use_visual)
        state_deque.append(next_state)
        next_state = np.stack(state_deque, axis=-1) \
                .squeeze(axis=0).transpose(0, -1, 1, 2)

        reward = env_info.rewards[0]
        done = env_info.local_done[0]

        show_agent(state, next_state, action)
        if first:
            input('Press enter to continue')
            first = False

        agent.step(
            state,
            action,
            reward,
            next_state,
            done,
        )

        score += reward
        if done:
            break
    logging.info(
        'Final score: %g', score
    )
    return score

def main():
    parser = argparse.ArgumentParser(description='Evaluates a learned agent')
    parser.add_argument('--checkpoint', dest='checkpoint', action='store',
                        default='checkpoint.pth')
    args = parser.parse_args()

    logging.basicConfig(level=logging.INFO)

    env = load_environment()
    evaldqn(env, checkpointfn=args.checkpoint)

if __name__ == '__main__':
    main()


'''
Created on Sep 25, 2010

@author: aadjemonkeyrock
'''
#import datetime, os, logging
#import StringIO
#import urllib

from google.appengine.api import users
#from google.appengine.api import mail

from google.appengine.ext import db
#from google.appengine.ext.db import djangoforms

#from django import newforms as forms

from pytz.gae import pytz

_REQUIRE_INVITE = True

# DB Model - Account
# ------------------
class Account(db.Model):
    id = db.StringProperty()
    name = db.StringProperty()
    image = db.BlobProperty()
    username = db.StringProperty()
#    tzname = db.StringProperty(choices=pytz.common_timezones_set)
    tzname = db.StringProperty()
    web = db.StringProperty()
    created = db.DateTimeProperty(auto_now_add=True)

    def tzinfo(self):
        return pytz.timezone(self.tzname)

    def localizeDate(self, dt):
        tz = self.tzinfo()
        if dt.tzinfo is None:
            dt = dt.replace(tzinfo=pytz.utc)
        else:
            dt = dt.astimezone(pytz.utc)

        if tz:
            dt = dt.astimezone(tz)

        return dt

    def emails(self):
        emails = AccountEmail.gql('WHERE account = :1', self)
        return emails

    def services(self):
        # must make sure defaults are set
        services = AccountService.gql('WHERE account = :1', self)
        return services

    def vehicles(self):
        # must make sure defaults are set
        vehicles = AccountVehicle.gql('WHERE account = :1', self)
        return vehicles

    def hasConnected(self, service):
        for s in self.services():
            if s.service == service and s.oauth_username != None:
                return True

        return False

    def toJSON(self):
        emails = []
        for e in self.emails():
            emails.append({'email': e.email, 'verified': e.verified})

        return {'account': {'name': self.name, 'username': self.username, 'timezone': self.tzname, 'web': self.web, 'emails': emails}}

class AccountEmail(db.Model):
    account = db.ReferenceProperty(Account)
    email = db.StringProperty()
    verified = db.BooleanProperty(default = False)

class AccountService(db.Model):
    account = db.ReferenceProperty(Account)
    service = db.StringProperty()

    oauth_token = db.StringProperty()
    oauth_secret = db.StringProperty()
    oauth_username = db.StringProperty()
    oauth_usertoken = db.StringProperty()
    oauth_usersecret = db.StringProperty()
    oauth_accesstoken = db.StringProperty()

    user_fullname = db.StringProperty()
    user_picture = db.StringProperty()

class AccountVehicle(db.Model):
    account = db.ReferenceProperty(Account)
    car_brand = db.StringProperty()
    car_type = db.StringProperty()
    car_license = db.StringProperty()
    date_from = db.DateProperty()
    date_to = db.DateProperty()

class Accounts():
    @staticmethod
    def loadAll():
        # Check cache first, if not load from DB
        accounts = Account.gql('ORDER BY created DESC LIMIT 50')
        return accounts

    @staticmethod
    def loadOne(key):
        # Check cache first, if not load from DB
        account = db.get(key)
        return account

    @staticmethod
    def loadByID(id, create=False):
        # Check cache first, if not load from DB
        q = Account.all()
        q.filter("id =", id)
        r = q.fetch(1)
        if len(r) > 0:
            account = r[0]
        else:
            # Automatic create of users or not
            if create or not _REQUIRE_INVITE:
                account = Accounts._createUser()
            else:
                account = None

        return account

    @staticmethod
    def loadByUsername(username):
        # Check cache first, if not load from DB
        accounts = Account.gql('WHERE username = :1 LIMIT 1', username)
        return accounts[0]

    @staticmethod
    def loadByEmail(email):
        # Check cache first, if not load from DB
        q = AccountEmail.all()
        q.filter("email =", email)
        r = q.fetch(1)
        if len(r) > 0:
            account = r[0].account
        else:
            account = None

        return account

    @staticmethod
    def loadService(account, service):
        q = AccountService.all()
        q.filter("account =", account)
        q.filter("service =", service)
        r = q.fetch(1)
        if len(r) > 0:
            s = r[0]
        else:
            # Create a default
            s = AccountService(account=account, service=service)
            #s.account = account
            #s.service = service
            s.put()

        return s

    @staticmethod
    def _createUser():
        # if account doesn't exist, make sure to create one.. later I can clean up if necessary
        user = users.get_current_user()
        name = user.nickname()
#            email = user.email()
        # fix the name and create unique username

        # create the account
        account = Account()
        account.id = user.user_id()
        account.name = name
        account.put()

        # create the email
        email = AccountEmail()
        email.account = account
        email.email = user.email()
        email.verified = True
        email.put()

        # create services... problem is when adding services for new users..

        return account

    @staticmethod
    def deleteOne(account):
        q = AccountEmail.all()
        q.filter("account =", account)
        for e in q:
            e.delete()

        q = AccountService.all()
        q.filter("account =", account)
        for e in q:
            e.delete()

        account.delete()


__author__ = 'lucas'

from pkg_resources import resource_filename
from os.path import join, sep
from configparser import ConfigParser

class Config(object):

    def __init__(self, config_parser):
        self.__sections = []
        parser_sections = config_parser.items()
        for section_name, section_parser in parser_sections:
            section = Section(section_name, section_parser)
            self.__sections.append(section)

    @property
    def sections(self):
        return [section.name for section in self.__sections]

    def get_section(self, section_name, default=None):
        name = section_name.lower()
        for section in self.__sections:
            if section.name == name:
                return section
        return default

    def get_attribute(self, section_name, attribute, default=None, type_=None):
        section = self.get_section(section_name)
        if section is None:
            raise KeyError('Section not found: %s' % section_name)

        value = section.get(attribute.lower(), default)
        if type_ is not None and value:
            value = type_(value)

class Section(dict):

    def __init__(self, name, parser_section):
        self.name = name.lower()
        super(Section, self).__init__(parser_section.items())

def parse_config():
    instance_dir = resource_filename('generatefiles', '').split('/')[:-5]
    instance_dir = sep.join(instance_dir)
    instance_dir = instance_dir
    ini_file = join(join(instance_dir), 'configure.ini')
    config_parser = ConfigParser()
    config_parser.read(ini_file)
    config = Config(config_parser)
    return config

CONFIG = parse_config()


# -*- coding: utf-8 -*-
import cv2
import numpy as np

# 重複要素の削除とソート
def unique2d(a):
    a = np.ascontiguousarray(a)
    unique_a = np.unique(a.view([('', a.dtype)]*a.shape[1]))
    return unique_a.view(a.dtype).reshape((unique_a.shape[0], a.shape[1]))

def main():
    im = cv2.imread("hz2.jpg")  # 入力画像を取得
    hsv = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)
    # マスク画像を用いて元画像から指定した色を抽出
    hsv_min = np.array([160,150,0])
    hsv_max = np.array([190,255,255])
    mask = cv2.inRange(hsv, hsv_min, hsv_max)
    x2,x1 = np.array(np.nonzero(mask))
    w = np.dstack((x1,x2))[0]
    cv2.imshow("View",mask)
    cv2.waitKey(0)
    cv2.destroyAllWindows()                     # ウィンドウ破棄
    w = unique2d(w)
    ws = np.c_[w.T[0],w.T[1]]
    np.savetxt("obstacle.csv",ws , delimiter="\t",fmt="%d")

if __name__ == '__main__':
    main()


from django.core.mail import send_mail
from django.http import JsonResponse
from django.shortcuts import render
from django.views.decorators.debug import sensitive_post_parameters

from mysite.settings import EMAIL_HOST_USER
from perguntas.models import Pergunta, Alternativa, Perguntaresposta
from resultado.models import Noticia
from usuario.models import Formacao, Secretaria, Escola, Privateprofessor

@sensitive_post_parameters()
def noticia(request):
    noticias = Noticia.objects.last()
    if request.POST.get('action') == 'post':
        nome = str(request.POST.get('nome'))
        sobrenome = str(request.POST.get('sobrenome'))
        email = str(request.POST.get('email'))
        mensagem = str(request.POST.get('mensagem'))
        send_mail(nome+sobrenome,
                  mensagem, EMAIL_HOST_USER, [email], fail_silently=False)
        response_data = 'sucesso'
        return JsonResponse(response_data, safe=False)
    return render(request, 'resultado/index2.html', {'noticias': noticias})

def posts(request):
    noticias = Noticia.objects.all()
    return render(request, 'resultado/page52.html', {'noticias': noticias})

def estatisticas(request):
    from django.shortcuts import render
    import plotly.graph_objects as go
    from plotly.offline import plot
    import plotly.graph_objs as go1
    perguntas = Pergunta.objects.filter(isAlternativa=True)

    response = []
    for pergunta in perguntas:
        alternativas = Alternativa.objects.filter(id_questao=pergunta)
        x = []
        y = []
        for alternativa in alternativas:
            x.append(alternativa.alternativa)
            respostas = Perguntaresposta.objects.filter(alternativa=alternativa.id)
            y.append(len(respostas))
        fig = go.Figure(data=[go.Bar(x=x, y=y)],
                        layout_title_text=pergunta.questao)
        plot_div = plot(fig, output_type='div', include_plotlyjs=False)
        response.append(plot_div)

    secretarias = Secretaria.objects.all()
    a = []
    b = []
    c = []
    for secretaria in secretarias:
        escolas = Escola.objects.filter(secretaria=secretaria)
        professor = Privateprofessor.objects.filter(secretaria=secretaria)
        a.append(secretaria.municipio)
        b.append(len(escolas))
        c.append(len(professor))
    a.append('Total')
    b.append(len(Escola.objects.all()))
    fig1 = go.Figure(data=[go.Bar(x=a, y=b)],
                     layout_title_text='Relação de escolas por município')
    plot_div1 = plot(fig1, output_type='div', include_plotlyjs=False)
    rel1= plot_div1
    c.append(len(Privateprofessor.objects.all()))
    fig2 = go.Figure(data=[go.Bar(x=a, y=c)],
                     layout_title_text='Relação de professores por município')
    plot_div2 = plot(fig2, output_type='div', include_plotlyjs=False)
    rel2 = plot_div2

    professores = Privateprofessor.objects.all()
    d = []
    e = []
    g = 0
    for p in professores:
        forma = Formacao.objects.filter(professor=p, isGraduacao=True)
        for f in forma:
            z = 0
            if len(d) > 0:
                for xx in d:
                    if f == xx:
                        e[d.index[xx]] = e[d.index[xx]] + 1
                        z = z + 1
                if z == 0:
                    d.append(f.curso)
                    e.append(1)

            else:
                d.append(f.curso)
                e.append(1)
            g = g + 1
    fig3 = go.Figure(data=[go.Bar(x=d, y=e)],
                     layout_title_text='Relação de áreas de graduação dos professores')
    plot_div3 = plot(fig3, output_type='div', include_plotlyjs=False)
    rel3 = plot_div3

    h = []
    i = []
    pg = 0
    for p in professores:
        forma = Formacao.objects.filter(professor=p, isPosgraduacao=True)
        for f in forma:
            z = 0
            if len(h) > 0:
                for xx in h:
                    if f == xx:
                        i[h.index[xx]] = i[h.index[xx]] + 1
                        z = z + 1
                if z == 0:
                    h.append(f.curso)
                    i.append(1)

            else:
                h.append(f.curso)
                i.append(1)
            pg = pg + 1
    fig4 = go.Figure(data=[go.Bar(x=h, y=i)],
                     layout_title_text='Relação de áreas de pós-graduação dos professores')
    plot_div4 = plot(fig4, output_type='div', include_plotlyjs=False)
    rel4 = plot_div4

    return render(request, 'resultado/page42.html', {'resultados': response, 'r1': rel1, 'r2': rel2, 'r3': rel3,
                                                    'r4': rel4})

def get_noticias(request):
    response_data = {}

    if request.POST.get('action') == 'post':
        id1 = request.POST.get('id')
        news = Noticia.objects.get(id=id1)
        response_data['titulo'] = news.titulo
        response_data['texto'] = news.texto
        response_data['autor'] = news.autor
        response_data['data'] = news.data_publi
        return JsonResponse(response_data)

    return render(request, 'resultado/page5.html')


#coding: utf-8
# +-------------------------------------------------------------------
# | 宝塔Linux面板
# +-------------------------------------------------------------------
# | Copyright (c) 2015-2099 宝塔软件(http://bt.cn) All rights reserved.
# +-------------------------------------------------------------------
# | Author: 黄文良 <287962566@qq.com>
# +-------------------------------------------------------------------
from flask import request,redirect,g
from BTPanel import session,cache
from datetime import datetime
import os,public,json,sys
class dict_obj:
    def __contains__(self, key):
        return getattr(self,key,None)
    def __setitem__(self, key, value): setattr(self,key,value)
    def __getitem__(self, key): return getattr(self,key,None)
    def __delitem__(self,key): delattr(self,key)
    def __delattr__(self, key): delattr(self,key)
    def get_items(self): return self

class panelSetup:
    def init(self):
        ua = request.headers.get('User-Agent')
        if ua:
            ua = ua.lower();
            if ua.find('spider') != -1 or ua.find('bot') != -1: return redirect('https://www.baidu.com');

        g.version = '6.8.4'
        g.title =  public.GetConfigValue('title')
        g.uri = request.path
        session['version'] = g.version;
        session['title'] = g.title
        return None

class panelAdmin(panelSetup):
    setupPath = '/www/server'

    #本地请求
    def local(self):
        result = panelSetup().init()
        if result: return result
        result = self.checkLimitIp()
        if result: return result
        result = self.setSession();
        if result: return result
        result = self.checkClose();
        if result: return result
        result = self.checkWebType();
        if result: return result
        result = self.checkDomain();
        if result: return result
        result = self.checkConfig();
        #self.checkSafe();
        self.GetOS();

    #检查IP白名单
    def checkAddressWhite(self):
        token = self.GetToken();
        if not token: return redirect('/login');
        if not request.remote_addr in token['address']: return redirect('/login');

    #检查IP限制
    def checkLimitIp(self):
        if os.path.exists('data/limitip.conf'):
            iplist = public.ReadFile('data/limitip.conf')
            if iplist:
                iplist = iplist.strip();
                if not request.remote_addr in iplist.split(','): return redirect('/login')

    #设置基础Session
    def setSession(self):
        session['menus'] = sorted(json.loads(public.ReadFile('config/menu.json')),key=lambda x:x['sort'])
        session['yaer'] = datetime.now().year
        session['download_url'] = 'http://download.bt.cn';
        if not 'brand' in session:
            session['brand'] = public.GetConfigValue('brand');
            session['product'] = public.GetConfigValue('product');
            session['rootPath'] = '/www'
            session['download_url'] = 'http://download.bt.cn';
            session['setupPath'] = session['rootPath'] + '/server';
            session['logsPath'] = '/www/wwwlogs';
            session['yaer'] = datetime.now().year
        if not 'menu' in session:
            session['menu'] = public.GetLan('menu');
        if not 'lan' in session:
            session['lan'] = public.GetLanguage();
        if not 'home' in session:
            session['home'] = 'http://www.bt.cn';

    #检查Web服务器类型
    def checkWebType(self):
        if os.path.exists(self.setupPath + '/nginx'):
            session['webserver'] = 'nginx'
        else:
            session['webserver'] = 'apache'
        if os.path.exists(self.setupPath+'/'+session['webserver']+'/version.pl'):
            session['webversion'] = public.ReadFile(self.setupPath+'/'+session['webserver']+'/version.pl').strip()
        filename = self.setupPath+'/data/phpmyadminDirName.pl'
        if os.path.exists(filename):
            session['phpmyadminDir'] = public.ReadFile(filename).strip()

    #检查面板是否关闭
    def checkClose(self):
        if os.path.exists('data/close.pl'):
            return redirect('/close');

    #检查域名绑定
    def checkDomain(self):
        try:
            if not 'login' in session:
                if not self.get_sk():
                    return redirect('/login')
            else:
                if session['login'] == False: return redirect('/login')
            tmp = public.GetHost()
            domain = public.ReadFile('data/domain.conf')
            if domain:
                if(tmp.strip().lower() != domain.strip().lower()): return redirect('/login')
        except:
            return redirect('/login')

    #获取sk
    def get_sk(self,):
        save_path = '/www/server/panel/config/api.json'
        if not os.path.exists(save_path): return False
        api_config = json.loads(public.ReadFile(save_path))
        if not api_config['open']: return False
        from BTPanel import get_input
        get = get_input()
        if not 'request_token' in get: return False
        if not 'request_time' in get: return False
        client_ip = public.GetClientIp()
        if not client_ip in api_config['limit_addr']: return False
        request_token = public.md5(get.request_time + api_config['token'])
        if get.request_token == request_token:
            return True
        return False

    #检查系统配置
    def checkConfig(self):
        if not 'config' in session:
            session['config'] = public.M('config').where("id=?",('1',)).field('webserver,sites_path,backup_path,status,mysql_root').find();
            if not 'email' in session['config']:
                session['config']['email'] = public.M('users').where("id=?",('1',)).getField('email');
            if not 'address' in session:
                session['address'] = public.GetLocalIp()

    def checkSafe(self):
        mods = ['/','/site','/ftp','/database','/plugin','/soft','/public'];
        if not os.path.exists('/www/server/panel/data/userInfo.json'):
            if 'vip' in session: del(session.vip);
        if not request.path in mods: return True
        if 'vip' in session: return True

        import panelAuth
        data = panelAuth.panelAuth().get_order_status(None);
        try:
            if data['status'] == True:
                session.vip = data
                return True
            return redirect('/vpro');
        except:pass
        return False

    #获取操作系统类型
    def GetOS(self):
        if not 'server_os' in session:
            tmp = {}
            if os.path.exists('/etc/redhat-release'):
                tmp['x'] = 'RHEL';
                tmp['osname'] = public.ReadFile('/etc/redhat-release').split()[0];
            elif os.path.exists('/usr/bin/yum'):
                tmp['x'] = 'RHEL';
                tmp['osname'] = public.ReadFile('/etc/issue').split()[0];
            elif os.path.exists('/etc/issue'):
                tmp['x'] = 'Debian';
                tmp['osname'] = public.ReadFile('/etc/issue').split()[0];
            session['server_os'] = tmp


from sklearn import svm
from hog import HOG
from sklearn.svm import LinearSVC
import detection
import argparse
import cPickle

# construct the argument parse and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-d", "--dataset", required = True,
	help = "path to the dataset file")
ap.add_argument("-m", "--model", required = True,
	help = "path to where the model will be stored")
args = vars(ap.parse_args())

# load the dataset and initialize the data matrix
(digits, target) = detection.load_digits(args["dataset"])
data = []

# initialize the HOG descriptor
hog = HOG(orientations = 18, pixelsPerCell = (10, 10),
	cellsPerBlock = (1, 1), normalize = True)

# loop over the images
for image in digits:
	# deskew the image, center it
	image = detection.deskew(image, 20)
	image = detection.center_extent(image, (20, 20))

	# describe the image and update the data matrix
	hist = hog.describe(image)

	data.append(hist)

model = LinearSVC(random_state=42)

model.fit(data, target)

# dump the model to file
f = open(args["model"], "w")
f.write(cPickle.dumps(model))
f.close()


import json
import pandas as pd
import pandas.testing as pdt
import pytest

from solarforecastarbiter.io import utils

# data for test Dataframe
TEST_DICT = {'value': [2.0, 43.9, 338.0, -199.7, 0.32],
             'quality_flag': [1, 1, 9, 5, 2]}

DF_INDEX = pd.DatetimeIndex(start=pd.Timestamp('2019-01-24T00:00'),
                            freq='1min',
                            periods=5,
                            tz='UTC', name='timestamp')
TEST_DATA = pd.DataFrame(TEST_DICT, index=DF_INDEX)

@pytest.mark.parametrize('dump_quality,default_flag,flag_value', [
    (False, None, 1),
    (True, 2, 2)
])
def test_obs_df_to_json(dump_quality, default_flag, flag_value):
    td = TEST_DATA.copy()
    if dump_quality:
        del td['quality_flag']
    converted = utils.observation_df_to_json_payload(td, default_flag)
    converted_dict = json.loads(converted)
    assert 'values' in converted_dict
    values = converted_dict['values']
    assert len(values) == 5
    assert values[0]['timestamp'] == '2019-01-24T00:00:00Z'
    assert values[0]['quality_flag'] == flag_value
    assert isinstance(values[0]['value'], float)

def test_obs_df_to_json_no_quality():
    td = TEST_DATA.copy()
    del td['quality_flag']
    with pytest.raises(KeyError):
        utils.observation_df_to_json_payload(td)

def test_obs_df_to_json_no_values():
    td = TEST_DATA.copy().rename(columns={'value': 'val1'})
    with pytest.raises(KeyError):
        utils.observation_df_to_json_payload(td)

def test_forecast_series_to_json():
    series = pd.Series([0, 1, 2, 3, 4], index=pd.date_range(
        start='2019-01-01T12:00Z', freq='5min', periods=5))
    expected = [{'value': 0.0, 'timestamp': '2019-01-01T12:00:00Z'},
                {'value': 1.0, 'timestamp': '2019-01-01T12:05:00Z'},
                {'value': 2.0, 'timestamp': '2019-01-01T12:10:00Z'},
                {'value': 3.0, 'timestamp': '2019-01-01T12:15:00Z'},
                {'value': 4.0, 'timestamp': '2019-01-01T12:20:00Z'}]
    json_out = utils.forecast_object_to_json(series)
    assert json.loads(json_out)['values'] == expected

def test_json_payload_to_observation_df(observation_values,
                                        observation_values_text):
    out = utils.json_payload_to_observation_df(
        json.loads(observation_values_text))
    pdt.assert_frame_equal(out, observation_values)

def test_json_payload_to_forecast_series(forecast_values,
                                         forecast_values_text):
    out = utils.json_payload_to_forecast_series(
        json.loads(forecast_values_text))
    pdt.assert_series_equal(out, forecast_values)

def test_empty_payload_to_obsevation_df():
    out = utils.json_payload_to_observation_df({'values': []})
    assert set(out.columns) == {'value', 'quality_flag'}
    assert isinstance(out.index, pd.DatetimeIndex)

def test_empty_payload_to_forecast_series():
    out = utils.json_payload_to_forecast_series({'values': []})
    assert isinstance(out.index, pd.DatetimeIndex)

def test_hidden_token():
    ht = utils.HiddenToken('THETOKEN')
    assert str(ht) != 'THETOKEN'
    assert ht.token == 'THETOKEN'


#  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License").
#    You may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

import time
from multiprocessing import Queue, synchronize
from typing import Any, Dict, Optional, cast

import boto3
import yaml
from aws_orbit_image_replicator import load_config, logger

def _generate_buildspec(repo_host: str, repo_prefix: str, src: str, dest: str) -> Dict[str, Any]:
    repo = dest.replace(f"{repo_host}/", "").split(":")[0]
    build_spec = {
        "version": 0.2,
        "phases": {
            "install": {
                "runtime-versions": {"python": 3.7, "docker": 19},
                "commands": [
                    (
                        "nohup /usr/sbin/dockerd --host=unix:///var/run/docker.sock "
                        "--host=tcp://0.0.0.0:2375 --storage-driver=overlay&"
                    ),
                    'timeout 15 sh -c "until docker info; do echo .; sleep 1; done"',
                ],
            },
            "pre_build": {
                "commands": [
                    f"aws ecr get-login-password | docker login --username AWS --password-stdin {repo_host}",
                    (
                        f"aws ecr create-repository --repository-name {repo} "
                        f"--tags Key=Env,Value={repo_prefix} || echo 'Already exists'"
                    ),
                ]
            },
            "build": {"commands": [f"docker pull {src}", f"docker tag {src} {dest}", f"docker push {dest}"]},
        },
    }
    logger.debug("BuildSpec: %s", build_spec)
    return build_spec

def _replicate_image(config: Dict[str, Any], src: str, dest: str) -> str:
    logger.info("Replicating Image: %s -> %s", src, dest)

    buildspec = yaml.safe_dump(_generate_buildspec(config["repo_host"], config["repo_prefix"], src, dest))
    logger.debug("BuildSpec:\n%s", buildspec)

    client = boto3.client("codebuild")
    build_id = client.start_build(
        projectName=config["codebuild_project"],
        sourceTypeOverride="NO_SOURCE",
        buildspecOverride=buildspec,
        timeoutInMinutesOverride=config["codebuild_timeout"],
        privilegedModeOverride=True,
    )["build"]["id"]

    logger.info("Started CodeBuild Id: %s", build_id)

    while True:
        build = client.batch_get_builds(ids=[build_id])["builds"][0]
        status: str = build["buildStatus"]
        phase: str = build["currentPhase"]

        logger.debug("CodeBuild Id: %s, Phase: %s,  Status: %s", build_id, phase, status)

        if status == "IN_PROGRESS":
            time.sleep(10)
            continue
        else:
            return status

def replicate(
    config: Dict[str, Any],
    lock: synchronize.Lock,
    replications_queue: Queue,  # type: ignore
    replication_statuses: Dict[str, str],
    replicator_id: int,
) -> int:
    logger.info("Started Replicator Id: %s", replicator_id)
    replication_task: Optional[Dict[str, str]] = None

    while True:
        try:
            load_config(config["in_cluster_deployment"])

            queue_size = replications_queue.qsize()
            logger.info(f"Queue Size: {queue_size}")

            replication_task = cast(Dict[str, str], replications_queue.get(block=True, timeout=None))
            src, dest = replication_task["src"], replication_task["dest"]

            with lock:
                logger.info("Got Replication Task: %s -> %s", src, replication_task["dest"])

                status = replication_statuses[dest]
                if status == "Complete":
                    logger.info("Skipping Completed Task: %s -> %s", src, dest)
                    continue
                elif status.startswith("Failed"):
                    logger.info("Skipping Failed Task: %s -> %s", src, dest)
                    continue
                elif status.startswith("Replicating"):
                    logger.info("Skipping Replicating Task: %s -> %s", src, dest)
                    continue
                else:
                    attempt = int(status.split(":")[1])
                    replication_statuses[dest] = f"Replicating:{attempt}"

            result = _replicate_image(config, src, dest)

            with lock:
                if result == "SUCCEEDED":
                    logger.info("Replication Complete: %s -> %s", src, dest)
                    replication_statuses[dest] = "Complete"
                else:
                    logger.error(
                        "Image Replication Attempt %s Failed: %s -> %s",
                        attempt,
                        src,
                        dest,
                    )
                    replication_statuses[dest] = f"Failed:{attempt}"

        except Exception as e:
            with lock:
                status = replication_statuses[dest]
                attempt = int(status.split(":")[1])
                logger.error(
                    "Image Replication Attempt %s Failed: %s -> %s",
                    attempt,
                    src,
                    dest,
                )
                logger.exception(e)
                replication_statuses[dest] = f"Failed:{attempt}"
        finally:
            replication_task = None
            time.sleep(5)


import numpy as np
import collections as co
import igraph as ig
import operator
import itertools
import argparse
from collections import Counter
import json

import overlap
import show_graph

# Possible optimizations and notes:
#   * Calculating the pair betweennesses is the large bottleneck.
#       * However, calculation of all-pairs-shortest-paths
#          and pair betweennesses are highly parallelizable.
#   * Keep a record of splits or merges?
#       * Right now, we store a lot of redundant information with a new
#           VertexCover item for every split.

def conga(OG, calculate_modularities=None, optimal_count=None):
    """
    Defines the CONGA algorithm outlined in the Gregory 2007 paper
    (An Algorithm to Find Overlapping Community Structure in Networks)

    Returns a CrispOverlap object of all of the covers.
    """

    G = OG.copy()

    comm = G.components()

    # Just in case the original graph is disconnected
    nClusters = len(comm)

    # Store the original ids of all vertices
    G.vs['CONGA_orig'] = [i.index for i in OG.vs]
    allCovers = {nClusters : ig.VertexCover(OG)}
    while G.es:
        split = remove_edge_or_split_vertex(G)
        if split:
            comm = G.components().membership
            cover = get_cover(G, OG, comm)
            nClusters += 1
            # short circuit stuff would go here.
            allCovers[nClusters] = cover
            # if nClusters > 1:
            #     break
    if calculate_modularities is None: calculate_modularities = "lazar"
    return overlap.CrispOverlap(OG, allCovers,
                                    modularity_measure=calculate_modularities,
                                    optimal_count=optimal_count)

def remove_edge_or_split_vertex(G):
    """
    The heart of the CONGA algorithm. Decides which edge should be
    removed or which vertex should be split. Returns True if the
    modification split the graph.
    """
    # has the graph split this iteration?
    split = False
    eb = G.edge_betweenness()

    maxIndex, maxEb = max(enumerate(eb), key=operator.itemgetter(1))
    # We might be able to calculate vertex betweenness and edge
    # betweenness at the same time. The current implementation is slower
    # than the builtin, though.
    vb = G.betweenness()

    # Only consider vertices with vertex betweenness >= max
    # edge betweenness. From Gregory 2007 step 3
    vi = [i for i, b in enumerate(vb) if b > maxEb] #HERE TODO TODO

    edge = G.es[maxIndex].tuple

    if not vi:
        split = delete_edge(G, edge)
    else:
        pb = pair_betweenness(G, vi)
        maxSplit, vNum, splitInstructions = max_split_betweenness(G, pb)
        if maxSplit > maxEb:
            split = split_vertex(G, vNum, splitInstructions[0])
        else:
            split = delete_edge(G, edge)
    return split

def get_cover(G, OG, comm):

    coverDict = co.defaultdict(list)
    for i, community in enumerate(comm):
        coverDict[community].append(int(G.vs[i]['CONGA_orig']))
    return ig.clustering.VertexCover(OG, clusters=list(coverDict.values()))

def delete_edge(G, edge):

    G.delete_edges(edge)
    return check_for_split(G, edge)

def check_for_split(G, edge):

    if edge[0] == edge[1]: return False
    return not G.edge_disjoint_paths(source=edge[0], target=edge[1])

def split_vertex(G, v, splitInstructions):

    new_index = G.vcount()
    G.add_vertex()
    G.vs[new_index]['CONGA_orig'] = G.vs[v]['CONGA_orig']

    # adding all relevant edges to new vertex, deleting from old one.
    for partner in splitInstructions:
        G.add_edge(new_index, partner)
        G.delete_edges((v, partner))

    # check if the two new vertices are disconnected.
    return check_for_split(G, (v, new_index))

def order_tuple(toOrder):

    if toOrder[0] <= toOrder[1]:
        return toOrder
    return (toOrder[1], toOrder[0])

def update_betweenness(G, path, pair, count, relevant):

    weight = 1./count
    pos = 0
    while pos < len(path) - 2:
        if path[pos + 1] in relevant:
            pair[path[pos + 1]][order_tuple((path[pos], path[pos + 2]))] += weight
        pos += 1

def pair_betweenness(G, relevant):

    pair_betweenness = {vertex : {uw : 0 for uw in itertools.combinations(G.neighbors(vertex), 2)} for vertex in relevant}

    for i in G.vs:
        pathCounts = Counter()

        shortest_paths_from_v = G.get_all_shortest_paths(i, to=G.vs[i.index+1:])
        for path in shortest_paths_from_v:
            pathCounts[path[-1]] += 1
        for path in shortest_paths_from_v:
            update_betweenness(G, path, pair_betweenness, pathCounts[path[-1]], set(relevant))
    return pair_betweenness

def create_clique(G, v, pb):
    """
    Given a vertex and its pair betweennesses, returns a k-clique
    representing all of its neighbors, with edge weights determined by the pair
    betweenness scores. Algorithm discussed on page 5 of the CONGA paper.
    """
    neighbors = G.neighbors(v)

    # map each neighbor to its index in the adjacency matrix
    mapping = {neigh : i for i, neigh in enumerate(neighbors)}
    n = len(neighbors)

    # Can use ints instead: (dtype=int). Only works if we use matrix_min
    # instead of mat_min.
    clique = np.matrix(np.zeros((n, n)))

    for uw, score in pb.items():

        clique[mapping[uw[0]], mapping[uw[1]]] = score
        clique[mapping[uw[1]], mapping[uw[0]]] = score

    # Ignore any self loops if they're there. If not, this line
    # does nothing and can be removed.
    np.fill_diagonal(clique, 0)
    return clique

def max_split_betweenness(G, dic):
    """
    Given a dictionary of vertices and their pair betweenness scores, uses the greedy
    algorithm discussed in the CONGA paper to find a (hopefully) near-optimal split.

    Returns a 3-tuple (vMax, vNum, vSpl) where vMax is the max split betweenness,
    vNum is the vertex with said split betweenness, and vSpl is a list of which
    vertices are on each side of the optimal split.
    """
    vMax = 0
    # for every vertex of interest, we want to figure out the maximum score achievable
    # by splitting the vertices in various ways, and return that optimal split
    for v in dic:
        clique = create_clique(G, v,dic[v])

        # initialize a list on how we will map the neighbors to the collapsing matrix
        vMap = [[ve] for ve in G.neighbors(v)]

        # we want to keep collapsing the matrix until we have a 2x2 matrix and its
        # score. Then we want to remove index j from our vMap list and concatenate
        # it with the vMap[i]. This begins building a way of keeping track of how
        # we are splitting the vertex and its neighbors
        while clique.size > 4:
            i,j,clique = reduce_matrix(clique)
            vMap[i] += vMap.pop(j)
        if clique[0,1] >= vMax:
            vMax = clique[0,1]
            vNum = v
            vSpl = vMap
    return vMax,vNum,vSpl

def mat_min(M):
    """
    Given a matrix, find an index of the minimum value (not including the
    diagonal).
    """
    # take a matrix we pass in, and fill the diagonal with the matrix max. This is
    # so that we don't grab any values from the diag.
    np.fill_diagonal(M, float('inf'))

    # figure out the indices of the cell with the lowest value.
    i,j = np.unravel_index(M.argmin(), M.shape)
    np.fill_diagonal(M,0)
    return i, j

def reduce_matrix(M):
    """
    Given a matrix M, collapses the row and column of the minimum value. This is just
    an adjacency matrix way of implementing the greedy "collapse" discussed in CONGA.

    Returns the new matrix and the collapsed indices.
    """
    i,j = mat_min(M)
    #i, j = matrix_min(M)
    # add the ith row to the jth row and overwrite the ith row with those values
    M[i,:] = M[j,:] + M[i,:]

    # delete the jth row
    M = np.delete(M, (j), axis=0)

    # similarly with the columns
    M[:,i] = M[:,j] + M[:,i]
    M = np.delete(M, (j), axis=1)
    np.fill_diagonal(M,0) # not sure necessary.
    return i,j,M

# def pretty_print_cover(G, cover, label='CONGA_index'):
#     """
#     Takes a cover in vertex-id form and prints it nicely
#     using label as each vertex's name.
#     """
#     pp = [G.vs[num] for num in [cluster for cluster in cover]]
#     for count, comm in enumerate(pp):
#         #print("Community {0}:".format(count))
#         for v in comm:
#             #print("\t", end=' ')
#             if label == 'CONGA_index':
#                 a = dict.fromkeys(v.index,format(count))
#                 print(a)
#                 print(v.index)
#             else:
#                 print(v[label])
#         print()

def main():
    # data = []
    # de = []
    # with open('D:\Thesis\Dong Tam\data_dongtam.txt') as json_file:
    #     database = json.load(json_file)
    # for x in database:
    #     datatest = (x["fbid1"], x["fbid2"])
    #     data.append(datatest)
    # G = ig.Graph()
    # G.add_vertices(37017)
    # G.add_edges(data)
    # comn = G.components()
    # de_cn = []
    # for cn in comn:
    #     if len(cn) < 35000:
    #         for i in cn:
    #             de_cn.append(i)
    # de_cn.sort(reverse=True)
    # for j in de_cn:
    #     G.delete_vertices(j)
    # vb = G.betweenness()
    # for x, y in enumerate(vb):
    #     if y < 1.0:
    #         de.append(x)
    # de.sort(reverse=True)
    # for i in range(len(de)):
    #     G.delete_vertices(de[i])
    # print(G)

    database = []
    f = open("D:\DATA\AA-DATN\DATN_GroupDetection\Source_Code\Data set\Data set\dolphins.txt", "r")
    for x in f:
        y = x.split('\n')
        z = y[0].split(' ')
        t = (int(z[0])-1, int(z[1])-1)
        database.append(t)
    G = ig.Graph()
    G.add_vertices(62)
    G.add_edges(database)

    # G = ig.Graph().Famous("Zachary").as_undirected()

    result = conga(G, calculate_modularities="lazar")
    data_graph = result.pretty_print_cover(G, result.optimal_count, label='CONGA_index')
    show_graph.show_graph(G, data_graph)

if __name__ == "__main__":
    main()


#coding:utf-8
import numpy as np
from sklearn import linear_model
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score
from sklearn.model_selection import GridSearchCV
from sklearn.externals import joblib

'''
フレーム数：５
特徴量
・フレーム間の平均(正規化後)*5ch*5kind
・フレーム間の分散(正規化後)*5ch*5kind
・フレーム間の平均(正規化前)*5ch*5kind
・フレーム間の分散(正規化前)*5ch*5kind
・フレーム間での傾き(正規化前)*5ch*5kind
・各チャンネルのパワー（正規化前）
・現フレームの各チャンネルの割合
'''

class ActiveEstimator():
    def __init__(self, channelNum, kindNum, frameNum):
        """
        コンストラクタ
        バッファの初期化を行う。

        Parameters
        ----------
        channelNum : int
            入力センサのチャンネル数
        kindNum : int
            １センサから得られる情報数（例：AF3のTheta,Alpha...など）
        frameNum : int
            バッファリングしておくフレーム数
        """
        self.reshape(channelNum, kindNum, frameNum)

    def clearBuf(self):
        """
        バッファを初期化する。
        """
        self.dataBuf = np.zeros(
            [self.frameNum, self.channelNum, self.kindNum]
            )
        self.dataBufNormed = np.zeros(
            [self.frameNum, self.channelNum, self.kindNum]
            )
        self.bufferCounter = 0

    def reshape(self, channelNum, kindNum, frameNum):
        """
        バッファのサイズを指定された引数でリシェイプする。

        Parameters
        ----------
        channelNum : int
            入力センサのチャンネル数
        kindNum : int
            １センサから得られる情報数（例：AF3のTheta,Alpha...など）
        frameNum : int
            バッファリングしておくフレーム数
        """
        self.channelNum = channelNum
        self.kindNum = kindNum
        self.frameNum = frameNum
        self.clearBuf()
        self.X_act = []
        self.X_sle = []

        self.Y_act = []
        self.Y_sle = []

    def estimate(self, data):
        if self._checkInvalidData(data):
            return -1
        self._buffering(data)
        f = self._calcFeature()
        if f.shape[0] == 1:
            return -1
        return self.clf.predict(f.reshape(1,len(f)))

    def train(self):

        if len(self.Y_sle) > len(self.Y_act):
            ind= np.random.randint(0,len(self.Y_sle),(len(self.Y_act)))
            self.Y_sle = np.array(self.Y_sle)[list(ind)]
            self.X_sle = np.array(self.X_sle)[list(ind)]
        else:
            ind= np.random.randint(0,len(self.Y_act),(len(self.Y_sle)))
            self.Y_act = np.array(self.Y_act)[list(ind)]
            self.X_act = np.array(self.X_act)[list(ind)]

        print(str(len(self.X_act)) + "," + str(len(self.X_sle)))
        self.X = np.r_[self.X_act,self.X_sle]
        self.Y = np.r_[self.Y_act,self.Y_sle]
        # return

        x = np.array(self.X)
        np.savetxt("train_data.csv",x,delimiter=',')
        for i in range(0,x.shape[1]):
            x[:][i] = (x[:][i] - np.min(x[:][i]) ) / (np.max(x[:][i]) - np.min(x[:][i]))
        X_train, X_test, y_train, y_test = train_test_split(
            self.X,
            self.Y,
            test_size=0.2)

        tuned_parameters = [
                            # {'C': [1, 10, 100, 1000], 'kernel': ['linear']},
                            {'C': [1, 10, 100, 1000, 5000, 10000], 'kernel': ['rbf'], 'gamma': [0.01, 0.001, 0.0001]},
                            # {'C': [1, 10, 100, 1000], 'kernel': ['poly'], 'degree': [2, 3, 4], 'gamma': [0.001, 0.0001]},
                            # {'C': [1, 10, 100, 1000], 'kernel': ['sigmoid'], 'gamma': [0.001, 0.0001]}
                            ]
        score = 'accuracy'
        clf = GridSearchCV(
                            SVC(), # 識別器
                            tuned_parameters, # 最適化したいパラメータセット
                            cv=5, # 交差検定の回数
                            scoring=score ) # モデルの評価関数の指定
        clf.fit(X_train, y_train)
        y_pred = clf.predict(X_test)
        print(clf.best_params_)
        print(accuracy_score(y_test, y_pred))
        joblib.dump(clf, 'clf.pkl')

    def loadModel(self, modelpath):
        self.clf = joblib.load('clf.pkl')

    def bufferTrainData(self, data, correctValue):
        """
        学習用データを作成し、バッファする。

        Parameters
        ----------
        data : numpy.array = (channelNum,kindNum)
            入力センサデータ
        correctValue : int
            入力データの正解ラベル
        """
        if self._checkInvalidData(data):
            return -1
        self._buffering(data)
        f = self._calcFeature()
        if f.shape[0] == 1:
            return
        if correctValue == 0:
            self.X_sle.append(f)
            self.Y_sle.append(correctValue)
        else:
            self.X_act.append(f)
            self.Y_act.append(correctValue)

    def _checkInvalidData(self,data):
        return np.any(np.sum(data,axis=1)<=0)

    def _buffering(self, data):
        """
        入力されたデータと過去frameNum分のファイルをバッファする。

        Parameters
        ----------
        data : numpy.array = (channelNum,kindNum)
            入力センサデータ
        """
        self.dataBuf[0:self.frameNum-1] = self.dataBuf[
            1:self.frameNum
            ]
        self.dataBufNormed[0:self.frameNum-1] = self.dataBufNormed[
            1:self.frameNum
            ]
        self.dataBuf[self.frameNum-1] = np.array(data)
        self.dataBufNormed[self.frameNum-1] = np.array(data)/(np.tile(
            np.sum(data, axis=1), self.kindNum).reshape(
                self.channelNum, self.kindNum)
            )
        if self.bufferCounter < self.frameNum:
            self.bufferCounter += 1
        return

    def _calcFeature(self):
        if self.bufferCounter < self.frameNum:
            return np.zeros(1)
        """
        # 生データのframe合計値
        f = np.sum(
            self.dataBuf[:], axis=0).reshape(
            self.channelNum*self.kindNum
            )
        # 生データのframe平均値
        f = np.r_[f, np.average(
            self.dataBuf, axis=0).reshape(
                self.channelNum*self.kindNum)
                ]
        # 生データのframe標準偏差
        f = np.r_[f, np.std(
            self.dataBuf, axis=0).reshape(
                self.channelNum*self.kindNum)
                ]
        """
        # 正規化データのframe合計値
        f = np.sum(
            self.dataBufNormed, axis=0).reshape(
                self.channelNum*self.kindNum)

        # 正規化データのframe平均値
        f = np.r_[f, np.average(
            self.dataBufNormed, axis=0).reshape(
                self.channelNum*self.kindNum)
                ]
        # 正規化データのframe標準偏差
        f = np.r_[f, np.std(
            self.dataBufNormed, axis=0).reshape(
                self.channelNum*self.kindNum)
                ]
        # 最新フレームの正規化データ
        f = np.r_[f, self.dataBufNormed[self.frameNum-1].reshape(
            self.channelNum*self.kindNum)
            ]

        # フレーム間のDiff値
        f = np.r_[f, np.sum(np.diff(self.dataBufNormed,axis=0),axis=0).reshape(self.channelNum*self.kindNum)]

        return f


from tensorflow.lite.python.interpreter import Interpreter
import cv2
import numpy as np

class TFLiteModel:
    def load_model(self, model_path):
        self.interpreter = Interpreter(
            model_path=model_path,)

        self.interpreter.allocate_tensors()
        self.input_details = self.interpreter.get_input_details()
        self.output_details = self.interpreter.get_output_details()
        self.height = self.input_details[0]['shape'][1]
        self.width = self.input_details[0]['shape'][2]
        return self

    def preprocess(self, image):
        image = cv2.resize(image, (self.width, self.height))
        image = (np.asarray(image, dtype=np.float32))/255.0
        image = np.expand_dims(image, axis=0)
        return image

    def get_model_output(self, image):
        self.interpreter.set_tensor(self.input_details[0]['index'], image)
        self.interpreter.invoke()
        outputs = [self.interpreter.get_tensor(self.output_details[i]['index']) for i in range(len(self.output_details))]
        outputs = [np.squeeze(x) for x in outputs]
        return outputs

    def get_model_details(self):
        return self.interpreter.get_tensor_details()

    def model_path(self):
        return None

    def speedtest(self, image):
        image = cv2.imread(image)
        image = self.preprocess(image)
        from tqdm import tqdm
        for i in tqdm(range(1000)):
            output = self.get_model_output(image)

if __name__ == "__main__":
    import argparse, os, glob, traceback
    parser = argparse.ArgumentParser()
    parser.add_argument("--model", default=None, help="pass either a directory or a tflite model", required=True)
    parser.add_argument("--image", default=None, help="pass a sample image", required=False)

    args = parser.parse_args()

    image = None
    if args.image:
        image = cv2.imread(args.image)
    path = args.model
    try:
        tfutil = TFLiteModel().load_model(path)
        print (f"--------{path}---------")
        print ([(i['name'], i['shape']) for i in tfutil.input_details])
        print ([(i['name'], i['shape']) for i in tfutil.output_details])
        for x in tfutil.get_model_details():
            print (x)

        if image is not None:
            image = tfutil.preprocess(image)
            from tqdm import tqdm
            for i in tqdm(range(1000)):
                output = tfutil.get_model_output(image)
            print(output[0].shape)

    except:
        print("Exception in user code:")
        print("-" * 60)
        traceback.print_exc()
        print("-" * 60)


from django.core.paginator import Paginator
from django.db import transaction
from django.shortcuts import render, HttpResponse
from banner.models import Bnner
import json
from django.views.decorators.csrf import csrf_exempt

# 存储轮播图
@csrf_exempt
def save_banner(request):
    try:
        # 获取名称
        title = request.POST.get('title')
        # 获取状态
        status = request.POST.get('status')
        # 获取文件
        pic = request.FILES.get('pic')
        print(title, status, pic)
        if not pic:
            # pic 不存在
            return HttpResponse("图片不能为空")
        # 存入数据库
        with transaction.atomic():
            Bnner.objects.create(title=title, status=status, pic=pic)
        return HttpResponse('成功保存')
    except:
        return HttpResponse('保存失败')

# 序列化所有默认函数
def my_default(b):
    if isinstance(b, Bnner):
        return {'id': b.id, 'title': b.title, 'status': b.status, 'create_time': str(b.create_time), 'pic': str(b.pic)}

# 展示轮播图数据
def show_banner(request):
    print(Bnner.objects.get(pk=1).pic)
    # 从前端获取每页显示行数
    row_num = request.GET.get('rows')
    # 从前端获取页码
    page_num = request.GET.get('page')
    # 从数据库获取数据 进行分页管理
    pgtor = Paginator(Bnner.objects.all(), per_page=row_num)
    # 生成page
    pg = pgtor.page(page_num)
    # 返回的数据
    data = {
        'page': page_num,
        'total': pgtor.num_pages,
        'records': pgtor.count,
        'rows': list(pg)
    }
    # 进行序列化
    json_str = json.dumps(data, default=my_default)
    return HttpResponse(json_str)

# 编辑轮播图列表
@csrf_exempt
def edit_banner(request):
    try:
        # 获取oper,添，修改edit，删除del
        oper = request.POST.get('oper')
        print(oper)
        # 判断是修改，还是删除
        if oper == 'edit':
            # 修改 , 获取 需要修改的数据
            id = request.POST.get('id')
            title = request.POST.get('title')
            status = request.POST.get('status')
            # 通过id获取modal对象
            banner = Bnner.objects.get(pk=id)
            # 进行修改
            with transaction.atomic():
                banner.title = title
                banner.status = status
                banner.save()
            return HttpResponse('成功修改')
        elif oper == 'del':
            # 获取id
            id = request.POST.get('id')
            # 从数据库删除
            Bnner.objects.get(pk=id).delete()
            return HttpResponse('成功删除')
    except:
        return HttpResponse('操作失败')


import urllib
import urllib2
import urlparse
import xml.etree.cElementTree as ET

from datetime import datetime, tzinfo
from decimal import Decimal
from datetime import datetime, timedelta
from django.contrib.sites.models import Site
from django.core import urlresolvers
from django.utils.datastructures import SortedDict
from urllib2 import HTTPError

from hiicart.gateway.amazon import fps
from hiicart.gateway.amazon.errors import AmazonGatewayError
from hiicart.gateway.amazon.settings import default_settings
from hiicart.gateway.base import IPNBase
from hiicart.models import HiiCart, Payment

_FPS_NS = "{http://fps.amazonaws.com/doc/2008-09-17/}"

class AmazonIPN(IPNBase):
    """Payment Gateway for Amazon Payments."""

    def __init__(self):
        super(AmazonIPN, self).__init__("amazon", default_settings)

    def _process_error(self, response):
        """Process error xml into (code, message, requestId)."""
        xml = ET.XML(response)
        code = xml.find(".//Code")
        message = xml.find(".//Message")
        request_id = xml.find(".//RequestID")
        if code is not None and message is not None and request_id is not None:
            return (code.text, message.text, request_id.text)
        return None

    def accept_payment(self, cart, data):
        """Record payment received from IPN."""
        self._update_with_cart_settings(cart)
        total = None
        if "transactionAmount" in data and data["transactionAmount"].startswith("USD "):
            total = Decimal(data["transactionAmount"][4:])
        if data["transactionStatus"] == "PENDING":
            transaction_id = data["transactionId"]
            pending = cart.payments.filter(transaction_id=transaction_id)
            if not pending: # Could already be created in make_pay_request
                self._create_payment(cart, total, transaction_id, "PENDING")
        elif data["transactionStatus"] == "SUCCESS":
            # Was this a pending payment?
            transaction_id = data["transactionId"]
            pending = cart.payments.filter(transaction_id=transaction_id, state="PENDING")
            if pending:
                pending[0].state = "PAID"
                pending[0].save()
            elif cart.payments.filter(transaction_id=transaction_id).count() == 0: # No duplicate payments
                self._create_payment(cart, total, transaction_id, "PAID")
            cart.update_state()
            self.begin_recurring(cart)
        elif data["transactionStatus"] == "CANCELLED":
            message = "Purchase %i (txn:%s) was cancelled with message '%s'" % (
                      cart.id, data['transactionId'], data['statusMessage'])
            self.log.warn(message)
            transaction_id = data["transactionId"]
            cancelled = cart.payments.filter(transaction_id=transaction_id)
            for p in cancelled:
                p.state = "CANCELLED"
                p.notes.create(text=message)
                p.save()
            cart.update_state()
        elif data["transactionStatus"] == "FAILURE":
            self.log.warn("Purchase %i (txn:%s) failed with message '%s'" % (
                cart.id, data['transactionId'], data['statusMessage']))
            cart.update_state()

    def begin_recurring(self, cart):
        """Save token and mark recurring item as active."""
        self._update_with_cart_settings(cart)
        if cart.recurringlineitems.count() == 0:
            return
        for ri in cart.recurringlineitems.all():
            ri.is_active = True
            ri.save()
        cart.update_state()

    def end_recurring(self, cart, token):
        """Mark a recurring item as inactive."""
        self._update_with_cart_settings(cart)
        if cart.recurringlineitems.count() == 0:
            return
        for ri in cart.recurringlineitems.all():
            ri.is_active = False
            ri.save()
        cart.update_state()

    def make_pay_request(self, cart, token, caller_reference=None):
        """
        Make a Pay request for the purchase with the given token.
        Return status received from Amazon.
        """
        self._update_with_cart_settings(cart)
        if caller_reference is None:
            caller_reference = cart.cart_uuid
        response = fps.do_fps("Pay", "GET", self.settings,
                   **{"CallerReference" : caller_reference,
                      "SenderTokenId" : token,
                      "TransactionAmount.CurrencyCode" : "USD",
                      "TransactionAmount.Value" : cart.total})
        xml = ET.XML(response)
        status = xml.find(".//%sTransactionStatus" % _FPS_NS)
        if status is None:
            error = self._process_error(response)
            if not error:
                msg = "Pay request failed for purchase '%s' with response: %s" % (
                            cart, response)
            else:
                msg = "Pay request failed for purchase '%s' with code/message: '%s' '%s' (request id: '%s')" % (
                            cart, error[0], error[1], error[2])
            self.log.warn(msg)
            cart.notes.create(text=msg)
            # An error code, maybe?
            code = xml.find(".//Code")
            if code is not None:
                return code.text
            return None
        else:
            # This work for pending payments?
            transaction_id = xml.find(".//%sTransactionId" % _FPS_NS).text
            if status.text == "Pending":
                self._create_payment(cart, cart.total, transaction_id, "PENDING")
            elif status.text == "Success":
                pending = cart.payments.filter(transaction_id=transaction_id,
                                               state="PENDING")
                if pending:
                    pending[0].state = "PAID"
                    pending[0].save()
                else:
                    self._create_payment(cart, cart.total, transaction_id, "PAID")
                self.begin_recurring(cart)
            else: # Cancelled or Failure
                msg = "Pay request for purchase '%s' returned status '%s'. \nFull response: %s" % (
                            cart, response)
                self.log.warn(msg)
                cart.notes.create(text=msg)
            cart.update_state()
            return status.text

    def save_recurring_token(self, cart, token):
        """Save recurring use token for future use."""
        self._update_with_cart_settings(cart)
        if cart.recurringlineitems.count() == 0:
            return
        ri = cart.recurringlineitems.all()[0]
        ri.payment_token = token
        ri.save()

    def verify_signature(self, raw_data, http_method, endpoint_uri, cart):
        self._update_with_cart_settings(cart)
        response = fps.do_fps("VerifySignature", http_method, self.settings,
                              UrlEndPoint=endpoint_uri,
                              HttpParameters=raw_data)
        xml = ET.XML(response)
        el = xml.find(".//{http://fps.amazonaws.com/doc/2008-09-17/}VerificationStatus")
        return el is not None and el.text == "Success"


#!/usr/bin/env python3
import socket, os, os.path, time, pika, logging, datetime, sys
from multiprocessing import Queue, Value
import multiprocessing
from waggle.protocol.PacketHandler import *
from waggle.protocol.utils import packetmaker
sys.path.append('../NC/')
from NC_configuration import *

sys.path.append('../DataCache/')
from send2dc import send

logging.basicConfig()
logger = logging.getLogger(__name__)

"""
    The external communicator is the communication channel between the cloud and the DC. It consists of four processes: two pika clients for pushing and pulling to the cloud and two clients for pushing
    and pulling to the data cache.
"""
class external_communicator(object):
    """
        This class is a convenience class that stores shared variables amongst all instances.
    """

    def __init__(self):
        pass

    incoming = Queue() #stores messages to push into DC
    outgoing = Queue() #stores messages going out to cloud
    cloud_connected = Value('i') #indicates if the cloud is or is not connected. Clients only connect to DC when cloud is connected.
    #params = pika.connection.URLParameters(CLOUD_ADDR) #SSL

"""
    A pika client for rabbitMQ to push messages to the cloud. When the cloud is connected, the pull client sends pull requests and puts messages into the outgoing queue.
    When the outgoing queue is not empty, this pika client will connect and push those messages to the cloud.
"""
def pika_push():

    comm = external_communicator()
    #params = comm.params
    logger.info('Pika push started...\n')
    while True:
        connection=None

        # try to connect to cloud
        try:
            connection = pika.BlockingConnection(pika_params)
        except Exception as e:
            logger.warning( 'Pika_push currently unable to connect to cloud (%s:%d) (queue: %s) : %s' % (pika_params.host, pika_params.port , conf['QUEUENAME'], str(e)) )
            comm.cloud_connected.value = 0 #set the flag to 0 when not connected to the cloud. I
            time.sleep(5)
            break

        logger.warning( 'Pika_push connected to cloud (%s:%d)' % (pika_params.host, pika_params.port) )

        # get channel
        try:
            channel = connection.channel()
            comm.cloud_connected.value = 1 #set the flag to true when connected to cloud
            #Declaring the queue
            channel.queue_declare(queue=conf['QUEUENAME'])
            logger.info("Pika push got queue \"%s\"." % (conf['QUEUENAME']))
        except Exception as e:
            logger.warning('Pika_push unable to get channel (%s:%d) (queue: %s) : %s' % (pika_params.host, pika_params.port , conf['QUEUENAME'], str(e)) )
            comm.cloud_connected.value = 0 #set the flag to 0 when not connected to the cloud. I
            time.sleep(5)
            break

        try:
            send_registrations() #sends registration for each node and node controller configuration file
        except Exception as e:
            logger.error('registration failed: %s' % (str(e)) )
            comm.cloud_connected.value = 0 #set the flag to 0 when not connected to the cloud. I
            time.sleep(5)
            break

        while True:

            #while comm.outgoing.empty(): #sleeps until there are messages to send
            #    time.sleep(1)

            time_0 = time.time()
            msg = comm.outgoing.get() # gets the first item in the outgoing queue
            time_1 = time.time()
            logger.debug('Pika_push: sending message to cloud. (waited %d seconds)' % (time_1 - time_0))

            try:

                channel.basic_publish(exchange='waggle_in', routing_key= 'in', body= msg) #sends to cloud
                #connection.close()

            except pika.exceptions.ConnectionClosed:
                logger.debug("Pika push connection closed. Waiting and trying again " + str(datetime.datetime.now()) + '\n')
                comm.cloud_connected.value = 0
                time.sleep(5)
                break #need to break this loop to reconnect
            except Exception as e:
                logger.error("Pika push encounterd an unexpected error: %s" % (str(e)))
                comm.cloud_connected.value = 0
                time.sleep(5)
                break
            time_2 = time.time()
            logger.debug('Pika_push: sending message took %d seconds' % (time_2 - time_1))

        if connection:
            try:
                connection.close(0)
            except:
                pass

"""
     A pika client for pulling messages from the cloud. If messages are sent from the cloud, this client will put them into the incoming queue.
"""
def pika_pull():

    logger.info('Pika pull started...\n')
    comm = external_communicator()
    #params = comm.params
    #params = pika.connection.URLParameters("amqps://waggle:waggle@10.10.10.108:5671/%2F") #SSL

    if not conf['QUEUENAME']:
        logger.warning("QUEUENAME is not defined... waiting..")
        time.sleep(5)
        return

    while True:

        try:
            connection = pika.BlockingConnection(pika_params)
        except Exception as e:
            logger.warning('(pika.BlockingConnection) Pika_pull currently unable to connect to cloud. Waiting before trying again. ' + str(e))
            comm.cloud_connected.value = 0 #set the flag to 0 when not connected to the cloud
            time.sleep(5)
            continue

        try:
            channel = connection.channel()
        except Exception as e:
            logger.warning('(pika.BlockingConnection) Could not create channel. ' + str(e))
            comm.cloud_connected.value = 0 #set the flag to 0 when not connected to the cloud
            time.sleep(5)
            continue

        logger.info('Pika pull connection successful.\n')
        comm.cloud_connected.value = 1 #sets indicator flag to 1 so clients will connect to data cache

        try:
            channel.queue_declare(queue=conf['QUEUENAME'])
        except Exception as e:
            logger.debug('Cannot declare queuename (%s) : %s' % (conf['QUEUENAME'], str(e)))
            pass

        try:
            channel.basic_consume(callback, queue=conf['QUEUENAME'])
        except:
            logger.warning('(channel.basic_consume) failed: '+ str(e))
            comm.cloud_connected.value = 0 #set the flag to 0 when not connected to the cloud
            time.sleep(5)
            continue

        try:
            #loop that waits for data
            channel.start_consuming() #TODO Can this process still publish to RabbitMQ while this is continuously looping? If so, The pika_pull and pika_push processes can and should be combined.
        except:
            logger.warning('(channel.start_consuming) something wrong:' + str(e))
            comm.cloud_connected.value = 0 #set the flag to 0 when not connected to the cloud

        time.sleep(5)

    connection.close()

#pulls the message from the cloud and puts it into incoming queue
def callback(ch, method, properties, body):
    comm = external_communicator()
    logger.debug('Callback received message from cloud, length %d ' % (len(body)))
    comm.incoming.put(body)
    ch.basic_ack(delivery_tag=method.delivery_tag) #RabbitMQ will not delete message until ack received

"""
    A client process that connects to the data cache if the cloud is currently connected.  Once a message is recieved, it is put into the outgoing queue.

"""

def external_client_pull():

    logger.info('External client pull started...\n')
    comm = external_communicator()
    while True:
        try:
            if comm.cloud_connected.value == 1: #checks if the cloud is connected
                try:
                    if os.path.exists('/tmp/Data_Cache_server'):
                        client_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
                        client_sock.connect('/tmp/Data_Cache_server') #connects to server
                        #logger.debug("Client_pull connected to data cache... ")
                        #sends the pull request indicating that it is an outgoing pull request.
                        #TODO This can be improved (both clients combined into one) if there is a better way to distinguish incoming vs outgoing pull and pull vs push requests.
                        data = '|o'.encode('iso-8859-1')
                        client_sock.send(data)
                        msg = client_sock.recv(4028) #can be changed
                        if msg:
                            if msg != 'False'.encode('iso-8859-1'):
                                logger.debug("put message from DC in outgoing queue")
                                comm.outgoing.put(msg) #puts the message in the outgoing queue
                            else:
                                #logger.debug("DC has no message for me.")
                                time.sleep(1)
                        else:
                            logger.debug("DC has no message for me. (msg==None !?)")
                            time.sleep(1)

                        client_sock.close()
                    else:
                        logger.warning('External client pull unable to connect to the data cache... path does not exist!\n')
                        time.sleep(5)
                except:
                    logger.info('External client pull disconnected from data cache. Waiting and trying again.\n')
                    client_sock.close()
                    time.sleep(5)
            else:
                logger.debug('External client pull...cloud is not connected. Waiting and trying again.')
                time.sleep(5)
        except KeyboardInterrupt as k:
                logger.info("External client pull shutting down.\n")
                break
    client_sock.close()

"""
    A client process that connects to the data cache and pushes incoming messages.
"""

def external_client_push():

    logger.info('External client push started...\n')
    comm = external_communicator()

    while True:
        while comm.incoming.empty(): #sleeps until a message arrives in the incoming queue
            time.sleep(1)
        try:
            if os.path.exists('/tmp/Data_Cache_server'):
                client_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
                client_sock.connect('/tmp/Data_Cache_server') #opens socket when a message is in the queue
                logger.debug("Client_push connected to data cache... ")
                msg = comm.incoming.get() #gets the incoming message from the queue
                client_sock.send(msg) #sends msg
                client_sock.close()
            else:
                logger.info("External client push-Unable to connect to Data Cache.\n")
                time.sleep(5)
        except KeyboardInterrupt as k:
            logger.info("External client push shutting down.\n")
            break
    client_sock.close()

def send_registrations():
    """
        Sends registration message for NC and GNs and configuration file for node controller.
    """
    #loops through the list of nodes and send a registration for each one
    #logger.debug("number of devices: "+str(len(DEVICE_DICT.keys())))

    #for device in DEVICE_DICT.keys():
    #    logger.debug("try to send registration for device "+device)
    #    header_dict = {
    #        "msg_mj_type" : ord('r'),
    #        "msg_mi_type" : ord('r'),
    #        "s_uniqid"    : int("0x" + device, 0)
    #        }
    #    msg = str(conf['QUEUENAME'])
    #    try:
    #        packet = pack(header_dict, message_data = msg)
    #    except Exception as e:
    #        logger.error("pack failed: "+str(e))
    #        raise
    #
    #
    #    try:
    #        for pack_ in packet:
    #            try:
    #                send(pack_)
    #            except Exception as e:
    #                logger.error("send(pack_) failed: "+str(e))
    #                raise
    #
    #    except Exception as e:
    #        logger.error("for loop send(pack_) failed: "+str(e))
    #        raise
    #
    #    logger.debug("sent registration for device "+device)

    #send nodecontroller configuration file
    config = get_config() #this function is in NC_configuration
    logger.info("Sending config for registration: "+config)
    try:
        packet = make_config_reg(config) # this is major, minor: "rn"
        for pack_ in packet:
            send(pack_)
    except Exception as e:
        logger.error(str(e))
        raise

external_communicator_name2func = {}

external_communicator_name2func["pika_pull"]=pika_pull
external_communicator_name2func["pika_push"]=pika_push
external_communicator_name2func["client_push"]=external_client_push
external_communicator_name2func["client_pull"]=external_client_pull

if __name__ == "__main__":
    name2process={}

    try:

        for name, function in external_communicator_name2func.items():
            new_process = multiprocessing.Process(target=function, name=name)
            new_process.start()
            name2process[name]=new_process
            logger.info(name+' has started.')

        while True:
            pass

    except KeyboardInterrupt as k:

        for name, subhash in external_communicator_name2func.items():
            logger.info( '(KeyboardInterrupt) shutting down ' + name)
            name2process[name].terminate()
        logger.info('Done.')


from copy import deepcopy
from dotenv import load_dotenv, find_dotenv
import os
import subprocess
import sys
import time

import click
import yaml

from dr import config
from dr import kubectl
from dr import travis as travis_lib
from dr.gcloud import auth as gcloud_auth, kubectl_path
from dr.aliyun import auth as aliyun_kube_init
from dr.options import global_options
from dr.templates import tpl
from dr.util import retry, shell, is_aliyun

def _get_old_versions(cfg):
    anti_selector = {
        'selector': 'app=%s,track=%s,version!=%s' % (
            cfg.get('app_name'), cfg.get('release_track'), cfg['version']),
        'namespace': cfg.get('namespace'), }
    svc = kubectl.get('deploy', kubectl=kubectl_path, **anti_selector)
    return [
        x['metadata']['labels']['version'] for x in svc
        if x['metadata']['labels']['version'] != cfg['version']]

def _login(fn):
    def wrapper(*args, **kwargs):
        if is_aliyun(**kwargs):
            aliyun_kube_init(**kwargs)
        else:
            gcloud_auth(**kwargs)
        return fn(*args, **kwargs)
    wrapper.__name__ = fn.__name__
    return wrapper

def _validate_cfg(cfg, method='create'):
    click.echo('Validating configuration')
    kwargs = {'dry-run': 'true'}
    getattr(kubectl, method)(stdin=cfg, kubectl=kubectl_path, **kwargs)
    click.echo('Configuration is valid')

def _log_pod(pod, cfg, follow=False):
    kubectl.logs(
        pod['metadata']['name'],
        kubectl=kubectl_path,
        container=cfg.get('app_name'),
        namespace=cfg.get('namespace'),
        follow=follow, )

@click.group()
def kube(*args, **kwargs):
    '''
    Tools for generating kubernetes configuration and deploying it to the
    cluster.
    '''
    pass

@click.command()
@click.option(
    '--commit', '-C', help='Define the git commit for annotations')
@click.option(
    '--output', '-o', help='Location to write output')
@click.option(
    '--image-id', '-i', help='Override default docker image id')
@click.option(
    '--image-pull-secret', '-I', help='Define an imagePullSecret')
@global_options
@config.apply
def gen(output=None, **kwargs):
    '''
    Generate kubernetes configuration
    '''
    cfg = kwargs['config']
    static_tpl = tpl.get_template('static.tpl.yaml')
    svc_tpl = tpl.get_template('svc.tpl.yaml')
    dynamic_tpl = tpl.get_template('dynamic.tpl.yaml')

    rendered = '\n'.join([
        static_tpl.render(**cfg),
        svc_tpl.render(**cfg),
        dynamic_tpl.render(**cfg)])

    if output is None:
        click.echo(rendered)
    else:
        with open(output, 'w') as f:
            f.write(rendered)

@click.command()
@global_options
@config.apply
@_login
def validate(**kwargs):
    '''
    Validate Kubernetes configuration
    '''
    cfg = kwargs['config']

    static_tpl = tpl.get_template('static.tpl.yaml')
    svc_tpl = tpl.get_template('svc.tpl.yaml')
    dynamic_tpl = tpl.get_template('dynamic.tpl.yaml')

    # Validate configuration before deploy
    _validate_cfg('\n'.join(
        x.render(**cfg) for x in [static_tpl, svc_tpl, dynamic_tpl]))

@click.command()
@global_options
@config.apply
def clean(output=None, **kwargs):
    '''Clean up kubernetes resources for the current project'''
    cfg = kwargs['config']

    # static_tpl = tpl.get_template('static.tpl.yaml')
    svc_tpl = tpl.get_template('svc.tpl.yaml')
    dynamic_tpl = tpl.get_template('dynamic.tpl.yaml')

    rendered = '\n'.join([
        # static_tpl.render(**cfg),
        svc_tpl.render(**cfg),
        dynamic_tpl.render(**cfg)])

    click.confirm('Are you sure you want to delete all resources?', abort=True)

    click.echo('Deleting all resources for service: %s' % cfg['app_name'])
    kubectl.deletef(stdin=rendered, kubectl=kubectl_path)

@click.command()
@click.option(
    '--commit', '-C', help='Define the git commit for annotations')
@click.option(
    '--image-id', '-i', help='Override default docker image id')
@click.option(
    '--image-pull-secret', '-I', help='Define an imagePullSecret')
@click.option(
    '--timeout',
    default=300,
    help='How long to wait for container test to pass before giving up.'
)
@global_options
@config.apply
@_login
def deploy(**kwargs):
    '''
    Deploy application to specified cluster using generated configuration
    '''
    cfg = kwargs['config']

    static_tpl = tpl.get_template('static.tpl.yaml')
    svc_tpl = tpl.get_template('svc.tpl.yaml')
    dynamic_tpl = tpl.get_template('dynamic.tpl.yaml')

    # Validate configuration before deploy
    _validate_cfg('\n'.join(
        x.render(**cfg) for x in [static_tpl, svc_tpl, dynamic_tpl]))

    selector = {
        'selector': 'app=%s,track=%s,version=%s' % (
            cfg.get('app_name'), cfg.get('release_track'), cfg.get('version')),
        'namespace': cfg.get('namespace'), }

    if kubectl.exists('deploy', kubectl=kubectl_path, **selector):
        click.secho(
            'Skipping deployment since replication controller'
            ' with version %s already exists' % cfg.get('version'), fg='red')
        return

    pf = None
    try:
        click.echo('Creating/updating static resources...')
        kubectl.apply(stdin=static_tpl.render(**cfg), kubectl=kubectl_path)

        test_cfg = deepcopy(cfg)
        test_cfg['test_replica_count'] = 1

        click.echo('Creating new dynamic resources...')
        kubectl.create(
            stdin=dynamic_tpl.render(**test_cfg), kubectl=kubectl_path)

        click.echo('Preparing to run container test...')
        pod = kubectl.wait_for_pods(
            kubectl=kubectl_path, timeout=kwargs['timeout'], **selector)[0]
        ports = [p['container_port'] for p in cfg.get('ports', [])]

        test = cfg.get('container_test')
        if test is not None:
            click.echo('Running test on pod: %s' % pod['metadata']['name'])

            if ports:
                pf = kubectl.port_forward(
                    pod['metadata']['name'],
                    kubectl_path,
                    *ports,
                    namespace=cfg['namespace'])

            for local in test.get('local_commands', []):
                click.echo(local)

                @retry(30)
                def _run_test():
                    out = shell(local)
                    if out:
                        click.echo(out)
                    click.secho('Test passed', fg='green')
                _run_test()

        click.echo('Scaling new replication controller...')
        deploy = kubectl.get(
            'deploy',
            '{app_name}-{version}-{site_id}'.format(**cfg),
            namespace=cfg['namespace'],
            kubectl=kubectl_path)
        kubectl.scale(
            'deploy',
            deploy['metadata']['name'],
            cfg['test_replica_count'],
            namespace=cfg['namespace'],
            kubectl=kubectl_path)

        kubectl.wait_for_pods(
            kubectl=kubectl_path, timeout=kwargs['timeout'], **selector)

        click.secho('%d pods running' % cfg['test_replica_count'], fg='green')

        if ports:
            click.echo(
                'Updating service to point to new replication controller')
            kubectl.apply(stdin=svc_tpl.render(**cfg), kubectl=kubectl_path)
    except Exception as err:
        click.secho(
            'Could not deploy service due to errors starting '
            'container', fg='red')
        click.secho(" ".join(err.args))
        click.secho('Outputting pod information:', fg='red')
        kubectl.describe('po', kubectl=kubectl_path, **selector)

        pods = kubectl.get('po', kubectl=kubectl_path, **selector)
        if len(pods) > 0:
            click.secho('Printing container logs:', fg='red')
            _log_pod(pods[0], cfg)
        else:
            click.secho('No container started, no container logs', fg='red')

        click.secho('Deleting resources due to failed deployment...', fg='red')
        kubectl.deletef(
            stdin=dynamic_tpl.render(**cfg),
            kubectl=kubectl_path,
            **{'ignore-not-found': 'true'})
        # Previously was "raise", but nobody caught that
        sys.exit(1)
    finally:
        if pf:
            pf.terminate()

    for version in _get_old_versions(cfg):
        click.echo('Deleting old dynamic resources for version: %s' % version)
        _cfg = deepcopy(cfg)
        _cfg['version'] = version
        kubectl.deletef(
            stdin=dynamic_tpl.render(**_cfg), kubectl=kubectl_path, **{
                'ignore-not-found': 'true', })

@click.command()
@global_options
@config.apply
@_login
def ssh(**kwargs):
    '''SSH into a running container'''
    cfg = kwargs['config']

    pods = kubectl.get('po', kubectl=kubectl_path, **{
        'selector': 'app=%s,track=%s' % (
            cfg.get('app_name'), cfg.get('release_track')),
        'namespace': cfg.get('namespace'), })
    if not pods:
        raise Exception('Pod could not be identified')

    kubectl.exec_(
        pods[0]['metadata']['name'],
        kubectl=kubectl_path,
        container=cfg.get('app_name'),
        namespace=cfg.get('namespace'),)

@click.command()
@global_options
@config.apply
@_login
def attach(**kwargs):
    '''Attach to a running container'''
    cfg = kwargs['config']

    pods = kubectl.get('po', kubectl=kubectl_path, **{
        'selector': 'app=%s,track=%s' % (
            cfg.get('app_name'), cfg.get('release_track')),
        'namespace': cfg.get('namespace'), })
    if not pods:
        raise Exception('Pod could not be identified')

    kubectl.attach(
        pods[0]['metadata']['name'],
        kubectl=kubectl_path,
        container=cfg.get('app_name'),
        namespace=cfg.get('namespace'), )

@click.command()
@click.option(
    '--follow',
    default=False,
    is_flag=True,
    help='Follow logs'
)
@global_options
@config.apply
@_login
def logs(**kwargs):
    '''Tail logs from a running container'''
    cfg = kwargs['config']

    pods = kubectl.get('po', kubectl=kubectl_path, **{
        'selector': 'app=%s,track=%s' % (
            cfg.get('app_name'), cfg.get('release_track')),
        'namespace': cfg.get('namespace'), })
    if not pods:
        raise Exception('Pod could not be identified')
    _log_pod(pods[0], cfg, follow=kwargs['follow'])

kube.add_command(gen)
kube.add_command(clean)
kube.add_command(deploy)
kube.add_command(ssh)
kube.add_command(attach)
kube.add_command(logs)
kube.add_command(validate)