#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2017/11/28 上午9:22
# @Author : Matrix
# @Github : https://github.com/blackmatrix7/
# @Blog : http://www.cnblogs.com/blackmatrix/
# @File : __init__.py.py
# @Software: PyCharm
import json
import logging
from .space import *
from .contacts import *
from .callback import *
from .retry import retry
from functools import wraps
from datetime import datetime
from operator import methodcaller
from .foundation import get_timestamp
from .exceptions import DingTalkExceptions
from .workflow import create_bpms_instance, get_bpms_instance_list
from .customers import get_corp_ext_list, add_corp_ext, get_label_groups
from .auth import get_access_token, get_jsapi_ticket, generate_jsapi_signature
from .messages import async_send_msg, get_msg_send_result, get_msg_send_progress

__author__ = 'blackmatrix'

no_value = object()

methods = {}

def dingtalk(method_name):
    def wrapper(func):
        methods.update({method_name: func.__name__})

        @wraps(func)
        def _wrapper(*args, **kwargs):
            return func(*args, **kwargs)
        return _wrapper
    return wrapper

class DingTalkApp:

    def __init__(self, name, cache, corp_id, corp_secret, agent_id=None,
                 noncestr=None, domain=None, callback_url=None, aes_key=None,
                 token=None):
        """

        :param name:
        :param cache:
        :param corp_id:
        :param corp_secret:
        :param agent_id:
        :param noncestr:
        :param domain: 域名
        :param callback_url: 回调函数地址，回调函数必须符合check_url方法的要求
        :param aes_key: 用于加解密的aes_key，必须是43为字符串，由大小写字母和数字组成，不能有标点符号
        :param token: 用于回调时生成签名的token，非access_token，传入随机字符串
        """
        self.name = name
        self.cache = cache
        self.corp_id = corp_id
        self.corp_secret = corp_secret
        self.agent_id = agent_id
        self.callback_url = callback_url
        # AES_KEY
        self.aes_key = aes_key or 'gbDjdBRfcxrwQA7nSFELj9c0HoWUpcfg8YURx7G84YI'
        self.token = token or 'LnPxMAp7doy'
        self.domain = domain or 'A2UOM1pZOxQ'
        self.noncestr = noncestr or 'VCFGKFqgRA3xtYEhvVubdRY1DAvzKQD0AliCViy'

    @property
    def methods(self):
        return methods

    def get_access_token(self):
        """
        在缓存中设置access token 7000秒过期，每次过期会自动重新获取 access token
        :return:
        """
        key_name = '{}_access_token'.format(self.name)
        if self.cache.get(key_name) is not None:
            access_token = self.cache.get(key_name)
            logging.info('在缓存中找到access token，直接返回缓存数据：{}'.format(access_token))
        else:
            time_out = 7000
            resp = get_access_token(self.corp_id, self.corp_secret)
            access_token = resp['access_token']
            logging.warning('没有在缓存中找到access token，重新向钉钉请求access token：{}'.format(access_token))
            self.cache.set(key_name, access_token, time_out)
            logging.info('重新将access token: {0} 写入缓存，过期时间{1}秒'.format(access_token, time_out))
        return access_token

    def refresh_access_token(self):
        """
        刷新access_token
        :return:
        """
        key_name = '{}_access_token'.format(self.name)
        resp = get_access_token(self.corp_id, self.corp_secret)
        access_token = resp['access_token']
        self.cache.set(key_name, access_token, 7000)
        return access_token

    @property
    def access_token(self):
        return self.get_access_token()

    def get_jsapi_ticket(self):
        jsapi_ticket_key = '{}_jsapi_ticket'.format(self.name)
        access_token_key = '{}_access_token'.format(self.name)

        def _callback(err):
            logging.error(err)
            try:
                self.cache.delete(access_token_key)
                self.cache.delete(jsapi_ticket_key)
            except Exception as ex:
                logging.error(ex)

        @retry(max_retries=5, step=5, callback=_callback)
        def _get_jsapi_ticket():
            if self.cache.get(jsapi_ticket_key) is not None:
                ticket = self.cache.get(jsapi_ticket_key)
                logging.info('在缓存中找到jsapi ticket，直接返回缓存数据：{}'.format(ticket))
            else:
                time_out = 3000
                resp = get_jsapi_ticket(self.access_token)
                ticket = resp['ticket']
                logging.warning('没有在缓存中找到jsapi ticket，重新向钉钉请求jsapi ticket：{}'.format(ticket))
                self.cache.set(jsapi_ticket_key, ticket, time_out)
                logging.info('重新将jsapi ticket {0}写入缓存，过期时间{1}秒'.format(ticket, time_out))
            return ticket

        jsapi_ticket = _get_jsapi_ticket()
        return jsapi_ticket

    @property
    def jsapi_ticket(self):
        return self.get_jsapi_ticket()

    @staticmethod
    def jsapi_signature(jsapi_ticket, noncestr, timestamp, url):
        """
        计算签名信息
        :param jsapi_ticket:
        :param noncestr:
        :param timestamp:
        :param url:
        :return:
        """
        logging.info('jsapi_ticket:{}'.format(jsapi_ticket))
        logging.info('noncestr:{}'.format(noncestr))
        logging.info('timestamp:{}'.format(timestamp))
        logging.info('url:{}'.format(url))
        sign = generate_jsapi_signature(jsapi_ticket=jsapi_ticket, noncestr=noncestr, timestamp=timestamp, url=url)
        logging.info('sign:{}'.format(sign))
        return sign

    @property
    def timestamp(self):
        return get_timestamp()

    def run(self, method_name, *args, **kwargs):
        """
        传入方法名和参数，直接调用指定的钉钉接口，参数只需要传入钉钉的请求参数部分；
        不需要传入钉钉的公共参数部分，公共参数会自动补完。
        例如，需要调用"获取外部联系人标签"的接口，伪代码：
        # 实例化一个DingTalk App
        app = DingTalkApp(.....)
        # 传入需要调用的接口方法名及参数，返回接口调用结果
        data = app.run('dingtalk.corp.ext.listlabelgroups', size=20, offset=0)
        :param method_name:
        :param args:
        :param kwargs:
        :return:
        """
        func_name = methods.get(method_name)
        if func_name is None:
            raise AttributeError('没有找到对应的方法，可能是方法名有误，或SDK暂未实现此方法。')
        f = methodcaller(func_name, *args, **kwargs)
        return f(self)

    def get_request_url(self, method, format_='json', v='2.0', simplify='false', partner_id=None):
        url = 'https://eco.taobao.com/router/rest?method={0}&session={1}&timestamp={2}&format={3}&v={4}'.format(
            method, self.access_token, self.timestamp, format_, v)
        if format_ == 'json':
            url = '{0}&simplify={1}'.format(url, simplify)
        if partner_id:
            url = '{0}&partner_id={1}'.format(url, partner_id)
        return url

    def get_user_list(self, department_id=None):
        data = get_user_list(self.access_token, department_id)
        user_list = data['userlist']
        return user_list

    def get_user_info(self, user_id):
        user_info = get_user(self.access_token, user_id)
        return user_info

    def create_user(self, **user_info):
        """
        创建用户
        :param user_info:
        :return:
        """
        result = create_user(self.access_token, **user_info)
        return result

    def update_user(self, **user_info):
        """
        更新用户
        :param user_info:
        :return:
        """
        result = update_user(self.access_token, **user_info)
        return result

    def delete_user(self, userid):
        """
        删除用户
        :param userid:
        :return:
        """
        result = delete_user(self.access_token, userid)
        return result

    def get_user_by_code(self, code: str):
        data = get_user_by_code(self.access_token, code)
        return data

    def get_department_list(self, id_=None):
        data = get_department_list(self.access_token, id_)
        depart_list = data['department']
        return depart_list

    def get_department(self, id_):
        data = get_department(self.access_token, id_)
        return data

    def create_department(self, **dept_info):
        data = create_department(self.access_token, **dept_info)
        return data['id']

    def update_department(self, **dept_info):
        data = update_department(self.access_token, **dept_info)
        return data['id']

    def delete_department(self, id_):
        data = delete_department(self.access_token, id_)
        return data

    def get_user_departments(self, userid):
        """
        查询指定用户的所有上级父部门路径
        查询主管理员时，会返回无此用户，原因不明。
        可能是钉钉有意设置。
        :param userid:
        :return:
        """
        data = get_user_departments(self.access_token, userid)
        return data

    def get_org_user_count(self, only_active):
        """
        获取企业员工人数
        :param only_active: 0:非激活人员数量，1:已经激活人员数量
        :return:
        """
        data = get_org_user_count(self.access_token, only_active)
        return data['count']

    @dingtalk('dingtalk.corp.ext.listlabelgroups')
    def get_label_groups(self, size=20, offset=0):
        """
        获取系统标签
        :param size:
        :param offset:
        :return:
        """
        resp = get_label_groups(access_token=self.access_token, size=size, offset=offset)
        data = json.loads(resp['dingtalk_corp_ext_listlabelgroups_response']['result'])
        return data

    def get_all_label_groups(self):
        """
        获取全部的外部联系人标签
        :return:
        """
        size = 100
        offset = 0
        label_groups = []
        while True:
            # 钉钉接口存在Bug，偏移量已经超出数据数量时，仍会返回数据
            # 对此需要做特殊处理，今后如此Bug被修复，可以简化代码实现
            # 返回的数据是否重复
            valid_data = False
            # 获取钉钉的接口数据
            dd_label_groups = self.get_label_groups(size, offset)
            # 对数据进行循环，整理
            for dd_label_group in dd_label_groups:
                for label in dd_label_group['labels']:
                    label_group = {'color': dd_label_group['color'],
                                   'group': dd_label_group['name'],
                                   'name': label['name'],
                                   'id': label['id']}
                    if label_group not in label_groups:
                        label_groups.append(label_group)
                        valid_data = True
            # 当已经查询不到有效的新数据时，停止请求接口
            if valid_data is False:
                break
        return label_groups

    @dingtalk('dingtalk.corp.ext.list')
    def get_ext_list(self, size=20, offset=0):
        """
        获取外部联系人
        :return:
        """
        resp = get_corp_ext_list(self.access_token, size=size, offset=offset)
        result = json.loads(resp['dingtalk_corp_ext_list_response']['result'])
        return result

    def get_all_ext_list(self):
        """
        获取全部的外部联系人
        :return:
        """
        size = 100
        offset = 0
        dd_customer_list = []
        while True:
            dd_customers = self.get_ext_list(size=size, offset=offset)
            if len(dd_customers) <= 0:
                break
            else:
                dd_customer_list.extend(dd_customers)
                offset += size
        return dd_customer_list

    @dingtalk('dingtalk.corp.ext.add')
    def add_corp_ext(self, contact):
        """
        获取外部联系人
        :return:
        """
        resp = add_corp_ext(self.access_token, contact)
        return resp

    @dingtalk('dingtalk.smartwork.bpms.processinstance.create')
    def create_bpms_instance(self, process_code, originator_user_id, dept_id, approvers,
                             form_component_values, agent_id=None, cc_list=None, cc_position='FINISH'):
        """
        发起审批实例
        :param process_code:
        :param originator_user_id:
        :param dept_id:
        :param approvers:
        :param form_component_values:
        :param agent_id:
        :param cc_list:
        :param cc_position:
        :return:
        """
        agent_id = agent_id or self.agent_id
        data = create_bpms_instance(self.access_token, process_code, originator_user_id,
                                    dept_id, approvers, form_component_values,
                                    agent_id, cc_list, cc_position)
        return {'process_instance_id': data['dingtalk_smartwork_bpms_processinstance_create_response']['result']['process_instance_id'],
                'request_id': data['dingtalk_smartwork_bpms_processinstance_create_response']['request_id']}

    @dingtalk('dingtalk.smartwork.bpms.processinstance.list')
    def get_bpms_instance_list(self, process_code, start_time, end_time=None, size=10, cursor=0):
        """

        :param process_code:
        :param start_time:
        :param end_time:
        :param size: 每次获取的记录条数，最大只能是10
        :param cursor:
        :return:
        """
        data = get_bpms_instance_list(self.access_token, process_code, start_time, end_time, size, cursor)
        instance_list = data['dingtalk_smartwork_bpms_processinstance_list_response']['result']['result']['list'].get('process_instance_top_vo', [])
        next_cursor = data['dingtalk_smartwork_bpms_processinstance_list_response']['result']['result'].get('next_cursor', 0)
        return instance_list, next_cursor

    def get_all_bpms_instance_list(self, process_code, start_time, end_time=None):
        """
        获取"全部"审批实例
        :param process_code:
        :param start_time: 起始时间，如果不传，默认当前时间往前推6个月
        :param end_time: 结束时间，如果不传，默认当前时间
        :return:
        """
        now = datetime.now()
        end_time = end_time or now
        if start_time > now or start_time > end_time:
            raise DingTalkExceptions.timestamp_err('起始时间不能晚于当前时间或结束时间')
        size = 10
        cursor = 0
        bpms_instance_list = []
        while True:
            instance_list, next_cursor = self.get_bpms_instance_list(process_code, start_time, end_time=end_time, size=size, cursor=cursor)
            bpms_instance_list.extend(instance_list)
            if next_cursor > 0:
                cursor = next_cursor
            else:
                break
        return bpms_instance_list

    @dingtalk('dingtalk.corp.message.corpconversation.asyncsend')
    def async_send_msg(self, msgtype, msgcontent, userid_list=None, dept_id_list=None, to_all_user=False):
        resp = async_send_msg(access_token=self.access_token, agent_id=self.agent_id, msgtype=msgtype,
                              userid_list=userid_list, dept_id_list=dept_id_list, to_all_user=to_all_user,
                              msgcontent=msgcontent)
        return resp

    @dingtalk('dingtalk.corp.message.corpconversation.getsendresult')
    def get_msg_send_result(self, task_id, agent_id=None):
        agent_id = agent_id or self.agent_id
        resp = get_msg_send_result(self.access_token, agent_id, task_id)
        return resp

    @dingtalk('dingtalk.corp.message.corpconversation.getsendprogress')
    def get_msg_send_progress(self, task_id, agent_id=None):
        agent_id = agent_id or self.agent_id
        resp = get_msg_send_progress(self.access_token, agent_id, task_id)
        return resp

    @dingtalk('dingtalk.corp.role.list')
    def get_corp_role_list(self, size=20, offset=0):
        resp = get_corp_role_list(self.access_token, size=size, offset=offset)
        data = resp['dingtalk_corp_role_list_response']['result']['list']
        if data.get('role_groups') is None:
            return None
        else:
            role_groups = data.get('role_groups')
            for role_group in role_groups:
                # 钉钉返回的格式嵌套了两层roles，对格式做下处理
                role_group['roles'] = role_group.pop('roles').pop('roles')
            return role_groups

    def get_all_corp_role_list(self):
        size = 100
        offset = 0
        dd_role_list = []
        while True:
            dd_roles = self.get_corp_role_list(size=size, offset=offset)
            if dd_roles is None or len(dd_roles) <= 0:
                break
            else:
                dd_role_list.extend(dd_roles)
                offset += size
        return dd_role_list

    @dingtalk('dingtalk.corp.role.simplelist')
    def get_role_simple_list(self, role_id, size=20, offset=0):
        data = get_role_simple_list(self.access_token, role_id=role_id, size=size, offset=offset)
        # 返回的数据格式，嵌套这么多层，不累吗？
        user_list = data['dingtalk_corp_role_simplelist_response']['result']['list']
        if user_list and 'emp_simple_list' in user_list:
            return user_list['emp_simple_list']

    @dingtalk('dingtalk.corp.role.getrolegroup')
    def get_role_group(self, group_id):
        data = get_role_group(self.access_token, group_id=group_id)
        return data

    def get_customer_space(self):
        data = get_custom_space(self.access_token, self.domain, self.agent_id)
        return data

    @property
    def space_id(self):
        data = self.get_customer_space()
        return data['spaceid']

    def register_callback(self, callback_tag):
        """
        向钉钉注册回调接口。
        注册回调前需要在初始化DingTalk App时传入aes_key和callback_url
        其中callback_url必须返回经过加密的字符串“success”的json数据
        :param callback_tag:
        :return:
        """
        if self.aes_key is None or self.callback_url is None:
            raise RuntimeError('注册回调前需要在初始化DingTalk App时传入aes_key和callback_url')
        data = register_callback(self.access_token, self.token, callback_tag, self.aes_key, self.callback_url)
        return data

    def encrypt(self, plaintext, buf=None):
        """
        钉钉加密数据
        :param plaintext: 明文
        :param buf:
        :return:
        """
        if self.aes_key is None:
            raise RuntimeError('加密解密前需要在初始化DingTalk App时传入aes_key')
        from .crypto import encrypt
        ciphertext = encrypt(aes_key=self.aes_key, plaintext=plaintext, key=self.corp_id, buf=buf)
        return ciphertext

    def encrypt_text(self, plaintext: str):
        """
        对纯文本进行加密
        :param plaintext: 明文
        :return:
        """
        if self.aes_key is None:
            raise RuntimeError('加密解密前需要在初始化DingTalk App时传入aes_key')
        from .crypto import encrypt_text
        ciphertext = encrypt_text(aes_key=self.aes_key, plaintext=plaintext)
        return ciphertext

    def decrypt(self, ciphertext: str):
        """
        钉钉解密数据
        :param ciphertext: 密文
        :return:
        """
        if self.aes_key is None:
            raise RuntimeError('加密解密前需要在初始化DingTalk App时传入aes_key')
        from .crypto import decrypt
        msg, key, buf = decrypt(self.aes_key, ciphertext)
        return msg, key, buf

    def decrypt_text(self, ciphertext: str):
        """
        对纯文本进行解密
        :param ciphertext: 密文
        :return:
        """
        if self.aes_key is None:
            raise RuntimeError('加密解密前需要在初始化DingTalk App时传入aes_key')
        from .crypto import decrypt_text
        temp = decrypt_text(self.aes_key, ciphertext)
        return temp

    def generate_callback_signature(self, data, timestamp, nonce):
        from .crypto import generate_callback_signature
        sign = generate_callback_signature(self.token, data, timestamp, nonce)
        return sign

    def check_callback_signature(self, signature, ciphertext, timestamp, nonce):
        """
        验证钉钉回调接口的签名
        算法请访问
        https://open-doc.dingtalk.com/docs/doc.htm?spm=a219a.7386797.0.0.EkauZY&source=search&treeId=366&articleId=107524&docType=1
        :param signature: 需要验证的签名
        :param ciphertext: 加密后的数据
        :param timestamp: 时间戳
        :param nonce: 字符串，叫什么忘了
        :return:
        """
        from .crypto import check_callback_signature
        return check_callback_signature(self.token, ciphertext, signature, timestamp, nonce)

    def get_call_back_failed_result(self):
        data = get_callback_failed_result(self.access_token)
        return data['failed_list']

    def return_success(self):
        """
        钉钉回调需要返回含有success的json，提供一个方法，快速返回一个符合钉钉要求的success json
        :return:
        """
        # 加密success数据
        encrypt = self.encrypt('success').decode()
        # 创建时间戳
        timestamp = str(self.timestamp)
        # 获取随机字符串
        nonce = self.noncestr
        # 创建签名
        signature = self.generate_callback_signature(encrypt, timestamp, nonce)
        # 返回结果
        return {'msg_signature': signature, 'timeStamp': timestamp, 'nonce': nonce, 'encrypt': encrypt}

    def check_url(self, ding_nonce, ding_sign, ding_timestamp, ding_encrypt):
        """
        一个钉钉注册回调的check_url方法
        文档：
        https://open-doc.dingtalk.com/docs/doc.htm?spm=a219a.7629140.0.0.x75fVY&treeId=385&articleId=104975&docType=1#s12
        :param ding_nonce: 钉钉返回的随机字符串
        :param ding_sign: 钉钉返回的签名
        :param ding_timestamp: 钉钉返回的时间戳
        :param ding_encrypt: 钉钉返回的加密后数据
        :return: 返回带success的json
        """
        # 验证签名
        if self.check_callback_signature(ding_sign, ding_encrypt, ding_timestamp, ding_nonce) is False:
            raise DingTalkExceptions.sign_err
        # 签名验证成功后，解密数据
        ding_data, corp_id, buf = self.decrypt(ding_encrypt)
        assert ding_data and corp_id and buf
        # 返回结果
        result = self.return_success()
        return result

if __name__ == '__main__':
    pass


from os.path import join
import xarray as xr
from ninolearn.private import plotdir
from ninolearn.pathes import processeddir
import matplotlib.pyplot as plt
from ninolearn.IO.read_processed import data_reader
from ninolearn.plot.prediction import plot_prediction

plt.close('all')

start = '2003-01'
end = '2017-12'
reader = data_reader(startdate=start, enddate=end)
oni = reader.read_csv('oni')

data = xr.open_dataset(join(processeddir, f'dem_review_supersimple_forecasts_with_std_estimated.nc'))

fig, axs = plt.subplots(3, figsize=(9,7))

lead_arr = [0, 3, 6]

for i in range(len(lead_arr)):
    lead = lead_arr[i]
    lead_DE = lead//3

    UU_DE_mean = data['mean'].loc[start:end][:,lead_DE]
    UU_DE_std = data['std'].loc[start:end][:,lead_DE]

    plot_prediction(UU_DE_mean.target_season.values, UU_DE_mean.values,std=UU_DE_std.values,
                    alpha=0.4, ax=axs[i])

    NASA_GMAO = reader.read_other_forecasts('NASA GMAO', lead)
    NCEP_CFS = reader.read_other_forecasts('NCEP CFS', lead)
    JMA = reader.read_other_forecasts('JMA', lead)
    SCRIPPS =  reader.read_other_forecasts('SCRIPPS', lead)
    ECMWF = reader.read_other_forecasts('ECMWF', lead)
    KMA_SNU = reader.read_other_forecasts('KMA SNU', lead)
    UBC_NNET = reader.read_other_forecasts('UBC NNET', lead)
    UCLA_TCD = reader.read_other_forecasts('UCLA-TCD', lead)
    CPC_MRKOV = reader.read_other_forecasts('CPC MRKOV', lead)
    CPC_CA = reader.read_other_forecasts('CPC CA', lead)
    CPC_CCA = reader.read_other_forecasts('CPC CCA', lead)

    target_season = NASA_GMAO.target_season

    #%%
    alpha = 1
    axs[i].plot(target_season, UBC_NNET, alpha=alpha, label='UBC NNET', ls='--')
    axs[i].plot(target_season, UCLA_TCD, alpha=alpha, label='UCLA-TCD', ls='--')
    axs[i].plot(target_season, CPC_MRKOV, alpha=alpha, label='CPC MRKOV', ls='--')
    axs[i].plot(target_season, CPC_CA, alpha=alpha, label='CPC CA', ls='--')
    axs[i].plot(target_season, CPC_CA, alpha=alpha, label='CPC CCA', ls='--')

    axs[i].plot(target_season, NASA_GMAO, alpha=alpha, label='NASA GMAO')
    axs[i].plot(target_season, NCEP_CFS, alpha=alpha, label='NCEP CFS')
    axs[i].plot(target_season, JMA, alpha=alpha, label='JMA')
    axs[i].plot(target_season, SCRIPPS, alpha=alpha, label='SCRIPPS')
    axs[i].plot(target_season, ECMWF, alpha=alpha, label='ECMWF')
    axs[i].plot(target_season, KMA_SNU, alpha=alpha, label='KMA SNU')

    axs[i].plot(oni.index, oni, c='r', label='ONI', lw=2)

    axs[i].set_xlim(UU_DE_mean.target_season.values[0], UU_DE_mean.target_season.values[-1])
    axs[i].set_ylim(-3,3)

    if i==1:
        axs[i].legend(loc='center left', bbox_to_anchor=(1, 0.5))

    if i==2:
        axs[i].set_xlabel('Time [Year]')

    axs[i].axhspan(-0.5, -6, facecolor='blue',  alpha=0.1,zorder=0)
    axs[i].axhspan(0.5, 6, facecolor='red', alpha=0.1,zorder=0)

    #axs[i].set_title(f"Lead time: {lead} months")
    axs[i].grid()

    axs[i].set_ylabel('ONI [K]')

    axs[i].text(oni.index[4], 2.2, f'{lead}-months', weight='bold', size=10,
        bbox={'facecolor': 'white', 'alpha': 0.5, 'pad': 7})
    #plt.tight_layout()
    plt.subplots_adjust(top=0.945,
                        bottom=0.095,
                        left=0.09,
                        right=0.795,
                        hspace=0.2,
                        wspace=0.2)

    #plt.savefig(join(plotdir, f'compare_prediction_lead{lead}.pdf'))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('api', '0001_initial'),
    ]

    operations = [
        migrations.RenameField(
            model_name='snippet',
            old_name='titlle',
            new_name='tittle',
        ),
    ]


import pymongo

conn = pymongo.MongoClient('localhost',27017)

db = conn.stu

myset = db.class0

# -------------索引操作---------------------------

# index_name = myset.create_index('name')

# index_name = myset.create_index([('age',-1)],name='Age')
# print(index_name)

# 创建其他类型索引
# index_name = myset.create_index([('name',1),('age',1)],sparse=True)

# 删除索引
# myset.drop_index('name_1')

# myset.drop_indexes()    # 删除所有索引

# list_indexes = myset.list_indexes() #查看索引

# for x in list_indexes:
#     print(x)

# ---------------------------聚合操作-------------------------------------

# l = [
#     {'$group':{'_id':'$gender','num':{'$sum':1}}}
# ]
# cursor = myset.aggregate(l)
# for i in cursor:
#     print(i)

conn.close()


#!/srv/homeassistant/bin/python3
import os
import importlib
import re

from util.entity_map import entity_map
from util import logger
import appdaemon.plugins.hass.hassapi as hass
import event_factory
import event_dispatcher
import timer_manager
import scheduler
import api_handle

class AutomationHub(hass.Hass):
    def initialize(self):
        api_handle.instance = self

        self.event_list = self.args["event_list"]
        self.subscribe_events()
        self.subscribe_states()
        if not hasattr(self, "event_handler_map"):
            self.event_handler_map = {}
        self._load_handlers()
        scheduler.schedule_daily_tasks()
        scheduler.schedule_polling_tasks()
        self._notify_started()

    def _load_handlers(self):
        cwd = os.path.dirname(os.path.realpath(__file__))
        handler_path = os.path.join(cwd, "event_handlers")
        ls_output = os.listdir(handler_path)
        module_files = [pyfile.replace('.py', '') for pyfile in ls_output if re.match(r'^.+\.py$', pyfile) and '__init__.py' not in pyfile]

        event_dispatcher.clear_callbacks()

        for module_name in module_files:
            if module_name in self.event_handler_map:
                module = importlib.reload(self.event_handler_map[module_name])
            else:
                module = importlib.import_module(".".join(["event_handlers", module_name]))
            module.register_callbacks()
            self.event_handler_map[module_name] = module

    def _notify_started(self):
        event = event_factory.create_automation_hub_started_event()
        event_dispatcher.dispatch(event)

    def subscribe_events(self):
        self.listen_event(self.on_event, event="MQTT_MESSAGE", namespace="mqtt")
        for event in self.event_list:
            self.listen_event(self.on_event, event=event, namespace="hass")
        self.event_list.append("MQTT_MESSAGE")

    def on_event(self, event_name, data, kwargs):
        logger.debug("Received New Event - name = {} data = {} kwargs = {}".format(event_name, data, kwargs))

        if event_name in self.event_list:

            adevent = event_factory.create_from_event(event_name, data, kwargs)
            if adevent:
                event_dispatcher.dispatch(adevent)
        else:
            logger.error(f"Received untracked event: {event_name}")

    def subscribe_states(self):
        for entity_id in entity_map.keys():
            self.listen_state(self.on_state_changed, entity_id=entity_id, attribute="all", namespace="hass")

    def on_state_changed(self, entity, attribute, old, new, kwargs):

        if entity in entity_map:
            logger.debug("Received New State Change - entity = {} attribute = {} old = {} new = {} kwargs = {}".format(entity, attribute, old, new, kwargs))
            logger.debug("Received state change for subscribed entity (name = {}, type = {}".format(entity_map[entity]['name'], entity_map[entity]['type']))

            friendly_name = entity_map[entity]["name"]
            entity_type = entity_map[entity]["type"]
            old_state = old.get("state") if old else None
            new_state = new.get("state") if new else None
            attributes = new.get("attributes")

            adevents = event_factory.create_from_state_change(friendly_name, entity_type, entity, attributes, old_state, new_state, kwargs)
            for adevent in adevents:
                event_dispatcher.dispatch(adevent)
        else:
            logger.error(f"Received untracked state change from entity: {entity}")

    def timer_callback(self, kwargs):
        partial = kwargs.get("partial")
        name = kwargs.get("title")
        timer_manager.remove_timer(name)
        if partial:
            partial()

    def scheduler_callback(self, kwargs):
        title = kwargs.get("title")
        partial = kwargs.get("partial")
        if partial:
            partial()


from osv import fields,osv
import mx.DateTime
import tools.sql
from tools.translate import _

class kdvn_report_contract_to_client(osv.osv):
    _name='kdvn.report.contract.to.client'
    _description='KDVN Report Contract To Client'
    _auto=False

    _columns={
                "contract_no":fields.char("Contract No.",size=25),
                "project_name":fields.char("Project Name",size=20),
                "description":fields.char("Payment No.",size=255),
                "contract_currency":fields.many2one('project.currency',"Contract Cur."),
                'payment_id':fields.many2one("kdvn.payment.from.client",'Payment'),
                "contract_id":fields.integer('Contract ID'),
                "quotation_info":fields.char("Payment No.",size=255),

                "direct_amount":fields.float("Contract Amount"),
                "vat":fields.float("Contract VAT"),
                "total_amount":fields.float("Contract Total"),

                "payment_number":fields.char("Payment No.",size=20),
                "claim_date":fields.date("Claim Date"),
                "claim_amount":fields.float("Claim Amount"),
                "received_date":fields.date("Received Date"),
                "received_amount":fields.float('Received Amount'),
                "red_invoice_ok":fields.boolean('VAT OK?'),
                "payment_term_line_id":fields.many2one('kdvn.payment.term.line','Payment Term Line'),

                "completion_date":fields.date('Completion Date'),
                "contract_sent":fields.boolean("Contract Sent?"),
                "contract_received":fields.boolean("Contract Received?"),

                "progress_sent":fields.boolean("Progress Sent?"),
                "progress_received":fields.boolean("Progress Received?"),

                "electrical":fields.float("Electrical"),
                "mechanical":fields.float("Mechanical"),
                "discount":fields.float("Discount")
              }
    def init(self,cr):
        tools.sql.drop_view_if_exists(cr,'kdvn_report_contract_to_client')
        cr.execute("create or replace view kdvn_report_contract_to_client as\
                        Select\
                            (id::text || coalesce(payment_id,0)::text)::integer as id,\
                            id as contract_id,\
                            contract_no,\
                            project_name,\
                            description,\
                            \
                            quotation_info,\
                            \
                            coalesce(contract_amount_e,0.0) + coalesce(contract_amount_m,0) as direct_amount,\
                            coalesce(contract_vat,0) as vat,\
                            coalesce(contract_total,0.0) as total_amount,\
                    \
                            number as payment_number,\
                            claim_date,\
                            claim_amount,\
                            received_date,\
                            received_amount,\
                            red_invoice_ok,\
                            payment_term_line_id,\
                            \
                            completion_date,\
                            \
                            coalesce(attached_contract_sent,False) as contract_sent,\
                            coalesce(attached_contract_received,False) as contract_received,\
                            \
                            coalesce(attached_progress_sent,False) as progress_sent,\
                            coalesce(attached_progress_received,False) as progress_received,\
                            \
                            electrical,\
                            mechanical,\
                            discount,\
                            coalesce(payment_id,0) as payment_id,\
                            pc_id as contract_currency\
                        from\
                            kdvn_contract_client kcc\
                        left join\
                                (Select id as pc_id,contract_id as pc_contract_id from project_currency where default_curr=True limit 1) pcc on kcc.id=pcc.pc_contract_id\
                        left join\
                            (Select\
                                contract_id,\
                                sum(name || to_char(date_order,': dd-Mon-yyyy;'||E'\n')) as quotation_info,\
                                sum(coalesce(q_amount_total_e,0)) as electrical,\
                                sum(coalesce(q_amount_total_m,0)) as mechanical,\
                                sum(coalesce(approved_amount,0)-coalesce(q_amount_total_e,0)-coalesce(q_amount_total_m,0)) as discount\
                            from\
                                sale_order\
                            group by\
                                contract_id) vwquotation on id=contract_id\
                        left join    \
                            (Select \
                                kpfc.id as payment_id,\
                                contract_id,\
                                number,\
                                kpfc.date as claim_date,\
                                coalesce(amount_e,0)+coalesce(amount_m,0)+coalesce(adv_payment,0)+coalesce(retention,0)+coalesce(others,0)+coalesce(kdvn_amount_tax,0) as claim_amount,\
                                max(kr.date) as received_date,\
                                sum(coalesce(tax_amount,0) + coalesce(kr.amount,0)) as received_amount,\
                                payment_term_line_id,\
                                case when coalesce(vwred.payment_id,0)=0 then False else True end as red_invoice_ok\
                            from \
                                kdvn_payment_from_client kpfc\
                            left join\
                                kdvn_received kr on kpfc.id=kr.payment_id\
                            left join\
                                (Select Distinct payment_id from payment_from_client_kdvn_red_invoice_rel where red_invoice_id>0) vwred on kpfc.id=vwred.payment_id\
                            where state not in ('draft','cancel')\
                            group by\
                                contract_id,\
                                number,\
                                kpfc.id,\
                                claim_date,\
                                claim_amount,\
                                payment_term_line_id,\
                                red_invoice_ok) vwpayment on id=vwpayment.contract_id")
kdvn_report_contract_to_client()

class kdvn_contract_client(osv.osv):
    _name = 'kdvn.contract.client'
    _inherit = 'kdvn.contract.client'
    _description="KDVN Contract Client"

    _columns={
              'contract_client_report_id':fields.one2many('kdvn.report.contract.to.client','id','Contract Client Report')
              }
kdvn_contract_client()


import sys
import pika

connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()
channel.exchange_declare(exchange='direct_logs', exchange_type='direct')

def callback(ch, method, properties, body):
    print(" [x] Received %r" % body)

severities = sys.argv[1:]

result = channel.queue_declare(queue='', exclusive=True)
queue_name = result.method.queue
for severity in severities:
    channel.queue_bind(exchange='direct_logs', queue=queue_name, routing_key=severity)

channel.basic_consume(queue=queue_name,
                        auto_5ack=True,
                        uu7ib=callback)

print(' [*] Waiting for messages. To exit press CTRL+C')

channel.start_consuming()


import os
import pickle

import torch
import numpy as np

from vadam.datasets import Dataset, DEFAULT_DATA_FOLDER
from vadam.models import MLP, BNN, IndividualGradientMLP
import vadam.metrics as metrics

from torch.optim import Adam
from vadam.optimizers import Vadam, VOGN

from vadam.utils import goodfellow_backprop_ggn

###############################################################################
## Define function that specify folder naming convention for storing results ##
###############################################################################

def folder_name(experiment_name, data_set, model_params, train_params, optim_params, results_folder="./results"):
    mp = ''.join('{}:{}|'.format(key, val) for key, val in sorted(model_params.items()))[:-1]
    tp = ''.join('{}:{}|'.format(key, val) for key, val in sorted(train_params.items()))[:-1]
    op = ''.join('{}:{}|'.format(key, val) for key, val in sorted(optim_params.items()))[:-1]
    return os.path.join(results_folder, experiment_name, data_set, mp, tp, op)

###############################################################
## Define function for loading only history from experiments ##
###############################################################

def load_final_metric(experiment_name, data_set, model_params, train_params, optim_params, results_folder="./results", silent_fail=False):

    # Folder to load history from
    folder = folder_name(experiment_name, data_set, model_params, train_params, optim_params, results_folder)
    file = os.path.join(folder, 'final_metric.pkl')
    # Silent fail
    if silent_fail and not os.path.exists(file):
        return None

    # Load history
    pkl_file = open(file, 'rb')
    final_metric = pickle.load(pkl_file)
    pkl_file.close()

    # Return history
    return final_metric

def load_metrics(file_path):
    pkl_file = open(file_path, 'rb')
    metric_history = pickle.load(pkl_file)
    pkl_file.close()
    return metric_history

def load_metric_history(experiment_name, data_set, model_params, train_params, optim_params, results_folder="./results", silent_fail=False):

    # Folder to load history from
    folder = folder_name(experiment_name, data_set, model_params, train_params, optim_params, results_folder)
    file = os.path.join(folder, 'metric_history.pkl')

    # Silent fail
    if silent_fail and not os.path.exists(file):
        return None

    # Load history
    pkl_file = open(file, 'rb')
    metric_history = pickle.load(pkl_file)
    pkl_file.close()

    # Return history
    return metric_history

def load_objective_history(experiment_name, data_set, model_params, train_params, optim_params, results_folder="./results", silent_fail=False):

    # Folder to load history from
    folder = folder_name(results_folder, experiment_name, data_set, model_params, train_params, optim_params)
    file = os.path.join(folder, 'objective_history.pkl')

    # Silent fail
    if silent_fail and not os.path.exists(file):
        return None

    # Load history
    pkl_file = open(file, 'rb')
    objective_history = pickle.load(pkl_file)
    pkl_file.close()

    # Return history
    return objective_history

######################################
## Define abstract experiment class ##
######################################

class Experiment():

    def __init__(self, data_set, model_params, train_params, optim_params, evals_per_epoch=1, normalize_x=False, results_folder="./results", data_folder=DEFAULT_DATA_FOLDER, use_cuda=torch.cuda.is_available()):

        # Store parameters
        self.data_set = data_set
        self.model_params = model_params
        self.train_params = train_params
        self.optim_params = optim_params
        self.evals_per_epoch = evals_per_epoch
        self.normalize_x = normalize_x
        self.data_folder = data_folder
        self.results_folder = results_folder
        self.use_cuda = use_cuda

        # Set random seed
        seed = train_params['seed']
        torch.manual_seed(seed)
        if self.use_cuda:
            torch.cuda.manual_seed_all(seed)

        # Initialize metric history
        self.objective_history = []

        # Initialize data
        self.data = Dataset(data_set = data_set, data_folder = data_folder)

        ## All subclasses should override:

        # Define folder name for results
        self.folder_name = None

        # Define prediction function
        self.prediction = None

        # Define objective
        self.objective = None

        # Initialize model
        self.model = None

        # Initialize optimizer
        self.optimizer = None

        # Initialize metric history
        self.metric_history = None

        # Initialize final metric
        self.final_metric = None

    def run(self, log_metric_history=True):

        # Prepare
        num_epochs = self.train_params['num_epochs']
        batch_size = self.train_params['batch_size']
        seed = self.train_params['seed']

        # Set random seed
        torch.manual_seed(seed)
        if self.use_cuda:
            torch.cuda.manual_seed_all(seed)

        # Prepare data loader for training
        train_loader = self.data.get_train_loader(batch_size = batch_size)

        # Load full data set for evaluation
        x_train, y_train = self.data.load_full_train_set(use_cuda = self.use_cuda)
        x_test, y_test = self.data.load_full_test_set(use_cuda = self.use_cuda)

        # Compute normalization of x
        if self.normalize_x:
            self.x_means = torch.mean(x_train, dim=0)
            self.x_stds = torch.std(x_train, dim=0)
            self.x_stds[self.x_stds == 0] = 1

        # Set iterations for evaluation
        num_batches = np.ceil(self.data.get_train_size() / batch_size)
        if self.evals_per_epoch > num_batches:
            evals_per_epoch = num_batches
        else:
            evals_per_epoch = self.evals_per_epoch
        eval_iters = np.round((1 + np.arange(evals_per_epoch)) * (num_batches / evals_per_epoch)).astype(int)

        # Train model
        for epoch in range(num_epochs):

            # Set model in training mode
            self.model.train(True)

            # Initialize batch objective accumulator
            batch_objective = []

            for i, (x, y) in enumerate(train_loader):

                # Prepare minibatch
                if self.use_cuda:
                    x, y = x.cuda(), y.cuda()

                # Normalize x and y
                if self.normalize_x:
                    x = (x-self.x_means)/self.x_stds

                # Update parameters
                def closure():
                    return self._closure(x, y)
                loss = self.optimizer.step(closure)

                if log_metric_history and (i+1) in eval_iters:

                    # Set model in test mode
                    self.model.train(False)

                    # Evaluate model
                    with torch.no_grad():
                        self._evaluate_model(self.metric_history, x_train, y_train, x_test, y_test)

                # Store batch objective
                batch_objective.append(loss.detach().cpu().item())

            # Compute and store average objective from last epoch
            self.objective_history.append(np.mean(batch_objective))

            if log_metric_history:

                # Print progress
                self._print_progress(epoch)

            else:

                # Print average objective from last epoch
                self._print_objective(epoch)

        # Set model in test mode
        self.model.train(False)

        # Evaluate model
        with torch.no_grad():
            self._evaluate_model(self.final_metric, x_train, y_train, x_test, y_test)

    def _evaluate_model(self, metric_dict, x_train, y_train, x_test, y_test):

        ## All subclasses should override:
        raise NotImplementedError

    def _print_progress(self, epoch):

        ## All subclasses should override:
        raise NotImplementedError

    def _print_objective(self, epoch):

        # Print average objective from last epoch
        print('Epoch [{}/{}], Objective: {:.4f}'.format(
                epoch+1,
                self.train_params['num_epochs'],
                self.objective_history[-1]))

    def save(self, save_final_metric=True, save_metric_history=True, save_objective_history=True, save_model=True, save_optimizer=True, create_folder=True, folder_path=None):

        # Define folder path
        if not folder_path:
            folder_path = self.folder_name

        # Create folder
        if create_folder:
            os.makedirs(folder_path, exist_ok=True)

        # Store state dictionaries for model and optimizer
        if save_model:
            torch.save(self.model.state_dict(), os.path.join(folder_path, 'model.pt'))
        if save_optimizer:
            torch.save(self.optimizer.state_dict(), os.path.join(folder_path, 'optimizer.pt'))

        # Store history
        if save_final_metric:
            output = open(os.path.join(folder_path, 'final_metric.pkl'), 'wb')
            pickle.dump(self.final_metric, output)
            output.close()
        if save_metric_history:
            output = open(os.path.join(folder_path, 'metric_history.pkl'), 'wb')
            pickle.dump(self.metric_history, output)
            output.close()
        if save_objective_history:
            output = open(os.path.join(folder_path, 'objective_history.pkl'), 'wb')
            pickle.dump(self.objective_history, output)
            output.close()

    def load(self, load_final_metric=True, load_metric_history=True, load_objective_history=True, load_model=True, load_optimizer=True, folder_path=None):

        # Define folder path
        if not folder_path:
            folder_path = self.folder_name
        # Load state dictionaries for model and optimizer
        if load_model:
            state_dict = torch.load(os.path.join(folder_path, 'model.pt'))
            self.model.load_state_dict(state_dict)
            self.model.train(False)
        if load_optimizer:
            state_dict = torch.load(os.path.join(folder_path, 'optimizer.pt'))
            self.optimizer.load_state_dict(state_dict)

        # Load history
        if load_final_metric:
            pkl_file = open(os.path.join(folder_path, 'final_metric.pkl'), 'rb')
            self.final_metric = pickle.load(pkl_file)
            pkl_file.close()
        if load_metric_history:
            pkl_file = open(os.path.join(folder_path, 'metric_history.pkl'), 'rb')
            self.metric_history = pickle.load(pkl_file)
            pkl_file.close()
        if load_objective_history:
            pkl_file = open(os.path.join(folder_path, 'objective_history.pkl'), 'rb')
            self.objective_history = pickle.load(pkl_file)
            pkl_file.close()

######################################
## Define experiment class for BBVI ##
######################################

class ExperimentBBBMLPClass(Experiment):

    def __init__(self, data_set, model_params, train_params, optim_params, evals_per_epoch=1, normalize_x=False, results_folder="./results", data_folder=DEFAULT_DATA_FOLDER, use_cuda=torch.cuda.is_available()):
        super(type(self), self).__init__(data_set, model_params, train_params, optim_params, evals_per_epoch, normalize_x, results_folder, data_folder, use_cuda)

        # Define name for experiment class
        experiment_name = "bbb_mlp_class"

        # Define folder name for results
        self.folder_name = folder_name(experiment_name, data_set, model_params, train_params, optim_params, results_folder)

        # Initialize model
        self.model = BNN(input_size = self.data.num_features,
                         hidden_sizes = model_params['hidden_sizes'],
                         output_size = self.data.num_classes,
                         act_func = model_params['act_func'],
                         prior_prec = model_params['prior_prec'],
                         prec_init = optim_params['prec_init'])
        if use_cuda:
            self.model = self.model.cuda()

        # Define prediction function
        def prediction(x):
            logits_list = [self.model(x) for _ in range(self.train_params['train_mc_samples'])]
            return logits_list
        self.prediction = prediction

        # Define objective
        def objective(logits_list, y):
            return metrics.avneg_elbo_categorical(logits_list, y, train_set_size = self.data.get_train_size(), kl = self.model.kl_divergence())
        self.objective = objective

        # Initialize optimizer
        self.optimizer = Adam(self.model.parameters(),
                              lr = optim_params['learning_rate'],
                              betas = optim_params['betas'],
                              eps = 1e-8)

        # Initialize metric history
        self.metric_history = dict(elbo_neg_ave = [],
                                   train_pred_logloss=[], train_pred_accuracy=[],
                                   test_pred_logloss=[], test_pred_accuracy=[])

        # Initialize final metric
        self.final_metric = dict(elbo_neg_ave = [],
                                 train_pred_logloss=[], train_pred_accuracy=[],
                                 test_pred_logloss=[], test_pred_accuracy=[])

    def _evaluate_model(self, metric_dict, x_train, y_train, x_test, y_test):

        # Normalize train x
        if self.normalize_x:
            x_train = (x_train-self.x_means)/self.x_stds

        # Get train predictions
        logits_list = [self.model(x_train) for _ in range(self.train_params['eval_mc_samples'])]

        # Store train metrics
        metric_dict['train_pred_logloss'].append(metrics.predictive_avneg_loglik_categorical(logits_list, y_train).detach().cpu().item())
        metric_dict['train_pred_accuracy'].append(metrics.softmax_predictive_accuracy(logits_list, y_train).detach().cpu().item())
        metric_dict['elbo_neg_ave'].append(metrics.avneg_elbo_categorical(logits_list, y_train, train_set_size = self.data.get_train_size(), kl = self.model.kl_divergence()).detach().cpu().item())

        # Normalize test x
        if self.normalize_x:
            x_test = (x_test-self.x_means)/self.x_stds

        # Get test predictions
        logits_list = [self.model(x_test) for _ in range(self.train_params['eval_mc_samples'])]

        # Store test metrics
        metric_dict['test_pred_logloss'].append(metrics.predictive_avneg_loglik_categorical(logits_list, y_test).detach().cpu().item())
        metric_dict['test_pred_accuracy'].append(metrics.softmax_predictive_accuracy(logits_list, y_test).detach().cpu().item())

    def _print_progress(self, epoch):

        # Print progress
        print('Epoch [{}/{}], Neg. Ave. ELBO: {:.4f}, Logloss: {:.4f}, Test Logloss: {:.4f}'.format(
                epoch+1,
                self.train_params['num_epochs'],
                self.metric_history['elbo_neg_ave'][-1],
                self.metric_history['train_pred_logloss'][-1],
                self.metric_history['test_pred_logloss'][-1]))

    def _closure(self, x, y):
        self.optimizer.zero_grad()
        logits = self.prediction(x)
        loss = self.objective(logits, y)
        loss.backward()
        return loss

#######################################
## Define experiment class for Vadam ##
#######################################

class ExperimentVadamMLPClass(Experiment):

    def __init__(self, data_set, model_params, train_params, optim_params, evals_per_epoch=1, normalize_x=False, results_folder="./results", data_folder=DEFAULT_DATA_FOLDER, use_cuda=torch.cuda.is_available()):
        super(type(self), self).__init__(data_set, model_params, train_params, optim_params, evals_per_epoch, normalize_x, results_folder, data_folder, use_cuda)

        # Define name for experiment class
        experiment_name = "vadam_mlp_class"

        # Define folder name for results
        self.folder_name = folder_name(experiment_name, data_set, model_params, train_params, optim_params, results_folder)

        # Initialize model
        self.model = MLP(input_size = self.data.num_features,
                         hidden_sizes = model_params['hidden_sizes'],
                         output_size = self.data.num_classes,
                         act_func = model_params['act_func'])
        if use_cuda:
            self.model = self.model.cuda()

        # Define prediction function
        def prediction(x):
            logits = self.model(x)
            return logits
        self.prediction = prediction

        # Define objective
        self.objective = metrics.avneg_loglik_categorical

        # Initialize optimizer
        self.optimizer = Vadam(self.model.parameters(),
                               lr = optim_params['learning_rate'],
                               betas = optim_params['betas'],
                               prior_prec = model_params['prior_prec'],
                               prec_init = optim_params['prec_init'],
                               num_samples = train_params['train_mc_samples'],
                               train_set_size = self.data.get_train_size())

        # Initialize metric history
        self.metric_history = dict(elbo_neg_ave = [],
                                   train_pred_logloss=[], train_pred_accuracy=[],
                                   test_pred_logloss=[], test_pred_accuracy=[])

        # Initialize final metric
        self.final_metric = dict(elbo_neg_ave = [],
                                 train_pred_logloss=[], train_pred_accuracy=[],
                                 test_pred_logloss=[], test_pred_accuracy=[])

    def _evaluate_model(self, metric_dict, x_train, y_train, x_test, y_test):

        # Normalize train x
        if self.normalize_x:
            x_train = (x_train-self.x_means)/self.x_stds

        # Get train predictions
        logits_list = self.optimizer.get_mc_predictions(self.model.forward, inputs = x_train, mc_samples = self.train_params['eval_mc_samples'], ret_numpy=False)

        # Store train metrics
        metric_dict['train_pred_logloss'].append(metrics.predictive_avneg_loglik_categorical(logits_list, y_train).detach().cpu().item())
        metric_dict['train_pred_accuracy'].append(metrics.softmax_predictive_accuracy(logits_list, y_train).detach().cpu().item())
        metric_dict['elbo_neg_ave'].append(metrics.avneg_elbo_categorical(logits_list, y_train, train_set_size = self.data.get_train_size(), kl = self.optimizer.kl_divergence()).detach().cpu().item())

        # Normalize test x
        if self.normalize_x:
            x_test = (x_test-self.x_means)/self.x_stds

        # Get test predictions
        logits_list = self.optimizer.get_mc_predictions(self.model.forward, inputs = x_test, mc_samples = self.train_params['eval_mc_samples'], ret_numpy=False)

        # Store test metrics
        metric_dict['test_pred_logloss'].append(metrics.predictive_avneg_loglik_categorical(logits_list, y_test).detach().cpu().item())
        metric_dict['test_pred_accuracy'].append(metrics.softmax_predictive_accuracy(logits_list, y_test).detach().cpu().item())

    def _print_progress(self, epoch):

        # Print progress
        print('Epoch [{}/{}], Neg. Ave. ELBO: {:.4f}, Logloss: {:.4f}, Test Logloss: {:.4f}'.format(
                epoch+1,
                self.train_params['num_epochs'],
                self.metric_history['elbo_neg_ave'][-1],
                self.metric_history['train_pred_logloss'][-1],
                self.metric_history['test_pred_logloss'][-1]))

    def _closure(self, x, y):
        self.optimizer.zero_grad()
        logits = self.prediction(x)
        loss = self.objective(logits, y)
        loss.backward()
        return loss


import torch
from torch import nn
import torch.optim as optim

from utils import evaluation

def training(batch_size, n_epoch, lr, model_dir, train, valid, model, device):
    # Keep the loss and accuracy at every iteration for plotting
    train_loss_list = []
    valid_loss_list = []
    train_acc_list = []
    valid_acc_list = []

    # print model status
    total = sum(p.numel() for p in model.parameters())
    trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
    print('\nstart training, parameter total:{}, trainable:{}\n'.format(total, trainable))

    # model params
    criterion = nn.BCELoss() #binary cross entropy loss
    t_batch_num = len(train)
    v_batch_num = len(valid)
    optimizer = optim.Adam(model.parameters(), lr=lr)
    #optimizer = torch.optim.SGD(model.parameters(), lr, momentum=0.9, weight_decay=1e-4)
    #optimizer = optim.Adadelta(model.parameters(), lr=lr) #lr=1

    # training loop
    model.train()
    train_total_loss, train_total_acc, best_acc, best_loss = 0, 0, 0, 10000
    for epoch in range(n_epoch):
        # training
        train_total_loss, train_total_acc = 0, 0
        for i, (inputs, labels) in enumerate(train):
            inputs = inputs.to(device, dtype=torch.long) # device is "cuda" inputs to torch.cuda.LongTensor
            labels = labels.to(device, dtype=torch.float) # device is "cuda" labels to torch.cuda.FloatTensor
            optimizer.zero_grad() # duo to loss.backward(), gradient will accumulate，so we need to zero it every batch
            outputs = model(inputs)
            outputs = outputs.squeeze() # squeeze (batch_size,1) to (batch_size)
            loss = criterion(outputs, labels) # calculate training loss
            loss.backward() # compute gradient from loss
            optimizer.step() # update model parameters

            correct = evaluation(outputs, labels)
            train_total_acc += (correct / batch_size)
            train_total_loss += loss.item()
            print('[ Epoch{}: {}/{} ] '.format(epoch+1, i+1, t_batch_num), end='\r')
        print('\nTrain | Loss:{:.5f} Acc: {:.3f}'.format(train_total_loss/t_batch_num, train_total_acc/t_batch_num*100))

        # validation
        model.eval() # set model to eval mode，fix model parameters
        with torch.no_grad():
            valid_total_loss, valid_total_acc = 0, 0
            for i, (inputs, labels) in enumerate(valid):
                inputs = inputs.to(device, dtype=torch.long)
                labels = labels.to(device, dtype=torch.float)
                outputs = model(inputs)
                outputs = outputs.squeeze()
                loss = criterion(outputs, labels)
                correct = evaluation(outputs, labels)
                valid_total_acc += (correct / batch_size)
                valid_total_loss += loss.item()

            print("Valid | Loss:{:.5f} Acc: {:.3f} ".format(valid_total_loss/v_batch_num, valid_total_acc/v_batch_num*100))
            if valid_total_acc > best_acc or valid_total_loss < best_loss:
                best_acc = valid_total_acc
                best_loss = valid_total_loss
                torch.save(model, "{}/ckpt_{}.model".format(model_dir,valid_total_acc/v_batch_num*100))
                print('saving model with acc {:.3f}'.format(valid_total_acc/v_batch_num*100))
        print('-----------------------------------------------')
        model.train() # set model to train mode，let model parameters updatable

        # store acc and loss result
        train_loss_list.append(train_total_loss/t_batch_num)
        valid_loss_list.append(valid_total_loss/v_batch_num)
        train_acc_list.append(train_total_acc/t_batch_num*100)
        valid_acc_list.append(valid_total_acc/v_batch_num*100)

    # plotting result
    import matplotlib.pyplot as plt

    # Loss curve
    plt.plot(train_loss_list)
    plt.plot(valid_loss_list)
    plt.title('Loss')
    plt.legend(['train', 'valid'])
    plt.savefig('loss.png')
    plt.show()

    # Accuracy curve
    plt.plot(train_acc_list)
    plt.plot(valid_acc_list)
    plt.title('Accuracy')
    plt.legend(['train', 'valid'])
    plt.savefig('acc.png')
    plt.show()


# -*- coding: utf-8 -*-
"""
Created on Fri Sep  4 11:41:26 2020

@author: QuYue
"""

#%%
import record
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

#%%
file_name = ['./result/e4_2-2020-09-04_16-33-41.pkl']
#%%
class PARM:
    def __init__(self):
        self.dataset_ID = 1

def mean_std(vector, dim=None):
    vector = np.array(vector)
    if dim == None:
        m = vector.mean()
        n = vector.std()
    else:
        m = vector.mean(dim)
        n = vector.std(dim)
    return m, n

num = len(file_name)
Parm = []
for name in file_name:
    Parm.append(record.read(name))

#%%
for parm in Parm:
    print(parm.random_seed)

#%%
length = len(Parm[0].result['SoloNet'])
Acc = np.empty([length, num])
Total = np.empty([length, num])
for i in range(num):
    parm = Parm[i]
    for j in range(length):
        Acc[j, i] = parm.result['SoloNet'][j]['Acc']
        Total[j, i] = parm.result['SoloNet'][j]['TotalAcc']

print(f"SoloNetAcc: {mean_std(Acc, 1)}")
print(f"SoloNetTotal: {mean_std(Total, 1)}")

# %%
Aver = np.empty([num])
AverAcc = np.empty([num, length])
for i in range(num):
    parm = Parm[i]
    Aver[i] = parm.result['FusionNet']['AverFusion']['TotalAcc']
    for j in range(length):
        AverAcc[i,j] = parm.result['FusionNet']['AverFusion']['Acc'][j]

print(f"AverAcc: {mean_std(AverAcc, 0)}")
print(f"AverTotal: {mean_std(Aver)}")

# %%
pinv = np.empty([num])
pinvAcc = np.empty([num, length])
for i in range(num):
    parm = Parm[i]
    pinv[i] = parm.result['FusionNet']['PinvFusion']['TotalAcc']
    for j in range(length):
        pinvAcc[i,j] = parm.result['FusionNet']['PinvFusion']['Acc'][j]

print(f"pinvAcc: {mean_std(pinvAcc, 0)}")
print(f"pinvTotal: {mean_std(pinv)}")

# %%
pinv_w = np.empty([num])
pinv_wAcc = np.empty([num, length])
for i in range(num):
    parm = Parm[i]
    pinv_w[i] = parm.result['FusionNet']['PinvFusion_W']['TotalAcc']
    for j in range(length):
        pinv_wAcc[i,j] = parm.result['FusionNet']['PinvFusion_W']['Acc'][j]

print(f"pinvwAcc: {mean_std(pinv_wAcc, 0)}")
print(f"pinvwTotal: {mean_std(pinv_w)}")

# %%
pinv_AF = np.empty([num])
pinv_AFAcc = np.empty([num, length])
for i in range(num):
    parm = Parm[i]
    pinv_AF[i] = parm.result['FusionNet']['PinvFusion+AF']['TotalAcc']
    for j in range(length):
        pinv_AFAcc[i,j] = parm.result['FusionNet']['PinvFusion+AF']['Acc'][j]

print(f"pinvAFAcc: {mean_std(pinv_AFAcc, 0)}")
print(f"pinvAFTotal: {mean_std(pinv_AF)}")

# %%
pinv_MAN = np.empty([num])
pinv_MANAcc = np.empty([num, length])
for i in range(num):
    parm = Parm[i]
    pinv_MAN[i] = parm.result['FusionNet']['PinvFusion+MAN']['TotalAcc']
    for j in range(length):
        pinv_MANAcc[i,j] = parm.result['FusionNet']['PinvFusion+MAN']['Acc'][j]

print(f"pinvMANAcc: {mean_std(pinv_MANAcc, 0)}")
print(f"pinvMANTotal: {mean_std(pinv_MAN)}")

# %%
Origin = np.empty([num])
for i in range(num):
    parm = Parm[i]
    Origin[i] = max(parm.result['Origin']['origin'])

print(f"Origin: {mean_std(Origin)}")

# %%
FineTune = np.empty([num])
for i in range(num):
    parm = Parm[i]
    FineTune[i] = max(parm.result['FusionNet']['FusionFineTune'])

print(f"FineTune: {mean_std(FineTune)}")

#%%
MLKD = np.empty([num])
for i in range(num):
    parm = Parm[i]
    MLKD[i] = max(parm.result['FusionNet']['FusionMLKD'])

print(f"MLKD: {mean_std(MLKD)}")

#%%
plt.figure(1)
plt.plot(Parm[0].time['Origin'], Parm[0].result['Origin']['origin'])
plt.plot(Parm[0].time['FusionFineTune'], Parm[0].result['FusionNet']['FusionFineTune'])
plt.plot(Parm[0].time['FusionMLKD'], Parm[0].result['FusionNet']['FusionMLKD'])
plt.legend(['Normal', 'FineTune', 'MLKD'])
plt.grid('on')
plt.show()

#%%
def counter(acc):
    if int(acc//0.1)==9:
        return 9 + int((acc-0.9)//0.01)
    else:
        return int(acc//0.1)

def speed(acc_list):
    acc = [counter(i) for i in acc_list]
    acc = np.array(acc)
    return acc

d = speed(Parm[0].result['Origin']['origin'])
d = pd.Series(d).value_counts()
d1 = dict(d)
print(d)

d = speed(Parm[0].result['FusionNet']['FusionFineTune'])
d = pd.Series(d).value_counts()
d2 = dict(d)
print(d)

d = speed(Parm[0].result['FusionNet']['FusionMLKD'])
d = pd.Series(d).value_counts()
d3 = dict(d)
print(d)

#%%
h = {}
for i in range(1,19):
    a = []
    if i in d1:
        a.append(d1[i])
    else:
        a.append(0)
    if i in d2:
        a.append(d2[i])
    else:
        a.append(0)
    if i in d3:
        a.append(d3[i])
    else:
        a.append(0)
    h[i] = a

#%%
def autolabel(rects):
    for rect in rects:
        height = rect.get_height()
        plt.text(rect.get_x()+rect.get_width()/2.- 0.2, 1.03*height, '%s' % int(height))
name_list = ['10-20', '20-30', '30-40', '40-50', '50-60', '60-70', '70-80', '80-90', '90-91', '91-92','92-93','93-94','94-95','95-96','96-97','97-98','98-99','99-100']
num_list = [h[i][0] for i in range(1, 19)]
num_list1 = [h[i][1] for i in range(1, 19)]
num_list2 = [h[i][2] for i in range(1, 19)]
plt.figure(2)
plt.grid('on')
x = list(range(len(name_list)))
total_width, n = 0.8, 3
width = total_width / n

plt.bar(x, num_list, width=width, label='Normal', fc='b')

for i in range(len(x)):
    x[i] += 2*width
plt.bar(x, num_list2, width=width, label='MLKD', tick_label=name_list, fc='r')

for i in range(len(x)):
    x[i] -= width
plt.bar(x, num_list2, width=width, label='FineTune', tick_label=name_list, fc='g')
plt.legend()
plt.xlabel('Accuray')
plt.ylabel()
plt.show()


#! /usr/bin/env python

import pygame, sys, time, string
from pygame.locals import *
from companyState import *

pygame.init()
blue = 0, 0, 255
background = 0,0,0
font = pygame.font.Font(None, 36)

def header(screen, companyState):

	screen.fill(background)

	width = screen.get_width()
	height = screen.get_height()

	# Draw header line
	point1 = 0, 30
	point2 = screen.get_width(), 30
	pygame.draw.line(screen, blue, point1, point2)

	# Draw company name in header text
	text = font.render(companyState.companyName, 1, blue)
	screen.blit(text, (2, 5))

	# Draw day in header text
	text = font.render(str(companyState.day), 1, blue)
	screen.blit(text, (width-30, 5))

	return screen

def main():

	company = CompanyState("Tinder")

	screen = pygame.display.set_mode((640, 400))
	running = True
	screen.fill((0, 0, 0))

	while running:
        # events for txtbx
		events = pygame.event.get()
        # process other events
		for event in events:
			# close it x button si pressed
			if event.type == QUIT: return

		screen = header(screen, company)
		pygame.display.flip()

		company.tickTime()
		time.sleep(2)

if __name__ == '__main__':
	main()


from sklearn.base import BaseEstimator
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.tree import export_graphviz

import xgboost as xgb
import numpy as np

class ClassifierRandomForest(BaseEstimator):
    def __init__(self):
        print("I don't like classifying people :(")

    def fit(self, X, y):
        self.clf = RandomForestClassifier(
            n_estimators=2, max_leaf_nodes=2, random_state=61)
        self.clf.fit(X, y)

    def predict(self, X):
        return self.clf.predict(X)

    def predict_proba(self, X):
        return self.clf.predict_proba(X)

# ----------------------------------------------------------------------

class ClassifierSimpleTree(BaseEstimator):
    def __init__(self, nleaf=80):
        self.clf = DecisionTreeClassifier(
            max_depth=80,
            max_features=57,
            max_leaf_nodes=nleaf
        )
        self.nleaf = nleaf
        self.feats = nleaf
        self.name = 'ClassifierSimpleTree' + '_nleaf' + str(self.nleaf)
        print(self.name)

    def fit(self, X, y):
        self.clf.fit(X, y)
        with open("simple_tree.dot", 'w') as f:
            f = export_graphviz(self.clf, out_file=f)

        importances = self.clf.feature_importances_
        indices = np.argsort(importances)[::-1]

        # Print the feature ranking
        print("Feature ranking:")
        for f in range(X.shape[1]):
            print("%d. feature %d (%f)" %
                  (f, indices[f], importances[indices[f]]))

    def predict(self, X):
        return self.clf.predict(X)

    def predict_proba(self, X):
        return self.clf.predict_proba(X)

class ClassifierXGB(BaseEstimator):
    def __init__(self, level=0):
        self.name = 'ClassifierXGB' + '_feat' + str(level)
        print(self.name)
        pass

    def fit(self, X, y):
        dtrain = xgb.DMatrix(X, y)
        param = {'objective': 'binary:logistic', 'max_depth': 8,
                 'subsample': 1, 'eta': 0.1, 'num_round': 400,
                 'gamma': 0, 'silent': 1}
        self.clf = xgb.train(param, dtrain,
                             num_boost_round=param['num_round'])
        print("weight scores")
        print(self.clf.get_score(importance_type="weight"))
        print("gain scores")
        print(self.clf.get_score(importance_type="gain"))
        print("cover scores")
        print(self.clf.get_score(importance_type="cover"))

    def predict(self, X):
        dpred = xgb.DMatrix(X)
        return (np.sign(self.clf.predict(dpred) - 0.5) / 2 + 0.5)

    def predict_proba(self, X):
        dpred = xgb.DMatrix(X)
        pred = self.clf.predict(dpred)
        pred = np.array([1 - pred, pred]).T
        return pred

Classifier = ClassifierRandomForest
#Classifier = ClassifierSimpleTree
#Classifier = ClassifierXGB

# class Classifier(ClassifierSimpleTree):
#    def __init__(self):
#        ClassifierSimpleTree.__init__(self)
#        pass


# This is the MIT license: http://www.opensource.org/licenses/mit-license.php
#
# Copyright (c) 2005-2012 the SQLAlchemy authors and contributors <see AUTHORS file>.
# SQLAlchemy is a trademark of Michael Bayer.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of this
# software and associated documentation files (the "Software"), to deal in the Software
# without restriction, including without limitation the rights to use, copy, modify, merge,
# publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
# to whom the Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all copies or
# substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
# FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

from __future__ import absolute_import
from __future__ import unicode_literals
import jaydebeapi
import os
import logging
from .base import DrillDialect, DrillCompiler_sadrill

class DrillDialect_jdbc(DrillDialect):
    jdbc_db_name = "drill"
    jdbc_driver_name = "org.apache.drill.jdbc.Driver"

    statement_compiler = DrillCompiler_sadrill
    logging.basicConfig(format='%(asctime)s - %(message)s', level=logging.INFO)

    def __init__(self, *args, **kwargs):
        super(DrillDialect_jdbc, self).__init__(*args, **kwargs)
        self.jdbc_driver_path = os.environ.get('DRILL_JDBC_DRIVER_PATH')
        self.jdbc_jar_name = os.environ.get('DRILL_JDBC_JAR_NAME')

        if self.jdbc_driver_path is None:
            raise Exception('To connect to Drill via JDBC, you must set the DRILL_JDBC_DRIVER path to the location of the Drill JDBC driver.')

        if self.jdbc_jar_name is None:
            raise Exception(
                'To connect to Drill via JDBC, you must set the DRILL_JDBC_JAR_NAME environment variable.')

    def initialize(self, connection):
        super(DrillDialect_jdbc, self).initialize(connection)

    """
    Open a connection to a database using a JDBC driver and return
        a Connection instance.
        jclassname: Full qualified Java class name of the JDBC driver.
        url: Database url as required by the JDBC driver.
        driver_args: Dictionary or sequence of arguments to be passed to
               the Java DriverManager.getConnection method. Usually
               sequence of username and password for the db. Alternatively
               a dictionary of connection arguments (where `user` and
               `password` would probably be included). See
               http://docs.oracle.com/javase/7/docs/api/java/sql/DriverManager.html
               for more details
        jars: Jar filename or sequence of filenames for the JDBC driver
        libs: Dll/so filenames or sequence of dlls/sos used as shared
              library by the JDBC driver
        """
    def create_connect_args(self, url):
        if url is not None:
            params = super(DrillDialect, self).create_connect_args(url)[1]
            driver = self.jdbc_driver_path + self.jdbc_jar_name

            cargs = (self.jdbc_driver_name,
                     self._create_jdbc_url(url),
                     [params['username'], params['password']],
                     driver)
            cparams = {p: params[p] for p in params if p not in ['host', 'username', 'password', 'port']}

            logging.info("Cargs:" + str(cargs))
            logging.info("Cparams" + str(cparams))

            return (cargs, cparams)

    def _create_jdbc_url(self, url):
        return "jdbc:drill:drillbit=%s:%s" % (
            url.host,
            url.port or 31010
        )

    @classmethod
    def dbapi(cls):
        return jaydebeapi

dialect = DrillDialect_jdbc


#!/usr/local/bin/python3.4
import kombu
from _socket import timeout
import os
import traceback
import sys
import pprint

from adhoc import TaxiLantern

def light_up_lantern(body, message):
    lantern = TaxiLantern(26)
    lantern.on()

try:
    connection = kombu.Connection(os.getenv('RABBIT_URL'))
    channel = connection.channel()
    queue = kombu.Queue('taxiLamp', durable=False, auto_delete=False)

    consumer = kombu.Consumer(channel, queues=queue, callbacks=[light_up_lantern])
    consumer.consume()

    try:
        connection.drain_events(timeout=1)
    except timeout as e:
        pass
    except connection.connection_errors + connection.channel_errors:
        connection.close()
        print("Host down, connecting to the next one.")
        connection.ensure_connection()
        channel = connection.channel()
        consumer = kombu.Consumer(channel, queues=queue, callbacks=[light_up_lantern])
        consumer.consume()

except Exception as e:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    print("*** print_tb:")
    traceback.print_tb(exc_traceback, limit=1, file=sys.stdout)
    print("*** print_exception:")
    traceback.print_exception(exc_type, exc_value, exc_traceback, limit=2, file=sys.stdout)
    pprint.pprint("Error = " + str(e))


import requests
import json
from urllib.request import urlopen
from bs4 import BeautifulSoup

# Función genérica que escribe un archivo a partir de un texto
# Puede utilizarse para escribir un archivo HTML copia local
# de la página que vamos a leer
def escribirArchivo(texto, archivo):
    f = open(archivo, "w")
    f.write(texto)
    f.close()

def devolverSoup(url):
  resp = requests.get(url)
  # resp = urlopen(url).read().decode('utf-8')

  if resp.status_code == 200:
    # print(resp.encoding)
    # resp.encoding = 'utf-8'
    # print(resp.encoding)
    contenido = resp.content

    print(contenido)
    soup = BeautifulSoup(contenido, 'html.parser')
    return soup

# Función genérica que crea un archivo JSON de nombre "nombre" y
# con los datos "datos"
def crearJSON(nombre, datos):
  result = {}
  result[nombre] = datos

  with open(nombre + ".json", "w") as write_file:
    json.dump(result, write_file, indent=2)

# Función específica a la página y la información que queremos
# extraer
def extraerProductos(soup):
  productos = soup.find_all('tr')
  result = []

  for producto in productos:
    objProducto = {}

    # Recuperamos el nombre del producto y el primer precio en COP
    nombre = producto.find('td', class_='product-name')
    precio = producto.find('td', class_='price')

    # Para no tratar el encabezado
    if None in (nombre, precio):
      continue

    objProducto['nombre'] = nombre.text.strip()
    precioAux = precio.text.strip().replace('$', '').replace(',', '.')
    objProducto['precio'] = float(precioAux)

    result.append(objProducto)

  return result

# Dada la URL, creamos un objeto soup que contiene la página
# Luego extraemos la información que necesitamos
# Finalemente creamos un archivo JSON con los datos extraidos
if __name__ == "__main__":

  url = "https://preciosmundi.com/colombia/precios-supermercado"

  soup = devolverSoup(url)
  productos = extraerProductos(soup)

  crearJSON("productos", productos)


import cv2 as cv
import numpy as np
from random import randint
import math as math
from population import *
from path import *
from point import *

global esc
esc = 27

class mainWindow:
    def __init__(self):
        self.heightmap = cv.imread('terrain.png')
        self.tmp_map = self.heightmap.copy()
        self.height, self.width = self.tmp_map.shape[:2]
        self.window_name = 'PAPAJ'
        self.mWindow = cv.namedWindow(self.window_name, cv.WINDOW_GUI_NORMAL)
        self.maphandle = cv.imshow(self.window_name, self.tmp_map)
        self.key = 155
        esc = 27
    def drawPath(self, path):
        cv.line(self.tmp_map, path.start_point.getPoint(), path.getPoint(0), (255, 0, 0), 1)
        for i in range(1, path.length):
            cv.line(self.tmp_map, path.getPoint(i-1), path.getPoint(i), (0, 255, 0), 1)
        cv.line(self.tmp_map, path.end_point.getPoint(), path.getPoint(path.length-1), (255, 0, 0), 1)
    def drawPop(self, _population):
        for i in range(0, len(_population.paths)):
            self.drawPath(_population.paths[i])


from flask import Flask, render_template, redirect
import pymongo
import scrape_weather

app = Flask(__name__)

conn = "mongodb://localhost:27017"
client = pymongo.MongoClient(conn)
db = client.weather
col = db.data

# weather_data = scrape_weather.scrape()
# from pprint import pprint
# pprint(weather_data['alabama'])
# col.remove()
# col.insert(weather_data)

@app.route("/")
def home():

    data = list(col.find())
    print ("squid")
    print (data)
    return render_template("index.html", weather=data)

@app.route("/scrape")
def scrape():

    weather_data = scrape_weather.scrape()
    col.insert(weather_data)
    print("start")
    print (list(col.find()))
    print("finish")
    return redirect("/")

@app.route("/hightemp")
def highmap():
    return render_template("hightemp.html", weather=data)

if __name__ == "__main__":
    app.run(debug=True)