# -*- coding:utf-8 -*-
__author__ = 'Threedog'
__Date__ = '2018/7/30 20:59'

import socket
import threading
import json

from PyQt5.QtWidgets import QLabel,QMessageBox,QWidget,QLineEdit,QVBoxLayout,QHBoxLayout,QPushButton,QListWidget,QListWidgetItem
from PyQt5.QtGui import QMouseEvent,QPixmap,QIcon,QCloseEvent
from PyQt5.QtCore import QPoint,pyqtSignal,Qt
import pygame

from Base import Chessman,is_win,trans_pos
from Base import BasePlayer

from TDWidgets import TDPushButton
import Base

# addr = ("www.threedog.top",3003)
addr = ('127.0.0.1',3003)
chessboard = Base.chessboard
# 列表记录走棋坐标，用于悔棋操作
history = []

pygame.mixer.init()
pygame.mixer.music.load("source/luozisheng.wav")
sound = pygame.mixer.Sound("source/cuicu.wav")

def recv_sockdata(the_socket):
    '''从网络接收数据'''
    total_data = ""
    while True:
        data = the_socket.recv(1024).decode()
        if "END" in data:
            # 注意这里不去掉末尾的END，直接转发给另一端
            total_data += data[:data.index("END")]
            break
        total_data += data
    # print("total data:" , total_data)
    return total_data

class NetworkConfig(QWidget):
    '''
    配置网络信息的窗体
    '''

    # 传输数据的信号
    dataSignal = pyqtSignal(dict, name='data')
    #  连接中断的信号
    disconnectSignal = pyqtSignal()

    def __init__(self,main_window,parent=None):
        super().__init__(parent)
        self.setWindowTitle('网络配置')
        self.setWindowIcon(QIcon('source/icon.ico'))
        self.main_window = main_window

        self.game_window = None
        self.is_join = False
        self.keep_recv = True

        # 启动网络
        try:
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.sock.connect(addr)
            self.is_connected = True
            threading.Thread(target=self.recv_data).start()
            self.dataSignal.connect(self.deal_data)
        except ConnectionRefusedError as e:
            QMessageBox.information(self,"消息","连接服务器失败，程序即将退出")
            raise e

        # 显示自己的昵称
        self.layout_h = QHBoxLayout()  # 水平布局1
        self.label_name = QLabel("昵称：",self)
        self.name_edit = QLineEdit(self)
        self.name_edit.setText("未命名")
        self.layout_h.addWidget(self.label_name,1)
        self.layout_h.addWidget(self.name_edit,3)

        # 玩家列表文字：
        self.layout_h1 = QHBoxLayout()  # 水平布局1
        self.label_players = QLabel("玩家列表：",self)
        self.refresh_btn = QPushButton("刷新",self)
        self.refresh_btn.clicked.connect(self.refresh)
        self.layout_h1.addWidget(self.label_players)
        self.layout_h1.addWidget(self.refresh_btn)

        # 加载玩家列表
        self.layout_h2 = QHBoxLayout()  # 水平布局1
        self.list_widget = QListWidget()
        self.refresh()  # 发送消息刷新用户列表
        self.list_widget.itemDoubleClicked.connect(self.item_double_clicked)
        self.layout_h2.addWidget(self.list_widget)

        self.layout_h3 = QHBoxLayout()  # 水平布局2
        self.join_btn = QPushButton("加入房间",self)
        self.join_btn.clicked.connect(self.join)
        self.battle_btn = QPushButton("选择对战",self)
        self.battle_btn.clicked.connect(self.battle)
        self.battle_btn.setEnabled(False)
        self.layout_h3.addWidget(self.join_btn)
        self.layout_h3.addWidget(self.battle_btn)

        self.layout_main = QVBoxLayout()  # 整体垂直布局
        self.layout_main.addLayout(self.layout_h)
        self.layout_main.addLayout(self.layout_h1)
        self.layout_main.addLayout(self.layout_h2)
        self.layout_main.addLayout(self.layout_h3)
        self.setLayout(self.layout_main)
        self.disconnectSignal.connect(self.dis_connect)

    def item_double_clicked(self,item):
        if not self.is_join:
            return  # 如果没有加入房间， 双击无效
        # print(item)
        # print(item.text())
        data = {
            "target": "server",
            "msg": "battle",
            "data": item.text()
        }
        self.sock.sendall((json.dumps(data) + " END").encode())

    def refresh(self):
        data = {
            "target":"server",
            "msg":"refresh",
            "data":""
        }
        self.sock.sendall((json.dumps(data)+" END").encode())

    def join(self):
        '''加入房间'''
        if not self.is_join:
            # listwidget_item = QListWidgetItem(self.name_edit.text(),self.list_widget)
            # self.list_widget.addItem(listwidget_item)
            data = {
                "target": "server",
                "msg": "join",
                "data": self.name_edit.text().strip()
            }
            # print(self.name_edit.text())
            self.sock.sendall((json.dumps(data) + " END").encode())
            self.battle_btn.setEnabled(True)
            self.name_edit.setEnabled(False)
            self.join_btn.setText("退出房间")
            self.is_join = True
        else:
            # 退出房间
            data = {
                "target": "server",
                "msg": "quit"
                # "data": self.name_edit.text()
            }
            self.sock.sendall((json.dumps(data) + " END").encode())
            self.battle_btn.setEnabled(False)
            self.name_edit.setEnabled(True)
            self.is_join = False
            self.join_btn.setText("加入房间")

    def battle(self):
        '''选择对战'''
        data = {
            "target": "server",
            "msg": "battle",
            "data": self.list_widget.currentItem().text()
        }
        self.sock.sendall((json.dumps(data) + " END").encode())

    def deal_data(self,json_data):
        '''数据处理'''
        if json_data['msg'] == 'player_list':
            # print(json_data['data'])
            self.list_widget.clear()
            for name in json_data['data']:
                self.widget_item = QListWidgetItem(name, self.list_widget)
                self.list_widget.addItem(self.widget_item)

        elif json_data['msg'] == 'get_name':  # 收到服务器分配的用户名
            self.name_edit.setText(json_data['data'])

        elif json_data['msg'] == "replay":
            if json_data['type'] == 'join':  # 加入游戏失败
                QMessageBox.information(self,"消息",json_data['data'])
                self.battle_btn.setEnabled(False)
                self.name_edit.setEnabled(True)
                self.is_join = False
                self.join_btn.setText("加入房间")

            if json_data['type'] == 'battle':  # 对战
                if json_data['data'] == True:
                    # 进入对战模式
                    self.game_window = NetworkPlayer(sock=self.sock,name=json_data['name'])
                    self.game_window.backSignal.connect(self.main_window.show)  # 点击后退按钮触发的信号
                    self.game_window.exitSignal.connect(self.main_window.game_over)  # 如果程序退出，触发的信号
                    self.game_window.show()
                    self.close()
                else :
                    QMessageBox.information(self,"提示",json_data['info'])

    def dis_connect(self):
        QMessageBox.information(self,"提示","与服务器断开连接，即将返回主界面")
        self.is_connected = False
        self.close()

    def recv_data(self):
        while self.keep_recv:
            try:
                res_data = recv_sockdata(self.sock)     # 本地收到数据
                json_data = json.loads(res_data)        # json转成字典
                # 在线程处理函数中不能直接进行界面的相关操作，所以用一个信号把数据发送出来
                self.dataSignal.emit(json_data)
                # self.deal_data(json_data,parent)

            except (ConnectionAbortedError,ConnectionResetError):
                # print("与服务器断开连接")
                self.disconnectSignal.emit()
                break  # 退出循环，线程结束

            except json.JSONDecodeError:
                # print("数据解析错误")
                print("Error Data:",res_data)

    def closeEvent(self, a0: QCloseEvent):
        if self.is_connected:
            data = {
                "target": "server",
                "msg": "quit"
            }
            self.keep_recv = False # 结束接收线程
            self.sock.sendall((json.dumps(data) + " END").encode())
        else:
            self.main_window.show()
        super().closeEvent(a0)

class NetworkPlayer(BasePlayer):
    dataSignal = pyqtSignal(dict,name='data')
    disconnectSignal = pyqtSignal()

    # 网络对战窗体
    def __init__(self,name,sock,parent=None):
        super().__init__(parent)
        self.name = name # 对方的昵称
        self.setWindowTitle('与 {} 对战中'.format(name))

        self.label_statu = QLabel("游戏状态：",self)
        self.label_statu.resize(100,20)
        self.label_statuvalue = QLabel("点击开始",self)
        self.label_statuvalue.resize(200,30)
        self.label_statuvalue.setAlignment(Qt.AlignTop)
        self.label_statu.move(630,200)
        self.label_statuvalue.move(690,204)

        # 清空棋盘
        global chessboard
        chessboard = [[None for i in range(0,19)] for i in range(0,19)]
        # 默认情况下游戏是结束状态， 通过开始按钮才可以触发
        self.is_over = True
        # 标识黑旗白旗
        self.color = 'w'
        # 是否是我的回合
        self.my_turn = False
        # 保持连接
        self.keep_connect = True
        # 三个按钮点击触发的函数
        self.restart_button.click_signal.connect(self.restart)
        self.huiqi_button.click_signal.connect(self.goback)
        self.renshu_button.click_signal.connect(self.lose)
        self.win_label = None

        self.cuicu_button = TDPushButton(self,"source/催促按钮_normal.png","source/催促按钮_hover.png","source/催促按钮_press.png",parent=self)
        self.cuicu_button.move(640,450)
        self.cuicu_button.click_signal.connect(self.cuicu)
        self.chess_pos.hide()  # 这个位置标识隐藏起来

        # 网络模块启动
        self.is_connected = True  # 默认情况下有连接
        self.is_listening = True  # 默认情况下启动了监听
        self.tcp_socket = sock
        self.label_statuvalue.setText("连接成功,\n点击开始")
        threading.Thread(target=self.recv_data, args=(self.tcp_socket, '')).start()

        self.dataSignal.connect(self.deal_data)
        self.disconnectSignal.connect(self.dis_connect)

    def dis_connect(self):
        QMessageBox.information(self,"提示","与服务器断开连接，即将返回主界面")
        self.back()

    def deal_data(self,data):
        '''
        对收到的数据进行处理
        '''
        print(data)
        if data['msg'] == 'action':

            if data['data'] == 'restart':
                result = QMessageBox.information(self,"消息","对方请求(重新)开始游戏，是否同意？",QMessageBox.Yes | QMessageBox.No)
                if result == QMessageBox.Yes:
                    data = {"msg": "replay", "data": True, "type":"restart","target":"player"}
                    self.tcp_socket.sendall((json.dumps(data) + " END").encode())
                    self.restart_func()
                    self.is_over = False
                    self.my_turn = True
                    if self.my_turn:
                        self.label_statuvalue.setText("己方回合")
                    else:
                        self.label_statuvalue.setText("对方回合")
                else:
                    data = {"msg": "replay", "data": False, "type": "restart","target":"player"}
                    self.tcp_socket.sendall((json.dumps(data) + " END").encode())
                    self.label_statuvalue.setText("点击开始")

            if data['data'] == 'lose':
                QMessageBox.information(self,"消息","对方认输")
                if self.my_turn:
                    self.win(color=self.color)
                else:
                    self.change_color()
                    self.win(color=self.color)

            if data['data'] == 'goback':
                result = QMessageBox.information(self, "消息", "对方请求悔棋，是否同意？", QMessageBox.Yes | QMessageBox.No)
                if result == QMessageBox.Yes:
                    data = {"msg": "replay", "data": True, "type": "goback","target":"player"}
                    self.tcp_socket.sendall((json.dumps(data) + " END").encode())
                    self.goback_func()
                    # self.is_over = False
                    if self.my_turn:
                        self.label_statuvalue.setText("己方回合")
                    else:
                        self.label_statuvalue.setText("对方回合")
                else:
                    data = {"msg": "replay", "data": False, "type": "goback","target":"player"}
                    self.tcp_socket.sendall((json.dumps(data) + " END").encode())
                    # self.label_statuvalue.setText("等待开始")

            if data['data'] == 'cuicu':
                # print(self.is_connected)
                if not self.is_connected:
                    return
                if self.is_over :
                    return
                # print("cuicu")
                sound.play()
                # pygame.mixer.music.load("source/luozisheng.wav")

            if data['data'] == 'ready':
                pass

            if data['data'] == 'exit':
                # 对方退出游戏  回到游戏大厅
                print("receive exit!!!")
                self.keep_connect = False
                QMessageBox.information(self,"消息","对方退出游戏对局，即将返回游戏主界面")
                self.back()
                # self.is_connected = False
                # self.is_listening = False
                # self.tcp_socket.close()
                # self.tcp_socket = None
            # print(data)

        elif data['msg'] == 'position':
            # print(data['data'])
            # 在对应位置落子
            pos = data['data']
            if chessboard[pos[1]][pos[0]] is not None:
                return  # 如果对应位置不为空，说明有棋子，则直接返回

            self.chess = Chessman(self.color, self)
            self.chess.move(QPoint(pos[0] * 30 + 50, pos[1] * 30 + 50))
            self.chess.show()
            pygame.mixer.music.play() # 播放声音
            self.logo_move()  # 移动小标
            self.change_color()

            # 在棋盘的对应位置放上棋子
            chessboard[pos[1]][pos[0]] = self.chess
            # 并且在列表中记录坐标
            history.append((pos[1], pos[0], self.chess.color))
            # 每次落子后，都判断一下胜负
            res = is_win(chessboard)
            if res:
                self.win(res) # 通过颜色，显示胜利的图片
                return
            self.my_turn = True
            self.label_statuvalue.setText("己方回合")

        elif data['msg'] == 'replay':
            if data['type'] == 'restart': # 重开回执
                if data['data'] == True:
                    self.restart_func()
                else:
                    QMessageBox.information(self,"消息","对方拒绝了你的请求")
                    self.label_statuvalue.setText("点击开始")
                    return
                if self.my_turn:
                    self.label_statuvalue.setText("己方回合")
                else:
                    self.label_statuvalue.setText("对方回合")
            if data['type'] == 'goback': # 悔棋回执

                if data['data'] == True:
                    self.is_over = False
                    self.goback_func()
                else:
                    QMessageBox.information(self,'消息','对方拒绝了你的请求')
                if self.my_turn:
                    self.label_statuvalue.setText("己方回合")
                else:
                    self.label_statuvalue.setText("对方回合")
                self.is_over = False

        elif data['msg'] == 'name':
            self.setWindowTitle('与 {} 对战中'.format(data['data']))

    def restart_func(self):
        # 清空所有棋子
        for j in range(0, 19):
            for i in range(0, 19):
                if chessboard[i][j] is not None:
                    chessboard[i][j].close()
                    chessboard[i][j] = None
        history.clear()  # 清空历史数组
        if self.is_over:  # 如果游戏已经结束
            if self.win_label is not None:
                self.win_label.close()
            self.win_label = None
            self.is_over = False
        self.chess_pos.hide()  # 这个位置标识隐藏起来

    def recv_data(self,sock,addr):
        self.is_connected = True  # 连接状态
        # print("start receiving data ...")
        while self.keep_connect:
            # print("start receiving data ...")
            try:
                res_data = recv_sockdata(sock)
            except (ConnectionAbortedError,ConnectionResetError):
                if self.keep_connect == False:
                    break
                # print("与服务器连接断开")
                # QMessageBox.information(self, "消息", "与服务器断开连接，即将退回到主界面")
                # QMessageBox.information(self,"提示","对方已经断开连接")
                self.is_connected = False
                # 连接断开
                # self.label_statuvalue.setText("对方断线,\n点击开始重试")
                self.disconnectSignal.emit()
                break
            try:
                data = json.loads(res_data)
                # print(type(data))
                # 在线程处理函数中不能直接进行界面的相关操作，所以用一个信号把数据发送出来
                self.dataSignal.emit(data)
            except json.decoder.JSONDecodeError as e:
                print(e)
                print("error data:\n"+res_data)
                continue
            # self.deal_data(data,parent)
        self.is_connected = False  # 连接断开
        self.tcp_socket.close()
        # self.back()

    def closeEvent(self, a0: QCloseEvent):
        self.keep_connect = False
        if self.tcp_socket is not None and self.is_connected == True:
            print("???")
            self.tcp_socket.sendall((json.dumps({"msg":"action","data":"exit","target":"player"})+" END").encode())
            self.tcp_socket.close()

        return super().closeEvent(a0)

    def win(self,color):
        '''
        黑旗胜利或者白棋胜利了
        '''
        if color == 'b':
            win_pic = QPixmap('source/黑棋胜利.png')
            self.label_statuvalue.setText("黑旗胜利")
        else:
            win_pic = QPixmap('source/白棋胜利.png')
            self.label_statuvalue.setText("白棋胜利")
        self.win_label = QLabel(parent=self)
        self.win_label.setPixmap(win_pic)
        self.win_label.resize(win_pic.size())
        self.win_label.move(50, 50)  # 显示游戏结束的图片
        self.win_label.show()
        self.is_over = True  # 游戏结束

    def mouseReleaseEvent(self, a0: QMouseEvent):
        if self.is_over: # 如果游戏已经结束，点击失效
            return
        if not self.my_turn:
            # print("not my turn")
            return
        # 如果点击在棋盘区域
        if a0.x() >= 50 and a0.x() <= 50+30*19 and a0.y() >= 50 and a0.y() <= 50+30*19:

            # 讲像素坐标转化成棋盘坐标，判断棋盘此位置是否为空
            pos = trans_pos(a0)
            if chessboard[pos[1]][pos[0]] is not None:
                return # 如果对应位置不为空，说明有棋子，则直接返回

            # 不为空，则生成棋子并显示
            self.chess = Chessman(self.color,self)
            self.chess.move(a0.pos())
            self.chess.show()
            pygame.mixer.music.play()  # 播放声音
            self.logo_move() # 移动小标
            self.change_color()

            # 在棋盘的对应位置放上棋子
            chessboard[pos[1]][pos[0]] = self.chess
            # 并且在列表中记录坐标
            history.append((pos[1], pos[0],self.chess.color))
            # 将坐标发送给另一方
            if self.tcp_socket is not None:
                data={"msg":"position","data":pos,"target":"player"}
                self.tcp_socket.sendall((json.dumps(data)+" END").encode())

            # 每次落子后，都判断一下胜负
            res = is_win(chessboard)
            if res:
                self.win(res) # 通过颜色，显示胜利的图片
                return
            self.my_turn = False
            self.label_statuvalue.setText("对方回合")

    def change_color(self):
        if self.color == 'w':
            self.color = 'b'
        else:
            self.color = 'w'

    def restart(self):
        '''
        重新开始游戏
        '''
        data = {"msg": "action", "data": "restart","target":"player"}
        self.tcp_socket.sendall((json.dumps(data) + " END").encode())
        self.label_statuvalue.setText("请求(重新)开始")
        self.my_turn = False

    def goback(self):
        '''
        悔棋按钮
        '''
        if self.my_turn is False:
            return # 如果是对方回合，不能悔棋
        if self.is_over:
            return
        if not self.is_connected:
            return
        else :
            data = {"msg": "action", "data": "goback","target":"player"}
            self.tcp_socket.sendall((json.dumps(data) + " END").encode())
            self.label_statuvalue.setText("请求悔棋")
            self.is_over = True

    def goback_func(self):
        if self.is_over:
            return None  # 如果游戏已经结束了
        if len(history) == 0:
            return None  # 没有落子，不能悔棋
        chess = history.pop(-1)
        chessboard[chess[0]][chess[1]].close()
        chessboard[chess[0]][chess[1]] = None
        # self.change_color()
        if len(history) == 0:
            return None  # 没有落子，不能悔棋
        chess = history.pop(-1)
        chessboard[chess[0]][chess[1]].close()
        chessboard[chess[0]][chess[1]] = None
        # self.change_color()
        self.chess_pos.hide()  # 这个位置标识隐藏起来

    def lose(self):
        '''
        认输按钮
        '''
        if self.is_over:
            return
        if not self.is_connected:
            return
        else :
            data = {"msg": "action", "data": "lose","target":"player"}
            self.tcp_socket.sendall((json.dumps(data) + " END").encode())
            self.label_statuvalue.setText("对方胜利")
            if self.my_turn:
                self.change_color()
                self.win(color=self.color)
            else:
                self.win(color=self.color)

    def cuicu(self):
        '''
        催促按钮
        '''
        if self.my_turn:
            return
        if self.is_over:
            return
        if not self.is_connected:
            return
        else:
            data = {"msg": "action", "data": "cuicu","target":"player"}
            self.tcp_socket.sendall((json.dumps(data) + " END").encode())
            # self.label_statuvalue.setText("请求(重新)开始")

    def logo_move(self):
        self.chess_pos.show()
        self.chess_pos.move(self.chess.pos())
        self.chess_pos.raise_()

    def __del__(self):
        # print("析构函数被执行")
        pass


#!/usr/bin/env python3
import asyncio
import re
from subprocess import Popen, PIPE
import sys

def run_fas(brand, fname, recipients):
    print('brand="%(brand)s", fname="%(fname)s", recipients="%(recipients)s"'%locals())
    pp = Popen(['/usr/bin/python3', '-u', './scripts/process_firmware_file_and_send_mail.py',
                brand, fname, recipients],
               cwd='/home/mikil/firmadyne/firmadyne/',
               bufsize=1,stdout=sys.stdout, stderr=sys.stdout,
               universal_newlines=True)
    print('pp.pid=', pp.pid)
    return

@asyncio.coroutine
def handle_req(reader, writer):
    data = yield from reader.read(100)
    cmdl = data.decode()
    addr = writer.get_extra_info('peername')
    print("Received %r from %r" % (cmdl, addr))
    import shlex
    args = shlex.split(cmdl)
    if args[0] == 'run_fas':
        run_fas(*args[1:])

loop = asyncio.get_event_loop()
coro = asyncio.start_server(handle_req, '127.0.0.1', 55688, loop=loop)
server = loop.run_until_complete(coro)

# Serve requests until Ctrl+C is pressed
print('Serving on {}'.format(server.sockets[0].getsockname()))
try:
    loop.run_forever()
except KeyboardInterrupt:
    pass

# Close the server
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()


"""
byceps.services.user.deletion_service
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

User account anonymization and removal

:Copyright: 2006-2021 Jochen Kupperschmidt
:License: Revised BSD (see `LICENSE` file for details)
"""

from ...database import db
from ...events.user import UserAccountDeleted
from ...typing import UserID

from ..authorization import service as authorization_service

from . import event_service
from .dbmodels.user import User as DbUser
from . import service as user_service

def delete_account(
    user_id: UserID, initiator_id: UserID, reason: str
) -> UserAccountDeleted:
    """Delete the user account."""
    user = user_service.get_db_user(user_id)
    initiator = user_service.get_user(initiator_id)

    user_screen_name_before_anonymization = user.screen_name

    user.deleted = True
    _anonymize_account(user)

    event = event_service.build_event(
        'user-deleted',
        user.id,
        {
            'initiator_id': str(initiator.id),
            'reason': reason,
        },
    )
    db.session.add(event)

    # Deassign authorization roles.
    authorization_service.deassign_all_roles_from_user(
        user.id, initiator.id, commit=False
    )

    db.session.commit()

    return UserAccountDeleted(
        occurred_at=event.occurred_at,
        initiator_id=initiator.id,
        initiator_screen_name=initiator.screen_name,
        user_id=user.id,
        user_screen_name=user_screen_name_before_anonymization,
    )

def _anonymize_account(user: DbUser) -> None:
    """Remove user details from the account."""
    user.screen_name = None
    user.email_address = None
    user.legacy_id = None

    # Remove details.
    user.detail.first_names = None
    user.detail.last_name = None
    user.detail.date_of_birth = None
    user.detail.country = None
    user.detail.zip_code = None
    user.detail.city = None
    user.detail.street = None
    user.detail.phone_number = None

    # Remove avatar association.
    if user.avatar_selection is not None:
        db.session.delete(user.avatar_selection)


# %%
import datetime
import os
import pandas as pd
from pathlib import Path
import platform
import logging
import numpy as np
import pickle
import tensorflow as tf
import matplotlib.pyplot as plt

from utils_stra import cal_model_r2, gen_filterIPO, filter_data
from utils_stra import save_res, setwd
from model_func import runModel

setwd()
# create logger with 'spam_application'
logger = logging.getLogger('records')
logger.setLevel(logging.DEBUG)
# create file handler which logs even debug messages
fh = logging.FileHandler('records.log')
fh.setLevel(logging.DEBUG)
logger.addHandler(fh)

# %%
p = Path('.') / 'data' / "data.h5"
data = pd.read_hdf(p, key="data")

ind_ftr = [i for i in data.columns if i.startswith('Ind_')]
mcr_ftr = [i for i in data.columns if i.startswith('Macro_')]
data = data[list(data.iloc[:, :89].columns) + ind_ftr + mcr_ftr + ["Y"]]

# %%
runGPU = 0
retrain = 1
runfreq = "M"

data = filter_data(data, ["IPO"])
pre_dir = "Filter IPO"

# train30% validation20% test50% split
def intiConfig():
    config = {"runOLS3":0,
              'runOLS3+H':0,
              'runOLS5':0,
              'runOLS5+H': 0,
                "runOLS":0,
                "runOLSH":0,
                "runENET":0,
                "runPLS":0,
                "runPCR":0,
                "runNN1":0,
                "runNN2":0,
                "runNN3":0,
                "runNN4":0,
                "runNN5": 0,
                "runNN6": 0,
                "runRF": 0,
                "runGBRT": 1,
                "runGBRT2": 0
              }
    return config

config = intiConfig()

for config_key in config.keys():
    if config[config_key] == 0:
        continue
    print(f"running model {config_key}")
    # data index should be ticker date
    # runNN = sum([config[i] for i in [i for i in config.keys() if re.match("runNN[0-9]", i)]])
    runNN = "runNN" in config_key[:5]
    if runNN and runGPU:
        device_name = tf.test.gpu_device_name()
        if device_name != '/device:GPU:0' and platform.system() == 'linus':
            raise SystemError('GPU device not found')
        print('Found GPU at: {}'.format(device_name))

    # for year in tqdm(range(2013, 2019)):
    #     p_t = [str(1900), str(year)] # period of training
    #     # p_t = [str(year-3), str(year)]
    #     p_v = [str(year+1), str(year + 1)] # period of valiation
    #     p_test = [str(year + 2), str(year+2)]
    if runNN:
        model_name, bcktst_df, container, nn_valid_r2, nn_oos_r2, model_dir = runModel(data, config, retrain, runGPU, runNN, runfreq, pre_dir)

        r2v, r2v_df = cal_model_r2(container, model_name, set_type="valid")
        r2is = r2v
        print(f"{model_name} valid R2: ", "{0:.3%}".format(r2v))
        c = np.array(nn_valid_r2)
        d = np.array(nn_oos_r2)
        print("the correlation coefficients is :", np.corrcoef(c, d))
        plt.scatter(c, d)
        plt.xlabel("validation scores")
        plt.ylabel("test scores")
        plt.title(f"{model_name}")
        plt.show()
        with open(model_dir / "validation_score.pkl", "wb") as f:
            pickle.dump(c, f)
        with open(model_dir / "testing_score.pkl", "wb") as f:
            pickle.dump(d, f)
    else:
        model_name, bcktst_df, container = runModel(data, config, retrain, runGPU, runNN, runfreq, pre_dir)

        r2is, r2is_df = cal_model_r2(container, model_name, set_type="is")
        print(f"{model_name} IS R2: ", "{0:.3%}".format(r2is))

    r2oos, r2oos_df = cal_model_r2(container, model_name, set_type="oos")
    print(f"{model_name} R2: ", "{0:.3%}".format(r2oos))

    r2oos_df.merge(r2is_df, right_index=True, left_index=True).sort_index().plot()
    plt.show()
    # nr2oos = cal_model_r2(container, model_name, normal=True)
    # print(f"{model_name} Normal R2: ", "{0:.3%}".format(nr2oos))

    # nr2is = cal_model_r2(container, model_name, oos=False, normal=True)
    # print(f"{model_name} ISN R2: ", "{0:.3%}".format(nr2is))

    save_res(model_name, pre_dir, r2is, r2oos, nr2is=0, nr2oos=0)

    pt = Path('code') / pre_dir/ model_name
    if not os.path.exists(pt):
        os.makedirs(pt)
    with open(pt / f"predictions.pkl", "wb+") as f:
        pickle.dump(container[model_name], f)

    bcktst_df.dropna().to_csv(pt / "predictions.csv")
    config[config_key] = 0

#%%
# combine = np.concatenate([c.reshape(-1,1),d.reshape(-1,1)], axis=1)
# slice = combine[combine[:,0].argsort()][:-100]
# np.corrcoef(slice[:,0], slice[:,1])
#%%
# c = np.array(nn_valid_r2)
# c.sort()
# c[-len(nn_valid_r2)//10]

# %%


from __future__ import unicode_literals

import pytz
from django.db.models import Sum
from django.utils import timezone

from money.models import Expense, Invoice, Refund, Contract, Certificate, Compliance, DocumentStatus
from pybilling.settings import logger, MONEY_DEFAULT_CURRENCY

class DefaultExpensesDistributor(object):
    """
    Analyze and automatically cover expenses by the invoices.
    """

    @staticmethod
    def distribute(contract, only_payed_invoices=True):
        """
        Optimally cover expenses by invoices. Expenses can be split into smaller parts to
        fill all the invoices.
        :param invoices:
        :param only_payed_invoices: Use only payed invoices.
        """
        assert contract
        assert isinstance(contract, Contract)

        invoices = Invoice.objects.filter(contract=contract,
                                          payed=only_payed_invoices).order_by('date', '-amount')

        for invoice in invoices:
            if only_payed_invoices and not invoice.is_payed:
                continue

            uncovered_expences = Expense.objects.filter(service__contract=contract).order_by('date', '-amount')

            if invoice.unused_amount <= 0:
                continue

            logger.debug("inv %s from %s by %s %s (%s)" % (
                invoice.id, invoice.date, invoice.amount, MONEY_DEFAULT_CURRENCY, invoice.payed))

            for expense in uncovered_expences:
                if expense.get_uncovered_amount() <= 0:
                    continue

                invoice_unused_amount = invoice.unused_amount
                if invoice.is_completed:
                    break

                if invoice_unused_amount < expense.amount:
                    logger.warning("Split expense %s into %s and %s" % (
                        expense.id, invoice_unused_amount, expense.amount - invoice_unused_amount))

                    splitted_expenses = expense.split(invoice_unused_amount, expense.amount - invoice_unused_amount)
                    expense = splitted_expenses[0]

                invoice.cover(expense)

                logger.debug("\texp %s from %s by %s %s service %s (cert %s)" % (
                    expense.id, expense.date, expense.amount, MONEY_DEFAULT_CURRENCY, expense.service.id,
                    expense.certificate))

            if not invoice.is_completed:
                logger.warning("Invoice not totally covered by. %s %s left." % (
                    invoice.unused_amount, MONEY_DEFAULT_CURRENCY))

class BalanceUpdater(object):
    @staticmethod
    def update(account):
        """
        Update user account balance.
        :param account:
        :return: New balance.
        """
        assert account

        total_payed_incomes = Invoice.objects.filter(contract__personal_data__account=account,
                                                     payed=True).aggregate(Sum('amount'))
        total_expenses = Expense.objects.filter(service__contract__personal_data__account=account).aggregate(
            Sum('amount'))
        total_refunds = Refund.objects.filter(invoice__contract__personal_data__account=account).aggregate(
            Sum('amount'))

        total_income = 0
        total_expense = 0
        total_refund = 0

        if total_payed_incomes['amount__sum']:
            total_income = total_payed_incomes['amount__sum']

        if total_expenses['amount__sum']:
            total_expense = total_expenses['amount__sum']

        if total_refunds['amount__sum']:
            total_refund = total_refunds['amount__sum']

        account.balance = total_income - total_expense - total_refund
        account.save()

        return account.balance

class ContractManager(object):
    """
    Retrieve account billing info.
    """

    def get(self, id):
        assert id > 0
        return Contract.objects.get(pk=id)

    def get_default(self, account):
        assert account

        contracts = Contract.objects.filter(personal_data__account=account).order_by('-default')
        if len(contracts) > 0:
            return contracts[0]

        return None

    def get_contracts(self, account=None, signed=None, **query):
        if account:
            query['personal_data__account'] = account

        if signed:
            query['status'] = DocumentStatus.SIGNED

        return Contract.objects.filter(**query)

    def find_by_account(self, account, **query):
        assert account

        return self.get_contracts(account=account)

    def create(self, name, personal_data, type=None, date=None):
        """
        Add new contract to the account.
        :param personal_data: Requisites, used to sign contract.
        :param date: Date of the contract.
        :param name: Name of the contract.
        :param type: Type key of the contract. If None, then 'oferta' is used.
        :return: New created contract.
        """
        assert name
        assert personal_data

        if not date:
            date = timezone.now()

        fields = dict(
            personal_data=personal_data,
            name=name,
            date=date,
        )

        if not Contract.objects.filter(personal_data=personal_data).exists():
            fields['default'] = True

        if not Contract.objects.filter(pk=personal_data.account.id).exists():
            fields['id'] = personal_data.account.id  # trying to create contract with the user account ID

        if type:
            fields['type'] = type

        return Contract.objects.create(**fields)

class RefundManager(object):
    """
    Retrieve account billing info.
    """

    def get(self, id):
        assert id > 0
        return Refund.objects.get(pk=id)

    def find_by_contract(self, contract, **query):
        assert contract

        return Refund.objects.filter(invoice__contract=contract, **query)

    def create(self, invoice, amount, date=None, note=''):
        """
        Return money from balance to the client bank. Amount must be less or equal to Invoice amount
        and not greater than account balance.
        :param invoice: Invoice to refund.
        :param amount: Amount to refund.
        :param date: Date of operation.
        :param note: Notes.
        :return: Refund record.
        """
        assert invoice
        assert amount > 0
        assert invoice.amount >= amount

        if date is None:
            date = timezone.now()

        return Refund.objects.create(invoice=invoice, date=date, amount=amount, note=note)

class ExpenseManager(object):
    """
    Manage expenses.
    """

    def get(self, id):
        assert id > 0
        return Expense.objects.get(pk=id)

    def find_by_contract(self, contract, **query):
        """
        Get all expenses by the contract.
        :param contract:
        :param query:
        :return:
        """
        assert contract
        assert isinstance(contract, Contract)

        return Expense.objects.filter(service__contract=contract, **query)

    def create(self, service, date=None, period_start=None, period_end=None, count=1.0):
        """
        Add expense that is not covered by any incomes.
        :param service: Service, which amount is blocked.
        :param amount:
        :param date:
        :return:
        """
        assert service

        if date is None:
            date = timezone.now()

        return Expense.objects.create(service=service,
                                      amount=service.period_payment,
                                      date=date,
                                      period_start=period_start,
                                      period_end=period_end,
                                      count=count)

class InvoiceManager(object):
    def build_invoice(self, contract):
        """
        Create invoice builder to create new Invoices.
        :param contract:
        :return:
        """
        assert contract

        return InvoiceBuilder(contract)

    def get(self, id):
        assert id > 0
        return Invoice.objects.get(pk=id)

    def find_not_payed(self, **query):
        """
        Returns not payed invoices.
        :param query:
        :return:
        """
        return Invoice.objects.filter(payed=False, **query)

    def find_by_contract(self, contract, **query):
        """
        Return Invoices filtered by contract if specified.
        :param contract:
        :return:
        """
        assert contract
        assert isinstance(contract, Contract)

        return Invoice.objects.filter(contract=contract, **query)

class CertificateManager(object):
    def get(self, id):
        assert id > 0

        return Certificate.objects.get(pk=id)

    def find_by_contract(self, contract):
        assert contract
        assert isinstance(contract, Contract)

        return Certificate.objects.filter(contract=contract)

    def create(self, contract, year, month):
        """
        Create certificate and refresh aligned expenses list by month. Creation date is in the current timezone.
        :param month: Period year.
        :param year: Period month.
        :return:
        """
        assert contract
        assert isinstance(contract, Contract)
        assert year > 0
        assert month > 0

        doc_date = timezone.localtime(timezone.now()).replace(year=year, month=month, hour=0, minute=0, second=0)
        doc_date = doc_date.astimezone(pytz.utc)

        # this date will be aligned as needed by the model
        certificate = Certificate.objects.create(
            contract=contract,
            date=doc_date
        )

        return certificate

class ComplianceManager(object):
    def get(self, id):
        assert id > 0

        return Compliance.objects.get(pk=id)

    def find_by_contract(self, contract):
        assert contract
        assert isinstance(contract, Contract)

        return Compliance.objects.filter(contract=contract)

    def create(self, contract, year):
        """
        Create compliance. Creation date is in the current timezone.
        :param year: Period.
        :return:
        """
        assert contract
        assert isinstance(contract, Contract)
        assert year > 1900

        doc_date = timezone.localtime(timezone.now()).replace(year=year, month=12, hour=0, minute=0, second=0)
        doc_date = doc_date.astimezone(pytz.utc)

        compliance, created = Compliance.objects.update_or_create(contract=contract,
                                                                  date=doc_date)

        return compliance

class InvoiceBuilder(object):
    """
    Build invoice to pay for specific selected services.
    """

    def __init__(self, contract):
        assert contract

        self.contract = contract
        self.services = []

    def add_service(self, service):
        """
        Add service to the Invoice to pay for.
        :param service: Specific service.
        """
        assert service

        self.services.append(service)

    def get_invoice(self, date=None, amount=0):
        """
        Get Invoice with or without covered expenses by the added services.
        :param amount: If amount is greater than 0, prepayment Invoice is created, else - invoice for specific services.
        :param date: Invoice date.
        :return:
        """
        if not date:
            date = timezone.now()

        fm = ExpenseManager()

        if amount <= 0:
            assert len(self.services) > 0, "There must be services added."

            total_amount = 0
            expenses = []
            for service in self.services:
                expenses.append(fm.create(service, date))
                total_amount += service.period_payment

            invoice = Invoice.objects.create(contract=self.contract, amount=total_amount, date=date)
            for expense in expenses:
                invoice.cover(expense)
        else:
            invoice = Invoice.objects.create(contract=self.contract, amount=amount, date=date)

        return invoice

class Accountant(object):
    """
    Accountant object, which controls contracts, invoices, balance updates and other aspects of finances.
    """

    """ This class used to cover expenses by the invoices. """
    EXPENSES_DISTRIBUTOR = DefaultExpensesDistributor

    def manage_invoices(self):
        """
        Returns InvoiceManager to manage invoices of the account.
        :return: InvoiceManager instance.
        """
        return InvoiceManager()

    def manage_expenses(self):
        """
        Manage payments
        :return:
        """
        return ExpenseManager()

    def manage_refunds(self):
        """
        Manage payments
        :return:
        """
        return RefundManager()

    def manage_contracts(self):
        """
        Manage contracts of the account.
        :return:
        """
        return ContractManager()

    def manage_certificates(self):
        """
        Manage contracts of the account.
        :return:
        """
        return CertificateManager()

    def manage_compliances(self):
        """
        Manage compliance reports of the account.
        :return:
        """
        return ComplianceManager()

    def sign_certificate(self, certificate):
        """
        Sing certificate.
        :param certificate:
        :return:
        """
        assert certificate
        assert isinstance(certificate, Certificate)

        certificate.signed = True
        certificate.save()

    def sign_contract(self, contract):
        """
        Sing certificate.
        :param certificate:
        :return:
        """
        assert contract
        assert isinstance(contract, Contract)

        contract.signed = True
        contract.save()

    def distribute_expenses(self, contract, only_payed_invoices=True):
        """
        Optimally distribute expenses along invoices, to maximize amount of covered expenses. This method is used
        as a service function to detalize prepayment invoices.
        :param contract:
        :param only_payed_invoices: Analyze only payed invoices.
        :return:
        """
        assert contract
        assert isinstance(contract, Contract)

        self.EXPENSES_DISTRIBUTOR.distribute(contract, only_payed_invoices)

    def update_balance(self, account):
        """
        Update user account balance.
        :return: New balance.
        """
        return BalanceUpdater.update(account)


from framework.targeting import ForwardTarget, HeadingTarget, PIDLoop
from framework.helpers import within_deadband, get_camera_size
from framework.task import Task
from framework.movement import VelocityX, VelocityY, RelativeToCurrentHeading, PositionN, PositionE, Depth, RelativeToInitialDepth, RelativeToInitialHeading, Pitch, Heading
from framework.primitive import Zero
from framework.combinators import Sequential, MasterConcurrent, Concurrent
from framework.timing import Timer, Timed
from framework.position import MoveX, MoveY, GoToPosition
from mission.opt_aux.aux import Mode
from framework.actuators import FireActuator, FireActuators

import shm
from vision.modules.torpedoes import CutoutSize, CutoutName, TorpedoesMode
import aslam

from enum import Enum
from collections import namedtuple
from time import sleep
import numpy as np

direction = ["N", "E"]
USE_ACTUATORS = True
COVER_INITALLY_PRESENT = True
known_letters = [
    None, None,
    None, None
]

board_coords = (shm.torpedoes_results.board_center_x.get, shm.torpedoes_results.board_center_y.get)
heading_target_board = HeadingTarget()

class Torpedoes(Task):
    pass

'''
Moves the sub to the position directly in front of the torpedoes board.
Must have successfully transitioned out of the align_and_characterize state for
this task to work.
'''
class GoToBoardFront(Task):
    def on_first_run(self):
        self.logi("Started GoToBoardFront")
        self.task = GoToPosition(Torpedoes.board_front_north, Torpedoes.board_front_east,
                                 heading=Torpedoes.board_front_heading,
                                 depth=Torpedoes.board_front_depth)
        shm.torpedoes_settings.mode.set(TorpedoesMode.board_subsequent.value)

    def on_run(self):
        if self.task.finished:
            self.logi("Finished GoToBoardFront")
            self.finish()
        else:
            self.task()

class LocateAndApproach(Task):
    def on_first_run(self):
        self.task = Sequential(LocateBoard(), ApproachBoard())

    def on_run(self):
        if Torpedoes.failed or self.task.finished:
            self.finish()
        else:
            self.task()

'''
Rotates the sub until it sees the torpedoes board
'''
BOARD_DEPTH = 1.14
class LocateBoard(Task):
    def on_first_run(self):
        self.inital_heading = shm.kalman.heading.get()
        self.num_headings = 8
        self.heading_inc = 360 / self.num_headings
        shm.torpedoes_settings.mode.set(TorpedoesMode.board_inital.value)

        headings = []
        for i in range(0, self.num_headings):
            headings.append(Heading(self.inital_heading + (self.heading_inc * i) % 360))
            headings.append(Timer(1))
        spin = Sequential(*headings)

        self.spin_search = Sequential(Depth(BOARD_DEPTH), spin)
        self.logi("Started LocateBoard")
        if not USE_ACTUATORS:
            self.logi("Actuators are disabled!!")

        # aslam_target = aslam.SimpleTarget(obj = aslam.world.torpedoes, relativePosition = np.array([-1, 0, 0]))
        #self.goto_board = aslam_target

    def on_run(self):
        # if not self.goto_board.finished:
        #    self.goto_board()

        if (shm.torpedoes_results.board_prob.get() > 0.7):
            if heading_target_board.finished:
                self.finish()
                self.logi("Finished LocateBoard")
            else:
                heading_target_board.run(board_coords, px=0.02, py=0.001, target=Torpedoes.camera_center, deadband=(60, 60))
        elif self.spin_search.finished:
            Torpedoes.failed = True
            self.finish()
        else:
            self.spin_search()

'''
Moves forward (or backwards) until the board is a certain distance from the sub
Perhaps not necessary because the bins are so close to torpedoes
'''
class ApproachBoard(Task):
    def on_first_run(self):
        self.logi("Started ApproachBoard")

        self.target_board = ForwardTarget()
        self.zero_vel_x = VelocityX(0.0)
        self.zero_vel_y = VelocityY(0.0)

        self.pid_loop_x = PIDLoop(output_function=VelocityX())
        shm.torpedoes_settings.mode.set(TorpedoesMode.board_inital.value)

    def on_run(self):
        if self.pid_loop_x.finished:
            # Zero speed so the sub doesn't keep moving after finish
            self.zero_vel_x()
            self.zero_vel_y()

            self.logi("Finished ApproachBoard")
            self.finish()
        else:
            self.target_board(board_coords, target=Torpedoes.camera_center)
            self.pid_loop_x(input_value=shm.torpedoes_results.board_height.get, p=.001, i=0, d=0, target=Torpedoes.camera_height*0.7, deadband=100)

'''
Aligns the sub to face normal to the board
'''
class AlignToBoard(Task):
    def on_first_run(self):
        self.logi("Started AlignToBoard")
        self.pid_loop_y = PIDLoop(output_function=VelocityY())
        self.zero_vel_y = VelocityY(0.0)
        shm.torpedoes_settings.mode.set(TorpedoesMode.board_inital.value)
        self.pid_loop_x = PIDLoop(output_function=VelocityX())

    def on_run(self):
        results = shm.torpedoes_results.get()

        # By keeping the board centered in the camera (via heading adjusment)
        # and swaying until skew is zero, we will align normal to the board
        heading_target_board.run(board_coords, target=Torpedoes.camera_center, px=0.015)
        self.pid_loop_y(input_value=results.board_skew, p=0.01, i=0, target=0, deadband=1.0)
        self.pid_loop_x(input_value=shm.torpedoes_results.board_height.get, p=.001, i=0, d=0, target=Torpedoes.camera_height*0.7, deadband=10)

        if self.pid_loop_y.finished and results.skew_valid:
            self.zero_vel_y()
            self.logi("Finished AlignToBoard, skew: {}, valid: {}".format(results.board_skew, results.skew_valid))
            self.finish()

class RemoveCover(Task):
    def on_first_run(self):
        self.task = Sequential(TargetCutout(), SlideCover())
        Torpedoes.target_cutout_failed = False

    def on_run(self):
        if self.task.finished:
            self.finish()
        else:
            self.task()

        if Torpedoes.target_cutout_failed:
            self.finish()

class ShootCutout(Task):
    def on_first_run(self):
        self.task = Sequential(TargetCutout(), FireTorpedo())
        Torpedoes.target_cutout_failed = False

    def on_run(self):
        if self.task.finished:
            self.finish()
        else:
            self.task()

        if Torpedoes.target_cutout_failed:
            self.logi("Wrong letter, exiting ShootCutout!")
            self.finish()
'''
Attempts to remove the cover from the covered cutout
'''
class SlideCover(Task):
    def on_first_run(self):
        if (Torpedoes.target.name == CutoutName.top_left or
            Torpedoes.target.name == CutoutName.bottom_left):
            align_for_swipe = Concurrent(
                Timed(RelativeToInitialDepth(-0.10330, error=0.005), 5.0),
                Sequential(
                    Timed(MoveY(0.37, deadband=0.03), 5.0),
                    Timed(MoveX(0.26), 5.0)
                )
            )
            self.remove_cover = Sequential(
                align_for_swipe,
                Timed(MoveY(-0.57, deadband=0.05), 5.0),
                Timed(MoveX(-0.2, deadband=0.05), 5.0),
                Timed(Concurrent(
                    RelativeToInitialDepth(0.1),
                    RelativeToInitialHeading(20, error=5)
                ), 5.0),
                Timed(MoveX(-0.2, deadband=0.05), 5.05)
            )
        else:
            align_for_swipe = Concurrent(
                Timed(RelativeToInitialDepth(-0.10330, error=0.005), 5.0),
                Sequential(
                    Timed(MoveY(-0.1, deadband=0.03), 5.0),
                    Timed(MoveX(0.26), 5.0)
                )
            )
            self.remove_cover = Sequential(
                align_for_swipe,
                Timed(MoveY(0.57, deadband=0.05), 5.0),
                Timed(MoveX(0.2, deadband=0.05), 5.0),
                Timed(Concurrent(
                    RelativeToInitialDepth(0.1),
                    RelativeToInitialHeading(-20, error=5)
                ), 5.0),
                Timed(MoveX(-0.2, deadband=0.05), 5.0)
            )

        self.logi("Started RemoveCover")
        # north = shm.kalman.north.get()
        # east = shm.kalman.east.get()
        # self.logi("north: {}, east: {}".format(north, east))
        depth = shm.kalman.depth.get()
        self.logi("depth: {}".format(depth))

    def on_run(self):
        if self.remove_cover.finished:
            self.logi("Finished RemoveCover")
            self.finish()
        else:
            # pass
            self.remove_cover()

'''
Select the covered cutout as the target
'''
class SelectCoveredCutout(Task):
    def on_run(self):
        global target
        # For now, just select the top left cutout (All cutouts have covers currently!)
        target = shm.torpedoes_cutout_top_left
        self.finish()

'''
Targets the selected cutout and shoots a torpedo thought it
'''
class TargetCutout(Task):
    def on_first_run(self):
        self.forward_target = ForwardTarget()
        self.zero_vel_y = VelocityY(0.0)

        def vel_x_bound(vel):
            VelocityX(min(vel, 0.3))()

        self.pid_loop_x = PIDLoop(output_function=vel_x_bound)
        self.zero_vel_x = VelocityX(0.0)

        self.targeted_once = False
        self.logi("Started TargetCutout")
        self.lost_target = False
        self.prev_lost_target = False
        self.lost_target_timer = None
        Torpedoes.target_cutout_failed = False

    def on_run(self):
        rel_target_height = Torpedoes.target.results.height / Torpedoes.camera_height
        target_letter = Torpedoes.target.results.letter.decode('utf-8')
        if rel_target_height > 0.23 and rel_target_height < 0.4:
            Torpedoes.target.known_letter = target_letter
            if Torpedoes.target.inital_letter != target_letter:
                self.logi("Wrong letter detected! Inital letter was %s but letter is %s now" % (Torpedoes.target.inital_letter, target_letter))

                if Torpedoes.target.results.covered:
                    self.logi("Continuing with cover removal...")
                else:
                    self.logi("Backing up to try another cutout!")
                    Torpedoes.target_cutout_failed = True
                    Torpedoes.target.attempted = False

                    self.zero_vel_y()
                    self.zero_vel_x()

                    self.finish()
                    return

        if self.forward_target.finished:
            self.targeted_once = True

        if self.pid_loop_x.finished and self.forward_target.finished:
            self.zero_vel_y()
            self.zero_vel_x()

            self.logi("Finished TargetCutout")
            self.finish()
        elif Torpedoes.target.results.visible:
            if self.lost_target:
                self.logi("Found Target!!")
                self.lost_target = False

            # actual_height = 0.1778
            # size = target.size.get()
            # if size == int(CutoutSize.small):
            #     actual_height = 0.1778
            # elif size == int(CutoutSize.large):
            #     actual_height = 0.3048
            # else:
            #     self.loge("Invalid size {}!!".format(size))

            # px_to_m = (actual_height / target.height.get())
            # coords = (Torpedoes.target.x.get() * px_to_m, Torpedoes.target.y.get() * px_to_m)
            coords = (Torpedoes.target.results.x, Torpedoes.target.results.y)

            if self.targeted_once:
                self.pid_loop_x(input_value=Torpedoes.target.results.width, p=.0005, i=0, d=0.0, target=Torpedoes.camera_height*0.55, deadband=10)
            # self.forward_target(coords, target=Torpedoes.camera_center, py=5, px=10, deadband=(0.05,0.05))
            self.forward_target(coords, target=Torpedoes.camera_center, py=0.0005, px=0.001, deadband=(80,80))
        else:
            if not self.lost_target:
                self.logi("Lost Target!!")
                self.lost_target = True
                self.lost_target_timer = Timer(2.0)

            self.zero_vel_y()
            VelocityX(-0.1)()
            self.lost_target_timer()

            if self.lost_target_timer.finished:
                Torpedoes.target_cutout_failed = True
                self.finish()

'''
Moves from targeting one cutout to another
'''
class TransitionToCutout(Task):
    def on_run(self):
        # Not implemented yet!
        pass

'''
Requests boad characterization from vision and continues once it has been done
'''
class CharacterizeBoard(Task):
    def on_first_run(self, target):
        self.logi("Started CharacterizeBoard, waiting for vision to characterize...")
        shm.torpedoes_settings.target.set(target.name.value)
        shm.torpedoes_settings.mode.set(1)

    def on_run(self):
        if shm.torpedoes_settings.mode.get() == 2:
            self.logi("Finished CharacterizeBoard")
            self.finish()
'''
Fires a torpedo through the cutout!
'''
class FireTorpedo(Task):
    def on_first_run(self):
        self.forward_target = ForwardTarget()
        self.pid_loop_x = PIDLoop(output_function=VelocityX())

        self.torpedo_top_actuators = ["torpedo_top_1", "torpedo_top_2"]
        if USE_ACTUATORS:
            if Torpedoes.num_torpedoes == 1:
                self.task = Sequential(RelativeToInitialDepth(0.03, error=0.01), FireActuator("torpedo_bottom", 0.1))
            else:
                self.task = Sequential(RelativeToInitialDepth(0.05, error=0.01), FireActuators(self.torpedo_top_actuators, 0.1))
        else:
            self.logi("Actuators disabled, set USE_ACTUATORS to True to enable")
            self.task = Zero()

        Torpedoes.num_torpedoes -= 1
        Torpedoes.current_letter = None

        self.logi("Started FireTorpedo")

    def on_run(self):
        if self.task.finished:
            self.finish()
        else:
            self.task()

class Cutout:
    def __init__(self, *, group, attempted, name):
        self.group = group
        self.attempted = attempted
        self.name = name
        self.inital_letter = None
        self.known_letter = None

    def __str__(self):
        return "Group: " + str(self.group) + ", Attempted: " + str(self.attempted) + ", Covered: " + str(self.results.covered) + ", Size: " + str(CutoutSize(self.results.size)) + ", Letter: " + self.results.letter.decode('utf-8')

    def update(self):
        self.results = self.group.get()

modes = [
    Mode(name='Torpedoes Full', expectedPoints=3500, expectedTime=5)
]

class Torpedoes(Task):
    def __init__(self, cover_present="cover", *args, **kwargs):
        super(Torpedoes, self).__init__(*args, **kwargs)
        global direction
        global known_letters

        # Set the direction variable in shm for vision
        dir_str = "".join(direction)
        shm.torpedoes_settings.direction.set(dir_str)

        # Set the known_letters for each cutout from the global var
        known_letters_str = ""
        for idx, letter in enumerate(known_letters):
            self.cutouts[idx].known_letter = letter

        shm.torpedoes_settings.cover_initally_present.set(COVER_INITALLY_PRESENT)

    def possibleModes(self):
        return modes

    def desiredModules(self):
        return [shm.vision_modules.Torpedoes]

    class State(Enum):
        locate_and_approach_board = 1
        align_and_characterize = 2
        remove_cover = 3
        target_uncovered_cutout = 4
        back_up = 5
        goto_board_front = 6
        end = 7

    board_front_north = 0
    board_front_east = 0
    board_front_heading = 0
    board_front_depth = 0
    state = State.locate_and_approach_board
    task = LocateAndApproach()
    current_letter = None
    camera_width = 0
    camera_height = 0
    camera_center = (0,0)

    num_tries = {
        'remove_cover': 0,
        'top_left_target': 0,
        'top_right_target': 0,
        'bottom_left_target': 0,
        'bottom_right_target': 0
    }

    # The order of these is important! It is depended on implictly by the known_letters
    # variable (probably a bad idea)
    cutouts = [Cutout(group=shm.torpedoes_cutout_top_left, attempted=False,
                      name=CutoutName.top_left),
              Cutout(group=shm.torpedoes_cutout_top_right, attempted=False,
                     name=CutoutName.top_right),
              Cutout(group=shm.torpedoes_cutout_bottom_left, attempted=False,
                     name=CutoutName.bottom_left),
              Cutout(group=shm.torpedoes_cutout_bottom_right, attempted=False,
                     name=CutoutName.bottom_right)]

    succeeded = False
    failed = False
    target = None
    lost_target = False

    def on_first_run(self, mode=modes[0]):
        Torpedoes.num_torpedoes = 2
        Torpedoes.lost_target = False
        Torpedoes.current_letter = None
        Torpedoes.target_cutout_failed = False

    def on_run(self, mode=modes[0]):
        camera_dims = get_camera_size(shm.torpedoes_results.get())
        Torpedoes.camera_width = camera_dims[0]
        Torpedoes.camera_height = camera_dims[1]
        Torpedoes.camera_center = (Torpedoes.camera_width/2, Torpedoes.camera_height/2)

        known_letters_str = ""
        for cutout in self.cutouts:
            # Populate the results variable of the cutout
            cutout.update()

            # Update the known_letters variable in shm
            letter = cutout.known_letter
            if letter == "N" or letter == "E" or letter == "S" or letter == "W":
                known_letters_str += letter
            elif letter == None:
                known_letters_str += "?"
            else:
                self.loge("Unknown letter found in known_letters!")
        shm.torpedoes_settings.known_letters.set(known_letters_str)

        # State transition table
        if self.task.finished:
            self.logi("Transitioning State!")
            if self.state == self.State.locate_and_approach_board:
                if (Torpedoes.failed):
                    self.logi("Couldn't find torpedoes board!")
                    self.finish()
                else:
                    self.logi("Finsihed Locate and Approach!")
                    self.state = self.State.align_and_characterize
                    self.task = AlignToBoard()
            elif self.state == self.State.align_and_characterize or self.state == self.State.goto_board_front:
                kalman = shm.kalman.get()
                Torpedoes.board_front_north = kalman.north
                Torpedoes.board_front_east = kalman.east
                Torpedoes.board_front_heading = kalman.heading
                Torpedoes.board_front_depth = kalman.depth

                covered_cutout = self.get_covered_cutout()
                if covered_cutout == None or self.num_tries['remove_cover'] >= 2:
                    self.state = self.State.target_uncovered_cutout

                    if Torpedoes.lost_target:
                        Torpedoes.lost_target = False
                    else:
                        Torpedoes.target = self.get_uncovered_cutout()

                    # Can't find a target! Exit (TODO: Re-align and try again?)
                    if self.target == None:
                        self.logi("Can't find a target!")
                        self.finish()
                    else:
                        self.task = ShootCutout()

                else:
                    Torpedoes.target = covered_cutout
                    self.state = self.State.remove_cover
                    self.task = RemoveCover()

            elif self.state == self.State.remove_cover:
                if not Torpedoes.target_cutout_failed:
                    self.num_tries['remove_cover'] += 1

                self.state = self.State.goto_board_front
                self.task = GoToBoardFront()

            elif self.state == self.State.target_uncovered_cutout:
                if self.num_torpedoes > 0:
                    self.state = self.State.goto_board_front
                    self.task = GoToBoardFront()
                else:
                    self.state = self.State.end
                    self.task = Sequential(GoToBoardFront(), Zero())
            elif self.state == self.State.back_up:
                if Torpedoes.target.results.visible:
                    self.state = self.State.target_uncovered_cutout
                    self.task = ShootCutout()
                else:
                    self.state = self.State.goto_board_front
                    self.task = ReturnToBoardFront()
            elif self.state == self.State.goto_board_front:
                self.state = self.State.align_and_characterize
                self.task = AlignToBoard()
            elif self.state == self.State.end:
                self.finish()

        self.task()

    def get_covered_cutout(self):
        for cutout in self.cutouts:
            if cutout.results.covered:
                shm.torpedoes_settings.target.set(cutout.name.value)
                shm.torpedoes_settings.mode.set(TorpedoesMode.track_target.value)

                cutout.inital_letter = cutout.results.letter.decode('utf-8')
                self.logi("Letter from results is %s" % cutout.results.letter.decode('utf-8'))
                self.logi("Inital letter for %s cutout is %s" % (cutout.name, cutout.inital_letter))
                return cutout
        return None

    def list_to_str(self, list):
        string = ""
        for e in list:
           string += str(e) + " "

        return string

    '''
    Decides which cutout should be targeted, the preference order is:
        1. Uncovered small cutout, with specified letter
        2. Big cutout, with specified letter
        3. Big cutout
        3. Small cutout
    '''

    def get_uncovered_cutout(self):
        global direction

        target = None

        self.logi("Direction is currently %s" % direction)

        if Torpedoes.current_letter == None:
            Torpedoes.current_letter = direction.pop(0)

        cutouts = [c for c in self.cutouts if not c.attempted and not c.results.covered and c.results.visible]

        small_cutouts = [c for c in cutouts if c.results.size == int(CutoutSize.small)]
        large_cutouts = [c for c in cutouts if c.results.size == int(CutoutSize.large)]

        correct_letter_small = [c for c in small_cutouts if c.results.letter.decode('utf-8') == self.current_letter]
        correct_letter_large = [c for c in large_cutouts if c.results.letter.decode('utf-8') == self.current_letter]

        self.logi("All Cutouts: %s" % self.list_to_str(self.cutouts))
        self.logi("Considered Cutouts: %s" % self.list_to_str(cutouts))
        self.logi("Small Cutouts: %s" % self.list_to_str(small_cutouts))
        self.logi("Large Cutouts: %s" % self.list_to_str(large_cutouts))
        self.logi("Correct Letter Small: %s" % self.list_to_str(correct_letter_small))
        self.logi("Correct Letter Large: %s" % self.list_to_str(correct_letter_large))

        if len(correct_letter_small) > 0:
            target = correct_letter_small[0]
            self.logi("Selected small cutout with correct letter: %s" % target)
        elif len(correct_letter_large) > 0:
            target = correct_letter_large[0]
            self.logi("Selected large cutout with correct letter: %s" % target)
        elif len(small_cutouts) > 0:
            target = small_cutouts[0]
            self.logi("Selected small cutout with unknown letter: %s" % target)
        elif len(large_cutouts) > 0:
            target = large_cutouts[0]
            self.logi("Selected large cutout with unknown letter: %s" % target)
        else:
            self.loge("Can't find a suitable target! Panic!!")
            return None

        target.attempted = True
        shm.torpedoes_settings.target.set(target.name.value)
        shm.torpedoes_settings.mode.set(TorpedoesMode.track_target.value)
        target.inital_letter = target.results.letter.decode('utf-8')
        self.logi("Initial letter for %s cutout is %s" % (target.name, target.inital_letter))

        return target

full = Torpedoes()


# -*- coding: utf-8 -*-

from odoo import api, fields, models, _
from odoo.exceptions import Warning
import time
import logging
import json

from odoo.addons.wecom_api.api.wecom_abstract_api import ApiException

_logger = logging.getLogger(__name__)

class SyncDepartment(models.AbstractModel):
    _name = "wecom.sync_task_department"
    _description = "Wecom Synchronize department tasks"

    def run(self, company):
        """[summary]
        运行同步
        """
        params = self.env["ir.config_parameter"].sudo()
        debug = params.get_param("wecom.debug_enabled")
        if debug:
            _logger.info(
                _("Start synchronizing departments of %s"), company.name,
            )

        result = ""
        times = 0
        try:
            wxapi = self.env["wecom.service_api"].InitServiceApi(
                company, "contacts_secret", "contacts"
            )
            response = wxapi.httpCall(
                self.env["wecom.service_api_list"].get_server_api_call(
                    "DEPARTMENT_LIST"
                ),
                {"id": str(company.contacts_sync_hr_department_id),},
            )
            # response 为 dict
            # response["department"] 为 list

            # 清洗数据
            departments = self.department_data_cleaning(response["department"])

            start1 = time.time()
            for obj in departments:
                self.run_sync_department(company, obj)
            end1 = time.time()
            times1 = end1 - start1

            start2 = time.time()
            self.run_set_department(company)
            end2 = time.time()
            times2 = end2 - start2

            times = times1 + times2
            result = (
                _("Successfully synchronized '%s''s WeCom Department") % company.name
            )
        except ApiException as e:
            times = time.time()
            result = (
                _("Failed to synchronized '%s''s WeCom Department") % company.name
            ) % (company.name, e.errMsg,)
            if debug:
                _logger.warning(
                    _(
                        "Department error synchronizing %s, error: %s",
                        company.name,
                        e.errMsg,
                    )
                )
        times = times
        if debug:
            _logger.info(
                _("End sync %s Department,Total time spent: %s seconds")
                % (company.name, times)
            )

        return times, result

    def department_data_cleaning(self, departments):
        """[summary]
        部门数据清洗
        Args:
            json ([type]): [description]index
        """

        for department in departments:
            if department.get("parentid") == 1:
                department["parentid"] = 0

        for index, department in enumerate(departments):
            if department.get("id") == 1:
                del departments[index]

        return departments

    def run_sync_department(self, company, wecom_department):
        """[summary]
        执行同步部门
        Args:
            wecom_department ([type]): [description]
        """

        # 查询数据库是否存在相同的企业微信部门ID，有则更新，无则新建

        department = (
            self.env["hr.department"]
            .sudo()
            .search(
                [
                    ("wecom_department_id", "=", wecom_department["id"]),
                    ("is_wecom_department", "=", True),
                    ("company_id", "=", company.id),
                    "|",
                    ("active", "=", True),
                    ("active", "=", False),
                ],
                limit=1,
            )
        )

        if not department:
            self.create_department(company, department, wecom_department)
        else:
            self.update_department(company, department, wecom_department)

    def create_department(self, company, records, obj):
        """[summary]
        创建部门
        Args:
            records ([type]): [description]
            obj ([type]): [description]
        """
        try:
            records.create(
                {
                    "name": obj["name"],
                    "wecom_department_id": obj["id"],
                    "wecom_department_parent_id": obj["parentid"],
                    "wecom_department_order": obj["order"],
                    "is_wecom_department": True,
                    "company_id": company.id,
                }
            )
        except Exception as e:
            _logger.warning(
                "Error creating Department [%s], error details:%s"
                % (obj["name"], str(e))
            )

    def update_department(self, company, records, obj):
        """[summary]
        更新部门
        Args:
            records ([type]): [description]
            obj ([type]): [description]
        """
        try:
            records.write(
                {
                    "name": obj["name"],
                    "wecom_department_parent_id": obj["parentid"],
                    "wecom_department_order": obj["order"],
                    "is_wecom_department": True,
                    "company_id": company.id,
                }
            )
        except Exception as e:
            _logger.warning(
                "Error updating Department [%s], error details:%s"
                % (obj["name"], str(e))
            )

    def run_set_department(self, company):
        """[summary]
        由于json数据是无序的，故在同步到本地数据库后，需要设置新增企业微信部门的上级部门
        """
        params = self.env["ir.config_parameter"].sudo()
        debug = params.get_param("wecom.debug_enabled")

        departments = (
            self.env["hr.department"]
            .sudo()
            .search([("is_wecom_department", "=", True),])
        )

        for department in departments:
            if not department.wecom_department_id:
                pass
            else:
                try:
                    department.write(
                        {
                            "parent_id": self.get_parent_department(
                                department, departments
                            ).id,
                        }
                    )
                except Exception as e:
                    if debug:
                        _logger.warning(
                            _(
                                "Error setting parent department for company %s, Error details:%s"
                            )
                            % (company.name, repr(e))
                        )

    def get_parent_department(self, department, departments):
        """[summary]
        获取上级部门
        Args:
            department ([type]): [description]
            departments ([type]): [description]
        """
        parent_department = departments.search(
            [
                ("wecom_department_id", "=", department.wecom_department_parent_id),
                ("is_wecom_department", "=", True),
            ]
        )
        return parent_department


from django.contrib import admin
from django.urls import path, include
from django.conf import settings
from django.conf.urls.static import static

urlpatterns = [
    path('admin/', admin.site.urls),

    # Authentication APIs
    path('api-auth/', include('rest_framework.urls')),
    path('dj-rest-auth/', include('dj_rest_auth.urls')),
    path('dj-rest-auth/registration/', include('dj_rest_auth.registration.urls')),

    # path('', include('task_management_app.urls')),
    path('api/', include('masters.apis.urls')),
    path('api/', include('product_management.apis.urls')),
]

if settings.DEBUG:
    urlpatterns += static(settings.MEDIA_URL,
                          document_root=settings.MEDIA_ROOT)


# -*- coding: utf-8 -*-
# @Author: Niccolò Bonacchi
# @Date:   2018-02-02 17:19:09
# @Last Modified by:   Niccolò Bonacchi
# @Last Modified time: 2018-02-26 16:07:42
import os
import datetime
import numpy as np
import scipy.stats as st
from pythonosc import udp_client
import json
import glob

class sounds():
    """Software solution for playing sounds"""
    def configure_sounddevice(SD=None):
        if SD is None:
            import sounddevice as SD
        SD.default.latency = 'low'
        SD.default.channels = 2
        return SD

    def make_sound(frequency=10000, duration=0.1, amplitude=1, fade_in=0.01):
        """builds sound to feed to sounddevice lib
        if frequency is set to -1 will produce white noise"""
        FsOut = 44100  # sample rate, depend on the sound card
        toneDuration = duration  # sec
        amplitude = amplitude  # [0->1]
        frequency = frequency  # Hz
        onsetDur = fade_in  # sec

        tvec = np.linspace(0, toneDuration, toneDuration * FsOut)
        sound = amplitude * np.sin(2 * np.pi * frequency * tvec)  # sound vec
        size = FsOut * onsetDur / 2  #
        dist = st.expon(0., size)  # distribution object provided by scipy
        F = dist.cdf  # cumulative density function
        ker = F(range(int(FsOut * toneDuration)))
        sound = sound * ker
        if frequency == -1:
            sound = amplitude * np.random.rand(sound.size)
        return sound

    def play_sound(sound):
        pass

class rotary_encoder(object):  # TODO: SAVE THIS IN SETTINGS FILE ALSO

    def __init__(self, port, stim_positions, quiescence_thresholds, gain):
        self.port = port
        self.wheel_perim = 31 * 2 * np.pi  # = 194,778744523
        self.wheel_deg_mm = 360 / self.wheel_perim
        self.factor = self.wheel_deg_mm / gain
        self.all_thresholds = stim_positions + quiescence_thresholds
        self.SET_THRESHOLDS = [x * self.factor for x in self.all_thresholds]
        self.ENABLE_THRESHOLDS = [(True if x != 0
                                   else False) for x in self.SET_THRESHOLDS]
        # ENABLE_THRESHOLDS needs 8 bools even if only 2 thresholds are set
        while len(self.ENABLE_THRESHOLDS) < 8:
            self.ENABLE_THRESHOLDS.append(False)

    def configure(self, RotaryEncoderModule):
        m = RotaryEncoderModule(self.port)
        m.set_zero_position()  # Not necessarily needed
        m.set_thresholds(self.SET_THRESHOLDS)
        m.enable_thresholds(self.ENABLE_THRESHOLDS)
        m.close()

class session_param_handler(object):
    """Session object copies settings parameters and calculates other secondary
    session parameters, saves all params in a settings file.json"""

    def __init__(self, task_settings):
        tsp = {i: task_settings.__dict__[i]
               for i in [x for x in dir(task_settings) if '__' not in x]}
        self.__dict__.update(tsp)

        self.VALVE_TIME = self.CALIBRATION_VALUE * self.TARGET_REWARD

        self.GO_TONE = sounds.make_sound(frequency=self.GO_TONE_FREQUENCY,
                                         duration=self.GO_TONE_DURATION,
                                         amplitude=self.GO_TONE_AMPLITUDE)
        self.WHITE_NOISE = sounds.make_sound(
            frequency=-1,
            duration=self.WHITE_NOISE_DURATION,
            amplitude=self.WHITE_NOISE_AMPLITUDE)
        self.SD = sounds.configure_sounddevice()
        self.OSC_CLIENT = self.osc_client_init()

        self.ALL_THRESHOLDS = (self.STIM_POSITIONS +
                               self.QUIESCENCE_THRESHOLDS)
        # Names of the RE events generated by Bpod
        self.ENCODER_EVENTS = ['RotaryEncoder{}_{}'.format(
            self.ROTARY_ENCODER_SERIAL_PORT_NUM, x) for x in
            list(range(1, len(self.ALL_THRESHOLDS) + 1))]
        # Dict mapping threshold crossings with name ov RE event
        self.THRESHOLD_EVENTS = dict(zip(self.ALL_THRESHOLDS,
                                         self.ENCODER_EVENTS))

        self.SESSION_NAME = datetime.datetime.now().isoformat().replace(':',
                                                                        '_')
        self.TASK_NAME = '{}_{}'.format('pycw', self.TASK.strip('ChoiceWorld'))

        self.SESSION_ID = '{}_{}_{}'.format(self.MOUSE_NAME,
                                            self.SESSION_NAME,
                                            self.TASK_NAME)

        self.ROOT_DATA_FOLDER = path_helper.root_data_folder(
            self.ROOT_DATA_FOLDER)
        self.MOUSE_DATA_FOLDER = path_helper.mouse_data_folder(self)
        self.PREVIOUS_DATA_FILE = path_helper.previous_data_file(self)
        self.SESSION_DATA_FOLDER = path_helper.session_data_folder(self)
        self.SETTINGS_FILE_PATH = path_helper.settings_file_path(self)
        self.DATA_FILE_PATH = path_helper.data_file_path(self)

        self.STATE_AFTER_START = ('stim_on' if self.QUIESCENT_PERIOD == 0
                                  else 'quiescent_period')

        self.save_session_settings()

    def reprJSON(self):
        d = self.__dict__.copy()
        d['GO_TONE'] = 'sounds.make_sound(frequency={}, duration={}, \
amplitude={})'.format(self.GO_TONE_FREQUENCY, self.GO_TONE_DURATION,
                      self.GO_TONE_AMPLITUDE)
        d['WHITE_NOISE'] = 'sounds.make_sound(frequency=-1, duration={}, \
amplitude={})'.format(self.WHITE_NOISE_DURATION, self.WHITE_NOISE_AMPLITUDE)
        d['SD'] = str(d['SD'])
        d['OSC_CLIENT'] = str(d['OSC_CLIENT'])
        return d

    def save_session_settings(self):
        with open(self.SETTINGS_FILE_PATH, 'a') as f:
            f.write(json.dumps(self.reprJSON()))
            f.write('\n')
        return

    def osc_client_init(self):
        osc_client = udp_client.SimpleUDPClient(self.OSC_CLIENT_IP,
                                                self.OSC_CLIENT_PORT)
        return osc_client

class path_helper:

    def root_data_folder(rdf):
        # if running as main
        if __name__ == '__main__' and os.name == 'nt':
            return 'C:\\IBL_root\\pybpod_projects\\IBL\\data/'
        elif __name__ == '__main__' and os.name == 'posix':
            return '/home/nico/Projects/IBL/IBL-github/pybpod_projects/IBL/\
data/'

        # If no data folder was defined in settings file use default folder
        if rdf is None:
            out = 'C:/IBL_root/pybpod_data/'  # '../pybpod_projects/IBL/data/'
        else:
            # If folder was defined in settings check if endswith '/'
            out = rdf if rdf.endswith('/') else rdf + '/'

        if not os.path.exists(out):
            os.mkdir(out)
        return out

    def mouse_data_folder(sph):
        mdf = sph.ROOT_DATA_FOLDER + sph.MOUSE_NAME + '/'
        if not os.path.exists(mdf):
            os.mkdir(mdf)
        return mdf

    def session_data_folder(sph):
        sdf = sph.MOUSE_DATA_FOLDER + sph.SESSION_NAME + '/'
        if not os.path.exists(sdf):
            os.mkdir(sdf)
        return sdf

    def settings_file_path(sph):
        return (sph.SESSION_DATA_FOLDER + sph.TASK_NAME + '.settings.json')

    def data_file_path(sph):
        return (sph.SESSION_DATA_FOLDER + sph.TASK_NAME + '.data.json')

    def previous_data_file(sph):
        session_list = glob.glob(sph.MOUSE_DATA_FOLDER + '*')
        if not session_list:
            return None
        latest_session = max(session_list, key=os.path.getctime)
        prev_session_files = glob.glob(latest_session + '/*')
        prev_data_file = [x for x in prev_session_files if 'data.' in x]
        if prev_data_file:
            return prev_data_file[0]
        else:
            # Alternatively could call myself again after deleting session...
            del_file = open(latest_session + '/NO_DATA_FOUND_DELETE_SESSION',
                            'a')
            del_file.close()
            return None

        return prev_data_file

if __name__ == '__main__':
    import task_settings
    sph = session_param_handler(task_settings)

    # session_settings = []
    # with open(sph.SETTINGS_FILE_PATH, 'r') as f:
    #     for line in f:
    #         dict_obj = json.loads(line)
    #         session_settings.append(dict_obj)
    # print(dict_obj)

    # f = open(sph.SETTINGS_FILE_PATH + 'dump', 'w')
    # json.dump(sph.reprJSON(), f)
    # f.close()

    # f = open(sph.SETTINGS_FILE_PATH + 'dump', 'r')
    # bla = json.load(f)
    # f.close()
    # print(bla)


# -*- coding: utf-8 -*-
#
# Copyright 2012-2015 Spotify AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import logging
from helpers import unittest

import luigi.notifications
import luigi.worker
from luigi import Parameter, RemoteScheduler, Task
from luigi.worker import Worker
from mock import Mock

luigi.notifications.DEBUG = True

class DummyTask(Task):
    param = Parameter()

    def __init__(self, *args, **kwargs):
        super(DummyTask, self).__init__(*args, **kwargs)
        self.has_run = False

    def complete(self):
        old_value = self.has_run
        self.has_run = True
        return old_value

    def run(self):
        logging.debug("%s - setting has_run", self.task_id)
        self.has_run = True

class MultiprocessWorkerTest(unittest.TestCase):

    def setUp(self):
        self.scheduler = RemoteScheduler()
        self.scheduler.add_worker = Mock()
        self.scheduler.add_task = Mock()
        self.worker = Worker(scheduler=self.scheduler, worker_id='X', worker_processes=2)

    def tearDown(self):
        self.worker.stop()

    def gw_res(self, pending, task_id):
        return dict(n_pending_tasks=pending,
                    task_id=task_id,
                    running_tasks=0, n_unique_pending=0)

    def test_positive_path(self):
        a = DummyTask("a")
        b = DummyTask("b")

        class MultipleRequirementTask(DummyTask):

            def requires(self):
                return [a, b]

        c = MultipleRequirementTask("C")

        self.assertTrue(self.worker.add(c))

        self.scheduler.get_work = Mock(side_effect=[self.gw_res(3, str(a)),
                                                    self.gw_res(2, str(b)),
                                                    self.gw_res(1, str(c)),
                                                    self.gw_res(0, None),
                                                    self.gw_res(0, None)])

        self.assertTrue(self.worker.run())
        self.assertTrue(c.has_run)

    def test_path_with_task_failures(self):
        class FailingTask(DummyTask):

            def run(self):
                raise Exception("I am failing")

        a = FailingTask("a")
        b = FailingTask("b")

        class MultipleRequirementTask(DummyTask):

            def requires(self):
                return [a, b]

        c = MultipleRequirementTask("C")

        self.assertTrue(self.worker.add(c))

        self.scheduler.get_work = Mock(side_effect=[self.gw_res(3, str(a)),
                                                    self.gw_res(2, str(b)),
                                                    self.gw_res(1, str(c)),
                                                    self.gw_res(0, None),
                                                    self.gw_res(0, None)])

        self.assertFalse(self.worker.run())

if __name__ == '__main__':
    unittest.main()


import os.path
import sys
import json
import urllib2
import base64
import time
import md5

working_dir = os.path.dirname(sys.argv[0])
update_only = len(sys.argv) > 1 and 'only' in str(sys.argv[1]).lower()

class PatreonAPI(object):
    def __init__(self, access_token):
        super(PatreonAPI, self).__init__()
        self.access_token = access_token

    def fetch_user(self):
        return self.__get_json('current_user')

    def fetch_campaign_and_patrons(self):
        return self.__get_json('current_user/campaigns?include=rewards,creator,goals,pledges')

    def __get_json(self, suffix):
        url = "https://api.patreon.com/oauth2/api/{}".format(suffix)
        headers = {'Authorization': "Bearer {}".format(self.access_token)}
        request = urllib2.Request(url, headers=headers)
        contents = urllib2.urlopen(request).read()
        return json.loads(contents)

class MailChimpAPI(object):
    def __init__(self, api_key, list_id):
        super(MailChimpAPI, self).__init__()
        self.api_key = api_key
        self.list_id = list_id

    def create_tags_and_update_emails(self, email_map):
        tags_result = self.create_tags()
        emails_result = self.update_emails(email_map)
        return [tags_result, emails_result]

    def create_tags(self):
        create_operations = [self.craft_single_mergevar_creation('PTR_AMT', 'Patreon - Pledge Amount (Cents)', 'number'),
                             self.craft_single_mergevar_creation('PTR_NAME', 'Patreon - Full Name', 'text'),
                             self.craft_single_mergevar_creation('PTR_EXISTS', 'Patreon - Is Patron', 'radio')]
        return self.__batch_operations(create_operations)

    def craft_single_mergevar_creation(self, key, name, type_):
        single_tag_creation_body = {
            'tag': key,
            'type': type_,
            'name': name,
            'required': False,
            'public': False
        }
        if type_ == 'radio':
            single_tag_creation_body['options'] = {
                'choices': ['true', '']
            }
        return {
            'method': 'POST',
            'path': '/lists/{}/merge-fields'.format(self.list_id),
            'operation_id': 'create mergevar {}'.format(key),
            'body': json.dumps(single_tag_creation_body)
        }

    def update_emails(self, email_map):
        update_operations = [self.craft_single_email_update(email, patronage_amount_and_full_name['amount_cents'],
                                                              patronage_amount_and_full_name['full_name'])
                             for email, patronage_amount_and_full_name in email_map.items()]
        update_result = self.__batch_operations(update_operations)
        # TODO: we could http-GET update_result['response_body_url'], un-gzip & un-tar it, iterate over the responses, and retry any operation_id that had a non-200 status_code
        # ... or, we could just issue create requests for all the emails, and trust that the intersection of failures for the updates and failures for the creations is size 0
        creation_result = None
        if not update_only and update_result['errored_operations'] != 0:
            create_operations = [self.craft_single_email_creation(email, patronage_amount_and_full_name['amount_cents'],
                                                                  patronage_amount_and_full_name['full_name'])
                                 for email, patronage_amount_and_full_name in email_map.items()]
            creation_result = self.__batch_operations(create_operations)
        return [update_result, creation_result]

    def craft_single_email_update(self, email, patronage_amount, full_name):
        single_email_update_body = {
            'email_address': email,
            'merge_fields': {
                'PTR_EXISTS': 'true' if (patronage_amount > 0) else '',
                'PTR_AMT': patronage_amount,
                'PTR_NAME': full_name
            }
        }
        return {
            'method': 'PATCH',
            'path': self.path_for_email(email),
            'operation_id': 'update {}'.format(email),
            'body': json.dumps(single_email_update_body)
        }

    def craft_single_email_creation(self, email, patronage_amount, full_name):
        single_email_create_body = {
            'email_address': email,
            'status': 'subscribed',
            'merge_fields': {
                'PTR_EXISTS': 'true' if (patronage_amount > 0) else '',
                'PTR_AMT': patronage_amount,
                'PTR_NAME': full_name
            }
        }
        return {
            'method': 'POST',
            'path': self.path_for_email(email, is_create=True),
            'operation_id': 'update {}'.format(email),
            'body': json.dumps(single_email_create_body)
        }

    def datacenter(self):
        return "us{}".format(self.api_key.split('us')[-1])

    def path_for_email(self, email, is_create=False):
        basic_path = '/lists/{}/members'.format(self.list_id)
        if is_create:
            return basic_path
        else:
            email_md5 = md5.new(email.lower())
            return '{}/{}'.format(basic_path, email_md5.hexdigest())

    def __mailchimp_request(self, suffix, json_payload):
        url = "https://{}.api.mailchimp.com/3.0/{}".format(self.datacenter(), suffix)
        print('url',url)
        basic_auth = base64.encodestring('{}:{}'.format('user', self.api_key)).replace('\n', '')
        headers = {'Authorization': "Basic {}".format(basic_auth)}
        if json_payload:
            payload = json.dumps(json_payload)
            request = urllib2.Request(url, data=payload, headers=headers)
        else:
            request = urllib2.Request(url, headers=headers)
        try:
            contents = urllib2.urlopen(request).read()
            return json.loads(contents)
        except urllib2.HTTPError as e:
            print('http error',e.read())
            raise e

    def __batch_operations(self, operations):
        batch_request_state = self.__mailchimp_request('batches', {'operations': operations})
        batch_request_id = batch_request_state['id']
        while batch_request_state['status'] != 'finished':
            time.sleep(3)
            batch_request_state = self.__mailchimp_request('batches/{}'.format(batch_request_id), None)
        return batch_request_state

with open("{}/Resources/Config/config.csv".format(working_dir), 'r') as config_file:
    access_token, mailchimp_api_key, mailchimp_list_id = config_file.read().split(',')
    access_token = access_token.strip()
    mailchimp_api_key = mailchimp_api_key.strip()
    mailchimp_list_id = mailchimp_list_id.strip()
    results = [access_token, mailchimp_api_key, mailchimp_list_id]

    # # GET from Patreon
    patreon_api_client = PatreonAPI(access_token)
    response = patreon_api_client.fetch_campaign_and_patrons()
    main_campaign = response['data'][0]
    creator_id = main_campaign['relationships']['creator']['data']['id']
    included = response.get('included')
    pledges = [obj for obj in included
               if obj['type'] == 'pledge' and obj['relationships']['creator']['data']['id'] == creator_id]
    patron_ids = [pledge['relationships']['patron']['data']['id'] for pledge in pledges]
    patrons = [obj for obj in included
               if obj['type'] == 'user' and obj['id'] in patron_ids]
    patron_attributes_map = {patron['id']: patron['attributes']
                             for patron in patrons
                             if 'email' in patron['attributes']}
    patronage_map = {}
    for pledge in pledges:
        if pledge['relationships']['patron']['data']['id'] in patron_attributes_map:
            patron_attributes = patron_attributes_map[pledge['relationships']['patron']['data']['id']]
            if 'email' in patron_attributes and 'amount_cents' in pledge['attributes']:
                relevant_info = {
                    'amount_cents': pledge['attributes']['amount_cents']
                }
                if 'full_name' in patron_attributes:
                    relevant_info['full_name'] = patron_attributes['full_name']
                patronage_map[patron_attributes['email']] = relevant_info

    # POST and PATCH to MailChimp
    mailchimp_api_client = MailChimpAPI(api_key=mailchimp_api_key,
                                        list_id=mailchimp_list_id)
    email, amt_and_name = patronage_map.items()[0]
    # results = mailchimp_api_client.update_one_email(email, amt_and_name['amount_cents'], amt_and_name['full_name'])
    # results = mailchimp_api_client.create_tags()
    results = mailchimp_api_client.create_tags_and_update_emails(patronage_map)