import random  # For generating random numbers
import sys
from turtle import isvisible  # We will use sys.exit to exit the program
import pygame
from pygame.locals import *
from scipy.fftpack import diff  # Basic pygame imports
from environment import environment
from agent import DQN, DQNagent
from agent_cn import DQN_cn, DQNagent_cn
import copy as cp
import torch
import numpy as np
import torchvision.transforms as T
from PIL import Image
import matplotlib.pyplot as plt
import collections
#CN trained bird worked good on manual input test environment!?!?!?
from agentPG_cp import PolicyNet

# Global Variables for the game
FPS = 32
scr_width = 289
scr_height = 511
display_screen_window = pygame.display.set_mode((scr_width, scr_height))
play_ground = scr_height * 0.8
game_image = {}
game_audio_sound = {}
player = 'images/bird.png'
bcg_image = 'images/background.png'
pipe_image = 'images/pipe.png'
resize = T.Compose([T.ToPILImage(),
            T.Resize(40, interpolation=Image.CUBIC),
            T.ToTensor()])

def welcome_main_screen():
    """
    Shows welcome images on the screen
    """

    p_x = int(scr_width / 5)
    p_y = int((scr_height - game_image['player'].get_height()) / 2)
    msgx = int((scr_width - game_image['message'].get_width()) / 2)
    msgy = int(scr_height * 0.13)
    b_x = 0
    while True:
        for event in pygame.event.get():
            # if user clicks on cross button, close the game
            if event.type == QUIT or (event.type == KEYDOWN and event.key == K_ESCAPE):
                pygame.quit()
                sys.exit()

            # If the user presses space or up key, start the game for them
            elif event.type == KEYDOWN and (event.key == K_SPACE or event.key == K_UP):
                return
            else:
                display_screen_window.blit(game_image['background'], (0, 0))
                display_screen_window.blit(game_image['player'], (p_x, p_y))
                display_screen_window.blit(game_image['message'], (msgx, msgy))
                display_screen_window.blit(game_image['base'], (b_x, play_ground))
                pygame.display.update()
                time_clock.tick(FPS)

def main_gameplay(game):
    while True:
        flapper = False
        for event in pygame.event.get():
            if event.type == QUIT or (event.type == KEYDOWN and event.key == K_ESCAPE):
                pygame.quit()
                sys.exit()
            if event.type == KEYDOWN and (event.key == K_SPACE or event.key == K_UP):
                if game.p_y > 0:
                    flapper = True
                    game_audio_sound['wing'].play()

        p_middle_positions = game.p_x + game_image['player'].get_width() / 2
        for pipe in game.up_pips:
            pip_middle_positions = pipe['x'] + game_image['pipe'][0].get_width() / 2
            if pip_middle_positions <= p_middle_positions < pip_middle_positions + 4:
                game_audio_sound['point'].play()

        if game.isGameOver:
            game_audio_sound['hit'].play()
            game.restart()
            return

        if flapper:
            flapper = False
            game.update(True)
        else:
            game.update(False)

        display_screen_window.blit(game_image['background'], (0, 0))
        for pip_upper, pip_lower in zip(game.up_pips, game.low_pips):
            display_screen_window.blit(game_image['pipe'][0], (pip_upper['x'], pip_upper['y']))
            display_screen_window.blit(game_image['pipe'][1], (pip_lower['x'], pip_lower['y']))

        display_screen_window.blit(game_image['base'], (game.b_x, play_ground))
        display_screen_window.blit(game_image['player'], (game.p_x, game.p_y))
        d = [int(x) for x in list(str(game.score))]
        w = 0
        for digit in d:
            w += game_image['numbers'][digit].get_width()
        Xoffset = (scr_width - w) / 2

        for digit in d:
            display_screen_window.blit(game_image['numbers'][digit], (Xoffset, scr_height * 0.12))
            Xoffset += game_image['numbers'][digit].get_width()
        #imgdata = pygame.surfarray.array3d(display_screen_window) ######
        pygame.display.update()
        time_clock.tick(FPS)

def get_screen(imdata):
        screen = imdata.transpose((2, 0, 1))
        screen = np.ascontiguousarray(screen, dtype=np.float32) / 255
        screen = torch.from_numpy(screen)
        return resize(screen).unsqueeze(0)

def read_action(action):
        if action == 0:
            return False
        else:
            return True

def SelectAction(policyNet, currentState):
    actionProbabilityDis = policyNet(currentState) #takes a state and returns policy dis

    #selects random action based on policy
    selection = random.random()
    if selection < actionProbabilityDis.item():
        return True
    else:
        return False

def manual(game):
    if game.cur_state[1] > game.cur_state[2] - 75.0:
        return True
    else:
        return False

if __name__ == "__main__":

    isHumanPlayer = False
    testAgent = True
    manual_is = False
    testAgent_PG = False
    test_cn_agent = False

    pygame.init()
    time_clock = pygame.time.Clock()
    pygame.display.set_caption('Flappy Bird Game')
    game_image['numbers'] = (
        pygame.image.load('images/0.png').convert_alpha(),
        pygame.image.load('images/1.png').convert_alpha(),
        pygame.image.load('images/2.png').convert_alpha(),
        pygame.image.load('images/3.png').convert_alpha(),
        pygame.image.load('images/4.png').convert_alpha(),
        pygame.image.load('images/5.png').convert_alpha(),
        pygame.image.load('images/6.png').convert_alpha(),
        pygame.image.load('images/7.png').convert_alpha(),
        pygame.image.load('images/8.png').convert_alpha(),
        pygame.image.load('images/9.png').convert_alpha(),
    )

    game_image['message'] = pygame.image.load('images/message.png').convert_alpha()
    game_image['base'] = pygame.image.load('images/base.png').convert_alpha()
    game_image['pipe'] = (pygame.transform.rotate(pygame.image.load(pipe_image).convert_alpha(), 180),
                          pygame.image.load(pipe_image).convert_alpha()
                          )

    # Game sounds
    game_audio_sound['die'] = pygame.mixer.Sound('sounds/die.wav')
    game_audio_sound['hit'] = pygame.mixer.Sound('sounds/hit.wav')
    game_audio_sound['point'] = pygame.mixer.Sound('sounds/point.wav')
    game_audio_sound['swoosh'] = pygame.mixer.Sound('sounds/swoosh.wav')
    game_audio_sound['wing'] = pygame.mixer.Sound('sounds/wing.wav')

    game_image['background'] = pygame.image.load(bcg_image).convert()
    game_image['player'] = pygame.image.load(player).convert_alpha()

    pipe_width = game_image['pipe'][0].get_width()
    pipe_height = game_image['pipe'][0].get_height()
    player_width = game_image['player'].get_width()
    player_height = game_image['player'].get_height()
    base_height = game_image['base'].get_height()
    #print(scr_width)
    #print(scr_height)
    #print(pipe_width)
    #print(pipe_height)
    #print(player_width)
    #print(player_height)
    #print(base_height)
    game = environment(scr_width, scr_height,pipe_width,pipe_height,player_width,player_height,base_height,difficulty=4)
    #Diff 0,2,4

    n_test = 2000000
    score_ls = np.zeros(n_test)

    if isHumanPlayer:
        while True:
            welcome_main_screen()  # Shows welcome screen to the user until he presses a button
            main_gameplay(game)  # This is the main game function

    ########  DENSE #########

    elif testAgent:
        #FPS = 256
        FPS = 64
        for ep in range(n_test):
            old_score = 0
            agent = DQNagent(0,0)
            model = DQN(agent.n_input, agent.n_actions, agent.n_hidden)
            #model.load_state_dict(torch.load('C:/Users/jespe/Documents/GitHub/Flappy_git/Flappy_jesper/net1.pt'))
            model.load_state_dict(torch.load('C:/Users/Jesper/OneDrive/Dokument/GitHub/Flappy_git/Flappy_jesper/net_dense.pt'))
            model.eval()
            game.update(False)
            while True:
                for event in pygame.event.get():
                    if event.type == QUIT or (event.type == KEYDOWN and event.key == K_ESCAPE):
                        pygame.quit()
                        sys.exit()
                state = torch.tensor(game.cur_state)
                if game.isGameOver:
                    score_ls[ep] = old_score
                    break
                action  = model(state).argmax().item()

                if read_action(action):
                    game_audio_sound['wing'].play()
                if game.score != old_score:
                    old_score = game.score
                    print(f"Your score is {old_score}")
                    game_audio_sound['point'].play()

                game.update(read_action(action))

                display_screen_window.blit(game_image['background'], (0, 0))
                for pip_upper, pip_lower in zip(game.up_pips, game.low_pips):
                    display_screen_window.blit(game_image['pipe'][0], (pip_upper['x'], pip_upper['y']))
                    display_screen_window.blit(game_image['pipe'][1], (pip_lower['x'], pip_lower['y']))

                display_screen_window.blit(game_image['base'], (game.b_x, play_ground))
                display_screen_window.blit(game_image['player'], (game.p_x, game.p_y))
                d = [int(x) for x in list(str(game.score))]
                w = 0
                for digit in d:
                    w += game_image['numbers'][digit].get_width()
                Xoffset = (scr_width - w) / 2

                for digit in d:
                    display_screen_window.blit(game_image['numbers'][digit], (Xoffset, scr_height * 0.12))
                    Xoffset += game_image['numbers'][digit].get_width()

                pygame.display.update()
                time_clock.tick(FPS)

    ########  MANUAL #########

    elif manual_is:
        FPS = 256
        for ep in range(n_test):
            old_score = 0

            game.update(False)
            while True:
                for event in pygame.event.get():
                    if event.type == QUIT or (event.type == KEYDOWN and event.key == K_ESCAPE):
                        pygame.quit()
                        sys.exit()
                state = torch.tensor(game.cur_state)
                if game.isGameOver:
                    score_ls[ep] = old_score
                    break

                if manual(game):
                    game_audio_sound['wing'].play()
                if game.score != old_score:
                    old_score = game.score
                    print(f"Your score is {old_score}")
                    game_audio_sound['point'].play()

                game.update(manual(game))

                display_screen_window.blit(game_image['background'], (0, 0))
                for pip_upper, pip_lower in zip(game.up_pips, game.low_pips):
                    display_screen_window.blit(game_image['pipe'][0], (pip_upper['x'], pip_upper['y']))
                    display_screen_window.blit(game_image['pipe'][1], (pip_lower['x'], pip_lower['y']))

                display_screen_window.blit(game_image['base'], (game.b_x, play_ground))
                display_screen_window.blit(game_image['player'], (game.p_x, game.p_y))
                d = [int(x) for x in list(str(game.score))]
                w = 0
                for digit in d:
                    w += game_image['numbers'][digit].get_width()
                Xoffset = (scr_width - w) / 2

                for digit in d:
                    display_screen_window.blit(game_image['numbers'][digit], (Xoffset, scr_height * 0.12))
                    Xoffset += game_image['numbers'][digit].get_width()

                pygame.display.update()
                time_clock.tick(FPS)

    ########## POLICY ##############

    elif testAgent_PG:
        FPS = 256
        for ep in range(n_test):
            old_score = 0
            model = PolicyNet()
            #model.load_state_dict(torch.load('C:/Users/jespe/Documents/GitHub/Flappy_git/Flappy_jesper/net1.pt'))
            model.load_state_dict(torch.load('C:/Users/Jesper/OneDrive/Dokument/GitHub/Flappy_git/Flappy_jesper/pissnisse.pt'))
            model.eval()
            game.update(False)
            while True:
                for event in pygame.event.get():
                    if event.type == QUIT or (event.type == KEYDOWN and event.key == K_ESCAPE):
                        pygame.quit()
                        sys.exit()
                state = torch.tensor(game.cur_state)
                if game.isGameOver:
                    score_ls[ep] = old_score
                    break
                #action  = model(state).argmax().item()

                if SelectAction(model, state):
                    game_audio_sound['wing'].play()
                if game.score != old_score:
                    old_score = game.score
                    print(f"Your score is {old_score}")
                    game_audio_sound['point'].play()

                game.update(SelectAction(model, state))

                display_screen_window.blit(game_image['background'], (0, 0))
                for pip_upper, pip_lower in zip(game.up_pips, game.low_pips):
                    display_screen_window.blit(game_image['pipe'][0], (pip_upper['x'], pip_upper['y']))
                    display_screen_window.blit(game_image['pipe'][1], (pip_lower['x'], pip_lower['y']))

                display_screen_window.blit(game_image['base'], (game.b_x, play_ground))
                display_screen_window.blit(game_image['player'], (game.p_x, game.p_y))
                d = [int(x) for x in list(str(game.score))]
                w = 0
                for digit in d:
                    w += game_image['numbers'][digit].get_width()
                Xoffset = (scr_width - w) / 2

                for digit in d:
                    display_screen_window.blit(game_image['numbers'][digit], (Xoffset, scr_height * 0.12))
                    Xoffset += game_image['numbers'][digit].get_width()

                pygame.display.update()
                time_clock.tick(FPS)

    ############ CONVOLUTION #############

    elif test_cn_agent:
        FPS = 128
        for ep in range(n_test):
            old_score = 0
            agent = DQNagent_cn(0,0)
            model = DQN_cn(agent.scr_height, agent.scr_width, agent.n_actions)
            #model.load_state_dict(torch.load('C:/Users/jespe/Documents/GitHub/Flappy_git/Flappy_jesper/net1.pt'))
            model.load_state_dict(torch.load('C:/Users/Jesper/OneDrive/Dokument/GitHub/Flappy_git/Flappy_jesper/net_cnn10.pt'))
            model.eval()
            game.update(False)

            display_screen_window.blit(game_image['background'], (0, 0))
            for pip_upper, pip_lower in zip(game.up_pips, game.low_pips):
                display_screen_window.blit(game_image['pipe'][0], (pip_upper['x'], pip_upper['y']))
                display_screen_window.blit(game_image['pipe'][1], (pip_lower['x'], pip_lower['y']))

            display_screen_window.blit(game_image['base'], (game.b_x, play_ground))
            display_screen_window.blit(game_image['player'], (game.p_x, game.p_y))
            d = [int(x) for x in list(str(game.score))]
            w = 0
            for digit in d:
                w += game_image['numbers'][digit].get_width()
            Xoffset = (scr_width - w) / 2

            for digit in d:
                display_screen_window.blit(game_image['numbers'][digit], (Xoffset, scr_height * 0.12))
                Xoffset += game_image['numbers'][digit].get_width()
            imgdata = pygame.surfarray.array3d(display_screen_window)

            last_screen = get_screen(imgdata)
            current_screen = get_screen(imgdata)

            state = current_screen - last_screen
            while True:
                for event in pygame.event.get():
                    if event.type == QUIT or (event.type == KEYDOWN and event.key == K_ESCAPE):
                        pygame.quit()
                        sys.exit()
                #state = torch.tensor(game.cur_state)
                if game.isGameOver:
                    score_ls[ep] = old_score
                    game_audio_sound['hit'].play()
                    break
                action  = model(state).argmax().item()

                if read_action(action):
                    game_audio_sound['wing'].play()
                if game.score != old_score:
                    old_score = game.score
                    print(f"Your score is {old_score}")
                    game_audio_sound['point'].play()

                game.update(read_action(action))

                display_screen_window.blit(game_image['background'], (0, 0))
                for pip_upper, pip_lower in zip(game.up_pips, game.low_pips):
                    display_screen_window.blit(game_image['pipe'][0], (pip_upper['x'], pip_upper['y']))
                    display_screen_window.blit(game_image['pipe'][1], (pip_lower['x'], pip_lower['y']))

                display_screen_window.blit(game_image['base'], (game.b_x, play_ground))
                display_screen_window.blit(game_image['player'], (game.p_x, game.p_y))
                d = [int(x) for x in list(str(game.score))]
                w = 0
                for digit in d:
                    w += game_image['numbers'][digit].get_width()
                Xoffset = (scr_width - w) / 2

                for digit in d:
                    display_screen_window.blit(game_image['numbers'][digit], (Xoffset, scr_height * 0.12))
                    Xoffset += game_image['numbers'][digit].get_width()
                imgdata = pygame.surfarray.array3d(display_screen_window)
                last_screen = cp.deepcopy(current_screen)
                current_screen = get_screen(imgdata)
                pygame.display.update()
                time_clock.tick(FPS)

    score_ls = score_ls.astype(int)
    meany = np.mean(score_ls)
    print(f'Best score obtained durring testing: {max(score_ls)}')
    print(f'Mean of scores: {meany}')
    print(f'Standard deviation of scores: {np.std(score_ls)}')
    plt.figure()
    plt.plot(score_ls,label='Score')
    plt.plot([0, len(score_ls)-1],[meany,meany],label='Mean Score')
    plt.xlabel('Episode Number')
    plt.ylabel('Score')
    plt.legend()

    plt.figure()
    #plt.hist(score_ls, bins=max(score_ls))
    d = collections.Counter(score_ls)
    plt.bar(d.keys(), d.values())

    plt.xlabel('Score')
    plt.ylabel('Frequency')

    plt.show()


# coding=utf-8
import xml.dom.minidom
import os, sys
import xml.etree.ElementTree as ET

def gaiming(imglist, xmllist, xmlpath, imgpath):
    print("img size : ", len(imglist))
    print("xml size : ", len(xmllist))
    for k in range(0, len(imglist)):
        for i in range(0, len(xmllist)):
            xmldir = os.path.join(xmlpath, xmllist[i])
            if os.path.splitext(imglist[k])[1] == '.bmp' and os.path.splitext(xmllist[i])[1] == '.xml' and os.path.splitext(imglist[k])[0] == os.path.splitext(xmllist[i])[0] and os.path.isfile(xmldir):
                #print(os.path.splitext(imglist[k])[0])
                #imgl=list(os.path.splitext(imglist[k]))
                #xmll = list(os.path.splitext(xmllist[i]))
                #imgl[0] = str(k)
                imgsrc = os.path.join(imgpath, imglist[k])
                imgdst = os.path.join(imgpath, str(k) + '.bmp')
                xmlsrc = os.path.join(xmlpath, xmllist[i])
                xmldst = os.path.join(xmlpath, str(k) + '.xml')
                #print(xmldst)
                os.rename(imgsrc, imgdst)
                os.rename(xmlsrc, xmldst)
def changexml(xmlpath):
    xmllist = os.listdir(xmlpath)
    a = []
    #print(xmllist)
    for c in xmllist:
        if os.path.splitext(c)[1] == '.xml':
            a.append(int(os.path.splitext(c)[0]))
            #print(os.path.splitext(c)[0])
            doc = ET.parse(os.path.join(xmlpath, c))
            root = doc.getroot()
            sub1 = root.find('filename')
            sub1.text = os.path.splitext(c)[0]
            #print(sub1.text)
            #os.path.splitext(k)[0]
            doc.write(os.path.join(xmlpath, c))
            #print(doc)
    print(sorted(a))
def xmlprthc(xmlpath):
    xmllist = os.listdir(xmlpath)
    #imglist = os.listdir(imgpath)
    for k in xmllist:
        if os.path.splitext(k)[1] == '.xml':
            doc = ET.parse(os.path.join(xmlpath, k))
            root = doc.getroot()
            sub2 = root.find('path')
            sub2list = sub2.text.split('/')
            #sub2list[3] = imglist
            #print(sub2list)
            sub2.text = os.path.join(sub2list[0], sub2list[1], sub2list[2], os.path.splitext(k)[0] +'.bmp')
            #print(sub2.text)
            doc.write(os.path.join(xmlpath, k))
def shanchu(xmlpath):
    xmllist = os.listdir(xmlpath)
    for k in xmllist:
        if os.path.splitext(k)[1] != '.xml' or len(os.path.splitext(k)) == 1:
            os.remove(os.path.join(xmlpath, k))

if __name__ == "__main__":
    xmlpath = '/home/nie/downloads/imgxml/xml'  # 存有xml的文件夹路径
    imgpath = '/home/nie/downloads/imgxml/grayimg'
    xmllist = os.listdir(xmlpath)  # 列出文件夹下所有的目录与文件
    imglist = os.listdir(imgpath)
    #gaiming(imglist, xmllist, xmlpath, imgpath)
    changexml(xmlpath)
    #xmlprthc(xmlpath)
    #shanchu(xmlpath)


import yaml
from types import SimpleNamespace

class DotDict(SimpleNamespace):
    def __init__(self, dictionary, **kwargs):
        super().__init__(**kwargs)
        for key, value in dictionary.items():
            if isinstance(value, dict):
                self.__setattr__(key, DotDict(value))
            else:
                self.__setattr__(key, value)

    def __repr__(self):
        keys = sorted(self.__dict__)
        items = ("{}={!r}".format(k, self.__dict__[k]) for k in keys)
        return "{}({})".format(type(self).__name__, ", ".join(items))

    def items(self):
        keys = sorted(self.__dict__)
        item_dict = {k: self.__dict__[k] for k in keys}
        return item_dict.items()

def read_config(config_path):
    with open(config_path, "r") as ymlfile:
        return DotDict(yaml.safe_load(ymlfile))


# coding: utf-8

# ## CNN

# In[1]:

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import glob
import numpy as np
import matplotlib.pyplot as plt
from keras.models import Sequential, Model
from keras.layers import Input, Dense, Conv2D, Activation, MaxPool2D, Dropout, Flatten,BatchNormalization

#-----------
# CNNmodel
#-----------

def create_cnn(dim=(24,4,1)):
    model = Sequential()

    model.add(Conv2D(8, input_shape=dim,
                     kernel_size=(3, 3),
                     strides=(1, 1),
                     padding='same'))
    model.add(Activation('relu'))

    model.add(Conv2D(8,
                     kernel_size=(3, 3),
                     strides=(1, 1),
                     padding='same'))
    model.add(Activation('relu'))

    model.add(Flatten())
    model.add(Dense(10))
    model.add(Activation('relu'))
    model.add(Dense(2))
    model.add(Activation('softmax'))

    return model

num_pairs = 3000

pos_dir = '../16_8/positive_x/*'
neg_dir = '../16_8/negative_x/*'

pro_name =list()
lig_name = list()

pos_files_path = sorted(glob.glob(pos_dir))
neg_files_path = sorted(glob.glob(neg_dir))

# Load first positive sample
X = np.loadtxt(pos_files_path[0], delimiter=',', dtype=np.float64, ndmin=2)
X = X[np.newaxis,:,:,np.newaxis]
# Concat first negative sample
X_neg = np.loadtxt(neg_files_path[0], delimiter=',', dtype=np.float64, ndmin=2)
X_neg = X_neg[np.newaxis,:,:,np.newaxis]
X = np.concatenate((X,X_neg), axis=0)

for i in range(0,num_pairs-1):
    text = str(pos_files_path[i])
    number =text[19:28]
    pro_name.append(number.split("_")[0])
    lig_name.append(number.split("_")[1])
    text = str(neg_files_path[i])
    pro_name.append(number.split("_")[0])
    lig_name.append(number.split("_")[1])

    # Concat positive sample
    X_pos = np.loadtxt(pos_files_path[i], delimiter=',', dtype=np.float64, ndmin=2)
    X_pos = X_pos[np.newaxis,:,:,np.newaxis]
    X = np.concatenate((X,X_pos), axis=0)
    # Concat negative sample
    X_neg = np.loadtxt(neg_files_path[i], delimiter=',', dtype=np.float64, ndmin=2)
    X_neg = X_neg[np.newaxis,:,:,np.newaxis]
    X = np.concatenate((X,X_neg), axis=0)

print('X.shape:', X.shape)

# In[2]:

pro_name =list()
lig_name = list()
for i in range(0,num_pairs):
    text = str(pos_files_path[i])
    number =text[19:28]
    pro_name.append(number.split("_")[0])
    lig_name.append(number.split("_")[1])
    text2 = str(neg_files_path[i])
    number2 =text2[19:28]
    pro_name.append(number2.split("_")[0])
    lig_name.append(number2.split("_")[1])

# In[3]:

len(pro_name)

# In[4]:

# Create Y: Dock-(1,0), No dock-(0,1)
Y = np.tile((1,0,0,1), num_pairs).astype(np.float64).reshape(-1,2)
print('Y.shape:', Y.shape)

# In[5]:

from sklearn.model_selection import train_test_split

indices = np.array(range(X.shape[0]))
X_train, X_test, Y_train, Y_test,pro_name_train,pro_name_test,lig_name_train,lig_name_test = train_test_split(X, Y, pro_name,lig_name,test_size=0.20, random_state=111)

# In[6]:

print(X_train.shape)
print(X_test.shape)

# In[7]:

# ミニバッチに含まれるサンプル数を指定
batch_size = 50
# epoch数を指定
n_epoch = 30

cnn = create_cnn(dim=(24,4,1))
print(cnn.summary())

cnn.compile(loss='binary_crossentropy',
            optimizer='adam',
            metrics=['categorical_accuracy'])

# 学習を開始します。
print('Training model...')
cnn_hist = cnn.fit(X_train,Y_train,
                   epochs=n_epoch,
                   validation_split=0.2,
                   verbose=1,
                   batch_size=batch_size)

print('\n', cnn_hist.history)

# In[8]:

## 学習結果の確認

def plot_history_loss(hist):
    # 損失値(Loss)の遷移のプロット
    plt.figure()
    plt.plot(hist.history['loss'],label="Training ")
    plt.title('Loss Curve')
    plt.xlabel('Epochs')
    plt.ylabel('Loss')
    plt.legend(loc='best')
    plt.show()

def plot_history_acc(hist):
    # 精度(Accuracy)の遷移のプロット
    plt.figure()
    plt.plot(hist.history['categorical_accuracy'],label="Training set")
    plt.title('Accuracy Curve')
    plt.xlabel('Epochs')
    plt.ylabel('Accuracy')
    plt.legend(loc='best')
    plt.ylim([0, 1])
    plt.show()

plot_history_loss(cnn_hist)
plot_history_acc(cnn_hist)

# In[9]:

# テストデータを使ってモデルの評価
loss_and_accuracy = cnn.evaluate(X_train, Y_train, batch_size=128)
print(loss_and_accuracy)

# In[10]:

# テストデータを使ってモデルの評価
loss_and_accuracy = cnn.evaluate(X_test, Y_test, batch_size=128)
print(loss_and_accuracy)

# ## VGGnet

# In[12]:

import keras
from keras.layers import Conv2D, MaxPooling2D, Lambda, Input, Dense, Flatten, BatchNormalization
from keras.models import Model
from keras.layers.core import Dropout
from keras import optimizers
import tensorflow as tf
from keras.callbacks import ReduceLROnPlateau,TensorBoard

from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report

from sklearn.preprocessing import OneHotEncoder
from keras.datasets import cifar10
import cv2
# import gc
import numpy as np

# In[20]:

inputs = Input(shape=(24, 4, 1))
# Due to memory limitation, images will resized on-the-fly.
#x = Lambda(lambda image: tf.image.resize_images(image, (224, 224)))(inputs)
x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(inputs)
x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(x)
#x = MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='block1_pool')(x)
x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(x)
x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(x)
#x = MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='block2_pool')(x)
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(x)
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(x)
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(x)
#x = MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='block3_pool')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(x)
#x = MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='block4_pool')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(x)
#x = MaxPooling2D((2, 2), strides=(2, 2), padding='same', name='block5_pool')(x)
flattened = Flatten(name='flatten')(x)
x = Dense(4096, activation='relu', name='fc1')(flattened)
x = Dropout(0.5, name='dropout1')(x)
x = Dense(4096, activation='relu', name='fc2')(x)
x = Dropout(0.5, name='dropout2')(x)
# Change to 2 from 10
predictions = Dense(2, activation='softmax', name='predictions')(x)

BATCH_SIZE = 256
sgd = optimizers.SGD(lr=0.01,
                     momentum=0.9,
                     decay=5e-4)#, nesterov=False)

model = Model(inputs=inputs, outputs=predictions)

model.compile(optimizer=sgd,
              loss='categorical_crossentropy',
              metrics=['accuracy'])

# In[21]:

# ミニバッチに含まれるサンプル数を指定
batch_size = 50
# epoch数を指定
n_epoch = 5

# 学習を開始します。
print('Training model...')
hist = model.fit(X_train,Y_train,
                   epochs=n_epoch,
                   validation_split=0.2,
                   verbose=1,
                   batch_size=batch_size)

print('\n', hist.history)

# ## CNN with dropout_BatchNormalization

# In[16]:

def create_cnn_drop(dim=(24,4,1)):
    model = Sequential()

    model.add(Conv2D(8, input_shape=dim,
                     kernel_size=(3, 3),
                     strides=(1, 1),
                     padding='same'))
    model.add(Dropout(0.4))
    model.add(BatchNormalization())

    model.add(Conv2D(8,
                     kernel_size=(3, 3),
                     strides=(1, 1),
                     padding='same'))
    model.add(Dropout(0.4))
    model.add(BatchNormalization())

    model.add(Flatten())
    model.add(Dense(10))
    model.add(Activation('relu'))
    model.add(Dense(2))
    model.add(Activation('softmax'))

    return model

# In[17]:

# ミニバッチに含まれるサンプル数を指定
batch_size = 50
# epoch数を指定
n_epoch = 30

cnn = create_cnn_drop(dim=(24,4,1))
print(cnn.summary())

cnn.compile(loss='categorical_crossentropy',
            optimizer='adam',
            metrics=['categorical_accuracy'])

# 学習を開始します。
print('Training model...')
cnn_hist = cnn.fit(X_train,Y_train,
                   epochs=n_epoch,
                   validation_split=0.2,
                   verbose=1,
                   batch_size=batch_size)

print('\n', cnn_hist.history)

# In[18]:

# テストデータを使ってモデルの評価
loss_and_accuracy = cnn.evaluate(X_test, Y_test, batch_size=128)
print(loss_and_accuracy)

# ## For test data

# In[10]:

import pandas as pd
df = pd.DataFrame(np.random.random([len(Y_test), 4]), columns=['pro_id','lig_id','dock','no_dock'])

for i in range(len(Y_test)):
    df.pro_id[i]=pro_name[i]
    df.lig_id[i] = lig_name[i]
    pred_1 = cnn.predict(X_test[i].reshape(1, 308, 4, 1))[0]
    pred_2 = np.array(pred_1, dtype=str)
    df.dock[i] = pred_2[0]
    df.no_dock[i] = pred_2[1]

# In[ ]:

df

# In[ ]:

df.sort_values(by='dock', ascending=False)[:10]

# In[ ]:

df[df.pro_id==314.0].sort_values(by='dock', ascending=False)[:10]


import streamlit as st
from PIL import Image, ImageDraw
import torch
import numpy as np

from streamlit_webrtc import webrtc_streamer, WebRtcMode, RTCConfiguration
import av

# trick to load model once at server start
if not hasattr(st, 'classifier'):
    st.model = torch.hub.load('ultralytics/yolov5', 'yolov5s',  _verbose=False)
    # if you train your custm model "yolov5s.pt" use the following line
    # st.model = torch.hub.load('ultralytics/yolov5', 'custom', path='yolov5s.pt', _verbose=False)

# Configuration for use camera from web deployed server
RTC_CONFIGURATION = RTCConfiguration(
    {"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]}
)

def hit(player, enemy, ray):
    xp, yp = player
    xe, ye = enemy
    dist = np.sqrt((xp - xe)**2 + (yp - ye)**2)
    return dist < ray

def compute_shape(arr):
    cx, cy = arr
    return [cx-10, cy-10, cx+10, cy+10]

class VideoProcessor:
    points = 0
    enemy = (256, 256)
    enemy2 = (400, 400)
    enemy_ray = 20
    player_pos = [50, 50]

    def recv(self, frame):
        img = frame.to_ndarray(format="bgr24")

        # vision processing
        flipped = img[:, ::-1, :]

        # model processing
        im_pil = Image.fromarray(flipped)
        # return av.VideoFrame.from_ndarray(np.array(im_pil, copy=False), format="bgr24")
        results = st.model(im_pil, size=226)
        # bbox_img = np.array(results.render()[0])
        # res_im_pil = Image.fromarray(bbox_img)
        res_im_pil = im_pil

        # il draw è molto lento
        draw = ImageDraw.Draw(im_pil)
        draw.text((28, 36), "Points:" +
                  str(VideoProcessor.points), fill=(255, 0, 0))

        if VideoProcessor.points >= 10:
            draw.text((res_im_pil.width//2, res_im_pil.height//2),
                      "You WIN", fill=(255, 0, 0))
        elif VideoProcessor.points < 0:
            draw.text((res_im_pil.width//2, res_im_pil.height//2),
                      "You Louse", fill=(255, 0, 0))
        else:
            draw.text(VideoProcessor.enemy, "ENEMY", fill=(255, 0, 0))
            draw.text(VideoProcessor.enemy2, "ENEMY", fill=(0, 0, 255))

            for p in results.pred[0]:
                x1, y1, x2, y2, conf, pred = list(p.numpy())
                cx,cy = np.abs(x1-x2) / 2 + min([x1, x2]), np.abs(y1-y2) / 2 + min([y1, y2])
                class_name = results.names[int(pred)]
                if class_name == 'scissors':
                    VideoProcessor.player_pos = [cx, cy]
                    break

            shape = compute_shape(VideoProcessor.player_pos)
            draw.rectangle(shape, fill ="#ffff33", outline ="red")

            if hit(VideoProcessor.player_pos, VideoProcessor.enemy, VideoProcessor.enemy_ray):
                VideoProcessor.points += 1
                VideoProcessor.enemy = list(np.random.randint(475 - 40 , size=2) +20 )
                VideoProcessor.enemy2 = list(np.random.randint(475 - 40 , size=2) +20 )
                print(VideoProcessor.enemy, VideoProcessor.enemy2)
            if hit(VideoProcessor.player_pos, VideoProcessor.enemy2, VideoProcessor.enemy_ray):
                VideoProcessor.points -=1
                VideoProcessor.enemy = list(np.random.randint(475 - 40 , size=2) +20 )
                VideoProcessor.enemy2 = list(np.random.randint(475 - 40 , size=2) +20 )
                print(VideoProcessor.enemy, VideoProcessor.enemy2)

            # if VideoProcessor.enemy[0] > x1 and VideoProcessor.enemy[0] < x2 and \
                    #         VideoProcessor.enemy[1] > y1 and VideoProcessor.enemy[1] < y2:
                    #     VideoProcessor.points += 1

        # res_im_pil.save('test.png')
        return av.VideoFrame.from_ndarray(np.array(res_im_pil, copy=False), format="bgr24")

if __name__ == '__main__':
    st.session_state['game'] = {
        'points': 0
    }

    webrtc_ctx = webrtc_streamer(
        key="WYH",
        mode=WebRtcMode.SENDRECV,
        rtc_configuration=RTC_CONFIGURATION,
        video_processor_factory=VideoProcessor,
        media_stream_constraints={"video": True, "audio": False},
        async_processing=False,
    )


import asyncio
import os
from datetime import datetime

# File locations
root_folder = r"<root_file_path>"
log_file = r"<log_file_path\output.txt>"

async def run():
    # Initialize the variables
    file_count = 0
    total_lines = 0
    special_count = 0
    event_count = 0

    for subdir, dirs, files in os.walk(root_folder):
        for file in files:
            file_count += 1
            msg_filename = os.path.join(subdir, file)
            num_lines = 0
            with open(msg_filename, "r") as f:
                print("File name: ", msg_filename)
                msg_str = ""
                # Check if it is a multi-line message
                for line in f:
                    num_lines += 1
                    if line.startswith("{ body: ") and ("timezone.utc) }" in line):
                        msg_str = line
                    else:
                        msg_str += line
                        if not (msg_str.startswith("{ body: ") and ("timezone.utc) }" in line)):
                            continue

                    # Process the message
                    try:
                        has_special = 0
                        if msg_str.find("\\x") > 0:
                            has_special = 1
                        special_count += has_special

                    except Exception as err:
                        print(str(err))
                        with open(log_file, "a+") as log_f:
                            log_f.write(
                                "ERROR: " + str(err) + ", File path: " + msg_filename + ", Event: " + msg_str)
                        pass

                    # Reset the message
                    msg_str = ""
                    event_count += 1

            # Log for each file
            with open(log_file, "a+") as log_f:
                output_time = str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
                log_f.write(output_time + " Completed processing: " + msg_filename + "\n")
            print("Number of lines in the file: ", num_lines)
            print(
                "------------------------------------------------------------------------------------------------------")
            total_lines += num_lines

    # Log for the total process
    with open(log_file, "a+") as log_f:
        log_f.write("Total num of files: " + str(file_count) + "\n")
        log_f.write("Total num of lines: " + str(total_lines) + "\n")
        log_f.write("Total num of events: " + str(event_count) + "\n")
        log_f.write("Total num of events with special characters: " + str(special_count) + "\n")

def main():
    startTime = datetime.now()
    with open(log_file, "a+") as log_f:
        log_f.write(str(startTime.strftime("%Y-%m-%d %H:%M:%S")) + " ----------SCRIPT EXECUTION STARTED----------" + "\n")
    # Start processing the files
    loop = asyncio.get_event_loop()
    loop.run_until_complete(run())
    # Finish processing the files
    with open(log_file, "a+") as log_f:
        log_f.write("Total processing time in seconds: " + str((datetime.now() - startTime)) + "\n")
        log_f.write(
            str(datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + " ----------SCRIPT EXECUTION ENDED ----------" + "\n")

if __name__ == "__main__":
    # This code won't run if this file is imported.
    main()


# import essentials
import time
import random
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler

# set the seed for reproducability
seed = 69
random.seed = seed
np.random.seed(seed)

# get data
data = pd.read_csv('../input/high_diamond_ranked_10min.csv')

# data cleaning time!

# Heat map to show which features correlate with y
corrmat = data.corr()
plt.subplots(figsize=(12,9))
sns.heatmap(corrmat, vmax=0.9, square=True)

# many features are linearly dependent on one another. Gotta drop.

# redGoldDiff is -1 * blueGoldDiff
# blueTotalGold is NOT dropped because it is not a
# linear combination of cs and kills for game-specific reasons
# check if blueFirstBlood and redFirstBlood are opposites
nokills = pd.DataFrame(data['blueFirstBlood'] + data['redFirstBlood'] == 0)
print('\nno kill games:', str(pd.DataFrame.sum(nokills).values[0]) + '\n')
# so if blueFirstBlood is false, redFirstBlood is true and vice-versa
# so drop redFirstBlood

# total experience is not a zero-sum game, so leave redTotalExperience in
# same with dragons, kills, wards placed/destroyed
# but red and blue deaths are almost opposites (not completely opposites since executes can occur
# so get rid of deaths
# also get rid of red total gold and exp, since blue gold - difference = red gold
# blueEliteMonsters = blueDragons + blueHeralds
dropset = ['redGoldDiff', 'redFirstBlood', 'blueDeaths', 'redDeaths',
           'redExperienceDiff', 'redTotalGold', 'redTotalExperience',
           'blueEliteMonsters', 'redEliteMonsters', 'gameId']

outcomes = data.pop('blueWins')
data = data.drop(columns=dropset)

# adding features time!
def square_feats(_columns, _data):
    for col in _columns :
        _data[col + '_2'] = _data[col] ** 2
    return _data

square_cols = [
    'blueWardsPlaced', 'redWardsPlaced', 'blueWardsDestroyed',
    'redWardsDestroyed', 'blueExperienceDiff'
    ]
data = square_feats(square_cols, data)

# check that all the variables are in
print(str(data.dtypes) + '\n')

# observe our dataset
print(data.describe())

# create the train and test sets
from sklearn.model_selection import \
    train_test_split, cross_val_predict, cross_val_score
from sklearn.model_selection import KFold

X_train, X_test, y_train, y_test = train_test_split(
    data, outcomes, test_size=0.20,
        random_state=seed, shuffle=True, stratify=outcomes)

# scale our data
scaler = StandardScaler().fit(X_train)
X_train, X_test = scaler.transform(X_train), scaler.transform(X_test)

# define how we will output scores to the screen:
def score_method(_y_actual :np.array, _y_pred :np.array) :
    errors = np.abs(_y_actual - _y_pred)
    _score = np.sum(errors) / _y_actual.shape[0]
    return 1 - _score

def score_models(_models :dict, _X=X_train, _y=y_train) :
    t_0 = time.time()
    kfold = KFold(n_splits=5, shuffle=False)
    output = list()
    for name, model in _models.items() :
        predictions = cross_val_predict(model, _X, _y, cv=kfold, n_jobs=-1)
        _t_1 = time.time() - t_0
        _pred_score = score_method(y_train,predictions)
        _score = cross_val_score(model, _X, _y, cv=kfold, n_jobs=-1, scoring='roc_auc')
        output.append([name,_pred_score,_t_1])
        print('\n{} ROC Score: {:.4f}, ({:.4f})'
              .format(name, _score.mean(), _score.std()))
        print('{} accuracy: {:.4f}\nTime elapsed: {:.1f}s'
              .format(name,_pred_score,_t_1))
        model.fit(X_train,y_train)
        test_eval = model.score(X_test,y_test)
        print('{} accuracy on the test set: {:.4f}\nVariance: {:.4f}'
              .format(name, test_eval, _score.mean() - _pred_score))
    return output

# regression tiiime
from sklearn.linear_model import ElasticNet, Lasso, \
    BayesianRidge, LassoLarsIC, LogisticRegression
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier

# create empty model dictionary that will contain the model under its name
models = {}

# define our models and add them to our model dictionary
logit = make_pipeline(RobustScaler(),
                      LogisticRegression(random_state=seed, max_iter=1000,n_jobs=-1))
models['Logit'] = logit

lasso = make_pipeline(RobustScaler(),
                      Lasso(alpha=.0005, random_state=seed))
models['Lasso'] = lasso

eNet = make_pipeline(RobustScaler(),
                     ElasticNet(alpha=.0005, l1_ratio=.9, random_state=seed))
models['ENet'] = eNet

rforest = RandomForestClassifier(max_depth=5,n_estimators=64)
models['Random Forest'] = rforest

svm = SVC(kernel='rbf',random_state=seed)
models['Basic SVM'] = svm

bayesian_ridge = BayesianRidge()
models['Bayesian Ridge'] = bayesian_ridge

lasso_lars_ic = LassoLarsIC(max_iter=15)
models['LassoLarsIC'] = lasso_lars_ic

score_models(models)

# Let's build our own stacking classifier
from sklearn.base import BaseEstimator, RegressorMixin, \
    TransformerMixin, clone
from sklearn.model_selection import cross_val_predict

print('--------------------------------------')
print('\nBuilding a stacking classifier...\n')

kfold = KFold(n_splits=5, shuffle=True, random_state=seed)

class StackedClassifier(BaseEstimator, RegressorMixin,
                      TransformerMixin):
    def __init__(self, models, final_estimator):
        self.models = models
        self.cloned_models = \
            [clone(x) for x in self.models]
        self.final_estimator = clone(final_estimator)

    # clone original models to fit data into
    def fit(self, X, y):
        # train clones
        predictions = np.column_stack([
            cross_val_predict(model,X,y,cv=kfold,n_jobs=-1,method='predict_proba')
            for model in self.cloned_models
        ])
        for model in self.cloned_models : model.fit(X,y)
        self.final_estimator.fit(predictions, y)
        return self

    # Now execute predictions for cloned models
    def predict(self, X):
        base_predictions = np.column_stack([
            model.predict(X) for
            model in self.cloned_models
        ])
        final_predictions = self.final_estimator.predict(base_predictions)
        return final_predictions

# instantiate our stacking regressor
stacked_classifier = StackedClassifier(
    [model for model in models.values()],
    LogisticRegression(penalty='l2',random_state=seed))

# fit and score our model
stacked_classifier.fit(X_train,y_train)
predictions = stacked_classifier.predict(X_test)

score = score_method(y_test, predictions)
print('\n\nStacked Classifier performance: {}'.format(score))

# now let's try a prepackadged stacking classifier
from sklearn.ensemble import StackingClassifier
from sklearn.svm import LinearSVC

estimators = [
    ('rf', RandomForestClassifier(n_estimators=128,
                                  random_state=seed)),
    ('svr', make_pipeline(LinearSVC(random_state=seed,
                                    max_iter=30000))),
    ('logit', make_pipeline(StandardScaler(),
                            LogisticRegression(max_iter=1000,n_jobs=-1,
                                               random_state=seed))),
    ('Random Forest', RandomForestClassifier(max_depth=5,n_estimators=64)),
    ('SVC',SVC(kernel='rbf',random_state=seed)),
]

print('\nNow evaluating prepackadged Stacking Classifier...')

score = score_method(y_test, predictions)
print('\n\nStacked Classifier performance: {}'.format(score))

quit()
# looks like our Logit model performs best. Let's see if a neural net does any better

#import tensorflow
import tensorflow as tf
from tensorflow.keras import layers

# make tensorflow a little quieter
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

def score_neural_net(_neural_net :tf.keras.models.Sequential,
                     X_data=X_train, y_data=y_train) :
    n_folds = 5
    kfold = KFold(n_splits=n_folds)
    _score = np.zeros(n_folds)
    for i, (train_index, test_index) in enumerate(kfold.split(X_data)) :
        _X_train,_X_test = X_data[train_index], X_data[test_index]
        _y_train,_y_test = y_data[train_index], y_data[test_index]
        _neural_net.fit(_X_train, _y_train)
        pred = _neural_net.predict(_X_test)
        _score[i] = score_method(_y_test, pred)
    return _score

def build_neural_net() :
    _neural_net = tf.keras.models.Sequential([
        layers.Dense(64,activation='relu'),
        layers.Dense(64,activation='sigmoid'),
        layers.Dense(1,activation='sigmoid')
    ])

    _neural_net.compile(
        optimizer='rmsprop',
        loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
    metrics=['accuracy'])

    return _neural_net

neural_net = build_neural_net()

print('\nFitting Neural Net...')
neural_net.fit(X_train,y_train,epochs=10)

# add a predictions layer
neural_net.add(
    layers.Lambda(
        lambda x: tf.math.round(x)
    )
)

neural_net.predict(X_test)

print('\nEvaluating model...')
neural_net.evaluate(X_test,y_test)

stacking_clf = StackingClassifier(
    estimators=estimators, final_estimator=
    LogisticRegression(max_iter=10000,random_state=seed),
    n_jobs=-1, verbose=1
)

# evaluate our final model
stacking_clf.fit(X_train,y_train)
score = stacking_clf.score(X_test,y_test)
print('\nStacking Classifier Score on the test set: {:.4f}'
      .format(score))


# -*- coding: utf-8 -*-
# @Author: Fule
# @Date:   2020-05-20 18:03:50
# @Last Modified by:   Fule
# @Last Modified time: 2020-05-21 11:30:25

# %%
import PySimpleGUI as psg
import os
import pandas as pd
from time import strftime
from pandas import DataFrame as df
from pandas import read_excel
from pathlib import Path
from secrets import token_hex
#from pandas.plotting._matplotlib import table

pro_dir = os.path.join(os.getcwd(), 'profile')
script_dir = os.path.join(os.getcwd(), 'script')
dirlist = []
event = ''

if Path(script_dir).is_dir():
    print('目录存在')
else:
    os.makedirs(script_dir)
    print('创建目录')

#%%
# 文件名函数

def file_name(file_dir, exten):
    file_list = []
    filename_list = []
    for root, dirs, files in os.walk(file_dir):
        for file in files:
            (name, extention) = os.path.splitext(file)
            if extention == exten:
                file_list.append(name)
                filename_list.append((name, file))
    return file_list, filename_list

# 替换函数

def replace_data_sing(table, template):
    with open(template) as f:
        template = f.read()
        pd_table = pd.read_excel(table)
        dict_table_index = pd_table.to_dict('index')
        for i in list(dict_table_index):
            maxlen = len(dict_table_index.keys())
            data_dict = dict_table_index[i]
            j = list(data_dict.keys())
            temp_cache = template
            for k in j:
                temp_cache = temp_cache.replace(k, str(data_dict[k]))
            yield [(i+1)/maxlen, temp_cache]

def replace_data_mult(table, template):
    with open(template) as f:
        template = f.read()
        pd_table = pd.read_excel(table)
        dict_table_index = pd_table.to_dict('index')
        for i in list(dict_table_index.keys()):
            maxlen = len(dict_table_index.keys())
            data_dict = dict_table_index[i]
            filename = str(data_dict['@filename'])
            del data_dict['@filename']
            j = list(data_dict.keys())
            temp_cache = template
            for k in j:
                temp_cache = temp_cache.replace(k, str(data_dict[k]))
            yield [(i+1)/maxlen, temp_cache,filename]

# 写入函数

def writer(data, filename):
    with open(filename, 'a+') as f:
        f.write(data)
        return filename
def exitsfile(filename):
    script_dir = os.path.join(os.getcwd(), 'script')
    filename = os.path.join(script_dir, filename)
    if Path(filename).exists():
        (name, extention) = os.path.splitext(filename)
        filename = name + '_' + str(token_hex(nbytes=2)) + extention
        return filename
    else:
        return filename

# 百分比函数

def percentage(i: int, n: int):
    print('\r{}{}%'.format(['/', '-', '\\'][i % 3] if n -
                           i else '', int(i/n*100)), end='' if n-i else '\n')

# if else简写方式
# 上面等效与
# a = '\r{}{}%'.format(['/', '-', '\\'][i % 3]
# if n-i:
#    print(a)
# else:
#    print('')
psg.theme('SystemDefault')
print_out = [
    [psg.Output(size=(85, 30))]
]
temp_input = [
             [psg.Text(text='刷新脚本模板后，单击选择')],
             [psg.Listbox(values=dirlist, size=(30, 6), key='模板列表'), ],
             [psg.Submit(button_text='刷新模板', key='flush', size=(25, 1))]
]

table_input = [
    [psg.Text(text='请选择脚本数据表格')],
    [psg.Input(key='excelname'), psg.FileBrowse(
        button_text='浏览',file_types=(('Table Files','*.xlsx'),) )],
    [psg.Text(text='如果需生成的脚本为单文件，则勾选此项')],
    [psg.Radio('单文件模式', 'mode', key='sing',default=True)],
    [psg.Text(text='如果需生成的脚本为多文件，则勾选此项')],
    [psg.Radio('多文件模式', 'mode', key='mult')],
]
script_creat = [
    [psg.Submit(button_text='生成脚本'), psg.Checkbox(
        '预览脚本，影响生成速度', key='preview')],
    [psg.Text('脚本生成进度'), psg.Text('  ', size=(5, 1), key='percentage')],
    [psg.ProgressBar(max_value=1, size=(61.5, 20), key='进度条')]
]
layout = [
    [psg.Frame('第一步', temp_input),
     psg.Frame('第二步', table_input)
     ],
    [psg.Frame('第三步', script_creat)],
    [psg.Frame('打印信息', print_out)],
    [psg.Cancel('退出')]
]
gui = psg.Window('脚本生成器(不好用也别告诉我)_v0.1').Layout(layout)

# %%
while True:
    event, var_read = gui.Read()
   # print(event,var_read)
    dirlist = file_name(pro_dir, '.pro')
    dict_template = dict(dirlist[1])
    if event == None:
        break
    elif event == '退出':
        break
    elif event == 'flush':
        gui.Element('模板列表').update(values=dirlist[0])
    elif event == '生成脚本':
        templist = gui.Element('模板列表').get()
        if templist == []:
            print('刷新模板列表后，再提交数据')
        elif var_read.get('excelname') == '':
            print('请选择数据表格')
        else:
            template = dict_template.get(templist[0])
            template = os.path.join(pro_dir, template)
            excelname = var_read.get('excelname')
            if  var_read['sing']==True:
                tablename = os.path.split(excelname)[1]
                name = os.path.splitext(tablename)[0]
                data = replace_data_sing(excelname, template)
                scriptname = name + '.txt'
                scriptname = exitsfile(scriptname)
                print(scriptname)
                try:
                    for [i, temp_cache] in data:
                        writer(temp_cache,scriptname)
                        gui.Element('percentage').update('{:.1%}'.format(i))
                        gui.Element('进度条').UpdateBar(i)
                        if var_read.get('preview') == True:
                            print(temp_cache)
                            pass
                        if i == 1:
                            print(scriptname)
                            psg.popup_ok('生成完毕！')
                            i = 0
                            gui.Element('进度条').UpdateBar(i)
                            gui.Element('percentage').update('{:.1%}'.format(i))
                except:
                    psg.popup('出错了，我也不知道为啥！！')
            elif var_read['mult']==True:
                data = replace_data_mult(tablename, template)
                try:
                    for [i, temp_cache,filename] in data:
                        writer(temp_cache,filename)
                        gui.Element('percentage').update('{:.1%}'.format(i))
                        gui.Element('进度条').UpdateBar(i)
                        if var_read.get('preview') == True:
                            print(filename)
                        else:
                            pass
                    if i == 1:
                        psg.popup_ok('生成完毕！')
                        i = 0
                        gui.Element('进度条').UpdateBar(i)
                        gui.Element('percentage').update('{:.1%}'.format(i))
                except:
                        psg.popup('出错了，我也不知道为啥！！')

gui.close() # 一定要加上这行，不然退出按钮无法使用

        # print('数据表格为')

# 2020-05-18 1:49完成脚本文件的读取，下一步计划进行表格遍历，然后根据表格读取值生成脚本
# 脚本生成两种方式：
# 1、 一个表格生成一个脚本文件；单文件方式
# 2、一个表格生成多个脚本文件；多文件方式
# %%
# def replace_data(table,template):
#    with open(template) as temp:                                    ##打开模板
#        template = temp.read()                                      ##读入模板
#        fl=pd.read_excel(table)                                     ##读入数据表，并dataframe
#        fl_test = fl.to_dict('index')                               ##以index为key转化为复合字典
#        for i in list(fl_test.keys()):                              ##遍历index值
#            data_dict=fl_test[i]                                    ##读取每行数据组成的字典
#            j = list(data_dict.keys())                              ##将每行数据组成的字典的key转化为list
#            temp_cache=template                                     ##对模板重新赋值
#            for k in j:                                             ##遍历字典key获取key的字符串
#                temp_cache=temp_cache.replace(k,str(data_dict[k]))  ##替换
#            yield [i,temp_cache]                                    ##使用迭代器yield，获取循环中的返回值，
# 2020-5-19 遗留问题：模板文件路径问题


from random import randint
from django.views.generic import TemplateView
from chartjs.views.lines import BaseLineChartView
from .models import Sensor, Owner, SensorValue
import random
import time
from django.utils import timezone
from datetime import timedelta

class TimeLineChartView(BaseLineChartView):
    def get_context_data(self, **kwargs):
        context = super(BaseLineChartView, self).get_context_data(**kwargs)
        context.update({'datasets': self.get_datasets()})
        return context
    def get_datasets(self):
        datasets = []
        color_generator = self.get_colors()
        data = self.get_data()
        providers = self.get_providers()
        num = len(providers)
        for i, entry in enumerate(data):
            color = tuple(next(color_generator))
            dataset = {'backgroundColor': "rgba(%d, %d, %d, 0.5)" % color,
                       'borderColor': "rgba(%d, %d, %d, 1)" % color,
                       'pointBackgroundColor': "rgba(%d, %d, %d, 1)" % color,
                       'pointBorderColor': "#fff",
                       'fill': False,
                        'pointRadius': 0,
                       'lineTension': 0,
                       'data': entry}
            if i < num:
                dataset['label'] = providers[i]  # series labels for Chart.js
                dataset['name'] = providers[i]  # HighCharts may need this
            datasets.append(dataset)
        return datasets

class LineChartJSONView(BaseLineChartView):
    sensor_id = 0
    def get_labels(self):
        """Return 7 labels for the x-axis."""
        return ["January", "February", "March", "April", "May", "June", "July"]

    def get_providers(self):
        """Return names of datasets."""
        return ["Central", "Eastside", "Westside"]

    def get_data(self):
        """Return 3 datasets to plot."""

        return [[75, 44, 92, 11, 44, 95, 35],
                [41, 92, 18, 3, 73, 87, 92],
                [87, 21, 94, 3, 90, 13, 65]]

class TimeChartJSONView(TimeLineChartView):
    sensor_ids = 0
    value_type = 'P1'
    def get_labels(self):
        """Return 7 labels for the x-axis."""
        return ["January", "February", "March", "April", "May", "June", "July"]

    def get_providers(self):
        """Return names of datasets."""
        return [str(d) for d in self.kwargs['sensor_ids']]

    def get_data(self):
        """Return 3 datasets to plot."""
        listDictList = []
        for id in self.kwargs['sensor_ids']:
            values = list(SensorValue.objects.filter(sensor=id,type=self.kwargs['value_type'],created__range=[timezone.now()-timedelta(days=3),timezone.now()]).order_by('created').values('created', 'value'))
            timeData = [time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.mktime(d['created'].timetuple()))) for d in values]
            ydata = [float(d['value']) for d in values]
            dictList = []
            for i in range(len(timeData)):
                dict =  {}
                dict["x"] = timeData[i]
                dict["y"] = ydata[i]

                dictList.append(dict)
            listDictList.append(dictList)

        return random.sample(listDictList,min(len(listDictList),5))


import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import os, sys

def cost(X, y, theta):
    h = np.dot(X, theta)
    cos = np.sum((h - y) * ( h - y))/(2 * len(y))
    return cos

def gradient_descent(X, y, theta, alpha, num_iters):
    m = len(y)
    costs = []
    for _ in range(num_iters):
        h = np.dot(X, theta)
        theta -= alpha * np.dot(X.T, (h - y))/m
        costs.append(cost(X, y, theta))
    return theta, costs

def linear_regression(X, y, alpha = 0.01,num_iters = 100):
    X = np.append(np.ones((X.shape[0], 1)), X, axis = 1)
    theta = np.zeros((X.shape[1], 1), dtype = np.float64)
    theta, costs = gradient_descent(X, y, theta, alpha, num_iters)
    predicted = np.dot(X, theta)
    return predicted, theta, costs

if __name__ == '__main__':
    images_dir = os.path.join(sys.path[0], 'images')
    if not os.path.exists(images_dir):
        os.makedirs(images_dir)

    X = np.array(range(25))
    y = (X ** 1.3 + np.random.normal(10, 10, X.shape[0]))
    X, y = X.reshape((-1, 1)), y.reshape((-1, 1))
    # data = np.loadtxt(os.path.join(sys.path[0], 'linear_regression_data.csv'), delimiter = ',', dtype = np.float64)
    # X, y = data[:, :-1], data[:, -1].reshape((-1, 1))

    predicted, theta, costs = linear_regression(X, y)

    sns.scatterplot(X[:, 0], y.flatten())
    plt.title('Dataset')
    plt.ylabel('Y')
    plt.xlabel('X')
    plt.savefig(os.path.join(images_dir, 'data.png'))
    plt.clf()

    sns.lineplot(range(100), costs)
    plt.title('Cost vs Number of Interations')
    plt.ylabel('Cost')
    plt.xlabel('No. of Interations')
    plt.savefig(os.path.join(images_dir, 'cost.png'))
    plt.clf()

    sns.lineplot(X[:, 0], predicted.flatten())
    sns.scatterplot(X[:, 0], y.flatten())
    for i, x in enumerate(X):
        plt.vlines(x, min(predicted[i], y[i]), max(predicted[i], y[i]))
    plt.ylabel('Y')
    plt.xlabel('X')
    plt.title('Error')
    plt.legend(('linear fit', 'data'))
    plt.savefig(os.path.join(images_dir, 'Error.png'))
    plt.clf()


import base64
import json
import logging
from datetime import datetime
from urllib.request import urlopen

import requests
import werkzeug.http

from odoo import api, models
from odoo.exceptions import AccessDenied

try:
    from jose import jwt
except ImportError:
    logging.getLogger(__name__).debug("jose library not installed")

_logger = logging.getLogger(__name__)

class ResUsers(models.Model):
    _inherit = "res.users"

    @api.model
    def _auth_oauth_signin(self, provider, validation, params):
        """retrieve and sign in the user corresponding to provider and validated access token
        :param provider: oauth provider id (int)
        :param validation: result of validation of access token (dict)
        :param params: oauth parameters (dict)
        :return: user login (str)
        :raise: AccessDenied if signin failed
        This method can be overridden to add alternative signin methods.
        """
        oauth_provider = self.env["auth.oauth.provider"].browse(provider)
        if oauth_provider.g2p_oidc_id_to_use:
            oauth_uid = validation["user_id"]
            try:
                oauth_user = self.search(
                    [
                        ("oauth_uid", "=", oauth_uid),
                        ("oauth_provider_id", "=", provider),
                    ]
                )
                if not oauth_user:
                    raise AccessDenied()
                assert len(oauth_user) == 1
                oauth_user.write({"oauth_access_token": params["access_token"]})
                return oauth_user.login
            except AccessDenied:
                json.loads(params["state"])
                partner = self.generate_partner_signup(
                    oauth_provider, validation, params
                )
                new_user = self.generate_partner_user_signup(
                    partner, oauth_provider, validation, params
                )
                return new_user.login
        else:
            return super(ResUsers, self)._auth_oauth_signin(
                provider, validation, params
            )

    def generate_partner_user_signup(self, partner, oauth_provider, validation, params):
        oauth_uid = validation["user_id"]
        login_attr = oauth_provider.login_attribute_mapping_on_user_creation
        login = validation.get(login_attr, oauth_uid)
        user_creation_dict = {
            "login": login,
            "partner_id": partner.id,
            "oauth_provider_id": oauth_provider.id,
            "oauth_uid": oauth_uid,
            "oauth_access_token": params["access_token"],
            "active": True,
        }
        if oauth_provider.default_group_user_creation:
            user_creation_dict["groups_id"] = [
                (
                    4,
                    oauth_provider.default_group_user_creation.id,
                ),
            ]
        return self.env["res.users"].create(user_creation_dict)

    def generate_partner_signup(self, oauth_provider, validation, params):
        oauth_uid = validation["user_id"]
        if oauth_provider.partner_creation_call_validate_url:
            userinfo_dict = self._auth_oauth_rpc(
                oauth_provider.validation_endpoint, params["access_token"]
            )
            update_dict = oauth_provider.map_validation_response_partner_creation(
                userinfo_dict
            )
            validation.update(update_dict)
            _logger.info(
                "Userinfo JWT payload after validation call. %s",
                json.dumps(userinfo_dict),
            )
            _logger.info(
                "Update dict after validation call. %s", json.dumps(update_dict)
            )
            _logger.info(
                "Validation Dict after validation call. %s", json.dumps(validation)
            )
        try:
            g2p_reg_id = self.env["g2p.reg.id"].search(
                [
                    ("value", "=", oauth_uid),
                    ("id_type", "=", oauth_provider.g2p_id_type.id),
                    ("partner_id.is_registrant", "=", True),
                    ("partner_id.is_group", "=", False),
                ]
            )
            if not g2p_reg_id:
                raise AccessDenied()
            assert len(g2p_reg_id) == 1
            return g2p_reg_id.partner_id
        except AccessDenied:
            # Create partner from validation dictionary
            # TODO: Improve following mapping.
            name = validation.pop("name", "")
            partner_dict = {
                "given_name": name.split(" ")[0],
                "family_name": name.split(" ")[-1],
                "addl_name": " ".join(name.split(" ")[1:-1]),
                "email": validation.pop(
                    "email", "provider_%s_user_%s" % (oauth_provider.id, oauth_uid)
                ),
                "is_registrant": True,
                "is_group": False,
            }
            partner_dict["name"] = self.process_name(
                partner_dict["family_name"],
                partner_dict["given_name"],
                partner_dict["addl_name"],
            )
            partner_dict["gender"] = self.process_gender(validation.pop("gender", ""))
            partner_dict["birthdate"] = self.process_birthdate(
                validation.pop("birthdate", None),
                date_format=oauth_provider.partner_creation_date_format,
            )
            partner_dict["reg_ids"] = self.process_ids(
                oauth_provider.g2p_id_type, oauth_uid
            )
            phone_numbers, primary_phone = self.process_phones(
                validation.pop("phone", "")
            )
            if primary_phone:
                partner_dict["phone"] = primary_phone
            if phone_numbers:
                partner_dict["phone_number_ids"] = phone_numbers

            partner_dict["image_1920"] = self.process_picture(
                validation.pop("picture", None)
            )

            partner_dict.update(
                self.process_other_fields(
                    validation,
                    oauth_provider.partner_creation_validate_response_mapping,
                )
            )

            return self.env["res.partner"].create(partner_dict)

    def _auth_oauth_rpc(self, endpoint, access_token):
        # This is recreated to suit that application/jwt response type
        if (
            self.env["ir.config_parameter"]
            .sudo()
            .get_param("auth_oauth.authorization_header")
        ):
            response = requests.get(
                endpoint,
                headers={"Authorization": "Bearer %s" % access_token},
                timeout=10,
            )
        else:
            response = requests.get(
                endpoint, params={"access_token": access_token}, timeout=10
            )

        if response.ok:  # nb: could be a successful failure
            if response.headers.get("content-type"):
                # TODO: Improve the following
                if "application/jwt" in response.headers["content-type"]:
                    return jwt.decode(
                        response.text, None, options={"verify_signature": False}
                    )
                if "application/json" in response.headers["content-type"]:
                    return response.json()
        auth_challenge = werkzeug.http.parse_www_authenticate_header(
            response.headers.get("WWW-Authenticate")
        )
        if auth_challenge.type == "bearer" and "error" in auth_challenge:
            return dict(auth_challenge)

        return {"error": "invalid_request"}

    def process_gender(self, gender):
        return gender.capitalize()

    def process_birthdate(self, birthdate, date_format="%Y/%m/%d"):
        if not birthdate:
            return None
        return datetime.strptime(birthdate, date_format).date()

    def process_name(self, family_name, given_name, addl_name):
        name = ""
        if family_name:
            name += family_name + ", "
        if given_name:
            name += given_name + " "
        if addl_name:
            name += addl_name + " "
        return name.upper()

    def process_ids(self, id_type, id_value, expiry_date=None):
        return [
            (
                0,
                0,
                {
                    "id_type": id_type.id,
                    "value": id_value,
                    "expiry_date": expiry_date,
                },
            )
        ]

    def process_phones(self, phone):
        phone_numbers = []
        if phone:
            phone_numbers.append(
                (
                    0,
                    0,
                    {
                        "phone_no": phone,
                    },
                )
            )
        return phone_numbers, phone

    def process_picture(self, picture):
        image_parsed = None
        if picture:
            with urlopen(picture) as response:
                image_parsed = base64.b64encode(response.read())
        return image_parsed

    def process_other_fields(self, validation: dict, mapping: str):
        res = {}
        all_fields = [pair.split(":")[0].strip() for pair in mapping.split(" ")]
        for key in list(validation):
            if key in all_fields and key in self.env["res.partner"]._fields:
                value = validation.pop(key)
                if isinstance(value, dict) or isinstance(value, list):
                    res[key] = json.dumps(value)
                else:
                    res[key] = value
        return res