from azureml.core import (
    Workspace,
    Experiment,
    Environment,
    RunConfiguration,
    ScriptRunConfig,
)

#Defines compute instance
compute_name = 'hackcompute22'

#Defines the name of experiment and environment
experiment_name = 'Hack-experiment'
environment_name = 'Hack-environment'

#Defines name of model and where it is saved
model_name = 'model.pkl'
model_path = 'outputs/model.pkl'

#Defines  directory to run the experiment from
source_directory = '.'

#Defines the entry script for the experiment
script_path = 'Hack Partners/traffic.py'

#Defines the location of the machine learning workspace
subscription_id = '4f67948b-2ff9-49ee-bf1f-90c32dc7545e'
resource_group  = 'cloud-shell-storage-westeurope'
workspace_name  = 'Hack_Workspace'

#Connect to workspace
ws = Workspace(subscription_id = subscription_id, resource_group = resource_group, workspace_name = workspace_name)

#Uses workspace to create an experiment
exp = Experiment(workspace=ws, name=experiment_name)

#Creates environment with the packages needed
env = Environment(name=environment_name)

for pip_package in ['numpy','pandas','install torch==1.8.1+cpu torchvision==0.9.1+cpu torchaudio==0.8.1 -f https://download.pytorch.org/whl/torch_stable.html', 'torch torchvision torchaudio', 'matplotlib','requirements.txt']:
    env.python.conda_dependencies.add_pip_package(pip_package)

#Creates a run configuration to connect environment and compute
run_config = RunConfiguration()
run_config.target = compute_name
run_config.environment = env

#Creates a script run config to tie all the elements together
config = ScriptRunConfig(
    source_directory=source_directory, script=script_path, run_config=run_config
)

# submitting the experiment to start it
run = exp.submit(config)

#waits for completion and shows output of the experiment
run.wait_for_completion(show_output=True, wait_post_processing=True)

#Registers the model with the experiment
run.register_model(model_name=model_name, model_path=model_path)


import sys
import pygame
import copy
from settings import *
from player import *
from Enemy import *

class Program:

    pygame.init()
    vec = pygame.math.Vector2  # for 2-D surface

    def __init__(self):
        self.window = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT))
        self.clock = pygame.time.Clock()  # to control the frames per second( to keep track of time)
        self.running = True
        self.state = "Start"
        self.cell_width = MAZE_WIDTH // COLUMNS
        self.cell_height = MAZE_HEIGHT // ROWS
        self.walls = []
        self.coins=[]
        self.enemies=[]
        self.enemy_pos = []
        self.player_pos=None
        self.load()
        self.player=Player(self,vec(self.player_pos))
        self.make_enemies()

    def run(self):
        while self.running:
            if self.state == "Start":
                self.start_events()
                self.start_update()
                self.start_draw()
            elif self.state == "playing":
                self.playing_events()
                self.playing_update()
                self.playing_draw()
            elif self.state == "game over":
                self.game_over_events()
                self.game_over_update()
                self.game_over_draw()
            else:  # so the game doesn't crash
                self.running = False
            self.clock.tick(FPS)  # This can be used to help limit the runtime speed of a game

        pygame.quit()
        sys.exit()

    # ############################################ HELP FUNCTIONS #########################################

    def draw_text(self, words, screen,pos, size,color,font_name, centered=False, custom_font=False):
        if custom_font:
            custom_font = pygame.font.Font(font_name, size)
            text = custom_font.render(words, True, color)
            text_size=text.get_size()
        else:
            system_font = pygame.font.SysFont(font_name, size)
            text = system_font.render(words, True, color)
            text_size = text.get_size()
        if centered:
            pos[0] = pos[0] - text_size[0]//2
            pos[1] = pos[1] - text_size[1]//2
            self.window.blit(text, pos)

    def load(self):
        self.start_maze = pygame.image.load("images\\start_maze.jpg")
        self.start_maze = pygame.transform.scale(self.start_maze, (WINDOW_WIDTH, WINDOW_HEIGHT))
        self.background = pygame.image.load("images\\maze.png")
        self.background = pygame.transform.scale(self.background, (MAZE_WIDTH, MAZE_HEIGHT))

        # we are opening walls file
        # we are creating walls list with cordinate of walls
        with open("walls.txt", 'r') as file:
            for yidx, line in enumerate(file):
                for xidx, char in enumerate(line):
                    if char == "1":
                        self.walls.append(vec(xidx, yidx))
                    elif char == "C":
                        self.coins.append(vec(xidx, yidx))
                    elif char == "P":
                        self.player_pos = [xidx, yidx]
                    elif char in ["2", "3", "4", "5"]:
                        self.enemy_pos.append([xidx, yidx])
                    elif char == "B":
                        pygame.draw.rect(self.background, BLACK, (xidx*self.cell_width, yidx*self.cell_height,
                                                                  self.cell_width, self.cell_height))

        print(self.walls)
    def make_enemies(self):
        for idx,pos in enumerate(self.enemy_pos):
            self.enemies.append(Enemy(self,vec(pos),idx))

    def draw_grid(self):
        for x in range(WINDOW_WIDTH//self.cell_width):
            pygame.draw.line(self.background, YELLOW, (x * self.cell_width, 0), (x * self.cell_width, WINDOW_HEIGHT))
        for x in range(WINDOW_HEIGHT // self.cell_height):
            pygame.draw.line(self.background, YELLOW, (0, x * self.cell_height), (WINDOW_WIDTH, x * self.cell_height))
        # for coin in self.coins:
        #     pygame.draw.rect(self.background,YELLOW,(coin.x*self.cell_width,
        #                      coin.y*self.cell_height,self.cell_width,self.cell_height))

    def reset(self):
        self.player.lives = 3
        self.player.current_score = 0
        self.player.grid_pos = vec(self.player.starting_pos)
        self.player.pix_pos = self.player.get_pix_pos()
        self.player.direction *= 0
        for enemy in self.enemies:
            enemy.grid_pos = vec(enemy.starting_pos)
            enemy.pix_pos = enemy.get_pix_pos()
            enemy.direction *= 0

        self.coins = []
        with open("walls.txt", 'r') as file:
            for yidx, line in enumerate(file):
                for xidx, char in enumerate(line):
                    if char == 'C':
                        self.coins.append(vec(xidx, yidx))
        self.state = "playing"
    # ############################################ START FUNCTIONS #########################################
    def start_events(self):
        for events in pygame.event.get():
            if events.type == pygame.QUIT:
                self.running = False
            if events.type == pygame.KEYDOWN or events.type == pygame.K_SPACE:
                self.state = "playing"

    def start_update(self):
        pass

    def start_draw(self):
        self.window.fill(BLACK)
        self.window.blit(self.start_maze, (0, WINDOW_HEIGHT // 2 + 50))
        self.draw_text("PAC-MAN",self.window,[WINDOW_WIDTH//2,WINDOW_HEIGHT//2], PAC_MAN,WHITE,PAC_MAN_FONT, centered=True, custom_font=True)
        # self.draw_text("HIGHEST SCORE ",self.window,[300,40], HIGHEST_SCORE,WHITE,PAC_MAN_FONT, centered=True, custom_font=True)

        pygame.display.set_caption("PAC_MAN Start")
        pygame.display.update()

    # ############################################ PLAYING FUNCTIONS #########################################
    def playing_events(self):
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                self.running = False
            if event.type== pygame.KEYDOWN:
                if event.key==pygame.K_LEFT:
                    self.player.move(vec(-1,0))  # one step back is minus
                if event.key==pygame.K_RIGHT:
                    self.player.move(vec(1, 0))
                if event.key==pygame.K_UP:
                    self.player.move(vec(0, -1))
                if event.key==pygame.K_DOWN:
                    self.player.move(vec(0, 1))
    def playing_update(self):
        self.player.update()
        for enemy in self.enemies:
            enemy.update()
        for enemy in self.enemies:
            if enemy.grid_pos == self.player.grid_pos:
                self.remove_life()

    def playing_draw(self):

        self.window.fill(BLACK)
        self.window.blit(self.background, (TOP_BOTTOM_BUFFER//2, TOP_BOTTOM_BUFFER//2))
        self.draw_coins()
        #self.draw_grid()
        self.draw_text("CURRENT SCORE: {}".format(self.player.current_score),self.window,[120 ,15],16,GRAY,"arial", centered=True, custom_font=False)
        self.draw_text("HIGHEST SCORE: ",self.window,[450 ,15],16,GRAY,"arial", centered=True, custom_font=False)
        pygame.display.set_caption("PAC_MAN playing")
        self.player.draw()
        for enemy in self.enemies:
            enemy.draw()
        pygame.display.update()
    def remove_life(self):
        self.player.lives -= 1
        if self.player.lives == 0:
            self.state = "game over"
        else:
            self.player.grid_pos = vec(self.player.starting_pos)
            self.player.pix_pos = self.player.get_pix_pos()
            self.player.direction *= 0
            for enemy in self.enemies:
                enemy.grid_pos = vec(enemy.starting_pos)
                enemy.pix_pos = enemy.get_pix_pos()
                enemy.direction *= 0

    def draw_coins(self):
        for coin in self.coins:
            pygame.draw.circle(self.window,COIN_COLOR,(int(coin.x*self.cell_width)+self.cell_width//2+TOP_BOTTOM_BUFFER//2,
                                                int(coin.y*self.cell_height)+self.cell_width//2+TOP_BOTTOM_BUFFER//2),5)

# ############################################ GAME OVER FUNCTIONS #########################################
    def game_over_events(self):
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                self.running = False
            if event.type == pygame.KEYDOWN and event.key == pygame.K_SPACE:
                self.reset()
            if event.type == pygame.KEYDOWN and event.key == pygame.K_ESCAPE:
                self.running = False

    def game_over_update(self):
        pass

    def game_over_draw(self):
        self.window.fill(BLACK)
        quit_text = "Press the escape button to QUIT"
        again_text = "Press SPACE bar to PLAY AGAIN"
        self.draw_text("GAME OVER", self.window,[WINDOW_WIDTH//2, 100], 52,WHITE,"arial", centered=True, custom_font=False)
        self.draw_text(again_text, self.window, [
                       WINDOW_WIDTH//2, WINDOW_HEIGHT//2],  36, (255, 212, 128), "arial", centered=True, custom_font=False)
        self.draw_text(quit_text, self.window, [
                       WINDOW_WIDTH//2, WINDOW_HEIGHT//1.5],  36, (255, 212, 128), "arial", centered=True,custom_font=False)
        pygame.display.update()


import random
import time

from PySide2.QtWidgets import *
from functools import partial
from qt_material import apply_stylesheet, list_themes, QtStyleTools
from JasonUI.buttons import DarkerButton

class NineBoxDraw(QWidget):
    def __init__(self, parent=None):
        super().__init__(parent)
        self.order = [(0, 0), (0, 1), (0, 2), (1, 2), (2, 2), (2, 1), (2, 0), (1, 0)]
        self.current_position = 0
        self.buttons = [[QPushButton("奖品") for j in range(3)] for i in range(9)]
        self.buttons[1][1] = DarkerButton()
        self.buttons[1][1].setStyleSheet(self.buttons[1][1].styleSheet() + "\nQPushButton{color: gray;}")
        self.buttons[1][1].setText("抽奖！")
        self.buttons[1][1].setFixedSize(100, 100)
        self.buttons[1][1].clicked.connect(self.draw)
        self.grid = QGridLayout(self)
        for line in range(3):
            for col in range(3):
                self.grid.addWidget(self.buttons[line][col], line, col)
        for (row, col) in self.order:
            self.buttons[row][col].clicked.connect(partial(self.on_text_edit, row, col))
            self.buttons[row][col].setFixedSize(100, 100)

    def on_text_edit(self, row, col):
        d = EditDialog(row, col, self)
        d.show()

    def draw(self):
        index = self.current_position
        row, col = self.order[index]
        self.buttons[row][col].setStyleSheet(self.buttons[row][col].styleSheet().rstrip("\nQPushButton{"
                                                                                "background-color: pink;}"))
        stop_point = random.randint(1, 8)
        for i in range(self.current_position, 50+stop_point):
            index = i % 8;
            row, col = self.order[index]
            button_to_shine = self.buttons[row][col]
            button_to_shine.setStyleSheet(button_to_shine.styleSheet() + "\nQPushButton{background-color: pink;}")
            QApplication.processEvents()
            if 0.2 - i*0.01 >= 0.03:
                time.sleep(0.2 - i*0.01)
            elif 50+stop_point - i <= 12:
                time.sleep(0.03+(12 - 50+stop_point + i)*0.01)
            else:
                time.sleep(0.03)
            if i != 50+stop_point - 1:
                button_to_shine.setStyleSheet(button_to_shine.styleSheet().rstrip("\nQPushButton{background-color: pink;}"))
        self.current_position = i % 8

class EditDialog(QDialog):
    def __init__(self, row, col, parent=None):
        super().__init__(parent)
        self.coordinates = (row, col)
        self.vbox = QVBoxLayout(self)
        self.label = QLabel("请输入奖品名称：")
        self.line_edit = QLineEdit(self)
        self.btn = QPushButton("确定")
        self.btn.clicked.connect(self.quit)
        self.vbox.addWidget(self.label)
        self.vbox.addWidget(self.line_edit)
        self.vbox.addWidget(self.btn)

    def quit(self):
        self.parentWidget().buttons[self.coordinates[0]][self.coordinates[1]].setText(self.line_edit.text())
        self.close()

class SelectBatchDialog(QDialog):
    def __init__(self, parent=None):
        super().__init__(parent)
        self.vbox = QVBoxLayout(self)
        self.label = QLabel("在这里批量添加奖品名称，一行一个奖品，共八个，从左上角开始顺时针顺序")
        self.text_edit = QTextEdit(self)
        self.submit = QPushButton("确定")
        self.vbox.addWidget(self.label)
        self.vbox.addWidget(self.text_edit)
        self.vbox.addWidget(self.submit)
        self.submit.clicked.connect(self.submit_names)

    def submit_names(self):
        text = self.text_edit.toPlainText().strip().split('\n')
        for index in range(8):
            w = self.parentWidget().w
            row, col = w.order[index]
            w.buttons[row][col].setText(text[index].strip())
        self.close()

class AboutDialog(QDialog):
    def __init__(self, parent=None):
        super().__init__(parent)
        self.vbox = QVBoxLayout(self)
        self.label = QLabel("这是关于作者、本程序，以及本程序的使用条款")
        self.text_edit = QTextEdit(self)
        self.text_edit.setHtml("
本程序的作者是河北省保定市的一名中学生，"
                               "欢迎通过jasoncoder@qq.com与作者联系，或进行关于编程问题的交流探讨。
"
                               "这个程序受版权保护，不得随意进行修改。未经作者同意，不得商用。
"
                               "这个程序是完全免费且开源的，可以"
                               "在"
                               "https://github.com/Jason-XII/Nine-Box-Draw上查看"
                               "此项目的源代码，以及更为详细的使用说明。
")
        self.text_edit.setReadOnly(True)
        self.submit = QPushButton("我已阅读")
        self.vbox.addWidget(self.label)
        self.vbox.addWidget(self.text_edit)
        self.vbox.addWidget(self.submit)

class MainWindow(QMainWindow, QtStyleTools):
    def __init__(self):
        super().__init__()
        self.setWindowTitle("九宫格抽奖程序")
        self.w = NineBoxDraw(self)
        self.setCentralWidget(self.w)
        self.menu = self.menuBar()
        self.more_menu = self.menu.addMenu("更多……")
        self.add_more = self.more_menu.addAction("批量添加奖品名称")
        self.add_more.triggered.connect(self.select_names)
        self.themes_menu = self.more_menu.addMenu("设置程序外观")
        self.about = self.more_menu.addAction("关于")
        self.about.triggered.connect(self.about_toggled)
        self.ac = []
        t = list_themes() + ['default']
        for theme_name in t:
            act = self.themes_menu.addAction(theme_name)
            act.triggered.connect(partial(self.set_theme, theme_name, t.index(theme_name)))
            act.setCheckable(True)
            self.ac.append(act)
        self.ac[-1].setChecked(True)
        self.checked_index = len(t) - 1

    def select_names(self):
        d = SelectBatchDialog(self)
        d.show()

    def set_theme(self, theme_name, index):
        self.apply_stylesheet(self, theme_name)
        self.ac[self.checked_index].setChecked(False)
        self.checked_index = index

    def about_toggled(self):
        d = AboutDialog(self)
        d.show()

app = QApplication([])
window = MainWindow()

# setup stylesheet
extra = {

    # Button colors
    'danger': '#dc3545',
    'warning': '#ffc107',
    'success': '#17a2b8',
}
apply_stylesheet(app, theme='default', extra=extra)

# run
window.show()
app.exec_()


from django.test import TestCase
from route .serializer import ManagerSerializer,MerchandiserSerializer,RouteSerializer
from route.models import Manager, Merchandiser,Address

class ManagerSerializerTest(TestCase):
    def setUp(self):
        self.manager_attributes ={
            'name':'melody',
            'description':' branch manager',
            'phone_number':'+28390129',
            'location':'Narok'
        }
        self.serializer_data ={
            'name':'melody',
            'description':' branch manager',
            'phone_number':'+28390129',
            'location':'Narok'
        }
        self.manager = Manager.objects.create(**self.manager_attributes)
        self.serializer = ManagerSerializer(instance=self.manager)

    def test_contains_expected_fields(self):
        data = self.serializer.data
        self.assertEqual(set(data.keys()),set(['name','description','phone_number','location']))

class MerchandiserSerializerTest(TestCase):
    def setUp(self):
        self.merchandiser_attributes ={
            'username':'melody_towett',
            'phone_number':'089708643',
            'email':'melody@gmail.com',
            'location':'Narok'
        }
        self.serializer_data ={
            'username':'melody_towett',
            'phone_number':'089708643',
            'email':'melody@gmail.com',
            'location':'Narok'
        }
        self.merchandiser = Merchandiser.objects.create(**self.merchandiser_attributes)
        self.serializer = MerchandiserSerializer(instance=self.merchandiser)

    def test_contains_expected_fields(self):
        data = self.serializer.data
        self.assertEqual(set(data.keys()),set(['username','email','phone_number','location']))

class RouteSerializerTest(TestCase):
    def setUp(self):
        self.route_attribute ={
            'city':'Nairobi',
            'location':'Kenya'
        }
        self.serializer_data ={
            'city':'Nairobi',
            'location':'Kenya'
        }
        self.route = Address.objects.create(**self.route_attribute)
        self.serializer = RouteSerializer(instance=self.route)

    def test_contains_expected_fields(self):
        data = self.serializer.data
        self.assertEqual(set(data.keys()),set(['city','location']))


import os
from dotenv import load_dotenv
import psycopg2
import tempfile
import pandas as pd
import numpy as np
import requests
load_dotenv()

def get_ask_connection():
    CONN_STRING = os.getenv("PSYCOPG2_CONN_STRING")
    return psycopg2.connect(CONN_STRING, port = 5433)
def player_exists_at_db(player_name: str):
    result = open_request(f"""
                    select player_name from player
                    where player_name='{player_name}'
                            """)
    if len(result) == 1:
        return True
    return False
def number_of_games(player_name):
    n_games = open_request(f"""
            SELECT COUNT(game) FROM game
            where game.black = '{player_name}'
            or game.white = '{player_name}';
            """)
    return n_games[0][0]
def open_request(sql_question:str):
    conn = get_ask_connection()
    with conn.cursor() as curs:
        curs.execute(
            sql_question
        )
        result = curs.fetchall()
    return result
def read_sql_tmpfile(query):
    conn = get_ask_connection()
    with tempfile.TemporaryFile() as tmpfile:
        copy_sql = "COPY ({query}) TO STDOUT WITH CSV {head}".format(
           query=query, head="HEADER"
        )
        cur = conn.cursor()
        cur.copy_expert(copy_sql, tmpfile)
        tmpfile.seek(0)
        df = pd.read_csv(tmpfile)
        return df
def get_games_for_month(player_name):
    dates =read_sql_tmpfile(f"""
            SELECT year, month FROM game
            Where game.black = '{player_name}'
            OR game.white = '{player_name}'
            """)
    dates = dates.pivot_table(index = ['year', 'month'], aggfunc ='size')
    dates_hist = np.array(
        [
            [
                f'{x[0]}_{x[1]}',
                int(dates[x])            ]
            for x in dates.index
        ]
    )
    return dates_hist
def post_profile(player_name):
    POST_PLAYER = os.getenv("POST_PLAYER_STRING")
    myobj = {"player_name":player_name}
    response = requests.post(POST_PLAYER, json = myobj)
    response = response.content.decode('utf-8')
    if 'RESPONSE' in response:
        return False
    return True
def read_pd_profile(player_name):
    profile = read_sql_tmpfile(f"""
            SELECT * FROM player
            Where player.player_name = '{player_name}'
            """)
    return profile.to_dict()
def get_profile(player_name):
    player_name = player_name.lower()
    if not player_exists_at_db(player_name):
        response = post_profile(player_name)
        if not response:
            return f'No such player {player_name} exists'
    profile = read_pd_profile(player_name)
    if profile['joined'] == 0:
        post_profile(player_name)
        profile = read_pd_profile(player_name)
    return profile


#!/usr/bin/env python
'''
.. module:: preprocess_data

Contains functions for normalizing and converting the raw data.

.. moduleauthor:: Rasmus Diederichsen
'''

from argparse import ArgumentParser
import numpy as np
import sys

from utils import compute_rgb_mean
from data_manager import DataManager

def main():
    parser = ArgumentParser('Preprocess robot data vor DeepVO, This process is destructive.')
    parser.add_argument('-d', '--data', type=str, required=True,
                        help='Path to dataset (a folder with "images" and "poses" subfolders.)')
    parser.add_argument('-f', '--to-float', required=False, default=False,
                        action='store_true', help='Convert images array to float')
    parser.add_argument('-m', '--mean-normalize', required=False, default=False,
                        action='store_true', help='Subtract rgb mean from images')
    parser.add_argument('-s', '--show', required=False, default=False,
                        action='store_true', help='Show the images')
    parser.add_argument('-p', '--pose', required=False, default=False,
                        action='store_true', help='Add pi to poses (for range 0-2pi)')
    parser.add_argument('-sp', '--subpi', required=False, default=False,
                        action='store_true', help='Add pi to poses (for range -pi - +pi)')
    args = parser.parse_args()

    data_manager = DataManager(args.data, dtype=np.float32, batch_size=1, sequence_length=1)
    if args.to_float:
        to_float(data_manager)

    if args.mean_normalize:
        mean_normalize(data_manager)

    if args.show:
        show_imgs(data_manager)

    if args.pose:
        add_pi_to_poses(data_manager)

    if args.subpi:
        sub_pi_from_poses(data_manager)

def show_imgs(data_manager):
    from matplotlib import pyplot as plt
    N = len(data_manager)
    for idx in range(N):
        img = data_manager.loadImage(idx)
        minimum, maximum = img.min(), img.max()
        print(f'Range: ({minimum}, {maximum})')
        plt.imshow((img - minimum) / (maximum - minimum))
        plt.show()

def to_float(data_manager):
    '''Convert dataset to range (0, 1)'''
    N = len(data_manager)
    print(f'Converting {data_manager.dataset_path}images/*.npy fo float ...')
    for idx in range(N):

        # print progress
        if idx % 10 == 0:
            print(f'\r{idx+1:4d}/{N}', end='')

        img = data_manager.loadImage(idx) / 255.
        data_manager.saveImage(idx, img.astype(np.float32))
    print('\nDone')

def mean_normalize(data_manager):
    '''Normalize data to the range -1 to 1'''
    assert data_manager.dtype == np.float32
    N = len(data_manager)
    print(f'Mean-normalizing {data_manager.dataset_path}/images/*.npy ...')
    mean_accumlator = np.zeros((3,), dtype=np.float32)

    # run over entire dataset to compute mean (fucking inefficient but I have other shit to do)
    for idx in range(N):
        img = data_manager.loadImage(idx)
        mean_accumlator += compute_rgb_mean(img)

    mean_accumlator /= N
    print(f'Mean: {mean_accumlator}')
    for idx in range(N):

        if idx % 10 == 0:
            print(f'\r{idx+1:4d}/{N}', end='')

        img = data_manager.loadImage(idx)
        data_manager.saveImage(idx, (img - mean_accumlator))
    print('\nDone')

def add_pi_to_poses(data_manager):
    '''Add Pi to every pose angle.'''
    N = len(data_manager)
    for idx in range(N):
        pose = data_manager.loadPose(idx)
        pose[..., 3:6] = pose[..., 3:6] + np.pi
        data_manager.savePose(idx, pose)

def sub_pi_from_poses(data_manager):
    '''Subtract Pi from every pose angle.'''
    N = len(data_manager)
    for idx in range(N):
        pose = data_manager.loadPose(idx)
        pose[..., 3:6] = pose[..., 3:6] - np.pi
        data_manager.savePose(idx, pose)

if __name__ == '__main__':
    main()


import pprint
import json
import itertools
import urllib.parse
import tqdm

from app import db
from app import app
from app.models import AlternateNames
from app.models import AlternateNamesChanges
from app.models import AlternateTranslatorNames
from app.models import AlternateTranslatorNamesChanges
from app.models import Author
from app.models import AuthorChanges
from app.models import Genres
from app.models import GenresChanges
from app.models import Illustrators
from app.models import IllustratorsChanges
from app.models import Series
from app.models import SeriesChanges
from app.models import Tags
from app.models import TagsChanges
from app.models import Translators
from app.models import Ratings

from app.models import Covers
from app.models import CoversChanges

from app.models import Releases
from app.models import ReleasesChanges
from app.models import Watches
from app.models import Publishers
from app.models import PublishersChanges

from sqlalchemy import or_

from flask import flash
from flask_babel import gettext
from flask_login import current_user

from flask_login import current_user
import app.series_tools
from app.api_common import getResponse

import FeedFeeder.FeedFeeder
import sqlalchemy.exc

import util.text_tools as text_tools

import app.api_handlers

def setSortOrder(data):
	if not current_user.is_mod():
		return getResponse(error=True, message="You have to have moderator privileges to do that!")

	# Json request: {'item-id': '32606', 'mode': 'set-sort-mode', 'sort-mode': 'chronological_order'}
	assert data['mode'] == 'set-sort-mode'
	assert 'item-id' in data
	assert 'sort-mode' in data
	mid = int(data['item-id'])
	mode = data['sort-mode']

	assert mode in ['chronological_order', 'parsed_title_order']
	itm = Series.query.filter(Series.id==mid).one()

	print(itm.sort_mode)
	print(itm)

	itm.sort_mode = mode

	db.session.commit()

	return getResponse("Success", False)

	# return getResponse(error=True, message="lolwut!")

def get_config_json():
	val = {}
	try:
		ret = json.load(open("volatile_config.json"))
		if isinstance(ret, dict):
			val = ret
	except FileNotFoundError:
		val = {}
	except json.JSONDecodeError:
		val = {}

	val.setdefault("no-merge-series", [])
	val.setdefault("no-merge-groups", [])
	val.setdefault("delete-tags", [])
	return val

def save_config_json(newdat):
	with open("volatile_config.json", "w") as fp:
		json.dump(newdat, fp, indent="	")

def preventMergeSeriesItems(data):
	if not current_user.is_mod():
		return getResponse(error=True, message="You have to have moderator privileges to do that!")

	assert 'mode' in data
	assert data['mode'] == 'block-merge-id'
	assert 'item-id' in data
	assert 'separate_id' in data

	m1, m2 = int(data['item-id']), int(data['separate_id'])

	m1, m2 = min(m1, m2), max(m1, m2)
	have = get_config_json()

	if not [m1, m2] in have['no-merge-series']:
		have['no-merge-series'].append([m1, m2])
	save_config_json(have)

	return getResponse("Success", False)

def mergeSeriesItems(data):
	if not current_user.is_mod():
		return getResponse(error=True, message="You have to have moderator privileges to do that!")

	assert 'mode' in data
	assert data['mode'] == 'merge-id'
	assert 'item-id' in data
	assert 'merge_id' in data

	m1, m2 = int(data['item-id']), int(data['merge_id'])
	return merge_series_ids(m1, m2)

def preventMergeGroupEntries(data):
	if not current_user.is_mod():
		return getResponse(error=True, message="You have to have moderator privileges to do that!")

	assert 'mode' in data
	assert data['mode'] == 'block-group-merge-id'
	assert 'item-id' in data
	assert 'separate_id' in data

	m1, m2 = int(data['item-id']), int(data['separate_id'])

	m1, m2 = min(m1, m2), max(m1, m2)
	have = get_config_json()

	if not [m1, m2] in have['no-merge-groups']:
		have['no-merge-groups'].append([m1, m2])
	save_config_json(have)

	return getResponse("Success", False)

def mergeGroupEntries(data):
	if not current_user.is_mod():
		return getResponse(error=True, message="You have to have moderator privileges to do that!")

	assert 'mode' in data
	assert data['mode'] == 'do-group-merge-id'
	assert 'item-id' in data
	assert 'merge_id' in data

	m1, m2 = int(data['item-id']), int(data['merge_id'])
	return merge_tl_group_ids(m1, m2)

def mergeGroupItems(data):
	if not current_user.is_mod():
		return getResponse(error=True, message="You have to have moderator privileges to do that!")

	assert 'mode' in data
	assert data['mode'] == 'merge-group'
	assert 'item-id' in data
	assert 'merge_id' in data

	m1, m2 = int(data['item-id']), int(data['merge_id'])

	return merge_tl_group_ids(m1, m2)

def merge_tl_group_ids(m1, m2):
	merge_from = max(m1, m2)
	merge_to   = min(m1, m2)

	itm_from = Translators.query.filter(Translators.id==merge_from).one()
	itm_to   = Translators.query.filter(Translators.id==merge_to).one()

	print(itm_from)
	print(itm_to)

	Releases       .query.filter(Releases.tlgroup       ==merge_from).update({'tlgroup': merge_to})
	ReleasesChanges.query.filter(ReleasesChanges.tlgroup==merge_from).update({'tlgroup': merge_to})

	app.api_handlers.updateGroupAltNames(itm_to, [itm.name for itm in itm_from.alt_names], delete=False)

	AlternateTranslatorNamesChanges.query                   \
		.filter(AlternateTranslatorNamesChanges.group==itm_from.id) \
		.delete(synchronize_session="fetch")

	db.session.delete(itm_from)
	db.session.commit()

	return getResponse("Success", False)

def merge_series_ids(m1, m2):
	merge_from = max(m1, m2)
	merge_to   = min(m1, m2)

	itm_from = Series.query.filter(Series.id==merge_from).scalar()
	itm_to = Series.query.filter(Series.id==merge_to).scalar()
	if not itm_from:
		return getResponse("Item from (id: {num}) not found?".format(num=merge_from), error=True)
	if not itm_to:
		return getResponse("Item to (id: {num}) not found?".format(num=merge_to), error=True)

	alts   = []
	author = []
	illust = []
	tags   = []
	genres = []
	publs  = []

	alts.append(itm_from.title)
	for altname in AlternateNames.query.filter(AlternateNames.series==itm_from.id).all():
		alts.append(altname.name)

	for val in Author.query.filter(Author.series==itm_from.id).all():
		author.append(val.name)

	for val in Illustrators.query.filter(Illustrators.series==itm_from.id).all():
		illust.append(val.name)

	for val in Tags.query.filter(Tags.series==itm_from.id).all():
		tags.append(val.tag)

	for val in Genres.query.filter(Genres.series==itm_from.id).all():
		genres.append(val.genre)

	for val in Publishers.query.filter(Publishers.series==itm_from.id).all():
		publs.append(val.name)

	Ratings.query.filter(Ratings.series_id==merge_from).update({'series_id': merge_to})

	# !Ordering here matters!
	# Change-tables have to go second.
	delete_from = [
			AlternateNames,
			AlternateNamesChanges,
			Author,
			AuthorChanges,
			Illustrators,
			IllustratorsChanges,
			Tags,
			TagsChanges,
			Genres,
			GenresChanges,
			Publishers,
			PublishersChanges,
		]

	for clearTable in delete_from:
		clearTable.query.filter(clearTable.series==itm_from.id).delete()

	app.series_tools.updateAltNames   ( itm_to, alts,                deleteother=False )
	app.series_tools.setAuthorIllust  ( itm_to, author     = author, deleteother=False )
	app.series_tools.setAuthorIllust  ( itm_to, illust     = illust, deleteother=False )
	app.series_tools.updateTags       ( itm_to, tags       = tags,   deleteother=False )
	app.series_tools.updateGenres     ( itm_to, genres     = genres, deleteother=False )
	app.series_tools.updatePublishers ( itm_to, publishers = publs,  deleteother=False )

	# For each user watch, if the user is already watching the merge-to item,
	# just delete it. If not, update the user-id
	for watch in Watches.query.filter(Watches.series_id==itm_from.id).all():
		if not Watches                              \
				.query                                  \
				.filter(Watches.series_id==itm_to.id)   \
				.filter(Watches.user_id==watch.user_id) \
				.scalar():

			watch.series_id = itm_to.id

		else:
			db.session.delete(watch)

	if itm_from.description and not itm_to.description:
		itm_to.description = itm_from.description

	if itm_from.description and not itm_to.description:
		itm_to.description = itm_from.description
	if itm_from.type and not itm_to.type:
		itm_to.type = itm_from.type
	if itm_from.origin_loc and not itm_to.origin_loc:
		itm_to.origin_loc = itm_from.origin_loc
	if itm_from.demographic and not itm_to.demographic:
		itm_to.demographic = itm_from.demographic
	if itm_from.orig_lang and not itm_to.orig_lang:
		itm_to.orig_lang = itm_from.orig_lang
	if itm_from.region and not itm_to.region:
		itm_to.region = itm_from.region
	if itm_from.license_en and not itm_to.license_en:
		itm_to.license_en = itm_from.license_en
	if itm_from.orig_status and not itm_to.orig_status:
		itm_to.orig_status = itm_from.orig_status

	if itm_from.pub_date and not itm_to.pub_date:
		itm_to.pub_date = itm_from.pub_date

	if itm_from.website and not itm_to.website:
		itm_to.website = itm_from.website.lower()

	# Merge item links
	elif itm_from.website and itm_to.website:
		websites = list(set(itm_from.website.split("\n") + itm_to.website.split("\n")))
		websites.sort()
		websites = "\n".join(websites)
		itm_to.website = websites

	db.session.flush()
	sid = itm_from.id
	tid = itm_to.id
	Covers.query.filter(Covers.series==sid).update({'series': tid})
	CoversChanges.query.filter(CoversChanges.series==sid).update({'series': tid})

	# Move releases over
	Releases.query.filter(Releases.series==sid).update({'series': tid})
	ReleasesChanges.query.filter(ReleasesChanges.series==sid).update({'series': tid})

	Series.query.filter(Series.id==sid).delete(synchronize_session="fetch")
	SeriesChanges.query.filter(SeriesChanges.id==sid).delete(synchronize_session="fetch")

	db.session.commit()

	app.series_tools.set_rating(sid, new_rating=None)

	return getResponse("Success", False)

def getReleaseFromId(inId):
	ret = Releases.query.filter(Releases.id==inId).one()
	return ret

def toggle_counted(data):
	release = getReleaseFromId(data['id'])
	release.include = not release.include

	app.utilities.update_latest_row(release.series_row)

	db.session.commit()

	flash(gettext('Release %(id)s count-state toggled. New state: %(state)s', id=release.id, state="counted" if release.include else "uncounted"))
	return getResponse("Item count-state toggled!", error=False)

def delete(data):
	release = getReleaseFromId(data['id'])
	db.session.delete(release)
	db.session.commit()
	flash(gettext('Release deleted.'))
	return getResponse("Release deleted.", error=False)

RELEASE_OPS = {
	'toggle-counted' : toggle_counted,
	'delete'         : delete,
}

BOOL_LUT = {
	"True"  : True,
	"False" : False,
}

def alterReleaseItem(data):

	if not current_user.is_mod():
		return getResponse(error=True, message="You have to have moderator privileges to do that!")
	assert 'op' in data
	assert 'mode' in data
	assert 'count' in data
	assert 'id' in data

	assert data['mode'] == "release-ctrl"

	try:
		data['id'] = int(data['id'])
	except ValueError:
		raise AssertionError("Failure converting item ID to integer!")
	assert data['count'] in BOOL_LUT
	data['count'] = BOOL_LUT[data['count']]

	assert data['op'] in RELEASE_OPS

	return RELEASE_OPS[data['op']](data)

def clean_bad_releases(data, admin_override=False):
	badurls = [
		"https://www.novelupdates.com/",
	]

	bad = 0
	for badurl in badurls:
		matches = Releases.query.filter(Releases.srcurl==badurl).all()
		if matches:
			for match in matches:
				bad += 1
				db.session.delete(match)
			db.session.commit()
	print("Deleted %s items." % bad)

def _consolidate_rrl_items():

	dups = db.engine.execute('''
		SELECT
			id, website
		FROM
			series
		WHERE
			website IS NOT NULL AND website != ''
		;''')

	match_num = 0
	paths = {}

	dups = list(dups)

	print("Found %s rows with non-null URLs" % (len(dups), ))
	for idno, website in dups:
		if not ("royalroadl.com" in website.lower() or "royalroad.com" in website.lower()):
			continue

		parsed = urllib.parse.urlsplit(website)
		if not parsed.path in paths:
			paths[parsed.path] = []
		paths[parsed.path].append((idno, website, parsed))

		# matches = Series.query.filter(Series.website==website).all()
		# ids = [match.id for match in matches]
		# zipped = list(zip(ids, ids[1:]))
		# for m1, m2 in zipped:
		# 	match_num += 1
		# 	merge_series_ids(m1, m2)

	long = []
	for key in paths.keys():
		if len(paths[key]) > 1:
			long.append(paths[key])

	for item in long:
		# if not all([(tmp[2].netloc == 'royalroadl.com' or tmp[2].netloc == 'www.royalroadl.com') for tmp in item]):
		if not all((
				all([item[0][2].path   == tmp[2].path for tmp in item]),
				all([item[0][2].query  == tmp[2].query for tmp in item]),
				all([(tmp[2].netloc == 'royalroadl.com' or tmp[2].netloc == 'www.royalroadl.com' or tmp[2].netloc == 'royalroad.com' or tmp[2].netloc == 'www.royalroad.com') for tmp in item]),
			)):
			print("Wat?")
			print([item[0][2].path   == tmp[2].path for tmp in item])
			print([item[0][2].query  == tmp[2].query for tmp in item])
			print([(tmp[2].netloc == 'royalroadl.com' or tmp[2].netloc == 'www.royalroadl.com' or tmp[2].netloc == 'royalroad.com' or tmp[2].netloc == 'www.royalroad.com') for tmp in item])
			print(item)
			assert True == False
		# if not all((
		# 		all([item[0][2].scheme == tmp[2].scheme for tmp in item]),
		# 		all([item[0][2].path   == tmp[2].path for tmp in item]),
		# 		all([item[0][2].query  == tmp[2].query for tmp in item]),
		# 		all([(tmp[2].netloc == 'royalroadl.com' or tmp[2].netloc == 'www.royalroadl.com') for tmp in item]),
		# 	)):
		# 	print("Wut: ", item)
		# 	print("	", all([tmp[2].scheme for tmp in item]))
		# 	print("	", all([tmp[2].path for tmp in item]))
		# 	print("	", all([tmp[2].query for tmp in item]))
		# 	print("	", all([(tmp[2].netloc == 'royalroadl.com' or tmp[2].netloc == 'www.royalroadl.com') for tmp in item]))
		# for sub in item:
		# 	print("	", sub[2])
		print("Need to merge: ", [tmp[0] for tmp in item])

		match_num += 1
		print("merging:", item[0][0], item[1][0])
		merge_series_ids(item[0][0], item[1][0])
		db.session.commit()

	print("Merged: ", match_num)
	return match_num

def consolidate_rrl_items(data, admin_override=False):
	if admin_override is False and (not current_user.is_mod()):
		return getResponse(error=True, message="You have to have moderator privileges to do that!")

	# We repeatedly execute the merge until it no longer results in any changes.
	match_num = 0
	merged = _consolidate_rrl_items()
	while merged:
		match_num += merged
		merged = _consolidate_rrl_items()

	return getResponse("%s Items merged." % match_num, error=False)

def dump_results(res_list):
	for row in res_list:
		print("	https://www.wlnupdates.com/series-id/%s/ , %s" % (row.series, (
					row.volume, row.chapter, row.fragment, row.postfix
				)
			))

	pass

class MatchTracker():
	def __init__(self):
		self.map = {}

	def add_match(self, s1, s2):

		assert isinstance(s1, int)
		assert isinstance(s2, int)

		if s1 == s2:
			return

		s1, s2 = min((s1, s2)), max((s1, s2))
		self.map.setdefault((s1, s2), 0)
		self.map[(s1, s2)] += 1

def pairwise(iterable):
	"s -> (s0,s1), (s1,s2), (s2, s3), ..."
	a, b = itertools.tee(iterable)
	next(b, None)
	return zip(a, b)

def _update_consolidate_release(mt, from_url, new_url):
	old = Releases.query.filter(Releases.srcurl == from_url).all()
	new = Releases.query.filter(Releases.srcurl == new_url).all()

	if not old:
		print("What?", from_url, new_url)
		return

	if len(old) > 1:
		# print("Multiple old results: ", from_url)
		# dump_results(old)
		map(mt.add_match, pairwise([tmp.series for tmp in old]))
		if new:
			map(mt.add_match, pairwise([tmp.series for tmp in old+new]))
			map(mt.add_match, pairwise([tmp.series for tmp in new+old]))
		return
	if len(new) > 1:
		# print("Multiple new results: ", new_url)
		# dump_results(new)
		map(mt.add_match, pairwise([tmp.series for tmp in new]))
		if old:
			map(mt.add_match, pairwise([tmp.series for tmp in old+new]))
			map(mt.add_match, pairwise([tmp.series for tmp in new+old]))
		return

	if old:
		old, = old
	if new:
		new, = new

	if old and not new:
		old.srcurl = new_url
		db.session.commit()
		return

	if old and new:
		if old.series == new.series:
			db.session.delete(old)
			db.session.commit()
			return

		mt.add_match(old.series, new.series)
		# print("Series mismatch! https://www.wlnupdates.com/series-id/%s , https://www.wlnupdates.com/series-id/%s" % (old.series, new.series))

def consolidate_rrl_releases(data, admin_override=False):
	if admin_override is False and (not current_user.is_mod()):
		return getResponse(error=True, message="You have to have moderator privileges to do that!")

	rrl = Translators.query.filter(Translators.name == "RoyalRoadL").scalar()

	print("RRL TL Group")
	print(rrl)

	mt = MatchTracker()

	print("Total releases:")
	print(Releases.query.filter(Releases.tlgroup == rrl.id).count())
	print("Loading releases....")
	db.session.execute("BEGIN;")
	urls = Releases.query.with_entities(Releases.srcurl).filter(Releases.tlgroup == rrl.id).all()
	try:
		for srcurl, in tqdm.tqdm(urls):
			if 'http://royalroadl.com/forum/showthread.php?tid=' in srcurl:
				to_delete = Releases.query.filter(Releases.srcurl == srcurl).all()
				for bad in to_delete:
					db.session.delete(bad)
				continue

			fixed = text_tools.clean_fix_url(srcurl)

			if fixed != srcurl:
				to_fix = Releases.query.filter(Releases.srcurl == srcurl).all()
				for row in to_fix:
					row.srcurl = fixed

				db.session.commit()

				# _update_consolidate_release(mt, srcurl, fixed)
				# print("Change: %s <- %s" % (fixed, release.srcurl))
	finally:
		with open("rrl_match_items.json", "w") as fp:
			json.dump(mt.map, fp=fp, indent=4)

	match_num = 0
	return getResponse("%s Items merged." % match_num, error=False)

def flatten_series_by_url(data, admin_override=False):
	if admin_override is False and (not current_user.is_mod()):
		return getResponse(error=True, message="You have to have moderator privileges to do that!")

	dups = db.engine.execute('''
		SELECT
			website, COUNT(*) AS dupes
		FROM
			series
		WHERE
			website IS NOT NULL AND website != ''
		GROUP
			BY website
		HAVING
			(COUNT(*) > 1);''')
	dups = list(dups)

	match_num = 0
	for website, number in dups:
		if not "royalroadl" in website.lower():
			continue

		matches = Series.query.filter(Series.website==website).all()
		ids = [match.id for match in matches]
		zipped = list(zip(ids, ids[1:]))
		for m1, m2 in zipped:
			match_num += 1
			merge_series_ids(m1, m2)

	return getResponse("%s Items merged." % match_num, error=False)

def delete_duplicate_releases(data, admin_override=False):
	if admin_override is False and (not current_user.is_mod()):
		return getResponse(error=True, message="You have to have moderator privileges to do that!")

	dups = db.engine.execute('''
		SELECT
			srcurl, COUNT(*) AS dupes
		FROM
			releases
		WHERE
			srcurl IS NOT NULL AND srcurl != ''
		GROUP
			BY srcurl
		HAVING
			(COUNT(*) > 1);''')
	dups = list(dups)

	match_num = 0
	mismatches = set()
	for release_url, number in dups:
		matches = Releases.query.filter(Releases.srcurl==release_url).all()
		zipped = list(zip(matches, matches[1:]))
		for m1, m2 in zipped:
			if m1.series != m2.series:
				tup = (int(m1.series), int(m2.series))
				if tup not in mismatches:
					print("Mismatch: ", m1.series, m2.series, m1.srcurl, m2.srcurl)
					mismatches.add(tup)
			else:
				match_num += 1
				# print(m1.series, m2.series)

				# Pick the older version, since untimed duplicates keep
				# cropping up from japtem.
				# However, if a newer release has a postfix, go with that instead.
				if m2.postfix and m2.postfix != m1.postfix:
					older = m2
					newer = m1
				elif m1.changetime < m2.changetime:
					older = m1
					newer = m2
				else:
					older = m2
					newer = m1
				if not older.include:
					# If the to-be-removed chapter has been expliclty excluded from
					# the chapter count, don't do the merge.
					pass
				else:
					db.session.delete(newer)
					db.session.commit()

	# print(dups)
	# print(list(dups))
	mismatches = list(mismatches)

	with open("rrl_match_items.json", "w") as fp:
		json.dump(mismatches, fp=fp, indent=4)

	return getResponse("%s Items merged." % match_num, error=False)

def fix_escaped_quotes(dummy_data, admin_override=False):
	if admin_override is False and (not current_user.is_mod()):
		return getResponse(error=True, message="You have to have moderator privileges to do that!")

	# SELECT * FROM series WHERE title LIKE E'%\\\'%';
	bad_title = 0

	q = Series.query.filter(or_(Series.title.like(r"%'%"), Series.title.like(r"%’%"), Series.title.like(r"%‘%"), Series.title.like(r"%“%"), Series.title.like(r"%”%")))
	items = q.all()
	print("Name fixing processing query resulted in %s items" % len(items))
	for item in items:
		old = item.title
		new = old
		while any([r"\"" in new, r"\'" in new, "’" in new, "‘" in new, "“" in new, "”" in new]):
			new = new.replace(r"\'", "'")
			new = new.replace(r'\"', '"')
			new = new.replace(r"’", "'")
			new = new.replace(r"‘", "'")
			new = new.replace(r"“", '"')
			new = new.replace(r"”", '"')

		have = Series.query.filter(Series.title == new).scalar()
		if old != new:
			if have:
				print("Duplicate item!", (old, new), old==new)
				merge_series_ids(have.id, item.id)
			else:
				print("Fixing title.")
				item.title = new
				db.session.commit()
			bad_title += 1

	bad_alt_title = 0

	q = AlternateNames.query.filter(or_(AlternateNames.name.like(r"%'%"), AlternateNames.name.like(r"%’%"), AlternateNames.name.like(r"%‘%"), AlternateNames.name.like(r"%“%"), AlternateNames.name.like(r"%”%")))

	conflicts = ''
	items = q.all()
	print("Alternate names processing query resulted in %s items" % len(items))
	for item in items:
		old = item.name
		new = old
		while any([r"\"" in new, r"\'" in new, "’" in new, "‘" in new, "“" in new, "”" in new]):
			new = new.replace(r"\'", "'")
			new = new.replace(r'\"', '"')
			new = new.replace(r"’", "'")
			new = new.replace(r"‘", "'")
			new = new.replace(r"“", '"')
			new = new.replace(r"”", '"')
		if old != new:
			haves = AlternateNames.query.filter(AlternateNames.name == new).all()
			# assert(len(have) <= 1), "too many results - '%s'" % [(t.id, t.series, t.name) for t in have]

			for have in haves:
				if have.series == item.series:
					print("Duplicate names")
					assert have.series == item.series
					# We don't care about duplicates if one is the escaped version of the other
					db.session.delete(item)
					db.session.commit()
				else:
					conflicts += """\nSeries mismatch?\nSeries %s: '%s'\nSeries %s: '%s'""" % (have.series, have.name, item.series, item.name)
					print("Wat?", have.name, item.name)
					print("Wat?", have.series, item.series)
			else:
				print("Fixing title.")
				item.name = new
				db.session.commit()
			bad_alt_title += 1

	bad_desc = 0

	# FUCK ALL SMART QUOTE BULLSHITS EVER
	q = Series.query.filter(or_(Series.description.like(r"%'%"), Series.description.like(r"%’%"), Series.description.like(r"%‘%"), Series.description.like(r"%“%"), Series.description.like(r"%”%")))

	items = q.all()
	print("Series description processing query resulted in %s items" % len(items))
	for item in items:
		old = item.description
		new = old

		while any([r"\"" in new, r"\'" in new, "’" in new, "‘" in new, "“" in new, "”" in new]):
			new = new.replace(r"\'", "'")
			new = new.replace(r'\"', '"')
			new = new.replace(r"’", "'")
			new = new.replace(r"‘", "'")
			new = new.replace(r"“", '"')
			new = new.replace(r"”", '"')
		if old != new:
			print("Fixing description smart-quotes and over-escapes for series: %s" % item.id)
			item.description = new
			db.session.commit()
			bad_desc += 1

	print("Update complete.")

	return getResponse("%s main titles, %s alt titles, %s descriptions required fixing.%s" % (bad_title, bad_alt_title, bad_desc, conflicts), error=False)

def clean_singleton_tags(dummy_data, admin_override=False):
	if admin_override is False and (not current_user.is_mod()):
		return getResponse(error=True, message="You have to have moderator privileges to do that!")
	bad_tags = 0

	bad_tags = db.session.execute('''
		SELECT
			COUNT(*)
		FROM
			tags
		WHERE
			tag IN (
			SELECT tag
			FROM (
				SELECT tag
				FROM tags
				GROUP BY tag
				HAVING COUNT(*) = 1
			) AS ONLY_ONCE
			)
		''')

	bad_tags = list(bad_tags)

	db.session.execute('''
		DELETE
		FROM
			tags
		WHERE
			tag IN (
			SELECT tag
			FROM (
				SELECT tag
				FROM tags
				GROUP BY tag
				HAVING COUNT(*) = 1
			) AS ONLY_ONCE
			)
		;
		''')
	db.session.commit()

	return getResponse("Found %s tags that required patching." % (bad_tags), error=False)

def delete_bad_tags(dummy_data, admin_override=False):
	if admin_override is False and (not current_user.is_mod()):
		return getResponse(error=True, message="You have to have moderator privileges to do that!")

	conf = get_config_json()
	print("Bad tags:", conf['delete-tags'])
	removed = 0
	for bad_tag in conf['delete-tags']:
		items = Tags.query.filter(Tags.tag == bad_tag).count()
		print("Found %s instances of tag: %s" % (items, bad_tag))
		removed += 1
		Tags.query.filter(Tags.tag == bad_tag).delete()

	save_config_json(conf)
	removed = 0
	return getResponse("Found %s tags that required patching." % (removed), error=False)

def check_merge(fromrow):
	other = Series.query.filter(Series.title == fromrow.title.strip()).one()
	if fromrow.title.strip() == other.title.strip():
		fromauths = [auth.name for auth in fromrow.author]
		toauths   = [auth.name for auth in other.author]
		if (fromrow.tl_type == other.tl_type and
			fromauths == toauths and
			fromauths ):

			print("Wut")

			print("	", (fromrow.title, other.title))
			print("	", (fromrow.tl_type, other.tl_type))
			print("	", (fromauths, toauths))

			print("Doing merge")
			merge_series_ids(fromrow.id, other.id)
			return
	print("Mismatch:")
	print("	URLs:")
	print("		1: https://www.wlnupdates.com/series-id/{}/".format(fromrow.id))
	print("		2: https://www.wlnupdates.com/series-id/{}/".format(other.id))

def clean_spaces(dummy_data, admin_override=False):
	if admin_override is False and (not current_user.is_mod()):
		return getResponse(error=True, message="You have to have moderator privileges to do that!")

	items = Series.query.all()

	for item in items:
		if item.title != item.title.strip():

			item.title = item.title.strip()

			try:
				db.session.commit()
			except sqlalchemy.exc.IntegrityError:
				db.session.rollback()
				check_merge(item)

	# return getResponse("Found %s tags that required patching." % (bad_tags), error=False)

def deleteSeries(data):

	if not current_user.is_mod():
		return getResponse(error=True, message="I see what you (tried) to do there!")
	assert 'item-id' in data
	assert 'mode' in data

	delete_id = data["item-id"]
	clean_item = Series.query.filter(Series.id==delete_id).one()

	# !Ordering here matters!
	# Change-tables have to go second.
	delete_from = [
			AlternateNames,
			AlternateNamesChanges,
			Author,
			AuthorChanges,
			Illustrators,
			IllustratorsChanges,
			Tags,
			TagsChanges,
			Genres,
			GenresChanges,
			Publishers,
			PublishersChanges,
			Covers,
			CoversChanges,
			Releases,
			ReleasesChanges,
			# Series,
			# SeriesChanges,
		]

	for clearTable in delete_from:
		clearTable.query.filter(clearTable.series==clean_item.id).delete()

	Ratings.query.filter(Ratings.series_id==clean_item.id).delete()
	Watches.query.filter(Watches.series_id==clean_item.id).delete()

	Series.query.filter(Series.id==clean_item.id).delete()
	SeriesChanges.query.filter(SeriesChanges.srccol==clean_item.id).delete()
	# db.session.delete(clean_item)
	db.session.commit()

	return getResponse("Series was deleted entirely!", error=False)

def deleteAutoReleases(data):

	if not current_user.is_mod():
		return getResponse(error=True, message="I see what you (tried) to do there!")

	assert 'item-id' in data
	assert 'mode' in data
	assert data['mode'] == "delete-auto-releases"

	try:
		delete_id = int(data["item-id"])
	except ValueError:
		raise AssertionError("Failure converting item ID to integer!")

	clean_item = Series.query.filter(Series.id==delete_id).one()

	print(clean_item)
	for release in clean_item.releases:
		if release.changeuser == FeedFeeder.FeedFeeder.RSS_USER_ID:
			db.session.delete(release)
			# print(release.id, release.volume, release.chapter, release.postfix, release.changeuser)
		else:
			print("Not deleting: ", release.id, release.volume, release.chapter, release.postfix, release.changeuser)

	app.utilities.update_latest_row(clean_item)

	db.session.commit()

	return getResponse("Autogen releases deleted. Reloading.", error=False)

def deleteGroup(data):
	return getResponse("Not implemented yet!", error=True)

	# if not current_user.is_mod():
	# 	return getResponse(error=True, message="I see what you (tried) to do there!")
	# assert 'item-id' in data
	# assert 'mode' in data

	# delete_id = data["item-id"]
	# clean_item = Series.query.filter(Series.id==delete_id).one()

	# # !Ordering here matters!
	# # Change-tables have to go second.
	# delete_from = [
	# 		AlternateNames,
	# 		AlternateNamesChanges,
	# 		Author,
	# 		AuthorChanges,
	# 		Illustrators,
	# 		IllustratorsChanges,
	# 		Tags,
	# 		TagsChanges,
	# 		Genres,
	# 		GenresChanges,
	# 		Publishers,
	# 		PublishersChanges,
	# 		Covers,
	# 		CoversChanges,
	# 		Releases,
	# 		ReleasesChanges,
	# 		# Series,
	# 		# SeriesChanges,
	# 	]

	# for clearTable in delete_from:
	# 	clearTable.query.filter(clearTable.series==clean_item.id).delete()

	# Watches.query.filter(Watches.series_id==clean_item.id).delete()
	# Series.query.filter(Series.id==clean_item.id).delete()
	# SeriesChanges.query.filter(SeriesChanges.srccol==clean_item.id).delete()
	# # db.session.delete(clean_item)
	# db.session.commit()

	return getResponse("Series was deleted entirely!", error=False)

def deleteGroupAutoReleases(data):

	if not current_user.is_mod():
		return getResponse(error=True, message="I see what you (tried) to do there!")

	assert 'item-id' in data
	assert 'mode' in data
	assert data['mode'] == "delete-auto-from-group"

	try:
		delete_id = int(data["item-id"])
	except ValueError:
		raise AssertionError("Failure converting item ID to integer!")

	clean_item = Translators.query.filter(Translators.id==delete_id).one()

	print(clean_item)
	for release in clean_item.releases:
		if release.changeuser == FeedFeeder.FeedFeeder.RSS_USER_ID:
			db.session.delete(release)
			# print(release.id, release.volume, release.chapter, release.postfix, release.changeuser)
		else:
			print("Not deleting: ", release.id, release.volume, release.chapter, release.postfix, release.changeuser)

	db.session.commit()

	return getResponse("Autogen releases deleted. Reloading.", error=False)

def bulkToggleVolumeCountedStatus(data):

	if not current_user.is_mod():
		return getResponse(error=True, message="I see what you (tried) to do there!")

	assert 'item-id' in data
	assert 'mode'    in data
	assert 'enable'  in data
	assert data['mode'] == "toggle-volume-releases"

	try:
		series_id = int(data["item-id"])
	except ValueError:
		raise AssertionError("Failure converting item ID to integer!")

	enable = data['enable']
	assert enable in ['Include', 'Exclude']

	should_include = enable == 'Include'

	item_row = Series.query.filter(Series.id==series_id).one()

	print(item_row)
	for release in item_row.releases:
		if release.changeuser == FeedFeeder.FeedFeeder.RSS_USER_ID or release.changeuser == FeedFeeder.FeedFeeder.NU_SRC_USER_ID:
			if release.volume:
				release.include = should_include

	app.utilities.update_latest_row(item_row)

	db.session.commit()

	return getResponse("Autogen releases deleted. Reloading.", error=False)


# -*- coding: utf-8 -*-

import codecs
import json
import re

from .jcconv import *

#
# Deinflection
#

class Deinflection:
    def __init__(self, term, tags=list(), rule=str()):
        self.children = list()
        self.term = term
        self.tags = tags
        self.rule = rule
        self.success = False

    def validate(self, validator):
        for tags in validator(self.term):
            if len(self.tags) == 0:
                return True

            for tag in self.tags:
                if self.searchTags(tag, tags):
                    return True

    def deinflect(self, validator, rules):
        if self.validate(validator):
            child = Deinflection(self.term)
            self.children.append(child)

        for rule, variants in rules.items():
            for variant in variants:
                tagsIn = variant['tagsIn']
                tagsOut = variant['tagsOut']
                kanaIn = variant['kanaIn']
                kanaOut = variant['kanaOut']

                allowed = len(self.tags) == 0
                for tag in self.tags:
                    if self.searchTags(tag, tagsIn):
                        allowed = True
                        break

                if not allowed or not self.term.endswith(kanaIn):
                    continue

                term = self.term[:-len(kanaIn)] + kanaOut

                child = Deinflection(term, tagsOut, rule)
                if child.deinflect(validator, rules):
                    self.children.append(child)

        if len(self.children) > 0:
            return True

    def searchTags(self, tag, tags):
        for t in tags:
            if re.search(tag, t):
                return True

    def gather(self):
        if len(self.children) == 0:
            return [{'root': self.term, 'rules': list()}]

        paths = list()
        for child in self.children:
            for path in child.gather():
                if self.rule:
                    path['rules'].append(self.rule)
                path['source'] = self.term
                paths.append(path)

        return paths

#
# Deinflector
#

class Deinflector:
    def __init__(self, filename):
        with codecs.open(filename, 'rb', 'utf-8') as fp:
            self.rules = json.load(fp)

    def deinflect(self, term, validator):
        node = Deinflection(term)
        if node.deinflect(validator, self.rules):
            return node.gather()
        else:
            node = Deinflection(kata2hira(term))
            if node.deinflect(validator, self.rules):
                return node.gather()


from ibm_cloud_sdk_core import ApiException, read_external_sources
import os
import pytest
from ibm_dpx_services.data_product_exchange_api_service_v1 import *

#
# This file provides an example of how to use the Data Product Exchange API Service service.
#
# The following configuration properties are assumed to be defined:
# DATA_PRODUCT_EXCHANGE_API_SERVICE_URL=<service base url>
# DATA_PRODUCT_EXCHANGE_API_SERVICE_AUTH_TYPE=iam
# DATA_PRODUCT_EXCHANGE_API_SERVICE_APIKEY=<IAM apikey>
# DATA_PRODUCT_EXCHANGE_API_SERVICE_AUTH_URL=<IAM token service base URL - omit this if using the production environment>
#
# These configuration properties can be exported as environment variables, or stored
# in a configuration file and then:
# export IBM_CREDENTIALS_FILE=<name of configuration file>
#
config_file = 'data_product_exchange_api_service_v1.env'

data_product_exchange_api_service_service = None

config = None

# Variables to hold link values
create_data_product_version_by_catalog_id_link = None
delete_data_product_version_by_user_id_link = None
deliver_data_product_version_by_user_id_link = None
get_data_product_by_user_id_link = None
get_data_product_version_by_user_id_link = None
get_list_of_data_product_by_catalog_id_link = None
get_status_by_catalog_id_link = None
update_data_product_version_by_user_id_link = None

##############################################################################
# Start of Examples for Service: DataProductExchangeApiServiceV1
##############################################################################
# region
class TestDataProductExchangeApiServiceV1Examples:
    """
    Example Test Class for DataProductExchangeApiServiceV1
    """

    @classmethod
    def setup_class(cls):
        global data_product_exchange_api_service_service
        if os.path.exists(config_file):
            os.environ['IBM_CREDENTIALS_FILE'] = config_file

            # begin-common

            data_product_exchange_api_service_service = DataProductExchangeApiServiceV1.new_instance()

            # end-common
            assert data_product_exchange_api_service_service is not None

            # Load the configuration
            global config
            config = read_external_sources(DataProductExchangeApiServiceV1.DEFAULT_SERVICE_NAME)

        print('Setup complete.')

    needscredentials = pytest.mark.skipif(
        not os.path.exists(config_file), reason="External configuration not available, skipping..."
    )

    @needscredentials
    def test_initialize_example(self):
        """
        initialize request example
        """
        try:
            print('\ninitialize() result:')
            # begin-initialize

            response = data_product_exchange_api_service_service.initialize(
                include=['delivery_methods', 'data_product_samples', 'domains_multi_industry'],
            )
            initialize_resource = response.get_result()

            print(json.dumps(initialize_resource, indent=2))

            # end-initialize

            global create_data_product_version_by_catalog_id_link
            create_data_product_version_by_catalog_id_link = initialize_resource['container']['id']
            global get_status_by_catalog_id_link
            get_status_by_catalog_id_link = initialize_resource['container']['id']
            global get_list_of_data_product_by_catalog_id_link
            get_list_of_data_product_by_catalog_id_link = initialize_resource['container']['id']
        except ApiException as e:
            pytest.fail(str(e))

    @needscredentials
    def test_create_data_product_version_example(self):
        """
        create_data_product_version request example
        """
        try:
            print('\ncreate_data_product_version() result:')
            # begin-create_data_product_version

            container_reference_model = {
                # 'id': 'd29c42eb-7100-4b7a-8257-c196dbcca1cd',
                'id': create_data_product_version_by_catalog_id_link,
                'type': 'catalog',
            }

            response = data_product_exchange_api_service_service.create_data_product_version(
                container=container_reference_model,
                name='My New Data Product',
            )
            data_product_version = response.get_result()

            print(json.dumps(data_product_version, indent=2))

            # end-create_data_product_version

            global get_data_product_version_by_user_id_link
            get_data_product_version_by_user_id_link = data_product_version['id']
            global update_data_product_version_by_user_id_link
            update_data_product_version_by_user_id_link = data_product_version['id']
            global delete_data_product_version_by_user_id_link
            delete_data_product_version_by_user_id_link = data_product_version['id']
            global get_data_product_by_user_id_link
            get_data_product_by_user_id_link = data_product_version['data_product']['id']
            global deliver_data_product_version_by_user_id_link
            deliver_data_product_version_by_user_id_link = data_product_version['id']
        except ApiException as e:
            pytest.fail(str(e))

    @needscredentials
    def test_get_initialize_status_example(self):
        """
        get_initialize_status request example
        """
        try:
            print('\nget_initialize_status() result:')
            # begin-get_initialize_status

            response = data_product_exchange_api_service_service.get_initialize_status()
            initialize_resource = response.get_result()

            print(json.dumps(initialize_resource, indent=2))

            # end-get_initialize_status

        except ApiException as e:
            pytest.fail(str(e))

    @needscredentials
    def test_get_data_product_example(self):
        """
        get_data_product request example
        """
        try:
            print('\nget_data_product() result:')
            # begin-get_data_product

            response = data_product_exchange_api_service_service.get_data_product(
                id=get_data_product_by_user_id_link,
            )
            data_product = response.get_result()

            print(json.dumps(data_product, indent=2))

            # end-get_data_product

        except ApiException as e:
            pytest.fail(str(e))

    @needscredentials
    def test_list_data_products_example(self):
        """
        list_data_products request example
        """
        try:
            print('\nlist_data_products() result:')
            # begin-list_data_products

            all_results = []
            pager = DataProductsPager(
                client=data_product_exchange_api_service_service,
                limit=10,
            )
            while pager.has_next():
                next_page = pager.get_next()
                assert next_page is not None
                all_results.extend(next_page)

            print(json.dumps(all_results, indent=2))

            # end-list_data_products
        except ApiException as e:
            pytest.fail(str(e))

    @needscredentials
    def test_list_data_product_versions_example(self):
        """
        list_data_product_versions request example
        """
        try:
            print('\nlist_data_product_versions() result:')
            # begin-list_data_product_versions

            all_results = []
            pager = DataProductVersionsPager(
                client=data_product_exchange_api_service_service,
                asset_container_id=get_list_of_data_product_by_catalog_id_link,
                # data_product='testString',
                # state='draft',
                # version='testString',
                limit=10,
            )
            while pager.has_next():
                next_page = pager.get_next()
                assert next_page is not None
                all_results.extend(next_page)

            print(json.dumps(all_results, indent=2))

            # end-list_data_product_versions
        except ApiException as e:
            pytest.fail(str(e))

    @needscredentials
    def test_get_data_product_version_example(self):
        """
        get_data_product_version request example
        """
        try:
            print('\nget_data_product_version() result:')
            # begin-get_data_product_version

            response = data_product_exchange_api_service_service.get_data_product_version(
                id=get_data_product_version_by_user_id_link,
            )
            data_product_version = response.get_result()

            print(json.dumps(data_product_version, indent=2))

            # end-get_data_product_version

        except ApiException as e:
            pytest.fail(str(e))

    @needscredentials
    def test_update_data_product_version_example(self):
        """
        update_data_product_version request example
        """
        try:
            print('\nupdate_data_product_version() result:')
            # begin-update_data_product_version

            json_patch_operation_model = {
                'op': 'replace',
                'path': '/description',
            }

            response = data_product_exchange_api_service_service.update_data_product_version(
                id=update_data_product_version_by_user_id_link,
                json_patch_instructions=[json_patch_operation_model],
            )
            data_product_version = response.get_result()

            print(json.dumps(data_product_version, indent=2))

            # end-update_data_product_version

        except ApiException as e:
            pytest.fail(str(e))

    # @needscredentials
    # def test_deliver_data_product_version_example(self):
    #     """
    #     deliver_data_product_version request example
    #     """
    #     try:
    #         print('\ndeliver_data_product_version() result:')
    #         # begin-deliver_data_product_version

    #         item_reference_model = {
    #             'id': 'd29c42eb-7100-4b7a-8257-c196dbcca1cd',
    #         }

    #         order_reference_model = {
    #             'id': '4705e047-1808-459a-805f-d5d13c947637',
    #             'items': [item_reference_model],
    #         }

    #         response = data_product_exchange_api_service_service.deliver_data_product_version(
    #             id=deliver_data_product_version_by_user_id_link,
    #             order=order_reference_model,
    #         )
    #         delivery_resource = response.get_result()

    #         print(json.dumps(delivery_resource, indent=2))

    #         # end-deliver_data_product_version

    #     except ApiException as e:
    #         pytest.fail(str(e))

    @needscredentials
    def test_delete_data_product_version_example(self):
        """
        delete_data_product_version request example
        """
        try:
            # begin-delete_data_product_version

            response = data_product_exchange_api_service_service.delete_data_product_version(
                id=delete_data_product_version_by_user_id_link,
            )

            # end-delete_data_product_version
            print('\ndelete_data_product_version() response status code: ', response.get_status_code())

        except ApiException as e:
            pytest.fail(str(e))

# endregion
##############################################################################
# End of Examples for Service: DataProductExchangeApiServiceV1
##############################################################################


import pandas as pd
import matplotlib.pyplot as plt
import requests
import numpy as np
from math import floor
from termcolor import colored as cl

plt.style.use('fivethirtyeight')
plt.rcParams['figure.figsize'] = (20, 10)

# The first thing we did is to define a function named ‘get_historical_data’ that takes the stock’s symbol (‘symbol’) as a required parameter
# and the starting date of the historical data (‘start_date’) as an optional parameter. Inside the function, we are defining the API key
# and the URL and stored them into their respective variable. Next, we are extracting the historical data in JSON format using the ‘get’ function
# and stored it into the ‘raw_df’ variable. After doing some processes to clean and format the raw JSON data, we are returning it in the form of
# a clean Pandas dataframe. Finally, we are calling the created function to pull the historic data of IBM from the starting of 2020 and stored it
# into the ‘ibm’ variable.

def get_historical_data(symbol, start_date = None):
    api_key = open(r'api_key.txt')
    api_url = f'https://www.alphavantage.co/query?function=TIME_SERIES_DAILY_ADJUSTED&symbol={symbol}&apikey={api_key}&outputsize=full'
    raw_df = requests.get(api_url).json()
    df = pd.DataFrame(raw_df[f'Time Series (Daily)']).T
    df = df.rename(columns = {'1. open': 'open', '2. high': 'high', '3. low': 'low', '4. close': 'close', '5. adjusted close': 'adj close', '6. volume': 'volume'})
    for i in df.columns:
        df[i] = df[i].astype(float)
    df.index = pd.to_datetime(df.index)
    df = df.iloc[::-1].drop(['7. dividend amount', '8. split coefficient'], axis = 1)
    if start_date:
        df = df[df.index >= start_date]
    return df

ibm = get_historical_data('IBM', '2020-01-01')
ibm
print(ibm)

# Firstly, we are defining a function named ‘get_rsi’ that takes the closing price of a stock (‘close’) and the lookback period (‘lookback’) as parameters.
# Inside the function, we are first calculating the returns of the stock using the ‘diff’ function provided by the Pandas package and stored it into the ‘ret’
# variable. This function basically subtracts the current value from the previous value. Next, we are passing a for-loop on the ‘ret’ variable to distinguish
# gains from losses and append those values to the concerning variable (‘up’ or ‘down’). Then, we are calculating the Exponential Moving Averages for both the
# ‘up’ and ‘down’ using the ‘ewm’ function provided by the Pandas package and stored them into the ‘up_ewm’ and ‘down_ewm’ variable respectively. Using these
# calculated EMAs, we are determining the Relative Strength by following the formula we discussed before and stored it into the ‘rs’ variable. By making use of
# the calculated Relative Strength values, we are calculating the RSI values by following its formula. After doing some data processing and manipulations, we
# are returning the calculated Relative Strength Index values in the form of a Pandas dataframe. Finally, we are calling the created function to store the RSI
# values of IBM with 14 as the lookback period.

def get_rsi(close, lookback):
    ret = close.diff()
    up = []
    down = []
    for i in range(len(ret)):
        if ret[i] < 0:
            up.append(0)
            down.append(ret[i])
        else:
            up.append(ret[i])
            down.append(0)
    up_series = pd.Series(up)
    down_series = pd.Series(down).abs()
    up_ewm = up_series.ewm(com = lookback - 1, adjust = False).mean()
    down_ewm = down_series.ewm(com = lookback - 1, adjust = False).mean()
    rs = up_ewm/down_ewm
    rsi = 100 - (100 / (1 + rs))
    rsi_df = pd.DataFrame(rsi).rename(columns = {0:'rsi'}).set_index(close.index)
    rsi_df = rsi_df.dropna()
    return rsi_df[3:]

ibm['rsi_14'] = get_rsi(ibm['close'], 14)
ibm = ibm.dropna()
ibm
print(ibm)

ax1 = plt.subplot2grid((10,1), (0,0), rowspan = 4, colspan = 1)
ax2 = plt.subplot2grid((10,1), (5,0), rowspan = 4, colspan = 1)
ax1.plot(ibm['close'], linewidth = 2.5)
ax1.set_title('IBM CLOSE PRICE')
ax2.plot(ibm['rsi_14'], color = 'orange', linewidth = 2.5)
ax2.axhline(30, linestyle = '--', linewidth = 1.5, color = 'grey')
ax2.axhline(70, linestyle = '--', linewidth = 1.5, color = 'grey')
ax2.set_title('IBM RELATIVE STRENGTH INDEX')
plt.show()

# First, we are defining a function named ‘implement_rsi_strategy’ which takes the stock prices (‘prices’), and the RSI values (‘rsi’) as parameters.
# Inside the function, we are creating three empty lists (buy_price, sell_price, and rsi_signal) in which the values will be appended while creating
# the trading strategy. After that, we are implementing the trading strategy through a for-loop. Inside the for-loop, we are passing certain conditions,
# and if the conditions are satisfied, the respective values will be appended to the empty lists. If the condition to buy the stock gets satisfied, the
# buying price will be appended to the ‘buy_price’ list, and the signal value will be appended as 1 representing to buy the stock. Similarly, if the
# condition to sell the stock gets satisfied, the selling price will be appended to the ‘sell_price’ list, and the signal value will be appended as -1
# representing to sell the stock. Finally, we are returning the lists appended with values. Then, we are calling the created function and stored the values
# into their respective variables. The list doesn’t make any sense unless we plot the values. So, let’s plot the values of the created trading lists.

def implement_rsi_strategy(prices, rsi):
    buy_price = []
    sell_price = []
    rsi_signal = []
    signal = 0

    for i in range(len(rsi)):
        if rsi[i-1] > 30 and rsi[i] < 30:
            if signal != 1:
                buy_price.append(prices[i])
                sell_price.append(np.nan)
                signal = 1
                rsi_signal.append(signal)
            else:
                buy_price.append(np.nan)
                sell_price.append(np.nan)
                rsi_signal.append(0)
        elif rsi[i-1] < 70 and rsi[i] > 70:
            if signal != -1:
                buy_price.append(np.nan)
                sell_price.append(prices[i])
                signal = -1
                rsi_signal.append(signal)
            else:
                buy_price.append(np.nan)
                sell_price.append(np.nan)
                rsi_signal.append(0)
        else:
            buy_price.append(np.nan)
            sell_price.append(np.nan)
            rsi_signal.append(0)

    return buy_price, sell_price, rsi_signal

buy_price, sell_price, rsi_signal = implement_rsi_strategy(ibm['close'], ibm['rsi_14'])

# We are plotting the Relative Strenght Index values along with the buy and sell signals generated by the trading strategy. We can
# observe that whenever the RSI line crosses from above to below the lower band or the oversold level, a green-colored buy signal
# is plotted in the chart. Similarly, the RSI line crosses from below to above the upper band or the overbought level, a red-colored
# sell signal is plotted in the chart.

ax1 = plt.subplot2grid((10,1), (0,0), rowspan = 4, colspan = 1)
ax2 = plt.subplot2grid((10,1), (5,0), rowspan = 4, colspan = 1)
ax1.plot(ibm['close'], linewidth = 2.5, color = 'skyblue', label = 'IBM')
ax1.plot(ibm.index, buy_price, marker = '^', markersize = 10, color = 'green', label = 'BUY SIGNAL')
ax1.plot(ibm.index, sell_price, marker = 'v', markersize = 10, color = 'r', label = 'SELL SIGNAL')
ax1.set_title('IBM RSI TRADE SIGNALS')
ax2.plot(ibm['rsi_14'], color = 'orange', linewidth = 2.5)
ax2.axhline(30, linestyle = '--', linewidth = 1.5, color = 'grey')
ax2.axhline(70, linestyle = '--', linewidth = 1.5, color = 'grey')
plt.show()


import pygame as pg

from data.colors import WHITE, COLOR_KEY
from data.config import *
from data.paths import *
import data.languages.english as eng
import data.languages.russian as rus

class StartMenuCaption:
    def __init__(self):
        self.image = pg.image.load(START_MENU_CAPTION_BG).convert_alpha()
        self.bg = pg.transform.scale(self.image, (SCR_W, int(4/15 * SCR_H)))
        self.text_surface = None
        self.set_language("English")

    def set_language(self, language):
        pg.font.init()
        font = pg.font.Font(FONT_1, int(7/60 * SCR_H))
        if language == "English":
            caption = font.render(eng.START_MENU_CAPTION, True, WHITE)
        else:
            caption = font.render(rus.START_MENU_CAPTION, True, WHITE)
        self.text_surface = pg.Surface((caption.get_size()))
        self.text_surface.fill(COLOR_KEY)
        self.text_surface.set_colorkey(COLOR_KEY)
        self.text_surface.blit(caption, (0, 0))

    def update_alpha(self, time, state):
        if state == START_MENU_HIDE:
            alpha = int(255 * (1 - time / START_MENU_ANIMATION_TIME))
        else:
            alpha = int(255 * time / START_MENU_ANIMATION_TIME)
        self.text_surface.set_alpha(alpha)
        self.bg.set_alpha(alpha)

    def reset(self):
        self.text_surface.set_alpha(255)
        self.bg.set_alpha(255)

    def draw(self, surface):
        surface.blit(self.bg, (0, int(11/30 * SCR_H)))
        surface.blit(self.text_surface, (1 / 32 * SCR_W, int(9 / 20 * SCR_H)))


########
# USDD #
########
# data trasmitted through websocket, so only mhtml captured
import datetime
import time
def archive(browser, wa, folder):
    name = 'USDD'
    time_str = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
    browser.retry('https://tdr.org/')
    time.sleep(10)
    browser.archive_mhtml('{}/{}-{}.mhtml'.format(folder, name, time_str))


import xml.etree.ElementTree as ET
import os
import sqlite3
import thread
import unicodedata
import threading
import json
import re

from multiprocessing.dummy import Pool as ThreadPool

########################################################################################################
############################### extract topics from raw files 	##############################
########################################################################################################
def extract_tag(in_path):
	"""
	return a set of tags the raw file has: (indexing_service, online_producer)
	"""
	tag_indexing = []
	tag_online = []
	tree = ET.parse(in_path)
	head = tree.getroot().find("head")
	for descr in head.findall('docdata/identified-content/classifier[@type="descriptor"]'):
		if descr.get('class') == 'indexing_service':
			tag_indexing.append(descr.text)
		if descr.get('class') == 'online_producer':
			tag_online.append(descr.text)
	return set(tag_indexing), set(tag_online)

def extract_tag_all(root_dir):
	"""
	save an index of all the tags and files
	"""
	index = {}
	online_prod = {}
	for root, dirs, files in os.walk(root_dir):
		path = [os.path.join(root,name) for name in files if len(name.split("."))==2 and name.split(".")[1]=="xml"]
		for p in path:
			try:
				tag_indexing, tag_online= extract_tag(p)
			except:
				tag_indexing, tag_online = set([]),set([])
				print("Both empty: "+p)
			#name = ("/").join(p.split("/")[-3:]).split(".")[0]
			name = p.split("/")[-1].split(".")[0]
			if tag_indexing == set([]):
				tag_indexing.update(["NO TAG"])
			if tag_online == set([]):
				tag_online.update(["NO TAG"])

			for tag in tag_indexing:
				item_index = index.get(tag)
				if item_index is None:
					index[tag] = [name]
				else:
					item_index.append(name)
					index.update([(tag,item_index)])
			for tag in tag_online:
				item_index = online_prod.get(tag)
				if item_index is None:
					online_prod[tag] = [name]
				else:
					item_index.append(name)
					online_prod.update([(tag,item_index)])
	f = open(root_dir+'topics_indexing_services.json','wb')
	json.dump(index,f)
	f.close()
	f = open(root_dir + 'topics_online_producer.json','wb')
	json.dump(online_prod,f)
	f.close()

def file_to_topic(root_path):
	"""
	for all the files under root_path, map file id to its topics.
	Save result: "file_to_topics.json"
	file_id:([Topics_by_indexing_service], [topics_by_online_service])
	"""
	processed = set()
	out = {}
	try:
		with open(root_path+"file_to_topics.json") as json_data:
			d = json.load(json_data)
			processed = set(d.keys())
	except:
		print("No previous work available!")

	for root, dirs, files in os.walk(root_path):
		path = [os.path.join(root,name) for name in files if len(name.split("."))==2 and name.split(".")[1]=="xml"]
		for p in path:
			tag_indexing, tag_online= extract_tag(p)
			file_id = p.split("/")[-1].split(".")[0]
			if file_id in processed:
				continue

			if tag_indexing == set([]):
				tag_indexing.update(["NO TAG"])
			if tag_online == set([]):
				tag_online.update(["NO TAG"])

			# tmp = {file_id:(list(tag_indexing), list(tag_online))}
			out[file_id] = (list(tag_indexing), list(tag_online))

	with open(root_path+'file_to_topics.json','w') as f_json:
		json.dump(out, f_json)
	print(">>Done for "+root_path)

########################################################################################################
################## extract summary, full-article, headlines from raw files 	#########################
########################################################################################################

def extract(in_path):
	tree = ET.parse(in_path)
	summary=""
	body = tree.getroot().find("body")
	for head in body.findall('body.head/abstract/p'):
		summary += head.text
	article=""
	body = tree.getroot().find("body")
	contents = body.find('body.content')

	for content in contents.iter('block'):
		para = content.get('class')
		if para!="full_text":
			continue
		for txt in content.findall('p'):
			article+=txt.text

	h1 = tree.getroot().find("body/body.head/hedline/hl1")
	meta = h1.text

	return summary,article, meta

# extract and parse plain text into root/summary, root/meta, root/content
def extractall(root_dir):
	paths = [] # list of all files under root
	new_storage = ["summary","content","meta"]
	process_path = [root_dir+ns for ns in new_storage]

	# make folders under root if they don't already exist
	for ps in process_path:
		if not os.path.exists(ps):
			os.makedirs(ps)

	for root, dirs, files in os.walk(root_dir):
		# get all .xml file path under year/XX/XX
		path = [os.path.join(root,name) for name in files if len(name.split("."))==2 and name.split(".")[1]=="xml"]
		for p in path:
			try:
				extraction = extract(p)
			except Exception:
				print("Parsing error: "+p)
				continue
			name = p.split("/")[-1].split(".")[0]

			# skip if contain non-ascii or some parts missing
			if extraction[0]=="" or extraction[1]=="" or extraction[2]=="":
				print("missing part on "+name)
				continue
			for i in range(3):
				f = open(process_path[i]+"/"+name+".txt","wb")
				try:
					f.write(extraction[i])
				except UnicodeEncodeError:
					print("Unicode encoding: "+p)
					f.write(extraction[i].encode("utf8"))
				f.close()

class myThreads(threading.Thread):
	def __init__(self,in_dir,number):
		threading.Thread.__init__(self)
		self._dir = in_dir
		self._number = number
	def run(self):
		print("Starting: "+str(self._number))
		extract_tag_all(self._dir)
		print("Existing: "+str(self._number))

def extract_multi(in_dir,start,finish):
	threads = []
	for i in range(start,finish+1):
		path= in_dir+str(i)+"/"
		thread = myThreads(path,i)
		threads.append(thread)

	for thread in threads:
		thread.start()

	for t in threads:
		t.join()
	print(str(start)+"-"+str(finish)+" Done")

########################################################################################################
################################ preprocess extracted files ###################################
########################################################################################################
def annotate(content, in_dir,start,end):
    for i in range(start,end):
        in_path = in_dir+content+'/'+str(i)+content
        out_path = in_dir+content+"_annotated/"+str(i)+content+"_annotated"
        annotate_all(in_path,out_path)

        print("Annotating "+content+"for "+str(i)+"Done!")

def annotate_all(in_dir, out_dirs):
	import corenlpy
	if not os.path.exists(out_dirs):
		os.makedirs(out_dirs)

	corenlpy.corenlp(in_dirs=in_dir, out_dir=out_dirs,
		annotators=['tokenize','ssplit','pos','lemma','ner','parse','depparse','dcoref','relation'],
		threads=10,output_format='xml')

########################################################################################################
############################### store paths of all files     ################################
########################################################################################################
# store all file paths under root into db
def store(root_dir):
	conn = sqlite3.connect("nyt.db")
	c = conn.cursor()
	c.execute("""CREATE TABLE IF NOT EXISTS nyt(file_id integer, orinigal_path text NOT NULL);""")

	# entry:[(id,path)]
	entry = []
	for root, dirs, files in os.walk(root_dir):
		# get all .xml file path under year/XX/XX
		path = [os.path.join(root,name) for name in files if len(name.split("."))==2 and name.split(".")[1]=="xml"]
		for p in path:
			name = p.split("/")[-1].split(".")[0]
			entry.append((name,p))

	c.executemany("INSERT INTO nyt VALUES (?,?)",entry)

	conn.commit()
	conn.close()

# given file id, return path to its original raw xml file
def get_path(file_id):
	conn = sqlite3.connect("nyt.db")
	c = conn.cursor()
	query = "SELECT * FROM nyt WHERE file_id=="+str(file_id)
	c.execute(query)
	out = c.fetchone()
	conn.close()
	return out[1]

#################################################################################################
################################ preprocess summary files ###################################
#################################################################################################
def substi(text):
	"""
	Find all ; and replace with '.' get rid of (M), photo(s)/drawing(s)
	"""

	text = re.sub(r'photo|drawing|map|chart|table|graph|diagram',"", text)
	text = re.sub(r'\([A-Z]\)','',text)

	text_sep = text.split(".")
	text_sep = [s.strip() for s in text_sep if len(s)>=3]
	text_new = ". ".join(text_sep)

	return text_new

def substi_all(root_dir):
	"""
	For all files under root_dir, preprocess and write back
	"""
	l = os.listdir(root_dir)
	for p in l:
		_dir= root_dir+p
		try:
			with open(_dir) as f:
				text = f.read()
				text = substi(text)
		except:
			print("Cannot modify "+ _dir)
			continue
		try:
			with open(_dir,'wb') as f:
				f.write(text)
		except:
			print("Cannot write into "+ _dir)
			continue

	print(root_dir+" Done!")

#######################################################################################################
################################################# Testing	###########################################
#######################################################################################################

# testing
def test_old():
	# test extract
	in_path ="/Users/liujingyun/Desktop/NLP/nyt_corpus/data/2007/02/04/1823747.xml"

	summary, content, meta = extract(in_path)
	# print(len(summary),len(content),len(meta))
	# f = open("test1","wb")
	# try:
	# 	f.write(content)
	# except UnicodeEncodeError:
	# 	print("Encode in Unicode!")
	# 	f.write(content.encode("utf8"))
	# f.close()

	#test extractall
	# in_dir =s
	in_dir = '/Users/liujingyun/Desktop/NLP/nyt_corpus/data/2006'
	in_dir_200701 = '/Users/liujingyun/Desktop/NLP/nyt_corpus/data/2007/01'
	in_dir_200702 = '/Users/liujingyun/Desktop/NLP/nyt_corpus/data/2007/02'
	tag_path = "/Users/liujingyun/Desktop/NLP/nyt_corpus/data/2006/01/02/1729112.xml"
	#extractall(in_dir_200701)
	# thread1 = myThreads(in_dir_200701,"0701")
	# thread2 = myThreads(in_dir_200702,"0702")
	# thread1.start()
	# thread2.start()
	#extract_multi(in_dir,2006,2007)
	extract_tag_all(in_dir)

	#test annotate_all
	in_dir = '/Users/liujingyun/Desktop/NLP/nyt_corpus/data/2007content'
	out_dir = '/Users/liujingyun/Desktop/NLP/nyt_corpus/summary_annotated'
	#annotate_all(in_dir, out_dir)

	for i in range(1987,2008):
		in_path = '/Users/liujingyun/Desktop/NLP/nyt_corpus/data/'+str(i)+'content'
		out_path = '/Users/liujingyun/Desktop/NLP/nyt_corpus/content_annotated/'+str(i)
		#annotate_all(in_path, out_path)

def test_substi():
	# _dir = "/home/ml/jliu164/corpus/nyt_corpus/summary/2007summary/"
	# substi_all(_dir)

	pool = ThreadPool(6)
	root_dir = '/home/ml/jliu164/corpus/nyt_corpus/summary/'
	dirs = [root_dir+str(i)+"summary/" for i in range(2002,2007)]
	results = pool.map(substi_all, dirs)
	pool.close()
	pool.join()

if __name__ == '__main__':
	# test()
	test_substi()


from gpiozero import DistanceSensor
from pyo import *
pyoserver = Server(sr=48000, nchnls=2, duplex=0, audio="jack", jackname="pyo")
pyoserver.setJackAuto(False, True)
#pyoserver.setJackAutoConnectOutputPorts(system:playback_1)
pyoserver.boot()
pyoserver.start()
pyoserver.amp = 0.5
sensor = DistanceSensor(echo=17, trigger=4, max_distance=1, queue_len=10)

amp = SigTo(value=0.3, time=0.05, init=0.3)
gliss = SigTo(value=200, time=0.025, init=200)
a = SineLoop(freq=gliss, feedback=0.08, mul=amp).out()
b = SineLoop(freq=gliss*1.005, feedback=0.08, mul=amp).out()
rev = Freeverb([a, b]).out()

def capteur():
    pitch = round(sensor.distance * 80 + 30, 4)
    pitch = 110 - pitch
    print("distance : ", sensor.distance, "m")
    if sensor.distance > 1:
        amp.value = 0.0
    amp.value = 0.3
    print(pitch)
    f = midiToHz(pitch)
    gliss.value = f

pat = Pattern(capteur, 0.025).play()

#pyoserver.gui(locals())