# 在本次会议中：
# # # # 用 OpenCV 画不同的几何图形
# # # # 你要学习这些函数：cv.line(), cv.circle() , cv.rectangle(), cv.ellipse(), cv.putText() 等。

#画线
import numpy as np
import cv2 as cv
# 创建一个黑色的图像
img = np.zeros((512,512,3), np.uint8)
# 画一条 5px 宽的蓝色对角线
cv.line(img,(0,0),(511,511),(255,0,0),5)

#画矩形
cv.rectangle(img, (384, 0), (510, 128), (0, 255, 0), 3)
#画圆
cv.circle(img,(447,63), 63, (0,0,255), -1)
#画椭圆
cv.ellipse(img, (256,256),(100,50),0,0,180,255,-1)

#画多边形
pts = np.array([[10,5],[20,30],[70,20],[50,10]], np.int32)
pts = pts.reshape((-1,1,2))
cv.polylines(img,[pts],True,(0,255,255))

#添加文字:图片,文字，位置，字体类型，字体大小，字体颜色，字体粗细
cv.putText(img, "guolei", (10, 30), cv.FONT_HERSHEY_SIMPLEX, 20, (0, 0, 255), 2)
cv.imshow('show', img)
cv.waitKey(0)
cv.destroyAllWindows()


import numpy
import torch
import torch.nn.functional as F
import scipy.optimize as opt

class BetterSecondOrderAttack():
    def __init__(self, step_size, epsilon, perturb_steps,
                 random_start=None):
        self.step_size = step_size
        self.epsilon = epsilon
        self.perturb_steps = perturb_steps
        self.random_start = random_start

    def __call__(self, model, x, y):
        model.eval()

        def minimized(cz):
            return torch.trace(F.softmax(model(cz), -1)[..., y])

        def fun(cx):
            with torch.no_grad():
                return minimized(
                    x.new_tensor(cx.reshape(*x.shape))).double().item()

        def jac(cx):
            return torch.autograd.functional.jacobian(
                minimized,
                x.new_tensor(cx.reshape(*x.shape))
            ).cpu().double().numpy().reshape(-1)

        def clbk(_):
            return False

        result = opt.minimize(
            fun, (x + torch.sign(torch.randn_like(x)) *
                  self.epsilon).cpu().numpy().reshape(-1),
            method="L-BFGS-B", jac=jac,
            bounds=[
                (max(0, v - self.epsilon), min(1, v + self.epsilon))
                for v in x.cpu().double().numpy().reshape(-1)
            ],
            callback=clbk,
            options={
                'maxiter': self.perturb_steps,
                'maxcor': 100,
                # 'minfev': -6,
                # 'disp': True,
                # 'maxfun': 300,
                # 'iprint': 1
            }
        )
        x_adv = torch.min(
            torch.max(
                x.new_tensor(result.x.reshape(*x.shape)),
                x - self.epsilon
            ),
            x + self.epsilon
        )
        x_adv = torch.clamp(x_adv, 0.0, 1.0)

        return x_adv


import argparse
import datetime
import glob
import os
import re
import subprocess
import sys
import time

import numpy as np

PATH_TO_PROCESSING_SCRIPT  =  "./RunBSCITE.sh" # path to B-SCITE executable
PATH_TO_EXECUTABLE  =  "/Users/seonghwanjun/ScRNACloneEvaluation/B-SCITE/src/bscite.exe" # path to B-SCITE executable
parser = argparse.ArgumentParser(description='Script to batch run simulation studies.')
parser.add_argument('sim_path', help="Path to simulation data.", type=str)
parser.add_argument('-t', '--threshold', help="Threshold for calling variants in the single cell data.", type=int, default = 1)
parser.add_argument('--rbegin', help="Rep begin.", type=int, default = 0)
parser.add_argument('--rend', help="Rep end (not inclusive).", type=int, default = 20)
parser.add_argument('--cbegin', help="Sim begin.", type=int, default = 1)
parser.add_argument('--cend', help="Sim end (not inclusive).", type=int, default = 4)
args = parser.parse_args()

SIM_PATH = args.sim_path
CASE_BEGIN = args.cbegin
CASE_END = args.cend
REP_BEGIN = args.rbegin
REP_END = args.rend
SC_VAR_THRESHOLD = args.threshold

fp = 0.01 # esimated false positive rate of SCS experiment
fn = 0.2     # estimated false negative rate of SCS experiment
n  = 100      # number of mutations
#m  = 100      # number of cells
r  = 4       # number of repeats
l  = 20000    # number of loops

for rep_no in range(REP_BEGIN, REP_END):
	REP_PATH = os.path.join(SIM_PATH, "rep" + str(rep_no))
	for case_no in range(CASE_BEGIN, CASE_END):
		CASE_PATH = os.path.join(REP_PATH, "case" + str(case_no))
		run_command = PATH_TO_PROCESSING_SCRIPT + " "
		run_command += CASE_PATH + " "
		run_command += str(SC_VAR_THRESHOLD)
		os.system(run_command)

		OUTPUT_FILES_PREFIX = CASE_PATH + "/bscite/bscite" # this is the prefix used for the three output files (.matrices, .gv and .newick) given as the output of B-SCITE
		SCFile   = os.path.join(CASE_PATH, "bscite/bscite.SC") # path input SC File
		bulkFile = os.path.join(CASE_PATH, "bscite/bscite.bulk")     # path input bulk file

		cell_count = np.loadtxt(os.path.join(CASE_PATH, "bscite", "cell_count.txt"), dtype=int, ndmin=1)[0]

		run_command = ""
		run_command += PATH_TO_EXECUTABLE + " "
		run_command += "-i "   + SCFile   + " "
		run_command += "-bulkFileLocation " + bulkFile + " "
		run_command += "-n "  + str(n)  + " "
		run_command += "-m "  + str(cell_count)  + " "
		run_command += "-fd " + str(fp) + " "
		run_command += "-ad " + str(fn) + " "
		run_command += "-r "  + str(r)  + " "
		run_command += "-l "  + str(l)  + " "
		run_command += "-o "  + OUTPUT_FILES_PREFIX + " "

		start = datetime.datetime.now()
		os.system(run_command)
		end = datetime.datetime.now()
		result = end - start
		exec_time_in_milli_seconds = result.total_seconds() * 1000
		with open(os.path.join(CASE_PATH, "bscite/timing.txt"), "w+") as f:
			f.write(str(exec_time_in_milli_seconds) + "\n")


# Grid Strategy
import asyncio
import json
from typing import Dict, Any, List
from datetime import datetime
from random import randint
from bisect import bisect_left
from multiprocessing import Process

from api.base_api import OrderType, Direction
from api.binance.binance_api import BinanceApi
from api.exchange import ExchangeidSymbolidAPI
from db.cache import RedisHelper
from base.consts import RedisKeys, BinanceWebsocketUri
from base.config import grid_logger as logger
import websockets
import pandas as pd
import click

from util.time_util import TimeUtil

DEBUG = True

class BinanceGridRobotV2(object):
    def __init__(self, exchange_api: BinanceApi, info: Dict[str, Any], info_key: str, trade_key: str, pair_key: str,
                 price_key: str):
        self.info = info
        self.exchange_api: BinanceApi = exchange_api
        self.redis = RedisHelper()

        # TODO: generate all price at init phrase
        self.all_prices: List[float] = []

        # keep digits same as exchange
        self.digits: int = self.exchange_api.symbol.price_precision

        # redis key related from above
        self.info_key: str = info_key
        self.trade_key: str = trade_key
        self.pair_key: str = pair_key
        self.price_key: str = price_key

        self.task_table = {
            "price": self.update_price,
            "order": self.keep_orders,
        }

        logger.info(
            f"successful init {self.exchange_api.api.exchange} grid robot,info is:\n {info}\n redis keys are {self.info_key, self.trade_key, self.pair_key, self.price_key} ")

    async def cancel(self, order_id: str, price: str):
        try:
            order_detail = self.redis.hget(redis_key=self.trade_key, key=price)
            if order_detail and order_detail != "H" and order_detail != "OnGoing":
                result = await self.exchange_api.cancel_order(client_order_id=order_detail)
                if result:
                    self.redis.hdel(redis_key=self.trade_key, key=price)
                else:
                    logger.warning(f"撤单失败:{price}-{order_detail}")
                    self.redis.hdel(redis_key=self.trade_key, key=price)
            else:
                logger.error(f"没有在redis中找到这个订单{price}-{order_id}-{order_detail}")
        except Exception as e:
            self.redis.hdel(redis_key=self.trade_key, key=price)
            logger.error(f"撤单请求异常，{e}")

    async def buy(self, price, order_id: str = None):
        price = round(price, self.digits)
        try:
            result = await self.trade(direction=Direction.OPEN_LONG, price=price, order_id=order_id)
            if result['status'] != "NEW":
                logger.error(f"买单过期，{result}")
            return result
        except Exception as e:
            logger.error(f"下单失败:{e}")
            return None

    async def sell(self, price, amount: int = 0, order_id: str = None):
        result = None
        retry = 3
        while not result and retry > 0:
            try:
                result = await self.trade(direction=Direction.CLOSE_LONG, price=price, amount=amount, order_id=order_id)
            except Exception as e:
                logger.error(f"卖单下单失败")
            retry -= 1
            await asyncio.sleep(0.5)
        return result

    async def trade(self, direction: str, price: float, amount: int = 0, order_id: str = None):
        try:
            info = self.redis.get(redis_key=self.info_key)
            if not amount:
                amount = info["order_amount"]

            # 平多的时候不限制 maker only
            if direction == Direction.CLOSE_LONG:
                order_type = OrderType.LIMIT
            else:
                order_type = OrderType.MAKER if info["maker_only"] else OrderType.LIMIT

            if not order_id:
                order_id = f"G_F_{randint(1, 10 ** 8)}_{price}"

            result = await self.exchange_api.create_order(
                amount=amount,
                order_type=order_type,
                direction=direction,
                price=price,
                client_oid=order_id,
                order_resp_type="RESULT"
            )
            return result

        except Exception as e:
            logger.error(f"下单失败{direction},{price}:{e}")
            return None

    async def update_buy_orders(self):
        """update buy orders

        """
        order_infos = await self.exchange_api.get_symbol_orders()
        if order_infos.shape[0]:
            self.buy_orders = {x['clientOrderId']: float(x['price']) for x in
                               order_infos[order_infos.side == "BUY"].to_dict(orient='records')}
            self.current_buy_prices = [float(x['price']) for x in
                                       order_infos[order_infos.side == "BUY"].to_dict(orient='records')]
        else:
            logger.warning("没有在交易所发现有任何挂单")
            self.buy_orders = {}
            self.current_buy_prices = []

    def generate_prices_up(self, start_price: float, number: int, info) -> List[float]:
        """ generate price from down to up

        Args:
            start_price: up price
            number: length of price list

        Returns:
            price list

        """
        prices: List[float] = []
        percent = info["grid_percent"]
        while number > 0:
            prices.append(round(start_price, self.digits))
            start_price *= (1 + percent)
            number -= 1
        return prices

    def generate_prices(self, start_price: float, number: int, info) -> List[float]:
        """ generate price from up to down

        Args:
            start_price: up price
            number: length of price list

        Returns:
            price list

        """
        prices: List[float] = []
        percent = info["grid_percent"]
        while number > 0:
            prices.append(round(start_price, self.digits))
            start_price /= (1 + percent)
            number -= 1
        return prices

    async def update_orders(self, price):
        """ update buy order on rising market
        1. make new orders with current price
        2. cancel old orders

        Args:
            price: current price

        """
        if not price:
            logger.warning(f"更新订单价格为 0")
            return
        info = self.redis.get(self.info_key)

        # 一开始生成好所有的格子信息

        # TODO：修改成5000
        number = 5000

        # number = 10
        price = info['middle_price']

        upper_prices = self.generate_prices_up(start_price=price, number=number, info=info)
        down_prices = self.generate_prices(start_price=price, number=number, info=info)
        self.all_prices = sorted(list(set(upper_prices + down_prices)))
        self.grid_percent = info["grid_percent"]
        self.middle_price = info["middle_price"]

    async def cancel_buy_orders(self, useful_order_ids: List[str]):
        """

        Args:
            useful_order_ids: order ids to preserve

        """
        await self.update_buy_orders()
        cancel_orders = []
        cancel_price = []

        for k, v in self.buy_orders.items():
            if k not in useful_order_ids:
                cancel_orders.append(k)
                cancel_price.append(v)
                # 删除掉redis里面的key
                self.redis.hdel(redis_key=self.trade_key, key=v)

        logger.info(f"开始撤单:{cancel_orders},保留单:{useful_order_ids}")
        await asyncio.gather(
            *[self.exchange_api.cancel_order(client_order_id=order_id) for order_id in cancel_orders]
        )

    async def update_price(self):
        stream_names = [
            f"{self.exchange_api.symbol.symbol.lower()}@aggTrade"
        ]

        uri = BinanceWebsocketUri.__dict__[
                  self.exchange_api.symbol.market_type] + f"/stream?streams={'/'.join(stream_names)}"

        subscribe = {
            "method": "SUBSCRIBE",
            "params": stream_names,
            "id": 1
        }

        while 1:
            try:
                async with websockets.connect(uri) as ws:
                    await ws.send(json.dumps(subscribe))
                    logger.info(f"成功连接到{uri},开始更新价格")
                    while 1:
                        data = json.loads(await asyncio.wait_for(ws.recv(), timeout=25))
                        if data.get("data"):
                            self.redis.set(redis_key=self.price_key, value=float(data["data"]["p"]))
                            logger.debug(f"更新价格成功: {data['data']['p']}")
                        else:
                            logger.warning(f"发现异常数据:{data}")
            except Exception as e:
                logger.error(f"连接断开，正在重连……{e}", exc_info=True)

    async def keep_orders(self):
        """整理订单
        初始化:
            挂最开始的几个单, 记录挂单ID
        运行中：
            1 (优化)，获取当前的挂单对比是否都在目前内存的挂单里面
            1. 判断目前的几个挂单是否有成交的，如果有，挂上对应的卖单, 并记录对应的持仓信息
            2. 根据当前的价格，计算新的挂单，和目前内存里面的挂单/持仓比较，确认需要挂哪几个 撤哪几个。
            3 同时发送撤单和挂单的请求
            4 sleep一下
        """
        logger.info("3s后，开始整理订单")
        await asyncio.sleep(3)

        # 在最开始预设置价格为0
        current_price = 0

        # 当前的挂单信息
        current_buy_orders_info: Dict[float:str] = {}
        # 当前的持仓信息
        current_holding_info: Dict[float:str] = {}

        while True:
            try:
                # 每次循环获取最新的价格
                new_price = self.redis.get(redis_key=self.price_key)
                # 每次循环获取最新的网格设置
                info = self.redis.get(redis_key=self.info_key)

                # 初始化, 最开始启动的时候，记录价格
                if current_price == 0:
                    logger.warning(f"第一次循环，设置最初的格子,清空redis,撤了挂单和持仓,debug:{DEBUG}")

                    if DEBUG:
                        # 开始的时候撤销所有的挂单
                        order_infos = await self.exchange_api.get_symbol_order()
                        for order in order_infos:
                            if order['clientOrderId'].startswith("G_F"):
                                await self.exchange_api.cancel_order(client_order_id=order['clientOrderId'])

                        # 开始时候平仓
                        long_amount, _, _ = await self.exchange_api.get_symbol_position_short_long()
                        if long_amount > 0:
                            await self.exchange_api.create_order(
                                amount=long_amount,
                                direction=Direction.CLOSE_LONG,
                                order_type=OrderType.MARKET,
                            )

                    # 生成所有的格子
                    await self.update_orders(price=new_price)

                    order_prices_index = bisect_left(a=self.all_prices, x=float(new_price))
                    # 生成当前应该有的挂单信息
                    order_prices = [x for x in self.all_prices[:order_prices_index]][-int(info["max_buy_order_size"]):]
                    order_results = await asyncio.gather(
                        *[
                            self.buy(price=p) for p in order_prices
                        ]
                    )
                    amount = info["order_amount"]
                    # 记录当前的挂单信息
                    current_buy_orders_info = {
                        x["price"]: {
                            "order_id": x["clientOrderId"],
                            "amount": int(amount)
                        } for x in order_results
                    }
                    # 记录当前的价格
                    current_price = new_price
                    logger.info(f"最开始的挂单为:{current_buy_orders_info}")
                else:
                    logger.info(
                        f"开始整理订单,new_price:{new_price},upper bound:{round(new_price * (1 + info['grid_percent']), self.digits)},current order:{current_buy_orders_info} holding {current_holding_info}")

                    exchange_open_orders = await self.exchange_api.get_symbol_orders()
                    exchange_open_orders_df = pd.DataFrame(exchange_open_orders)
                    if exchange_open_orders_df.shape[0] > 0:
                        exchange_open_buy_orders_df = exchange_open_orders_df[exchange_open_orders_df['side'] == "BUY"]
                    else:
                        exchange_open_buy_orders_df = pd.DataFrame()

                    exchange_buy_orders = {x['price']: x['clientOrderId']
                                           for x in exchange_open_buy_orders_df.to_dict(orient="records")}

                    logger.info(f"开始检查卖单...")
                    # 检查有没有卖单成交了
                    for price in list(current_holding_info):
                        order_id = current_holding_info[price]
                        order_df = exchange_open_orders_df[exchange_open_orders_df["clientOrderId"] == order_id]
                        if not order_df.shape[0]:
                            logger.info(f'发现卖单成交,补一个买单{price}')
                            del current_holding_info[price]

                    logger.info(f"开始检查买单...")
                    # 检查有没有之前的买单成交了
                    for price in list(current_buy_orders_info):
                        order_id = current_buy_orders_info[price]["order_id"]
                        order_df = exchange_open_buy_orders_df[exchange_open_buy_orders_df["clientOrderId"] == order_id]
                        if not order_df.shape[0]:
                            buy_price = round(float(price), self.digits)
                            sell_price = round(float(buy_price) * (1 + info["grid_percent"]), self.digits)
                            logger.info(f"有买单成交:{buy_price}，补一个卖单:{sell_price}")
                            # TODO: make it at same time
                            sell_order_results = await self.sell(price=sell_price,
                                                                 amount=current_buy_orders_info[price]["amount"])
                            if sell_order_results:
                                current_holding_info[buy_price] = sell_order_results["clientOrderId"]
                            else:
                                logger.error(f"没有成功补上卖单,buy price:{buy_price}")

                    # 重新对齐内存里面的挂单信息
                    current_buy_orders_info = {
                        float(x['price']): {
                            'order_id': x['clientOrderId'],
                            'amount': int(x['amount'])
                        } for x in exchange_open_buy_orders_df.to_dict(orient="records")
                        if x['clientOrderId'].startswith("G_F")
                    }

                    # 如果修改了格子的比例/中位价，重新生成格子
                    if self.grid_percent != info["grid_percent"] or self.middle_price != info["middle_price"]:
                        # 生成所有的格子
                        logger.info(f"发现格子信息改了:{info}")
                        await self.update_orders(price=current_price)

                    order_prices_index = bisect_left(a=self.all_prices, x=float(new_price))
                    # 生成当前应该有的挂单信息
                    should_order_prices = [x for x in self.all_prices[:order_prices_index] if
                                           not current_holding_info.get(x)][-int(info["max_buy_order_size"]):]
                    order_prices = [x for x in should_order_prices if not current_buy_orders_info.get(x)]

                    # 撤掉多余的单
                    cancel_order_id = [order_id for price, order_id in exchange_buy_orders.items() if
                                       float(price) not in should_order_prices]
                    cancel_order_id.extend(
                        [
                            x['clientOrderId'] for x in exchange_open_buy_orders_df[
                            exchange_open_buy_orders_df.duplicated(subset=['price'])].to_dict(orient="records")
                        ]
                    )
                    if len(cancel_order_id) > 0:
                        logger.info(f"发现需要撤的单子:{cancel_order_id},open buy orders{exchange_buy_orders}")
                        await asyncio.gather(
                            *[
                                self.exchange_api.cancel_order(client_order_id=order_id) for order_id in cancel_order_id
                            ]
                        )
                        # 在内存挂单里面把撤掉的单子删了
                        for price in list(current_buy_orders_info):
                            info = current_buy_orders_info[price]
                            if info['order_id'] in cancel_order_id:
                                del current_buy_orders_info[price]

                    if len(order_prices) > 0:
                        order_results = await asyncio.gather(
                            *[
                                self.buy(price=p) for p in order_prices
                            ]
                        )
                        order_results_info = {
                            float(x["price"]): {
                                "order_id": x["clientOrderId"],
                                "amount": x["origQty"]
                            }
                            for x in order_results
                        }
                        current_buy_orders_info.update(
                            order_results_info
                        )
                        logger.info(f"下单成功:{order_results_info},当前挂单信息:{current_buy_orders_info}")

                # 每次循环休息一秒♨
                await asyncio.sleep(0.5)
            except Exception as e:
                logger.error(f"管理订单出错:{e}")
                await asyncio.sleep(1)

    async def main(self, task: str):
        await asyncio.gather(
            self.task_table[task]()
        )

class FutureGridStrategy(object):
    """Robot For Grid Strategy
    """

    def setup_test_data(self, info_key: str):
        """ test handler for generate test data
        """
        info = {
            "api_id": 32,
            # BTC
            # "symbol_id": 765,
            # ETH
            "symbol_id": 768,
            "order_amount": 1,
            "middle_price": 1800,
            "grid_percent": 0.001,
            # "grid_percent": 0.002,
            # "grid_percent": 0.003,
            "max_buy_order_size": 2,
            "maker_only": True,
            "start_time": TimeUtil.format_time(datetime.now()),
            "start_money": None,
        }

        self.redis.set(redis_key=info_key, value=info)

    def __init__(self, info_key: str, trade_key: str, pair_key: str, price_key: str, mode: str = "test"):
        self.redis = RedisHelper()

        # TODO: delete me
        if mode == "test":
            self.setup_test_data(info_key=info_key)

        #  info data
        info = self.redis.get(info_key)

        self.exchange_api: BinanceApi = ExchangeidSymbolidAPI(symbol_id=info["symbol_id"], api_id=info["api_id"])
        logger.info(
            f"successfully init exchange api {self.exchange_api.api.exchange}-{self.exchange_api.symbol.symbol}")
        if self.exchange_api.api.exchange == BinanceApi.EXCHANGE:
            self.robot = BinanceGridRobotV2(exchange_api=self.exchange_api, info=info, price_key=price_key,
                                            info_key=info_key, pair_key=pair_key, trade_key=trade_key)

            # 设置开始的资金
            info = self.redis.get(redis_key=info_key)
            if not info.get("start_money"):
                info['start_money'] = asyncio.run(self.robot.exchange_api.get_symbol_balance())["equity"]
                self.redis.set(redis_key=info_key, value=info)

    def run(self, task: str):
        asyncio.run(self.robot.main(task))

def run_task(task: str, info_key, pair_key, trade_key, price_key, mode):
    strategy = FutureGridStrategy(
        info_key=info_key,
        price_key=price_key,
        pair_key=pair_key,
        trade_key=trade_key,
        mode=mode
    )
    strategy.run(task)

@click.command()
@click.argument("task")
def start_grid(task):
    logger.info(f"开始运行:{task}")
    tasks = [
        "price", "order"
    ]
    if task == "test":
        info_key = RedisKeys.TEST_GRID_STRATEGY_FUTURE_INFO
        price_key = RedisKeys.TEST_GRID_STRATEGY_FUTURE_PRICE
        pair_key = RedisKeys.TEST_GRID_STRATEGY_FUTURE_ORDER_PAIR
        trade_key = RedisKeys.TEST_GRID_STRATEGY_FUTURE_TRADE
        mode = "test"
    elif task == "btc":
        info_key = RedisKeys.GRID_STRATEGY_FUTURE_INFO
        price_key = RedisKeys.GRID_STRATEGY_FUTURE_PRICE
        pair_key = RedisKeys.GRID_STRATEGY_FUTURE_ORDER_PAIR
        trade_key = RedisKeys.GRID_STRATEGY_FUTURE_TRADE
        mode = "btc"
    else:
        # 特殊处理单任务的情况
        DEBUG = True
        if DEBUG:
            info_key = RedisKeys.TEST_GRID_STRATEGY_FUTURE_INFO
            price_key = RedisKeys.TEST_GRID_STRATEGY_FUTURE_PRICE
            pair_key = RedisKeys.TEST_GRID_STRATEGY_FUTURE_ORDER_PAIR
            trade_key = RedisKeys.TEST_GRID_STRATEGY_FUTURE_TRADE
            mode = "test"
        else:
            info_key = RedisKeys.GRID_STRATEGY_FUTURE_INFO
            price_key = RedisKeys.GRID_STRATEGY_FUTURE_PRICE
            pair_key = RedisKeys.GRID_STRATEGY_FUTURE_ORDER_PAIR
            trade_key = RedisKeys.GRID_STRATEGY_FUTURE_TRADE
            mode = "btc"

        strategy = FutureGridStrategy(
            info_key=info_key,
            price_key=price_key,
            pair_key=pair_key,
            trade_key=trade_key,
            mode=mode
        )
        strategy.run(task)
        # 直接退出
        return

    # 正常情况
    running_tasks = []

    for task in tasks:
        p = Process(args=(task, info_key, pair_key, trade_key, price_key, mode), target=run_task)
        p.start()
        running_tasks.append(p)

    for running_task in running_tasks:
        running_task.join()

if __name__ == '__main__':
    start_grid()


# Make predictions using a machine learning model

#choosing the correct model and import
from sklearn.ensemble import RandomForestClassifier

#setup random seed
np.random.seed(42)

#create the input and output
X = heart_disease.drop('target', axis = 1)
y = heart_disease['target']

#split into test and train
X_train, X_test, y_trian, y_test = train_test_split(X, y, test_size = 0.2)

#train the model and instatntiate Random Forest classifier
clf = RandomForestClassifier()
clf.fit(X_train, y_trian)

#Evalute
clf.score(X_test, y_test)

#################
#M-1 to make prediction
clf.predict(X_test)

#################
#M-2
# predict_proba returns probability of a classification label
# returns the probability of 0, 1 for each input case
clf.predict_proba(X_test[:5])


import torch

class MyCollator(object):
    def __init__(self, tokenizer, max_seq_len):
        self.max_seq_len = max_seq_len  # this will be our model's maximum sequence length
        self.tokenizer = tokenizer   # we still need our tokenizer to know that the pad token's id is

    def __call__(self, input_batch):
        # Question for you: print the input_batch to see what it contains ;)
        output_batch = {
            "input_ids": [],
            "labels": [],
            "token_idx": [],
            "attention_mask": []
        }

        max_len = 0  # we'll need first to find out what is the longest line and then pad the rest to this length

        for instance in input_batch:
            instance_len = min(len(instance["instance_ids"]), self.max_seq_len-2)  # we will never have instances > max_seq_len-2
            max_len = max(max_len, instance_len)  # update max

        for instance in input_batch: # for each instance
            instance_ids = instance["instance_ids"]  # it's clearer if we use variables again
            instance_ner_ids = instance[ "instance_ner_ids"]
            instance_token_idx = instance["instance_token_idx"]

            # create the attention mask
            # this is a vector of 1s if the token is to be processed (0 if it's padding)
            instance_attention_mask = [1] * len(instance_ids)  # just a list of 1s for now

            # cut to max sequence length, if needed
            # notice how easy it is to process them together
            if len(instance_ids) > self.max_seq_len - 2:  # we need the -2 to accomodate for special tokens, this is a transformer's quirk
                instance_ids = instance_ids[:self.max_seq_len - 2]
                instance_ner_ids = instance_ner_ids[:self.max_seq_len - 2]
                instance_token_idx = instance_token_idx[:self.max_seq_len - 2]
                instance_attention_mask = instance_attention_mask[:self.max_seq_len - 2]

            """ Depending on your chosen model, the transformer might not have cls and sep, so don't use them
            if self.tokenizer.cls_token_id and self.tokenizer.sep_token_id:
                instance_ids = [self.tokenizer.cls_token_id] + instance_ids + [self.tokenizer.sep_token_id]
                instance_labels = [0] + instance_labels + [0]
                instance_token_idx = [-1] + instance_token_idx  # no need to pad the last, will do so automatically at return
            """

            # how much would we need to pad?
            pad_len = max_len - len(instance_ids)  # with this much

            if pad_len > 0:
                # pad the instance_ids
                instance_ids.extend( [self.tokenizer.pad_token_id] * pad_len)  # notice we're padding with tokenizer.pad_token_id

                # pad the instance_ner_ids
                instance_ner_ids.extend( [0] * pad_len)  # pad with zeros

                # pad the token_ids
                instance_token_idx.extend( [-1] * pad_len)  # notice we're padding with -1 as 0 is a valid word index

                # pad the attention mask
                instance_attention_mask.extend( [0] * pad_len)  # pad with zeros as well

            # add to batch
            output_batch["input_ids"].append(instance_ids)
            output_batch["labels"].append(instance_ner_ids)
            output_batch["token_idx"].append(instance_token_idx)
            output_batch["attention_mask"].append(instance_attention_mask)

        # we're done cutting and padding, let's transform them to tensors
        output_batch["input_ids"] = torch.tensor(output_batch["input_ids"])
        output_batch["labels"] = torch.tensor(output_batch["labels"])
        output_batch["token_idx"] = torch.tensor(output_batch["token_idx"])
        output_batch["attention_mask"] = torch.tensor(output_batch["attention_mask"])

        return output_batch


#!/usr/bin/env python
# encoding: utf-8
"""
@author: youfeng
@email: youfeng243@163.com
@license: Apache Licence
@file: view.py
@time: 2017/10/17 23:22
"""
import json

from flask import Blueprint
from flask import request
from flask_login import login_required

from exts.common import log, HTTP_OK, fail, success
from service.charge.impl import ChargeService
from service.charge.model import Charge

bp = Blueprint('charge', __name__, url_prefix='/admin')

# 创建费率
@bp.route('/charge', methods=['POST'])
@login_required
def new_charge():
    if not request.is_json:
        log.warn("参数错误...")
        return fail(HTTP_OK, u"need application/json!!")

    # json = {
    #     'name': 'xxx',
    #     'charge_mode': 'xxxx',
    # }

    name = request.json.get('name', None)
    charge_mode = request.json.get('charge_mode', None)
    if name is None or not isinstance(charge_mode, int) or charge_mode <= 0:
        log.warn("参数错误: name = {} charge_mode = {}".format(name, charge_mode))
        return fail(HTTP_OK, u"name or charge_mode 参数错误!")

    if ChargeService.find_by_name(name) is not None:
        log.warn("费率命名冲突, 数据库中已存在名字相同的费率模板: name = {}".format(name))
        return fail(HTTP_OK, u"费率命名冲突, 数据库中已存在名字相同的费率模板: name = {}".format(name))

    charge, is_success = ChargeService.create(name, charge_mode)
    if not is_success:
        log.warn("创建费率模板失败: name = {} charge_mode = {}".format(name, charge_mode))
        return fail(HTTP_OK, u"创建费率模板失败: name = {} charge_mode = {}".format(name, charge_mode))

    return success(charge.to_dict())

# 获得费率列表
@bp.route('/charge/list', methods=['POST'])
@login_required
def get_charge_list():
    return Charge.search_list()

# 删除费率
# 批量删除设备
@bp.route('/charge', methods=['DELETE'])
@login_required
def delete_charges():
    if not request.is_json:
        log.warn("参数错误...")
        return fail(HTTP_OK, u"need application/json!!")

    id_list = request.json.get('list', None)
    if not isinstance(id_list, list):
        log.warn("参数错误: id_list = {}".format(id_list))
        return fail(HTTP_OK, u"传入不是id列表")

    result_list = []
    for charge_id in id_list:
        charge = Charge.get(charge_id)
        if charge is None:
            log.warn("当前ID设备信息不存在: {}".format(charge_id))
            continue

        if not charge.delete():
            log.warn("费率信息删除失败: {}".format(json.dumps(charge.to_dict(), ensure_ascii=False)))
            continue

        result_list.append(charge_id)

    # 如果当前费率有被成功删除的，则需要更新redis中的费率信息
    # if len(result_list) > 0:
    #     charge = ChargeService.update_charge_to_redis()
    #     if charge is not None:
    #         log.info("完成一次 redis 中费率更新: 最新费率 charge_mode = {}".format(charge.charge_mode))
    #     else:
    #         log.info("更新费率到redis失败!")

    return success(result_list)

# 修改费率
@bp.route('/charge', methods=['PUT'])
@login_required
def update_charges():
    if not request.is_json:
        log.warn("参数错误...")
        return fail(HTTP_OK, u"need application/json!!")

    charge_id = request.json.get('id', None)
    if charge_id is None:
        log.warn("参数错误: charge_id 为None")
        return fail(HTTP_OK, u"参数错误: charge_id 为None")

    charge = Charge.get(charge_id)
    if charge is None:
        log.warn("当前费率模板不存在: charge_id = {}".format(charge_id))
        return fail(HTTP_OK, u"当前费率模板不存在: charge id = {}".format(charge))

    name = request.json.get('name', None)
    charge_mode = request.json.get('charge_mode', None)
    if not isinstance(name, basestring) and not isinstance(charge_mode, int):
        return fail(HTTP_OK, u"需要修改的参数错误，至少需要传入一个需要修改的参数")

    if isinstance(name, basestring) and name.strip() != '':
        other_charge = ChargeService.find_by_name(name.strip())
        if other_charge is not None and other_charge.id != charge.id:
            log.error("修改名称重复: charge_id = {} charge_name = {}".format(charge_id, name))
            return fail(HTTP_OK, u"需要修改的模板名称重复!")
        charge.name = name.strip()
    else:
        return fail(HTTP_OK, u'费率模板名称不能为空字符串!')

    if isinstance(charge_mode, int) and charge_mode > 0:
        charge.charge_mode = charge_mode
    else:
        return fail(HTTP_OK, u"charge_mode 参数类型错误!")

    if not charge.save():
        return fail(HTTP_OK, u"费率模板存储错误!")

    return success(charge.to_dict())

# 获得当前最新费率
# @bp.route('/charge/newest', methods=['GET'])
# @login_required
# def newest_charge():
#     charge = ChargeService.update_charge_to_redis()
#     if charge is not None:
#         log.info("完成一次 redis 中费率更新: 最新费率 charge_mode = {} ctime = {} ".format(
#             charge.charge_mode, charge.ctime.strftime('%Y-%m-%d %H:%M:%S')))
#         return success(charge.to_dict())
#
#     log.info("更新费率到redis失败!")
#     return fail(HTTP_OK, u"费率更新失败!")


import os, requests, logging, json
from datetime import datetime, timedelta
from time import sleep

absPath = os.path.abspath(__file__) # This little chunk makes sure the working directory is correct.
dname = os.path.dirname(absPath)
os.chdir(dname)
logging.basicConfig(filename=os.path.join(dname, 'logs', 'twitch_update.txt'), filemode='a', format='%(asctime)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.INFO)

class Twitch_Auth:
    def __init__(self, client_id: str, client_secret: str):
        self.ClientID = client_id
        self.ClientSecret = client_secret
        self.Authorization = None
        self.AuthExpiration = datetime.now()

    def GetAuthorization(self):
        url = "https://id.twitch.tv/oauth2/token"
        p = {"client_id": self.ClientID, "client_secret": self.ClientSecret, "grant_type": "client_credentials"}
        r = requests.post(url, params=p)
        if r.status_code != 200:
            logging.error(f"Request returned bad response. Response expected: 200. Response received: {str(r.status_code)}")
            return
        logging.info("Twitch response successful.")
        js = None
        try:
            js = r.json()
        except requests.exceptions.JSONDecodeError:
            logging.error(f"Could not decode JSON response. Response reads: \"{r.text}\"")
            return

        self.Authorization = js["access_token"]
        self.AuthExpiration = datetime.now() + timedelta(seconds=js["expires_in"])
        logging.info(f"Authorization key ready.")
        return

    def Validate(self) -> bool: # Returns whether or not the authorization is valid based on if it's expired or not.
        exp = self.AuthExpiration.timestamp()
        now = datetime.now().timestamp()
        if (exp - now) < 0:
            return False
        return True

    def GetSecondsRemaining(self) -> int: # Returns the approximate number of seconds left before the authorization key expires.
        exp = self.AuthExpiration.timestamp()
        now = datetime.now().timestamp()
        if (exp - now) < 0:
            return 0
        return (exp - now)

    def PreparedAuth(self) -> str:
        return f"Bearer {self.Authorization}"

def GetStreamData(User_ID: int, Client_ID: str, Auth: str):
    url = "https://api.twitch.tv/helix/streams"
    h = {'Authorization': Auth, "Client-Id": Client_ID}
    p = {'user_id': str(User_ID)}
    r = requests.get(url, headers=h, params=p)
    if r.status_code != 200:
        logging.error(f"Twitch returned non-standard response code: {str(r.status_code)} when acquiring stream data.")
        return
    with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dat', 'twitch_live_data.json'), 'w') as f:
        f.write(r.text)
    return

def GetUserData(User_ID: int, Client_ID: str, Auth: str):
    url = "https://api.twitch.tv/helix/users"
    h = {'Authorization': Auth, "Client-Id": Client_ID}
    p = {'id': str(User_ID)}
    r = requests.get(url, headers=h, params=p)
    if r.status_code != 200:
        logging.error(f"Twitch returned non-standard response code: {str(r.status_code)} when acquiring user data.")
        return
    with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dat', 'twitch_user_data.json'), 'w') as f:
        f.write(r.text)
    return

def GetChannelData(User_ID: int, Client_ID: str, Auth: str):
    url = "https://api.twitch.tv/helix/channels"
    h = {'Authorization': Auth, "Client-Id": Client_ID}
    p = {'broadcaster_id': str(User_ID)}
    r = requests.get(url, headers=h, params=p)
    if r.status_code != 200:
        logging.error(f"Twitch returned non-standard response code: {str(r.status_code)} when acquiring channel data.")
        return
    with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dat', 'twitch_channel_data.json'), 'w') as f:
        f.write(r.text)
    return

def main():
    ViccUserId = 518336885
    raw = None
    with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dat', 'twitch.json'), 'r') as f:
        raw = f.read()
    j = json.loads(raw)
    ClientID = j["client_id"]
    logging.info(f"Twitch Update Client starting with client ID: {ClientID}")
    auth = Twitch_Auth(ClientID, j["client_secret"])
    auth.GetAuthorization()
    ClientID = j["client_id"]
    del j, f, raw
    while True:
        if auth.GetSecondsRemaining() < 3600:
            logging.info("Refreshing Authorization token...")
            auth.GetAuthorization()
        GetStreamData(ViccUserId, ClientID, auth.PreparedAuth())
        GetUserData(ViccUserId, ClientID, auth.PreparedAuth())
        GetChannelData(ViccUserId, ClientID, auth.PreparedAuth())
        sleep(60)

# PREVENTS ERRORS FROM UNINTENTIONAL EXECUTION

if __name__ == "__main__":
    main()


import logging.config
import sys
from pnu.config import pub_config

class ConfigureLogging:

    def __init__(self):
        logging.config.fileConfig(pub_config["logging"]["location"],
                                  disable_existing_loggers=False)
        logging.getLogger('apscheduler').setLevel(logging.ERROR)
        logging.getLogger('pgoapi').setLevel(logging.ERROR)
        log = logging.getLogger('pnu')
        sys.stdout = LoggerWriter(log.debug)
        sys.stderr = LoggerWriter(log.warning)

class LoggerWriter:

    def __init__(self, level):
        self.level = level

    def write(self, message):
        # eliminate extra newlines in default sys.stdout
        if message != '\n':
            self.level(message)

    def flush(self):
        self.level(sys.stderr)


from sklearn.cluster import KMeans
from sklearn.datasets import load_iris
from sklearn.datasets import fetch_mldata
import matplotlib.pyplot as plt
import numpy as np
x = np.array([[5,3],[10,15],[15,12],[24,10],[30,45],[85,70],[71,80],[60,78],[55,52],[80,91]])
plt.scatter(x[:,0],x[:,1])

sos = []
for k in range(1,10):                           #Elbow method to find the number of cluster
    km = KMeans(n_clusters = k)
    km.fit(x)
    a = km.inertia_
    sos.append(a)
klist = [k for k in range(1,10)]
plt.plot(klist,sos)

print(km.cluster_centers_)
print(km.inertia_)
print(km.labels_)
print(km.n_iter_)

plt.scatter(x[:,0],x[:,1],c = km.labels_)
plt.scatter(km.cluster_centers_[:,0],km.cluster_centers_[:,1])

km.predict(np.array([25,25]).reshape(1,-1))

# Iris Dataset
ir = load_iris()
d = ir.data
sos1 = []
for k in range(1,15):                           #Elbow method to find the number of cluster
    km = KMeans(n_clusters = k)
    km.fit(d)
    a = km.inertia_
    sos1.append(a)
klist1 = [k for k in range(1,15)]
plt.plot(klist1,sos1)

print(km.cluster_centers_)
print(km.inertia_)
print(km.labels_)
print(km.n_iter_)

km.predict(np.array([1.5,4.8,3.6,2.4]).reshape(1,-1))

#mnist Dataset
mnist = fetch_mldata("mnist-original")
d = mnist.data
sos1 = []
for k in range(1,15):                           #Elbow method to find the number of cluster
    km = KMeans(n_clusters = k)
    km.fit(d)
    a = km.inertia_
    sos1.append(a)
klist1 = [k for k in range(1,15)]
plt.plot(klist1,sos1)

print(km.cluster_centers_)
print(km.inertia_)
print(km.labels_)
print(km.n_iter_)


import pytest

from diff_performance import diff_functions, diff2_functions, diff_n

def compare(diff_result, str1, str2):
    print(diff_result, str1, str2)
    wp, np = diff_result
    wp2 = [None if c == '*' else c for c in str1]
    np2 = list(str2)
    assert wp == wp2
    assert(sorted(np) == sorted(np2))

diff_test_data = [
    (("aaaaa", "bbbbb"), ("*****", "")),
    (("abcde", "axxxx"), ("a****", "")),
    (("abcde", "abcde"), ("abcde", "")),
    (("aabcd", "aabcd"), ("aabcd", "")),
    (("abcde", "bcdea"), ("*****", "abcde")),
    (("aayyy", "axxxa"), ("a****", "a")),
    (("axaxc", "xxaab"), ("*xa**", "ax")),
]

@pytest.mark.parametrize("input,expected", diff_test_data)
def test_diff(input, expected):
    for diff_func in diff_functions:
        compare(diff_func(*input), *expected)

diff2_test_data = [
    (("aaaaa", "bbbbb", "ccccc"), ('*****', '')),
    (("aaaaa", "abbbb", "aaccc"), ('aa***', '')),
    (("aaxxx", "abbbb", "acxca"), ('a*x**', 'a')),
    (("aaxxx", "bbbaa", "cxxca"), ('**x**', 'aax')),
    (("toxic", "train", "taper"), ('t**i*', '')),
    (("rapid", "train", "taper"), ('*api*','r')),
    (("affix", "taper", "bring"), ('*****','ai')),
    (("tapir", "train", "taper"), ('tapir', '')),
    (("tapir", "train", "paste"), ('ta*i*', 'rp')),
]

@pytest.mark.parametrize("input,expected", diff2_test_data)
def test_diff2(input, expected):
    for diff_func in diff2_functions:
        compare(diff_func(*input), *expected)

@pytest.mark.parametrize("input,expected", diff2_test_data)
def test_diff_n(input, expected):
    base, *words = input
    compare(diff_n(base, words), *expected)

idempotency_test_data = [
    (("aaaaa", "bbbbb"), ("aaaaa", "bbbbb", "bbbbb")),
    (("abcde", "axxxx"), ("abcde", "axxxx", "axxxx")),
    (("abcde", "abcde"), ("abcde", "abcde", "abcde")),
    (("aabcd", "aabcd"), ("aabcd", "aabcd", "aabcd")),
    (("abcde", "bcdea"), ("abcde", "bcdea", "bcdea")),
    (("aayyy", "axxxa"), ("aayyy", "axxxa", "axxxa")),
    (("axaxc", "xxaab"), ("axaxc", "xxaab", "xxaab")),
]

@pytest.mark.parametrize("diff_input,diff2_input", idempotency_test_data)
def test_idempotency_diff2(diff_input, diff2_input):
    for diff, diff2 in zip(diff_functions, diff2_functions):
        assert diff(*diff_input) == diff2(*diff2_input)

@pytest.mark.parametrize("diff_input,diff2_input", idempotency_test_data)
def test_idempotency_diff_n(diff_input, diff2_input):
    base1, *words1 = diff_input
    base2, *words2 = diff2_input
    words3 = words2 + [words2[-1]]
    assert diff_n(base1, words1) == diff_n(base2, words2)
    assert diff_n(base1, words1) == diff_n(base2, words3)


#importing libraries:
from airflow import DAG
from airflow.operators.dummy import DummyOperator
from airflow.utils.email import send_email
from airflow.providers.apache.spark.operators.spark_submit import SparkSubmitOperator
from datetime import timedelta,datetime

def Success_Email(context):
    dr = context.get("dag_run")
    msg = "DAG has been completed"
    subject = f"DAG {dr} Completed successfully"
    send_email(to=['shreeparakhi96@gmail.com'],subject=subject,html_content=msg)

def Failure_Email(context):
    dr = context.get("dag_run")
    msg = "DAG has been failied"
    subject = f"DAG {dr} failed"
    send_email(to=['shreeparakhi96@gmail.com'],subject=subject,html_content=msg)

# These args will get passed on to the python operator
default_args = {
    'owner': 'shreepad',
    'depends_on_past': True,
    'start_date': datetime(2022, 3, 14),
    'email': ['shreeparakhi96@gmail.com'],
    'email_on_failure': True,
    'email_on_retry': False,
    'retries': 1,
    'retry_delay': timedelta(minutes=1),
    'on_failure_callback' : Failure_Email
}

# define the DAG
dag = DAG(
    'emp_airflow_dag',
    default_args=default_args,
    description='emp table airflow dag',
    schedule_interval=timedelta(days=1),
    catchup=False
)

#spark submit task
emp_spark_submit_job = SparkSubmitOperator(application='/home/saif/PycharmProjects/cohort_c9_training/emp_airflow.py',
    task_id='emp_spark_submit_job',
    conf={'master':'yarn','deploy-mode':'client'},
    driver_memory='1g',
    executor_memory='2g',
    num_executors=1,
    executor_cores=4,
    on_success_callback = Success_Email,
    dag=dag
)

#dummy
start_task = DummyOperator(
    task_id = 'start_Task',
    dag=dag
)

end_task = DummyOperator(
    task_id = 'end_task',
    dag=dag
)

start_task >> emp_spark_submit_job >> end_task


"""Marshmallow bindings for indy proofs."""

from typing import Mapping, Sequence

from marshmallow import EXCLUDE, fields, validate

from ...messaging.models.base import BaseModel, BaseModelSchema
from ...messaging.valid import (
    INDY_CRED_DEF_ID_EXAMPLE,
    INDY_CRED_DEF_ID_VALIDATE,
    INDY_REV_REG_ID_EXAMPLE,
    INDY_REV_REG_ID_VALIDATE,
    INDY_SCHEMA_ID_EXAMPLE,
    INDY_SCHEMA_ID_VALIDATE,
    INT_EPOCH_EXAMPLE,
    INT_EPOCH_VALIDATE,
    NUM_STR_ANY_EXAMPLE,
    NUM_STR_ANY_VALIDATE,
    NUM_STR_WHOLE_EXAMPLE,
    NUM_STR_WHOLE_VALIDATE,
)
from ...utils.tracing import AdminAPIMessageTracingSchema
from .predicate import Predicate
from .requested_creds import (
    IndyRequestedCredsRequestedAttrSchema,
    IndyRequestedCredsRequestedPredSchema,
)

class IndyEQProof(BaseModel):
    """Equality proof for indy primary proof."""

    class Meta:
        """Equality proof metadata."""

        schema_class = "IndyEQProofMeta"

    def __init__(
        self,
        revealed_attrs: Mapping[str, str] = None,
        a_prime: str = None,
        e: str = None,
        v: str = None,
        m: Mapping[str, str] = None,
        m2: str = None,
        **kwargs,
    ):
        """Initialize equality proof object."""
        super().__init__(**kwargs)
        self.revealed_attrs = revealed_attrs
        self.a_prime = a_prime
        self.e = e
        self.v = v
        self.m = m
        self.m2 = m2

class IndyEQProofSchema(BaseModelSchema):
    """Indy equality proof schema."""

    class Meta:
        """Indy equality proof metadata."""

        model_class = IndyEQProof
        unknown = EXCLUDE

    revealed_attrs = fields.Dict(
        keys=fields.Str(metadata={"example": "preference"}),
        values=fields.Str(
            validate=NUM_STR_ANY_VALIDATE, metadata={"example": NUM_STR_ANY_EXAMPLE}
        ),
    )
    a_prime = fields.Str(
        validate=NUM_STR_WHOLE_VALIDATE, metadata={"example": NUM_STR_WHOLE_EXAMPLE}
    )
    e = fields.Str(
        validate=NUM_STR_WHOLE_VALIDATE, metadata={"example": NUM_STR_WHOLE_EXAMPLE}
    )
    v = fields.Str(
        validate=NUM_STR_WHOLE_VALIDATE, metadata={"example": NUM_STR_WHOLE_EXAMPLE}
    )
    m = fields.Dict(
        keys=fields.Str(metadata={"example": "master_secret"}),
        values=fields.Str(
            validate=NUM_STR_WHOLE_VALIDATE, metadata={"example": NUM_STR_WHOLE_EXAMPLE}
        ),
    )
    m2 = fields.Str(
        validate=NUM_STR_WHOLE_VALIDATE, metadata={"example": NUM_STR_WHOLE_EXAMPLE}
    )

class IndyGEProofPred(BaseModel):
    """Indy GE proof predicate."""

    class Meta:
        """Indy GE proof predicate metadata."""

        schema_class = "IndyGEProofPredSchema"

    def __init__(
        self,
        attr_name: str = None,
        p_type: str = None,
        value: int = None,
        **kwargs,
    ):
        """Initialize indy GE proof predicate."""
        super().__init__(**kwargs)
        self.attr_name = attr_name
        self.p_type = p_type
        self.value = value

class IndyGEProofPredSchema(BaseModelSchema):
    """Indy GE proof predicate schema."""

    class Meta:
        """Indy GE proof predicate metadata."""

        model_class = IndyGEProofPred
        unknown = EXCLUDE

    attr_name = fields.Str(
        metadata={"description": "Attribute name, indy-canonicalized"}
    )
    p_type = fields.Str(
        validate=validate.OneOf([p.fortran for p in Predicate]),
        metadata={"description": "Predicate type"},
    )
    value = fields.Integer(
        metadata={"strict": True, "description": "Predicate threshold value"}
    )

class IndyGEProof(BaseModel):
    """Greater-than-or-equal-to proof for indy primary proof."""

    class Meta:
        """GE proof metadata."""

        schema_class = "IndyGEProofMeta"

    def __init__(
        self,
        u: Mapping[str, str] = None,
        r: Mapping[str, str] = None,
        mj: str = None,
        alpha: str = None,
        t: Mapping[str, str] = None,
        predicate: IndyGEProofPred = None,
        **kwargs,
    ):
        """Initialize GE proof object."""
        super().__init__(**kwargs)
        self.u = u
        self.r = r
        self.mj = mj
        self.alpha = alpha
        self.t = t
        self.predicate = predicate

class IndyGEProofSchema(BaseModelSchema):
    """Indy GE proof schema."""

    class Meta:
        """Indy GE proof schema metadata."""

        model_class = IndyGEProof
        unknown = EXCLUDE

    u = fields.Dict(
        keys=fields.Str(),
        values=fields.Str(
            validate=NUM_STR_WHOLE_VALIDATE, metadata={"example": NUM_STR_WHOLE_EXAMPLE}
        ),
    )
    r = fields.Dict(
        keys=fields.Str(),
        values=fields.Str(
            validate=NUM_STR_WHOLE_VALIDATE, metadata={"example": NUM_STR_WHOLE_EXAMPLE}
        ),
    )
    mj = fields.Str(
        validate=NUM_STR_WHOLE_VALIDATE, metadata={"example": NUM_STR_WHOLE_EXAMPLE}
    )
    alpha = fields.Str(
        validate=NUM_STR_WHOLE_VALIDATE, metadata={"example": NUM_STR_WHOLE_EXAMPLE}
    )
    t = fields.Dict(
        keys=fields.Str(),
        values=fields.Str(
            validate=NUM_STR_WHOLE_VALIDATE, metadata={"example": NUM_STR_WHOLE_EXAMPLE}
        ),
    )
    predicate = fields.Nested(IndyGEProofPredSchema)

class IndyPrimaryProof(BaseModel):
    """Indy primary proof."""

    class Meta:
        """Indy primary proof metadata."""

        schema_class = "IndyPrimaryProofSchema"

    def __init__(
        self,
        eq_proof: IndyEQProof = None,
        ge_proofs: Sequence[IndyGEProof] = None,
        **kwargs,
    ):
        """Initialize indy primary proof."""
        super().__init__(**kwargs)
        self.eq_proof = eq_proof
        self.ge_proofs = ge_proofs

class IndyPrimaryProofSchema(BaseModelSchema):
    """Indy primary proof schema."""

    class Meta:
        """Indy primary proof schema metadata."""

        model_class = IndyPrimaryProof
        unknown = EXCLUDE

    eq_proof = fields.Nested(
        IndyEQProofSchema,
        allow_none=True,
        metadata={"description": "Indy equality proof"},
    )
    ge_proofs = fields.Nested(
        IndyGEProofSchema,
        many=True,
        allow_none=True,
        metadata={"description": "Indy GE proofs"},
    )

class IndyNonRevocProof(BaseModel):
    """Indy non-revocation proof."""

    class Meta:
        """Indy non-revocation proof metadata."""

        schema_class = "IndyNonRevocProofSchema"

    def __init__(
        self,
        x_list: Mapping = None,
        c_list: Mapping = None,
        **kwargs,
    ):
        """Initialize indy non-revocation proof."""
        super().__init__(**kwargs)
        self.x_list = x_list
        self.c_list = c_list

class IndyNonRevocProofSchema(BaseModelSchema):
    """Indy non-revocation proof schema."""

    class Meta:
        """Indy non-revocation proof schema metadata."""

        model_class = IndyNonRevocProof
        unknown = EXCLUDE

    x_list = fields.Dict(keys=fields.Str(), values=fields.Str())
    c_list = fields.Dict(keys=fields.Str(), values=fields.Str())

class IndyProofProofProofsProof(BaseModel):
    """Indy proof.proof.proofs constituent proof."""

    class Meta:
        """Indy proof.proof.proofs constituent proof schema."""

        schema_class = "IndyProofProofProofsProofSchema"

    def __init__(
        self,
        primary_proof: IndyPrimaryProof = None,
        non_revoc_proof: IndyNonRevocProof = None,
        **kwargs,
    ):
        """Initialize proof.proof.proofs constituent proof."""
        super().__init__(**kwargs)
        self.primary_proof = primary_proof
        self.non_revoc_proof = non_revoc_proof

class IndyProofProofProofsProofSchema(BaseModelSchema):
    """Indy proof.proof.proofs constituent proof schema."""

    class Meta:
        """Indy proof.proof.proofs constituent proof schema metadata."""

        model_class = IndyProofProofProofsProof
        unknown = EXCLUDE

    primary_proof = fields.Nested(
        IndyPrimaryProofSchema, metadata={"description": "Indy primary proof"}
    )
    non_revoc_proof = fields.Nested(
        IndyNonRevocProofSchema,
        allow_none=True,
        metadata={"description": "Indy non-revocation proof"},
    )

class IndyProofProofAggregatedProof(BaseModel):
    """Indy proof.proof aggregated proof."""

    class Meta:
        """Indy proof.proof aggregated proof metadata."""

        schema_class = "IndyProofProofAggregatedProofSchema"

    def __init__(
        self,
        c_hash: str = None,
        c_list: Sequence[Sequence[int]] = None,
        **kwargs,
    ):
        """Initialize indy proof.proof agreggated proof."""
        super().__init__(**kwargs)
        self.c_hash = c_hash
        self.c_list = c_list

class IndyProofProofAggregatedProofSchema(BaseModelSchema):
    """Indy proof.proof aggregated proof schema."""

    class Meta:
        """Indy proof.proof aggregated proof schema metadata."""

        model_class = IndyProofProofAggregatedProof
        unknown = EXCLUDE

    c_hash = fields.Str(metadata={"description": "c_hash value"})
    c_list = fields.List(
        fields.List(fields.Int(metadata={"strict": True})),
        metadata={"description": "c_list value"},
    )

class IndyProofProof(BaseModel):
    """Indy proof.proof content."""

    class Meta:
        """Indy proof.proof content metadata."""

        schema_class = "IndyProofProofSchema"

    def __init__(
        self,
        proofs: Sequence[IndyProofProofProofsProof] = None,
        aggregated_proof: IndyProofProofAggregatedProof = None,
        **kwargs,
    ):
        """Initialize indy proof.proof content."""
        super().__init__(**kwargs)
        self.proofs = proofs
        self.aggregated_proof = aggregated_proof

class IndyProofProofSchema(BaseModelSchema):
    """Indy proof.proof content schema."""

    class Meta:
        """Indy proof.proof content schema metadata."""

        model_class = IndyProofProof
        unknown = EXCLUDE

    proofs = fields.Nested(
        IndyProofProofProofsProofSchema,
        many=True,
        metadata={"description": "Indy proof proofs"},
    )
    aggregated_proof = fields.Nested(
        IndyProofProofAggregatedProofSchema,
        metadata={"description": "Indy proof aggregated proof"},
    )

class RawEncoded(BaseModel):
    """Raw and encoded attribute values."""

    class Meta:
        """Raw and encoded attribute values metadata."""

        schema_class = "RawEncodedSchema"

    def __init__(
        self,
        raw: str = None,
        encoded: str = None,
        **kwargs,
    ):
        """Initialize raw and encoded attribute values."""
        super().__init__(**kwargs)
        self.raw = raw
        self.encoded = encoded

class RawEncodedSchema(BaseModelSchema):
    """Raw and encoded attribute values schema."""

    class Meta:
        """Raw and encoded attribute values schema metadata."""

        model_class = RawEncoded
        unknown = EXCLUDE

    raw = fields.Str(metadata={"description": "Raw value"})
    encoded = fields.Str(
        validate=NUM_STR_ANY_VALIDATE,
        metadata={"description": "Encoded value", "example": NUM_STR_ANY_EXAMPLE},
    )

class IndyProofRequestedProofRevealedAttr(RawEncoded):
    """Indy proof requested proof revealed attr."""

    class Meta:
        """Indy proof requested proof revealed attr metadata."""

        schema_class = "IndyProofRequestedProofRevealedAttrSchema"

    def __init__(
        self,
        sub_proof_index: int = None,
        **kwargs,
    ):
        """Initialize indy proof requested proof revealed attr."""
        super().__init__(**kwargs)
        self.sub_proof_index = sub_proof_index

class IndyProofRequestedProofRevealedAttrSchema(RawEncodedSchema):
    """Indy proof requested proof revealed attr schema."""

    class Meta:
        """Indy proof requested proof revealed attr schema metadata."""

        model_class = IndyProofRequestedProofRevealedAttr
        unknown = EXCLUDE

    sub_proof_index = fields.Int(
        metadata={"strict": True, "description": "Sub-proof index"}
    )

class IndyProofRequestedProofRevealedAttrGroup(BaseModel):
    """Indy proof requested proof revealed attr group."""

    class Meta:
        """Indy proof requested proof revealed attr group metadata."""

        schema_class = "IndyProofRequestedProofRevealedAttrGroupSchema"

    def __init__(
        self,
        sub_proof_index: int = None,
        values: Mapping[str, RawEncoded] = None,
        **kwargs,
    ):
        """Initialize indy proof requested proof revealed attr."""
        super().__init__(**kwargs)
        self.sub_proof_index = sub_proof_index
        self.values = values

class IndyProofRequestedProofRevealedAttrGroupSchema(BaseModelSchema):
    """Indy proof requested proof revealed attr group schema."""

    class Meta:
        """Indy proof requested proof revealed attr group schema metadata."""

        model_class = IndyProofRequestedProofRevealedAttrGroup
        unknown = EXCLUDE

    sub_proof_index = fields.Int(
        metadata={"strict": True, "description": "Sub-proof index"}
    )
    values = fields.Dict(
        keys=fields.Str(),
        values=fields.Nested(RawEncodedSchema),
        metadata={
            "description": "Indy proof requested proof revealed attr groups group value"
        },
    )

class IndyProofRequestedProofPredicate(BaseModel):
    """Indy proof requested proof predicate."""

    class Meta:
        """Indy proof requested proof requested proof predicate metadata."""

        schema_class = "IndyProofRequestedProofPredicateSchema"

    def __init__(
        self,
        sub_proof_index: int = None,
        **kwargs,
    ):
        """Initialize indy proof requested proof predicate."""
        super().__init__(**kwargs)
        self.sub_proof_index = sub_proof_index

class IndyProofRequestedProofPredicateSchema(BaseModelSchema):
    """Indy proof requested prrof predicate schema."""

    class Meta:
        """Indy proof requested proof requested proof predicate schema metadata."""

        model_class = IndyProofRequestedProofPredicate
        unknown = EXCLUDE

    sub_proof_index = fields.Int(
        metadata={"strict": True, "description": "Sub-proof index"}
    )

class IndyProofRequestedProof(BaseModel):
    """Indy proof.requested_proof content."""

    class Meta:
        """Indy proof.requested_proof content metadata."""

        schema_class = "IndyProofRequestedProofSchema"

    def __init__(
        self,
        revealed_attrs: Mapping[str, IndyProofRequestedProofRevealedAttr] = None,
        revealed_attr_groups: Mapping[
            str,
            IndyProofRequestedProofRevealedAttrGroup,
        ] = None,
        self_attested_attrs: Mapping = None,
        unrevealed_attrs: Mapping = None,
        predicates: Mapping[str, IndyProofRequestedProofPredicate] = None,
        **kwargs,
    ):
        """Initialize indy proof requested proof."""
        super().__init__(**kwargs)
        self.revealed_attrs = revealed_attrs
        self.revealed_attr_groups = revealed_attr_groups
        self.self_attested_attrs = self_attested_attrs
        self.unrevealed_attrs = unrevealed_attrs
        self.predicates = predicates

class IndyProofRequestedProofSchema(BaseModelSchema):
    """Indy proof requested proof schema."""

    class Meta:
        """Indy proof requested proof schema metadata."""

        model_class = IndyProofRequestedProof
        unknown = EXCLUDE

    revealed_attrs = fields.Dict(
        keys=fields.Str(),
        values=fields.Nested(IndyProofRequestedProofRevealedAttrSchema),
        allow_none=True,
        metadata={"description": "Proof requested proof revealed attributes"},
    )
    revealed_attr_groups = fields.Dict(
        keys=fields.Str(),
        values=fields.Nested(IndyProofRequestedProofRevealedAttrGroupSchema),
        allow_none=True,
        metadata={"description": "Proof requested proof revealed attribute groups"},
    )
    self_attested_attrs = fields.Dict(
        metadata={"description": "Proof requested proof self-attested attributes"}
    )
    unrevealed_attrs = fields.Dict(metadata={"description": "Unrevealed attributes"})
    predicates = fields.Dict(
        keys=fields.Str(),
        values=fields.Nested(IndyProofRequestedProofPredicateSchema),
        metadata={"description": "Proof requested proof predicates."},
    )

class IndyProofIdentifier(BaseModel):
    """Indy proof identifier."""

    class Meta:
        """Indy proof identifier metadata."""

        schema_class = "IndyProofIdentifierSchema"

    def __init__(
        self,
        schema_id: str = None,
        cred_def_id: str = None,
        rev_reg_id: str = None,
        timestamp: int = None,
        **kwargs,
    ):
        """Initialize indy proof identifier."""
        super().__init__(**kwargs)
        self.schema_id = schema_id
        self.cred_def_id = cred_def_id
        self.rev_reg_id = rev_reg_id
        self.timestamp = timestamp

class IndyProofIdentifierSchema(BaseModelSchema):
    """Indy proof identifier schema."""

    class Meta:
        """Indy proof identifier schema metadata."""

        model_class = IndyProofIdentifier
        unknown = EXCLUDE

    schema_id = fields.Str(
        validate=INDY_SCHEMA_ID_VALIDATE,
        metadata={
            "description": "Schema identifier",
            "example": INDY_SCHEMA_ID_EXAMPLE,
        },
    )
    cred_def_id = fields.Str(
        validate=INDY_CRED_DEF_ID_VALIDATE,
        metadata={
            "description": "Credential definition identifier",
            "example": INDY_CRED_DEF_ID_EXAMPLE,
        },
    )
    rev_reg_id = fields.Str(
        allow_none=True,
        validate=INDY_REV_REG_ID_VALIDATE,
        metadata={
            "description": "Revocation registry identifier",
            "example": INDY_REV_REG_ID_EXAMPLE,
        },
    )
    timestamp = fields.Int(
        allow_none=True,
        validate=INT_EPOCH_VALIDATE,
        metadata={
            "strict": True,
            "description": "Timestamp epoch",
            "example": INT_EPOCH_EXAMPLE,
        },
    )

class IndyProof(BaseModel):
    """Indy proof."""

    class Meta:
        """Indy proof metadata."""

        schema_class = "IndyProofSchema"

    def __init__(
        self,
        proof: IndyProofProof = None,
        requested_proof: IndyProofRequestedProof = None,
        identifiers: Sequence[IndyProofIdentifier] = None,
        **kwargs,
    ):
        """Initialize indy proof."""
        super().__init__(**kwargs)
        self.proof = proof
        self.requested_proof = requested_proof
        self.identifiers = identifiers

class IndyProofSchema(BaseModelSchema):
    """Indy proof schema."""

    class Meta:
        """Indy proof schema metadata."""

        model_class = IndyProof
        unknown = EXCLUDE

    proof = fields.Nested(
        IndyProofProofSchema, metadata={"description": "Indy proof.proof content"}
    )
    requested_proof = fields.Nested(
        IndyProofRequestedProofSchema,
        metadata={"description": "Indy proof.requested_proof content"},
    )
    identifiers = fields.Nested(
        IndyProofIdentifierSchema,
        many=True,
        metadata={"description": "Indy proof.identifiers content"},
    )

class IndyPresSpecSchema(AdminAPIMessageTracingSchema):
    """Request schema for indy proof specification to send as presentation."""

    self_attested_attributes = fields.Dict(
        required=True,
        keys=fields.Str(metadata={"example": "attr_name"}),
        values=fields.Str(
            metadata={
                "example": "self_attested_value",
                "description": (
                    "Self-attested attribute values to use in requested-credentials"
                    " structure for proof construction"
                ),
            }
        ),
        metadata={"description": "Self-attested attributes to build into proof"},
    )
    requested_attributes = fields.Dict(
        required=True,
        keys=fields.Str(metadata={"example": "attr_referent"}),
        values=fields.Nested(IndyRequestedCredsRequestedAttrSchema),
        metadata={
            "description": (
                "Nested object mapping proof request attribute referents to"
                " requested-attribute specifiers"
            )
        },
    )
    requested_predicates = fields.Dict(
        required=True,
        keys=fields.Str(metadata={"example": "pred_referent"}),
        values=fields.Nested(IndyRequestedCredsRequestedPredSchema),
        metadata={
            "description": (
                "Nested object mapping proof request predicate referents to"
                " requested-predicate specifiers"
            )
        },
    )
    trace = fields.Bool(
        required=False,
        metadata={
            "description": "Whether to trace event (default false)",
            "example": False,
        },
    )


# -*- coding: utf-8 -*-
import csv
import requests
import config as cfg
from requests.packages.urllib3.exceptions import InsecureRequestWarning
#import subprocess
#import sys
import os
#from requests.auth import HTTPBasicAuth
import json
import re
from dbconn import Dbconn
#from fulltext import Fulltext

_author_ = "Colin Sippl"
_organization_ = "University Library of Regensburg"
_email_ = "colin.sippl@ur.de"
_license_ = "GPL 3"

"""
VLP data parser for Invenio.
"""

class Parser:
    """
    This class parses CSV files.
    """

    INVENIO_SERVICE_URL = cfg.INVENIO_SERVICE_URL
    ROW_LIMIT = cfg.ROW_LIMIT
    REQUESTS_SSL_VERIFY = cfg.REQUESTS_SSL_VERIFY
    query_a = 'SELECT * FROM vl_literature and id >= 1 and id <=' + \
        str(ROW_LIMIT)
    #query_a = "SELECT * FROM vl_literature where volumeid is not null or startpages is not null LIMIT " + str(ROW_LIMIT)
    query_b = "SELECT * FROM vl_literature where volumeid is null and startpages is null and referencetype != 'Journal Article' LIMIT " + \
        str(ROW_LIMIT)

    # hide InsecureRequestWarning
    # https://stackoverflow.com/questions/27981545/suppress-insecurerequestwarning-unverified-https-request-is-being-made-in-pytho
    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

    @staticmethod
    def parse_file(path):
        print(path)
        # https://docs.python.org/2/library/csv.html
        #response_array = []

        with open(path, 'r') as csvfile:
            reader = csv.DictReader(csvfile)
            number_rows = 0
            for row in reader:
                if number_rows < Parser.ROW_LIMIT:
                    record = Parser.create_records(row, True)
                    # response_array.append(record)
                    # Parser.create_people(row)
                    print(
                        "==========================================================================")
                    print("Zeile " + str(number_rows))
                    number_rows += 1

    # generate people entries from vl-people2.csv
    @staticmethod
    def parse_people_file(path):
        print(path)
        # https://docs.python.org/2/library/csv.html
        with open(path, 'rU') as csvfile:
            reader = csv.DictReader(csvfile, delimiter=';')
        #reader = csv.reader(open(path, 'rU'), dialect=csv.excel_tab, delimiter=';')
            number_rows = 0
            for row in reader:
                if number_rows < Parser.ROW_LIMIT:
                    # print(row)
                    Parser.create_people(row)
                    number_rows += 1

    @staticmethod
    def create_records_db(query, do_lines):
        dbconn = Dbconn()
        dbconn.connect()
        # reference, id, referencetype, author, title, "year", "date", volume,
        # secondarytitle, startpages, pages, sql_year, sql_date, fullreference,
        # shortreference, liseurl, lastupdate, volumeid, authorized, online, cd,
        # project, authorrefdisplay, yearrefdisplay, titlerefdisplay, detailsrefdisplay,
        # related, endnoteexport, volumeid_search, notes, transcription, url

        # id, author, title, "year", "date", volume,
        # secondarytitle, startpages, pages, sql_year, sql_date, fullreference,
        # shortreference, liseurl, lastupdate, volumeid, authorized, online, cd,
        # project, authorrefdisplay, yearrefdisplay, titlerefdisplay, detailsrefdisplay,
        # related, endnoteexport, volumeid_search, notes, transcription, url
        dbconn.execute_st(query)
        result = dbconn.fetch_result()
        dbconn.disconnect()
        records = []
        for data in result:

            json_data = {}
            json_data['identifier'] = data['reference']
            cref_data = {}
            cref = Parser.get_related_authors(data['reference'])
            if cref is not None:
                cref_data['$ref'] = "https://vlp.ur.de/api/resolver/author/" + cref
                json_data['authid'] = cref
                json_data['creator'] = cref_data
            json_data['title'] = data['title']
            if 'secondarytitle' in data:
                json_data['alternative'] = data['secondarytitle']
            #json_data['keywords'] = [data['referencetype']]
            json_data['type'] = data['referencetype']
            if 'year' in data:
                json_data['issued'] = data['year']
            #else:
            #    json_data['issued'] = data['created']
            #json_data['ocr'] = Fulltext.parse_ocr_files_for_id(data['reference'])

            if data['author'] is not None:
                #json_data['creator'] = data['author']
                #print(data[year'author'])
                json_data['contributors'] = Parser.get_authors(data['author'], cref, "author")
                #json_data['contributors'] = Parser.rebuild_refs_sites(data['author'], cref, "author")

            if 'volume' in data and data['volume'] is not None:
                json_data['volume'] = data['volume']

            if 'volume' in data and data['volume'] is not None:
                json_data['pages'] = data['pages']

            if data['referencetype'] == '(Journal)' or data['referencetype'] == '(JournalVolume)':
                json_data['title'] = data['secondarytitle']
            # else:
            # if  data['secondarytitle'] is not None and not Parser.is_empty(data['secondarytitle']):

            if 'detailsrefdisplay' in data and data['detailsrefdisplay'] is not None and not Parser.is_empty(data['detailsrefdisplay']):
                json_data['detailsRefDisplay'] = data['detailsrefdisplay']

            if 'volumeid' in data and data['volumeid'] is not None and not Parser.is_empty(data['volumeid']) and data['volumeid'] is not '':
                json_data['volume_id'] = data['volumeid']
                ref = {}
                ref['$ref'] = "https://vlp.ur.de/api/resolver/record/" + \
                    data['volumeid']
                json_data['record'] = ref

            if do_lines:
                Parser.post_record(json_data, 'records')
            else:
                records.append(json_data)

        return records

    @staticmethod
    def is_empty(field):
        return field.isspace()

    @staticmethod
    def get_related_authors(lit_reference):
        dbconn = Dbconn()
        dbconn.connect()
        query = "SELECT vl_people.reference FROM vl_literature, vl_people WHERE vl_people.fullname=vl_literature.author AND vl_literature.reference='" + lit_reference + "'"
        dbconn.execute_st(query)
        result = dbconn.fetch_result()
        dbconn.disconnect()
        if result is not None:
            for row in result:
                print(result)
                return row['reference']

    @staticmethod
    def create_people(query):
        dbconn = Dbconn()
        dbconn.connect()
        # id, created, created_by, modified_by, reference, marker, clip,
        # complete, "name", experiments, related, born, died, born_day, born_month,
        # born_year, died_day, died_month, died_year, first_name, career, degrees,
        # positions, technologies, main1, born_place, died_place, shortreference,
        # main2, main3, main4, main5, sources, image, comment_intern, literature,
        # sources2, fullname, lifespan4frontier, works, sourceslist, variationsname,
        # variationsfirstname, clip2, searchstring, lifespandisplay, marker2,
        # category, statistik, completed, customsearchstring, in_essay, bibliography,
        # imagesource, modified, article_autor, article_autor_url
        dbconn.execute_st(query)
        result = dbconn.fetch_result()
        dbconn.disconnect()
        people = []
        for row in result:
            people_data = {}
            #people_data['name'] = row['shortreference']
            # https://invenio-indexer.readthedocs.io/en/latest/usage.html#indexing-a-record
            people_data['$schema'] = 'http://vlp.ur.de/schemas/authors/author-v1.0.0.json'
            people_data['name'] = row['name']
            related_literature_ids = []
            if row['first_name'] is not None and row['first_name'] is not '':
                people_data['first_name'] = row['first_name']
            people_data['identifier'] = row['reference']
            if row['career'] is not None:
                people_data['career'] = Parser.rebuild_hyperlinks(row['career'], None)[0]
            if row['born_year'] is not None and row['died_year'] is not None:
                people_data['life_span'] = {"year_of_birth": int(
                    row['born_year']), "year_of_death": int(row['died_year'])}
            #life_span = Parser.get_life_span(row['shortReference'])
            #people_data['life_span'] = life_span
            if row['born_place'] is not None:
                people_data['born_place'] = row['born_place']
            if row['died_place'] is not None:
                people_data['died_place'] = row['died_place']
            if row['image'] is not None:
                people_data['image'] = row['image']
            if row['degrees'] is not None:
                people_data['degrees'] = row['degrees']

            if row['works'] is not None:
                people_data['references'] = Parser.rebuild_refs_sites(row['works'], None , None)

            #if do_lines:
            #    Parser.post_record(people_data, 'authors')
            #else:
            people.append(people_data)
            #Parser.post_record(people_data, 'authors')
            # people.append(people_data)

        for person in people:
            if 'references' in person:
                for reference in person['references']:
                    if 'identifier' in reference:
                        if reference['identifier'].startswith('lit'):
                            related_literature_ids.append(reference['identifier'])

        #print(related_literature_ids)
        people_literature = Parser.create_sites_literature(list(set(related_literature_ids)))
        Parser.create_json_file(people_literature, 'people_literature')

        return people

    @staticmethod
    def create_sites():
        #query = "SELECT * FROM public.vl_sites where reference='sit137'"
        query = 'SELECT * FROM vl_sites LIMIT ' + str(Parser.ROW_LIMIT)
        dbconn = Dbconn()
        dbconn.connect()
        dbconn.execute_st(query)
        result = dbconn.fetch_result()
        dbconn.disconnect()
        sites = []
        related_literature_ids = []
        for row in result:
            site = {}
            site['$schema'] = 'http://vlp.ur.de/schemas/sites/site-v1.0.0.json'
            site['identifier'] = row['reference']
            site['name'] = row['labname']

            if row['established'] is not None:
                site['date'] = row['established']
            if row['location'] is not None:
                site['location'] = row['location']

            keywordsb = Parser.get_labs_disciplines(row['labs_disciplines'])
            if len(keywordsb) > 0:
                site['keywordsb'] = keywordsb

            #site['keywordsb'] = [{'tag':'text'},{'tag':'toll'}]

            contributors = []
            contributor = {}
            if row['people_head'] is not None:
                contributors = contributors + Parser.rebuild_refs_sites(row['people_head'], None , "head")
            if row['people_architect'] is not None:
                contributors = contributors + Parser.rebuild_refs_sites(row['people_architect'], None, "architect")
            if row['people_deptheads'] is not None:
                contributors = contributors + Parser.rebuild_refs_sites(row['people_deptheads'], None, "depthead")
            if row['people_others'] is not None:
                contributors = contributors + Parser.rebuild_refs_sites(row['people_others'],None, "N/A")
            if row['people_assistants'] is not None:
                contributors = contributors + Parser.rebuild_refs_sites(row['people_assistants'],None, "assistant")

            if len(contributors) > 0:
                site['contributors'] = contributors
                #print(contributors)

            images = []
            if row['img1id'] is not None and row['img1id'] != '':
                images.append({"identifier" : row['img1id']})
            if row['img2id'] is not None and row['img2id'] != '':
                images.append({"identifier" : row['img2id']})
            if row['img3id'] is not None and row['img3id'] != '':
                images.append({"identifier" : row['img3id']})
            if row['img4id'] is not None and row['img4id'] != '':
                images.append({"identifier" : row['img4id']})

            if len(images) > 0:
                site['images'] = images
                #print(images)

            if row['further_descriptions'] != '':
                site['references'] = Parser.rebuild_refs_sites(row['further_descriptions'], None , None)

            sites.append(site)

        for site in sites:
            if 'references' in site:
                for reference in site['references']:
                    if 'identifier' in reference:
                        if reference['identifier'].startswith('lit'):
                            related_literature_ids.append(reference['identifier'])

        #print(related_literature_ids)
        site_literature = Parser.create_sites_literature(list(set(related_literature_ids)))
        Parser.create_json_file(site_literature, 'site_literature')

        return sites

    @staticmethod
    def create_sites_literature(related_literature_ids):
        literature = []
        for reference in related_literature_ids:
            query = "SELECT * FROM vl_literature WHERE reference='" + reference + "'"
            result = Parser.create_records_db(query, False)
            literature.append(result[0])

        return literature

    @staticmethod
    def create_people_old(data):
        people_data = {}
        people_data['name'] = data['shortReference']
        people_data['reference'] = data['reference']
        life_span = Parser.get_life_span(data['shortReference'])
        people_data['life_span'] = life_span
        Parser.post_record(people_data, 'authors')

    # delete?
    @staticmethod
    def get_authors(authors_string, pid, role):
        if authors_string is not None:
            authors_string = authors_string + '\n'
            authors = []
            pattern = re.compile(r"(.*?)\n")
            #x = re.findall(r"(.*?)\n", authors_string)
            # for author in x:
            #    print(author)
            for m in re.finditer(pattern, authors_string):
                contributor = {}
                if pid is not None and pid is not '':
                    contributor['identifier'] = pid
                if role is not None:
                    contributor['role'] = role
                if role is None:
                    contributor['role'] = "author"
                #if authors_string != "\n" and m.group(1) != "" and pid is not None:
                if authors_string != "\n" and m.group(1) != "":
                    contributor['name'] =  m.group(1)
                    print(contributor)
                    authors.append(contributor)

            # print(authors)
            return authors
        #else:
        #    return [{"name": "none"}]

    # delete?
    @staticmethod
    def get_date_span(shortReference):
        life_span = {}
        x = re.search(r"\((\d{4}) - (\d{4})\)", shortReference)
        if x:
            life_span["issued"] = int(x.group(1))
            life_span["year_of_death"] = int(x.group(2))
        else:
            life_span["year_of_birth"] = 0
            life_span["year_of_death"] = 0
        #print(x.group(1) + ' ' + x.group(2))
        return life_span

    # delete?
    @staticmethod
    def get_life_span(shortReference):
        life_span = {}
        x = re.search(r"\((\d{4}) - (\d{4})\)", shortReference)
        if x:
            life_span["year_of_birth"] = int(x.group(1))
            life_span["year_of_death"] = int(x.group(2))
        else:
            life_span["year_of_birth"] = 0
            life_span["year_of_death"] = 0
        #print(x.group(1) + ' ' + x.group(2))
        return life_span

    @staticmethod
    def get_labs_disciplines(labsdisciplines):
        labs_disciplines = []
        pattern = re.compile(r"(.*?)\n")
        for m in re.finditer(pattern, labsdisciplines):
            if m.group() is not '\n':
                labs_disciplines.append({"tag":m.group(1)})
        print(labs_disciplines)
        return labs_disciplines

    @staticmethod
    def rebuild_hyperlinks(pers_career, pid):
        #print(pers_career, pid)
        hit = re.search(r"(<link (.*?)>)(.*?)(<\/link>)", pers_career)
        #pid = None
        #print(pers_career, pid, hit)
        if hit is None:
            return pers_career, pid
        if hit is not None:
            pid = re.search(r"(ref=\"(.*?)\")",
                            hit.group().replace('""', '"')).group(2)
            replacement = hit.group().replace('""', '"').replace('<link ref', '<a href').replace('</link', '</a').replace(pid, Parser.get_url_from_pid(pid))
            pers_career = pers_career[:hit.span()[0]] + \
                    replacement + pers_career[hit.span()[1]:]
            pers_career, pid = Parser.rebuild_hyperlinks(pers_career, pid)
            return pers_career, pid

        #return '', ''
        return Parser.rebuild_hyperlinks(pers_career, pid)

    @staticmethod
    def rebuild_refs_sites(pers_career, references, role):
        hit = re.search(r"(<link (.*?)>)(.*?)(<\/link>)", pers_career)
        if references is None:
            references = []
        if hit is None and pers_career == '':
            return references
        if hit is None and pers_career != '':
            pattern = re.compile(r"(.*?)\n")
            for m in re.finditer(pattern, pers_career):
                if m.group() is not '\n':
                    reference = dict()
                    reference['name'] = m.group().replace('\n', '')
                    if role is not None:
                        reference['role'] = role
                    references.append(reference)
            return references
        if hit is not None:
            reference = {}
            if role is not None:
                reference['role'] = role
            #print(pers_career, "hit " + str(hit))
            pid_hit = re.search(r"(ref=\"(.*?)\")",
                            hit.group().replace('""', '"'))
            if pid_hit is None:
                return references
            pid = pid_hit.group(2)
            if pid.startswith('per'):
                reference['identifier'] = pid
                reference['$ref'] = "https://vlp.ur.de/api/resolver/author/" + \
                    pid
                pers_career = pers_career[hit.span()[1]:]
                references.append(reference)
                return Parser.rebuild_refs_sites(pers_career, references, role)
            if pid.startswith('lit'):
                reference['identifier'] = pid
                reference['$ref'] = "https://vlp.ur.de/api/resolver/record/" + \
                    pid
                pers_career = pers_career[hit.span()[1]:]
                references.append(reference)
                return Parser.rebuild_refs_sites(pers_career, references, None)
            # TODO Essays!!

            #if pid.startswith('sit') and pid:
            #    reference['$ref'] = "https://vlp.ur.de/api/resolver/site/" + \
            #        pid
            #    pers_career = pers_career[hit.span()[1]:]
            #    references.append(reference)
            #    return Parser.rebuild_refs_sites(pers_career, references, None)
            reference['identifier'] = pid
            replacement = hit.group().replace('""', '"').replace('<link ref', '<a href').replace('</link', '</a').replace(pid, Parser.get_url_from_pid(pid))
            reference['name'] = hit.group(3)
            reference['link'] = replacement
            pers_career = pers_career[hit.span()[1]:]
            references.append(reference)
            return Parser.rebuild_refs_sites(pers_career, references, role)

    @staticmethod
    def get_url_from_pid(pid):
        if pid.startswith('lit'):
            return "/records/" + pid
        if pid.startswith('per'):
            return "/authors/" + pid
        if pid.startswith('sit'):
            return "/sites/" + pid
        return "/records/" + pid

    @staticmethod
    def post_record(json_data, api_root):
        url = Parser.INVENIO_SERVICE_URL + "/api/" + api_root + "/?prettyprint=1"
        data = json.dumps(json_data, ensure_ascii=False).encode('utf-8')
        headers = {'content-type': 'application/json'}
        print("==================================" +
              json_data['identifier'] + "===================================")
        response = requests.post(
            url, data=data, headers=headers, verify=Parser.REQUESTS_SSL_VERIFY)
        if response.ok:
            print("Status code: " + str(response.status_code))
            print(json_data['identifier'] + " - success!")
            print("Message: " + response.text)
        else:
            print("Status code: " + str(response.status_code))
            print("Message: " + response.text)
            print(json.dumps(json_data, sort_keys=True, indent=4))
        print("===================================END====================================")

    @staticmethod
    def create_json_file(records_array, name):

        outfile = open(name + '.json', 'w', encoding='utf-8')
        #print(name)
        data = json.dumps(records_array, ensure_ascii=False, sort_keys=True, indent=4)
        #print(data)
        outfile.write(data)
        outfile.close()
        #json.dump(data, outfile, ensure_ascii=False, indent=4)

    @staticmethod
    def do_reindex_via_cli():
        """
        $ pipenv run invenio index reindex --pid-type recid --yes-i-know
        $ pipenv run invenio index run
        """
        p = os.popen(
            "pipenv run invenio index reindex --pid-type recid --yes-i-know")
        print(p.read())
        p = os.popen("pipenv run invenio index run")
        print(p.read())

if __name__ == "__main__":
    # Parser.parse_file('data/vlp.csv')
    # Parser.parse_people_file('data/VL-People2.csv')
    aut_str = """
Delacroix, Henri
E.Cassirer
L.Jordan
A.Sechehaye
W.Doroszewski
R.A.S.Paget
K.Bühler
H.Pongs
A.Meillet
J.Vendryes
P.Meriggi
V.Broendal
N.Trubetzkoy
Edward Sapir
J.van Ginneken
A.Sommerfelt
A.W.de Groot
O.Jespersen
Ch.Bally
Gustave Guillaume
A.Grégoire
M.Cohen
A.Gelb
K.Goldstein
"""
    Parser.get_authors(aut_str)

    vlp_links = '1633-1635 studied medicine in Leiden and Basel; M.D. 1637, University of Basel; 1641 moved to Amsterdam and worked there as a physician and was member of the <i>Amsterdam College of Physicians</i>; 1658 professor of medicine, University of Leiden; 1669-1670 vice chancellor of the University; 1671-1674 his main work <i>Praxeos medicae idea nova</i> was published in four volumes." \
"1858 studies of medicine at the University of Breslau under <link ref=""per86"">Rudolf Heidenhain</link> and in Berlin under <link ref=""per64"">Emil Du Bois-Reymond</link>; 1862 medical degree from the University of Berlin with a dissertation on invertebrate muscle physiology; 1864 assistant to <link ref=""per87"">Helmholtz</link> at the <link ref=""sit108"">Physiological Institute of the University of Heidelberg</link>; 1870 Privatdozent; 1871 associate professor and acting head of the Physiological Institute at Heidelberg; later that year return to Berlin; 1872 professor of physiology at the University of Halle in succession to <link ref=""per169"">Friedrich Goltz</link>; founder and director of the physiology institute at Halle (1881)." \
"Studied medicine for several years in Göttingen; 1823 M.D. at the University of Göttingen with a dissertation entitled <i>De cauterio actuali seu de igne ut medicamento</i>; 1824 worked on practical medicine at the University of Berlin and 1825 on zoology and comparative anatomy at the Sorbonne, Paris; 1825 habilitation, University of Göttingen; 1835 associate professor, same institution; 1836 professor of zoology and comparative anatomy at the faculty of medicine, University of Göttingen; since 1840 director of the zoological collection; founded the scientific study of hormones; 1861 retirement.<link ref="lit15702" page="p0007tablevii">Rabbit holder according to Czermak</link>'
    Parser.rebuild_hyperlinks(vlp_links)


import argparse
from players import PLAYERS

parser = argparse.ArgumentParser()
parser.add_argument(
    '-b',
    '--benchmark',
    action='store_true',
    dest='benchmark',
    help='execute 100 iterations')

parser.add_argument(
    '-t',
    '--tournament',
    action='store_true',
    dest='tournament',
    help='execute a tournament among all agents except random')

parser.add_argument(
    '-tr',
    '--tournament_against_random',
    action='store_true',
    dest='tournament_against_random',
    help='execute a tournament of all agents against random')

parser.add_argument(
    '-p0',
    choices=PLAYERS.keys(),
    help='selects player 0',
    required=True)
parser.add_argument(
    '-p1',
    choices=PLAYERS.keys(),
    help='selects player 1',
    required=True)

parser.add_argument('-v', '--verbose', action='count',
                    default=0, help='increase log verbosity')
parser.add_argument('-d',
                    '--debug',
                    action='store_const',
                    dest='verbose',
                    const=2,
                    help='log debug messages (same as -vv)')