#!/usr/bin/env python3

import paho.mqtt.client as mqtt
import importlib
import time
import argparse
import pkgutil
from socket import error as socket_error
from threading import Timer

from api import API
from ghoust_player import Player

class GHOUST:

    def __init__(self, game_list, host, port, join_mode='auto'):

        self.clients = dict()
        self.host = host
        self.port = port
        self.join_mode = join_mode

        self.client = mqtt.Client("GHOUST_SRV", clean_session=False)
        self.client.will_set("GHOUST/server/status", "EXIT")
        self.client._on_connect = self._on_connect
        self.client._on_message = self._on_message

        # config parameters
        self.max_games = 4

        # game modules
        self.games = []
        self.activegames = []
        self.setgames(game_list)

        self.gamemodes = []
        for i,m, ispkg in pkgutil.walk_packages(path="games/."):
            if 'games.' in m:
                self.gamemodes.append(m[6:])

        self.api = API(self)

    #### game functions ####

    def setgames(self, game_list):
        # stop old games
        for g in self.games:
            g.stop()

        self.games = []
        self.activegames = []

        # start new games
        if self.join_mode == "auto":
            game_list = game_list[:1]

        for i, g in enumerate(game_list):
            m = importlib.import_module("games." + g)
            C = getattr(m, g)
            game = C(i)
            game.setup()
            self.games.append(game)
            self.activegames.append("{}:{}".format(i, g))

        if self.join_mode == "auto":
            for _,p in self.clients.items():
                p.set_game(self.games[0])

    # buzzer, vibro val: [0-1023, 0-1023], [duration (ms), frequency (hz)]
    # led val: [0-1023, 0-1023, 0-10123], [r, g, b]
    # parameter: ["motor", "buzzer", "led"]

    def _game_config(self, game, parameter, val=None, preset=None):
        if not (0 <= game <= 4) or parameter not in ["motor", "buzzer", "led"]:
            print("game number or parameter not valid")
            return
        topic = "GHOUST/game/{}/{}".format(game, parameter)

        if preset != None:
            if not (0 <= int(preset) <= 9):
                print("vibrate preset not in range")
            self.client.publish(topic, "PRESET:" + preset)

        if val != None:
            if (not (0 <= val[0] <= 1023) or
                not (0 <= val[1] <= 1023) or
                    (parameter == "led" and not(0 <= val[2] <= 1023))):
                print("vibrate values not in range")
            fstring = "RAW:{:04},{:04}"
            if parameter == "led":
                fstring = "RAW:{:04},{:04},{:04}"
            self.client.publish(topic, fstring.format(*val))

    #### mqtt callbacks ####

    def _on_connect(self, client, userdata, flags, rc):
        print(("Connected with result code " + str(rc)))

        client.subscribe("GHOUST/server/changegame")
        client.subscribe("GHOUST/clients/+/status")
        client.subscribe("GHOUST/clients/+/events/button")
        client.subscribe("GHOUST/clients/+/events/accelerometer")
        client.subscribe("GHOUST/clients/+/events/gestures")

    def _on_message(self, client, userdata, msg):
        topic = msg.topic.split("/")
        payload = str(msg.payload, 'utf-8')
        if len(topic) < 3:
            print(("msg tree too short! debug: " + msg.topic + " " + payload))
            return

        if topic[1] == "server":
            if topic[2] == "changegame":
                self.setgames(payload.split(","))
            return

        pid = topic[2]
        subtree = topic[3]
        if subtree == "status":
            if payload == "CONNECTED":

                pobj = Player(pid, client)
                self.clients.update({pid: pobj})
                if len(self.games) == 1:
                    pobj.set_game(self.games[0])

            elif payload == "DISCONNECTED" and pid in self.clients:
                pobj = self.clients[pid]
                self.clients.pop(pid)
                pobj.set_game(None)

                del pobj

        elif subtree == "events":
            # pass message to game engine callbacks
            elem = topic[4]
            if pid not in self.clients:
                print("{} not in client list. msg: {} {}".format(pid, topic, payload))
                return
            pobj = self.clients[pid]

            if pobj.status == "SELECT_GAME":
                if elem == "button":
                    if payload == "CLICK":
                        # dirty...
                        pobj.select_game(
                            (pobj.select_game_n + 1) % len(self.games))
                    elif payload == "LONGPRESS":
                        pobj.set_game(self.games[pobj.select_game_n])
            else:
                if elem == "button":
                    # Leave game
                    if payload == "LONGPRESS" and len(self.games) > 1:
                        pobj.set_game(None)
                    else:    # let game know of button press
                        pobj.game._on_button(pobj, payload)
                elif elem == "accelerometer":
                    pobj.game._on_accelerometer(pobj, payload)
                elif elem == "gestures":
                    pobj.game._on_gestures(pobj, payload)

    def stop(self):
        for g in self.games:
            g.stop()
        self.client.loop_stop()

    def run(self):
        # Start API
        self.api.run(threaded=True, debug=False)

        # Start Ghoust
        for i in range(3):
            try:
                self.client.connect(self.host, self.port, 10)
                break
            except socket_error as e:
                print(("socket.error: [{}] {}".format(e.errno, e.strerror)))
                if i == 2:
                    raise e
                print("retrying after 10s")
                time.sleep(10)

        mqtt_game_modes_list = ','.join(self.gamemodes)
        mqtt_active_games = ','.join(self.activegames)
        self.client.publish("GHOUST/server/status", "ACTIVE")
        self.client.publish("GHOUST/server/status/activegames", mqtt_active_games, retain=True)
        self.client.publish("GHOUST/server/status/gamemodes", mqtt_game_modes_list, retain=True)

        self.client.loop_forever()

#############################

def filter_clients(c, status=""):
    if status != "":
        return [e for _, e in list(c.items()) if e.status == status]
    return []

if __name__ == "__main__":

    parser = argparse.ArgumentParser(
        description="GHOUST. it is a game. it is very good")
    parser.add_argument('games', metavar='game', type=str,
                        nargs='+', help="the games to be run")
    parser.add_argument('-H', '--host', nargs='?', type=str,
                        default='localhost', help="Host where MQTT server is running")
    parser.add_argument('-p', '--port', nargs='?', type=int,
                        default=1883, help="Port where MQTT server is running")
    parser.add_argument('--debug', action='store_true', help="run debug clients")
    args = parser.parse_args()

    g = GHOUST(args.games, args.host, args.port)

    if args.debug:
        import ghoust_debug_clients
        debugclients = ghoust_debug_clients.ghoust_debug(num_clients=3)

    try:
        g.run()
    except KeyboardInterrupt:
        pass
    except:
        raise
    finally:
        g.stop()
        if args.debug:
            debugclients.stop()


import numpy as np
import torch

def as_numpy(x:torch.Tensor) -> np.ndarray:
    if isinstance(x, np.ndarray):
        return x
    x = x.detach()
    if x.device.type >= 'cuda':
        x = x.cpu()
    x = x.numpy()
    return x

def accuracy(y:torch.Tensor, t:torch.Tensor) -> float:
    y = as_numpy(y)
    t = as_numpy(t)

    i = np.argmax(y, axis=1)
    c = i == t
    a = float(np.sum(c)) / np.prod(t.shape)
    return a


import os
import sys
import argparse
import cv2
import numpy as np
prj_path = os.path.join(os.path.dirname(__file__), '..')
if prj_path not in sys.path:
    sys.path.append(prj_path)

from lib.test.evaluation import Tracker

def run_imgdir(tracker_name, tracker_param, imgdir, optional_box=None, debug=None, save_results=False,index=None):
    """Run the tracker on your webcam.
    args:
        tracker_name: Name of tracking method.
        tracker_param: Name of parameter file.
        debug: Debug level.
    """
    tracker = Tracker(tracker_name, tracker_param, "video")

    tracker.run_imgdir(videofilepath=imgdir, optional_box=optional_box, debug=debug, save_results=save_results,index=index)

def main():
    parser = argparse.ArgumentParser(description='Run the tracker on your webcam.')
    parser.add_argument('tracker_name', type=str,help='Name of tracking method.')

    parser.add_argument('--optional_box', type=float, default=None, nargs="+", help='optional_box with format x y w h.')
    parser.add_argument('--debug', type=int, default=0, help='Debug level.')
    parser.add_argument('--save_results', dest='save_results', action='store_true', help='Save bounding boxes')
    parser.set_defaults(save_results=False)

    args = parser.parse_args()

    tracker_param='vitb_384_mae_ce_32x4_ep300'
    imgdir=''
    while True:
        panduan=input('b for box, p for point:')
        if (panduan=='b' or panduan=="B" or panduan=='p' or panduan=="P"):
            break
    if (panduan=='b' or panduan=='B'):
        run_imgdir(args.tracker_name,tracker_param, imgdir, args.optional_box, args.debug, args.save_results,index=(0,128,0))
    else:
        run_imgdir(args.tracker_name, tracker_param, imgdir, False, args.debug, args.save_results,index=(0,128,0))

if __name__ == '__main__':
    main()


import xlrd
from xlutils.copy import copy

rb = xlrd.open_workbook('name.xls', formatting_info=True)
r_sheet = rb.sheet_by_index(0)
r = r_sheet.nrows
wb = copy(rb)
current_sheet = wb.get_sheet(0)

for sheet in rb.sheets():
    for rowidx in range(r):
        row = sheet.row(rowidx)
        for colidx, cell in enumerate(row):
            if cell.value == "cell_value":
                try:
                    coming_number_in_stock_red = float(
                        str(sheet.cell(rowidx, colidx + 3)).partition('number:')[2].partition('(X')[0])
                    coming_money_red = float(
                        str(sheet.cell(rowidx, colidx + 4)).partition('number:')[2].partition('(X')[0])
                    consumption_number_in_stock_red = float(
                        str(sheet.cell(rowidx, colidx + 5)).partition('number:')[2].partition('(X')[0])
                    consumption_money_red = float(
                        str(sheet.cell(rowidx, colidx + 6)).partition('number:')[2].partition('(X')[0])

                    coming_number_in_stock_white = float(
                        str(sheet.cell(rowidx - 1, colidx + 3)).partition('number:')[2].partition('(X')[0])
                    coming_money_white = float(
                        str(sheet.cell(rowidx - 1, colidx + 4)).partition('number:')[2].partition('(X')[0])
                    consumption_number_in_stock_white = float(
                        str(sheet.cell(rowidx - 1, colidx + 5)).partition('number:')[2].partition('(X')[0])
                    consumption_money_white = float(
                        str(sheet.cell(rowidx - 1, colidx + 6)).partition('number:')[2].partition('(X')[0])

                    coming_number_in_stock_done = coming_number_in_stock_white - coming_number_in_stock_red
                    coming_money_done = coming_money_white - coming_money_red
                    consumption_number_in_stock_done = \
                        consumption_number_in_stock_white - consumption_number_in_stock_red
                    consumption_money_done = consumption_money_white - consumption_money_red

                    current_sheet.write(rowidx - 1, colidx + 3, coming_number_in_stock_done)
                    current_sheet.write(rowidx - 1, colidx + 4, coming_money_done)
                    current_sheet.write(rowidx - 1, colidx + 5, consumption_number_in_stock_done)
                    current_sheet.write(rowidx - 1, colidx + 6, consumption_money_done)

                    wb.save('name.xls')

                except ValueError:
                    continue


import numpy as np
from terminaltables import AsciiTable

from src.Environment import Environment
from src.algorithms import step1, expected_profit
from src.bandit.banditEnvironments.BidBanditEnvironment import BidBanditEnvironment
from src.bandit.LearningStats import plot_results
from src.bandit.learner.ucb.UCBOptimalBidLearner import UCBOptimalBidLearner
from src.constants import _Const
from src.scripts.environment_plotter import plot_everything

def main():
    CONST = _Const()

    interactive = False
    prices = np.arange(1, 101)
    bids = np.linspace(1, CONST.BID_MAX+10, num=10, dtype=np.int32)

    n_rounds = 365
    future_visits_delay = 30

    for envN, seedV in enumerate([3077083550, 538541279]):
        env = Environment(random_seed=seedV) if seedV is not None else Environment()
        print(f'Running with seed {env.get_seed()}')

        opt_price, opt_bid, profit = step1(env, prices, bids)

        bandit_env = BidBanditEnvironment(env, opt_price, bids, future_visits_delay)

        expected_profits = np.array([expected_profit(env, opt_price, b) for b in bids])
        gaps = np.max(expected_profits) - expected_profits
        rescaled_gaps = gaps/np.max(gaps)
        clairvoyant_cumulative_profits = np.cumsum([np.max(expected_profits)]*n_rounds)

        ucb_leaner = UCBOptimalBidLearner(bandit_env, lambda r: r % 400 == 401)
        ucb_leaner.learn(n_rounds)
        ucb_cumulative_profits = ucb_leaner.compute_cumulative_exp_profits(expected_profits)

        env.print_summary()

        for name, learner in [("UCB", ucb_leaner)]:
            print(name)
            print("Bid:               " + " ".join(f'{p:10d}' for p in bandit_env.bids))
            print("Projected profits: " + " ".join(f'{p:10.2f}' for p in learner.compute_projected_profits()))
            print("Expected profits:  " + " ".join(f'{p:10.2f}' for p in learner.compute_expected_profits()))
            print("Averages:          " + " ".join(f'{p:10.2f}' for p in learner.compute_average_new_clicks_per_arm()))
            print("Number of pulls:   " + " ".join(f'{p:10d}' for p in learner.get_number_of_pulls()))
            print()

        print(f"Optimal bid is {opt_bid} with expected profit {profit:.2f}")

        print("Bid:      " + " ".join(f'{p:10d} ' for p in bandit_env.bids))
        print("Expected: " + " ".join(f'{p:10.2f} ' for p in expected_profits))
        print("Gaps:     " + " ".join(f'{p:10.2f} ' for p in gaps))
        print("Rescaled: " + " ".join(f'{p*100:10.2f}%' for p in rescaled_gaps))

        if interactive:
            plot_results(["ucb"], [ucb_cumulative_profits], clairvoyant_cumulative_profits, n_rounds, smooth=False)

        table_data = [['Bid']]
        for p in bids:
            table_data.append([f'{p:.2f}'])

        table_data[0].append('True expected')
        for i, p in enumerate(bids):
            table_data[i + 1].append(f'{expected_profits[i]:.2f}')

        table_data[0].append('Gaps')
        for i, p in enumerate(bids):
            table_data[i + 1].append(f'{rescaled_gaps[i]*100:4.1f}')

        table_data[0] += ['Pulls', 'Learner expected']
        for i, (n, e) in enumerate(zip(ucb_leaner.get_number_of_pulls(), ucb_leaner.compute_expected_profits())):
            table_data[i + 1].append(f'{n}')
            table_data[i + 1].append(f'{e:.2f}')

        table = AsciiTable(table_data)
        for i in range(len(table_data[0])):
            table.justify_columns[i] = 'right'
        print(table.table)

        dir = '../../../report/figures/step5'

        with open(dir+f'/output{envN}.txt', 'w', encoding='utf8') as output_file:
            output_file.write(f' Seed: {env.get_seed()}\n')
            output_file.write(table.table)
            output_file.write(f'\n\n Optimal price: {opt_price:.2f}, Optimal bid: {opt_bid:.2f}')

        plot_results(["UCB"],
                     [ucb_cumulative_profits],
                     clairvoyant_cumulative_profits, n_rounds,
                     dest_file_path=dir+f'/plot{envN}')

        plot_everything(environment=env, path=dir+f'/env{envN}')

if __name__ == "__main__":
    main()


import unittest
import requests
import json
import sys
import os
sys.path.append('../..')

from config.config import *
from lib.db import *
from lib.read_excel import *
from lib.case_log import log_case_info

class TestUserReg(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.data_list = excel_to_list(os.path.join(data_path, "test_user_data.xlsx"), "TestUserReg")  # 读取TestUserReg工作簿的所有数据

    def test_user_reg_normal(self):
        case_data = get_test_data(self.data_list, 'test_user_reg_normal')
        if not case_data:
            logging.error("用例数据不存在")
        url = case_data.get('url')
        data = json.loads(case_data.get('data'))  # 转为字典，需要取里面的name进行数据库检查
        expect_res = json.loads(case_data.get('expect_res'))  # 转为字典，断言时直接断言两个字典是否相等
        name = data.get("name")  # 范冰冰

        # 环境检查
        if check_user(name):
            del_user(name)
        # 发送请求
        res = requests.post(url=url, json=data)  # 用data=data 传字符串也可以
        # 期望响应结果，注意字典格式和json格式的区别（如果有true/false/null要转化为字典格式）
        log_case_info('test_user_reg_normal', url, data, expect_res, json.dumps(res.json(), ensure_ascii=False))

        # 响应断言（整体断言）
        self.assertDictEqual(res.json(), expect_res)
        # 数据库断言
        self.assertTrue(check_user(name))
        # 环境清理（由于注册接口向数据库写入了用户信息）
        del_user(name)

    def test_user_reg_exist(self):
        case_data = get_test_data(self.data_list, 'test_user_reg_exist')
        if not case_data:
            print("用例数据不存在")
        url = case_data.get('url')
        data = json.loads(case_data.get('data'))  # 转为字典，需要取里面的name进行数据库检查
        expect_res = json.loads(case_data.get('expect_res'))  # 转为字典，断言时直接断言两个字典是否相等
        name = data.get("name")  # 张三

        # 环境检查
        if not check_user(name):
            add_user(name, '123456')

        # 发送请求
        res = requests.post(url=url, json=data)
        log_case_info('test_user_reg_exist', url, data, expect_res, json.dumps(res.json(), ensure_ascii=False))
        # 响应断言（整体断言）
        self.assertDictEqual(res.json(), expect_res)

if __name__ == '__main__':
    unittest.main(verbosity=2)  # 运行所有用例


# import discord
from discord.ext import commands
from discord.ext.commands import Context
import coc

from config import Config
from commands.war import create_war_commands
from coc_events.war import create_war_events

config = Config()
bot = commands.Bot(command_prefix='!')
coc_client = coc.login(
  config.coc_dev_email,
  config.coc_dev_password,
  key_names=config.coc_key_names,
  client=coc.EventsClient
)

@bot.event
async def on_ready():
  print(f'We have logged in as {bot.user}')

# Ping pong test command
@bot.command(name='ping')
@commands.guild_only()
@commands.has_permissions(administrator=True)
async def ping(ctx:Context):
  await ctx.send('Pong')

# Commands
create_war_commands(config, bot, coc_client)

# CoC Events
create_war_events(config, bot, coc_client)

bot.run(config.bot_token)


import ast
import sys
import json
from huepy import *
import os.path
import argparse
import pandas as pd

indentation = '    '

imports = '''
import shared3p;
import shared3p_random;
import shared3p_string;
import shared3p_sort;
import stdlib;

import oblivious;
import shared3p_oblivious;

import shared3p_table_database;
import table_database;
import c45_db;
'''
main_f = '''void main(){
'''

def is_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False

def quote(x):
    if is_number(x):
        return str(x)
    else:
        return '"' + x + '"'

def main():
    global main_f

    parser = argparse.ArgumentParser()
    parser.add_argument('configuration', help = 'Configuration file of the request')
    parser.add_argument('--mesh_mapping', help = 'File with the mesh term mapping (values to integers).', default = 'mhmd-driver/mesh_mapping.json')
    parser.add_argument('--cvi_mapping', help = 'File with the cvi categorical attributes mapping (values to integers).', default = 'datasets/analysis_test_data/cvi_mapping.json')
    parser.add_argument('--summary', help = 'CSV file with the summary of the dataset.', default = 'datasets/analysis_test_data/cvi_summary.csv')
    parser.add_argument('--DNS', help = 'File with the Hospitals names and IPS.', default = 'web/MHMDdns.json')
    args = parser.parse_args()

    print(run('Generating main..'))

    uid = args.configuration.split('_')[-1].split('.')[0]
    configuration = json.load(open(args.configuration))
    mesh_mapping = json.load(open(args.mesh_mapping))
    cvi_mapping = json.load(open(args.cvi_mapping))
    summary = pd.read_csv(args.summary, sep = ',')

    if 'datasources' in configuration:
        numberOfDatasets = len(configuration['datasources'])
        data_providers = '\n'.join([indentation + "string table_" + quote(i) + " = " + quote(configuration['datasources'][i] + '_' + uid) + ";" for i in range(len(configuration['datasources']))])
    else:
        dns = json.load(open(args.DNS))
        available_datasources = dns.keys()
        numberOfDatasets = len(available_datasources)
        data_providers = '\n'.join([indentation + "string table_" + quote(i) + " = " + quote(available_datasources[i] + '_' + uid) + ";" for i in range(len(available_datasources))])

    main_f += data_providers
    main_f += '''
    // Create the data-providers list
    providers_vmap = tdbVmapNew();
    data_providers_num = ''' + quote(numberOfDatasets) + ''';
'''
    for i in range(numberOfDatasets):
        main_f += '''
    tdbVmapAddString(providers_vmap, "0", table_'''+ quote(i) +''');
'''
    attributes = configuration['attributes']
    class_attribute = configuration['class_attribute']['name']
    original_attributes = list(range(len(attributes)+1))
    class_index = len(attributes)
    main_f += '''
    original_attributes = {'''+','.join(map(quote,original_attributes))+'''};
    uint64[[1]] original_attributes_without_class = {'''+','.join(map(quote,original_attributes[:-1]))+'''};
    class_index = ''' + quote(class_index) + ''';
    possible_values = tdbVmapNew();
    float64[[1]] values;
'''

    columns = len(attributes) + 1
    main_f += '''
    columns = ''' + quote(columns) + ''';
'''
    possible_values = {}
    categorical_attributes = []
    for counter, attribute in enumerate([attr['name'] for attr in attributes] + [class_attribute]):
        if attribute in cvi_mapping: # if attribute is categorical
            possible_values[counter] = [mapping for value, mapping in cvi_mapping[attribute].items()]
            categorical_attributes.append(counter)
        else:
            categorical_attributes.append(-1)

    if class_attribute not in cvi_mapping:
        class_min = summary[summary['Field'] == class_attribute][' Min'].item()
        class_max = summary[summary['Field'] == class_attribute][' Max'].item()
        class_cells = int(configuration['class_attribute']['cells'])
        possible_values[class_index] = list(range(class_cells))
    else:
        class_min = -1
        class_max = -1
        class_cells = -1

    for attribute, values in possible_values.items():
        main_f += '''
    values = {''' + ','.join(map(quote, values)) + '''};
    tdbVmapAddValue(possible_values, "''' + quote(attribute) + '''", values);
 '''

    main_f += '''
    categorical_attributes = {''' + ','.join(map(quote,categorical_attributes)) + '''};
    class_min = ''' + quote(class_min) + ''';
    class_max = ''' + quote(class_max) + ''';
    class_cells = ''' + quote(class_cells) + ''';
 '''
    main_f += '''
    // Open connection to DB and Insert data to different tables
    datasource = "DS1";
    print("Opening connection to db: ", datasource);
    tdbOpenConnection(datasource);

    uint64 original_example_indexes_vmap = tdbVmapNew();
    for (uint64 i = 0 ; i < data_providers_num ; i++) {
        string table = tdbVmapGetString(providers_vmap, "0", i :: uint64);
        uint64 rows = tdbGetRowCount(datasource, table);
        pd_shared3p int64[[1]] original_example_indexes(rows);
        original_example_indexes = 1;
        tdbVmapAddValue(original_example_indexes_vmap, "0", original_example_indexes);
    }
'''

    main_f += '''
    print("Running C4.5 ...");
    string root = c45(original_example_indexes_vmap, original_attributes_without_class);
    print(root);

    for (uint64 i = 0 ; i < data_providers_num ; i++) {
        string table = tdbVmapGetString(providers_vmap, "0", i :: uint64);
        // Check if a table exists
        if (tdbTableExists(datasource, table)) {
          // Delete existing table
          print("Deleting table: ", table);
          tdbTableDelete(datasource, table);
        }
    }
}'''

    if os.path.isdir("./decision-tree/"):
        OUTPUT_DIR = './decision-tree/'
    elif os.path.isdir("../decision-tree/"):
        OUTPUT_DIR = '../decision-tree/'
    else:
        OUTPUT_DIR = './'
    with open(OUTPUT_DIR + 'main_' + uid + '.sc', 'w') as output:
        output.write(imports)
        output.write(main_f)
    print(good('Main generated at ' + OUTPUT_DIR + 'main_' + uid + '.sc'))

if __name__ == '__main__':
    main()


from fastapi import APIRouter
from models.portfolio import Portfolio

router = APIRouter(
    prefix="/api/portfolio",
    tags=["portfolio"]
)

@router.get("/positions")
async def get_positions():
    positions = await Portfolio.get_positions()
    return positions

@router.get("/activities")
async def get_activities():
    activities = await Portfolio.get_activities()
    return activities

@router.get("/deposits")
async def get_deposits():
    deposits = await Portfolio.get_deposits()
    return deposits

@router.get("/orders")
async def get_orders():
    orders = await Portfolio.get_orders()
    return orders

@router.get("/history")
async def get_history():
    history = await Portfolio.get_history()
    return history

@router.get("/metrics")
async def get_metrics():
    metrics = await Portfolio.get_metrics()
    return metrics


import dash
import numpy as np
import pandas as pd
import plotly.express as px
from dash.dependencies import Input, Output, State, ClientsideFunction
import dash_core_components as dcc
import dash_html_components as html

app = dash.Dash(
    __name__, meta_tags=[{"name": "viewport", "content": "width=device-width"}]
)
server = app.server
origin_df=pd.read_csv('./game_data.csv')
df = pd.read_csv('./cleaned_game_data.csv')
cat=pd.read_csv('./cat.csv')
plot_cat=cat.sum()[cat.sum()>200].sort_values()
variable_list=['min_players','max_players','avg_time', 'min_time', 'max_time',
               'avg_rating','geek_rating', 'num_votes']
total_list=list(df.columns[~pd.Series(df.columns).isin(origin_df.columns)])
variable_options=[
    {"label": variable, "value": variable}
    for variable in variable_list
]
app.layout = html.Div(
    [

                html.Div(
                    [
                        html.Div(
                            [
                                html.H3(
                                    "Game data analysis",
                                    style={'text-align':'center',
                                           'font-size':'45px'},

                                ),
                                html.H5(
                                    "demo presentation",
                                    style={'text-align':'center',
                                           'font-size':'15px'}
                                ),
                            ]
                        )
                    ],
                                        id="title",
                ),

        html.Div(
            [
                html.Div(
                    [

                        html.P("Geek rating or average rating for box plot:", className="control_label"),
                        dcc.RadioItems(
                            id="rating",
                            options=[
                                {"label": "Geek rating", "value": "geek_rating"},
                                {"label": "Average rating ", "value": "avg_rating"}

                            ],
                            labelStyle={"display": "inline-block"},
                            className="dcc_control",
                        ),
                        html.P("variable distribution selector", className="control_label"),
                        dcc.Dropdown(
                            id="distr_var",
                            options=variable_options,
                            multi=False,
                            value=list(variable_list),
                            className="dcc_control",
                            style={'width':'240px'},
                        )
                    ],
                    className="pretty_container four columns",
                    id="cross-filter-options",
                )]),
        html.Div(
            [
                html.Div(
                    [dcc.Graph(id="distribution_graph")],
                    className="pretty_container seven columns",
                ),
                html.Div(
                    [dcc.Graph(id="boxplot_graph")],
                    className="pretty_container five columns",
                ),
            ],
            className="row flex-display",
        )
    ])

@app.callback(
    Output("distribution_graph", "figure"),
    [
        Input("distr_var", "value"),
    ],
)
def distribution_variable(variable):
    plot_values=df[variable][df[variable]<=df[variable].quantile(0.95)]
    distribution_figure=px.histogram(x=plot_values, title='distribution plot of '+variable)

    return distribution_figure

@app.callback(
    Output("boxplot_graph", "figure"),
    [
         Input("rating", "value"),
    ],
)
def boxplot_variable(rating):
    boxplot_df = pd.merge(df[[rating]], cat, left_index=True, right_index=True)
    melted_boxplot = pd.melt(boxplot_df, id_vars=[rating], value_vars=np.unique(total_list), var_name='category',
                             value_name='belong')
    melted_boxplot = melted_boxplot.loc[
                     (melted_boxplot['belong'] == 1) & (melted_boxplot['category'].isin(plot_cat.index)), :]
    melted_boxplot = pd.merge(melted_boxplot, pd.DataFrame(melted_boxplot.groupby('category')[rating].mean()),
                              left_on='category', right_index=True, how='left')
    melted_boxplot.columns = [rating, 'category', 'belong', 'mean_rating']
    melted_boxplot = melted_boxplot.sort_values('mean_rating', ascending=False)
    boxplot_figure=px.box(melted_boxplot, y=rating, x='category',color='category', title='box plot of '+rating)

    return boxplot_figure

if __name__ == "__main__":
    app.run_server()


import sys

sys.path.append("./result/")

import numpy as np
import random
from tqdm import tqdm

# Pytorch
import torch
from torch.utils.data import DataLoader

# Dataset
from Data.Dataset import NTUDataSet

# Feature
from Model.Custom.timestamp import print_time as Timestamp

# AAGCN
from Src.AAGCN.GCN_Unsupervised.unsupervised_globalgcn_3layer_action_KD_atu import (
    Model_student as AAGCN_Model_student,
)

# MS-G3D
from model import Training_Student_Model as Training_Student_Model

# Loss function
from Model.Function.mpjpe import *

#################### * Random Seed Set * ####################
torch.manual_seed(0)
np.random.seed(0)
random.seed(0)
np.set_printoptions(threshold=sys.maxsize, precision=3, suppress=True, linewidth=800)

#################### * Config parameters * ####################
# Model selection
option = "MS_G3D_VA_RICH"
# [1]: AAGCN
# [2]: MSG3D
# [3]: MSG3D_VA_RICH

# Training & Testing
num_epochs = 60
train_batch_size = 12  # AAGCN = 16 | MS-G3D = 12 (Experiment)
test_batch_size = 32
frame = 300

# Path
data_path = "/media/user/DATA/VA+RICH5+MS-G3D_EF/"
epoch_path = "./Batch/"
log_path = "./Log/"
vis_path = "./Archive/"

# Device
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# arg_config
args = {
    "dataset_config": {
        "directory": "{}data/ntu/".format(data_path),
        "partition": "xsub/",
        "use_mmap": True,
        "debug": False,
        "downsample_rate": 1,
    }
}
args = args["dataset_config"]

# val_dataset_config
val_dataset_config = {
    "data_path": args["directory"] + args["partition"] + "/val_multi_joint_60.npy",
    "label_path": args["directory"] + args["partition"] + "/val_label_60.pkl",
    "use_mmap": args["use_mmap"],
    "debug": args["debug"],
}

#################### * Dataset config * ####################
# Testing
test_dataset = NTUDataSet(val_dataset_config)
test_dataloader = DataLoader(test_dataset, batch_size=test_batch_size, shuffle=False)

#################### * Model * ####################

model_student = Training_Student_Model(
    num_class=60,
    num_point=25,
    num_person=2,
    num_gcn_scales=13,
    num_g3d_scales=6,
    in_channels=3,
    ATU_layer=2,
    adaptive=True,
    attention=True,
)

#################### * Testing model * ####################
num_best_epoch = 60
lowest_loss = 0.06364738941192627
print("Best epoch: {}, Lowest loss: {}".format(num_best_epoch, lowest_loss))
model_student.load_state_dict(
    torch.load("{}{}_post_best_epoch{}.pth".format(epoch_path, option, num_best_epoch))
)

model_student.to(device)
model_student.eval()

with torch.no_grad():
    total_error = 0.0
    total_error_all = 0.0
    total_mpjpe = 0.0
    total_mpjpe_all = 0.0
    total_acc_student = 0.0
    tqdm_time = 0

    for data in tqdm(test_dataloader):
        tqdm_time += 1
        label = data["label"].to(device)
        feature = data["data"].to(device)[:, :, :frame, :, :]

        mask_data = feature.clone()

        reconstructed = model_student(feature)
        predict_action_student = reconstructed[2]
        x_mask = reconstructed[3]

        mask_data = mask_data * x_mask
        reconstructed_all = reconstructed[0]
        reconstructed = reconstructed[0] * (1 - x_mask) + feature * (x_mask)

        error_recon_all = mpjpe(reconstructed_all, feature)
        error_recon = mpjpe(reconstructed, feature)

        total_error_all += error_recon_all.item()
        total_error += error_recon.item()
        total_mpjpe_all = total_error_all / tqdm_time
        total_mpjpe = total_error / tqdm_time

        predict_label_student = torch.max(predict_action_student, 1)[1]
        acc_student = torch.mean((predict_label_student == label).float())

        # Testing log file
        f = open("{}Test_Student_log.txt".format(log_path), "a")
        f.write(
            "Best epoch: {}, Lowest loss: {:.4f}, Acc: {} \n Predict_label: {} \n Label: {} \n".format(
                num_best_epoch, lowest_loss, acc_student, predict_label_student, label
            )
        )
        f.close()

        total_acc_student += acc_student.item()

total_acc_student = total_acc_student / len(test_dataloader.dataset) * test_batch_size

#################### * Output * ####################
print("Best epoch: {}, Lowest loss: {:.4f} \n".format(num_best_epoch, lowest_loss))
print("Testing_Student_Acc: {}".format(total_acc_student))

print("total_mpjpe_all", total_mpjpe_all)
print("total_mpjpe", total_mpjpe)
print("feature.shape: ", feature.shape)

# video_number
video_number = feature.shape[0]
print("video_number: ", video_number)
print("label", label, label.shape)
print("predict_label", predict_label_student, predict_label_student.shape)

# label
label = label
np.save("{}label".format(vis_path), label.cpu().detach().numpy())

# predict_label
predict_label_student = predict_label_student
np.save(
    "{}predict_label".format(vis_path), predict_label_student.cpu().detach().numpy()
)

# 3d_pose_ori
feature = feature[:, :, :, :, :]
feature = (
    feature.permute(0, 2, 3, 1, 4)
    .contiguous()
    .view(video_number, frame, 25, 3, feature.shape[4])
)
np.save("{}3d_pose_ori".format(vis_path), feature.cpu().detach().numpy())

# data
f = open("{}data".format(vis_path), "w")
f.write(str(feature.cpu().detach().numpy()))
f.close()

# 3d_pose0
mask_data = mask_data[:, :, :, :, :]
mask_data = (
    mask_data.permute(0, 2, 3, 1, 4)
    .contiguous()
    .view(video_number, frame, 25, 3, feature.shape[4])
)
np.save("{}3d_pose0".format(vis_path), mask_data.cpu().detach().numpy())

# 3d_pose
reconstructed = reconstructed[:, :, :, :, :]
reconstructed = (
    reconstructed.permute(0, 2, 3, 1, 4)
    .contiguous()
    .view(video_number, frame, 25, 3, feature.shape[4])
)
np.save("{}3d_pose".format(vis_path), reconstructed.cpu().detach().numpy())

# reconstructed
f = open("{}reconstructed".format(vis_path), "w")
f.write(str(reconstructed.cpu().detach().numpy()))
f.close()

print(
    Timestamp(),
    "#################### Testing Action Recognition Model Complete ####################",
)


import numpy as np
import tensorflow as tf
from skimage.data import chelsea

from gcvit import GCViTXXTiny

def test_inference():
    # load test image, resize, and create batch
    img = tf.keras.applications.imagenet_utils.preprocess_input(
        chelsea(), mode="torch"
    )
    img = tf.image.resize(img, (224, 224))[
        None,
    ]

    model = GCViTXXTiny(pretrain=True)

    pred = model(img).numpy()
    result = tf.keras.applications.imagenet_utils.decode_predictions(pred, top=5)[0]
    result = np.array([x[2] for x in result])

    # should be five predictions for this pretrained model and image
    assert len(result) == 5

    # check if confidence values match with expected results
    expected_result = [
        ("n02124075", "Egyptian_cat", 0.7402193),
        ("n02123045", "tabby", 0.07757583),
        ("n02123159", "tiger_cat", 0.07006007),
        ("n02127052", "lynx", 0.0043750308),
        ("n04040759", "radiator", 0.00091264246),
    ]
    expected_result = np.array([x[2] for x in expected_result])
    np.allclose(result, expected_result, atol=1e-4)

def test_feature_extraction():
    # load test image, resize, and create batch
    img = tf.keras.applications.imagenet_utils.preprocess_input(
        chelsea(), mode="torch"
    )
    img = tf.image.resize(img, (224, 224))[
        None,
    ]

    model = GCViTXXTiny(pretrain=True)
    model.reset_classifier(num_classes=0, head_act=None)
    feature = model(img)
    shape_ = feature.shape
    print(shape_)

    assert shape_ == (1, 512)


import dash
import dash_table
import dash_core_components as dcc
import dash_html_components as html
import plotly.express as px
import pandas as pd
import dash_bootstrap_components as dbc
import json
import plotly.graph_objs as go
import numpy as np
from dash.dependencies import Input, Output
import math

df= pd.read_csv('4_weeks_PA_data.csv')
df['R_minus'] = df['R_mean'] - df['R_low']
df['R_plus'] = df['R_high'] - df['R_mean']
df['dailycases_p100k_7d_avg'] = df['dailycases_p100k_7d_avg'].round(5)

date_list = list(df.Date.unique())
counties = list(df.county_name.unique())

df.rename(columns={'county_name': 'County',
                   'population': 'Population',
                   'dailycases_p100k_7d_avg': 'Daily Cases',
                   'R_mean': 'R mean',
                   'R_low': 'R Low',
                   'R_high': 'R High'},
        inplace=True)

low_x = df['R mean'].min()
high_x = df['R mean'].max()
low_y = df['Daily Cases'].min()
high_y = df['Daily Cases'].max()

fig_dict = {
    "data": [],
    "layout": {},
    "frames": []
}

# fill in most of layout
fig_dict["layout"]["xaxis"] = {"range": [low_x, high_x], "title": "Reproduction Rate"}
fig_dict["layout"]["yaxis"] = {"range": [low_y-10, high_y], "title": "Infection Rate (7 day average)"}
fig_dict["layout"]["width"] = 900
fig_dict["layout"]["height"] = 1100
fig_dict["layout"]["hoverlabel"] =dict(bgcolor="white",
                                        font_size=16,
                                        font_family="Rockwell",
                                        )

fig_dict["layout"]["hovermode"] = "closest"
fig_dict["layout"]["shapes"] = [dict(
            type='line',
            x0=1,
            x1=1,
            y0=low_y-10,
            y1=high_y,
            line=dict(
                color='Black',
                dash='dash'))
                ]
fig_dict["layout"]["title"] = {
            'text': 'Pennsylvania Counties',
            'font_size': 20,
            'y': 0.95,
            'x': 0.5,
            'xanchor': 'center',
            'yanchor': 'top'}

fig_dict["layout"]["updatemenus"] = [
    {
        "buttons": [
            {
                "args": [None, {"frame": {"duration": 500, "redraw": False},
                                "fromcurrent": True, "transition": {"duration": 300,
                                                                    "easing": "quadratic-in-out"}}],
                "label": "Play",
                "method": "animate"
            },
            {
                "args": [[None], {"frame": {"duration": 0, "redraw": False},
                                  "mode": "immediate",
                                  "transition": {"duration": 0}}],
                "label": "Pause",
                "method": "animate"
            }
        ],
        "direction": "left",
        "pad": {"r": 10, "t": 87},
        "showactive": False,
        "type": "buttons",
        "x": 0.1,
        "xanchor": "right",
        "y": 0,
        "yanchor": "top"
    }
]

sliders_dict = {
    "active": 0,
    "yanchor": "top",
    "xanchor": "left",
    "currentvalue": {
        "font": {"size": 20},
        "prefix": "Date:",
        "visible": True,
        "xanchor": "right"
    },
    "transition": {"duration": 300, "easing": "cubic-in-out"},
    "pad": {"b": 10, "t": 50},
    "len": 0.9,
    "x": 0.1,
    "y": 0,
    "steps": []
}

# make data
date = list(df.Date.unique())[0]
for county in counties:
    dataset_by_date = df[df["Date"] == date]
    dataset_by_date_and_cont = dataset_by_date[
        dataset_by_date["County"] == county]

    data_dict = {
        "x": list(dataset_by_date_and_cont["R mean"]),
        "error_x": dict(type='data',
                                symmetric=False,
                                array=dataset_by_date_and_cont['R_plus'],
                                arrayminus=dataset_by_date_and_cont['R_minus'],
                                thickness=0.5,
                            ),
        "y": list(dataset_by_date_and_cont["Daily Cases"]),
        "mode": "markers",
        "marker" : dict(size=np.log2(dataset_by_date_and_cont['Population'])),
        "hovertemplate":dataset_by_date_and_cont['County'] +
                                '
R Rate (mean): %{x:.2f}' +
                                '
Daily Cases: %{y}
<extra></extra>',
        "text": list(dataset_by_date_and_cont["County"]),
        "name": county
    }
    fig_dict["data"].append(data_dict)

# make frames
for Date in date_list:
    frame = {"data": [], "name": str(Date)}
    for county in counties:
        dataset_by_Date = df[df["Date"] == Date]
        dataset_by_Date_and_cont = dataset_by_Date[
            dataset_by_Date["County"] == county]

        data_dict = {
            "x": list(dataset_by_Date_and_cont["R mean"]),
            "error_x": dict(type='data',
                                             symmetric=False,
                                             array=dataset_by_Date_and_cont['R_plus'],
                                             arrayminus=dataset_by_Date_and_cont['R_minus'],
                                             thickness=0.5,
                                             ),
            "y": list(dataset_by_Date_and_cont["Daily Cases"]),
            "mode": "markers",
            "marker" : dict(size=np.log2(dataset_by_Date_and_cont['Population'])),
            "hovertemplate":dataset_by_Date_and_cont['County'] +
                                '
R Rate (mean): %{x:.2f}' +
                                '
Daily Cases: %{y}
<extra></extra>',
            "text": list(dataset_by_Date_and_cont["County"]),
            "name": county
        }
        frame["data"].append(data_dict)

    fig_dict["frames"].append(frame)
    slider_step = {"args": [
        [Date],
        {"frame": {"duration": 300, "redraw": False},
         "mode": "immediate",
         "transition": {"duration": 300}}
    ],
        "label": Date,
        "method": "animate"}
    sliders_dict["steps"].append(slider_step)

fig_dict["layout"]["sliders"] = [sliders_dict]

fig = go.Figure(fig_dict)

fig.show()

# fig0 = go.Figure(data=go.Scatter(x=df['R mean'],
#                             y=df['Daily Cases'],
#                             error_x=dict(type='data',
#                                             symmetric=False,
#                                             array=df['R_plus'],
#                                             arrayminus=df['R_minus'],
#                                             thickness=0.5,
#                                             ),
#                             mode='markers',
#                             hovertemplate=df['County']+
#                                 '
R Rate (mean): %{x:.2f}' +
#                                 '
Daily Cases: %{y}
<extra></extra>',
#             )
# )
# fig0.update_layout(
#     autosize=False,
#     width=900,
#     height=900,
#     hoverlabel=dict(
#         bgcolor="white",
#         font_size=16,
#         font_family="Rockwell",
#     ),
#     font_family="Ariel",
#     font_color='Black',
#     title={
#         'text': 'Pennsylvania Counties',
#         'font_size': 20,
#         'y': 0.95,
#         'x': 0.5,
#         'xanchor': 'center',
#         'yanchor': 'top'},
#     xaxis_title={
#         'text': "R rate",
#         'font_size': 16
#     },
#     yaxis_title={
#         'text': "Daily Cases per 100k (7 Day Average)",
#         'font_size': 16
#     },
#     shapes=[dict(
#         type='line',
#         x0=1,
#         x1=1,
#         y0=0,
#         y1=68,
#         line=dict(
#             color='Black',
#             dash='dash'))
#             ],
#     showlegend=False,
# )
# fig0.update_xaxes(
#     range=[0.2, 6]
# )
# fig0.update_yaxes(
#     range=(0, 68)
# )
# fig0.show()


#!/usr/bin/env python3

# tidalnorway-py
# Quick and dirty scraper for water level data in Norway
# Simple version for water level only: http://api.sehavniva.no/tidepos_en.html
# More fancy one is here: http://api.sehavniva.no/tideapi_en.html
# Garten Position 63.642022, 9.498116
# A full request for one day looks like this:
# http://api.sehavniva.no/tideapi.php?lat=63.642022&lon=9.498116&fromtime=2017-07-05T00%3A00&totime=2017-07-06T00%3A00&datatype=tab&refcode=cd&place=&file=&lang=en&interval=10&dst=1&tzone=1&tide_request=locationdata

from lxml import html
from bs4 import BeautifulSoup
import requests
import datetime
from pytz import timezone

if __name__ == '__main__':
    print("tidalnorway-py supposed to be a class in the future")

    #t = timezone(settings.TIME_ZONE).localize(datetime.now()).replace(microsecond=0)
    current_time_utc_iso = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
    print("Current datetime =", current_time_utc_iso)
    print("### RESPONSE FROM SERVER ###")
    #page = requests.get("http://forecast.weather.gov/MapClick.php?lat=37.7772&lon=-122.4168")
    lat = "63.642022"
    lon = "9.498116"
    fromtime = "2017-07-05T00:00"
    totime = "2017-07-06T00:00"
    datatype = "tab"
    refcode = "cd"
    place = ""
    file = ""
    lang = "en"
    interval = "10"
    dst = "1"
    tzone = "0"
    tide_request = "locationdata"

    payload = {'lat': lat, 'lon': lon, 'fromtime': fromtime, 'totime': totime, 'datatype': datatype, 'refcode': refcode, 'place': place, 'file': file, 'lang': lang, 'interval': interval, 'dst': dst, 'tzone': tzone, 'tide_request': tide_request}

    page = requests.get("http://api.sehavniva.no/tideapi.php", params=payload)
    soup = BeautifulSoup(page.content, 'lxml-xml')
    print(soup.prettify())


import torch.nn as nn
from torchvision.models import densenet
from models.boundary_aware_densenet import DenseNet
import torch.utils.model_zoo as model_zoo
import re
import numpy as np
import torch

class Dense201(DenseNet):
    def __init__(self, pretrained=True, num_classes=212, fmf_pretrain=None, **kwargs):
        super(Dense201, self).__init__(num_init_features=64, growth_rate=32, block_config=(6, 12, 48, 32),
                         num_classes=num_classes, **kwargs)
        if pretrained:
            pretrained_dict = dict(model_zoo.load_url(densenet.model_urls['densenet201']))
            tmp = pretrained_dict['features.conv0.weight']
            tmp = [tmp] * 16
            tmp = np.concatenate(tmp, axis=1)
            pretrained_dict['features.conv0.weight'] = torch.tensor(tmp)
            del pretrained_dict['classifier.weight']
            del pretrained_dict['classifier.bias']
            model_dict = self.state_dict()
            pattern = re.compile(
                r'^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$')
            for key in list(pretrained_dict.keys()):
                res = pattern.match(key)
                if res:
                    new_key = res.group(1) + res.group(2)
                    pretrained_dict[new_key] = pretrained_dict[key]
                    del pretrained_dict[key]

            model_dict.update(pretrained_dict)
            if not fmf_pretrain is None:
                used = {}
                state_dict = torch.load(fmf_pretrain)
                for k, v in state_dict.items():
                    if k == 'features.conv0.weight':
                        used[k] = v
                    elif 'fmf' in k:
                        used[k] = v
                model_dict.update(used)
            self.load_state_dict(model_dict)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x, heatmap):
        out = super(Dense201, self).forward(x, heatmap)
        out = self.sigmoid(out)
        return out

if __name__ == '__main__':
    import numpy as np
    import torch
    # heatmap =np.zeros((128, 128, 3), np.uint8)
    heatmap = torch.FloatTensor(np.zeros((1, 15, 128, 128), np.uint8))
    # print(heatmap)
    # print(heatmap.shape)
    featuremap = torch.FloatTensor(np.random.random((1, 48, 256, 256)))
    #假设featuremap是64通道的32*32
    net = Dense201(num_classes=212)

    out = net(featuremap, heatmap)


# -*- coding: utf-8 -*-
import scrapy
import requests
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from scrapy.selector import Selector
from scrapy.http.request import Request
from MVID.settings import ANY_URL_REGEX_NEW
import re
from math import ceil
import json

from MVID.items import MvidItem

# def make_request():
#     domen = 'https://ru.investing.com'
#     url = "https://ru.investing.com/stock-screener/Service/SearchStocks"
#     referrer = [
#         #"https://ru.investing.com/stock-screener/?sp=country::5|sector::a|industry::a|equityType::a|exchange::a%3Ceq_market_cap;1",
#         "https://ru.investing.com/stock-screener/?sp=country::56|sector::a|industry::a|equityType::a%3Ceq_market_cap;1",
#     ]

#     bodies = [
#         #'country%5B%5D=56&sector=6%2C11%2C12%2C8%2C5%2C7%2C4%2C10%2C2%2C1%2C3%2C9&industry=72%2C51%2C88%2C56%2C8%2C19%2C67%2C43%2C89%2C15%2C80%2C59%2C41%2C37%2C55%2C52%2C2%2C93%2C48%2C57%2C1%2C22%2C71%2C82%2C76%2C46%2C94%2C95%2C102%2C58%2C101%2C30%2C29%2C35%2C53%2C16%2C34%2C47%2C12%2C66%2C90%2C14%2C49%2C17%2C44%2C68%2C27%2C11%2C74%2C85%2C63%2C83%2C84%2C26%2C21%2C79%2C36%2C64%2C62%2C60%2C24%2C20%2C54%2C33%2C4%2C81%2C40%2C98%2C70%2C99%2C42%2C39%2C92%2C65%2C18%2C73%2C77%2C25%2C7%2C10%2C78%2C86%2C87%2C38%2C31%2C6%2C91%2C61%2C97%2C45%2C13%2C100%2C23%2C96%2C9%2C69%2C32%2C75%2C5%2C28%2C50%2C3&equityType=ORD%2CDRC%2CPreferred%2CUnit%2CClosedEnd%2CREIT%2CELKS%2COpenEnd%2CRight%2CParticipationShare%2CCapitalSecurity%2CPerpetualCapitalSecurity%2CGuaranteeCertificate%2CIGC%2CWarrant%2CSeniorNote%2CDebenture%2CETF%2CADR%2CETC%2CETN&exchange%5B%5D=40&pn={}&order%5Bcol%5D=eq_market_cap&order%5Bdir%5D=d',
#         'country%5B%5D=5&sector=6%2C11%2C12%2C8%2C5%2C7%2C4%2C10%2C2%2C1%2C3%2C9&industry=72%2"C51%2C88%2C56%2C8%2C19%2C67%2C43%2C89%2C15%2C80%2C59%2C41%2C37%2C55%2C52%2C2%2C93%2C48%2C57%2C1%2C22%2C71%2C82%2C76%2C46%2C94%2C95%2C102%2C58%2C101%2C30%2C29%2C35%2C53%2C16%2C34%2C47%2C12%2C66%2C90%2C14%2C49%2C17%2C44%2C68%2C27%2C11%2C74%2C85%2C63%2C83%2C84%2C26%2C21%2C79%2C36%2C64%2C62%2C60%2C24%2C20%2C54%2C33%2C4%2C81%2C40%2C98%2C70%2C99%2C42%2C39%2C92%2C65%2C18%2C73%2C77%2C25%2C7%2C10%2C78%2C86%2C87%2C38%2C31%2C6%2C91%2C61%2C97%2C45%2C13%2C100%2C23%2C96%2C9%2C69%2C32%2C75%2C5%2C28%2C50%2C3&equityType=ORD%2CDRC%2CPreferred%2CUnit%2CClosedEnd%2CREIT%2CELKS%2COpenEnd%2CRight%2CParticipationShare%2CCapitalSecurity%2CPerpetualCapitalSecurity%2CGuaranteeCertificate%2CIGC%2CWarrant%2CSeniorNote%2CDebenture%2CETF%2CADR%2CETC%2CETN&exchange%5B%5D=2&exchange%5B%5D=95&exchange%5B%5D=1&pn={}&order%5Bcol%5D=eq_market_cap&order%5Bdir%5D=d',
#     ]

#     headers = {
#         "Host": "ru.investing.com",
#         "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:66.0) Gecko/20100101 Firefox/66.0",
#         "Accept": "application/json, text/javascript, */*; q=0.01",
#         "Accept-Language": "ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3",
#         "Accept-Encoding": "gzip, deflate, br",
#         "Referer": "{}",
#         "Content-Type": "application/x-www-form-urlencoded",
#         "X-Requested-With": "XMLHttpRequest",
#     }

#     #scrapy.FormRequest( url = url, headers= headers, method= "POST", body = bodies)

#     links = []
#     for b , r in zip(bodies, referrer):
#         i = 1
#         while True:
#                 headers['Refferer'] = r
#                 req = requests.post( url = url, headers = headers, data= b.format(i))
#                 data = req.json()
#                 if 'error' in data  :
#                     break
#                 numPages = ceil(data['totalCount'] / 50) - 1

#                 print(data['pageNumber'],'/',numPages)
#                 for d in data['hits']:
#                     link = d['viewData']['link']
#                     if '?' in link:
#                         redirect_link = domen + link.replace('?', '-commentary?') + '\n'
#                     else:
#                         redirect_link = domen + link + '-commentary' + '\n'
#                     #print(redirect_link)
#                     links.append(redirect_link)
#                 if data['pageNumber'] == numPages:
#                     break

#                 i = i + 1
#     return links

# def change_link(value):
#     return value + "-commentary/"

class InvestingComSpider(CrawlSpider):
    name = 'investing_com'
    allowed_domains = ['ru.investing.com']
    #my_urls = make_request()
    #start_urls =  my_urls
    # ['https://ru.investing.com/stock-screener/?sp=country::56|sector::a|industry::a|equityType::a|exchange::40%3Ceq_market_cap;1',
    #               #'https://ru.investing.com/stock-screener/?sp=country::5|sector::a|industry::a|equityType::a%3Ceq_market_cap;1',
    #             ]
    rules = (
        #Rule(LinkExtractor( restrict_xpaths= ("//div[@class = 'resultsContainer stockScreener']"), process_value= change_link, allow='/equities/', unique=True), follow=True),
		Rule(LinkExtractor( restrict_xpaths= ("//div[@class = 'midDiv inlineblock']"), unique= True), callback='parse_page', follow=True),   #//div[contains(@class , 'sideDiv inlineblock text_align_lang_base_2')]//a[contains(.,@href])]
	)

    def parse_start_url(self, response):
        return self.parse_page(response)

    # def parse(self, response):
    #     headers= {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:48.0) Gecko/20100101 Firefox/48.0'}
    #     self.parse_item(response)
    #     #yield Request(self.start_urls[0], headers=headers, callback= self.parse_item)

    def parse_page(self, response):

        sel = Selector(response)

        comment_set = sel.xpath("//div[@class = 'comment js-comment']")
        for comment in comment_set:
            item = MvidItem()
            item['position'] = sel.xpath("//div[@class = 'instrumentHead']/h1/text()").extract_first()
            item['source'] = self.allowed_domains[0]
            main_comment = comment.xpath(".//div[@class = 'mainComment js-content']")
            item['main_comment_id'] = comment.xpath("./@data-comment-id").extract_first()
            item['comment_text'] = main_comment.xpath(".//span[@class = 'js-text']/text()").extract()
            item['comment_id'] = comment.xpath("./@data-comment-id").extract_first()
            item['user_id'] = comment.xpath("./@data-user-id").extract_first()
            item['user_name'] = main_comment.xpath(".//a[@class = 'js-user-link']//@alt").extract_first()
            item['date'] = main_comment.xpath(".//span[@class = 'js-date']/@comment-date-formatted").extract_first()
            item['selfurl'] = response.url
            if not item['comment_id'] or not item['user_name'] or not item['comment_text']:
                continue
            item['link'] = re.findall(ANY_URL_REGEX_NEW, main_comment.xpath(".//span[@class = 'js-text']/text()").extract()[0])

            reply_comments = comment.xpath(".//div[@class = 'commentReply js-comment js-comment-reply']")
            for reply in reply_comments:
                 self.parse_details(reply, item['comment_id'], item['position'], item['selfurl']) # comment.xpath("./@data-comment-id/text()")
            yield item

    def parse_details(self, response, main_comment_id, position, url):

        sel = response
        item = MvidItem()

        item['position'] = position
        item['source'] = self.allowed_domains[0]
        item['main_comment_id'] = main_comment_id
        item['comment_text'] = sel.xpath(".//span[@class = 'js-text']/text()").extract_first()
        item['comment_id'] = sel.xpath("./@data-comment-id").extract_first()
        item['user_id'] = sel.xpath("./@data-user-id").extract_first()
        item['user_name'] = sel.xpath(".//a[@class = 'js-user-link']//@alt").extract_first()
        item['date'] = sel.xpath(".//span[@class = 'js-date']/@comment-date-formatted").extract_first()
        item['selfurl'] = url
        if not item['comment_id'] or not item['user_name'] or not item['comment_text']:
            return
        item['link'] = re.findall(ANY_URL_REGEX_NEW, sel.xpath(".//span[@class = 'js-text']/text()").extract()[0]) #r'(https?://[^\s]+)'

        itemproc = self.crawler.engine.scraper.itemproc
        itemproc.process_item(item, self)

        return item


import calendar
from datetime import date

days = list(calendar.day_name)
def meetup_day(year, month, day_of_the_week, which):
    day_index = days.index(day_of_the_week)
    dates = [
        week[day_index]
        for week in calendar.monthcalendar(year, month)
        if week[day_index]]
    if which == 'teenth':
        for day_num in dates:
            if 13 <= day_num <= 19:
                return date(year, month, day_num)
    elif which == 'last':
        day_index = -1
    elif which == 'first':
        day_index = 0
    else:
        day_index = int(which[0]) - 1
    return date(year, month, dates[day_index])


"""Contain set of CLI applications."""
from collections.abc import Callable
from pathlib import Path
from re import IGNORECASE
from re import search
from sys import argv
from typing import Any

from click import argument
from click import command
from click import option

from shell_tools.dirs.search import find_empty_dirs
from shell_tools.files.random import make_random_file
from shell_tools.git.misc import find_repositories
from shell_tools.git.misc import update_repository

def determine_file_size(size_string: str) -> int:
    multiplier = {"gb": 1024 * 1024 * 1024, "mb": 1024 * 1024, "kb": 1024}

    result = search(r"^(?P<number>\d+\.?\d*)(?P<multiplier>gb|mb|kb)?$", size_string, IGNORECASE)
    if result:
        number = float(result.group("number"))
        ratio = 1 if not result.group("multiplier") else multiplier[result.group("multiplier").lower()]
        return int(number * ratio)
    return 0

def get_program_name() -> str:
    return Path(argv[0]).stem

Processor = Callable[[list[Path]], None]

class ProcessorFactory:
    _state: dict[str, Any] = {}

    def __init__(self) -> None:
        self.__dict__ = self._state
        if not hasattr(self, "_registry"):
            self._registry: dict[str, Processor] = {}

    def register(self, name: str, processor: Processor) -> Processor:
        self._registry[name] = processor
        return processor

    @property
    def processors(self) -> dict[str, Processor]:
        return self._registry

    def get_processor(self, name: str) -> Processor:
        processor = self._registry.get(name)
        if not processor:
            raise KeyError(f"Unknown processor '{name}'")

        return processor

def dirs_processor(name: str) -> Callable[[Processor], Processor]:
    factory = ProcessorFactory()

    def decorator(processor: Processor) -> Processor:
        return factory.register(name, processor)

    return decorator

@dirs_processor(name="print")
def print_dirs(dirs: list[Path]) -> None:
    print(*dirs, sep="\n")

@dirs_processor(name="remove")
def remove_dirs(dirs: list[Path]) -> None:
    for path in dirs:
        print(f"removing '{path}'")
        path.rmdir()

def get_dirs_processor(name: str) -> Processor:
    factory = ProcessorFactory()
    return factory.get_processor(name)

@command()
@argument("path", type=Path)
@option("--size", type=str, default="10mb", help="File size in bytes[kb|mb|gb].", show_default=True)
@option("--line-size", type=int, default=64, help="Length of line/chunk.", show_default=True)
def generate_file(path: Path, size: str, line_size: int) -> None:
    path = path.absolute()
    if path.is_dir():
        print(f"Script aborted: '{path}' exists and it's directory")
        return

    if path.is_file():
        answer = input(f"file '{path}' already exists. override? [yes/no]: ")
        if not answer.lower().startswith("y"):
            return

    file_size = determine_file_size(size)
    if not file_size:
        print(f"Invalid file size was requested (file-size={size}).")
        print(f"Use '{get_program_name()} --help' to see allowed options.")
        return

    print(f"The file of {size} will be generated at '{path}'.")
    make_random_file(path, file_size, line_size)
    print("The generation is finished.")

@command()
@argument("root-dir", type=Path, default=Path.cwd())
@option("--ignore-empty-files", is_flag=True, help="Treat empty files as absent.")
@option("--remove/--no-remove", is_flag=True, help="Indicate remove or not empty dirs")
def discover_empty_dirs(root_dir: Path, ignore_empty_files: bool, remove: bool) -> None:
    root_dir = root_dir.absolute()
    if not root_dir.exists():
        print("The root directory is not set or doesn't exists.")

    empty_dirs = find_empty_dirs(root_dir, ignore_empty_files)
    processor = get_dirs_processor("remove" if remove else "print")
    processor(empty_dirs)

@command()
@argument("root-dir", type=Path, default=Path.cwd())
@option("-r", "--recursive", is_flag=True, help="Search git repos recursively")
@option("--submodules/--no-submodules", is_flag=True, help="Update or not submodules")
def sync_repos(root_dir: Path, recursive: bool, submodules: bool) -> None:
    root_dir = root_dir.absolute()
    suffix = "recursively " if recursive else ""
    print(f"Scanning '{root_dir}' for git repositories {suffix}...")

    repos = find_repositories(root_dir, recursive)
    for repo in repos:
        update_repository(repo, submodules)
        print(f"{repo.relative_to(root_dir)} was synced")

    input("Press Enter to exit...")