# pylint: disable=W0611, E0401
"""
Main goal of this module is to scrape and parse data from "visityerevan.am" website
"""
import logging
import sys

from dataclasses import dataclass
from urllib.parse import urljoin

from httpx import Client
from selectolax.parser import HTMLParser, Node

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.WARNING)
stream_handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter(
    "[%(asctime)s] %(levelname)s:%(name)s:%(lineno)d:%(message)s")
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
           "AppleWebKit/537.36 (KHTML, like Gecko) " +
           "Chrome/109.0.0.0 Safari/537.36"}

@dataclass
class Event:
    """ Class contains all info about event """
    title: str
    description: str
    url_to_original: str
    time: str
    price: str
    img: str

@dataclass
class Response:
    """ Class contains html of page and info about existing of the next page """
    body_html: HTMLParser
    status_code: int

def serialize_event(event):
    """ Resulting format for each event """
    return {
        "id": "work in progress...",
        "type": "parsed_v1",
        "parserName": "visityerevan",
        "title": event.title,
        "description": event.description,
        "date": event.time,
        "durationInSeconds": 0,
        "location": {
                "country": "Armenia",
                "city": "Erevan",
        },
        "image": event.img,
        "price": {
            "amount": event.price,
            "currency": "AMD"
        },
        "timezone": {
            "timezoneName": "AMT",
            "timezoneOffset": "UTC +4",
        },
        "url": event.url_to_original,
    }

def get_page(client: Client, url: str) -> Response:
    """ Scrape html from page and check if next pages appears """
    resp = client.get(url, headers=HEADERS)
    html = HTMLParser(resp.text)

    return Response(body_html=html, status_code=resp.status_code)

def get_pages_amount(client: Client, url: str) -> int:
    """ func to get number of pages with events """
    resp = client.get(url, headers=HEADERS)
    html = HTMLParser(resp.text)

    pages_amount = html.css("ul[class='pagination justify-content-center'] >" +
                            "li[class='page-item']")[-1:][0].text()

    return int(pages_amount)

def is_valid(data):
    """ Helps us to catch website's structure changes """
    if data is None:
        logger.warning(
            "Seems that website changed structure. Please recheck code and website")

        return False
    else:
        return True

def parse_detail(blocks: list) -> list:
    """ Clean and prepare all data that we need """
    result = []

    # In this loop we will extract all
    #  Info that we can from each event's div
    for block in blocks:
        # Extract and prepare "time"
        month_day = block.css_first(
            "div[class='col-12 mt-n1'] > div")
        # Need validate data each parsing attempt
        if is_valid(month_day):
            month_day = month_day.text().replace('\n', '').strip()

        time = block.css_first(
            "div[class='text-grey text-md mb-2']")

        if is_valid(time):
            time = time.text().replace('\n', '').strip().split(' ')
            cleaned_time = f"{month_day} {time[-1:][0]}"
        else:
            cleaned_time = None
        # Extract and prepare "description"
        description = block.css_first("p")
        if is_valid(description):
            description = description.text().strip()
        # Clean and prepare "url"
        url = block.css_first("a").attrs["href"]
        if is_valid(url):
            url = "https://www.visityerevan.am" + url
        # Extract price
        price = ''
        cards = block.css("p.card-text > span")

        if len(cards) == 0:
            logger.warning(
                "Seems that website changed structure. Please recheck code and website")
        else:
            for card in cards:
                card = card.text()
                if "AMD" in card:
                    price = card.replace("AMD", "").strip()
                else:
                    price = "no info"
        # Extract img link
        img = block.css_first("img").attrs["src"]

        if is_valid(img):
            img = "https://www.visityerevan.am" + img
        # There is not need in cleaning "title"
        # With data we have create a new event object
        event = Event(
            title=block.css_first("h5").text(),
            description=description,
            url_to_original=url,
            time=cleaned_time,
            price=price,
            img=img
        )

        result.append(serialize_event(event))

    return result

def scrape_blocks(html: HTMLParser) -> list:
    """ Getting all divs with information from page """
    blocks = html.css("div[class='row px-lg-7']" +
                      " > div")

    return blocks

def pagination_loop(client: Client) -> list:
    """ Loop through all pages """
    url = "https://www.visityerevan.am/browse/things-to-do-events/ru/"
    # How many pages we will scrape
    pages_amount = get_pages_amount(client, url)
    # Blocks contains all divs that we need
    blocks = []
    # Iterating through all pages
    for page_number in range(1, pages_amount + 1):
        # Mutating a url to get page with current page number
        url = urljoin(url, f"?sel_filters=&current_page={page_number}")
        # Get object with scraped html markup from current page
        page = get_page(client, url)
        # Grad all divs with events data and append to list
        blocks += scrape_blocks(page.body_html)

    # Scraping is done, time to close session
    client.close()

    return blocks

async def scrape_website() -> list:
    """ Main function which contains all logic """
    # Start a new session
    client = Client()
    # Create list with all divs which contain info about events
    all_blocks = pagination_loop(client)
    # Parsing data from divs
    parsed_data = parse_detail(all_blocks)

    return parsed_data


import pyshorteners

def shorten(url):
    link = pyshorteners.Shortener()
    return link.tinyurl.short(url)

if __name__ == "__main__":
    url = input("Enter link for sorting:")
    print(f"\n {shorten(url)}")

# https://github.com/urmil89


import os
import sys

# 修改工作目录为程序所在目录，这样通过注册表实现开机自动启动时也能获取到正确的工作目录
# PS: 放到这个地方，是确保在所有其他初始化代码之前先修改掉工作目录
dirpath = os.path.dirname(os.path.realpath(sys.argv[0]))
old_path = os.getcwd()
os.chdir(dirpath)

import argparse
import datetime
import time
from multiprocessing import freeze_support

import psutil

import ga
from check_first_run import check_first_run_async
from config import config, load_config
from db_def import try_migrate_db
from first_run import is_weekly_first_run
from log import color, log_directory, logger
from main_def import (
    auto_send_cards,
    check_all_skey_and_pskey,
    check_djc_role_binding,
    check_proxy,
    check_update,
    get_user_buy_info,
    print_update_message_on_first_run_new_version,
    run,
    sas,
    show_ask_message_box,
    show_buy_info,
    show_extra_infos,
    show_lottery_status,
    show_multiprocessing_info,
    show_notices,
    show_pay_info,
    show_recommend_reward_tips,
    try_auto_update,
    try_auto_update_ignore_permission_on_special_case,
    try_join_xinyue_team,
    try_load_old_version_configs_from_user_data_dir,
    try_report_usage_info,
    try_save_configs_to_user_data_dir,
    try_take_dnf_helper_chronicle_task_awards_again_after_all_accounts_run_once,
    try_take_xinyue_team_award,
)
from pool import close_pool, init_pool
from qq_login import QQLogin
from show_usage import show_usage
from update import notify_manual_check_update_on_release_too_long
from usage_count import increase_counter
from util import (
    MiB,
    async_call,
    async_message_box,
    change_console_window_mode_async,
    change_title,
    clean_dir_to_size,
    disable_pause_after_run,
    disable_quick_edit_mode,
    is_run_in_github_action,
    kill_other_instance_on_start,
    pause,
    remove_old_version_portable_chrome_files,
    show_head_line,
    show_unexpected_exception_message,
)
from version import author, now_version, ver_time

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--no_max_console", default=False, action="store_true", help="是否不将窗口调整为最大化")
    parser.add_argument(
        "--wait_for_pid_exit",
        default=0,
        type=int,
        help="启动后是否等待对应pid的进程结束后再启动，主要用于使用配置工具启动小助手的情况，只有配置工具退出运行，自动更新才能正常进行",
    )
    parser.add_argument("--max_wait_time", default=5, type=int, help="最大等待时间")
    args = parser.parse_args()

    return args

def prepare_env():
    args = parse_args()

    # 最大化窗口
    if not args.no_max_console:
        logger.info("尝试调整窗口显示模式，打包exe可能会运行的比较慢")
        change_console_window_mode_async()

    if args.wait_for_pid_exit != 0:
        # 通过配置工具打开
        increase_counter(ga_category="open_by", name="config_tool", ga_misc_params={"dr": "config_tool"})
        logger.info(f"等待pid为{args.wait_for_pid_exit}的配置工具退出运行，从而确保可能有的自动更新能够正常进行，最大将等待{args.max_wait_time}秒")

        wait_time = 0.0
        retry_time = 0.1
        while wait_time <= args.max_wait_time:
            if not psutil.pid_exists(args.wait_for_pid_exit):
                logger.info("配置工具已成功退出，将开始运行小助手~")
                break

            time.sleep(retry_time)
            wait_time += retry_time
    else:
        # 直接打开
        increase_counter(ga_category="open_by", name="directly", ga_misc_params={"dr": "directly"})

def main():
    try_migrate_db()

    increase_counter(name="run/begin", ga_type=ga.GA_REPORT_TYPE_PAGE_VIEW)

    prepare_env()

    # 启动时检查是否需要同步本机数据目录备份的旧版本配置
    try_load_old_version_configs_from_user_data_dir()

    change_title()

    print_update_message_on_first_run_new_version()

    logger.warning(f"开始运行DNF蚊子腿小助手，ver={now_version} {ver_time}，powered by {author}")
    logger.warning(color("fg_bold_cyan") + "如果觉得我的小工具对你有所帮助，想要支持一下我的话，可以帮忙宣传一下或打开付费指引/支持一下.png，扫码打赏哦~")

    # 读取配置信息
    load_config("config.toml", "config.toml.local")
    cfg = config()

    if len(cfg.account_configs) == 0:
        raise Exception("未找到有效的账号配置，请检查是否正确配置。ps：多账号版本配置与旧版本不匹配，请重新配置")

    try_auto_update_ignore_permission_on_special_case(cfg)

    notify_manual_check_update_on_release_too_long(cfg.common)

    check_proxy(cfg)

    try_report_usage_info(cfg)

    if cfg.common.disable_cmd_quick_edit:
        disable_quick_edit_mode()

    show_notices()

    if cfg.common.allow_only_one_instance:
        logger.info("当前仅允许单个实例运行，将尝试干掉其他实例~")
        async_call(kill_other_instance_on_start)
    else:
        logger.info("当前允许多个实例同时运行~")

    pool_size = cfg.get_pool_size()
    init_pool(pool_size)

    change_title(multiprocessing_pool_size=pool_size, enable_super_fast_mode=cfg.common.enable_super_fast_mode)

    show_multiprocessing_info(cfg)

    account_names = []
    for account_cfg in cfg.account_configs:
        account_names.append(account_cfg.name)

    logger.info(f"当前共配置{len(account_names)}个账号，具体如下：{account_names}")

    clean_dir_to_size(log_directory, cfg.common.max_logs_size * MiB, cfg.common.keep_logs_size * MiB)
    clean_dir_to_size(f"utils/{log_directory}", cfg.common.max_logs_size * MiB, cfg.common.keep_logs_size * MiB)

    current_chrome_version = QQLogin(cfg.common).get_chrome_major_version()
    remove_old_version_portable_chrome_files(current_chrome_version)

    show_ask_message_box(cfg)

    # 检查是否有更新，用于提示未购买自动更新的朋友去手动更新~
    if cfg.common.check_update_on_start:
        check_update(cfg)

    check_all_skey_and_pskey(cfg)

    check_djc_role_binding()

    # 确保道聚城绑定OK后在活动运行同时进行异步的弹窗提示
    check_first_run_async(cfg)

    # 挪到所有账号都登陆后再尝试自动更新，从而能够判定是否已购买DLC
    try_auto_update(cfg)

    # 查询付费信息供后面使用
    show_head_line("查询付费信息")
    logger.warning("开始查询付费信息，请稍候~")
    user_buy_info = get_user_buy_info(cfg.get_qq_accounts())
    show_buy_info(user_buy_info, cfg, need_show_message_box=False)

    sas(cfg, "启动时展示账号概览", user_buy_info)

    # 预先尝试创建和加入固定队伍，从而每周第一次操作的心悦任务也能加到队伍积分中
    try_join_xinyue_team(cfg, user_buy_info)

    # 正式进行流程
    run(cfg, user_buy_info)

    try_take_dnf_helper_chronicle_task_awards_again_after_all_accounts_run_once(cfg, user_buy_info)

    # 尝试领取心悦组队奖励
    try_take_xinyue_team_award(cfg, user_buy_info)

    # # 尝试派赛利亚出去打工
    # try_xinyue_sailiyam_start_work(cfg)

    # 活动开启关闭时调这个开关即可
    enable_card_lottery = True

    if enable_card_lottery:
        auto_send_cards(cfg)

    show_extra_infos(cfg)
    sas(cfg, "运行完毕展示账号概览", user_buy_info)

    if enable_card_lottery:
        show_lottery_status("卡片赠送完毕后展示各账号抽卡卡片以及各礼包剩余可领取信息", cfg, need_show_tips=True)

    show_pay_info(cfg)

    show_recommend_reward_tips(user_buy_info)

    # 显示小助手的使用概览
    if cfg.common._show_usage:
        show_usage()

    # 运行结束展示下多进程信息
    show_multiprocessing_info(cfg)

    # 检查是否有更新，用于提示未购买自动更新的朋友去手动更新~
    if cfg.common.check_update_on_end:
        check_update(cfg)

    # 运行完毕备份配置到本机数据目录
    try_save_configs_to_user_data_dir()

    increase_counter(name="run/end", ga_type=ga.GA_REPORT_TYPE_PAGE_VIEW)

    show_head_line("运行完毕")

def main_wrapper():
    freeze_support()

    logger.info(color("bold_green") + f"已将工作目录设置为小助手所在目录：{dirpath}，之前为：{old_path}")

    try:
        run_start_time = datetime.datetime.now()
        main()
        total_used_time = datetime.datetime.now() - run_start_time
        logger.warning(color("fg_bold_yellow") + f"运行完成，共用时{total_used_time}")

        # 如果总用时太高的情况时，尝试提示开启多进程和超快速模式
        cfg = config()
        if total_used_time > datetime.timedelta(minutes=10) and (
            not cfg.common.enable_multiprocessing or not cfg.common.enable_super_fast_mode
        ):
            msg = (
                f"当前累计用时似乎很久({total_used_time})，是否要尝试多进程和超快速模式？\n"
                "多进程模式下，将开启多个进程并行运行不同账号的领取流程\n"
                "额外开启超快速模式，会进一步将不同账号的不同活动都异步领取，进一步加快领取速度\n"
                "\n"
                "如果需要开启，请打开配置工具，在【公共配置】tab中勾选【是否启用多进程功能】和【是否启用超快速模式（并行活动）】"
            )
            logger.warning(color("bold_cyan") + msg)
            if is_weekly_first_run("用时过久提示"):
                async_message_box(msg, "用时过久", print_log=False)

        # 按照分钟级别来统计使用时长
        total_minutes = int(total_used_time.total_seconds()) // 60
        increase_counter(ga_category="run_used_time_minutes", name=total_minutes)
    except Exception as e:
        show_unexpected_exception_message(e)
        # 如果在github action，则继续抛出异常
        if is_run_in_github_action():
            raise e
    finally:
        # 暂停一下，方便看结果
        if not disable_pause_after_run() and not is_run_in_github_action():
            async_call_close_pool_after_some_time()
            pause()
        close_pool()

def async_call_close_pool_after_some_time():
    def _close():
        wait_time = 10 * 60
        logger.info(f"{wait_time} 秒后将自动关闭进程池，方便有足够时间查看进程池中触发的弹窗信息")
        time.sleep(wait_time)
        close_pool()

    async_call(_close)

if __name__ == "__main__":
    main_wrapper()


from imagenet_c import *
from torchvision.datasets import ImageNet
import torchvision.transforms as transforms
from torch.utils.data import DataLoader
import os
import torch
import gorilla

DATA_ROOT = './data'
CORRUPTION_PATH = './corruption'

corruption_tuple = (gaussian_noise, shot_noise, impulse_noise, defocus_blur,
                    glass_blur, motion_blur, zoom_blur, snow, frost, fog,
                    brightness, contrast, elastic_transform, pixelate, jpeg_compression)

corruption_dict = {corr_func.__name__: corr_func for corr_func in corruption_tuple}

class corrupt(object):
    def __init__(self, corruption_name, severity=5):
        self.corruption_name = corruption_name
        self.severity = severity
        return

    def __call__(self, x):
        # x: PIL.Image
        x_corrupted = corruption_dict[self.corruption_name](x, self.severity)
        return np.uint8(x_corrupted)

    def __repr__(self):
        return "Corruption(name=" + self.corruption_name + ", severity=" + str(self.severity) + ")"

if os.path.exists(os.path.join(DATA_ROOT, CORRUPTION_PATH)) is False:
    os.mkdir(os.path.join(DATA_ROOT, CORRUPTION_PATH))

for corruption in corruption_dict.keys():
    if os.path.exists(os.path.join(DATA_ROOT, CORRUPTION_PATH, corruption + '.pth')):
        continue
    print(corruption)
    val_transform = transforms.Compose([
        transforms.Resize(256),
        transforms.CenterCrop(224),
        corrupt(corruption, 5)
    ])

    target_dataset = ImageNet(DATA_ROOT, 'val', transform=val_transform)

    target_dataloader = DataLoader(target_dataset, batch_size=256, shuffle=False, drop_last=False, num_workers=2)

    datas = []
    for batch in gorilla.track(target_dataloader):
        datas.append(batch[0])
    datas = torch.concat(datas)
    torch.save(datas, os.path.join(DATA_ROOT, CORRUPTION_PATH, corruption + '.pth'))


from flask import Flask
import requests

URL="https://en.wikipedia.org/w/api.php"

app = Flask(__name__)

#configuring the server name as required
app.config['SERVER_NAME'] = "wiki-search.com:5000"

@app.route("/")
def home():
    return 'Enter you query as the subdomain.'

@app.route('/', subdomain="<SEARCHPAGE>")

#function that searches for the URL
def url_search(SEARCHPAGE):
    if SEARCHPAGE is None:
        return 'Enter you search query as the subdomain'

    title_data = requests.get(URL, params={
        "action": "query",
        "format": "json",
        "list": "search",
        "srsearch": SEARCHPAGE}).json()

    #creating a list named titles and appending the titles of every search result into it
    titles = []
    for title in title_data['query']['search']:
        titles.append(title['title'])

    #creating a list named urls to which the url of every title is appended
    urls = []
    for title in titles:
        url_data = requests.get(URL, params={
            "action": "query",
            "format": "json",
            "titles": title,
            "prop": "info",
            "inprop": "url"}).json()

        for key in url_data['query']['pages']:
            urls.append(url_data['query']['pages'][key]['fullurl'])

    #creating a dictionary that contains the links appended as a list to the key named links
    results={"links":urls}
    return results

if __name__ == '__main__':
    app.run(debug=True,port=5000)


import math
from os import TMP_MAX

MAX = 700

lookup = [[0 for i in range(100001)]
                 for j in range(20)]

def buildSparseTable(arr, n):

    for i in range(0, n):
        lookup[i][0] = arr[i]

    j = 1

    while (1 << j) <= n:

        i = 0
        while (i + (1 << j) - 1) < n:

            if (lookup[i][j - 1] >
                lookup[i + (1 << (j - 1))][j - 1]):
                lookup[i][j] = lookup[i][j - 1]
            else:
                lookup[i][j] = \
                        lookup[i + (1 << (j - 1))][j - 1]

            i += 1
        j += 1

def query(L, R):

    j = int(math.log2(R - L + 1))

    if lookup[L][j] >= lookup[R - (1 << j) + 1][j]:
        return lookup[L][j]

    else:
        return lookup[R - (1 << j) + 1][j]

n = int(input())

nums = [int(i) for i in input().split()]

num_q = int(input())

queries = []

sparse_table = buildSparseTable(nums, len(nums))

for i in range(num_q):
    l, r = [int(i) for i in input().split()]
    if l > r:
        tmp = l
        l = r
        r = tmp
        print(query(l-1, r-1))


import pandas as pd
import timeit

pd.set_option('mode.chained_assignment', None)

def main():
        print('ok')

def sortim(df):

        new_cols = ['PERFIL','BANDEIRA','LOCAL_NOVO','LOCAL','TAMANHO','REGIÃO','REGIAO 2','CONCATENADO','CONCATENADO 2','CONCATENADO 3','CONCATENADO 4','CONCATENADO 5']
        for i in new_cols:
                df[i] = ''

        print('computado!')
        for i in range(len(df)):
                if 'RUA' in df["DESCTAMANHO"][i]:
                        df['LOCAL_NOVO'][i] = "CONVENCIONAL"
                        df['LOCAL'][i] = "RUA"
                elif 'SHOPPING' in df["DESCTAMANHO"][i]:
                        df['LOCAL_NOVO'][i] = "CONVENCIONAL"
                        df['LOCAL'][i] = "SHOPPING"
                elif 'QUIOSQUE' in df["DESCTAMANHO"][i]:
                        df['LOCAL_NOVO'][i] = "QUISOQUE"
                        df['LOCAL'][i] = "QUISOQUE"
                else:
                        df['LOCAL_NOVO'][i] = "COMPACTA"
                        df['LOCAL'][i] = "COMPACTA"

                if '-CB-' in df["DESCTAMANHO"][i]:
                        df['BANDEIRA'][i] = "CB"
                else:
                        df['BANDEIRA'][i] = "PF"

                if '-CB-' in df["DESCTAMANHO"][i]:
                        df['BANDEIRA'][i] = "CB"
                else:
                        df['BANDEIRA'][i] = "PF"

                if '-PERFIL 1-' in df["DESCTAMANHO"][i]:
                        df['PERFIL'][i] = "PERFIL 1"
                elif '-PERFIL 2-' in df["DESCTAMANHO"][i]:
                        df['PERFIL'][i] = "PERFIL 2"
                elif '-PERFIL 3-' in df["DESCTAMANHO"][i]:
                        df['PERFIL'][i] = "PERFIL 3"
                elif '-PERFIL 4-' in df["DESCTAMANHO"][i]:
                        df['PERFIL'][i] = "PERFIL 4"
                else:
                        df['PERFIL'][i] = "PERFIL 5"

                if '-PP-' in df["DESCTAMANHO"][i]:
                        df['TAMANHO'][i] = "PP"
                elif '-P-' in df["DESCTAMANHO"][i]:
                        df['TAMANHO'][i] = "P"
                elif '-M-' in df["DESCTAMANHO"][i]:
                        df['TAMANHO'][i] = "M"
                elif '-G-' in df["DESCTAMANHO"][i]:
                        df['TAMANHO'][i] = "G"
                elif '-GG-' in df["DESCTAMANHO"][i]:
                        df['TAMANHO'][i] = "GG"
                else:
                        df['TAMANHO'][i] = "MEGA"

                if '-RJ/ES' in df["DESCTAMANHO"][i]:
                        df['REGIÃO'][i] = "RJ/ES"
                        df['REGIAO 2'][i] = "RJ/ES"
                elif '-SPI' in df["DESCTAMANHO"][i]:
                        df['REGIÃO'][i] = "SPI"
                        df['REGIAO 2'][i] = "SPI"
                elif '-MG' in df["DESCTAMANHO"][i]:
                        df['REGIÃO'][i] = "MG"
                        df['REGIAO 2'][i] = "MG"
                elif '-SUL' in df["DESCTAMANHO"][i]:
                        df['REGIÃO'][i] = "SUL"
                        df['REGIAO 2'][i] = "SUL"
                elif '-GDE SP' in df["DESCTAMANHO"][i]:
                        df['REGIÃO'][i] = "GDE SP"
                        df['REGIAO 2'][i] = "GDE SP"
                elif '-CO/N' in df["DESCTAMANHO"][i]:
                        df['REGIÃO'][i] = "CO/N"
                        df['REGIAO 2'][i] = "CON/N"
                else:
                        df['REGIÃO'][i] = "NE"
                        df['REGIAO 2'][i] = "NE"
                df['CONCATENADO'][i] = f"{df['LOCAL_NOVO'][i]}-{df['PERFIL'][i]}-{df['TAMANHO'][i]}"
                df['CONCATENADO 2'][i] = f"{df['LOCAL_NOVO'][i]}-{df['PERFIL'][i]}-{df['REGIÃO'][i]}-{df['TAMANHO'][i]}"

                df['CONCATENADO 3'] = df['LOCAL_NOVO'].map(str) + '-' + df['PERFIL'].map(str) + '-' + df['REGIÃO'].map(
                        str)
                df['CONCATENADO 4'] = df['LOCAL'].map(str) + '-' + df['TAMANHO'].map(str)
                df['CONCATENADO 5'] = df['LOCAL_NOVO'].map(str) + '-' + df['PERFIL'].map(str) + '-' + df['BANDEIRA'].map(
                        str) + '-' + df['TAMANHO']
                df['CONCATENADO 6'] = df['PERFIL'].map(str) + '-' + df['BANDEIRA'].map(str) + '-' + df['LOCAL'].map(
                        str) + '-' + df['REGIÃO']

        colorder=['CODPUBLICO','DESCPUBLICO','CODSECAO','DESCSECAO','CODSORTIMENTO','CODTAMANHO','DESCTAMANHO','PERFIL','BANDEIRA','LOCAL_NOVO','LOCAL','TAMANHO','REGIÃO','REGIAO 2','CONCATENADO','CONCATENADO 2','CONCATENADO 3','CONCATENADO 4','CONCATENADO 5','CONCATENADO 6','EMPRESA','FILIAL','TIPOPUBLICO','SOLICVOLTDIF']
        df = df[colorder]

        return df

def sortemp(df):
        new_cols = ['PERFIL', 'BANDEIRA', 'LOCAL_NOVO', 'LOCAL', 'TAMANHO', 'REGIÃO', 'REGIAO 2', 'CONCATENADO',
                    'CONCATENADO 2']
        start = timeit.default_timer()
        for i in new_cols:
                df[i] = ' '

        df = df.reindex()

        print('novas colunas')

        df['LOCAL_NOVO'] = df['DESCTAMANHO'].apply(lambda x: 'CONVENCIONAL' if 'RUA' in x else x)
        df['LOCAL_NOVO'] = df['LOCAL_NOVO'].apply(lambda x: 'CONVENCIONAL' if 'SHOPPING' in x else x)
        df['LOCAL_NOVO'] = df['LOCAL_NOVO'].apply(lambda x: 'QUIOSQUE' if 'QUIOSQUE' in x else x)
        df['LOCAL_NOVO'] = df['LOCAL_NOVO'].apply(lambda x: 'COMPACTA' if 'COMPACTA' in x else x)
        df['LOCAL_NOVO'] = df['LOCAL_NOVO'].apply(lambda x: 'DIGITAL' if 'DIGITAL' in x else x)

        df['LOCAL'] = df['DESCTAMANHO'].apply(lambda x: 'RUA' if 'RUA' in x else x)
        df['LOCAL'] = df['LOCAL'].apply(lambda x: 'SHOPPING' if 'SHOPPING' in x else x)
        df['LOCAL'] = df['LOCAL'].apply(lambda x: 'QUIOSQUE' if 'QUIOSQUE' in x else x)
        df['LOCAL'] = df['LOCAL'].apply(lambda x: 'COMPACTA' if 'COMPACTA' in x else x)
        df['LOCAL'] = df['LOCAL'].apply(lambda x: 'DIGITAL' if 'DIGITAL' in x else x)

        df['BANDEIRA'] = df['DESCTAMANHO'].apply(lambda x: 'CB' if '-CB-' in x else x)
        df['BANDEIRA'] = df['BANDEIRA'].apply(lambda x: 'PF' if '-PF-' in x else x)

        df['PERFIL'] = df['DESCTAMANHO'].apply(lambda x: 'PERFIL 1' if '-PERFIL 1-' in x else x)
        df['PERFIL'] = df['PERFIL'].apply(lambda x: 'PERFIL 2' if '-PERFIL 2-' in x else x)
        df['PERFIL'] = df['PERFIL'].apply(lambda x: 'PERFIL 3' if '-PERFIL 3-' in x else x)
        df['PERFIL'] = df['PERFIL'].apply(lambda x: 'PERFIL 4' if '-PERFIL 4-' in x else x)
        df['PERFIL'] = df['PERFIL'].apply(lambda x: 'PERFIL 5' if '-PERFIL 5-' in x else x)

        df['TAMANHO'] = df['DESCTAMANHO'].apply(lambda x: 'PP' if '-PP-' in x else x)
        df['TAMANHO'] = df['TAMANHO'].apply(lambda x: 'P' if '-P-' in x else x)
        df['TAMANHO'] = df['TAMANHO'].apply(lambda x: 'M' if '-M-' in x else x)
        df['TAMANHO'] = df['TAMANHO'].apply(lambda x: 'G' if '-G-' in x else x)
        df['TAMANHO'] = df['TAMANHO'].apply(lambda x: 'GG' if '-GG-' in x else x)
        df['TAMANHO'] = df['TAMANHO'].apply(lambda x: 'MEGA' if '-MEGA-' in x else x)

        df['REGIÃO'] = df['DESCTAMANHO'].apply(lambda x: 'RJ/ES' if '-RJ/ES' in x else x)
        df['REGIÃO'] = df['REGIÃO'].apply(lambda x: 'SPI' if '-SPI' in x else x)
        df['REGIÃO'] = df['REGIÃO'].apply(lambda x: 'MG' if '-MG' in x else x)
        df['REGIÃO'] = df['REGIÃO'].apply(lambda x: 'SUL' if '-SUL' in x else x)
        df['REGIÃO'] = df['REGIÃO'].apply(lambda x: 'GDE SP' if '-GDE SP' in x else x)
        df['REGIÃO'] = df['REGIÃO'].apply(lambda x: 'CO/N' if '-CO/N' in x else x)
        df['REGIÃO'] = df['REGIÃO'].apply(lambda x: 'NE' if '-NE' in x else x)

        df['REGIAO 2'] = df['DESCTAMANHO'].apply(lambda x: 'RJ/ES' if '-RJ/ES' in x else x)
        df['REGIAO 2'] = df['REGIAO 2'].apply(lambda x: 'SPI' if '-SPI' in x else x)
        df['REGIAO 2'] = df['REGIAO 2'].apply(lambda x: 'MG' if '-MG' in x else x)
        df['REGIAO 2'] = df['REGIAO 2'].apply(lambda x: 'SUL' if '-SUL' in x else x)
        df['REGIAO 2'] = df['REGIAO 2'].apply(lambda x: 'GDE SP' if '-GDE SP' in x else x)
        df['REGIAO 2'] = df['REGIAO 2'].apply(lambda x: 'CO/N' if '-CO/N' in x else x)
        df['REGIAO 2'] = df['REGIAO 2'].apply(lambda x: 'NE' if '-NE' in x else x)

        df['CONCATENADO'] = df['LOCAL_NOVO'].map(str) + '-' + df['PERFIL'].map(str) + '-' + df['TAMANHO']
        df['CONCATENADO 2'] = df['LOCAL_NOVO'].map(str) + '-' + df['PERFIL'].map(str) + '-' + df['REGIÃO'].map(
                str) + '-' + df['TAMANHO']

        colorder = ['CODSORTIMENTO', 'CODSECAO', 'DESCSECAO', 'CODTAMANHO', 'DESCTAMANHO', 'CODPUBLICO',
                    'DESCPUBLICO',
                    'CODMERCADORIA', 'DESCMERCADORIA', 'MERCCONJ', 'PRIORIDADE', 'QTDE', 'VOLTAGEM',
                    'VIGENCIAINICIO',
                    'VIGENCIAFIM', 'PERFIL', 'BANDEIRA', 'LOCAL', 'LOCAL_NOVO', 'TAMANHO', 'REGIÃO', 'STATUS',
                    'DESCMARCA', 'DESCESPECIE', 'CONCATENADO', 'CONCATENADO 2']
        df = df[colorder]

        return df


from wordbank import wbank
import random

state = { 'word': '',
          'history' : [],
          'gc' : 0
          }

def userguess(inp, state) -> dict:
    guess = list(inp.upper())
    wo = list(state['word'])
    out = []
    for lg in guess:
        if lg in wo:
            out.append(('yellow', "*" + lg))
        else:
            out.append(('white', lg))
    for i in range(len(guess)):
        if guess[i] == wo[i]:
            out[i] = ('green', "#" + guess[i])
    state['history'].append(out)
    return state

def userinput() -> tuple:
    inp = input("[][][][][]")
    inp = inp.strip()
    if len(inp) == 5:
        return True, inp
    else:
        return False, inp

def wordle(state) -> None:
    print("Hello, let's play!")
    print("A * next to a letter means it's in the word, a # means it's in the right place!")
    while True:
        while True:
            valid, inp = userinput()
            if valid:
                break
            else:
                print(f"{inp} wasn't a valid guess, your guess needs to be 5 letters long")
        state = userguess(inp, state)
        state['gc'] += 1
        for h in state['history']:
            print(f"[{']['.join([x[1] for x in h])}]")
        if len([x[0] for x in state['history'][-1] if x[0] == 'green']) == 5:
            print("You win!")
            break
        else:
            if state['gc'] == 5:
                print(f"The word was {state['word']}, you didn't win!")
                break

if __name__ == "__main__":
    while True:
        state = { 'word': random.choice(wbank).upper(),
          'history' : [],
          'gc' : 0
          }
        wordle(state)
        inp = input("Play again? y/n")
        if inp.strip().lower() == "y":
            continue
        else:
            print("Goodbye!  Thanks for playing!")
            break


__author__ = "https://github.com/kdha0727"

import os
import functools
import contextlib

import torch
import torch.distributed as dist
from torch.cuda import is_available as _cuda_available

RANK = 0
WORLD_SIZE = 1

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#                                       Setup Tools                                       #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

def is_initialized():
    # if pytorch isn't compiled with c10d, is_initialized is omitted from namespace.
    # this function wraps
    """
    Returns c10d (distributed) runtime is initialized.
    """
    return dist.is_available() and getattr(dist, "is_initialized", lambda: False)()

def setup_dist(temp_dir, rank, world_size):
    """
    Set up a distributed process group.
    """

    if is_initialized():
        return True

    init_file = os.path.abspath(os.path.join(temp_dir, '.torch_distributed_init'))
    if os.name == 'nt':
        init_method = 'file:///' + init_file.replace('\\', '/')
        dist.init_process_group(
            backend='gloo', init_method=init_method, rank=rank, world_size=world_size)
    else:
        init_method = f'file://{init_file}'
        dist.init_process_group(
            backend='nccl', init_method=init_method, rank=rank, world_size=world_size)

    global RANK, WORLD_SIZE
    RANK = rank
    WORLD_SIZE = world_size

    torch.cuda.set_device(dev())
    torch.cuda.empty_cache()

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#                                      General Tools                                      #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

@functools.lru_cache(maxsize=None)
def get_rank(group=None):
    if group is not None and is_initialized():
        return dist.get_rank(group=group)
    return RANK

@functools.lru_cache(maxsize=None)
def get_world_size(group=None):
    if group is not None and is_initialized():
        return dist.get_world_size(group=group)
    return WORLD_SIZE

def barrier(*args, **kwargs):
    if is_initialized():
        return dist.barrier(*args, **kwargs)

@contextlib.contextmanager
def synchronized_ops():
    barrier()
    yield
    barrier()
    return

@functools.lru_cache(maxsize=None)
def dev(group=None):
    """
    Get the device to use for torch.distributed.
    """
    if _cuda_available():
        return torch.device(get_rank(group))
    return torch.device("cpu")

def load_state_dict(local_or_remote_path, **kwargs):
    """
    Load a PyTorch file.
    """
    with open(local_or_remote_path, "rb") as f:
        return torch.load(f, **kwargs)

def broadcast(tensor, src=0, group=None, async_op=False):
    """
    Synchronize a Tensor across ranks from {src} rank. (default=0)
    :param tensor: torch.Tensor.
    :param src: source rank to sync params from. default is 0.
    :param group:
    :param async_op:
    """
    if not is_initialized():
        return
    with torch.no_grad():
        dist.broadcast(tensor, src, group=group, async_op=async_op)

def sync_params(params, src=0, group=None, async_op=False):
    """
    Synchronize a sequence of Tensors across ranks from {src} rank. (default=0)
    :param params: Sequence of torch.Tensor.
    :param src: source rank to sync params from. default is 0.
    :param group:
    :param async_op:
    """
    if not is_initialized():
        return
    for p in params:
        broadcast(p, src, group=group, async_op=async_op)


from deep_rl_for_swarms.common import explained_variance, zipsame, dataset
from deep_rl_for_swarms.common import logger
import deep_rl_for_swarms.common.tf_util as U
import tensorflow as tf, numpy as np
import time
import os
from deep_rl_for_swarms.common import colorize
from mpi4py import MPI
from collections import deque
from deep_rl_for_swarms.common.mpi_adam import MpiAdam
from deep_rl_for_swarms.common.cg import cg
from contextlib import contextmanager
from deep_rl_for_swarms.common.act_wrapper import ActWrapper
import sys
from gym import spaces
import matplotlib.pyplot as plt

def traj_segment_generator(pi, env, horizon, stochastic):

    # Initialize state variables
    t = 0
    n_agents = env.nr_agents
    new = True
    ob = env.reset()

    cur_ep_ret = 0
    cur_ep_len = 0
    ep_rets = []
    ep_lens = []
    time_steps = []

    # Initialize history arrays
    obs = np.array([ob for _ in range(horizon)])
    rews = np.zeros([horizon, n_agents], 'float32')
    vpreds = np.zeros([horizon, n_agents], 'float32')
    news = np.zeros([horizon, n_agents], 'int32')
    if isinstance(env.action_space, spaces.Box):
        ac = np.vstack([env.action_space.sample() for _ in range(n_agents)])  # Used only to initialize vectors!!
        acs = np.array([ac for _ in range(horizon)])
    elif isinstance(env.action_space, spaces.Discrete):
        ac = np.array([env.action_space.sample() for _ in range(n_agents)])
        acs = np.zeros([horizon, n_agents], 'int32')  # For discrete actions
    else:
        raise NotImplementedError
    prevacs = acs.copy()
    time = np.zeros(horizon, 'float32')  # To store the time of acting
    # Info to be saved in the logger
    keys_to_save = ['attackers_caught', 'attackers_not_caught', 'mean_total_rwd', 'total_rwd']
    if env.attack_mode == 'phy':
        keys_to_save.extend(['phy_fc_error_rate'])
    if env.attack_mode == 'mac':
        keys_to_save.extend(['total_mac_tx', 'total_mac_col', 'total_bits_tx', 'prop_t_tx', 'mean_prop_bits_tx_at',
                             'mean_prop_bits_tx_no'])
    info_indiv = []
    while True:
        prevac = ac
        ac, vpred = pi.act(stochastic, np.vstack(ob))
        if isinstance(env.action_space, spaces.Box):
            ac = np.clip(ac, env.action_space.low, env.action_space.high)  # To ensure actions are in the right limit!
        # Slight weirdness here because we need value function at time T
        # before returning segment [0, T-1] so we get the correct
        # terminal value
        if t > 0 and t % horizon == 0:
            info_total = {}
            for key in keys_to_save:
                aux = 0
                for i in range(len(info_indiv)):
                    aux += info_indiv[i][key]
                info_total[key] = aux / len(info_indiv)
            if isinstance(env.action_space, spaces.Box):
                yield [
                    dict(
                        ob=np.array(obs[:, na, :]),
                        rew=np.array(rews[:, na]),
                        vpred=np.array(vpreds[:, na]),
                        new=np.array(news[:, na]),
                        ac=np.array(acs[:, na, :]),
                        prevac=np.array(prevacs[:, na, :]),
                        nextvpred=vpred[na] * (1 - new),
                        ep_rets=[epr[na] for epr in ep_rets],
                        ep_lens=ep_lens,
                        time_steps=np.array(time_steps),
                        time=time,
                    ) for na in range(n_agents)
                ], info_total
            elif isinstance(env.action_space, spaces.Discrete):
                yield [
                    dict(
                        ob=np.array(obs[:, na, :]),
                        rew=np.array(rews[:, na]),
                        vpred=np.array(vpreds[:, na]),
                        new=np.array(news[:, na]),
                        ac=np.array(acs[:, na]),
                        prevac=np.array(prevacs[:, na]),
                        nextvpred=vpred[na] * (1 - new),
                        ep_rets=[epr[na] for epr in ep_rets],
                        ep_lens=ep_lens,
                        time_steps=np.array(time_steps),
                        time=time
                    ) for na in range(n_agents)
                ], info_total
            else:
                raise NotImplementedError
            _, vpred = pi.act(stochastic, ob)
            # Be careful!!! if you change the downstream algorithm to aggregate
            # several of these batches, then be sure to do a deepcopy
            ep_rets = []
            ep_lens = []
            time_steps = []
            info_indiv = []
        i = t % horizon
        time_steps.append(t)
        obs[i] = ob
        vpreds[i] = vpred
        news[i] = new
        acs[i] = ac
        prevacs[i] = prevac
        if env.attack_mode == 'mac':
            time[i] = sum(env.t_counter)
        elif env.attack_mode == 'phy':
            time[i] = env.timestep
        else:
            raise RuntimeError('Environment not recognized')
        ob, rew, new, info = env.step(ac)
        rews[i] = rew
        #mask_undetected[i] = np.logical_not(env.banned[0:env.nr_agents])

        cur_ep_ret += rew
        cur_ep_len += 1
        if new:
            info_indiv.append(info)
            ep_rets.append(cur_ep_ret)
            ep_lens.append(cur_ep_len)
            cur_ep_ret = 0
            cur_ep_len = 0
            ob = env.reset()
            sys.stdout.write('\r Current horizon length = ' + str((t + 1) % horizon) + '/' + str(horizon))
            sys.stdout.flush()
        t += 1

def add_vtarg_and_adv(seg, gamma, lam):
    new = [np.append(p["new"], 0) for p in seg]  # last element is only used for last vtarg, but we already zeroed it if last new = 1
    vpred = [np.append(p["vpred"], p["nextvpred"]) for p in seg]

    for i, p in enumerate(seg):
        T = len(p["rew"])
        p["adv"] = gaelam = np.empty(T, 'float32')
        rew = p["rew"]
        lastgaelam = 0
        for t in reversed(range(T)):
            nonterminal = 1 - new[i][t + 1]
            delta = rew[t] + gamma * vpred[i][t + 1] * nonterminal - vpred[i][t]
            gaelam[t] = lastgaelam = delta + gamma * lam * nonterminal * lastgaelam
        p["tdlamret"] = p["adv"] + p["vpred"]

'''
def add_vtarg_and_adv(seg, gamma, lam):  # Modified version to include time!
    new = [np.append(p["new"], 0) for p in seg]  # last element is only used for last vtarg, but we already zeroed it if last new = 1
    vpred = [np.append(p["vpred"], p["nextvpred"]) for p in seg]

    for i, p in enumerate(seg):
        T = len(p["rew"])
        p["adv"] = gaelam = np.empty(T, 'float32')
        rew = p["rew"]
        lastgaelam = 0
        time = np.append(p['time'], p['time'][-1] + 1)  # Increase the final time by 1 to obtain the differential time
        difft = time[1:] - time[0: -1]
        for t in reversed(range(T)):
            nonterminal = 1 - new[i][t + 1]
            gt = gamma ** difft[t]  # Note that when difft is negative, nonterminal is 0!!
            lt = lam ** difft[t]
            delta = rew[t] + gt * vpred[i][t + 1] * nonterminal - vpred[i][t]
            gaelam[t] = lastgaelam = delta + gt * lt * nonterminal * lastgaelam
        p["tdlamret"] = p["adv"] + p["vpred"]
'''

def learn(env, policy_fn, *,
          timesteps_per_batch, # what to train on
          max_kl, cg_iters,
          gamma, lam, # advantage estimation
          entcoeff=0.0,
          cg_damping=1e-2,
          vf_stepsize=3e-4,
          vf_iters =3,
          max_timesteps=0, max_episodes=0, max_iters=0,  # time constraint
          callback=None,
          save_dir=None,
          save_flag=False,
          plot_flag=False
          ):
    nworkers = MPI.COMM_WORLD.Get_size()
    rank = MPI.COMM_WORLD.Get_rank()
    np.set_printoptions(precision=3)
    # Setup losses and stuff
    # ----------------------------------------
    ob_space = env.observation_space
    ac_space = env.action_space
    pi = policy_fn("pi", ob_space, ac_space)
    oldpi = policy_fn("oldpi", ob_space, ac_space)
    atarg = tf.placeholder(dtype=tf.float32, shape=[None])  # Target advantage function (if applicable)
    ret = tf.placeholder(dtype=tf.float32, shape=[None])  # Empirical return

    ob = U.get_placeholder_cached(name="ob")
    ac = pi.pdtype.sample_placeholder([None])

    kloldnew = oldpi.pd.kl(pi.pd)
    ent = pi.pd.entropy()
    meankl = tf.reduce_mean(kloldnew)
    meanent = tf.reduce_mean(ent)
    entbonus = entcoeff * meanent

    vferr = tf.reduce_mean(tf.square(pi.vpred - ret))

    ratio = tf.exp(pi.pd.logp(ac) - oldpi.pd.logp(ac))  # advantage * pnew / pold (advantage--> Next line)
    surrgain = tf.reduce_mean(ratio * atarg)

    optimgain = surrgain + entbonus
    losses = [optimgain, meankl, entbonus, surrgain, meanent]
    loss_names = ["optimgain", "meankl", "entloss", "surrgain", "entropy"]

    dist = meankl

    all_var_list = pi.get_trainable_variables()
    var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("pol")]  # Policy variables
    var_list.extend([v for v in all_var_list if v.name.split("/")[1].startswith("me")])  # Mean embedding variables
    vf_var_list = [v for v in all_var_list if v.name.split("/")[1].startswith("vf")]  # Value function variables
    vfadam = MpiAdam(vf_var_list)

    get_flat = U.GetFlat(var_list)
    set_from_flat = U.SetFromFlat(var_list)
    klgrads = tf.gradients(dist, var_list)
    flat_tangent = tf.placeholder(dtype=tf.float32, shape=[None], name="flat_tan")
    shapes = [var.get_shape().as_list() for var in var_list]
    start = 0
    tangents = []
    for shape in shapes:
        sz = U.intprod(shape)
        tangents.append(tf.reshape(flat_tangent[start:start + sz], shape))
        start += sz
    gvp = tf.add_n([tf.reduce_sum(g * tangent) for (g, tangent) in zipsame(klgrads, tangents)])  # pylint: disable=E1111
    fvp = U.flatgrad(gvp, var_list)

    assign_old_eq_new = U.function([], [], updates=[tf.assign(oldv, newv)
                                                    for (oldv, newv) in
                                                    zipsame(oldpi.get_variables(), pi.get_variables())])
    compute_losses = U.function([ob, ac, atarg], losses)
    compute_lossandgrad = U.function([ob, ac, atarg], losses + [U.flatgrad(optimgain, var_list)])
    compute_fvp = U.function([flat_tangent, ob, ac, atarg], fvp)
    compute_vflossandgrad = U.function([ob, ret], U.flatgrad(vferr, vf_var_list))

    @contextmanager
    def timed(msg):
        if rank == 0:
            print(colorize(msg, color='magenta'))
            tstart = time.time()
            yield
            print(colorize("done in %.3f seconds" % (time.time() - tstart), color='magenta'))
        else:
            yield

    def allmean(x):
        assert isinstance(x, np.ndarray)
        out = np.empty_like(x)
        MPI.COMM_WORLD.Allreduce(x, out, op=MPI.SUM)
        out /= nworkers
        return out

    act_params = {
        'name': "pi",
        'ob_space': ob_space,
        'ac_space': ac_space,
    }

    pi = ActWrapper(pi, act_params)

    U.initialize()
    th_init = get_flat()
    MPI.COMM_WORLD.Bcast(th_init, root=0)
    set_from_flat(th_init)
    vfadam.sync()
    print("Init param sum", th_init.sum(), flush=True)

    # Prepare for rollouts
    # ----------------------------------------
    seg_gen = traj_segment_generator(pi, env, timesteps_per_batch, stochastic=True)

    episodes_so_far = 0
    timesteps_so_far = 0
    iters_so_far = 0
    tstart = time.time()
    lenbuffer = deque(maxlen=40)  # rolling buffer for episode lengths
    rewbuffer = deque(maxlen=40)  # rolling buffer for episode rewards

    assert sum([max_iters > 0, max_timesteps > 0, max_episodes > 0]) == 1

    while True:
        if callback: callback(locals(), globals())
        if max_timesteps and timesteps_so_far >= max_timesteps:
            break
        elif max_episodes and episodes_so_far >= max_episodes:
            break
        elif max_iters and iters_so_far >= max_iters:
            break

        if max_timesteps:
            print(colorize(str(100 * timesteps_so_far / max_timesteps) + ' % of timesteps', color='magenta'))
        elif max_episodes:
            print(colorize(str(100 * episodes_so_far / max_episodes) + ' % of episodes', color='magenta'))
        elif max_iters:
            print(colorize(str(100 * iters_so_far / max_iters) + ' % of iters', color='magenta'))
        logger.log("********** Iteration %i ************" % iters_so_far)

        with timed("sampling"):
            seg, info = seg_gen.__next__()
        add_vtarg_and_adv(seg, gamma, lam)

        ob = np.concatenate([s['ob'] for s in seg], axis=0)
        ac = np.concatenate([s['ac'] for s in seg], axis=0)
        atarg = np.concatenate([s['adv'] for s in seg], axis=0)
        tdlamret = np.concatenate([s['tdlamret'] for s in seg], axis=0)
        vpredbefore = np.concatenate([s["vpred"] for s in seg], axis=0)  # predicted value function before udpate
        atarg = (atarg - atarg.mean()) / atarg.std()  # standardized advantage function estimate

        # if hasattr(pi, "ret_rms"): pi.ret_rms.update(tdlamret)
        # if hasattr(pi, "ob_rms"): pi.ob_rms.update(ob) # update running mean/std for policy

        args = ob, ac, atarg
        fvpargs = [arr[::5] for arr in args]

        def fisher_vector_product(p):
            return allmean(compute_fvp(p, *fvpargs)) + cg_damping * p

        assign_old_eq_new()  # set old parameter values to new parameter values
        with timed("computegrad"):
            *lossbefore, g = compute_lossandgrad(*args)
        lossbefore = allmean(np.array(lossbefore))
        g = allmean(g)
        if np.allclose(g, 0):
            logger.log("Got zero gradient. not updating")
        else:
            with timed("cg"):
                stepdir = cg(fisher_vector_product, g, cg_iters=cg_iters, verbose=rank == 0)
            assert np.isfinite(stepdir).all()
            shs = .5 * stepdir.dot(fisher_vector_product(stepdir))
            lm = np.sqrt(shs / max_kl)
            # logger.log("lagrange multiplier:", lm, "gnorm:", np.linalg.norm(g))
            fullstep = stepdir / lm
            expectedimprove = g.dot(fullstep)
            surrbefore = lossbefore[0]
            stepsize = 1.0
            thbefore = get_flat()
            for _ in range(10):
                thnew = thbefore + fullstep * stepsize
                set_from_flat(thnew)
                meanlosses = surr, kl, *_ = allmean(np.array(compute_losses(*args)))
                improve = surr - surrbefore
                logger.log("Expected: %.3f Actual: %.3f" % (expectedimprove, improve))
                if not np.isfinite(meanlosses).all():
                    logger.log("Got non-finite value of losses -- bad!")
                elif kl > max_kl * 1.5:
                    logger.log("violated KL constraint. shrinking step.")
                elif improve < 0:
                    logger.log("surrogate didn't improve. shrinking step.")
                else:
                    logger.log("Stepsize OK!")
                    break
                stepsize *= .5
            else:
                logger.log("couldn't compute a good step")
                set_from_flat(thbefore)
            if nworkers > 1 and iters_so_far % 20 == 0:
                paramsums = MPI.COMM_WORLD.allgather((thnew.sum(), vfadam.getflat().sum()))  # list of tuples
                assert all(np.allclose(ps, paramsums[0]) for ps in paramsums[1:])

        for (lossname, lossval) in zip(loss_names, meanlosses):
            logger.record_tabular(lossname, lossval)

        with timed("vf"):

            for _ in range(vf_iters):
                for (mbob, mbret) in dataset.iterbatches((ob, tdlamret),
                                                         include_final_partial_batch=False, batch_size=64):
                    g = allmean(compute_vflossandgrad(mbob, mbret))
                    vfadam.update(g, vf_stepsize)

        lrlocal = (seg[0]["ep_lens"], seg[0]["ep_rets"])  # local values
        listoflrpairs = MPI.COMM_WORLD.allgather(lrlocal) # list of tuples
        lens, rews = map(flatten_lists, zip(*listoflrpairs))

        episodes_so_far += len(lens)
        timesteps_so_far += sum(lens)
        iters_so_far += 1

        logger.record_tabular("EpisodesSoFar", episodes_so_far)
        logger.record_tabular("TimestepsSoFar", timesteps_so_far)
        logger.record_tabular("TimeElapsed", time.time() - tstart)
        # Add info values
        logger.record_tabular("AttC", info['attackers_caught'])
        logger.record_tabular("AttNC", info['attackers_not_caught'])
        logger.record_tabular("MtR", info['mean_total_rwd'])  # Mean total reward
        logger.record_tabular("TtR", info['total_rwd'])  # Total reward
        if env.attack_mode == 'phy':
            logger.record_tabular("Fce", info['phy_fc_error_rate'])
        if env.attack_mode == 'mac':
            logger.record_tabular("Tmt", info['total_mac_tx'])
            logger.record_tabular("Tmc", info['total_mac_col'])
            logger.record_tabular("Tbt", info['total_bits_tx'])
            logger.record_tabular("Ptt", info['prop_t_tx'])
            logger.record_tabular("MpbtA", info['mean_prop_bits_tx_at'])
            logger.record_tabular("MpbtN", info['mean_prop_bits_tx_no'])

        if rank == 0:
            logger.dump_tabular()

    if save_flag:
        pi.save(os.path.normpath(save_dir + '/models_trpo/model_{}.pkl'.format(iters_so_far)))

def flatten_lists(listoflists):
    return [el for list_ in listoflists for el in list_]


import boto3
import json
import uuid

print('Loading function')
def lambda_handler(event, context):
    bucketName = event['Records'][0]['s3']['bucket']['name']
    fileName = event['Records'][0]['s3']['object']['key']

    return detect_labels_and_put_dynamoDB(fileName, bucketName)

def detect_labels_and_put_dynamoDB(photo, bucket):
    rekognitionClient=boto3.client('rekognition', 'us-east-2')
    dynamoClient = boto3.client('dynamodb')

    response = rekognitionClient.detect_labels(Image={'S3Object':{'Bucket':bucket,'Name':photo}},
        MaxLabels=10)

    print('Detected labels for ' + photo)
    for label in response['Labels']:
        dynamoClient.put_item(
            TableName='RekognitionDetails',
            Item= {
                'ID' : {
                    'S': str(uuid.uuid4())
                },
                'Filename': {
                    'S': photo
                },
                'Category': {
                    'S' : label['Name']
                },
                'Confidence': {
                   'N': str(label['Confidence'])
                }
            })

def main():
    photo=''
    bucket=''
    label_count=detect_labels(photo, bucket)
    print("Labels detected: " + str(label_count))


import firebase_admin
import googleapiclient
from firebase_admin import credentials
from firebase_admin import db
import os
from os.path import join, dirname
from dotenv import load_dotenv
from XmlParser import XmlParser

class FirebaseService:
    dotenv_path = join(dirname(__file__), '.env')
    load_dotenv(dotenv_path)
    DATABASE_URL = os.environ.get("DATABASE_URL")
    if not firebase_admin._apps:
        print('初期化')
        cred = credentials.Certificate('hologram-test-firebase-adminsdk.json')

        firebase_admin.initialize_app(cred, {
            'databaseURL': DATABASE_URL,
        })

    ref_db = db.reference('/video')

    def __init__(self, video_item):
        self.video_item = video_item

    # FirestoreのドキュメントIDを取得
    def get_db_id(self):
        print("FirebaseService", 'get_db_id')
        id_list = []
        key_val = self.ref_db.get()
        # DB上に書き込まれたアイテムのvideoIdを取得
        for key, val in key_val.items():
            id_list.append(key)
        return id_list

    #
    def write_video_item(self):
        print("FirebaseService", "write_video_item")
        self.ref_db.update(self.video_item)

    def delete_video_item(self, update_db_items, xml_video_ids, error_channel_ids):
        print("FirebaseService", 'update_db_items', len(update_db_items), update_db_items)
        print("FirebaseService", 'xml_video_ids', len(xml_video_ids), xml_video_ids)
        print("FirebaseService", 'error_channel_ids', len(error_channel_ids), error_channel_ids)

        # 一週間以上まえのアイテムのみ抽出
        # （更新後のDB上のアイテム）-（XMLで取得したアイテム）
        last_week_ids = set(update_db_items).difference(set(xml_video_ids))
        print('last_week_ids', last_week_ids)
        for single_id in last_week_ids:
            db_channelId = self.ref_db.child(f'{single_id}').child('channelId').get()
            print('db_channel_id', db_channelId)
            # dbから取得したアイテムのチャンネルIDがエラーが発生したチャンネルIDリストの中に含まれていなければ削除
            # xml_parseで取得できなかったチャンネルの動画情報が削除されてしまうため
            if db_channelId not in set(error_channel_ids):
                print('delete', f'{single_id}')
                self.ref_db.child(f'{single_id}').delete()


import numpy as np
import pandas as pd
import time
from metric import SampleScore,EventScore, AdjustedMutualInfoScore
from joblib import Parallel, delayed

class Experiment:

    def __init__(self,algorithms:list, configurations:list, thresholds = np.linspace(0,1,101),njobs=1,verbose = True) -> None:

        """Initialization

        Args:
            algorithms (list): list of algorithm classes
            configurations (list): list of list of configurations as dictionnaries for each algorithms classes
            thresholds (np.ndarray, optional): numpy array of thresholds to consider for the event based metric. Defaults to numpy.linspace(0,1,101).
        """
        self.algorithms = algorithms
        self.configurations = configurations
        self.thresholds = thresholds
        self.njobs = njobs
        self.verbose = verbose

    def compute_scores(self,label,prediction):

        single_pred = np.clip(np.sum(prediction,axis=0),0,1).reshape(1,-1)
        single_label = np.clip(np.sum(label,axis=0),0,1).reshape(1,-1)

        scores = []

        #single sample score
        p,r,f = SampleScore().score(single_label,single_pred)
        scores.append(["sss-precision",p])
        scores.append(["sss-recall",r])
        scores.append(["sss-fscore",f])

        #sample score
        p,r,f = SampleScore().score(label,prediction)
        scores.append(["ss-precision",p])
        scores.append(["ss-recall",r])
        scores.append(["ss-fscore",f])

        # weigthed sample score
        p,r,f = SampleScore(averaging="weighted").score(label,prediction)
        scores.append(["w-ss-precision",p])
        scores.append(["w-ss-recall",r])
        scores.append(["w-ss-fscore",f])

        #single event score
        lp,lr,lf = EventScore().score(single_label,single_pred,self.thresholds)
        for t,p,r,f in zip(self.thresholds,lp,lr,lf):
            scores.append([f"ses-precision_{np.round(t,2)}",p])
            scores.append([f"ses-recall_{np.round(t,2)}",r])
            scores.append([f"ses-fscore_{np.round(t,2)}",f])
        scores.append(["ses-auc-precision",np.mean(lp)])
        scores.append(["ses-auc-recall",np.mean(lr)])
        scores.append(["ses-auc-fscore",np.mean(lf)])

        #event score
        lp,lr,lf = EventScore().score(label,prediction,self.thresholds)
        for t,p,r,f in zip(self.thresholds,lp,lr,lf):
            scores.append([f"es-precision_{np.round(t,2)}",p])
            scores.append([f"es-recall_{np.round(t,2)}",r])
            scores.append([f"es-fscore_{np.round(t,2)}",f])
        scores.append(["es-auc-precision",np.mean(lp)])
        scores.append(["es-auc-recall",np.mean(lr)])
        scores.append(["es-auc-fscore",np.mean(lf)])

        # weighted event score
        lp,lr,lf = EventScore(averaging="weighted").score(label,prediction,self.thresholds)
        for t,p,r,f in zip(self.thresholds,lp,lr,lf):
            scores.append([f"w-es-precision_{np.round(t,2)}",p])
            scores.append([f"w-es-recall_{np.round(t,2)}",r])
            scores.append([f"w-es-fscore_{np.round(t,2)}",f])
        scores.append(["w-es-auc-precision",np.mean(lp)])
        scores.append(["w-es-auc-recall",np.mean(lr)])
        scores.append(["w-es-auc-fscore",np.mean(lf)])

        #ajusted mutual information
        scores.append(["amis",AdjustedMutualInfoScore().score(label,prediction)])

        return scores

    def signal_algo_class_experiement(self,signal_idx,signal,label,algo_class,config,config_idx):
        "Return a DF"
        #keep only labels row that are activated by the signal
        label = label[label.sum(axis=1)>0]

        #update the number of patterns to predict if required
        t_config = config.copy()
        if ("n_patterns" in t_config.keys()):
            if (isinstance(t_config["n_patterns"],int)):
                t_config["n_patterns"] = label.shape[0]
            else:
                t_config["n_patterns"] = None

        try:
            #get predictions
            algo = algo_class(**t_config)
            start = time.time()
            algo.fit(signal)
            end = time.time()

            #compute scores
            scores = self.compute_scores(label,algo.prediction_mask_)

            tdf = pd.DataFrame(scores,columns=["metric","score"])
            tdf["algorithm"] = algo_class.__name__
            tdf["config_idx"] = config_idx
            tdf["execution_time"] = end - start
            tdf["signal_idx"] = signal_idx
            tdf["n_patterns"] = label.shape[0]
            tdf["predicted_n_patterns"] = algo.prediction_mask_.shape[0]

            if self.verbose:
                s1 = np.round(tdf[tdf["metric"] == "es-auc-fscore"].score.values[0],2)
                s2 = np.round(tdf[tdf["metric"] == "amis"].score.values[0],2)
                print(f"signal_id: {signal_idx}, algo: {algo_class.__name__}, config_id: {config_idx}, f-auc: {s1}, ami: {s2}")

            return tdf

        except:
            s= f"signal_id: {signal_idx}, algo: {algo_class.__name__}, config_id: {config_idx} failed to fit."
            if self.verbose:
                print(s)
            if self.logs_path_ is not None:
                with open(self.logs_path_,"a") as f:
                    f.write(s +"\n")

    def run_experiment(self,dataset:np.ndarray,labels:np.ndarray,backup_path = None,batch_size=10,logs_path = None,verbose = True)->np.ndarray:
        """_summary_

        Args:
            dataset (np.ndarray): array of signals, signal shape (L,), variable length allowed
            labels (np.ndarray): array of labels, label shape (L,), variable length allowed
            signal_configs (pd.DataFrame, optional): Dataframe containing the configuration of the synthetic generator for each signals.
            backup_path (str, optional): Path to store df in case of big experiment. If None no saving. Defaults to None.
            batch_size (int, optional)
            verbose (bool, optional): verbose. Defaults to True.

        Returns:
            pd.DataFrame: scores_df
        """
        self.logs_path_ = logs_path

        n_signals = len(dataset)
        n_configs = np.sum([len(conf) for conf in self.configurations])
        total = n_signals*n_configs

        if backup_path != None:
            n_batches  = n_signals//batch_size
            if n_batches >0:
                batches =[zip(dataset[i*batch_size:(i+1)*batch_size],labels[i*batch_size:(i+1)*batch_size]) for i in range(n_batches)]
            else:
                batches = []
            if n_signals % batch_size !=0:
                batches.append(zip(dataset[n_batches*batch_size:],labels[n_batches*batch_size:]))
        else:
            batches = [zip(dataset,labels)]

        self.df_ = pd.DataFrame()

        counts = 0
        for batch in batches:
            results = Parallel(n_jobs=self.njobs)(
                delayed(self.signal_algo_class_experiement)(counts+id_s,signal,label,algo,config,id_c)
                for id_s,(signal,label) in enumerate(batch)
                for id_a,algo in enumerate(self.algorithms)
                for id_c,config in enumerate(self.configurations[id_a])
                )
            counts = min(counts+batch_size,n_signals)
            self.df_= pd.concat((self.df_,*results)).reset_index(drop = True)
            self.df_ = self.df_.astype({'metric':str, "score":float, "algorithm":str,'config_idx':int,"signal_idx":int, "n_patterns":int, "predicted_n_patterns":int})

            if backup_path != None:
                self.df_.to_csv(backup_path)

            if verbose:
                print(f"Achieved [{counts*n_configs}/{total}]")

        return self.df_