"""
超市发清洗逻辑
销售，成本，库存，商品， 分类

"""
from datetime import datetime
import pandas as pd
import boto3
import tempfile
import pytz
import time
import numpy as np
_TZINFO = pytz.timezone("Asia/Shanghai")

S3_BUCKET = "ext-etl-data"
S3 = boto3.resource("s3")

CLEANED_PATH = "clean_data/source_id={source_id}/clean_date={date}/target_table={target_table}/dump={timestamp}&rowcount={rowcount}.csv.gz"

def clean_fujirongtong(source_id, date, target_table, data_frames):
    if target_table == "goodsflow":
        return clean_goodsflow(source_id, date, target_table, data_frames)
    elif target_table == "cost":
        return clean_cost(source_id, date, target_table, data_frames)
    elif target_table == "store":
        return clean_store(source_id, date, target_table, data_frames)
    elif target_table == "goods":
        return clean_goods(source_id, date, target_table, data_frames)
    elif target_table == "category":
        return clean_category(source_id, date, target_table, data_frames)
    else:
        pass

def clean_goodsflow(source_id, date, target_table, data_frames):
    """
    清洗销售流水
    :param source_id:
    :param date:
    :param target_table:
    :param data_frames:
    :return:
    """

    cmid = source_id.split("Y")[0]
    flow_frames = data_frames["sale_j"]
    pay_frames = data_frames["pay_j"]
    pay_frames = pay_frames[pay_frames["paytype"] == "h"].drop_duplicates()

    if len(flow_frames) == 0:
        goodsflow_frames = pd.DataFrame(columns=[
            'source_id', 'cmid', 'foreign_store_id', 'store_name', 'receipt_id', 'consumer_id', 'saletime', 'last_updated',
            'foreign_item_id', 'barcode', 'item_name', 'item_unit', 'saleprice', 'quantity', 'subtotal',
            'foreign_category_lv1', 'foreign_category_lv1_name', 'foreign_category_lv2', 'foreign_category_lv2_name',
            'foreign_category_lv3', 'foreign_category_lv3_name', 'foreign_category_lv4', 'foreign_category_lv4_name',
            'foreign_category_lv5', 'foreign_category_lv5_name', 'pos_id'])
    else:
        goodsflow_frames = pd.merge(flow_frames, pay_frames, how="left", on=["listno", "posid", "sdate", "shopid", "stime"])
        goodsflow_frames = goodsflow_frames.merge(data_frames["goods"], how="left", on="goodsid")
        goodsflow_frames = goodsflow_frames.merge(data_frames["shop"], how="left", left_on="shopid", right_on="id", suffixes=("", ".store"))

        sgroup_frames = data_frames["sgroup"]
        lv3 = sgroup_frames[sgroup_frames["deptlevelid"] == "3"]
        lv2 = sgroup_frames[sgroup_frames["deptlevelid"] == "2"]
        lv1 = sgroup_frames[sgroup_frames["deptlevelid"] == "1"]
        sgroup_frames["id_1"] = sgroup_frames.apply(lambda row: row["id"][:1], axis=1)
        sgroup_frames["id_2"] = sgroup_frames.apply(lambda row: row["id"][:3], axis=1)
        sgroup_frames["id_3"] = sgroup_frames.apply(lambda row: row["id"][:5], axis=1)
        sgroup_frames = sgroup_frames.merge(lv3, how="left", left_on="id_3", right_on="id", suffixes=("", "_lv3"))
        sgroup_frames = sgroup_frames.merge(lv2, how="left", left_on="id_2", right_on="id", suffixes=("", "_lv2"))
        sgroup_frames = sgroup_frames.merge(lv1, how="left", left_on="id_1", right_on="id", suffixes=("", "_lv1"))
        sgroup_frames = sgroup_frames[sgroup_frames["deptlevelid"] == "4"]
        sgroup_frames = sgroup_frames[["id", "name", "id_lv3", "name_lv3", "id_lv2", "name_lv2", "id_lv1", "name_lv1"]]
        sgroup_frames = sgroup_frames.rename(columns={
            "id": "foreign_category_lv4",
            "name": "foreign_category_lv4_name",
            "id_lv3": "foreign_category_lv3",
            "name_lv3": "foreign_category_lv3_name",
            "id_lv2": "foreign_category_lv2",
            "name_lv2": "foreign_category_lv2_name",
            "id_lv1": "foreign_category_lv1",
            "name_lv1": "foreign_category_lv1_name",
        })

        goodsflow_frames["deptid1"] = goodsflow_frames.apply(lambda row: row["deptid"][:7], axis=1)
        goodsflow_frames = goodsflow_frames.merge(sgroup_frames, how="left", left_on="deptid1", right_on="foreign_category_lv4")
        goodsflow_frames = goodsflow_frames.merge(data_frames["dept"], how="left", left_on="deptid", right_on="id", suffixes=("", "_lv5"))
        goodsflow_frames = goodsflow_frames[(goodsflow_frames["id"].notna()) & (goodsflow_frames["goodsid"].notna())]

        goodsflow_frames["source_id"] = source_id
        goodsflow_frames["cmid"] = cmid
        goodsflow_frames["last_updated"] = datetime.now(_TZINFO)

        def quatity(row):

            x = row["amount"]/row["x"] if row["x"] != 0 else 0
            return x
        goodsflow_frames["quantity"] = goodsflow_frames.apply(quatity, axis=1)

        def subtotal(row):
            x = row["salevalue"] - row["discvalue"]
            return x
        goodsflow_frames["subtotal"] = goodsflow_frames.apply(subtotal, axis=1)

        def saletime(row):
            date = row["sdate"].split(" ")[0]
            hour = row["stime"][0:2]
            minute = row["stime"][2:4]
            secend = row["stime"][4:6]
            time = f"{date} {hour}:{minute}:{secend}"
            return time

        goodsflow_frames["saletime"] = goodsflow_frames.apply(saletime, axis=1)
        goodsflow_frames = goodsflow_frames.rename(columns={
            "id": "foreign_store_id",
            "name.store": "store_name",
            "listno": "receipt_id",
            "cardno": "consumer_id",
            "goodsid": "foreign_item_id",
            "usebarcodeid": "barcode",
            "name": "item_name",
            "unitname": "item_unit",
            "price": "saleprice",
            "id_lv5": "foreign_category_lv5",
            "name_lv5": "foreign_category_lv5_name",
            "posid": "pos_id"
        })

        goodsflow_frames = goodsflow_frames[[
            'source_id', 'cmid', 'foreign_store_id', 'store_name', 'receipt_id', 'consumer_id', 'saletime', 'last_updated',
            'foreign_item_id', 'barcode', 'item_name', 'item_unit', 'saleprice', 'quantity', 'subtotal',
            'foreign_category_lv1', 'foreign_category_lv1_name', 'foreign_category_lv2', 'foreign_category_lv2_name',
            'foreign_category_lv3', 'foreign_category_lv3_name', 'foreign_category_lv4', 'foreign_category_lv4_name',
            'foreign_category_lv5', 'foreign_category_lv5_name', 'pos_id'
        ]]

    return upload_to_s3(goodsflow_frames, source_id, date, target_table)

def clean_cost(source_id, date, target_table, data_frames):
    """
    清洗成本
    :param source_id:
    :param date:
    :param target_table:
    :param frames:
    :return:
    """
    cmid = source_id.split("Y")[0]
    cost_frames = data_frames["salecost"]
    cost_frames = cost_frames.merge(data_frames["goods"], on="goodsid")
    if len(cost_frames) == 0:
        cost_frames = pd.DataFrame(columns=[
            "source_id", "foreign_store_id", "foreign_item_id", "date", "cost_type", "total_quantity", "total_sale",
            "total_cost", "foreign_category_lv1", "foreign_category_lv2", "foreign_category_lv3", "foreign_category_lv4",
            "foreign_category_lv5", "cmid"])
    else:
        sgroup_frames = data_frames["sgroup"]
        lv3 = sgroup_frames[sgroup_frames["deptlevelid"] == "3"]
        lv2 = sgroup_frames[sgroup_frames["deptlevelid"] == "2"]
        lv1 = sgroup_frames[sgroup_frames["deptlevelid"] == "1"]
        sgroup_frames["id_1"] = sgroup_frames.apply(lambda row: row["id"][:1], axis=1)
        sgroup_frames["id_2"] = sgroup_frames.apply(lambda row: row["id"][:3], axis=1)
        sgroup_frames["id_3"] = sgroup_frames.apply(lambda row: row["id"][:5], axis=1)

        sgroup_frames = sgroup_frames.merge(lv3, how="left", left_on="id_3", right_on="id", suffixes=("", "_lv3"))
        sgroup_frames = sgroup_frames.merge(lv2, how="left", left_on="id_2", right_on="id", suffixes=("", "_lv2"))
        sgroup_frames = sgroup_frames.merge(lv1, how="left", left_on="id_1", right_on="id", suffixes=("", "_lv1"))
        sgroup_frames = sgroup_frames[sgroup_frames["deptlevelid"] == "4"]
        sgroup_frames = sgroup_frames[["id", "id_lv3", "id_lv2", "id_lv1"]]
        sgroup_frames = sgroup_frames.rename(columns={
            "id": "foreign_category_lv4",
            "id_lv3": "foreign_category_lv3",
            "id_lv2": "foreign_category_lv2",
            "id_lv1": "foreign_category_lv1",
        })

        cost_frames["deptid1"] = cost_frames.apply(lambda row: row["deptid"][:7], axis=1)
        cost_frames = cost_frames.merge(sgroup_frames, how="left", left_on="deptid1", right_on="foreign_category_lv4")
        cost_frames = cost_frames.merge(data_frames["dept"], how="left", left_on="deptid", right_on="id", suffixes=("", "_lv5"))

        def generate_total_sale(row):
            res = row["salevalue"] - row["discvalue"]
            return res
        cost_frames["total_sale"] = cost_frames.apply(generate_total_sale, axis=1)
        cost_frames = cost_frames.groupby([
            "shopid", "goodsid", "sdate", "foreign_category_lv1", "foreign_category_lv2", "foreign_category_lv3", "foreign_category_lv4", "id"], as_index=False).\
            agg({"qty": np.sum, "costvalue": np.sum, "total_sale": np.sum})
        cost_frames["source_id"] = source_id
        cost_frames["cmid"] = cmid
        cost_frames["cost_type"] = ""
        cost_frames = cost_frames.rename(columns={
            "shopid": "foreign_store_id",
            "goodsid": "foreign_item_id",
            "sdate": "date",
            "qty": "total_quantity",
            "costvalue": "total_cost",
            "id": "foreign_category_lv5",
        })

        cost_frames = cost_frames[[
            "source_id", "foreign_store_id", "foreign_item_id", "date", "cost_type", "total_quantity", "total_sale",
            "total_cost", "foreign_category_lv1", "foreign_category_lv2", "foreign_category_lv3", "foreign_category_lv4",
            "foreign_category_lv5", "cmid"
        ]]

    return upload_to_s3(cost_frames, source_id, date, target_table)

def clean_goods(source_id, date, target_table, data_frames):
    """
    清洗商品
    :param source_id:
    :param date:
    :param target_table:
    :param frames:
    :return:
    """
    """
    origin_table_columns = {
        "goods": ['barcodeid', 'goodsid', 'name', 'unitname', 'flag', 'deptid', 'keepdays'],
        "cost": ['cost', 'goodsid', 'shopid', 'flag'],
        "goodsshop": ['price', 'goodsid', 'shopid'],
    }

    coverts = {"goods": {"barcodeid": str, "goodsid": str, "deptid": str, "goodsid": str},
               "cost": {"goodsid": str, "shopid": str},
               "goodsshop": {"goodsid": str}}
    """
    cmid = source_id.split("Y")[0]

    goods_frames = data_frames["goods"]
    goods_frames = goods_frames.merge(data_frames["cost"], on="goodsid", suffixes=("", ".b"))
    goods_frames = goods_frames[(goods_frames["shopid"] == "A00A") & (goods_frames["flag.b"] == "0")]
    goods_frames = goods_frames.merge(data_frames["goodsshop"], on="goodsid", suffixes=("", ".c"))
    goods_frames = goods_frames[goods_frames["shopid.c"] == 'A00A']

    def item_status(x):
        if x == "0":
            res = "正常"
        elif x == "8":
            res = "新品"
        elif x == "1":
            res = "暂停订货"
        elif x == "2":
            res = "暂停销售"
        elif x == "3":
            res = "清理"
        elif x == "4":
            res = "暂停经营"
        else:
            res = ""
        return res
    goods_frames['item_status'] = goods_frames.flag.apply(lambda x: item_status(x))
    goods_frames["cmid"] = cmid
    goods_frames['foreign_category_lv1'] = goods_frames.deptid.apply(lambda x: str(x)[:1])
    goods_frames['foreign_category_lv2'] = goods_frames.deptid.apply(lambda x: str(x)[:3])
    goods_frames['foreign_category_lv3'] = goods_frames.deptid.apply(lambda x: str(x)[:5])
    goods_frames['foreign_category_lv4'] = goods_frames.deptid.apply(lambda x: str(x)[:7])
    goods_frames["storage_time"] = datetime.now(_TZINFO)
    goods_frames["last_updated"] = datetime.now(_TZINFO)
    goods_frames["isvalid"] = "1"
    goods_frames["allot_method"] = ""
    goods_frames["supplier_name"] = ""
    goods_frames["supplier_code"] = ""
    goods_frames["brand_name"] = ""
    goods_frames["show_code"] = goods_frames["goodsid"]
    goods_frames = goods_frames.rename(columns={
        "barcodeid": "barcode",
        "goodsid": "foreign_item_id",
        "name": "item_name",
        "cost": "lastin_price",
        "price": "sale_price",
        "unitname": "item_unit",
        "deptid": "foreign_category_lv5",
        "keepdays": "warranty",
    })

    goods_frames = goods_frames[[
        "cmid",
        "barcode",
        "foreign_item_id",
        "item_name",
        "lastin_price",
        "sale_price",
        "item_unit",
        "item_status",
        "foreign_category_lv1",
        "foreign_category_lv2",
        "foreign_category_lv3",
        "foreign_category_lv4",
        "storage_time",
        "last_updated",
        "isvalid",
        "warranty",
        "show_code",
        "foreign_category_lv5",
        "allot_method",
        "supplier_name",
        "supplier_code",
        "brand_name",
    ]]

    return upload_to_s3(goods_frames, source_id, date, target_table)

def clean_category(source_id, date, target_table, data_frames):
    """
    分类清洗
    :param source_id:
    :param date:
    :param target_table:
    :param frames:
    :return:
    """
    cmid = source_id.split("Y")[0]
    sgroup_frames = data_frames["sgroup"]
    lv1 = sgroup_frames[sgroup_frames["id"].str.len() == 1].rename(columns=lambda x: f"lv1_{x}")
    lv2 = sgroup_frames[sgroup_frames["deptlevelid"] == "2"].rename(columns=lambda x: f"lv2_{x}")
    lv3 = sgroup_frames[sgroup_frames["deptlevelid"] == "3"].rename(columns=lambda x: f"lv3_{x}")
    lv4 = sgroup_frames[sgroup_frames["deptlevelid"] == "4"].rename(columns=lambda x: f"lv4_{x}")
    lv2["lv1_id"] = sgroup_frames.id.map(lambda x: x[:1])
    lv3["lv2_id"] = sgroup_frames.id.map(lambda x: x[:3])
    lv4["lv3_id"] = sgroup_frames.id.map(lambda x: x[:5])

    category1 = lv1.copy()
    category1 = category1[category1["lv1_deptlevelid"] == "1"]
    category1 = category1.rename(columns={
        "lv1_deptlevelid": "level",
        "lv1_id": "foreign_category_lv1",
        "lv1_name": "foreign_category_lv1_name"
    })
    category1["foreign_category_lv2"] = ""
    category1["foreign_category_lv2_name"] = ""
    category1["foreign_category_lv3"] = ""
    category1["foreign_category_lv3_name"] = ""
    category1["foreign_category_lv4"] = ""
    category1["foreign_category_lv4_name"] = ""
    category1["foreign_category_lv5"] = ""
    category1["foreign_category_lv5_name"] = ""
    category1["cmid"] = cmid
    category1["last_updated"] = datetime.now(_TZINFO)
    category1 = category1[[
        'cmid', 'level', 'foreign_category_lv1', 'foreign_category_lv1_name', 'foreign_category_lv2',
        'foreign_category_lv2_name', 'foreign_category_lv3', 'foreign_category_lv3_name',
        'last_updated', 'foreign_category_lv4', 'foreign_category_lv4_name', 'foreign_category_lv5',
        'foreign_category_lv5_name'
    ]]

    category2 = pd.merge(lv2.copy(), lv1.copy(), how="left", on="lv1_id")
    category2 = category2.rename(columns={
        "lv2_deptlevelid": "level",
        "lv1_id": "foreign_category_lv1",
        "lv1_name": "foreign_category_lv1_name",
        "lv2_id": "foreign_category_lv2",
        "lv2_name": "foreign_category_lv2_name"
    })
    category2["foreign_category_lv3"] = ""
    category2["foreign_category_lv3_name"] = ""
    category2["foreign_category_lv4"] = ""
    category2["foreign_category_lv4_name"] = ""
    category2["foreign_category_lv5"] = ""
    category2["foreign_category_lv5_name"] = ""
    category2["cmid"] = cmid
    category2["last_updated"] = datetime.now(_TZINFO)
    category2 = category2[[
        'cmid', 'level', 'foreign_category_lv1', 'foreign_category_lv1_name', 'foreign_category_lv2',
        'foreign_category_lv2_name', 'foreign_category_lv3', 'foreign_category_lv3_name',
        'last_updated', 'foreign_category_lv4', 'foreign_category_lv4_name', 'foreign_category_lv5',
        'foreign_category_lv5_name'
    ]]

    category3 = pd.merge(lv3.copy(), lv2.copy(), how="left", on="lv2_id").merge(lv1.copy(), how="left", on="lv1_id")
    category3 = category3.rename(columns={
        "lv3_deptlevelid": "level",
        "lv1_id": "foreign_category_lv1",
        "lv1_name": "foreign_category_lv1_name",
        "lv2_id": "foreign_category_lv2",
        "lv2_name": "foreign_category_lv2_name",
        "lv3_id": "foreign_category_lv3",
        "lv3_name": "foreign_category_lv3_name",
    })
    category3["foreign_category_lv4"] = ""
    category3["foreign_category_lv4_name"] = ""
    category3["foreign_category_lv5"] = ""
    category3["foreign_category_lv5_name"] = ""
    category3["cmid"] = cmid
    category3["last_updated"] = datetime.now(_TZINFO)
    category3 = category3[[
        'cmid', 'level', 'foreign_category_lv1', 'foreign_category_lv1_name', 'foreign_category_lv2',
        'foreign_category_lv2_name', 'foreign_category_lv3', 'foreign_category_lv3_name',
        'last_updated', 'foreign_category_lv4', 'foreign_category_lv4_name', 'foreign_category_lv5',
        'foreign_category_lv5_name'
    ]]

    category4 = pd.merge(lv4.copy(), lv3.copy(), how="left", on="lv3_id").merge(lv2.copy(), how="left", on="lv2_id").\
        merge(lv1.copy(), how="left", on="lv1_id")
    category4 = category4.rename(columns={
        "lv4_deptlevelid": "level",
        "lv1_id": "foreign_category_lv1",
        "lv1_name": "foreign_category_lv1_name",
        "lv2_id": "foreign_category_lv2",
        "lv2_name": "foreign_category_lv2_name",
        "lv3_id": "foreign_category_lv3",
        "lv3_name": "foreign_category_lv3_name",
        "lv4_id": "foreign_category_lv4",
        "lv4_name": "foreign_category_lv4_name"
    })
    category4["foreign_category_lv5"] = ""
    category4["foreign_category_lv5_name"] = ""
    category4["cmid"] = cmid
    category4["last_updated"] = datetime.now(_TZINFO)
    category4 = category4[[
        'cmid', 'level', 'foreign_category_lv1', 'foreign_category_lv1_name', 'foreign_category_lv2',
        'foreign_category_lv2_name', 'foreign_category_lv3', 'foreign_category_lv3_name',
        'last_updated', 'foreign_category_lv4', 'foreign_category_lv4_name', 'foreign_category_lv5',
        'foreign_category_lv5_name'
    ]]

    category5 = data_frames["dept"]
    category5["lv4_id"] = category5.id.apply(lambda x: x[:7])
    category5 = category5.merge(lv4.copy(), how="left", on="lv4_id").merge(lv3.copy(), how="left", on="lv3_id").\
        merge(lv2.copy(), how="left", on="lv2_id").merge(lv1.copy(), how="left", on="lv1_id")
    category5 = category5.rename(columns={
        "lv1_id": "foreign_category_lv1",
        "lv1_name": "foreign_category_lv1_name",
        "lv2_id": "foreign_category_lv2",
        "lv2_name": "foreign_category_lv2_name",
        "lv3_id": "foreign_category_lv3",
        "lv3_name": "foreign_category_lv3_name",
        "lv4_id": "foreign_category_lv4",
        "lv4_name": "foreign_category_lv4_name",
        "id": "foreign_category_lv5",
        "name": "foreign_category_lv5_name",
    })
    category5["cmid"] = cmid
    category5["last_updated"] = datetime.now(_TZINFO)
    category5["level"] = "5"
    category5 = category5[[
        'cmid', 'level', 'foreign_category_lv1', 'foreign_category_lv1_name', 'foreign_category_lv2',
        'foreign_category_lv2_name', 'foreign_category_lv3', 'foreign_category_lv3_name',
        'last_updated', 'foreign_category_lv4', 'foreign_category_lv4_name', 'foreign_category_lv5',
        'foreign_category_lv5_name'
    ]]

    category = pd.concat([category1, category2, category3, category4, category5])

    return upload_to_s3(category, source_id, date, target_table)

def clean_store(source_id, date, target_table, data_frames):
    """
    门店清洗
    :param source_id:
    :param date:
    :param target_table:
    :param frames:
    :return:
    """
    cmid = source_id.split("Y")[0]
    store_frames = data_frames["shop"]
    store_frames["cmid"] = cmid
    store_frames["address_code"] = ""
    store_frames["device_id"] = ""

    def generate_stor_status(x):
        if x == 11:
            res = "正常营业"
        elif x == 98:
            res = "闭店"
        else:
            res = "其它"
        return res
    store_frames["store_status"] = store_frames.shoptype.apply(generate_stor_status)
    store_frames["create_date"] = datetime.now(_TZINFO)
    store_frames["lat"] = None
    store_frames["lng"] = None
    store_frames["contacts"] = None
    store_frames["area_code"] = None
    store_frames["area_name"] = None
    store_frames["business_area"] = None
    store_frames["property_id"] = None
    store_frames["property"] = "直营店"
    store_frames["source_id"] = source_id
    store_frames["last_updated"] = datetime.now(_TZINFO)
    store_frames["show_code"] = store_frames["id"]
    store_frames = store_frames.rename(columns={
        "id": "foreign_store_id",
        "name": "store_name",
        "address": "store_address",
        "linktele": "phone_number"
    })
    store_frames = store_frames[
        ['cmid', 'foreign_store_id', 'store_name', 'store_address', 'address_code', 'device_id', 'store_status',
         'create_date', 'lat', 'lng', 'show_code', 'phone_number', 'contacts', 'area_code', 'area_name',
         'business_area', 'property_id', 'property', 'source_id', 'last_updated']]

    return upload_to_s3(store_frames, source_id, date, target_table)

def upload_to_s3(frame, source_id, date, target_table):
    filename = tempfile.NamedTemporaryFile(mode="w", encoding="utf-8")
    count = len(frame)
    frame.to_csv(filename.name, index=False, compression="gzip", float_format='%.4f')
    filename.seek(0)
    key = CLEANED_PATH.format(
        source_id=source_id,
        target_table=target_table,
        date=date,
        timestamp=now_timestamp(),
        rowcount=count,
    )
    S3.Bucket(S3_BUCKET).upload_file(filename.name, key)
    return key

def now_timestamp():
    _timestamp = datetime.fromtimestamp(time.time(), tz=_TZINFO)
    return _timestamp


import json
def parse_student_json_data(file_name):
     """
     :param file_name:
     :return:
     """
     fh = open(file_name)
     for line in fh:
          raw_data = json.loads(line)
          print("{}".format(raw_data['first_name']))

def parse_provisioner_json_data(file_name):
    """
    :param file_name:
    :return:
    """
    # fh = open(file_name)
    # raw_data = json.loads(fh)

    with open(file_name, "r") as read_file:
         data = json.load(read_file)
         #print(json.dumps(data, indent=4, sort_keys=True)) #Sorting
         #print(json.dumps(data, indent=4, sort_keys=False))  # dumps ->printing
         #print(type(data)) # object comes in as a dictionary
         #print(data) # contents of the dictionary are shown for clarity here
         # navigate through the dictionary items
         provisioner_details = data["members"][0]["name"] #- > Molecule Man
         print(type(data["members"])) # ->List
         print(type(data["members"][0]))  # ->Dic
         #provisioner_details = data['name']
         print("Provisioner Details: {}".format(provisioner_details)) # this is a list of dictionaries
         print("\n\nItems in Provisioner List:")

         for item in provisioner_details:
             print(item, "\n") # iterate through the dictionaries in the list of configuration items

if __name__ == "__main__":
     # parse_student_json_data("jsons_prov.jason")
     parse_provisioner_json_data("jsons_prov.json")
     new_data = {"student_id": "L000223344", "first_name": "Jo",
     "last_name": "Michaels"}
     print(new_data)
     with open("jsons", "a") as write_file:
          write_file.write('\n') # move onto a newline before writing object
          json.dump(new_data, write_file, indent=4,)


#!/usr/bin/env python3

"""
CockroachDB the Definitive Guide sample code
This code  is public domain software, i.e. not copyrighted.

Warranty Exclusion
------------------
You agree that this software is a non-commercially developed program that may
contain "bugs"   and that it may not function as intended.
The software is licensed "as is". The authors make no, and hereby expressly
disclaim all, warranties, express, implied, statutory, or otherwise
with respect to the software, including noninfringement and the implied
warranties of merchantability and fitness for a particular purpose.

@author:  gharriso (guy.a.harrison@gmail.com)

Using AS OF SYSTEM TIME

"""

import sys
import time
import csv

import psycopg2
from psycopg2 import pool

def main():

  if ((len(sys.argv)) !=2):
    sys.exit("Error:No URL provided on command line")
  uri=sys.argv[1]

  pool= psycopg2.pool.ThreadedConnectionPool(10, 40, uri) # min connection=10, max=40

  connection = pool.getconn()

  cursor=connection.cursor()
  connection.autocommit=False
  cursor.execute('ROLLBACK')
  cursor.execute("BEGIN AS OF SYSTEM TIME '-10s'" )
  top10cities=cursor.execute('''SELECT city,count(*)
                                  FROM movr.rides GROUP BY city
                                 ORDER BY 2 DESC LIMIT 10''')
  top10users=cursor.execute('''SELECT name, rides
                                 FROM movr.user_ride_counts
                                ORDER BY  rides DESC LIMIT 10''')
  cursor.execute('COMMIT')

  print (now)
  cursor.close()
  connection.close()

main()

chapters/Ch06 - Application Development/python/asOfSystemTime.py


from collections import defaultdict

t = int(input())
for _ in range(t):
	n = int(input())
	arr = [int(j) for j in input().split()]
	d = defaultdict(int)

	for x in arr:
		d[x] += 1

	num = sum(arr)
	denom = n

	s = []
	for x in d:
		s += [[x, d[x]]]

	s.sort()
	# print(s)

	ind1, ind2 = 0, len(s)-1

	ans = 0
	while ind1 <= ind2:
		# print(ind1, ind2)
		curnum = s[ind1][0] + s[ind2][0]
		curdenom = 2
		if num*curdenom < curnum*denom:
			ind2 -= 1
		elif num*curdenom > curnum*denom:
			ind1 += 1
		else:
			if ind1 != ind2:
				ans += s[ind1][1]*s[ind2][1]
			else:
				ans += s[ind1][1]*(s[ind1][1]-1)//2
			ind1 += 1
			ind2 -= 1

	print(ans)


import json

from common.lock.py_redis_lock import Lock
from inventory_srv.model.models import Inventory, InventoryHistory
from inventory_srv.proto import inventory_pb2, inventory_pb2_grpc
from inventory_srv.settings import settings

import grpc
from google.protobuf import empty_pb2
from loguru import logger
from peewee import DoesNotExist
from rocketmq.client import ConsumeStatus, ReceivedMessage

def reback_inventory(msg: ReceivedMessage) -> ConsumeStatus:
    """
    通过 msg body 中的 order_sn 确认归还库存
    :param msg: ReceivedMessage
    :return: ConsumeStatus
    """
    msg_body = json.loads(msg.body.decode("utf-8"))
    order_sn = msg_body["orderSn"]

    with settings.DB.atomic() as txn:
        try:
            inv_history = InventoryHistory.get(
                InventoryHistory.order_sn == order_sn,
                InventoryHistory.status == 1
            )
            inv_details = json.loads(inv_history.order_inv_detail)
            for item in inv_details:
                goods_id = item["goods_id"]
                num = item["num"]
                Inventory.update(stocks=Inventory.stocks+num).where(
                    Inventory.goods == goods_id).execute()
            inv_history.status = 2
            inv_history.save()
            return ConsumeStatus.CONSUME_SUCCESS
        except DoesNotExist:
            return ConsumeStatus.CONSUME_SUCCESS
        except Exception:
            txn.rollback()
            return ConsumeStatus.RECONSUME_LATER

class InventoryServicer(inventory_pb2_grpc.InventoryServicer):

    @logger.catch
    def SetInv(
        self,
        request: inventory_pb2.GoodsInvInfo,
        context: grpc.ServicerContext
    ) -> empty_pb2.Empty:
        """
        设置商品库存
        :param request: inventory_pb2.GoodsInvInfo
        :param context: grpc.ServicerContext
        :return: empty_pb2.Empty
        """
        force_insert = False
        inv = Inventory.select().where(
            Inventory.goods == request.goodsId).first()
        if not inv:
            inv = Inventory()
            inv.goods = request.goodsId
            force_insert = True
        inv.stocks = request.num
        inv.save(force_insert=force_insert)

        return empty_pb2.Empty()

    @logger.catch
    def InvDetail(
        self,
        request: inventory_pb2.GoodsInvInfo,
        context: grpc.ServicerContext
    ) -> inventory_pb2.GoodsInvInfo:
        """
        获取某个商品库存详情
        :param request: inventory_pb2.GoodsInvInfo
        :param context: grpc.ServicerContext
        :return: inventory_pb2.GoodsInvInfo
        """
        inv = Inventory.select().where(
            Inventory.goods == request.goodsId).first()
        if not inv:
            context.set_code(grpc.StatusCode.NOT_FOUND)
            context.set_details("inventory not found")
            return inventory_pb2.GoodsInvInfo()
        return inventory_pb2.GoodsInvInfo(goodsId=inv.goods, num=inv.stocks)

    @logger.catch
    def Sell(
        self, request: inventory_pb2.SellInfo, context: grpc.ServicerContext
    ) -> empty_pb2.Empty:
        """
        扣减库存
        解决超卖问题，使用 peewee 事务
        :param request: inventory_pb2.SellInfo
        :param context: grpc.ServicerContext
        :return: empty_pb2.Empty
        """
        inv_history = InventoryHistory(order_sn=request.orderSn)
        inv_details = []
        with settings.DB.atomic() as txn:
            for item in request.goodsInfo:
                lock = Lock(settings.REDIS_CLIENT,
                            f"lock:goods:{item.goodsId}",
                            auto_renewal=True, expire=10)
                lock.acquire()
                goods_inv = Inventory.select().where(
                    Inventory.goods == item.goodsId).first()
                # 库存不存在或者库存不足
                if not goods_inv:
                    context.set_code(grpc.StatusCode.NOT_FOUND)
                    context.set_details("inventory not found")
                    txn.rollback()
                    return empty_pb2.Empty()
                if goods_inv.stocks < item.num:
                    context.set_code(grpc.StatusCode.RESOURCE_EXHAUSTED)
                    context.set_details("insufficient inventory")
                    txn.rollback()
                    return empty_pb2.Empty()
                # 扣减库存
                inv_details.append({"goods_id": item.goodsId, "num": item.num})
                goods_inv.stocks -= item.num
                goods_inv.save()
                lock.release()

            inv_history.order_inv_detail = json.dumps(inv_details)
            inv_history.save()
            return empty_pb2.Empty()

    @logger.catch
    def Reback(
        self,
        request: inventory_pb2.GoodsInvInfo,
        context: grpc.ServicerContext
    ) -> empty_pb2.Empty:
        """
        归还库存
        1. 订单超时
        2. 订单创建失败
        3. 订单取消
        :param request: inventory_pb2.GoodsInvInfo
        :param context: grpc.ServicerContext
        :return: empty_pb2.Empty
        """
        with settings.DB.atomic() as txn:
            for item in request.goodsInfo:
                lock = Lock(settings.REDIS_CLIENT,
                            f"lock:goods:{item.goodsId}",
                            auto_renewal=True, expire=10)
                lock.acquire()
                goods_inv = Inventory.select().where(
                    Inventory.goods == item.goodsId).first()
                # 库存不存在或者库存不足
                if not goods_inv:
                    context.set_code(grpc.StatusCode.NOT_FOUND)
                    context.set_details("inventory not found")
                    txn.rollback()
                    return empty_pb2.Empty()
                # 归还库存
                goods_inv.stocks += item.num
                goods_inv.save()
                lock.release()

        return empty_pb2.Empty()


## PyQT5
from PyQt5 import uic
from PyQt5.QtCore import Qt, QRunnable, pyqtSlot, pyqtSignal, QThreadPool,  \
                         QObject, QTimer, QEventLoop
from PyQt5.QtWidgets import QApplication, QMainWindow, QPushButton, \
                            QVBoxLayout, QWidget, QSlider, QFileDialog, \
                            QMessageBox, QProgressBar, QGraphicsScene, \
                            QInputDialog, QDialog
from PyQt5.QtGui import QImage, QPixmap, QPen, QPainter
from PyQt5.QtTest import QTest

## General packages
import sys
import time
import json
import os

## Custome imports
from OPTIMAQS.utils.signals import Signals
from OPTIMAQS.utils.worker import Worker
from OPTIMAQS.utils.custom_sleep_function import custom_sleep_function
from OPTIMAQS.utils.json_functions import jsonFunctions
from OPTIMAQS.utils.debug import Pyqt_debugger

### View model imports
from OPTIMAQS.view.camera.camera_gui import CameraGui
from OPTIMAQS.view.dlp.dlp_gui import DLPGui
from OPTIMAQS.view.laser.laser_gui import LaserGui
from OPTIMAQS.view.controller.controller_gui import ControllerGui
from OPTIMAQS.view.electrophysiology.electrophysiology_gui import ElectrophysiologyGui

class AutomationGui(QWidget):
    def __init__(self, info_logfile_path, timings_logfile_path):
        super(AutomationGui, self).__init__()
        self.automation_ui = uic.loadUi('OPTIMAQS/view/automation/automation.ui', self)
        self.automation_ui.show()

        self.actions()
        self.threadpool = QThreadPool()

        self.n_experiment = 1

        # Init path variables
        if jsonFunctions.find_json('OPTIMAQS/config_files/path_init.json'):
            self.path_init = self.set_main_path_from_config_file()
        else:
            self.path_init = self.set_main_path_from_user_input()

        self.path = jsonFunctions.open_json('OPTIMAQS/config_files/path_init.json')
        self.path_experiment = jsonFunctions.open_json('OPTIMAQS/config_files/last_experiment.json')
        #self.initialize_experiment()

        self.info_logfile_path = info_logfile_path
        self.timings_logfile_path = timings_logfile_path

        self.load_modules()

    def set_main_path_from_config_file(self):
        """
        Set the main path of the experiment from the path saved in the
        path_init config file.
        """
        return jsonFunctions.open_json('OPTIMAQS/config_files/path_init.json')

    def actions(self):
        """
        Define actions for buttons and items.
        """
        self.export_experiment_button.clicked.connect(self.export_experiment)
        self.load_experiment_button.clicked.connect(self.load_experiment)
        self.run_experiment_dlp_static_button.clicked.connect(self.run_experiment_dlp_static_image)
        self.run_experiment_dlp_hdmi_button.clicked.connect(self.run_experiment_dlp_hdmi)
        self.end_experiment_function = Signals()
        self.end_experiment_function.finished.connect(self.end_experiment)

    def reset(self, info_logfile_path, timings_logfile_path, path):
        ## update automation paths
        self.timings_logfile_path = timings_logfile_path
        self.info_logfile_path = info_logfile_path
        self.path = path
        ## update path of other dependancies:
        self.dlp_gui.reset(timings_logfile_path = self.timings_logfile_path,
                                       info_logfile_path = self.info_logfile_path,
                                       path_experiment = self.path)
        self.camera_gui.reset(timings_logfile_path = self.timings_logfile_path,
                                       info_logfile_path = self.info_logfile_path,
                                       path_experiment = self.path)
        self.laser_gui.reset(timings_logfile_path = self.timings_logfile_path,
                                       info_logfile_path = self.info_logfile_path)

#    def initialize_experiment(self):
#        """
#        Initialize/Reinitilize experiment logfiles and variables
#        """
#        ## reinitilize JSON files
#        self.init_log_dict()
#
#        self.images = []
#        self.image_list = []
#        self.image_reshaped = []
#        self.ephy_data = []
#        self.run_nb = 0
#
#        ## init perf_counter for timing events
#        self.perf_counter_init = time.perf_counter()
#        jsonFunctions.write_to_json(self.perf_counter_init,
#                                    'OPTIMAQS/config_files/perf_counter_init.json')
#
#        ## generate folder to save the data
#        self.path = self.path_init
#        self.path_raw_data = self.path + '\\raw_data'
#        date = time.strftime("%Y_%m_%d")
#        self.path = os.path.join(self.path, date)
#        if not os.path.exists(self.path):
#            os.makedirs(self.path) ## make a folder with the date of today if it does not already exists
#        self.path_temp = os.path.join(self.path, 'experiment_' + str(self.n_experiment))  ## in case of multiple experiments a day, need to create several subdirectory
#        while os.path.exists(self.path_temp):                             ## but necesarry to check which one aleady exists
#            self.n_experiment += 1
#            self.path_temp = os.path.join(self.path, 'experiment_' + str(self.n_experiment))
#        self.path = self.path_temp
#        self.path_raw_data = self.path + '\\raw_data'
#        os.makedirs(self.path)
#        os.makedirs(self.path + '\\dlp_images')
#        os.makedirs(self.path + '\\raw_data')
#        jsonFunctions.write_to_json(self.path, 'OPTIMAQS/config_files/last_experiment.json')
#
#        ## generate log files
#        self.info_logfile_path = self.path + "/experiment_{}_info.json".format(self.n_experiment)
#        experiment_time = time.asctime(time.localtime(time.time()))
#        self.info_logfile_dict['experiment creation date'] = experiment_time
##        with open(self.info_logfile_path,"w+") as file:       ## store basic info and comments
##            json.dump(self.info_logfile_dict, file)
#        jsonFunctions.write_to_json(self.info_logfile_dict, self.info_logfile_path)
#
#        self.timings_logfile_path = self.path + "/experiment_{}_timings.json".format(self.n_experiment)
##        with open(self.timings_logfile_path, "w+") as file:
##            json.dump(self.timings_logfile_dict, file)
#        jsonFunctions.write_to_json(self.timings_logfile_dict, self.timings_logfile_path)
#
#        # self.current_folder_label_2.setText(str(self.path)) ## show current directory in the GUI  # this doesn't work here but works in main
#
#        ## update path of other dependancies:
#        self.dlp_gui.reset(timings_logfile_path= self.timings_logfile_path,
#                                       info_logfile_path = self.info_logfile_path)
#        self.camera_gui.reset(timings_logfile_path= self.timings_logfile_path,
#                                       info_logfile_path = self.info_logfile_path,
#                                       path_experiment = self.path)
#        self.laser_gui.reset(timings_logfile_path= self.timings_logfile_path,
#                                       info_logfile_path = self.info_logfile_path)

    def load_modules(self):
        ## can there be a better way than having automation on top of main ?
        self.dlp_gui = DLPGui(info_logfile_path = self.info_logfile_path,
                              timings_logfile_path = self.timings_logfile_path)
        self.dlp_gui.setGeometry(1200, 400, 500, 100)
        self.camera_gui = CameraGui(info_logfile_path = self.info_logfile_path,
                              timings_logfile_path = self.timings_logfile_path)
        self.camera_gui.setGeometry(100, 30, 200, 900)
        self.laser_gui = LaserGui(info_logfile_path = self.info_logfile_path,
                              timings_logfile_path = self.timings_logfile_path)
        self.laser_gui.setGeometry(1200, 250, 500, 100)
#        self.controller_gui = ControllerGui
#        self.electrophysiology_gui = ElectrophysiologyGui()

#    def init_log_dict(self):
#        """
#        Initialize dictionaries that will populate the json files saving info
#        and timings
#        """
#        ## initilizing dict for timings
#        self.timings_logfile_dict = {}
#        self.timings_logfile_dict['timer_init'] = {}
#        self.timings_logfile_dict['timer_init']['main'] = []
#        self.timings_logfile_dict['timer_init']['camera'] = []
#        self.timings_logfile_dict['timer_init']['dlp'] = []
#        self.timings_logfile_dict['timer_init']['laser'] = []
#        self.timings_logfile_dict['timer_init']['ephy'] = []
#        self.timings_logfile_dict['timer_init']['ephy_stim'] = [] # do I need this one ?
#        self.timings_logfile_dict['laser'] = {}
#        self.timings_logfile_dict['laser']['on'] = []
#        self.timings_logfile_dict['laser']['off'] = []
#        self.timings_logfile_dict['dlp'] = {}
#        self.timings_logfile_dict['dlp']['on'] = []
#        self.timings_logfile_dict['dlp']['off'] = []
#        self.timings_logfile_dict['camera'] = []
#        self.timings_logfile_dict['camera_bis'] = []
#        self.timings_logfile_dict['ephy'] = {}
#        self.timings_logfile_dict['ephy']['on'] = []
#        self.timings_logfile_dict['ephy']['off'] = []
#        self.timings_logfile_dict['ephy_stim'] = {}
#        self.timings_logfile_dict['ephy_stim']['on'] = []
#        self.timings_logfile_dict['ephy_stim']['off'] = []
#        ## initilizing dict for info
#        self.info_logfile_dict = {}
#        self.info_logfile_dict['experiment creation date'] = None
#        self.info_logfile_dict['roi'] = []
#        self.info_logfile_dict['exposure time'] = []
#        self.info_logfile_dict['binning'] = []
#        self.info_logfile_dict['fov'] = []
#        self.info_logfile_dict['fps'] = []

    def dlp_static_image(self):
        ## getting values from automation gui
        mode_index = 0 ## refer to mode_index of the function selection for the dlp
        # print(str(self.dlp_mode_comboBox.currentText())) # this doesn't work here but works in main
        dlp_on = int(self.dlp_time_on_lineEdit.text())
        dlp_off = int(self.dlp_time_off_lineEdit.text())
        dlp_interval = int(self.intervalle_between_sequences_lineEdit.text())
        dlp_sequence = int(self.number_of_sequence_lineEdit.text())
        dlp_repeat_sequence = int(self.number_sequence_repetition_lineEdit.text())

        ## lauching auto protocol
        for i in range(dlp_repeat_sequence):
            for j in range(dlp_sequence):
                self.laser_gui.laser_on()
                custom_sleep_function(1000)
#                if self.electrophysiology_gui.record_electrophysiological_trace_radioButton.isChecked():
#                    self.electrophysiology_gui.ephy_stim_start()
                ## ON
                self.dlp_gui.display_mode(0) ## static image
                self.dlp_gui.choose_action(2) ## display_image
#                self.timings_logfile_dict['dlp']['on'].append((time.perf_counter() - self.perf_counter_init)*1000)
                custom_sleep_function(dlp_on)
#                if self.record_electrophysiological_trace_radioButton.isChecked():
#                    self.ephy_stim_end()
                self.dlp_gui.turn_dlp_off()
#                self.timings_logfile_dict['dlp']['off'].append((time.perf_counter() - self.perf_counter_init)*1000)
                custom_sleep_function(dlp_off)
                self.laser_gui.laser_off()
            custom_sleep_function(dlp_interval)
        self.camera_gui.camera_signal.finished.emit()
#        self.ephy_signal.finished.emit()
        self.dlp_gui.dlp_signal.finished.emit()
#        self.end_experiment_function.finished.emit()

    def dlp_static_image_repetition(self):
        experiment_repetition = int(self.experiment_repetition_lineEdit.text())
        for repeat in range(experiment_repetition):
            print(f"experiment repetition number {repeat}")
            self.run_experiment_dlp_static_image()

    def run_experiment_dlp_static_image(self):
#        self.initialize_experiment()
        dlp_worker = Worker(self.dlp_static_image)
#        ephy_worker = Worker(self.ephy_recording_thread)
        self.camera_gui.stream()
        custom_sleep_function(2000)
        self.threadpool.start(dlp_worker)  ## in those experiment the dlp function drives the camera and laser stops
        self.recording = True
        self.ephy_data = []
#        if self.record_electrophysiological_trace_radioButton.isChecked():
#            self.recording = True
#            self.threadpool.start(ephy_worker)
#
#            while not self.end_expe:
#                time.sleep(1)
#                QApplication.processEvents()

    def dlp_hdmi_video_gen(self):
        # getting value from automation gui to create video
        print('generating video for hdmi display')
        duration_of_each_image_stimulus = int(self.duration_of_each_image_stimulus_lineEdit.text())
        time_between_stimulus_image = int(self.time_between_stimulus_image_lineEdit.text())

        #TODO: other way to do that? To Simplify and reuse function in dlp_gui ?
        self.info_logfile_dict = jsonFunctions.open_json(self.info_logfile_path)
        self.roi_list = self.info_logfile_dict['roi']
        self.dlp_gui.generate_one_image_per_roi(self.roi_list)
        self.dlp_gui.movie_from_images(time_stim_image=duration_of_each_image_stimulus,
                                       inter_image_interval=time_between_stimulus_image)
        # self.dlp_gui.generate_every_paired_combination_movie(self.roi_list,
        #                                                      time_stim_image = duration_of_each_image_stimulus,
        #                                                      inter_image_interval = time_between_stimulus_image)
    def dlp_hdmi(self):
        self.laser_gui.laser_on()
        custom_sleep_function(100)
        ## ON
        self.dlp_gui.display_mode(2)
        self.dlp_gui.choose_action(1)
        custom_sleep_function(10000) #TODO: estimate length of video instead
        ## OFF
        self.dlp_gui.turn_dlp_off()
        self.laser_gui.laser_off()
        ## DELAY
        self.camera_gui.camera_signal.finished.emit()
        self.dlp_gui.dlp_signal.finished.emit()

    def dlp_hdmi_repetition(self):
        number_of_repetition = int(self.number_of_repetition_lineEdit.text())
        delay_between_repetition = int(self.delay_between_repetition_lineEdit.text())
        for i in range(number_of_repetition):
            self.run_experiment_dlp_hdmi()
            custom_sleep_function(delay_between_repetition)

    def run_experiment_dlp_hdmi(self):
        if self.run_nb == 0:
            self.dlp_hdmi_video_gen()
        dlp_hdmi_worker = Worker(self.dlp_hdmi)
        self.camera_gui.stream()
        custom_sleep_function(2000)
        self.threadpool.start(dlp_hdmi_worker)
        self.recording = True
        self.run_nb += 1

    def export_experiment(self):
        self.info_logfile_dict['roi'].append(self.roi_list)
        self.info_logfile_dict['exposure time'].append(self.exposure_time_value.value())
        self.info_logfile_dict['binning'].append(self.bin_size)
        self.info_logfile_dict['fps'].append(self.internal_frame_rate)
        self.info_logfile_dict['fov'].append((self.x_init, self.x_dim, self.y_init, self.y_dim))
        jsonFunctions.write_to_json(self.info_logfile_dict, self.info_logfile_path)
#        with open(self.info_logfile_path, "w") as file:
#            file.write(json.dumps(self.info_logfile_dict, default=lambda x:list(x), indent=4, sort_keys=True))

    def load_experiment(self):
        ## experiment json file
        self.path = QFileDialog.getExistingDirectory(None, 'Experiment folder:',
                                                        'C:/', QFileDialog.ShowDirsOnly)[0]
        experiment_path = QFileDialog.getOpenFileName(self, 'Select Experiment file',
                                                        'C:/',"Experiment file (*.json)")[0]
        # experiment_path = '/media/jeremy/Data/Data_Jeremy/2019_10_29/experiment_1/experiment_1_info.json'
        ## load/write camera related stuff
#        with open(experiment_path) as file:
#            self.info_logfile_dict = dict(json.load(file))
        self.info_logfile_dict = jsonFunctions.open_json(self.path)
        self.roi_list = self.info_logfile_dict['roi'][0]
        self.cam.write_exposure(self.info_logfile_dict['exposure time'][0])
        self.cam.write_binning(self.info_logfile_dict['binning'][0])
        self.cam.write_subarray_mode(2)
        self.x_init = self.info_logfile_dict['fov'][0][0]
        self.x_dim = self.info_logfile_dict['fov'][0][1]
        self.y_init = self.info_logfile_dict['fov'][0][2]
        self.y_dim = self.info_logfile_dict['fov'][0][3]
        self.cam.write_subarray_size(self.x_init, self.x_dim, self.y_init, self.y_dim)

    def end_experiment(self):
        print('end experiment signal received')
        self.end_expe = True

if __name__ == "__main__":
    app = QApplication(sys.argv)
    win = AutomationGui()
    win.showMaximized()
    app.exit(app.exec_())


"""Install file for RADIS.

Typical install procedure, plus:

- auto-convert README.rst to long_description, removing some sphinx-only syntax
so it can be rendered by PyPi
- read version number from __version__.txt
- some dependencies should be installed manually (typically: all packages with
compiled components such as numpy, pandas, etc.)

Examples
--------

Install (normal, use-only)::

    python setup.py install

Or (create an alias, so you can still edit)::

    python setup.py develop

Notes
-----

For developers:

when creating a new version, just update the __version__.txt file

to register it on Pypi see register.py::

    python register.py
"""
import io
import re
import sys
from os.path import abspath, dirname, exists, join

from setuptools import Extension, find_packages, setup

# Build description from README (PyPi compatible)
# -----------------------------------------------

# Utils to format RST
def yield_sphinx_only_markup(lines):
    """Cleans-up Sphinx-only constructs (ie from README.rst), so that *PyPi*
    can format it properly.

    To check for remaining errors, install ``sphinx`` and run::

        python setup.py --long-description | sed -file 'this_file.sed' | rst2html.py  --halt=warning

    :param file_inp:     a `filename` or ``sys.stdin``?
    :param file_out:     a `filename` or ``sys.stdout`?`

    References
    ----------

    https://stackoverflow.com/questions/16367770/my-rst-readme-is-not-formatted-on-pypi-python-org

    Notes
    -----

    Check output with::

        python setup.py --long-description | rst2html.py > output.html
    """
    substs = [
        ## Selected Sphinx-only Roles.
        #
        (r":abbr:`([^`]+)`", r"\1"),
        (r":ref:`([^`]+)`", r"`\1`_"),
        (r":term:`([^`]+)`", r"**\1**"),
        (r":dfn:`([^`]+)`", r"**\1**"),
        (r":(samp|guilabel|menuselection):`([^`]+)`", r"``\2``"),
        ## Sphinx-only roles:
        #        :foo:`bar`   --> foo(``bar``)
        #        :a:foo:`bar` XXX afoo(``bar``)
        #
        # (r'(:(\w+))?:(\w+):`([^`]*)`', r'\2\3(``\4``)'),
        (r":(\w+):`([^`]*)`", r"\1(``\2``)"),
        ## Sphinx-only Directives.
        #
        (r"\.\. doctest", r"code-block"),
        (r"\.\. plot::", r".. "),
        (r"\.\. seealso", r"info"),
        (r"\.\. glossary", r"rubric"),
        (r"\.\. figure::", r".. "),
        ## Other
        #
        (r"\|version\|", r"x.x.x"),
        ## added to make RADIS docs Pypi compatible
        #        (r'\.\. image::',          r'.. '),
        #        (r'\.\. |CO2| replace:: CO\ :sub:`2`',          r'.. '),
    ]

    regex_subs = [(re.compile(regex, re.IGNORECASE), sub) for (regex, sub) in substs]

    def clean_line(line):
        try:
            for (regex, sub) in regex_subs:
                line = regex.sub(sub, line)
        except Exception as ex:
            print(("ERROR: %s, (line(%s)" % (regex, sub)))
            raise ex

        return line

    for line in lines:
        yield clean_line(line)

# (note: README.rst has been converted to README.md by register.py, and cleaned afterwards )
description = "A fast line-by-line code for high-resolution infrared molecular spectra"
readme_path = join(abspath(dirname(__file__)), "README.md")
if not exists(readme_path):
    long_description = description
else:
    with io.open(readme_path, encoding="utf-8") as f:
        long_description = f.read()

# Read version number from file
with open(join(dirname(__file__), "radis", "__version__.txt")) as version_file:
    __version__ = version_file.read().strip()

# %% Cython extensions:
# Ensures RADIS still builds if without Cython

class BuildFailed(Exception):
    pass

from distutils.errors import CCompilerError, DistutilsExecError, DistutilsPlatformError

def show_message(*lines):
    """Note : will only happen if user installs with `pip install -v`"""
    print("=" * 74, file=sys.stderr)
    for line in lines:
        print(line, file=sys.stderr)
    print("=" * 74, file=sys.stderr)

def get_ext_modules(with_binaries):
    """
    Parameters
    ----------
    with_binaries: bool
        if False, do not try to build Cython extensions
    """
    # Based on code from https://github.com/pallets/markupsafe/blob/main/setup.py

    print(sys.version)
    ext_modules = []
    cmdclass = {}

    if not with_binaries:
        # skip building extensions
        return {"cmdclass": cmdclass, "ext_modules": ext_modules}

    # TO-DO: set language level

    try:
        import cython
    except (ModuleNotFoundError) as err:
        raise BuildFailed(
            "Cython not found : Skipping all Cython extensions...!"
        ) from err

    print("Cython " + cython.__version__)

    from Cython.Distutils import build_ext

    class build_ext_subclass(build_ext):
        def build_extensions(self):
            c = self.compiler.compiler_type
            copt = {
                "msvc": ["/openmp", "/Ox", "/fp:fast", "/favor:INTEL64"],
                "mingw32": ["-fopenmp", "-O3", "-ffast-math", "-march=native"],
            }

            lopt = {"mingw32": ["-fopenmp"]}

            print("Compiling with " + c + "...")
            try:
                for e in self.extensions:
                    e.extra_compile_args = copt[c]
            except (KeyError):
                pass
            try:
                for e in self.extensions:
                    e.extra_link_args = lopt[c]
            except (KeyError):
                pass
            try:
                build_ext.build_extensions(self)
            except (CCompilerError, DistutilsExecError, DistutilsPlatformError) as err:
                raise BuildFailed() from err

    from numpy import get_include

    ext_modules.append(
        Extension(
            "radis_cython_extensions",
            sources=[
                "./radis/cython/radis_cython_extensions.pyx",
            ],
            include_dirs=[get_include()],
            language="c++",
            extra_link_args=[],
            define_macros=[("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")],
        )
    )

    cmdclass["build_ext"] = build_ext_subclass

    return {"cmdclass": cmdclass, "ext_modules": ext_modules}

#%% Main install routine
def run_setup(with_binary):
    setup(
        name="radis",
        version=__version__,
        description=description,
        long_description=long_description,
        long_description_content_type="text/markdown",
        url="https://github.com/radis/radis",
        author="Erwan Pannier",
        author_email="erwan.pannier@gmail.com",
        license="GNU Lesser General Public License v3 (LGPLv3)",
        keywords=[
            "spectrum",
            "infrared",
            "spectra",
            "radiation",
            "nonequilibrium",
            "spectroscopy",
            "molecules",
            "HITRAN",
            "hitemp",
            "exomol",
            "line-by-line",
        ],
        packages=find_packages(),
        install_requires=[],
        # do not add requirements here, add them directly in the requirements.txt file
        # note EP : generate a requirement.txt by Mamba & pip freeze for Pypi deployment ? see https://stackoverflow.com/a/75239980/5622825
        extras_require={
            "dev": [
                "numpydoc",  # for Jedi (autocompletion) to recognize
                "black>=20.8b1",  # for code-linting in accordance to PEP8
                "isort",  # for sorting imports
                "pre-commit",  # to enforce Black before each commit
                "pytest",  # to run test suite
                "ipython>=7.0.0",  # useful for fast debugging
            ],
            "docs": [
                "sphinx-autodoc-annotation",
                "sphinx_autodoc_defaultargs>=0.1.2",
                "sphinx>=1.7.0",
                "astroquery>=0.3.9",
                "sphinxcontrib-apidoc",
                "sphinx-gallery",
                "lmfit",
                "pytest",  # Sphinx autodoc also parses test suite
            ],
        },
        classifiers=[
            "Development Status :: 4 - Beta",
            "Intended Audience :: Science/Research",
            "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",
            "Topic :: Scientific/Engineering",
            "Programming Language :: Python",
            "Programming Language :: Python :: 3.7",
            "Programming Language :: Python :: 3.8",
            "Programming Language :: Python :: 3.9",
            "Operating System :: OS Independent",
        ],
        **get_ext_modules(with_binary),
        include_package_data=True,  # add non .py data files in MANIFEST.in
        # package_data={'radis': ['radis/phys/units.txt']},
        zip_safe=False,  # impossible as long as we have external files read with __file__ syntax
        platforms="any",
    )

# %% Run Main install routine
try:
    run_setup(with_binary=True)
except BuildFailed:
    import traceback

    traceback.print_exc()
    show_message(
        "WARNING: RADIS C-extension could not be compiled, speedups",
        " are not enabled.",
        "Failure information, if any, is above.",
        "Retrying the build without the C extension now.",
    )
    run_setup(with_binary=False)
    show_message(
        "WARNING: RADIS C-extension could not be compiled, speedups"
        " are not enabled.",
        "Plain-Python build succeeded.",
    )


from django.views.generic import DetailView, ListView, TemplateView

from .models import Event

class AboutView(TemplateView):
    template_name = "main/index.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context["event_types"] = Event.TypeChoice.choices
        return context

class EventsView(ListView):
    template_name = "main/events.html"

    def get_queryset(self):
        return Event.objects.filter(type=self.kwargs["event_type"]).order_by("-date")

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context["event_types"] = Event.TypeChoice.choices
        return context

class EventDetailView(DetailView):
    model = Event
    template_name = "main/event_detail.html"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context["event_types"] = Event.TypeChoice.choices
        return context


from pymongo import MongoClient
import datetime

mongo=MongoClient(host='localhost',port=27017)
a_document={'name':'jane',
                        'age':34,
                        'interests':['python','database','statistics'],
                        'date_added':datetime.datetime.now()}
# 选择一个尚未建立的db
db=mongo.my_data
# 选择一个尚未建立的collection
collection =db.docs
#寻找资料，即使数据库或资料不存在也不会产生例外异常
collection.find_one()
#列出目前db内所有的collection
print(db.list_collection_names())
collection.insert_one(a_document)
print(db.list_collection_names())
#不设条件寻找一笔资料
print(collection.find_one())
#寻找一笔资料
collection.find_one({'name':'jane'})

#MongoDB是用字典(键加上值)来进行搜寻以找出文件
#update
collection.update_one({'name':'jane'},{"$set":{'age':29}})
print(collection.find_one())
#replace 替换
collection.replace_one({'name':'jane'},{'username':'flag','date_added':datetime.datetime.now()})
print(collection.find_one())
#delete 删除
collection.delete_one({'username':'flag'})
print(collection.find_one())
print(db.list_collection_names())
collection.drop()
print(db.list_collection_names())


import aeidon
import os

class TestPreviewAgent(aeidon.TestCase):

    def setup_method(self, method):
        self.project = self.new_project()
        self.delegate = self.project.preview.__self__

    def test_find_video__avi(self):
        video_path = aeidon.temp.create(".avi")
        sub_path = video_path.replace(".avi", ".srt")
        file = aeidon.files.new(aeidon.formats.SUBRIP,
                                sub_path,
                                "ascii")

        self.project.save_main(file)
        self.project.find_video()
        assert self.project.video_path == video_path
        os.remove(sub_path)

    def test_find_video__none(self):
        self.project.find_video()
        assert self.project.video_path is None

    def test_preview__encoding(self):
        doc = aeidon.documents.MAIN
        self.project.video_path = self.new_subrip_file()
        self.project.preview("00:00:00.000", doc, "echo", 0, "utf_8")
        assert os.path.isfile(self.project.get_file(doc).path)

    def test_preview__main(self):
        doc = aeidon.documents.MAIN
        self.project.video_path = self.new_subrip_file()
        self.project.preview("00:00:00.000", doc, "echo", 0)
        assert os.path.isfile(self.project.get_file(doc).path)

    def test_preview__translation(self):
        doc = aeidon.documents.TRAN
        self.project.video_path = self.new_subrip_file()
        self.project.preview("00:00:00.000", doc, "echo", 0)
        assert os.path.isfile(self.project.get_file(doc).path)


import os
from ppadb.client import Client
import sys
import argparse
import zipfile
from tqdm import tqdm
import win32file
import requests
from bs4 import BeautifulSoup
import urllib.request
__author__ = "Paver(Zhen_Bo)"
local_version = "1.3"

def app_path():
    if hasattr(sys, 'frozen'):
        return os.path.dirname(sys.executable)  # 使用pyinstaller打包后的exe目录
    return os.path.dirname(__file__)  # 没打包前的py目录

def setup():
    adb_path = "{}/adb/adb.exe".format(app_path())
    os.system("{0} start-server".format(adb_path))
    client = Client(host="127.0.0.1", port=5037)
    devices = client.devices()
    dev = []
    print("請輸入有幾個隊員(1/2)?")
    main_count = input()
    for i in range(int(main_count)):
        print("請選擇隊員 {}".format(i+1))
        get_dev, devices = select_devices(devices)
        dev.append(get_dev)
    print("請選擇隊長")
    get_dev, devices = select_devices(devices)
    dev.append(get_dev)
    return dev

def select_devices(devices, error=0):
    for i in range(len(devices)):
        print("\033[1;32m{}: {}\033[0m".format(i+1, devices[i].serial))
    if error == 1:
        print("\033[1;31m{}\033[0m".format("!!!輸入設備編號過大!!!"))
    elif error == 2:
        print("\033[1;31m{}\033[0m".format("!!!編號輸入錯誤,請在試一次!!!"))
    inputIndex = input("請輸入編號 [1 ~ {0}]:".format(len(devices)))
    try:
        value = int(inputIndex)
        if value < 1:
            exit()
        elif value > len(devices):
            return select_devices(devices, 1)
        else:
            device = devices[value-1]
            del devices[value-1]
            return (device, devices)
    except (KeyboardInterrupt, SystemExit):
        raise Exception("KeyboardInterrupt")
    except:
        return select_devices(devices, 2)

def get_info():
    battle_info = list()
    boss = {"0": "大蛇", "1": "亞多明尼斯", "2": "白虎", "3": "Sec-5200L", "4": "詛咒艾基爾之長",
            "5": "寄居蟹船長", "6": "怒潮海怪", "7": "遺跡魔像", "8": "不死族之王雷希塔洛", "9": "風之隱者"}
    boss_code = {"0": "snake", "1": "robot", "2": "tiger", "3": "sec-5200L", "4": "curse",
                 "5": "crab", "6": "squid", "7": "golem", "8": "undead", "9": "owl"}
    level = {"0": "超級", "1": "高級+", "2": "高級", "3": "中級", "4": "初級"}
    level_code = {"0": "level_super", "1": "level_high+",
                  "2": "level_high", "3": "level_middle", "4": "level_begin"}
    select_boss = None
    while True:
        os.system('cls')
        print("請問要選擇的共鬥王為")
        for index in boss:
            print("{}: {}".format(index, boss[index]))
        print("====================================")
        number = int(input("請輸入數字代號(0~7): "))
        if number >= 0 and number <= 7:
            select_boss = boss[str(number)]
            battle_info.append(boss_code[str(number)])
            break
    while True:
        os.system('cls')
        print("已選定BOSS: {}".format(select_boss))
        print("====================================")
        print("請問要選擇的難度為")
        for index in level:
            print("{}: {}".format(index, level[index]))
        print("====================================")
        number = int(input("請輸入數字代號(0~4): "))
        if number >= 0 and number <= 4:
            battle_info.append(level_code[str(number)])
            break
    return [battle_info[0], battle_info[1]]

def check_update(version):
    print("檢查更新中...")
    # remote_version = requests.get(
    #     "https://zhen-bo.github.io/Auto-worldflipper/").text
    remote_version = urllib.request.urlopen(
        "https://zhen-bo.github.io/Auto-worldflipper/")
    remote_version = remote_version.read().decode("utf8")
    soup = BeautifulSoup(remote_version, features="html.parser")
    remote_version = soup.find_all("p", {"class": "version"})
    remote_version = remote_version[0].text.split("\n")[0]
    remote_link = "https://github.com/Zhen-bo/Auto-worldflipper/releases/download/{}/Auto-worldflipper.zip".format(
        remote_version)
    if remote_version == version:
        print("當前為最新版: {}".format(local_version))
    elif remote_version != version:
        print("即將從 {} 版 升級成 {} 版".format(version, remote_version))
        r = requests.get(remote_link, stream=True, allow_redirects=True)
        print("開始下載更新...")
        total_size = int(r.headers.get('content-length', 0))
        progress_bar = tqdm(total=total_size, unit='ib', unit_scale=True)
        with open(os.path.join(app_path(), "update.zip"), 'wb') as file:
            for data in r.iter_content(1024):
                progress_bar.update(len(data))
                file.write(data)
        progress_bar.close()
        print("下載完成")
        print("正在更新中...")
        temp_dir = os.path.join(app_path(), "_update_tmp/")
        with zipfile.ZipFile(os.path.join(app_path(), "update.zip"), 'r') as zip_ref:
            if not os.path.exists(temp_dir):
                os.mkdir(temp_dir)
            for file in tqdm(iterable=zip_ref.namelist(), total=len(zip_ref.namelist())):
                try:
                    zip_ref.extract(member=file, path=app_path())
                except:
                    try:
                        win32file.MoveFileEx(os.path.join(app_path(), file), os.path.join(
                            temp_dir, file), win32file.MOVEFILE_REPLACE_EXISTING)
                    except Exception as e:
                        if e.args[0] == 3:
                            dir_path = temp_dir
                            for folder in file.split('/')[:-1]:
                                dir_path = os.path.join(dir_path, folder)
                            file_name = file.split('/')[-1]
                            os.makedirs(dir_path)
                        win32file.MoveFileEx(os.path.join(app_path(), file), os.path.join(
                            dir_path, file_name), win32file.MOVEFILE_REPLACE_EXISTING)
                    zip_ref.extract(member=file, path=app_path())
        #os.system("start {}".format(os.path.join(app_path(), "Auto-worldflipper.exe")))
        os.system("start {}".format(
            "choice /C Y /N /D Y /T 1 & Auto-worldflipper.exe"))
        sys.exit()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    # 如果只有一個隊員的話是否招募並等待
    parser.add_argument("--wait_people", type=str, default="False")
    # quit=退出模式 close=重啟模式
    parser.add_argument("--mode", type=str, default="quit",)
    args = parser.parse_args()
    mode = args.mode
    os.system('cls')
    if os.path.exists(os.path.join(app_path(), "_update_tmp/")):
        os.system('choice /C Y /N /D Y /T 1 & rmdir /s {}'.format(
            os.path.join(app_path(), "_update_tmp/")))
    check_update(local_version)
    from core.bot import auto
    from core.worker import worker
    dev = setup()
    boss, level = get_info()
    try:
        if len(dev) == 2:
            main = worker(dev[0], "隊員")
            ap = worker(dev[1], "隊長")
            bot = auto(boss=boss, level=level,
                       wait_people=args.wait_people, main=main, ap=ap)
            bot.start(mode)
        else:
            main_1 = worker(dev[0], "隊員1")
            main_2 = worker(dev[1], "隊員2")
            ap = worker(dev[2], "隊長")
            bot = auto(boss=boss, level=level, wait_people=args.wait_people,
                       main=main_1, main2=main_2, ap=ap)
            bot.start(mode)
    except KeyboardInterrupt:
        print("STOP")
        sys.exit()
    except:
        os.system("PAUSE")


import torch
import numpy as np
import matplotlib.pyplot as plt
import os
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))))
from tqdm import tqdm
from copy import deepcopy
from collections import defaultdict as ddict
import matplotlib as mpl
from torch.utils.data import TensorDataset, DataLoader,Dataset, random_split
import json
from tqdm import tqdm
import torchvision
import torchvision.transforms as transforms
import torch.nn as nn
import torch.nn.functional as F
import math
import time
from torchvision import models
from XAI.Integrated_gradient.implement.ig_image_pt import *
from XAI.guided_integrated_gradients.Implement.gig import *
from vision.utils.data import *
from vision.utils.model import *
from vision.utils.vision import *

from matplotlib import pylab as P
import numpy as np
import PIL.Image

import torch
from torchvision import models, transforms

# From our repository.
import saliency.core as saliency

# Boilerplate methods.
def ShowImage(im, title='', ax=None):
    if ax is None:
        P.figure()
    P.axis('off')
    P.imshow(im)
    P.title(title)

def ShowGrayscaleImage(im, title='', ax=None):
    if ax is None:
        P.figure()
    P.axis('off')
    P.imshow(im, cmap=P.cm.gray, vmin=0, vmax=1)
    P.title(title)

def ShowHeatMap(im, title, ax=None):
    if ax is None:
        P.figure()
    P.axis('off')
    P.imshow(im, cmap='inferno')
    P.title(title)

def LoadImage(file_path):
    im = PIL.Image.open(file_path)
    im = im.resize((32, 32))
    im = np.asarray(im)
    return im

transformer = transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
def PreprocessImages(images):
    # assumes input is 4-D, with range [0,255]
    #
    # torchvision have color channel as first dimension
    # with normalization relative to mean/std of ImageNet:
    #    https://pytorch.org/vision/stable/models.html
    images = np.array(images)
    images = images/255
    images = np.transpose(images, (0,3,1,2))
    images = torch.tensor(images, dtype=torch.float32)
    images = transformer.forward(images)
    return images.requires_grad_(True)

im_orig = LoadImage('./doberman.png')
im_tensor = PreprocessImages([im_orig])
# Show the image
ShowImage(im_orig)

model=ResNet18(3)
path=os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__)))))
model.load_state_dict(torch.load(path+'/Distillation/assets/model/resnet18-baseline-1.pt'))
model.eval()
class_idx_str = 'class_idx_str'
predictions = model(im_tensor)
predictions = predictions.detach().numpy()
prediction_class = np.argmax(predictions[0])
call_model_args = {class_idx_str: prediction_class}

print("Prediction class: " + str(prediction_class))  # Should be a doberman, class idx = 236
im = im_orig.astype(np.float32)

## Gig test

test_set=experiment_cifar10(path+'/Distillation/assets/data/cifar10/experiment',train=False)

test_loader=DataLoader(test_set,batch_size=128,shuffle=False)

for j in test_loader:
    test_img=j[0][:,:3]
    test_ig=j[0][:,3:]
    test_label=j[1]
    break

test_img.shape

baseline=torch.zeros(1,3,32,32).to('cuda:1')

model.to('cuda:1')

mine=guided_ig(im_tensor.to('cuda:1'), baseline, torch.LongTensor([5]).to('cuda:1') ,model, steps=128, fraction=0.1, max_dist=0.02)

plt.imshow(mine[0].to('cpu').detach().numpy().transpose(1,2,0))

def call_model_function(images, call_model_args=None, expected_keys=None):
    images = PreprocessImages(images)
    target_class_idx =  call_model_args[class_idx_str]
    output = model(images)
    m = torch.nn.Softmax(dim=1)
    output = m(output)

    outputs = output[:,target_class_idx]
    grads = torch.autograd.grad(outputs, images, grad_outputs=torch.ones_like(outputs))
    grads = torch.movedim(grads[0], 1, 3)
    gradients = grads.detach().numpy()

    return {saliency.base.INPUT_OUTPUT_GRADIENTS: gradients}

# Construct the saliency object. This doesn't yet compute the saliency mask, it just sets up the necessary ops.
integrated_gradients2 = saliency.IntegratedGradients()
guided_ig2 = GuidedIGnp()

# Baseline is a black image for vanilla integrated gradients.
baseline = np.zeros(im.shape)

# Compute the vanilla mask and the Guided IG mask.
vanilla_integrated_gradients_mask_3d = integrated_gradients2.GetMask(
  im, call_model_function, call_model_args, x_steps=25, x_baseline=baseline, batch_size=20)

baseline2=torch.zeros(1,3,32,32)

black=torch.zeros(3,32,32)
black[0,:,:]=(black[0,:,:]-0.4914)/0.247
black[1,:,:]=(black[1,:,:]-0.4822)/0.243
black[2,:,:]=(black[2,:,:]-0.4465)/0.261
baseline2=black.unsqueeze(0)

mine,b2,c2=guided_ig_torch(im_tensor, baseline2, torch.LongTensor([5]), model, steps=25, fraction=0.1, max_dist=0.02,device='cpu')

guided_ig_mask_3d,b,c = guided_ig2.GetMask(
  im, call_model_function, call_model_args, x_steps=25, x_baseline=baseline, max_dist=0.02, fraction=0.1)

guided_ig_mask_3d,b,c

len(b2)
len(c2)

b2[0]
b[0]

len(b)
len(c)

# Call the visualization methods to convert the 3D tensors to 2D grayscale.

mine.shape
guided_ig_mask_3d.shape
plt.imshow(mine[0].detach().numpy().transpose(1,2,0))
plt.imshow(guided_ig_mask_3d)
torch.movedim(mine[0],0,2).detach().numpy()
mine[0].detach().numpy().transpose(1,2,0)
guided_ig_mask_3d

mine[0].detach().numpy().transpose(1,2,0)

gimg=mine[0].detach().numpy().transpose(1,2,0)
guided_ig_mask_3d

mine[0].detach().numpy().transpose(1,2,0)

vanilla_mask_grayscale = saliency.VisualizeImageGrayscale(vanilla_integrated_gradients_mask_3d)
guided_ig_mask_grayscale = saliency.VisualizeImageGrayscale(guided_ig_mask_3d)
gimg2=saliency.VisualizeImageGrayscale(mine[0].detach().numpy().transpose(1,2,0))
ROWS = 1
COLS = 3
UPSCALE_FACTOR = 20
P.figure(figsize=(ROWS * UPSCALE_FACTOR, COLS * UPSCALE_FACTOR))

# Render the saliency masks.
ShowImage(im_orig, title='Original Image', ax=P.subplot(ROWS, COLS, 1))
ShowGrayscaleImage(vanilla_mask_grayscale, title='Vanilla Integrated Gradients', ax=P.subplot(ROWS, COLS, 2))
ShowGrayscaleImage(guided_ig_mask_grayscale, title='Guided Integrated Gradients', ax=P.subplot(ROWS, COLS, 3))

ShowGrayscaleImage(guided_ig_mask_grayscale, title='Guided Integrated Gradients')
ShowGrayscaleImage(gimg2, title='Guided Integrated Gradients')

def l1_distancenp(x1, x2):
  """Returns L1 distance between two points."""
  return np.abs(x1 - x2).sum()

def translate_x_to_alphanp(x, x_input, x_baseline):
  """Translates a point on straight-line path to its corresponding alpha value.
  Args:
    x: the point on the straight-line path.
    x_input: the end point of the straight-line path.
    x_baseline: the start point of the straight-line path.
  Returns:
    The alpha value in range [0, 1] that shows the relative location of the
    point between x_baseline and x_input.
  """
  with np.errstate(divide='ignore', invalid='ignore'):
    return np.where(x_input - x_baseline != 0,
                    (x - x_baseline) / (x_input - x_baseline), np.nan)

def translate_alpha_to_xnp(alpha, x_input, x_baseline):
  """Translates alpha to the point coordinates within straight-line interval.
   Args:
    alpha: the relative location of the point between x_baseline and x_input.
    x_input: the end point of the straight-line path.
    x_baseline: the start point of the straight-line path.
  Returns:
    The coordinates of the point within [x_baseline, x_input] interval
    that correspond to the given value of alpha.
  """
  assert 0 <= alpha <= 1.0
  return x_baseline + (x_input - x_baseline) * alpha

def guided_ig_implnp(x_input, x_baseline, grad_func, steps=200, fraction=0.25,
    max_dist=0.02):
  """Calculates and returns Guided IG attribution.
  Args:
    x_input: model input that should be explained.
    x_baseline: chosen baseline for the input explanation.
    grad_func: gradient function that accepts a model input and returns
      the corresponding output gradients. In case of many class model, it is
      responsibility of the implementer of the function to return gradients
      for the specific class of interest.
    steps: the number of Riemann sum steps for path integral approximation.
    fraction: the fraction of features [0, 1] that should be selected and
      changed at every approximation step. E.g., value `0.25` means that 25% of
      the input features with smallest gradients are selected and changed at
      every step.
    max_dist: the relative maximum L1 distance [0, 1] that any feature can
      deviate from the straight line path. Value `0` allows no deviation and,
      therefore, corresponds to the Integrated Gradients method that is
      calculated on the straight-line path. Value `1` corresponds to the
      unbounded Guided IG method, where the path can go through any point within
      the baseline-input hyper-rectangular.
  """

  x_input = np.asarray(x_input, dtype=np.float64)
  x_baseline = np.asarray(x_baseline, dtype=np.float64)
  x = x_baseline.copy()
  l1_total = l1_distancenp(x_input, x_baseline)
  attr = np.zeros_like(x_input, dtype=np.float64)
  mid_out=[]
  mid_grad=[]

  # If the input is equal to the baseline then the attribution is zero.
  total_diff = x_input - x_baseline
  if np.abs(total_diff).sum() == 0:
    return attr

  # Iterate through every step.
  for step in range(steps):
    # Calculate gradients and make a copy.
    grad_actual = grad_func(x)
    grad = grad_actual.copy()
    # Calculate current step alpha and the ranges of allowed values for this
    # step.
    alpha = (step + 1.0) / steps
    alpha_min = max(alpha - max_dist, 0.0)
    alpha_max = min(alpha + max_dist, 1.0)
    x_min = translate_alpha_to_xnp(alpha_min, x_input, x_baseline)
    x_max = translate_alpha_to_xnp(alpha_max, x_input, x_baseline)
    # The goal of every step is to reduce L1 distance to the input.
    # `l1_target` is the desired L1 distance after completion of this step.
    l1_target = l1_total * (1 - (step + 1) / steps)

    # Iterate until the desired L1 distance has been reached.
    gamma = np.inf
    while gamma > 1.0:
      x_old = x.copy()
      x_alpha = translate_x_to_alphanp(x, x_input, x_baseline)
      x_alpha[np.isnan(x_alpha)] = alpha_max
      # All features that fell behind the [alpha_min, alpha_max] interval in
      # terms of alpha, should be assigned the x_min values.
      x[x_alpha < alpha_min] = x_min[x_alpha < alpha_min]

      # Calculate current L1 distance from the input.
      l1_current = l1_distancenp(x, x_input)
      # If the current L1 distance is close enough to the desired one then
      # update the attribution and proceed to the next step.
      if math.isclose(l1_target, l1_current, rel_tol=EPSILON, abs_tol=EPSILON):
        attr += (x - x_old) * grad_actual
        break

      # Features that reached `x_max` should not be included in the selection.
      # Assign very high gradients to them so they are excluded.
      grad[x == x_max] = np.inf

      # Select features with the lowest absolute gradient.
      threshold = np.quantile(np.abs(grad), fraction, interpolation='lower')
      s = np.logical_and(np.abs(grad) <= threshold, grad != np.inf)

      # Find by how much the L1 distance can be reduced by changing only the
      # selected features.
      l1_s = (np.abs(x - x_max) * s).sum()

      # Calculate ratio `gamma` that show how much the selected features should
      # be changed toward `x_max` to close the gap between current L1 and target
      # L1.
      if l1_s > 0:
        gamma = (l1_current - l1_target) / l1_s
      else:
        gamma = np.inf

      if gamma > 1.0:
        # Gamma higher than 1.0 means that changing selected features is not
        # enough to close the gap. Therefore change them as much as possible to
        # stay in the valid range.
        x[s] = x_max[s]
      else:
        assert gamma > 0, gamma
        x[s] = translate_alpha_to_xnp(gamma, x_max, x)[s]
      # Update attribution to reflect changes in `x`.
      mid_out.append(x)
      mid_grad.append(attr)
      attr += (x - x_old) * grad_actual
  return attr, mid_out, mid_grad

class GuidedIGnp(saliency.GuidedIG):
  """Implements ML framework independent version of Guided IG."""

  def GetMask(self, x_value, call_model_function, call_model_args=None,
      x_baseline=None, x_steps=200, fraction=0.25, max_dist=0.02):

    """Computes and returns the Guided IG attribution.
    Args:
      x_value: an input (ndarray) for which the attribution should be computed.
      call_model_function: A function that interfaces with a model to return
        specific data in a dictionary when given an input and other arguments.
        Expected function signature:
        - call_model_function(x_value_batch,
                              call_model_args=None,
                              expected_keys=None):
          x_value_batch - Input for the model, given as a batch (i.e. dimension
            0 is the batch dimension, dimensions 1 through n represent a single
            input).
          call_model_args - user defined arguments. The value of this argument
            is the value of `call_model_args` argument of the nesting method.
          expected_keys - List of keys that are expected in the output. For this
            method (Guided IG), the expected keys are
            INPUT_OUTPUT_GRADIENTS - Gradients of the output being
              explained (the logit/softmax value) with respect to the input.
              Shape should be the same shape as x_value_batch.
      call_model_args: The arguments that will be passed to the call model
        function, for every call of the model.
      x_baseline: Baseline value used in integration. Defaults to 0.
      x_steps: Number of integrated steps between baseline and x.
      fraction: the fraction of features [0, 1] that should be selected and
        changed at every approximation step. E.g., value `0.25` means that 25%
        of the input features with smallest gradients are selected and changed
        at every step.
      max_dist: the relative maximum L1 distance [0, 1] that any feature can
        deviate from the straight line path. Value `0` allows no deviation and;
        therefore, corresponds to the Integrated Gradients method that is
        calculated on the straight-line path. Value `1` corresponds to the
        unbounded Guided IG method, where the path can go through any point
        within the baseline-input hyper-rectangular.
    """

    x_value = np.asarray(x_value)
    if x_baseline is None:
      x_baseline = np.zeros_like(x_value)
    else:
      x_baseline = np.asarray(x_baseline)

    assert x_baseline.shape == x_value.shape

    return guided_ig_implnp(
        x_input=x_value,
        x_baseline=x_baseline,
        grad_func=self._get_grad_func(call_model_function, call_model_args),
        steps=x_steps,
        fraction=fraction,
        max_dist=max_dist)

def guided_ig_torch(x_input, x_baseline, label ,model, steps=128, fraction=0.25, max_dist=0.02,device='cuda:1'):

    x = x_baseline.clone().detach()
    l1_total = l1_distance(x_input, x_baseline)
    attr = torch.zeros_like(x_input,dtype=torch.float32)

    total_diff = x_input - x_baseline

    mid_out=[]
    mid_grad=[]

    if torch.abs(total_diff).sum() == 0:
        return attr

    for step in range(steps):

        x = torch.tensor(x, requires_grad=True, dtype=torch.float32, device=device)
        # Calculate gradients and make a copy.
        grad_actual = guided_compute_gradients(x, model, label)
        grad = grad_actual.clone().detach()

        # Calculate current step alpha and the ranges of allowed values for this step.

        alpha = (step + 1.0) / steps
        alpha_min = max(alpha - max_dist, 0.0)
        alpha_max = min(alpha + max_dist, 1.0)

        x_min = translate_alpha_to_x(alpha_min, x_input, x_baseline)
        x_max = translate_alpha_to_x(alpha_max, x_input, x_baseline)

        # The goal of every step is to reduce L1 distance to the input.
        l1_target = l1_total * (1 - (step+1)/steps)

        gamma = torch.tensor(float('inf'))

        while gamma > 1.0:
            x_old = x.clone().detach()
            x_alpha = translate_x_to_alpha(x, x_input, x_baseline)
            x_alpha = torch.where(torch.isnan(x_alpha),alpha_max,x_alpha)

            # All features that fell behind the [alpha_min, alpha_max] interval in
            # terms of alpha, should be assigned the x_min values.

            x = torch.where(x_alpha<alpha_min, x_min, x)

            # Calculate current L1 distance from the input.
            l1_current = l1_distance(x, x_input)

            # If the current L1 distance is close enough to the desired one then
            # update the attribution and proceed to the next step.

            if torch.isclose(l1_target, l1_current, rtol=EPSILON, atol=EPSILON):
                attr += (x - x_old) * grad_actual
                break

            # Features that reached `x_max` should not be included in the selection.
            # Assign very high gradients to them so they are excluded.
            grad[x == x_max] = torch.tensor(float('inf'))

            threshold = torch.quantile(torch.abs(grad), fraction, interpolation='lower')
            s = torch.logical_and(torch.abs(grad) <= threshold, ~torch.isinf(grad))

            # Find by how much the L1 distance can be reduced by changing only the
            # selected features.
            l1_s = (torch.abs(x - x_max) * s).sum()

            if l1_s > 0:
                gamma = (l1_current - l1_target) / l1_s
            else:
                gamma = torch.tensor(float('inf'))

            if gamma > 1.0:
                # Gamma higher than 1.0 means that changing selected features is not
                # enough to close the gap. Therefore change them as much as possible to
                # stay in the valid range.
                x[s] = x_max[s]
            else:
                assert gamma > 0, gamma
                x[s] = translate_alpha_to_x(gamma, x_max, x)[s]
            # Update attribution to reflect changes in `x`.
            mid_out.append(x)
            mid_grad.append(attr)
            attr += (x - x_old) * grad_actual
    return attr, mid_out, mid_grad


import requests
from bs4 import BeautifulSoup

def Check(auth, mirrors):
    response = requests.get("https://mesa.freedesktop.org/archive/")
    html = response.content.decode("utf-8")
    parsedHtml = BeautifulSoup(html, "html.parser")

    links = parsedHtml.find_all("a")

    maxVersionMajor = -1
    maxVersionMinor = -1
    maxVersionPatch = -1
    maxVersionRc = -1
    for link in links:
        if ("mesa-" in link.text and ".tar.xz" in link.text
         and ".sig" not in link.text):
            if "rc" in link.text:
                version = link.text.split("-")[1].split(".")
                versionMajor = int(version[0])
                versionMinor = int(version[1])
                try:
                    versionPatch = int(version[2])
                except IndexError:
                    versionPatch = int(link.text.split("-")[2])
                versionRc = int(link.text.split("rc")[1][0])
            else:
                version = link.text.split("-")[1].split(".tar.xz")[0].split(
                 ".")
                versionMajor = int(version[0])
                versionMinor = int(version[1])
                versionPatch = int(version[2])
                versionRc = 0

            if versionMajor > maxVersionMajor:
                maxVersionMajor = versionMajor
                maxVersionMinor = versionMinor
                maxVersionPatch = versionPatch
                maxVersionRc = versionRc
                versionText = link.text.split("-")[1].split(".tar.xz")[0]
            elif (versionMajor == maxVersionMajor
             and versionMinor > maxVersionMinor):
                maxVersionMinor = versionMinor
                maxVersionPatch = versionPatch
                maxVersionRc = versionRc
                versionText = link.text.split("-")[1].split(".tar.xz")[0]
            elif (versionMajor == maxVersionMajor
             and versionMinor == maxVersionMinor
             and versionPatch > maxVersionPatch):
                maxVersionPatch = versionPatch
                maxVersionRc = versionRc
                versionText = link.text.split("-")[1].split(".tar.xz")[0]
            elif (versionMajor == maxVersionMajor
             and versionMinor == maxVersionMinor
             and versionPatch == maxVersionPatch
             and (
              versionRc != 0 and maxVersionRc != 0 and versionRc > maxVersionRc
               or versionRc == 0 and maxVersionRc != 0
             )
            ):
                maxVersionRc = versionRc
                versionText = link.text.split("-")[1].split(".tar.xz")[0]

    return versionText