from contextlib import contextmanager
import logging
import typing as t

import numpy as np

from .. import annotation, show

try:
    import matplotlib.pyplot as plt
    from mpl_toolkits.axes_grid1 import make_axes_locatable
except ImportError:
    plt = None
    make_axes_locatable = None

LOG = logging.getLogger(__name__)
IMAGENOTGIVEN = object()

def itemsetter(list_, index, value):
    list_[index] = value
    return list_

class Base:
    all_indices = []
    common_ax = None
    processed_image_intensity_spread = 2.0

    _image = None
    _processed_image = None
    _image_meta = None
    _ground_truth: t.Optional[t.List[annotation.Base]] = None

    def __init__(self, head_name):
        self.head_name = head_name
        self._ax = None

        LOG.debug('%s: indices = %s', head_name, self.indices())

    @classmethod
    def image(cls, image=IMAGENOTGIVEN, meta=None):
        if image is IMAGENOTGIVEN:  # getter
            if callable(Base._image):  # check whether stored value is lazy
                Base._image = Base._image()  # pylint: disable=not-callable
            return Base._image

        if image is None:
            Base._image = None
            Base._image_meta = None
            return cls

        Base._image = lambda: np.asarray(image)
        Base._image_meta = meta
        return cls

    @classmethod
    def processed_image(cls, image=IMAGENOTGIVEN):
        if image is IMAGENOTGIVEN:  # getter
            if callable(Base._processed_image):  # check whether stored value is lazy
                Base._processed_image = Base._processed_image()  # pylint: disable=not-callable
            return Base._processed_image

        if image is None:
            Base._processed_image = None
            return cls

        def process_image(image):
            image = np.moveaxis(np.asarray(image), 0, -1)
            image = np.clip(image / cls.processed_image_intensity_spread * 0.5 + 0.5, 0.0, 1.0)
            return image

        Base._processed_image = lambda: process_image(image)
        return cls

    @staticmethod
    def ground_truth(ground_truth):
        Base._ground_truth = ground_truth

    @staticmethod
    def reset():
        Base._image = None
        Base._image_meta = None
        Base._processed_image = None
        Base._ground_truth = None

    @classmethod
    def normalized_index(cls, data):
        if isinstance(data, str):
            data = data.split(':')

        # unpack comma separation
        for di, d in enumerate(data):
            if ',' not in d:
                continue
            multiple = [cls.normalized_index(itemsetter(data, di, unpacked))
                        for unpacked in d.split(',')]
            # flatten before return
            return [item for items in multiple for item in items]

        if len(data) >= 2 and len(data[1]) == 0:
            data[1] = -1

        if len(data) == 3:
            return [(data[0], int(data[1]), data[2])]
        if len(data) == 2:
            return [(data[0], int(data[1]), 'all')]
        return [(data[0], -1, 'all')]

    @classmethod
    def set_all_indices(cls, all_indices):
        cls.all_indices = [d for dd in all_indices for d in cls.normalized_index(dd)]

    def indices(self, type_=None, with_all=True):
        head_names = self.head_name
        if not isinstance(head_names, (tuple, list)):
            head_names = (head_names,)
        return [
            f for hn, f, r_type in self.all_indices
            if hn in head_names and (
                type_ is None
                or (with_all and r_type == 'all')
                or type_ == r_type
            )
        ]

    @staticmethod
    def colorbar(ax, colored_element, size='3%', pad=0.01):
        assert make_axes_locatable is not None
        assert plt is not None

        divider = make_axes_locatable(ax)
        cax = divider.append_axes('right', size=size, pad=pad)
        cb = plt.colorbar(colored_element, cax=cax)
        cb.outline.set_linewidth(0.1)

    @contextmanager
    def image_canvas(self, image, *args, **kwargs):
        ax = self._ax or self.common_ax
        if ax is not None:
            ax.set_axis_off()
            ax.imshow(np.asarray(image))
            yield ax
            return

        with show.image_canvas(image, *args, **kwargs) as ax:
            yield ax

    @contextmanager
    def canvas(self, *args, **kwargs):
        ax = self._ax or self.common_ax
        if ax is not None:
            yield ax
            return

        with show.canvas(*args, **kwargs) as ax:
            yield ax

    @staticmethod
    def scale_scalar(field, stride):
        field = np.repeat(field, stride, 0)
        field = np.repeat(field, stride, 1)

        # center (the result is technically still off by half a pixel)
        half_stride = stride // 2
        return field[half_stride:-half_stride + 1, half_stride:-half_stride + 1]


import numpy as np
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import LSTM
from keras.callbacks import ModelCheckpoint
from keras.utils import np_utils
import argparse
import generate as g
import codecs
import os
import io
from model import make_model, MODEL_NAME
input_file = "input.txt"

def main(args):
    with codecs.open(os.path.join(args.data_dir, input_file), "r", encoding='utf-8') as f:
        raw_text = f.read()

    chars = sorted(list(set(raw_text)))
    char_to_int = dict((c, i) for i, c in enumerate(chars))

    n_chars = len(raw_text)
    n_vocab = len(chars)
    print("Total Characters: ", n_chars)
    print("Total Vocab: ", n_vocab)

    seq_length = 100
    dataX = []
    dataY = []
    for i in range(0, n_chars - seq_length, 1):
        seq_in = raw_text[i:i + seq_length]
        seq_out = raw_text[i + seq_length]
        dataX.append([char_to_int[char] for char in seq_in])
        dataY.append(char_to_int[seq_out])
    n_patterns = len(dataX)
    print("Total Patterns: ", n_patterns)
    X = np.reshape(dataX, (n_patterns, seq_length, 1))

    X = X / float(n_vocab)

    y = np_utils.to_categorical(dataY)
    model = make_model(X, y)
    # define the checkpoint
    filepath = os.path.join('models', "snapshot_"+MODEL_NAME+"_{epoch:03d}_{loss:.4f}.hdf5")
    checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')
    callbacks_list = [checkpoint]
    # fit the model
    model.fit(X, y, epochs=20, batch_size=128, callbacks=callbacks_list)
    g.sample(model)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('data_dir', type=str, help='data directory containing input.txt')

    args = parser.parse_args()

    main(args)


import datetime

from django.test import TestCase

from raffle.models import RaffleExecution

class RaffleExecutionManagerTest(TestCase):

    def test_create_from_raffle_param_list_validation(self):
        with self.assertRaises(RuntimeError):
            RaffleExecution.objects.create_from_raffle(items='not a list')

    def test_create_from_raffle_create_new_execution(self):
        items = ['a', 'b', 'c', 'd', 'e']
        obj = RaffleExecution.objects.create_from_raffle(items=items)
        self.assertIsNotNone(obj)
        self.assertEqual(obj.items, items)
        self.assertIn(obj.result, items)
        self.assertTrue(isinstance(obj.executed_at, datetime.datetime))


from __future__ import print_function
import sys

try:
    if hasattr(sys, '_run_from_cmdl') is True:
        raise ImportError
    from pycompss.api.parameter import FILE_IN, FILE_OUT
    from pycompss.api.task import task
except ImportError:
    print("[Warning] Cannot import \"pycompss\" API packages.")
    print("          Using mock decorators.")

    from utils.dummy_pycompss import FILE_IN, FILE_OUT
    from utils.dummy_pycompss import task

from basic_modules.metadata import Metadata
from basic_modules.tool import Tool
from utils import logger   # pylint: disable=ungrouped-imports

# -----------------------------------------------------------------------------
class SimpleTool2(Tool):
    """
    Mockup Tool that defines a task with two FILE_IN inputs and one
    FILE_OUT output.
    """

    # @constraint()
    @task(file1=FILE_IN, file2=FILE_IN, file3=FILE_OUT,
          returns=bool, isModifier=False)
    def sumTwoFiles(self, file1, file2, file3):  # pylint: disable=invalid-name,no-self-use
        """
        Task that merges the contents of two files and returns the value.
        @param file1 The first input file with initial content
        @param file2 The second input file with initial content
        @param file3 The file where the results will be
        @return bool True if done successfully. False on the contrary.
        """
        result = 0
        try:
            with open(file1, 'r+') as f1_handle:
                result += int(f1_handle.read())
                print(result)
            with open(file2, 'r+') as f2_handle:
                result += int(f2_handle.read())
            with open(file3, 'w') as f3_handle:
                f3_handle.write(str(result))
            return True
        except (IOError, OSError):
            return False

    def run(self, input_files, input_metadata, output_files):
        """
        Standard function to call a task
        """

        # input and output share most metadata
        output_metadata = Metadata.get_child(
            input_metadata["input1"], output_files["output"])

        # Run the tool 2
        logger.info("SimpleTool2: Running task sumTwoFiles")
        task_status = self.sumTwoFiles(input_files["input1"],
                                       input_files["input2"],
                                       output_files["output"])

        if task_status:
            logger.info("SimpleTool2: run successful")
            return (output_files,
                    {"output": output_metadata})

        logger.fatal("SimpleTool2: run failed")
        return {}, {}

# ------------------------------------------------------------------------------


##### PARSING COMMAND LINE ARGUMENTS #####
import argparse as arp
import os

def main():
    prs = arp.ArgumentParser()

    prs.add_argument('sp_data_path', type = str, help = 'path to spatial data')

    prs.add_argument('result_dir', type = str, help = 'directory to regression model and results')

    prs.add_argument('cuda_device', type = str, help = "index of cuda device ID, from 0-7")

    prs.add_argument('-a','--annotation_column', default = 'celltype',
             type = str, help = 'column name for covariate')

    prs.add_argument('-r', '--regression_model_path', default = None,
                     type = str, help = 'path to regression model')

    prs.add_argument('-s', '--slide', default="1",
                     type = str, help = 'select slide 1-4, or all')

    args = prs.parse_args()

    cuda_device = args.cuda_device
    sp_data_path = args.sp_data_path
    results_folder = args.result_dir
    covariate_col_names = args.annotation_column
    slide = args.slide

    if args.regression_model_path is None:
        regression_model_output = os.listdir(results_folder + "/regression_model")[0]
        reg_path = f'{results_folder}regression_model/{regression_model_output}/'
    else:
        reg_path = args.regression_model_path

    assert cuda_device in ["0", "1", "2", "3", "4", "5", "6", "7"], "invalid device id"
    assert slide in ["1", "2", "3", "4"] or slide == "all", "slide does not exist"
    if slide.isdigit():
        assert 'filtered_feature_bc_matrix.h5' in os.listdir(sp_data_path + "/JBO0" + slide), "file path does not contain h5 feature matrix"
    else:
        assert all('filtered_feature_bc_matrix.h5' in os.listdir(sp_data_path + "/JBO0" + str(i)) for i in range(1,5)),\
        "one or more file path does not contain h5 feature matrix"

    ##### MAIN PART #####
    os.environ["CUDA_VISIBLE_DEVICES"]=cuda_device
    os.environ["CPATH"]="/usr/local/cuda/include:$CPATH" #To use cuDNN

    import sys
    import scanpy as sc
    import anndata
    import pandas as pd
    import numpy as np

    data_type = 'float32'

    # this line forces theano to use the GPU and should go before importing cell2location
    os.environ["THEANO_FLAGS"] = 'device=cuda,floatX=' + data_type + ',force_device=True'

    import cell2location
    import matplotlib as mpl
    from matplotlib import rcParams
    import matplotlib.pyplot as plt
    import seaborn as sns
    mpl.use('Agg')

    # silence scanpy that prints a lot of warnings
    import warnings
    warnings.filterwarnings('ignore')

    if not os.path.exists(results_folder + "std_model/"):
        os.makedirs(results_folder + "std_model/")

    ## READ IN SPATIAL DATA ##
    if slide == "all":
        # We will merge all slides together in one adata object
        adata_list, sample_name = [], []
        for i in range(1,5):
            name = 'JBO0'+ str(i)
            temp_adata = sc.read_visium(sp_data_path + "/" + name)
            print("Read in file from " + sp_data_path + "/" + name)
            temp_adata.var_names_make_unique()
            temp_adata.var["mt"] = temp_adata.var_names.str.startswith("mt-")
            sc.pp.calculate_qc_metrics(temp_adata, qc_vars=["mt"], inplace=True)
            temp_adata.obs['sample'] = name
            sample_name.append(name)
            adata_list.append(temp_adata)

        adata = adata_list[0].concatenate(adata_list[1:], batch_key="sample", uns_merge="unique", \
                                      batch_categories = sample_name, index_unique=None)
    else:
        adata = sc.read_visium(sp_data_path + "/JBO0" + slide)
        print("Read in file from " + sp_data_path)
        adata.var_names_make_unique()
        adata.obs['sample'] = "JBO0" + slide
        adata.var['mt'] = adata.var_names.str.startswith("mt-")
        sc.pp.calculate_qc_metrics(adata, qc_vars=["mt"], inplace=True)

    adata.obs_names_make_unique()
    # Calculate QC metrics and filter
    print("Before filtering: {} spots and {} genes".format(*adata.shape))
    adata.var['SYMBOL'] = adata.var_names
    sc.pp.filter_cells(adata, min_counts=11000)
    sc.pp.filter_cells(adata, max_counts=50000)
    adata = adata[adata.obs["pct_counts_mt"] < 20]
    sc.pp.filter_genes(adata, min_cells=10)

    # mitochondria-encoded (MT) genes should be removed for spatial mapping
    adata.obsm['mt'] = adata[:, adata.var['mt'].values].X.toarray()
    adata = adata[:, ~adata.var['mt'].values]
    print("After filtering: {} spots and {} genes".format(*adata.shape))

    adata_vis = adata.copy()
    adata_vis.raw = adata_vis

    ## READ IN REFERENCE DATA
    adata_raw = sc.read(f'{reg_path}sc.h5ad')

    # Export cell type expression signatures:
    inf_aver = adata_raw.raw.var.copy()
    inf_aver.index = adata_raw.raw.var['SYMBOL']
    inf_aver = inf_aver.loc[:, [f'means_cov_effect_{covariate_col_names}_{i}' for i in adata_raw.obs[covariate_col_names].unique()]]
    from re import sub
    inf_aver.columns = [sub(f'means_cov_effect_{covariate_col_names}_{i}', '', i) for i in adata_raw.obs[covariate_col_names].unique()]
    inf_aver = inf_aver.iloc[:, inf_aver.columns.argsort()]

    # scale up by average sample scaling factor
    inf_aver = inf_aver * adata_raw.uns['regression_mod']['post_sample_means']['sample_scaling'].mean()

    ## RUN CELL2LOCATION ##
    r = cell2location.run_cell2location(

          # Single cell reference signatures as pd.DataFrame
          # (could also be data as anndata object for estimating signatures analytically - `sc_data=adata_snrna_raw`)
          sc_data=inf_aver,
          # Spatial data as anndata object
          sp_data=adata_vis,
          verbose=True,
          # the column in sc_data.obs that gives cluster idenitity of each cell
          summ_sc_data_args={'cluster_col': covariate_col_names},

          train_args={'use_raw': True, # By default uses raw slots in both of the input datasets.
                      'n_iter': 15000, # Increase the number of iterations if needed (see below)

                      # Whe analysing the data that contains multiple samples,
                      # cell2location will select a model version which pools information across samples
                      # For details see https://cell2location.readthedocs.io/en/latest/cell2location.models.html#module-cell2location.models.CoLocationModelNB4E6V2
                      'sample_name_col': 'sample'}, # Column in sp_data.obs with Sample ID

          # Number of posterios samples to use for estimating parameters,
          # reduce if not enough GPU memory
          posterior_args={'n_samples': 1000},

          export_args={'path': results_folder + 'std_model/', # path where to save results
                       'run_name_suffix': '' # optinal suffix to modify the name the run
                      },

          model_kwargs={ # Prior on the number of cells, cell types and co-located combinations

                        'cell_number_prior': {
                            # Use visual inspection of the tissue image to determine
                            # the average number of cells per spot,
                            # an approximate count is good enough:
                            'cells_per_spot': 8,
                            # Prior on the number of cell types (or factors) in each spot
                            'factors_per_spot': 7,
                            # Prior on the number of correlated cell type combinations in each spot
                            'combs_per_spot': 2.5
                        },

                         # Prior on change in sensitivity between technologies
                        'gene_level_prior':{
                            # Prior on average change in expression level from scRNA-seq to spatial technology,
                            # this reflects your belief about the sensitivity of the technology in you experiment
                            'mean': 1/2,
                            # Prior on how much individual genes differ from that average,
                            # a good choice of this value should be lower that the mean
                            'sd': 1/4
                        }
          }
    )

if __name__ == '__main__':
    main()


import json
import numpy as np
from PIL import Image
from torch import zeros, from_numpy
from torch.utils.data import Dataset
from torchvision.transforms import transforms

class Loader(Dataset):
    '''
    加载数据集的类
    '''
    def __init__(self, sum_num, data_path, json_path):
        '''
        sum_num   : 总共数据的数量
        data_path : 数据集的相对路径
        json_path : json 文件相对路径

        '''
        self._data_num = sum_num
        self._data_path = data_path
        self._json_path = json_path

        # 图片格式转换
        self._transform = transforms.Compose([
            # RGB 形式打开
            lambda x: Image.open(self._data_path + x).convert('RGB'),
            # 将取值范围 [0, 255] 的 [高度 宽度 通道] 图片转换为
            # [0, 1] 的 [通道，高度，宽度] 图片
            transforms.ToTensor(),
            # 标准化到 [-1, 1]
            # transforms.Normalize(
            #     # 每个通道的均值
            #     (0.5, 0.5, 0.5),
            #     # 每个通道的方差
            #     (0.2, 0.2, 0.2)
            # )
        ])

        # 创建数据
        self.init()

    def init(self):
        # 文件名
        self._x_path = []
        # 标签
        self._y = []
        # 尺寸
        self._size = []
        # 位置
        self._box = []

        with open(self._json_path, 'r') as f:
            data_dict = json.load(f)
            for i in range(len(data_dict)):
                self._x_path.append(data_dict[i]['name'])
                self._y.append(data_dict[i]['category'])
                self._size.append([data_dict[i]['image_height'],
                                   data_dict[i]['image_width']])
                self._box.append(data_dict[i]['bbox'])

    # 访问数据
    def __getitem__(self, index):
        x = self._x_path[index]
        x = self._transform(x)
        y = self._y[index]
        tmp = zeros((6, 1))
        tmp[y-1, 0] = 1
        return x, tmp

    def __len__(self):
        return len(self._x_path)

    def calc_pic_normal_sigma(self):
        '''
        计算图像的均值和方差
        '''
        pass


import os
import json
import re
import sys
from tqdm import tqdm
from allennlp.predictors.predictor import Predictor
import allennlp_models.tagging

__all__ = ['convert_to_entailment']

# String used to indicate a blank
BLANK_STR = "___"

def convert_to_entailment(qa_file: str, output_file: str, ans_pos: bool=False):
    print(f'converting {qa_file} to entailment dataset...')
    nrow = sum(1 for _ in open(qa_file, 'r'))
    # predictor = Predictor.from_path("ner-elmo.2021-02-12.tar.gz")
    predictor = Predictor.from_path("/home/zhengchen/codes/python_codes/wiqa_research/MRRG/graph_utils/openie-model.2020.03.26.tar.gz") ### link: https://storage.googleapis.com/allennlp-public-models/openie-model.2020.03.26.tar.gz

    path, _ = os.path.split(output_file) ### _ is the filename, path is the folder name
    if not os.path.exists(path):
        os.makedirs(path)

    with open(output_file, 'w+') as output_handle, open(qa_file, 'r') as qa_handle:
        # print("Writing to {} from {}".format(output_file, qa_file))
        for line in tqdm(qa_handle, total=nrow):
            json_line = json.loads(line)
            output_dict = convert_qajson_to_entailment(json_line, predictor)
            output_handle.write(json.dumps(output_dict))
            output_handle.write("\n")
    print(f'converted statements saved to {output_file}')
    print()

### use AllenNLP NER tool
# def convert_qajson_to_entailment(qa_json, predictor):
#     question_text = qa_json["question"]["stem"]
#     para_steps = " ".join(qa_json["question"]['para_steps'])
#     answer_labels = qa_json["question"]["answer_label"]

#     queston_and_para = question_text+para_steps
#     # ner_res = predictor.predict(sentence="Did Uriah honestly think he could beat The Legend of Zelda in under three hours?.")
#     ner_res = predictor.predict(sentence=queston_and_para)
#     statement = ''
#     for i in range(len(ner_res['tags'])):
#         if ner_res['tags'][i] != 'O':
#             statement += ner_res['words'][i] + ' '
#     # statement = create_hypothesis(get_fitb_from_question(question_text), choice_text, ans_pos)
#     create_output_dict(qa_json, statement, True)

#     return qa_json

# predictor = Predictor.from_path("./ner-elmo.2021-02-12.tar.gz")
# convert_qajson_to_entailment('...', predictor)

### use AllenNLP OpenIE tool
def convert_qajson_to_entailment(qa_json, predictor):
    question_text = qa_json["question"]["stem"]
    # para_steps = " ".join(qa_json["question"]['para_steps'])
    para_steps = qa_json["question"]['para_steps']
    answer_labels = qa_json["question"]["answer_label"]

    queston_and_para = [question_text] + para_steps

    set_res = set()
    for sen in queston_and_para:
        res = predictor.predict(sentence=sen.lower())
        for i in range(len(res['verbs'])):
            description = res['verbs'][i]['description']
            start, end= 0, 0
            for i in range(len(description)):
                if description[i] == '[':
                    start = i+1
                elif description[i] == ']':
                    end = i
                    if description[start:end+1].startswith('ARG'):
                        set_res.add(description[start:end].split(':')[1][1:])

    statement = ', '.join(set_res)
    create_output_dict(qa_json, statement, True)

    return qa_json

# Get a Fill-In-The-Blank (FITB) statement from the question text. E.g. "George wants to warm his
# hands quickly by rubbing them. Which skin surface will produce the most heat?" ->
# "George wants to warm his hands quickly by rubbing them. ___ skin surface will produce the most
# heat?
def get_fitb_from_question(question_text: str) -> str:
    fitb = replace_wh_word_with_blank(question_text)
    if not re.match(".*_+.*", fitb):
        # print("Can't create hypothesis from: '{}'. Appending {} !".format(question_text, BLANK_STR))
        # Strip space, period and question mark at the end of the question and add a blank
        fitb = re.sub(r"[\.\? ]*$", "", question_text.strip()) + " " + BLANK_STR
    return fitb

# Create a hypothesis statement from the the input fill-in-the-blank statement and answer choice.
def create_hypothesis(fitb: str, choice: str, ans_pos: bool) -> str:

    if ". " + BLANK_STR in fitb or fitb.startswith(BLANK_STR):
        choice = choice[0].upper() + choice[1:]
    else:
        choice = choice.lower()
    # Remove period from the answer choice, if the question doesn't end with the blank
    if not fitb.endswith(BLANK_STR):
        choice = choice.rstrip(".")
    # Some questions already have blanks indicated with 2+ underscores
    if not ans_pos:
        hypothesis = re.sub("__+", choice, fitb)
        return hypothesis
    choice = choice.strip()
    m = re.search("__+", fitb)
    start = m.start()

    length = (len(choice) - 1) if fitb.endswith(BLANK_STR) and choice[-1] in ['.', '?', '!'] else len(choice)
    hypothesis = re.sub("__+", choice, fitb)

    return hypothesis, (start, start + length)

# Identify the wh-word in the question and replace with a blank
def replace_wh_word_with_blank(question_str: str):
    # if "What is the name of the government building that houses the U.S. Congress?" in question_str:
    #     print()
    question_str = question_str.replace("What's", "What is")
    question_str = question_str.replace("whats", "what")
    question_str = question_str.replace("U.S.", "US")
    wh_word_offset_matches = []
    wh_words = ["which", "what", "where", "when", "how", "who", "why"]
    for wh in wh_words:
        # Some Turk-authored SciQ questions end with wh-word
        # E.g. The passing of traits from parents to offspring is done through what?

        if wh == "who" and "people who" in question_str:
            continue

        m = re.search(wh + r"\?[^\.]*[\. ]*$", question_str.lower())
        if m:
            wh_word_offset_matches = [(wh, m.start())]
            break
        else:
            # Otherwise, find the wh-word in the last sentence
            m = re.search(wh + r"[ ,][^\.]*[\. ]*$", question_str.lower())
            if m:
                wh_word_offset_matches.append((wh, m.start()))
            # else:
            #     wh_word_offset_matches.append((wh, question_str.index(wh)))

    # If a wh-word is found
    if len(wh_word_offset_matches):
        # Pick the first wh-word as the word to be replaced with BLANK
        # E.g. Which is most likely needed when describing the change in position of an object?
        wh_word_offset_matches.sort(key=lambda x: x[1])
        wh_word_found = wh_word_offset_matches[0][0]
        wh_word_start_offset = wh_word_offset_matches[0][1]
        # Replace the last question mark with period.
        question_str = re.sub(r"\?$", ".", question_str.strip())
        # Introduce the blank in place of the wh-word
        fitb_question = (question_str[:wh_word_start_offset] + BLANK_STR +
                         question_str[wh_word_start_offset + len(wh_word_found):])
        # Drop "of the following" as it doesn't make sense in the absence of a multiple-choice
        # question. E.g. "Which of the following force ..." -> "___ force ..."
        final = fitb_question.replace(BLANK_STR + " of the following", BLANK_STR)
        final = final.replace(BLANK_STR + " of these", BLANK_STR)
        return final

    elif " them called?" in question_str:
        return question_str.replace(" them called?", " " + BLANK_STR + ".")
    elif " meaning he was not?" in question_str:
        return question_str.replace(" meaning he was not?", " he was not " + BLANK_STR + ".")
    elif " one of these?" in question_str:
        return question_str.replace(" one of these?", " " + BLANK_STR + ".")
    elif re.match(r".*[^\.\?] *$", question_str):
        # If no wh-word is found and the question ends without a period/question, introduce a
        # blank at the end. e.g. The gravitational force exerted by an object depends on its
        return question_str + " " + BLANK_STR
    else:
        # If all else fails, assume "this ?" indicates the blank. Used in Turk-authored questions
        # e.g. Virtually every task performed by living organisms requires this?
        return re.sub(r" this[ \?]", " ___ ", question_str)

def create_output_dict(input_json: dict, statement: str, label: bool) -> dict:
    if "statements" not in input_json:
        input_json["statements"] = []
    input_json["statements"].append({"label": True, "statement": statement})
    return input_json

if __name__ == "__main__":
    if len(sys.argv) < 3:
        raise ValueError("Provide at least two arguments: "
                         "json file with hits, output file name")
    convert_to_entailment(sys.argv[1], sys.argv[2])


import torch, time
from model import Model,LossFN

def display(inp):
    if isinstance(inp, torch.Tensor):
        return list(inp.shape)
    elif isinstance(inp, tuple):
        shape = []
        for t in inp:
            shape.append(display(t))
        return shape
    return "<Unable to display>"

def checkModel(train=True):
    model = Model(3, 12000, 32)
    inp = torch.rand(2, 12000, 3, requires_grad=True)
    print(
        "Model Parameters :",
        sum(p.numel() for p in model.train().parameters() if p.requires_grad),
    )
    if train:
        with torch.no_grad():
            s = time.time()
            y = model.eval()(inp)
            e = time.time()
        print("Shape :", display(y))
        print("Time taken :", e - s)

def checkLoss():
    cls_target = torch.rand(4, 100,10)
    reg_target = torch.rand(4,100,7)
    pred = torch.cat([cls_target, reg_target], dim=-1)
    pred.requires_grad=True
    cls_target = torch.randint(10,(4, 100,10))
    reg_target = torch.rand(4,100,7)
    target = torch.cat([cls_target, reg_target], dim=-1)
    # target = pred
    print(pred.shape,target.shape)
    loss_fn = LossFN(n_classes=10)
    loss = loss_fn(pred, target)
    print("Loss shape :", loss.shape)
    try:
        print(f"Loss : {loss.item():.3f}")
        loss.backward()
        print("Accepted!!")
    except:

        print("Unable to do backward")
        loss.backward()

# checkModel(True)
checkLoss()


from beancount.ingest.importer import ImporterProtocol
from beancount.ingest.importers.mixins import identifier, filing
from beancount.core import data, flags, number
from beancount.utils.date_utils import parse_date_liberally

from collections import namedtuple, OrderedDict

import csv

GIFT_CERTIFICATE_STRING = 'Gift Certificate/Card'
RECEIPT_URL = 'https://www.amazon.com/gp/your-account/order-details?orderID={}'

RowFields = OrderedDict()
RowFields['Order Date'] = 'order_date'
RowFields['Order ID'] = 'order_id'
RowFields['Title'] = 'description'
RowFields['Category'] = 'category'
RowFields['ASIN/ISBN'] = 'asin_or_isbn'
RowFields['UNSPSC Code'] = 'unspsc'
RowFields['Website'] = 'website'
RowFields['Release Date'] = 'release_date'
RowFields['Condition'] = 'condition'
RowFields['Seller'] = 'seller'
RowFields['Seller Credentials'] = 'seller_credentials'
RowFields['List Price Per Unit'] = 'list_price_per_unit'
RowFields['Purchase Price Per Unit'] = 'purchase_price_per_unit'
RowFields['Quantity'] = 'quantity'
RowFields['Payment Instrument Type'] = 'payment_instrument'
RowFields['Purchase Order Number'] = 'purchase_order_number'
RowFields['PO Line Number'] = 'po_line_number'
RowFields['Ordering Customer Email'] = 'ordering_customer_email'
RowFields['Shipment Date'] = 'shipment_date'
RowFields['Shipping Address Name'] = 'shipping_address_name'
RowFields['Shipping Address Street 1'] = 'shipping_address_street_1'
RowFields['Shipping Address Street 2'] = 'shipping_address_street_2'
RowFields['Shipping Address City'] = 'shipping_address_city'
RowFields['Shipping Address State'] = 'shipping_address_state'
RowFields['Shipping Address Zip'] = 'shipping_address_zip'
RowFields['Order Status'] = 'order_status'
RowFields['Carrier Name & Tracking Number'] = 'carrier_name_and_tracking'
RowFields['Item Subtotal'] = 'item_subtotal'
RowFields['Item Subtotal Tax'] = 'item_subtotal_tax'
RowFields['Item Total'] = 'item_total'
RowFields['Tax Exemption Applied'] = 'tax_exemption_applied'
RowFields['Tax Exemption Type'] = 'tax_exemption_type'
RowFields['Exemption Opt-Out'] = 'exemption_opt_out'
RowFields['Buyer Name'] = 'buyer_name'
RowFields['Currency'] = 'currency'
RowFields['Group Name'] = 'group_name'

Row = namedtuple('Row', RowFields.values())

class Importer(identifier.IdentifyMixin, filing.FilingMixin, ImporterProtocol):
    '''
    Importer for Amazon Items reports.
    Create a new "Items" report at https://www.amazon.com/gp/b2b/reports and
    this importer will convert those into transactions.
    '''

    matchers = [  # Ways we know this is an Amazon Items file
        ('mime', 'text/csv'),
        ('content', ','.join(RowFields.keys()))
    ]

    funding_sources = {}
    expense_account = None
    tags = {'amazon-purchase'}
    currency_symbols = {'$', '€', '£'}  # I know, very English-centric
    debug = False

    def __init__(
            self,
            *args,
            funding_sources={},
            expense_account=None,
            tags=set(),
            currency_symbols={},
            prefix='Amazon',
            debug=False,
            **kwargs):
        '''
        Create an importer for Amazon Items reports. Available keyword
        arguments:
            funding_sources a dict mapping the last four digits of a store card
                            (or some other card/funding source identifier) to
                            its matching liability account.
            expense_account the account to offset every transaction against (or
                            if None, unbalanced transactions will be printed).
            tags            a set of tags to add to every transaction.
            currency_symbols    a set of currency symbols to strip from input.
            prefix          the filename prefix to use when beancount-file
                            moves files (defaults to "Amazon").
            debug           if True, every row will be printed to stdout.
        Additional keyword arguments for IdentifyMixin and FilingMixin are
        permitted. Most significantly:
            filing          the name of the account used to set the storage
                            location for these files.
        '''
        self.funding_sources.update(funding_sources)
        self.expense_account = expense_account
        self.tags.update(tags)
        self.currency_symbols.update(currency_symbols)
        self.debug = debug
        super().__init__(prefix=prefix, *args, **kwargs)

    def extract(self, file, existing_entries=None):
        transactions = []

        for row_number, row in self.get_rows(file):
            if self.debug:
                print("Row #{}: {}".format(row_number, row))
            transactions.append(self.get_transaction(
                row, file.name, row_number))

        transactions.sort(key=lambda txn: txn.date)

        return transactions

    def file_date(self, file):
        max_date = None
        for _, row in self.get_rows(file):
            parsed_date = parse_date_liberally(row.order_date)
            if max_date is None or parsed_date > max_date:
                max_date = parsed_date
        return max_date

    def get_rows(self, file):
        reader = csv.DictReader(open(file.name))
        assert reader.fieldnames == list(RowFields.keys()), "Header mismatch!"
        for row_number, row_dict in enumerate(reader, 1):
            if not row_dict or list(row_dict)[0][0].startswith('#'):
                continue
            yield (
                row_number,
                Row(**{RowFields[k]: v for k, v in row_dict.items()})
            )

    def get_transaction(self, row: Row, file_name, row_number):
        meta = data.new_metadata(file_name, row_number, {
            'receipt-url': RECEIPT_URL.format(row.order_id),
            'payment-method': row.payment_instrument,
        })
        postings = self.get_postings(row)
        if self.expense_account:
            postings.append(self.get_expense_posting(row))
        t = data.Transaction(
            meta,
            parse_date_liberally(row.order_date),
            flags.FLAG_WARNING if len(postings) == 0 else flags.FLAG_OKAY,
            row.seller.strip(),
            row.description.strip(),
            self.tags,
            data.EMPTY_SET,  # links
            postings)
        return t

    def get_postings(self, row: Row):
        postings = []
        if 'and' in row.payment_instrument:
            flag = flags.FLAG_WARNING
        else:
            flag = None
        for src, account in self.funding_sources.items():
            if src.isnumeric() and len(src) == 4:
                src = 'Amazon.com Store Card - {}'.format(src)
            if src in row.payment_instrument:
                postings.append(data.Posting(
                    account=account,
                    units=-self.get_amount(row),
                    cost=None,
                    price=None,
                    flag=flag,
                    meta={},
                ))
        return postings

    def get_expense_posting(self, row: Row):
        return data.Posting(
            account=self.expense_account,
            units=self.get_amount(row),
            cost=None,
            price=None,
            flag=None,
            meta={},
        )

    def get_amount(self, row: Row):
        total = row.item_total.strip(''.join(self.currency_symbols))
        return data.Amount(number.D(total), row.currency)


"""
思路：
1、准备两个进程
2、准备一个队列 一个进程向队列中写入数据，然后吧队列传递到另外一个进程
3、另外一个进程读取数据
"""
import time
import multiprocessing

# 1、写入数据到队列的函数
def write_queue(queue):
    # for循环，向队列写入数据
    for i in range(10):
        # 判断队列是否已满
        if queue.full():
            print("队列已满!")
            break

        # 向队列中放入值
        queue.put(i)
        print("写入成功，已经写入:", i)
        time.sleep(0.5)

# 2、读取队列数据并显示的函数
def read_queue(queue):
    while True:
        # 判断队列是否已经为空
        if queue.qsize() == 0:
            print("队列已空!")
            break

        # 从队列中读取数据
        value = queue.get()
        print("已经读取:", value)

if __name__ == '__main__':
    # 3、创建一个空的队列
    queue = multiprocessing.Queue(5)

    # 4、创建2个进程，分别写数据、读数据
    write_queue1 = multiprocessing.Process(target=write_queue, args=(queue,))
    read_queue1 = multiprocessing.Process(target=read_queue, args=(queue,))
    write_queue1.start()

    # 优先让写数据进程执行结束后，再读取数据
    write_queue1.join()

    read_queue1.start()

09-进程间的通信.py


from pprint import pprint

from flask import render_template, request, jsonify, current_app as app
from datatables import ColumnDT, DataTables

from ...logging import get_logger
from ...models.db_engine.column import Column, db
from ...models.db_engine.model import (
    get_model_data_by_table_id, automapped_classes
)
from ...models.genesis.utils import get_by_id_or_first_genesis_db_id
from ...datatables import DataTablesExt

logger = get_logger(app)

@app.route("/db-engine/table/<int:id>/columns")
def columns(id):
    table, model, model_name = get_model_data_by_table_id(id)
    column_names = ['Name', 'Type', 'Default', 'AutoIncrement',
                  'Primary Key']
    valid_db_id = get_by_id_or_first_genesis_db_id(id)
    return render_template('db_engine/columns.html',
                            project=app.config.get('PRODUCT_BRAND_NAME') + ' Block Explorer',
                            table_id=table.id, table_name=table.name,
                            valid_db_id=valid_db_id,
                            db_id=table.database.id,
                            db_name=table.database.name,
                            db_bind_name=table.database.bind_name,
                            model_name=model.__name__,
                            column_names=column_names,
                            columns_num=len(column_names))

@app.route('/dt/db-engine/table/<int:id>/columns')
def dt_columns(id):
    model = Column
    column_ids = ['name', 'type', 'default', 'autoincrement',
                  'primary_key']
    columns = [getattr(model, col_id) for col_id in column_ids]
    dt_columns = [ColumnDT(m) for m in columns]
    query = db.session.query(*columns).filter_by(table_id=id)
    params = request.args.to_dict()
    rowTable = DataTablesExt(params, query, dt_columns)
    return jsonify(rowTable.output_result())


import pygame

class Font:
    def __init__(self, size, bold, text, color):
        self.name = 'blackjack'
        self.size = size
        self.bold = bold
        self.t = text
        self.c = color

    def create(self, surf, x, y):
        self.font = pygame.font.SysFont(self.name, self.size, self.bold)
        self.text = self.font.render(self.t, False, self.c)
        surf.blit(self.text, (x, y))


# 地图节点
'''
流程：
    1. 初始化：从地图图片读入物理地图数据
    2. 保持服务畅通
        - 服务1： agent站点向地图发送，用来更新自己的位置
             - 输入： 坐标x，y 就从0开始用数组下标格式吧，不要折腾了
             - 输出：成功与否
        - 服务2： 向站点发送地图数据
             - 输入：无
             - 输出：真地图，但是打成一维，开头附上宽高数据，方便恢复。
'''

# map3: 有碰撞时：大约在25-30
      # 无碰撞时：大约在20-23

from interfaces.srv import MapSending, MapLocationUpdate
from interfaces.msg import MapVisualiserSiteMoves
from rclpy.node import Node
import rclpy
from PIL import Image

collision_on  = True # 是否打开节点间碰撞体积

map_path = "/mnt/a/OneDrive/MScRobotics/Dissertation2022/codes/map_builder/map2.png"

def map_reader(map_path):
    '''
    将地图图片读取为二维数组的函数

    Args:
        map_path (string): 地图图片的路径，编写此函数时用的是绝对路径。不推荐相对路径，因为ros2功能包里的src路径不太适合放图片吧

    Returns:
        list: 二维数组，其中地图图片中白色的地方是True，黑色像素的地方是False
    '''
    with Image.open(map_path) as img:
        width, height = img.size
        pixels = img.load()
        bool_array = [[False for _ in range(width)] for _ in range(height)]
        for i in range(height):
            for j in range(width):
                if pixels[j, i] == (255, 255, 255):
                    bool_array[i][j] = True
        return bool_array

def map_condenser(map_list_2d):
    '''
    将二维的数组压缩为一维的函数，压缩后在开头附上宽高数据，以便接收端恢复

    Args:
        map_list_2d (list[list[]]): 用来保存地图数据的二维数组

    Returns:
        list: 被压缩为一维的数组，且开头附有宽高信息
    '''
    width = len(map_list_2d[0])
    height = len(map_list_2d)

    flattened = [_ for row in map_list_2d for _ in row]

    flattened.insert(0, width)
    flattened.insert(1, height)

    return flattened

def map_uncondenser(map_list_1d):
    '''
    将输入的一维数组恢复为二维数组的函数，输入的一维数组第一位是原数组宽，第二位是原数组高

    Args:
        map_list_1d (list): 输入的一维数组第一位是原数组宽，第二位是原数组高

    Returns:
        list[list[]]: 恢复的二维数组
    '''
    width = map_list_1d[0]
    height = map_list_1d[1]

    del map_list_1d[:2]

    map_list_2d = [map_list_1d[i:i+width]
                   for i in range(0, len(map_list_1d), width)]

    return map_list_2d

class MapNode(Node):
    def __init__(self, name):
        super().__init__(name)

        self.get_logger().info("物理世界地图结点已启动")

        # 物理地图
        self.physical_map_ = map_reader(map_path)

        # 站点在地图中的位置，起始为空。
        self.sites_in_map_ = [[None for _ in row]
                              for row in self.physical_map_]

        # 向节点发送地图的服务
        self.map_sender_ = self.create_service(
            MapSending, "MapSender", self.map_init_sending_callback)

        # 节点申请移动的服务
        self.site_loaction_updater_ = self.create_service(
            MapLocationUpdate, "MapLocationUpdater", self.map_location_update_callback)

        # 每次有节点移动后告诉可视化器新消息的话题发布者
        self.site_moves_to_visualiser_publisher_ = self.create_publisher(
            MapVisualiserSiteMoves, "SiteMoves", 10)

    def map_location_update_callback(self, request, response):
        # 检查目的地是否是阻挡物，如果是则直接返回False
        x = request.x # 行号（纵坐标）
        y = request.y  # 列号（横坐标）
        node = request.applicant
        response.success = True  # 先初始化一个True, 后续根据条件结果修改
        # 检查目标位置是否超出范围。
        if y >= len(self.physical_map_) or x >= len(self.physical_map_[0]):
            response.success = False  # 目标位置出界了
        # 检查目标位置是否被阻挡

        if not self.physical_map_[x][y]:
            response.success = False

        if (self.sites_in_map_[x][y] != None and self.sites_in_map_[x][y] != node) and collision_on:
            response.success = False

        '''
        if (self.sites_in_map_[x][y] != None and self.sites_in_map_[x][y] != node) or not self.physical_map_[x][y]:
            response.success = False  # 目标位置有别人或者有墙
        '''

        if response.success:  # 以下操作仅在成功移动时进行
            # 将图中所有此发送者占有的地方清空
            self.sites_in_map_ = [
                [None if element == node else element for element in sublist] for sublist in self.sites_in_map_]
            # 将新位置赋为发送者标号
            self.sites_in_map_[x][y] = node

            # 每有节点成功完成一次移动请求：向可视化节点发送移动情况。
            msg = MapVisualiserSiteMoves()  # 实例化信息对象
            msg.site_no = node
            msg.x = x
            msg.y = y
            self.site_moves_to_visualiser_publisher_.publish(msg)

        return response

    def map_init_sending_callback(self, request, response):
        '''
        服务回调函数：返回压缩为一维的地图，起始两位是地图宽，高。
        '''
        condensed_map = map_condenser(self.physical_map_)
        response.map_data = condensed_map
        print("进行了一次地图初始化服务")
        return response

def main(args=None):

    rclpy.init(args=args)

    map = MapNode("MapNode")

    rclpy.spin(map)

    rclpy.shutdown()


from collections import defaultdict
from copy import deepcopy
from itertools import chain

import numpy as np

from r2d2.camera_utils.info import camera_type_to_string_dict
from r2d2.data_processing.data_transforms import ImageTransformer

class TimestepProcesser:
    def __init__(
        self,
        ignore_action=False,
        action_space="cartesian_velocity",
        robot_state_keys=["cartesian_position", "gripper_position", "joint_positions", "joint_velocities"],
        camera_extrinsics=["hand_camera", "varied_camera", "fixed_camera"],
        state_dtype=np.float32,
        action_dtype=np.float32,
        image_transform_kwargs={},
    ):
        assert action_space in ["cartesian_position", "joint_position", "cartesian_velocity", "joint_velocity"]

        self.action_space = action_space
        self.gripper_key = "gripper_velocity" if "velocity" in action_space else "gripper_position"
        self.ignore_action = ignore_action

        self.robot_state_keys = robot_state_keys
        self.camera_extrinsics = camera_extrinsics

        self.state_dtype = state_dtype
        self.action_dtype = action_dtype

        self.image_transformer = ImageTransformer(**image_transform_kwargs)

    def forward(self, timestep):
        # Make Deep Copy #
        timestep = deepcopy(timestep)

        # Get Relevant Camera Info #
        camera_type_dict = {k: camera_type_to_string_dict[v] for k, v in timestep["observation"]["camera_type"].items()}
        sorted_camera_ids = sorted(camera_type_dict.keys())

        ### Get Robot State Info ###
        sorted_state_keys = sorted(self.robot_state_keys)
        full_robot_state = timestep["observation"]["robot_state"]
        robot_state = [np.array(full_robot_state[key]).flatten() for key in sorted_state_keys]
        if len(robot_state):
            robot_state = np.concatenate(robot_state)

        ### Get Extrinsics ###
        calibration_dict = timestep["observation"]["camera_extrinsics"]
        sorted_calibrated_ids = sorted(calibration_dict.keys())
        extrinsics_dict = defaultdict(list)

        for serial_number in sorted_camera_ids:
            cam_type = camera_type_dict[serial_number]
            if cam_type not in self.camera_extrinsics:
                continue

            for full_cam_id in sorted_calibrated_ids:
                if serial_number in full_cam_id:
                    cam2base = calibration_dict[full_cam_id]
                    extrinsics_dict[cam_type].append(cam2base)

        sorted_extrinsics_keys = sorted(extrinsics_dict.keys())
        extrinsics_state = list(chain(*[extrinsics_dict[cam_type] for cam_type in sorted_extrinsics_keys]))
        if len(extrinsics_state):
            extrinsics_state = np.concatenate(extrinsics_state)

        ### Get High Dimensional State Info ###
        high_dim_state_dict = defaultdict(lambda: defaultdict(list))

        for obs_type in ["image", "depth", "pointcloud"]:
            obs_type_dict = timestep["observation"].get(obs_type, {})
            sorted_obs_ids = sorted(obs_type_dict.keys())

            for serial_number in sorted_camera_ids:
                cam_type = camera_type_dict[serial_number]

                for full_obs_id in sorted_obs_ids:
                    if serial_number in full_obs_id:
                        data = obs_type_dict[full_obs_id]
                        high_dim_state_dict[obs_type][cam_type].append(data)

        ### Finish Observation Portion ###
        low_level_state = np.concatenate([robot_state, extrinsics_state], dtype=self.state_dtype)
        processed_timestep = {"observation": {"state": low_level_state, "camera": high_dim_state_dict}}
        self.image_transformer.forward(processed_timestep)

        ### Add Proper Action ###
        if not self.ignore_action:
            arm_action = timestep["action"][self.action_space]
            gripper_action = timestep["action"][self.gripper_key]
            action = np.concatenate([arm_action, [gripper_action]], dtype=self.action_dtype)
            processed_timestep["action"] = action

        return processed_timestep


from io import BufferedReader  # used for types
import sys
import os
import time
import threading
from typing import NoReturn

class Node:
    """Definition of a Node for a Double Ended Queue"""

    def __init__(self, req: bytes, res: bytes, ttl: int) -> None:
        self.request: bytes = req
        self.response: bytes = res
        self.ttl: float = ttl
        self.last_access: float = time.time()
        self.next: Node | None = None
        self.prev: Node | None = None

class Deque:
    """Definition of a Double Ended Queue"""

    def __init__(self) -> None:
        self.head: Node | None = None
        self.tail: Node | None = None
        self.size: int = 0

    def add(self, args) -> Node | None:
        """Add a new node to the front of the queue and return it"""

        temp: Node = Node(*args)
        temp.next = self.head
        if self.head:
            self.head.prev = temp
        self.head = temp

        if self.size == 0:
            self.tail = self.head

        self.size += 1

        return temp

    def to_front(self, node: Node) -> None:
        """Move a node to the front of the queue"""

        if not node or self.size == 0:
            return

        back, front = node.prev, node.next

        # If there is only one node in the queue or if the node is already the head, dont do anything
        if (not back and not front) or (not back and front):
            return

        node.prev, node.next = None, None

        if back and not front:  # If the node is the tail, move to the front AND set the new tail
            back.next = None
            self.tail = back
        else:
            front.prev = back
            back.next = front

        # Move the node to the front
        node.next = self.head

        # And set the new head
        self.head = node

    def pop(self) -> Node | None:
        """Remove the last node from the queue and return it"""

        # If the Deque is empty, there is nothing to pop, so return None
        if self.size == 0:
            return None

        # Reduce size by 1
        self.size -= 1

        # If there is only one node in the queue, set the head and tail to None and return the it
        if self.size == 1:
            node = self.head
            node.next, node.prev = None, None
            self.head, self.tail = None, None
            return node

        # If there are more than one node in the queue, set the tail to the second to last node and return the last node
        deleted = self.tail
        self.tail = deleted.prev
        deleted.prev, self.tail.next = None, None

        return deleted

class Cache:
    """Definition of Cache"""

    def __init__(self, cache_size: int, ttl: int, delimiter: str, path_to_persistence: str, sleep: int) -> None:
        # Config variables
        self.PATH_TO_PERSISTENCE: str = path_to_persistence
        self.MAX_SIZE: int = cache_size
        self.DELIMITER: str = delimiter
        self.TTL: int = ttl
        self.lock: threading.Lock = threading.Lock()

        # Cache variables
        self.CURRENT_SIZE: int = 0
        self.in_deq: dict[bytes, Node | None] = {}
        self.deq: Deque = Deque()

        # Time
        self.SLEEP = sleep

        self.read_cache()

    def parse_descriptor(self, line: bytes) -> dict[str, int]:
        items: list[bytes] = line.split(b', ')
        descriptor: dict[str, int] = {}
        for item in items:
            key, val = item.split(b': ')
            key = key.decode('utf-8')
            val = int(val.decode('utf-8'))
            descriptor[key] = val
        return descriptor

    def read_segment(self, file: BufferedReader, descriptor: dict[str, int], key: str, other: bytes) -> tuple[bytes, bytes]:
        request: bytes = b''
        buffer: bytes = b''
        while descriptor[key] > 0:
            line: bytes = other + file.readline()
            other = b''
            length: int = min(len(line), descriptor[key])
            descriptor[key] -= length
            request += line[:length]
            buffer += line[length:]
        return request, buffer

    def read_cache(self) -> None:

        if not os.path.exists(self.PATH_TO_PERSISTENCE):
            print('Cache file not found')
            print('Procceeding to start with empty cache')
            return

        print('Cache file found, loading...')

        # Load cache from persistence
        with open(self.PATH_TO_PERSISTENCE, 'rb') as file:

            buffer = b''
            while True:
                line = buffer + file.readline()
                if not line:
                    break
                descriptor = self.parse_descriptor(line)

                request, buffer_request = self.read_segment(
                    file, descriptor, 'request', b'')
                response, buffer_response = self.read_segment(
                    file, descriptor, 'response', buffer_request)
                ttl, _ = self.read_segment(
                    file, descriptor, 'ttl', buffer_response)
                node: Node | None = self.deq.add(
                    [request, response, int(float(ttl))])
                self.in_deq[request] = node

    def add(self, request: bytes, response: bytes) -> Node | None:
        with self.lock:
            # Check if cache is full and delete oldest nodes while it is
            while self.CURRENT_SIZE + sys.getsizeof(response) > self.MAX_SIZE:
                old_node: Node = self.deq.pop()
                del self.in_deq[old_node.request]
                self.CURRENT_SIZE -= sys.getsizeof(old_node.response)

            # Add request to cache
            node = self.deq.add([request, response, self.TTL])
            self.CURRENT_SIZE += sys.getsizeof(response)
            self.in_deq[request] = node

    def get(self, request: bytes) -> bytes:
        with self.lock:
            # Check if request is in cache
            if request in self.in_deq:
                node = self.in_deq[request]  # Get node from cache
                node.ttl = self.TTL  # Reset TTL
                node.last_access = time.time()  # Update last access time
                self.deq.to_front(node)  # Move node to front of cache
                return node.response

            # Request not in cache, return None
            return b''

    def check_time(self) -> NoReturn:
        while True:
            self.update_time()
            self.save_cache()
            time.sleep(self.SLEEP)

    def update_time(self) -> None:
        with self.lock:
            curr: Node = self.deq.tail

            # Iterate through cache
            while curr != None:
                node = curr

                # Get current time
                now = time.time()

                # Check if TTL is 0 or less since last access
                node.ttl -= now - node.last_access

                # Update last access time
                node.last_access = now

                # If TTL is 0 or less, remove node from cache
                if node.ttl <= 0:
                    self.deq.pop()
                    self.CURRENT_SIZE -= sys.getsizeof(node)
                    del self.in_deq[node.request]

                # Move to next node
                curr = curr.prev

    def save_cache(self) -> None:
        with self.lock:
            curr: Node | None = self.deq.head
            with open(self.PATH_TO_PERSISTENCE, 'wb') as file:
                # Iterate through cache
                while curr != None:
                    node = curr

                    buffer = bytes()
                    request = node.request
                    response = node.response
                    ttl = str(int(node.ttl)).encode('utf-8')
                    len_request = str(len(request)).encode('utf-8')
                    len_response = str(len(response)).encode('utf-8')
                    len_ttl = str(len(ttl)).encode('utf-8')
                    descriptor: bytes = b'request: ' + len_request
                    descriptor += b', response: ' + len_response
                    descriptor += b', ttl: ' + len_ttl + b'\n'

                    buffer += descriptor + request + response + ttl

                    file.write(buffer)
                    curr = curr.next


from __future__ import print_function
import os
import re
import csv
import copy
import glob
import datetime
import json

FREESURFER_QUESTIONNAIRES = [
    "LH.APARC.FOLDIND", "LH.APARC.AREA", "RH.APARC.CURVIND",
    "LH.APARC.THICKNESSSTD", "MEASURE:VOLUME", "LH.APARC.MEANCURV",
    "RH.APARC.THICKNESSSTD", "LH.APARC.VOLUME", "RH.APARC.MEANCURV",
    "RH.APARC.VOLUME", "LH.APARC.CURVIND", "LH.APARC.THICKNESS",
    "RH.APARC.FOLDIND", "RH.APARC.GAUSCURV", "RH.APARC.AREA",
    "LH.APARC.GAUSCURV", "RH.APARC.THICKNESS"]
FREESURFER = [
    "FREESURFER"]

RQL_T1 = ("Any SC Where S is Subject, S code_in_study '{0}', "
          "S subject_scans SC, SC in_assessment A, A timepoint '{1}', "
          "SC label 'ADNI_MPRAGE'")

def freesurfer_stats(fsstatdirs, study_name, subject_age_map, savedir=None,
                     is_openanswer=False):
    """ Parse the freesurfer stats files and create a structure that
    fulfill the questionnaire importer synthax.

    Parameters
    ----------
    fsstatdirs: dict (mandatory)
        the location of the FreeSurfer stats directories with the timepoints
        as key.
    study_name: str (mandatory)
        the name of the study.
    subject_age_map: dict (mandatory)
        a map between subject ids as keys and subject ages as
        values with the time points as a fist key.
    savedir: str (optional, defafult None)
        if a valid directory is specified write the generated structure as
        a json file.
    is_openanswer: bool (optional, default False)
        if True use types open answers.

    Returns
    -------
    questionnaires: dict
        the generated structure with the stat files information.
    """
    # Go through timepoints
    questionnaires = {}
    for timepoint, statdir in fsstatdirs.items():

        # List all csv stat files
        files = glob.glob(os.path.join(statdir, "*.csv"))

        # Go through stat files
        for filepath in files:

            # Parse the stat file
            with open(filepath, "rb") as csvfile:
                dialect = csv.Sniffer().sniff(csvfile.read())
                csvfile.seek(0)
                reader = csv.reader(csvfile, dialect)
                rows = [row for row in reader]
                headers = rows[0]
                qname = headers[0]
                for row in rows[1:]:
                    subject = row[0]
                    age = subject_age_map[timepoint][subject]
                    qstruct = {
                        "Questionnaires": {
                            qname: None
                        },
                        "Assessment": {
                            "age_of_subject": float(age),
                            "identifier": u"{0}_{1}_{2}_{3}".format(
                                    timepoint, qname.upper(),
                                    study_name.upper(), subject),
                            "timepoint": unicode(timepoint)
                        }
                    }
                    if is_openanswer:
                        qstruct["Questionnaires"][qname] = {
                            "{0}: float".format(k): float(v)
                            for k, v in zip(headers[1:], row[1:])}
                    else:
                        qstruct["Questionnaires"][qname] = {
                            k: float(v) for k, v in zip(headers[1:], row[1:])}
                    questionnaires.setdefault(subject, []).append(qstruct)

    # Save the generated structure
    if savedir is not None and os.path.isdir(savedir):
        date = datetime.date.today()
        save_file = os.path.join(savedir, "freesurfer_stats_{0}.json".format(
            date.isoformat()))
        with open(save_file, "w") as openfile:
            json.dump(questionnaires, openfile, indent=4, sort_keys=True)

    return questionnaires

def freesurfer(fsdirs, study_name, subject_pattern, tool_version,
               tool_parameters=None, savedir=None, rql_template=RQL_T1):
    """ Parse the freesurfer files and create a structure that
    fulfill the processing importer synthax.

    Parameters
    ----------
    fsdirs: dict (mandatory)
        the location of the FreeSurfer directory with the
        timepoints as key.
    study_name: str (mandatory)
        the name of the study.
    subject_pattern: str (mandatory)
        a pattern used to extract the subject name from the FreeSurfer
        directories names.
    tool_version: str (mandatory)
        the FreeSurfer tool version.
    tool_parameters: object (optional, defafult None)
        a structure describing the used options.
    savedir: str (optional, defafult None)
        if a valid directory is specified write the generated structure as
        a json file.
    rql_template: str (optional, default RQL_T1)
        the rql used to retrieve the t1 scan attached to a FreeSurfer
        processing.

    Returns
    -------
    processings: dict
        the generated structure with the FreeSurfeer files information.
    """
    # Go through timepoints
    processings = {}
    for timepoint, fsdir in fsdirs.items():

        # Create an assessment
        assessment_id = u"{0}_FREESURFER_{1}".format(
            timepoint, study_name.upper())
        assessment_struct = {
            "age_of_subject": 0,
            "identifier": assessment_id,
            "timepoint": unicode(timepoint)
        }

        # Go through subjects
        for subject in os.listdir(fsdir):

            # Get the subject code
            subjectfsdir = os.path.join(fsdir, subject)
            if len(re.findall(subject_pattern, subject)) != 1:
                print("Skip '{0}' since no valid subject can be extracted "
                      "with pattern '{1}'.".format(subjectfsdir,
                                                   subject_pattern))
                continue

            # Build a RQL to get the input T1
            rql_t1 = rql_template.format(subject, timepoint)

            # Create the processingrun: multiple filsets
            processingrun_id = u"{0}_{1}".format(assessment_id, subject)
            processingrun_struct = {
                "identifier": processingrun_id,
                "label": u"FreeSurfer",
                "tool": u"FreeSurfer",
                "version": unicode(tool_version),
                "parameters": unicode(json.dumps(tool_parameters))
            }
            fileset_structs = []
            extresources_structs = []

            # Go through FreeSurfer subdirectories
            for dirname in os.listdir(subjectfsdir):
                fsetpath = os.path.join(subjectfsdir, dirname)

                # If subdirectory not empty
                if os.path.isdir(fsetpath) and len(os.listdir(fsetpath)) > 0:

                    # Create a fileset
                    fset_id = u"{0}_{1}".format(processingrun_id, dirname)
                    fileset_structs.append({
                        "identifier": fset_id,
                        "name": u"FreeSurfer {0}".format(dirname.upper())})
                    extresources_structs.append([])

                    # Create the external files
                    for root, dirs, files in os.walk(fsetpath):
                        for fname in files:
                            fpath = os.path.join(root, fname)
                            file_struct = {
                                "identifier": unicode(fpath),
                                "absolute_path": True,
                                "name": unicode(fname),
                                "filepath": unicode(fpath)
                            }
                            extresources_structs[-1].append(file_struct)

            # Create the final structure
            processing_struct = {
                "Assessment": copy.deepcopy(assessment_struct),
                "Processings": [{
                    "Inputs": [rql_t1],
                    "ExternalResources": extresources_structs,
                    "FileSets": fileset_structs,
                    "ProcessingRun": processingrun_struct
                }]
            }
            processings.setdefault(subject, []).append(processing_struct)

    # Save the generated structure
    if savedir is not None and os.path.isdir(savedir):
        date = datetime.date.today()
        save_file = os.path.join(savedir, "freesurfer_{0}.json".format(
            date.isoformat()))
        with open(save_file, "w") as openfile:
            json.dump(processings, openfile, indent=4, sort_keys=True)

    return processings


import os
import csv
import requests

crime = 'SIMPLE ASSAULT'
full_csv = open('../crime_data/Full_' + crime + '.csv', 'w')

def get_long_lat(addr):
    response = requests.get(
        'https://maps.googleapis.com/maps/api/geocode/json?address=' + addr + ',+Milwaukee,+WI')
    resp_json_payload = response.json()
    return resp_json_payload['results'][0]['geometry']['location']

def add_long_lat(fn):
    global crime
    global full_csv

    try:
        with open(fn) as csv_file:
            for row in csv.reader(csv_file, delimiter=','):
                if row[4] != crime:
                    continue
                if row[9] != '' and row[9] != 'LOCATION':
                    addr = row[9].replace(' ', '+')
                    print(addr)
                    loc = get_long_lat(addr)
                    row.append(str(loc['lat']))
                    row.append(str(loc['lng']))
                    row.append(fn.strip('.csv'))
                    full_csv.write(",".join(row) + '\n')
    except Exception as e:
        print(e)

#loop through each alderman district
for fn in os.listdir('../crime_data'):
    add_long_lat('../crime_data/' + fn)
full_csv.close()


from replit import clear
from art import logo
#HINT: You can call clear() to clear the output in the console.
new_dictionary = {}
checker = 0
winner = ""
should_continue = True
def auction_dictionary(name, bid):
  new_dictionary[str(name)] = bid

print(logo)
print("Welcome to the blind auction")

while should_continue:
 name =input("What is your name? ")
 bid = int(input("what is your bid? $"))
 auction_dictionary(name,bid)
 choice = input("Do you want to continue? Type Yes or No ").lower()
 if choice == "yes":
   clear()
 elif choice == "no":
   should_continue = False

for key in new_dictionary:
  score = new_dictionary[key]
  if score > checker:
    checker = score
    winner = key

print(f"The highest bidder is {winner} with a bid of ${checker}.")


import sys
from collections import deque

def direction(q, maps, x, y):
    dx = [0, 1, 0, -1]
    dy = [1, 0, -1, 0]
    cnt = 0
    for i in range(4):
        nx = x + dx[i]
        ny = y + dy[i]
        if -1 < nx < len(maps) and -1 < ny < len(maps) and maps[ny][nx] == 1:
            q.append((nx, ny))
            cnt += 1
            maps[ny][nx] = 0
    return cnt

def bfs(maps, y, x):
    q = deque()
    q.append((x, y))
    cnt = 1
    maps[y][x] = 0

    while q:
        (x, y) = q.popleft()
        cnt += direction(q, maps, x, y)

    return cnt

def solution(N: int, maps: list):
    cnt = 0
    answer = []
    for i in range(N):
        for j in range(N):
            if maps[i][j] == 1:
                result = bfs(maps, i, j)
                answer.append(result)
                cnt += 1
    answer.sort()

    return (cnt, answer)

def main():
    N = int(sys.stdin.readline())
    maps = []
    for i in range(N):
        maps.append(list(map(int, str(sys.stdin.readline().split()[0]))))

    (cnt, answer) = solution(N, maps)
    print(cnt)
    for element in answer:
        print(element)

if __name__ == "__main__":
    main()


import socket
import pickle
import dill

class Network:
    def __init__(self):
        self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # 소켓 생성
        self.server = "192.168.0.127"
        self.port = 10134
        self.addr = (self.server, self.port)
        self.p = self.connect()

    def getP(self):
        return self.p

    def connect(self):
        try:
            self.client.connect(self.addr) # 서버에 연결
            return self.client.recv(4096 * 128).decode() # 서버로부터 데이터를 받는다
        except:
            pass

    def send(self, data):
        try:
            self.client.send(str.encode(data)) # 데이터를 인코딩해서 서버에 보낸다
            received_data = self.client.recv(4096 * 128) # 서버로부터 데이터를 받는다

            # if not received_data:
            #     print(">> No data received")
            #     return None

            return dill.loads(received_data) # 받은 데이터를 디코딩해서 반환한다
        except socket.error as e:
            print(e)


import six

from holster.enum import Enum

from disco.util.snowflake import to_snowflake
from disco.util.functional import cached_property, one_or_many, chunks
from disco.types.user import User
from disco.types.base import SlottedModel, Field, snowflake, enum, listof, dictof, text
from disco.types.permissions import Permissions, Permissible, PermissionValue
from disco.voice.client import VoiceClient

ChannelType = Enum(
    GUILD_TEXT=0,
    DM=1,
    GUILD_VOICE=2,
    GROUP_DM=3,
)

PermissionOverwriteType = Enum(
    ROLE='role',
    MEMBER='member'
)

class ChannelSubType(SlottedModel):
    channel_id = Field(None)

    @cached_property
    def channel(self):
        return self.client.state.channels.get(self.channel_id)

class PermissionOverwrite(ChannelSubType):
    """
    A PermissionOverwrite for a :class:`Channel`.

    Attributes
    ----------
    id : snowflake
        The overwrite ID
    type : :const:`disco.types.channel.PermissionsOverwriteType`
        The overwrite type
    allowed : :class:`PermissionValue`
        All allowed permissions
    denied : :class:`PermissionValue`
        All denied permissions
    """
    id = Field(snowflake)
    type = Field(enum(PermissionOverwriteType))
    allow = Field(PermissionValue)
    deny = Field(PermissionValue)

    channel_id = Field(snowflake)

    @classmethod
    def create(cls, channel, entity, allow=0, deny=0):
        from disco.types.guild import Role

        ptype = PermissionOverwriteType.ROLE if isinstance(entity, Role) else PermissionOverwriteType.MEMBER
        return cls(
            client=channel.client,
            id=entity.id,
            type=ptype,
            allow=allow,
            deny=deny,
            channel_id=channel.id
        ).save()

    def save(self):
        self.client.api.channels_permissions_modify(self.channel_id,
                                                    self.id,
                                                    self.allow.value or 0,
                                                    self.deny.value or 0,
                                                    self.type.name)
        return self

    def delete(self):
        self.client.api.channels_permissions_delete(self.channel_id, self.id)

class Channel(SlottedModel, Permissible):
    """
    Represents a Discord Channel.

    Attributes
    ----------
    id : snowflake
        The channel ID.
    guild_id : Optional[snowflake]
        The guild id this channel is part of.
    name : str
        The channel's name.
    topic : str
        The channel's topic.
    position : int
        The channel's position.
    bitrate : int
        The channel's bitrate.
    recipients: list(:class:`disco.types.user.User`)
        Members of this channel (if this is a DM channel).
    type : :const:`ChannelType`
        The type of this channel.
    overwrites : dict(snowflake, :class:`disco.types.channel.PermissionOverwrite`)
        Channel permissions overwrites.
    """
    id = Field(snowflake)
    guild_id = Field(snowflake)
    name = Field(text)
    topic = Field(text)
    last_message_id = Field(snowflake)
    position = Field(int)
    bitrate = Field(int)
    recipients = Field(listof(User))
    type = Field(enum(ChannelType))
    overwrites = Field(dictof(PermissionOverwrite, key='id'), alias='permission_overwrites')

    def __init__(self, *args, **kwargs):
        super(Channel, self).__init__(*args, **kwargs)

        self.attach(six.itervalues(self.overwrites), {'channel_id': self.id, 'channel': self})

    def get_permissions(self, user):
        """
        Get the permissions a user has in the channel.

        Returns
        -------
        :class:`disco.types.permissions.PermissionValue`
            Computed permission value for the user.
        """
        if not self.guild_id:
            return Permissions.ADMINISTRATOR

        member = self.guild.members.get(user.id)
        base = self.guild.get_permissions(user)

        for ow in six.itervalues(self.overwrites):
            if ow.id != user.id and ow.id not in member.roles:
                continue

            base -= ow.deny
            base += ow.allow

        return base

    @property
    def mention(self):
        return '<#{}>'.format(self.id)

    @property
    def is_guild(self):
        """
        Whether this channel belongs to a guild.
        """
        return self.type in (ChannelType.GUILD_TEXT, ChannelType.GUILD_VOICE)

    @property
    def is_dm(self):
        """
        Whether this channel is a DM (does not belong to a guild).
        """
        return self.type in (ChannelType.DM, ChannelType.GROUP_DM)

    @property
    def is_voice(self):
        """
        Whether this channel supports voice.
        """
        return self.type in (ChannelType.GUILD_VOICE, ChannelType.GROUP_DM)

    @property
    def messages(self):
        """
        a default :class:`MessageIterator` for the channel.
        """
        return self.messages_iter()

    @cached_property
    def guild(self):
        """
        Guild this channel belongs to (if relevant).
        """
        return self.client.state.guilds.get(self.guild_id)

    def messages_iter(self, **kwargs):
        """
        Creates a new :class:`MessageIterator` for the channel with the given
        keyword arguments.
        """
        return MessageIterator(self.client, self, **kwargs)

    def get_message(self, message):
        return self.client.api.channels_messages_get(self.id, to_snowflake(message))

    def get_invites(self):
        """
        Returns
        -------
        list(:class:`disco.types.invite.Invite`)
            All invites for this channel.
        """
        return self.client.api.channels_invites_list(self.id)

    def create_invite(self, *args, **kwargs):
        from disco.types.invite import Invite
        return Invite.create(self, *args, **kwargs)

    def get_pins(self):
        """
        Returns
        -------
        list(:class:`disco.types.message.Message`)
            All pinned messages for this channel.
        """
        return self.client.api.channels_pins_list(self.id)

    def create_pin(self, message):
        self.client.api.channels_pins_create(self.id, to_snowflake(message))

    def delete_pin(self, message):
        self.client.api.channels_pins_delete(self.id, to_snowflake(message))

    def get_webhooks(self):
        return self.client.api.channels_webhooks_list(self.id)

    def create_webhook(self, name=None, avatar=None):
        return self.client.api.channels_webhooks_create(self.id, name, avatar)

    def send_message(self, content, nonce=None, tts=False):
        """
        Send a message in this channel.

        Parameters
        ----------
        content : str
            The message contents to send.
        nonce : Optional[snowflake]
            The nonce to attach to the message.
        tts : Optional[bool]
            Whether this is a TTS message.

        Returns
        -------
        :class:`disco.types.message.Message`
            The created message.
        """
        return self.client.api.channels_messages_create(self.id, content, nonce, tts)

    def connect(self, *args, **kwargs):
        """
        Connect to this channel over voice.
        """
        assert self.is_voice, 'Channel must support voice to connect'
        vc = VoiceClient(self)
        vc.connect(*args, **kwargs)
        return vc

    def create_overwrite(self, *args, **kwargs):
        return PermissionOverwrite.create(self, *args, **kwargs)

    def delete_message(self, message):
        """
        Deletes a single message from this channel.

        Args
        ----
        message : snowflake|:class:`disco.types.message.Message`
            The message to delete.
        """
        self.client.api.channels_messages_delete(self.id, to_snowflake(message))

    @one_or_many
    def delete_messages(self, messages):
        """
        Deletes a set of messages using the correct API route based on the number
        of messages passed.

        Args
        ----
        messages : list[snowflake|:class:`disco.types.message.Message`]
            List of messages (or message ids) to delete. All messages must originate
            from this channel.
        """
        messages = map(to_snowflake, messages)

        if not messages:
            return

        if self.can(self.client.state.me, Permissions.MANAGE_MESSAGES) and len(messages) > 2:
            for chunk in chunks(messages, 100):
                self.client.api.channels_messages_delete_bulk(self.id, chunk)
        else:
            for msg in messages:
                self.delete_message(msg)

    def delete(self):
        assert (self.is_dm or self.guild.can(self.client.state.me, Permissions.MANAGE_GUILD)), 'Invalid Permissions'
        self.client.api.channels_delete(self.id)

    def close(self):
        """
        Closes a DM channel. This is intended as a safer version of `delete`,
        enforcing that the channel is actually a DM.
        """
        assert self.is_dm, 'Cannot close non-DM channel'
        self.delete()

class MessageIterator(object):
    """
    An iterator which supports scanning through the messages for a channel.

    Parameters
    ----------
    client : :class:`disco.client.Client`
        The disco client instance to use when making requests.
    channel : `Channel`
        The channel to iterate within.
    direction : :attr:`MessageIterator.Direction`
        The direction in which this iterator will move.
    bulk : bool
        If true, this iterator will yield messages in list batches, otherwise each
        message will be yield individually.
    before : snowflake
        The message to begin scanning at.
    after : snowflake
        The message to begin scanning at.
    chunk_size : int
        The number of messages to request per API call.
    """
    Direction = Enum('UP', 'DOWN')

    def __init__(self, client, channel, direction=Direction.UP, bulk=False, before=None, after=None, chunk_size=100):
        self.client = client
        self.channel = channel
        self.direction = direction
        self.bulk = bulk
        self.before = before
        self.after = after
        self.chunk_size = chunk_size

        self.last = None
        self._buffer = []

        if not any((before, after)) and self.direction == self.Direction.DOWN:
            raise Exception('Must specify either before or after for downward seeking')

    def fill(self):
        """
        Fills the internal buffer up with :class:`disco.types.message.Message` objects from the API.
        """
        self._buffer = self.client.api.channels_messages_list(
                self.channel.id,
                before=self.before,
                after=self.after,
                limit=self.chunk_size)

        if not len(self._buffer):
            raise StopIteration

        self.after = None
        self.before = None

        if self.direction == self.Direction.UP:
            self.before = self._buffer[-1].id

        else:
            self._buffer.reverse()
            self.after = self._buffer[-1].id

    def next(self):
        return self.__next__()

    def __iter__(self):
        return self

    def __next__(self):
        if not len(self._buffer):
            self.fill()

        if self.bulk:
            res = self._buffer
            self._buffer = []
            return res
        else:
            return self._buffer.pop()


import os
import re
from abc import ABC
import difflib

import torch
from gensim import corpora
from gensim import  models
import itertools

import copy
import json
import os
import re
import warnings
from collections import OrderedDict, UserDict
from collections.abc import Mapping, Sized
from contextlib import contextmanager
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional, Sequence, Tuple, Union

import numpy as np
from packaging import version

from torch import TensorType
from transformers import PreTrainedTokenizerBase, BatchEncoding
from transformers.tokenization_utils_base import EncodedInput
from transformers.utils import PaddingStrategy, is_tf_tensor, is_torch_tensor, to_py_obj

from utils.names import EX_FEATURE_NAMES

def get_custom_tokenizer(tokenizer: PreTrainedTokenizerBase):
    class CustomTokenizer(tokenizer.__class__, ABC):
        def _pad(self, encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding], max_length: Optional[int] = None,
                 padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD, pad_to_multiple_of: Optional[int] = None,
                 return_attention_mask: Optional[bool] = None) -> dict:

            if return_attention_mask is None:
                return_attention_mask = "attention_mask" in self.model_input_names

            required_input = encoded_inputs[self.model_input_names[0]]

            if padding_strategy == PaddingStrategy.LONGEST:
                max_length = len(required_input)

            if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
                max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

            needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD

            # Initialize attention mask if not present.
            if return_attention_mask and "attention_mask" not in encoded_inputs:
                encoded_inputs["attention_mask"] = [1] * len(required_input)

            if needs_to_be_padded:
                difference = max_length - len(required_input)

                if self.padding_side == "right":
                    if return_attention_mask:
                        encoded_inputs["attention_mask"] = encoded_inputs["attention_mask"] + [0] * difference
                    if "token_type_ids" in encoded_inputs:
                        token_type_ids_difference = max_length - len(encoded_inputs["token_type_ids"])
                        encoded_inputs["token_type_ids"] = (
                            encoded_inputs["token_type_ids"] + [self.pad_token_type_id] * token_type_ids_difference
                        )
                    if "special_tokens_mask" in encoded_inputs:
                        encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"] + [1] * difference
                    for custom_input in EX_FEATURE_NAMES:
                        if custom_input in encoded_inputs:
                            custom_difference = max_length - len(encoded_inputs[custom_input])
                            encoded_inputs[custom_input] = encoded_inputs[custom_input] + [0] * custom_difference
                    encoded_inputs[self.model_input_names[0]] = required_input + [self.pad_token_id] * difference
                elif self.padding_side == "left":
                    if return_attention_mask:
                        encoded_inputs["attention_mask"] = [0] * difference + encoded_inputs["attention_mask"]
                    if "token_type_ids" in encoded_inputs:
                        token_type_ids_difference = max_length - len(encoded_inputs["token_type_ids"])
                        encoded_inputs["token_type_ids"] = [self.pad_token_type_id] * token_type_ids_difference + encoded_inputs[
                            "token_type_ids"
                        ]
                    if "special_tokens_mask" in encoded_inputs:
                        encoded_inputs["special_tokens_mask"] = [1] * difference + encoded_inputs["special_tokens_mask"]
                    for custom_input in EX_FEATURE_NAMES:
                        if custom_input in encoded_inputs:
                            custom_difference = max_length - len(encoded_inputs[custom_input])
                            encoded_inputs[custom_input] = [0] * custom_difference + encoded_inputs[custom_input]
                    encoded_inputs[self.model_input_names[0]] = [self.pad_token_id] * difference + required_input
                else:
                    raise ValueError("Invalid padding strategy:" + str(self.padding_side))

            return encoded_inputs

    tokenizer.__class__ = CustomTokenizer
    return tokenizer

class TF_IDFExtractor:
    @staticmethod
    def get_token_tfidf_list(token_id_lists):
        texts = [[str(word) for word in document] for document in token_id_lists]
        processed_corpus = texts
        dictionary = corpora.Dictionary(processed_corpus)
        bow_corpus = [dictionary.doc2bow(text) for text in processed_corpus]
        tfidf = models.TfidfModel(bow_corpus)
        word_tfidf_list = []
        id2token = {item[1]: item[0] for item in dictionary.token2id.items()}
        for text in processed_corpus:
            word_tfidf = {id2token[_[0]]: _[1] for _ in tfidf[dictionary.doc2bow(text)]}
            word_tfidf_list.append([word_tfidf.get(token, 0) for token in text])
        return word_tfidf_list

    @staticmethod
    def get_keyword_list(token_id_lists, tfidf_list):
        keyword_list = []
        for instance_num, instance_tfidf in enumerate(tfidf_list):
            keyword_index = instance_tfidf.index(max(instance_tfidf))
            keyword_list.append([token_id_lists[instance_num][keyword_index]])
        return keyword_list

    @staticmethod
    def batch_analyze(batch, token_field, **kwargs):
        tfidf_list =  TF_IDFExtractor.get_token_tfidf_list(batch[token_field])
        batch['tfidfs'] = tfidf_list
        keyword_list = TF_IDFExtractor.get_keyword_list(batch[token_field], tfidf_list)
        batch['keyword_ids'] = keyword_list
        return batch

def singleton(class_):
    instances = {}
    def getinstance(*args, **kwargs):
        if class_ not in instances:
            instances[class_] = class_(*args, **kwargs)
        return instances[class_]
    return getinstance

@singleton
class WordClean:
    def __init__(self):
        import nltk
        nltk.download('wordnet')
        nltk.download('omw-1.4')
        from nltk.corpus import stopwords
        nltk.download('stopwords')
        self.stoplist = set(stopwords.words('english'))
        self.stoplist.update(('cm', 'kg', 'mr', 'wa', 'nv', 'ore', 'da', 'pm', 'am', 'cx'))
        self.stoplist.remove('not')
        self.APPOS_LOWERCASE = {"ain't": "am not", "aren't": "are not", "can't": "cannot",
                 "can't've": "cannot have", "'cause": "because",
                 "could've": "could have", "couldn't": "could not",
                 "couldn't've": "could not have", "didn't": "did not",
                 "doesn't": "does not", "don't": "do not", "hadn't": "had not",
                 "hadn't've": "had not have", "hasn't": "has not",
                 "haven't": "have not", "he'd": "he would", "he'd've": "he would have",
                 "he'll": "he will", "he'll've": "he will have",
                 "he's": "he is", "how'd": "how did",
                 "how'd'y": "how do you", "how'll": "how will", "how's": "how is",
                 "i'd": "i would", "i'd've": "i would have", "i'll": "i will",
                                "i'll've": "i will have", "i'm": "i am", "i've": "i have",
                                "isn't": "is not", "it'd": "it would", "it'd've": "it would have",
                                "it'll": "it will", "it'll've": "it will have", "it's": "it is",
                                "let's": "let us", "ma'am": "madam", "mayn't": "may not",
                                "might've": "might have", "mightn't": "might not",
                                "mightn't've": "might not have", "must've": "must have",
                                "mustn't": "must not", "mustn't've": "must not have",
                                "needn't": "need not", "needn't've": "need not have",
                                "o'clock": "of the clock", "oughtn't": "ought not",
                                "oughtn't've": "ought not have", "shan't": "shall not",
                                "sha'n't": "shall not", "shan't've": "shall not have",
                                "she'd": "she would", "she'd've": "she would have",
                                "she'll": "she will", "she'll've": "she will have",
                                "she's": "she is", "should've": "should have",
                                "shouldn't": "should not", "shouldn't've": "should not have",
                                "so've": "so have", "so's": "so is",
                                "that'd": "that had", "that'd've": "that would have",
                                "that's": "that that is", "there'd": "there would",
                                "there'd've": "there would have", "there's": "there is",
                                "they'd": "they would", "they'd've": "they would have",
                                "they'll": "they will", "they'll've": "they will have",
                                "they're": "they are", "they've": "they have",
                                "to've": "to have", "wasn't": "was not", "we'd": "we would",
                                "we'd've": "we would have", "we'll": "we will",
                                "we'll've": "we will have", "we're": "we are",
                                "we've": "we have", "weren't": "were not",
                                "what'll": "what will", "what'll've": "what will have",
                                "what're": "what are", "what's": "what is",
                                "what've": "what have", "when's": "when is",
                                "when've": "when have", "where'd": "where did",
                                "where's": "where is", "where've": "where have",
                                "who'll": "who will", "who'll've": "who will have",
                                "who's": "who is", "who've": "who have",
                                "why's": "why is", "why've": "why have", "will've": "will have",
                                "won't": "will not", "won't've": "will not have",
                                "would've": "would have", "wouldn't": "would not",
                                "wouldn't've": "would not have", "y'all": "you all",
                                "y'all'd": "you all would", "y'all'd've": "you all would have",
                                "y'all're": "you all are", "y'all've": "you all have",
                                "you'd": "you would", "you'd've": "you would have",
                                "you'll": "you will", "you'll've": "you will have",
                                "you're": "you are", "you've": "you have"}
        self.PUNCT_TO_REMOVE = '!"#$%&\'()*+,./:;<=>?@[\\]^_`{|}~'

    def _clean(self, text):
        text = text.lower()
        # remove html
        regex = re.compile(r'<[^>]+>')
        text = regex.sub('', text)
        # replace_appos
        cleaned_words = []
        for word in text.split():
            if word in self.APPOS_LOWERCASE.keys():
                cleaned_words.append(self.APPOS_LOWERCASE[word])
            else:
                cleaned_words.append(word)
        text = ' '.join(cleaned_words)
        # remove_stopwords
        # text = " ".join([word for word in str(text).split() if word not in self.stoplist])
        # remove punc
        text = text.translate(str.maketrans('', '', self.PUNCT_TO_REMOVE))
        return text

    def _simple_clean(self, text):

        clean_line = ""

        text = text.replace("’", "")
        text = text.replace("'", "")
        text = text.replace("-", " ")  # replace hyphens with spaces
        text = text.replace("\t", " ")
        text = text.replace("\n", " ")
        text = text.lower()

        for char in text:
            if char in 'qwertyuiopasdfghjklzxcvbnm ':
                clean_line += char
            else:
                clean_line += ' '

        clean_line = re.sub(' +', ' ', clean_line)  # delete extra spaces
        if len(clean_line) and clean_line[0] == ' ':
            clean_line = clean_line[1:]
        return clean_line

    def __call__(self, texts):
        if isinstance(texts, str):
            return self._simple_clean(texts)
        else:
            return [self._simple_clean(text) for text in texts]

def mark_deleted_words(a, b):
    prev_s = None
    text = ""
    for i, s in enumerate(difflib.ndiff(a, b)):
        if s[0] == ' ':
            if prev_s == ' ':
                text += s[-1]
            elif prev_s == '+':
                text += "]" + s[-1]
            elif prev_s == '-':
                text += ")" + s[-1]
            else:
                text += s[-1]
        elif s[0] == '-':
            if prev_s == '-':
                text += s[-1]
            elif prev_s == ' ':
                text += "(" + s[-1]
            elif prev_s == '+':
                text += "] (" + s[-1]
            else:
                text += s[-1]
        elif s[0] == '+':
            if prev_s == '+':
                text += s[-1]
            elif prev_s == ' ':
                text += "[" + s[-1]
            elif prev_s == '+':
                text += ") [" + s[-1]
            else:
                text += s[-1]
        prev_s = s[0]
    return text

def mark_replaced_word(a, b):
    a_words = a.split()
    b_words = b.split()
    result = []
    i = 0
    j = 0
    while i < len(a_words) and j < len(b_words):
        if a_words[i] == b_words[j]:
            result.append(b_words[j])
            i += 1
            j += 1
        else:
            replaced = []
            while j < len(b_words) and b_words[j] not in a_words:
                replaced.append(b_words[j])
                j += 1
            if len(replaced) > 0:
                result.append('【' + " ".join(replaced) + '】')
            else:
                result.append(b_words[j])
                j += 1
    while j < len(b_words):
        result.append(b_words[j])
        j += 1
    return " ".join(result)


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="dilated-attention",
    version="0.1",
    description="",
    license='MIT',
    long_description=long_description,
    long_description_content_type="text/markdown",
    author = 'Boris Albar',
    author_email = 'b.albar@catie.fr',
    url="https://github.com/catie-aq/dilated-attention/",
    packages=setuptools.find_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "Operating System :: OS Independent",
    ],
    python_requires='>=3.7',
    install_requires=[
    ]
)


# -*- coding: utf-8 -*-
"""
Created on Fri Sep 30 11:18:36 2022

@author: mc16535
"""

import matplotlib.pyplot as plt
from matplotlib.widgets import Slider
import numpy as np
import scipy.interpolate as sit
import scipy.io as sio
import os

os.chdir(r"C:\Users\mc16535\OneDrive - University of Bristol\Documents\Postgrad\Experimental Data\2022\L-shape FMCs with 1mm SDHs\22.09.12 - test + ind sample\FE")
mat = sio.loadmat("a_0mm_d_1_b_BP.mat")

data = np.abs(mat['data']).T
time = mat['time'][:, 0]

# Reduce resolution
grid_x, grid_y   = np.meshgrid(time, range(data.shape[0]))
ngrid_x, ngrid_y = np.meshgrid(time[::2], range(data.shape[0])[::4])
pts = np.asarray([grid_x.ravel(), grid_y.ravel()]).T
vls = data.ravel()
grid_z = sit.griddata(pts, vls, (ngrid_x, ngrid_y), method='nearest')

max_, min_ = np.max(grid_z)/10, np.min(grid_z)
threshold  = (max_ + min_) / 2
mask = np.ones(grid_z.shape)
mask[grid_z < threshold] = 0

fig = plt.figure()
plt.subplots_adjust(bottom=0.25)
ax = fig.subplots()
p = ax.imshow(mask, cmap="gist_yarg", extent=[time[0], time[-1], data.shape[1], 1], aspect='auto')

ax_slide = plt.axes([0.25, 0.1, 0.65, 0.03])

frame_slider = Slider(ax_slide, 'threshold',
                  min_, max_, valinit=(max_ + min_) / 2, valstep=(max_ + min_) / 50)

def update_frame(val):
    threshold = frame_slider.val
    mask[grid_z >= threshold] = 1
    mask[grid_z < threshold]  = 0
    p.set_data(mask)
    fig.canvas.draw()

frame_slider.on_changed(update_frame)
plt.show()

# fig = plt.figure()
# plt.subplots_adjust(bottom=0.25)
# ax = fig.subplots()
# p = ax.imshow(data[0, :, :], extent=[xlims[0], xlims[1], ylims[0], ylims[1]], vmin=0, vmax=1e-11)

# ax_slide = plt.axes([0.25, 0.1, 0.65, 0.03])

# frame_slider = Slider(ax_slide, 'Frame',
#                   0, 160, valinit=0, valstep=1)

# def update_frame(val):
#     current_v = frame_slider.val
#     p.set_data(data[current_v, :, :])
#     fig.canvas.draw()

frame_slider.on_changed(update_frame)
plt.show()