from typing import List, Optional, Tuple, TypedDict

import torch, torchvision
from torch.utils.data import DataLoader
from torchvision.transforms import ToTensor

from torchvision.models.detection.generalized_rcnn import GeneralizedRCNN
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
from torchvision.models.detection import MaskRCNN
from torchvision.models.detection.mask_rcnn import MaskRCNNPredictor

from vision.dataset import AbstractCocoDataset

from vision.dataset_factory import create_dataset

from vision.helpers.engine import train_one_epoch, evaluate
from vision.helpers import utils

from os import path
from constants import ROOT_DIR

from PIL import Image
from datetime import datetime

SAVED_MODELS_DIR = path.join(ROOT_DIR, "vision/saved_models/")

class EvalutationResult(TypedDict):
    categories: List[str]
    supercategories: List[str]
    scores: List[int]
    boxes: List[Tuple[float, float, float, float]]

class AbstractLitgoModel:
    model: GeneralizedRCNN
    dataset: AbstractCocoDataset

    def __init__(self, name: str) -> None:
        self.name

    def train(self, num_epochs: int, batch_size: int, test_batch_size: int):
        raise NotImplementedError

    def evaluate(self, image: Image, score_threshold: float) -> EvalutationResult:
        raise NotImplementedError

    def name(self):
        return self.name

    def save(self):
        timestamp = int(datetime.today().timestamp())
        state_filename = f"{self.dataset.get_name()}-{self.name()}-state-{timestamp}.pt"
        torch.save(self.model.state_dict(), path.join(SAVED_MODELS_DIR, state_filename))

    def load_saved_state(self, saved_model_filename: str):
        if not path.isfile(path.join(SAVED_MODELS_DIR, saved_model_filename)):
            raise Exception(f"Could not find {saved_model_filename} in saved_models directory")

        if torch.cuda.is_available():
            self.model.load_state_dict(torch.load(path.join(SAVED_MODELS_DIR, saved_model_filename)))
        else:
            self.model.load_state_dict(torch.load(
                path.join(SAVED_MODELS_DIR, saved_model_filename),
                map_location=torch.device('cpu')
            ))

class LitgoModel(AbstractLitgoModel):
    model: MaskRCNN

    def train(self, num_epochs: int, batch_size: int, test_batch_size: int):
        # get data loaders
        data_loader, data_loader_test = self._get_data_loaders(batch_size, test_batch_size)

        # train on the GPU or on the CPU, if a GPU is not available
        device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')

        # move model to the right device
        self.model.to(device)

        # construct an optimizer
        params = [p for p in self.model.parameters() if p.requires_grad]
        optimizer = torch.optim.SGD(params, lr=0.005,
                                    momentum=0.9, weight_decay=0.0005)
        # and a learning rate scheduler
        lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
                                                        step_size=3,
                                                        gamma=0.1)

        print("Starting Training...")
        for epoch in range(num_epochs):
            # train for one epoch, printing every 10 iterations
            train_one_epoch(self.model, optimizer, data_loader, device, epoch, print_freq=10)
            # update the learning rate
            lr_scheduler.step()
            # evaluate on the test dataset
            evaluate(self.model, data_loader_test, device=device)
        print("Done Training!")

    def evaluate(self, image: Image, score_threshold: float):
        self.model.eval()
        image_tensor = ToTensor()(image)

        with torch.no_grad():
            prediction = self.model([image_tensor])[0]

            # scores are guanteed to be in descending order so find left sublist that meets threshold
            cutoff = 0
            while prediction["scores"][cutoff] >= score_threshold: cutoff += 1 # TODO: Do binary search instead of linear
            labels = prediction["labels"][:cutoff].tolist()

            return EvalutationResult(
                categories=list(map(lambda l: self.dataset.get_category_from_label(l)[0],labels)),
                supercategories=list(map(lambda l: self.dataset.get_category_from_label(l)[1],labels)),
                scores= prediction["scores"][:cutoff].tolist(),
                boxes=prediction["boxes"][:cutoff].tolist()
            )

    def _get_data_loaders(self,  batch_size: int, test_batch_size: int) -> Tuple[DataLoader, DataLoader]:

        # split the dataset into train and validation sets
        indices = torch.randperm(len(self.dataset)).tolist()
        training_dataset = torch.utils.data.Subset(self.dataset, indices[:-50])
        validation_dataset = torch.utils.data.Subset(
            create_dataset(self.dataset.get_name(), train=True), indices[-50:]
        )

        # define training and validation data loaders
        data_loader = torch.utils.data.DataLoader(
            training_dataset, batch_size=batch_size, shuffle=True, num_workers=2,
            collate_fn=utils.collate_fn)

        data_loader_test = torch.utils.data.DataLoader(
            validation_dataset, batch_size=test_batch_size, shuffle=False, num_workers=2,
            collate_fn=utils.collate_fn)

        return data_loader, data_loader_test


import numpy as np
import matplotlib.pyplot as plt

# part a
# simulate a particle choosing 1 of 4 directions on a lattice of shape LxL, let L=50, and choosing a separate direction if
# the particle chooses to move outside the lattice
# defining conditions
step = 5000 # number of time steps, increase to 5000 later
time = np.arange(1,step+1,1)
L = 101 # originally 101
position_x = np.zeros(step,dtype=int)
position_x[0] = (L-1)//2
position_y = np.zeros(step,dtype=int)
position_y[0] = (L-1)//2

for i in range(step-1):
    # choose a random integer [1,4] from a uniform distribution (equal probability for each) to determine step direction
    choice = np.random.randint(1, high=5, size=None, dtype=int)
    if position_x[i] >= L:
        choice = 3
    if position_y[i] >= L:
        choice = 2
    if position_x[i] <= 0:
        choice = 4
    if position_y[i] <= 0:
        choice = 1
    #########################
    if choice == 1: # symbols 'up' direction
        position_x[i+1] = position_x[i]
        position_y[i+1] = position_y[i] + 1
    if choice == 2: # symbols 'down' direction
        position_x[i+1] = position_x[i]
        position_y[i+1] = position_y[i] - 1
    if choice == 3: # symbols 'left' direction
        position_x[i+1] = position_x[i] - 1
        position_y[i+1] = position_y[i]
    if choice == 4: # symbols 'right' direction
        position_x[i+1] = position_x[i] + 1
        position_y[i+1] = position_y[i]

fig, (ax0,ax1,ax2) = plt.subplots(figsize=(14,4),ncols=3,nrows=1)

ax0.plot(position_x,position_y)
ax0.set_title('x vs y position of particle in a 100x100 box')
ax0.set_xlabel('x position')
ax0.set_ylabel('y position')
ax1.plot(time,position_x)
ax1.set_title('position of particle in x direction over time')
ax2.plot(time,position_y)
ax2.set_title('position of particle in y direction over time')


import re
import json
import torch
import random
import numpy as np
from datetime import datetime
from util import print_time_info
from transformers import BertTokenizer, BertModel

from time_graph import build_value_edges_in_cluster, build_entity_and_value_edges
from time_graph import build_dynamic_value_edges_in_cluster, build_dynamic_time_graphs
from cluster import ClusterModel
from embeddings import BERT, ValueEmbedding, get_cache_file_path

def read_mapping(path):
    def _parser(lines):
        for idx, line in enumerate(lines):
            i, name = line.strip().split('\t')
            lines[idx] = (int(i), name)
        return dict(lines)

    return read_file(path, _parser)

def read_triples(path):
    '''
    triple pattern: (head_id, tail_id, relation_id)
    '''
    return read_file(path, lambda lines: [tuple([int(item) for item in line.strip().split('\t')]) for line in lines])

def read_seeds(path):
    return read_file(path, lambda lines: [tuple([int(item) for item in line.strip().split('\t')]) for line in lines])

def read_file(path, parse_func):
    num = -1
    with open(path, 'r', encoding='utf8') as f:
        line = f.readline().strip()
        if line.isdigit():
            num = int(line)
        else:
            f.seek(0)
        lines = f.readlines()

    lines = parse_func(lines)

    if len(lines) != num and num >= 0:
        print_time_info('File: %s has corruptted, data_num: %d/%d.' %
                        (path, num, len(lines)))
        raise ValueError()
    return lines

def _load_language(directory, language):
    triples = read_triples(directory / ('triples_' + language + '.txt'))
    id2entity = read_mapping(directory / ('id2entity_' + language + '.txt'))
    id2relation = read_mapping(directory / ('id2relation_' + language + '.txt'))
    return triples, id2entity, id2relation

def _load_seeds(directory, train_seeds_ratio, load_new_seed_split):
    train_data_path = directory / 'train_entity_seeds.txt'
    valid_data_path = directory / 'valid_entity_seeds.txt'
    test_data_path = directory / 'test_entity_seeds.txt'
    entity_seeds = read_seeds(directory / 'entity_seeds.txt')
    if load_new_seed_split:
        train_data_path = directory / 'hard_split' / 'train_entity_seeds.txt'
        valid_data_path = directory / 'hard_split' / 'valid_entity_seeds.txt'
        test_data_path = directory / 'hard_split' / 'test_entity_seeds.txt'
        print_time_info("Loading adversarially-splitted train/valid/test set from %s." % str(directory / 'hard_split'))
        train_entity_seeds = read_seeds(train_data_path)
        valid_entity_seeds = read_seeds(valid_data_path)
        test_entity_seeds = read_seeds(test_data_path)
    elif train_data_path.exists() and valid_data_path.exists() and test_data_path.exists():
        print_time_info("Loading pre-splitted train/valid/test set from %s." % str(directory))
        train_entity_seeds = read_seeds(train_data_path)
        valid_entity_seeds = read_seeds(valid_data_path)
        test_entity_seeds = read_seeds(test_data_path)
    else:
        test_sr_ids_path = directory / ('test_sr_ids_%d.txt' % int(train_seeds_ratio * 100))
        if not test_sr_ids_path.exists():
            print_time_info("Randomly split train/valid set from %s." % str(directory))
            tmp_entity_seeds = [seed for seed in entity_seeds]
            random.shuffle(tmp_entity_seeds)
            train_entity_seeds = tmp_entity_seeds[:int(len(entity_seeds) * train_seeds_ratio)]
            valid_entity_seeds = tmp_entity_seeds[int(len(entity_seeds) * train_seeds_ratio):]
            test_entity_seeds = valid_entity_seeds
            test_sr_ent_set = set(x[0] for x in test_entity_seeds)
            with open(test_sr_ids_path, 'w', encoding='utf8') as f:
                for idx in test_sr_ent_set:
                    f.write(str(idx) + '\n')
        else:
            print_time_info('Loading previously random splitted data set.')
            with open(test_sr_ids_path, 'r', encoding='utf8') as f:
                test_sr_ent_set = [int(line.strip()) for line in f.readlines()]
                test_sr_ent_set = set(test_sr_ent_set)
            train_entity_seeds = [seed for seed in entity_seeds if seed[0] not in test_sr_ent_set]
            valid_entity_seeds = [seed for seed in entity_seeds if seed[0] in test_sr_ent_set]
            test_entity_seeds = valid_entity_seeds
    return train_entity_seeds, valid_entity_seeds, test_entity_seeds, entity_seeds

def _load_trans(directory, language):
    with open(directory / ('id2trans_%s.txt' % language), 'r') as f:
        lines = [line.strip().split('\t') for line in f.readlines()]
    id2trans = {int(idx): sr_trans for idx, sr_trans, sr_ent in lines}
    return id2trans

def _load_dbpedia_properties(data_path, entity2id, language, filter_alias=False):
    # filter_name: mask all the attribute that is potentially an alias of the entity
    potential_alias_pattern = ['name', 'alias', '名字', '别名']

    if language in {'en', 'zh', 'ja', 'fr'}:
        Prefix.set_language(language)
        remove_prefix = Prefix.remove_prefix
    else:
        remove_prefix = lambda x: x

    with open(data_path, 'r', encoding='utf8') as f:
        lines = [line.strip().split('\t') for line in f.readlines()]
    att_triples = []
    for line in lines:
        try:
            subject, property, value, _ = line
        except ValueError:
            subject, property, value = line
        subject = remove_prefix(subject)

        # filter the alias
        if filter_alias:
            for alias in potential_alias_pattern:
                if property.lower().find(alias) >= 0:
                    value = ''
        try:
            value = value.encode('utf8').decode('unicode_escape')
        except UnicodeDecodeError:
            pass
        ent_id = entity2id[subject]
        att = property
        att_triples.append((ent_id, value, att))
    # For all the triples: (head, tail, relation)
    return att_triples

def _get_train_value_and_attribute(train_ent_ids, att_triples):
    train_value_and_attribute = []
    train_ent_ids = set(train_ent_ids)
    for ent_id, value, att in att_triples:
        if ent_id in train_ent_ids:
            train_value_and_attribute.append((value, att))
    return train_value_and_attribute

class TimeExpression(object):
    def __init__(self):
        self.regex = {'year': re.compile(r'^\d{4}$'), 'date': re.compile(r'^(\d+)-(\d+)-(\d+)$'),
                      'month_day': re.compile(r'^--(\d{2})-(\d{2})$')}
        self.regex_func = {'year': lambda x: (int(x.group(0)), None, None),
                           'date': lambda x: (int(x.group(1)), int(x.group(2)), int(x.group(3))),
                           'month_day': lambda x: (None, int(x.group(1)), int(x.group(2)))}

    def is_time_exp(self, text):
        for regex_name in self.regex:
            is_numeral, result = self.__regex_pattern(text, regex_name)
            if is_numeral:
                return is_numeral, result
        is_numeral, result = self.__month_year_pattern(text)
        if is_numeral:
            return is_numeral, result
        return False, None

    def __regex_pattern(self, text, regex_name):
        regex = self.regex[regex_name]
        result = regex.match(text)
        if result:
            return True, self.regex_func[regex_name](result)
        return False, None

    def __month_year_pattern(self, text):
        try:
            data = datetime.strptime(text, '%B %Y')
            return True, (data.year, data.month, None)
        except ValueError:
            return False, None
        except:
            raise Exception()

    # todo, we can have a more accuarete way to find time expressions

def _split_time_attribute(value_and_attribute_pairs, threshold, att_set):
    time_exp = TimeExpression()
    att_is_time = {}
    for value, att_id in value_and_attribute_pairs:
        # 0 digit, 1 literal
        is_time_exp, reuslt = time_exp.is_time_exp(value)
        if att_id in att_is_time:
            if is_time_exp:
                att_is_time[att_id][0] += 1
            else:
                att_is_time[att_id][1] += 1
        else:
            if is_time_exp:
                att_is_time[att_id] = [1, 0]
            else:
                att_is_time[att_id] = [0, 1]

    time_atts = {att for att, count in att_is_time.items() if count[0] / sum(count) > threshold}
    time_atts = list(time_atts)
    time_atts.sort()
    time_att2id = {digit_att: idx for idx, digit_att in enumerate(time_atts)}
    return time_att2id

def _split_digit_attribute_and_literal_attribute(value_and_attribute_pairs, digit_threshold, att_set):
    numeral = Numeral()
    att_is_number = {}
    for value, att_id in value_and_attribute_pairs:
        # 0 digit, 1 literal
        is_numeral, number = numeral.is_numeral(value)
        if att_id in att_is_number:
            if is_numeral:
                att_is_number[att_id][0] += 1
            else:
                att_is_number[att_id][1] += 1
        else:
            if is_numeral:
                att_is_number[att_id] = [1, 0]
            else:
                att_is_number[att_id] = [0, 1]

    digit_atts = {att for att, count in att_is_number.items() if count[0] / sum(count) > digit_threshold}
    literal_atts = {att for att in att_set if att not in digit_atts}

    digit_atts = list(digit_atts)
    digit_atts.sort()
    literal_atts = list(literal_atts)
    literal_atts.sort()
    digit_att2id = {digit_att: idx for idx, digit_att in enumerate(digit_atts)}
    literal_att2id = {literal_att: idx for idx, literal_att in enumerate(literal_atts)}
    return digit_att2id, literal_att2id

def _split_time_triple(att_triples, time_att2id):
    time_triples = []
    for ent_id, value, att in att_triples:
        if att in time_att2id:
            time_triples.append((ent_id, value, time_att2id[att]))
    return time_triples

def _split_digit_and_literal_triple(att_triples, digit_att2id, literal_att2id):
    digit_triples = []
    literal_triples = []
    digit_num = 0
    literal_num = 0
    for ent_id, value, att in att_triples:
        if att in digit_att2id:
            digit_triples.append((ent_id, value, digit_att2id[att]))
            digit_num += 1
        else:
            literal_triples.append((ent_id, value, literal_att2id[att]))
            literal_num += 1
    return digit_triples, literal_triples

class LoadData(object):
    def __init__(self, train_seeds_ratio, directory, nega_sample_num, name_channel,
                 attribute_channel, digit_literal_channel, time_channel,
                 load_new_seed_split=False,
                 device='cpu'):
        self.device = device
        self.directory = directory
        self.nega_sample_num = nega_sample_num
        self.train_seeds_ratio = train_seeds_ratio
        self.language_sr, self.language_tg = directory.name.split('_')
        self.load_seed_alignment(load_new_seed_split)
        self.load_structure_feature()
        if name_channel:
            self.load_name_feature()
        if attribute_channel or digit_literal_channel or time_channel:
            self.load_attribute_feature(attribute_channel, digit_literal_channel, time_channel)
        self.negative_sample()
        self.to_torch()

    def to_torch(self):
        self.valid_sr_ent_seeds = torch.from_numpy(self.valid_sr_ent_seeds).to(self.device)
        self.valid_tg_ent_seeds = torch.from_numpy(self.valid_tg_ent_seeds).to(self.device)

    def update_negative_sample(self, sr_nega_sample, tg_nega_sample):
        # nega sample shape = (data_len, negative_sample_num)
        assert sr_nega_sample.shape == (len(self.train_sr_ent_seeds_ori), self.nega_sample_num)
        assert tg_nega_sample.shape == (len(self.train_tg_ent_seeds_ori), self.nega_sample_num)

        if not (hasattr(self, "sr_posi_sample") and hasattr(self, "tg_posi_sample")):
            sr_posi_sample = np.tile(self.train_sr_ent_seeds_ori.reshape((-1, 1)), (1, self.nega_sample_num))
            tg_posi_sample = np.tile(self.train_tg_ent_seeds_ori.reshape((-1, 1)), (1, self.nega_sample_num))
            self.sr_posi_sample = torch.from_numpy(sr_posi_sample.reshape((-1, 1))).to(self.device)
            self.tg_posi_sample = torch.from_numpy(tg_posi_sample.reshape((-1, 1))).to(self.device)

        sr_nega_sample = sr_nega_sample.reshape((-1, 1))
        tg_nega_sample = tg_nega_sample.reshape((-1, 1))

        # sr_nega_sample = sr_nega_sample.to(self.device)
        # tg_nega_sample = tg_nega_sample.to(self.device)
        self.train_sr_ent_seeds = torch.cat((self.sr_posi_sample, sr_nega_sample), dim=1)
        self.train_tg_ent_seeds = torch.cat((self.tg_posi_sample, tg_nega_sample), dim=1)

    def negative_sample(self):
        # Randomly negative sample
        sr_nega_sample = negative_sample(self.train_sr_ent_seeds_ori, self.sr_ent_num, self.nega_sample_num)
        tg_nega_sample = negative_sample(self.train_tg_ent_seeds_ori, self.tg_ent_num, self.nega_sample_num)
        sr_nega_sample = torch.from_numpy(sr_nega_sample).to(self.device)
        tg_nega_sample = torch.from_numpy(tg_nega_sample).to(self.device)

        self.update_negative_sample(sr_nega_sample, tg_nega_sample)

    def load_structure_feature(self):
        # Load triples and entity mapping
        id2atts = read_mapping(self.directory / 'id2atts.txt')
        self.att2id = {att: idx for idx, att in id2atts.items()}
        self.att_num = len(self.att2id)
        self.triples_sr, self.id2entity_sr, self.id2relation_sr = _load_language(self.directory, self.language_sr)
        self.triples_tg, self.id2entity_tg, self.id2relation_tg = _load_language(self.directory, self.language_tg)
        self.sr_ent_num = len(self.id2entity_sr)
        self.tg_ent_num = len(self.id2entity_tg)

    def load_name_feature(self):
        # Load translations
        if self.language_sr in {'zh', 'ja', 'fr'}:
            id2trans_sr = _load_trans(self.directory, self.language_sr)
            id2trans_sr = sorted(id2trans_sr.items(), key=lambda x: x[0])
            sr_text = [x[1] for x in id2trans_sr]
        else:
            id2entity_sr = sorted(self.id2entity_sr.items(), key=lambda x: x[0])
            sr_text = [x[1] for x in id2entity_sr]
        id2entity_tg = sorted(self.id2entity_tg.items(), key=lambda x: x[0])
        tg_text = [x[1] for x in id2entity_tg]
        bert = BERT()
        bert.to(self.device)
        self.sr_embed = bert.pooled_encode_batched(sr_text, layer=1)
        self.tg_embed = bert.pooled_encode_batched(tg_text, layer=1)
        del bert

    def load_seed_alignment(self, load_new_seed_split):
        # Load alignment seeds
        train_entity_seeds, valid_entity_seeds, test_entity_seeds, entity_seeds = _load_seeds(self.directory,
                                                                                self.train_seeds_ratio,
                                                                                load_new_seed_split)
        self.entity_seeds = entity_seeds  # The entity seeds in the original order

        # train_ent_seeds shape = [length, 2]
        train_sr_ent_seeds_ori, train_tg_ent_seeds_ori = zip(*train_entity_seeds)

        self.train_sr_ent_seeds_ori = np.asarray(train_sr_ent_seeds_ori)
        self.train_tg_ent_seeds_ori = np.asarray(train_tg_ent_seeds_ori)

        # valid_ent_seeds shape = [length]
        valid_sr_ent_seeds, valid_tg_ent_seeds = zip(*valid_entity_seeds)
        self.valid_sr_ent_seeds = np.asarray(valid_sr_ent_seeds)
        self.valid_tg_ent_seeds = np.asarray(valid_tg_ent_seeds)
        test_sr_ent_seeds, test_tg_ent_seeds = zip(*test_entity_seeds)
        self.test_sr_ent_seeds = np.asarray(test_sr_ent_seeds)
        self.test_tg_ent_seeds = np.asarray(test_tg_ent_seeds)

    def load_attribute_feature(self, load_attribute, load_digit_literal, time_channel):
        directory = self.directory
        language_sr = self.language_sr
        language_tg = self.language_tg

        entity2id_sr = {ent: idx for idx, ent in self.id2entity_sr.items()}
        entity2id_tg = {ent: idx for idx, ent in self.id2entity_tg.items()}
        att_triples_sr = _load_dbpedia_properties(directory / ("atts_properties_%s.txt" % language_sr),
                                                  entity2id_sr, language_sr)
        att_triples_tg = _load_dbpedia_properties(directory / ("atts_properties_%s.txt" % language_tg),
                                                  entity2id_tg, language_tg)
        temp_file_dir = directory / 'running_temp'
        value_embed_encoder = ValueEmbedding(self.device)

        if load_attribute:
            self.att_triples_sr = [(ent_id, value, self.att2id[att]) for ent_id, value, att in att_triples_sr]
            self.att_triples_tg = [(ent_id, value, self.att2id[att]) for ent_id, value, att in att_triples_tg]

            ent_id_seq_sr, att_id_seq_sr, value_seq_sr = transform_triple2seq(self.att_triples_sr, language_sr)
            ent_id_seq_tg, att_id_seq_tg, value_seq_tg = transform_triple2seq(self.att_triples_tg, language_tg)
            value_embed_cache_path, id2value_cache_path = get_cache_file_path(temp_file_dir, 'Attribute')
            self.value_embedding, self.id2value = value_embed_encoder.load_value(value_seq_sr + value_seq_tg,
                                                                                value_embed_cache_path,
                                                                                id2value_cache_path, )
            value2id = {value: idx for idx, value in enumerate(self.id2value)}
            value_id_seq_sr = [[value2id.get(value, value2id['[PAD]']) for value in value_seq] for value_seq in
                               value_seq_sr]
            value_id_seq_tg = [[value2id.get(value, value2id['[PAD]']) for value in value_seq] for value_seq in
                               value_seq_tg]

            attribute_triples_sr = []
            for ent_id, att_seq, val_seq in zip(ent_id_seq_sr, att_id_seq_sr, value_id_seq_sr):
                for att, val in zip(att_seq, val_seq):
                    attribute_triples_sr.append((ent_id, val, att))
            self.attribute_triples_sr = torch.tensor(attribute_triples_sr)

            attribute_triples_tg = []
            for ent_id, att_seq, val_seq in zip(ent_id_seq_tg, att_id_seq_tg, value_id_seq_tg):
                for att, val in zip(att_seq, val_seq):
                    attribute_triples_tg.append((ent_id, val, att))
            self.attribute_triples_tg = torch.tensor(attribute_triples_tg)

        if time_channel:
            train_value_and_attribute_sr = _get_train_value_and_attribute(self.train_sr_ent_seeds_ori, att_triples_sr)
            train_value_and_attribute_tg = _get_train_value_and_attribute(self.train_tg_ent_seeds_ori, att_triples_tg)

            time_threshold = 0.5
            time_att2id = _split_time_attribute(
                train_value_and_attribute_sr + train_value_and_attribute_tg, time_threshold,
                set(self.att2id.keys()))

            # manually insert a virtual edge label
            time_att2id['Value_Edge'] = len(time_att2id)
            self.VALUE_LABEL_ID = time_att2id['Value_Edge']
            self.time_att2id = time_att2id
            self.time_att_num = len(time_att2id)

            time_triples_sr = list(set(_split_time_triple(att_triples_sr, time_att2id)))
            time_triples_tg = list(set(_split_time_triple(att_triples_tg, time_att2id)))

            print('------------------')
            print_time_info('time attribute number: %d' % (len(time_att2id)))
            print_time_info('time triple number: %d, %d' % (len(time_triples_sr), len(time_triples_tg)))
            print('------------------')

            time_ent_id_seq_sr, time_att_id_seq_sr, time_value_seq_sr = transform_triple2seq(time_triples_sr,
                                                                                language_sr, False)
            time_ent_id_seq_tg, time_att_id_seq_tg, time_value_seq_tg = transform_triple2seq(time_triples_tg,
                                                                                language_tg, False)

            time_value_embed_cache_path, time_id2value_cache_path = get_cache_file_path(temp_file_dir, 'Time')
            self.time_value_embedding, self.time_id2value = value_embed_encoder.load_value(
                time_value_seq_sr + time_value_seq_tg, time_value_embed_cache_path,
                time_id2value_cache_path)
            time_value2id = {value: idx for idx, value in enumerate(self.time_id2value)}

            time_value_id_seq_sr = [[time_value2id.get(value, time_value2id['[PAD]']) for value in value_seq] for
                                    value_seq in time_value_seq_sr]
            time_value_id_seq_tg = [[time_value2id.get(value, time_value2id['[PAD]']) for value in value_seq] for
                                    value_seq in time_value_seq_tg]

            time_triples_sr = []
            for ent_id, att_seq, val_seq in zip(time_ent_id_seq_sr, time_att_id_seq_sr, time_value_id_seq_sr):
                for att, val in zip(att_seq, val_seq):
                    time_triples_sr.append((ent_id, val, att))
            time_triples_tg = []
            for ent_id, att_seq, val_seq in zip(time_ent_id_seq_tg, time_att_id_seq_tg, time_value_id_seq_tg):
                for att, val in zip(att_seq, val_seq):
                    time_triples_tg.append((ent_id, val, att))

            # time_triples_sr = list(set(time_triples_sr))
            # time_triples_tg = list(set(time_triples_tg))
            time_triples_sr.sort(key=lambda x: self.time_id2value[x[2]], reverse=False)
            self.time_triples_sr = torch.tensor(time_triples_sr)

            time_triples_tg.sort(key=lambda x: self.time_id2value[x[2]], reverse=False)
            self.time_triples_tg = torch.tensor(time_triples_tg)

            self.clusterModel = ClusterModel(self.device,
                                             self.directory,
                                             self.time_triples_sr,
                                             self.time_triples_tg,
                                             self.time_att2id)
            self.clusters = self.clusterModel.get_clusters()

            ent_value_edge_sr = build_entity_and_value_edges(self.sr_ent_num, time_triples_sr)
            ent_value_edge_tg = build_entity_and_value_edges(self.tg_ent_num, time_triples_tg)
            value_edges_sr, edge_attrs_sr = build_value_edges_in_cluster(self.sr_ent_num,
                                                                         self.time_id2value,
                                                                         time_triples_sr, self.clusters)
            value_edges_tg, edge_attrs_tg = build_value_edges_in_cluster(self.tg_ent_num,
                                                                         self.time_id2value,
                                                                         time_triples_tg, self.clusters)
            self.time_graph_ent_value_edges_sr = torch.tensor(ent_value_edge_sr)
            self.time_graph_ent_value_edges_tg = torch.tensor(ent_value_edge_tg)
            self.time_graph_value_value_edges_sr = torch.tensor(value_edges_sr)
            self.time_graph_value_value_edges_tg = torch.tensor(value_edges_tg)

            dy_value_edges_sr, dy_value_edge_attrs_sr = build_dynamic_time_graphs(self.sr_ent_num,
                                                                                self.time_id2value,
                                                                                time_triples_sr,
                                                                                self.clusters)
            dy_value_edges_tg, dy_value_edge_attrs_tg = build_dynamic_time_graphs(self.tg_ent_num,
                                                                                self.time_id2value,
                                                                                time_triples_tg,
                                                                                self.clusters)
            self.dy_value_edges_sr = [torch.tensor(x) for x in dy_value_edges_sr]
            self.dy_value_edge_attrs_sr = [torch.tensor(x) for x in dy_value_edge_attrs_sr]
            self.dy_value_edges_tg = [torch.tensor(x) for x in dy_value_edges_tg]
            self.dy_value_edge_attrs_tg = [torch.tensor(x) for x in dy_value_edge_attrs_tg]

            def concate_all_seq(time_triples, idx):
                seqs = []
                for triple in time_triples:
                    seqs.append(triple[idx])
                return seqs

            ent_val_ids_sr = concate_all_seq(time_triples_sr, 2)
            ent_val_ids_tg = concate_all_seq(time_triples_tg, 2)
            val_edge_ids_sr = [self.VALUE_LABEL_ID] * len(value_edges_sr)
            val_edge_ids_tg = [self.VALUE_LABEL_ID] * len(value_edges_tg)
            self.time_graph_ent_value_edge_labels_sr = torch.tensor(ent_val_ids_sr)
            self.time_graph_ent_value_edge_labels_tg = torch.tensor(ent_val_ids_tg)
            self.time_graph_value_attr_value_labels_sr = torch.tensor(edge_attrs_sr)
            self.time_graph_value_attr_value_labels_tg = torch.tensor(edge_attrs_tg)
            self.time_graph_value_value_edge_labels_sr = torch.tensor(val_edge_ids_sr)
            self.time_graph_value_value_edge_labels_tg = torch.tensor(val_edge_ids_tg)
            self.time_graph_sr = {
                'ent_value_edges': self.time_graph_ent_value_edges_sr,
                'value_value_edges': self.time_graph_value_value_edges_sr,
                'ent_value_edge_labels': self.time_graph_ent_value_edge_labels_sr,
                'value_attr_value_labels': self.time_graph_value_attr_value_labels_sr,
                'value_value_edge_labels': self.time_graph_value_value_edge_labels_sr,
                'dy_value_edges': self.dy_value_edges_sr,
                'dy_value_edge_attrs': self.dy_value_edge_attrs_sr
            }
            self.time_graph_tg = {
                'ent_value_edges': self.time_graph_ent_value_edges_tg,
                'value_value_edges': self.time_graph_value_value_edges_tg,
                'ent_value_edge_labels': self.time_graph_ent_value_edge_labels_tg,
                'value_attr_value_labels': self.time_graph_value_attr_value_labels_tg,
                'value_value_edge_labels': self.time_graph_value_value_edge_labels_tg,
                'dy_value_edges': self.dy_value_edges_tg,
                'dy_value_edge_attrs': self.dy_value_edge_attrs_tg
            }
            self.time_graph = {
                'sr': self.time_graph_sr,
                'tg': self.time_graph_tg
            }
            print_time_info('finish loading time info.')

        if load_digit_literal:
            train_value_and_attribute_sr = _get_train_value_and_attribute(self.train_sr_ent_seeds_ori, att_triples_sr)
            train_value_and_attribute_tg = _get_train_value_and_attribute(self.train_tg_ent_seeds_ori, att_triples_tg)

            digit_threshold = 0.5
            digit_att2id, literal_att2id = _split_digit_attribute_and_literal_attribute(
                train_value_and_attribute_sr + train_value_and_attribute_tg,
                digit_threshold, set(self.att2id.keys()))

            self.digit_att2id = digit_att2id
            self.literal_att2id = literal_att2id
            self.digit_att_num = len(digit_att2id)
            self.literal_att_num = len(literal_att2id)

            digit_triples_sr, literal_triples_sr = _split_digit_and_literal_triple(att_triples_sr, digit_att2id,
                                                                                literal_att2id)
            digit_triples_tg, literal_triples_tg = _split_digit_and_literal_triple(att_triples_tg, digit_att2id,
                                                                                literal_att2id)

            print_time_info('digit attribute number: %d' % (len(self.digit_att2id)))
            print_time_info('digit triple number: %d, %d' % (len(digit_triples_sr), len(digit_triples_tg)))
            print_time_info('literal attribute number: %d' % (len(self.literal_att2id)))
            print_time_info('literal triple number: %d, %d' % (len(digit_triples_tg), len(literal_triples_tg)))
            digit_ent_id_seq_sr, digit_att_id_seq_sr, digit_value_seq_sr = transform_triple2seq(digit_triples_sr,
                                                                                language_sr, False)
            digit_ent_id_seq_tg, digit_att_id_seq_tg, digit_value_seq_tg = transform_triple2seq(digit_triples_tg,
                                                                                language_tg, False)

            literal_ent_id_seq_sr, literal_att_id_seq_sr, literal_value_seq_sr = transform_triple2seq(
                literal_triples_sr, language_sr, False)
            literal_ent_id_seq_tg, literal_att_id_seq_tg, literal_value_seq_tg = transform_triple2seq(
                literal_triples_tg, language_tg, False)

            literal_value_embed_cache_path, literal_id2value_cache_path = get_cache_file_path(temp_file_dir,
                                                                                'Literal')
            digit_value_embed_cache_path, digit_id2value_cache_path = get_cache_file_path(temp_file_dir, 'Digit')

            self.literal_value_embedding, self.literal_id2value = value_embed_encoder.load_value(
                literal_value_seq_sr + literal_value_seq_tg, literal_value_embed_cache_path,
                literal_id2value_cache_path)

            self.digit_value_embedding, self.digit_id2value = value_embed_encoder.load_value(
                digit_value_seq_sr + digit_value_seq_tg, digit_value_embed_cache_path, digit_id2value_cache_path, )

            literal_value2id = {value: idx for idx, value in enumerate(self.literal_id2value)}
            digit_value2id = {value: idx for idx, value in enumerate(self.digit_id2value)}

            digit_value_id_seq_sr = [[digit_value2id.get(value, digit_value2id['[PAD]']) for value in value_seq] for
                                     value_seq in digit_value_seq_sr]
            digit_value_id_seq_tg = [[digit_value2id.get(value, digit_value2id['[PAD]']) for value in value_seq] for
                                     value_seq in digit_value_seq_tg]

            literal_value_id_seq_sr = [
                [literal_value2id.get(value, literal_value2id['[PAD]']) for value in value_seq]
                for value_seq in literal_value_seq_sr]
            literal_value_id_seq_tg = [
                [literal_value2id.get(value, literal_value2id['[PAD]']) for value in value_seq]
                for value_seq in literal_value_seq_tg]

            literal_triples_sr = []
            for ent_id, att_seq, val_seq in zip(literal_ent_id_seq_sr, literal_att_id_seq_sr,
                                                literal_value_id_seq_sr):
                for att, val in zip(att_seq, val_seq):
                    literal_triples_sr.append((ent_id, val, att))
            self.literal_triples_sr = torch.tensor(literal_triples_sr)

            literal_triples_tg = []
            for ent_id, att_seq, val_seq in zip(literal_ent_id_seq_tg, literal_att_id_seq_tg,
                                                literal_value_id_seq_tg):
                for att, val in zip(att_seq, val_seq):
                    literal_triples_tg.append((ent_id, val, att))
            self.literal_triples_tg = torch.tensor(literal_triples_tg)

            digital_triples_sr = []
            for ent_id, att_seq, val_seq in zip(digit_ent_id_seq_sr, digit_att_id_seq_sr, digit_value_id_seq_sr):
                for att, val in zip(att_seq, val_seq):
                    digital_triples_sr.append((ent_id, val, att))
            self.digital_triples_sr = torch.tensor(digital_triples_sr)

            digital_triples_tg = []
            for ent_id, att_seq, val_seq in zip(digit_ent_id_seq_tg, digit_att_id_seq_tg, digit_value_id_seq_tg):
                for att, val in zip(att_seq, val_seq):
                    digital_triples_tg.append((ent_id, val, att))
            self.digital_triples_tg = torch.tensor(digital_triples_tg)

        del value_embed_encoder

def negative_sample(pos_ids, data_range, nega_sample_num):
    # Output shape = (data_len, negative_sample_num)
    nega_ids_arrays = np.random.randint(low=0, high=data_range - 1, size=(len(pos_ids), nega_sample_num))
    for idx, pos_id in enumerate(pos_ids):
        for j in range(nega_sample_num):
            if nega_ids_arrays[idx][j] >= pos_id:
                nega_ids_arrays[idx][j] += 1
    assert nega_ids_arrays.shape == (len(pos_ids), nega_sample_num), print(nega_ids_arrays.shape)
    return nega_ids_arrays

class Prefix(object):
    pattern_en_value_type = ["http://www.w3.org/1999/02/22-rdf-syntax-ns#", "http://www.w3.org/2001/XMLSchema#",
                             "http://dbpedia.org/datatype/"]

    @classmethod
    def set_language(cls, language):
        if language == 'en':
            cls.regex_ent = re.compile(r'http:\/\/dbpedia\.org\/resource\/(.*)')
            cls.pattern_prop = 'http://dbpedia.org/property/'
        elif language in {'zh', 'fr', 'ja'}:
            cls.regex_ent = re.compile(r'http:\/\/%s\.dbpedia\.org\/resource\/(.*)' % language)
            cls.pattern_prop = 'http://%s.dbpedia.org/property/' % language
        else:
            raise Exception()

    @classmethod
    def remove_prefix(cls, input):
        if isinstance(input, str):
            input = cls.regex_ent.match(input).group(1)
            return input.replace('_', ' ')
        return [cls.remove_prefix(item) for item in input]

    @classmethod
    def remove_prop_prefix(cls, input):
        if isinstance(input, str):
            if input.find(cls.pattern_prop) >= 0:
                return input.split(cls.pattern_prop)[1]
            raise Exception()
        return [cls.remove_prop_prefix(item) for item in input]

    @classmethod
    def remove_value_type(cls, input):
        if isinstance(input, str):
            for pattern in cls.pattern_en_value_type:
                if input.find(pattern) >= 0:
                    return input.split(pattern)[1]
            raise Exception()
        return [cls.remove_value_type(item) for item in input]

def transform_triple2seq(att_triples, language, concate_values=False):
    # ent_id_seq = [ent1_id, ent2_id, ent3_id...]
    # prop_num = [ent1_num_prop, ent2_num_prop...]
    # att_id_seq = [[ent1_prop1_id, ent1_prop2_id, ...]...]
    # value_seq = [[ent1_value1, ent1_value2, ...]...]
    # Fixme: select the first 20 attributes
    # Fixme: Original average property number 26 --> only one property average property number 15.9 --> top 20 property 10.09
    if language in {'zh', 'en', 'ja', 'fr'}:
        top_k_att = 20
    else:
        top_k_att = 3
    ent_id_seq = []
    prop2value_seq = []
    for ent_id, value, att_id in att_triples:
        if len(ent_id_seq) == 0:
            ent_id_seq.append(ent_id)
            prop2value_seq.append(dict())
        if ent_id != ent_id_seq[-1]:
            ent_id_seq.append(ent_id)
            prop2value_seq.append(dict())
        if not concate_values:
            prop2value_seq[-1][att_id] = value
        else:
            if att_id in prop2value_seq[-1]:
                prop2value_seq[-1][att_id] += '. ' + value
            else:
                prop2value_seq[-1][att_id] = value
    att_id_seq = []
    value_seq = []
    for prop2value in prop2value_seq:
        att_ids, values = zip(*list(prop2value.items()))
        assert len(values) == len(att_ids)
        att_id_seq.append(att_ids[:top_k_att])
        value_seq.append(values[:top_k_att])
    return ent_id_seq, att_id_seq, value_seq

def construct_ent_id2info(ent_num, ent_id_seq, att_id_seq, value_id_seq, att_pad_id, value_pad_id, language):
    if language in {'zh', 'en', 'fr', 'ja'}:
        top_k_att = 20
    else:
        top_k_att = 3

    assert len(ent_id_seq) == len(att_id_seq) == len(value_id_seq)
    entid2atts = [[] for _ in range(ent_num)]
    entid2values = [[] for _ in range(ent_num)]
    for ent_id, att_ids, value_ids in zip(ent_id_seq, att_id_seq, value_id_seq):
        entid2atts[ent_id] += att_ids
        entid2values[ent_id] += value_ids

    entid2atts = [item[:top_k_att] for item in entid2atts]
    entid2values = [item[:top_k_att] for item in entid2values]

    max_len1 = max(len(item) for item in entid2atts)
    max_len2 = max(len(item) for item in entid2values)
    assert max_len1 == max_len2 == top_k_att
    ent2att_num = np.zeros(ent_num, dtype=np.int)
    ent2atts = np.ones((ent_num, max_len1), dtype=np.int) * att_pad_id
    ent2values = np.ones((ent_num, max_len1), dtype=np.int) * value_pad_id

    att_num = 0
    for idx, (atts, values) in enumerate(zip(entid2atts, entid2values)):
        assert len(atts) == len(values)
        ent2att_num[idx] = len(atts)
        ent2atts[idx, :len(atts)] = atts
        ent2values[idx, :len(atts)] = values
        att_num += len(atts)
    return ent2att_num, ent2atts, ent2values

class Numeral(object):
    def __init__(self):
        self.regex = {'year': re.compile(r'^\d{3,4}$'), 'date': re.compile(r'^(\d+)-(\d+)-(\d+)$'),
                      'month_day': re.compile(r'^--(\d{2})-(\d{2})$')}
        self.regex_func = {'year': lambda x: (int(x.group(0)), None, None),
                           'date': lambda x: (int(x.group(1)), int(x.group(2)), int(x.group(3))),
                           'month_day': lambda x: (None, int(x.group(1)), int(x.group(2)))}

    def is_numeral(self, text):
        # for regex_name in self.regex:
        #     is_numeral, result = self.__regex_pattern(text, regex_name)
        #     if is_numeral:
        #         return is_numeral, result
        # is_numeral, result = self.__month_year_pattern(text)
        # if is_numeral:
        #     return is_numeral, result
        is_numeral, result = self.__float_pattern(text)
        if is_numeral:
            return is_numeral, result
        return False, None

    def __regex_pattern(self, text, regex_name):
        regex = self.regex[regex_name]
        result = regex.match(text)
        if result:
            return True, self.regex_func[regex_name](result)
        return False, None

    def __month_year_pattern(self, text):
        try:
            data = datetime.strptime(text, '%B %Y')
            return True, (data.year, data.month, None)
        except ValueError:
            return False, None
        except:
            raise Exception()

    def __float_pattern(self, text):
        special_patterns = ['inf', 'nan']
        for pattern in special_patterns:
            if text.lower().find(pattern) >= 0:
                return False, None
        try:
            data = float(text)
            return True, data
        except ValueError:
            return False, None
        except:
            raise Exception()


from django import forms
from .models import Order

class OrderForm(forms.ModelForm):
    class Meta:
        model = Order
        fields = ('landlord_profile', 'full_name', 'email', 'phone_number',
                  'country', 'postcode', 'town_or_city',
                  'street_address1', 'street_address2',)

    def __init__(self, *args, **kwargs):
        """
        Add placeholders, remove auto-generated labels
        """
        super().__init__(*args, **kwargs)
        for field in self.fields:
            self.fields[field].label = False
            if self.fields[field].required:
                placeholder = f'{field}*'
            else:
                placeholder = field
            placeholder = placeholder.replace('_',  ' ')
            if field != 'country':
                self.fields[field].widget.attrs['placeholder'] = \
                                                placeholder.capitalize()


import pyfirmata
from pyfirmata import Arduino, INPUT, PWM
import time

board = pyfirmata.Arduino("COM4")

it = pyfirmata.util.Iterator(board)
it.start()

red = board.get_pin('d:3:p')
blue = board.get_pin('d:5:p')
green = board.get_pin('d:6:p')

r=0.50
b=0.50
g=0.50

for i in range(100):

    while r>=0.5 and r<=1:
        red.write(r)
        time.sleep(0.01)
        red.write(0)
        r = r + 0.01

    while r>0:
        red.write(r)
        green.write(g)
        time.sleep(0.01)
        red.write(0)
        green.write(0)
        r = r - 0.02
        g = g + 0.01

    r=0.51

    while g>=0.5 and g<=1:
        green.write(g)
        time.sleep(0.01)
        green.write(0)
        g = g + 0.01

    while g>0:
        green.write(g)
        blue.write(b)
        time.sleep(0.01)
        green.write(0)
        blue.write(0)
        g = g - 0.02
        b = b + 0.01

    g=0.51

    while b>=0.5 and b<=1:
        blue.write(b)
        time.sleep(0.01)
        blue.write(0)
        b = b + 0.01

    while b>0:
        blue.write(b)
        red.write(r)
        time.sleep(0.01)
        blue.write(0)
        red.write(0)
        b = b - 0.02
        r = r + 0.01

    b=0.51


#!/usr/local/bin/python
# -*- coding: utf-8 -*-
# @Time    : 2021/7/7 21:48
# @File    : jd_wishing_pool.py
# @Project : jd_scripts
# @Cron    : 45 */12 * * *
# @Desc    : 京东APP首页->京东众筹->众筹许愿池

from utils.jd_common import JdCommon
from config import USER_AGENT

CODE_KEY = 'jd_wishing_pool'

class JdWishingPool(JdCommon):
    """
    众筹许愿池
    """
    appid = '1E1NXxq0'

    code_key = CODE_KEY

    # 请求头
    headers = {
        'Content-Type': 'application/x-www-form-urlencoded',
        'Referer': 'https://h5.m.jd.com/babelDiy/Zeus/4FdmTJQNah9oDJyQN8NggvRi1nEY/index.html',
        'User-Agent': USER_AGENT
    }

if __name__ == '__main__':
    from utils.process import process_start
    process_start(JdWishingPool, '众筹许愿池', code_key=CODE_KEY)


from django.urls import path
from . import views
# app_name='projectapp'
urlpatterns =[
    path("home",views.home,name="home"),
    path("movie/<int:movie_id>/",views.detail,name="detail"),
    path("add_movies",views.add_movies,name='add_movies'),
    path("update/<int:id>/",views.updatehtml,name="update"),
    path("delete/<int:id>/",views.delete,name="delete"),
    path("register",views.register_request,name="register"),
    path("login", views.login_request, name="login"),
    path("logout",views.logout_request,name="logout")
]


# -*- coding: utf-8 -*-

import jieba
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.cluster import KMeans
from text_analysis_tools.api.text_cluster import STOPWORDS

class KmeansClustering():
    def __init__(self, stopwords_path=STOPWORDS):
        self.stopwords = self.load_stopwords(stopwords_path)
        self.vectorizer = CountVectorizer()
        self.transformer = TfidfTransformer()

    def load_stopwords(self, stopwords=None):
        """
        加载停用词
        :param stopwords:
        :return:
        """
        if stopwords:
            with open(stopwords, 'r', encoding='utf-8') as f:
                return [line.strip() for line in f]
        else:
            return []

    def preprocess_data(self, corpus_path):
        """
        文本预处理，每行一个文本
        :param corpus_path:
        :return:
        """
        corpus = []
        with open(corpus_path, 'r', encoding='utf-8') as f:
            for line in f:
                corpus.append(' '.join([word for word in jieba.lcut(line.strip()) if word not in self.stopwords]))
        return corpus

    def get_text_tfidf_matrix(self, corpus):
        """
        获取tfidf矩阵
        :param corpus:
        :return:
        """
        tfidf = self.transformer.fit_transform(self.vectorizer.fit_transform(corpus))

        # 获取词袋中所有词语
        # words = self.vectorizer.get_feature_names()

        # 获取tfidf矩阵中权重
        weights = tfidf.toarray()
        return weights

    def kmeans(self, corpus_path, n_clusters=5):
        """
        KMeans文本聚类
        :param corpus_path: 语料路径（每行一篇）,文章id从0开始
        :param n_clusters: ：聚类类别数目
        :return: {cluster_id1:[text_id1, text_id2]}
        """
        corpus = self.preprocess_data(corpus_path)
        weights = self.get_text_tfidf_matrix(corpus)

        clf = KMeans(n_clusters=n_clusters)

        # clf.fit(weights)

        y = clf.fit_predict(weights)

        # 中心点
        # centers = clf.cluster_centers_

        # 用来评估簇的个数是否合适,距离约小说明簇分得越好,选取临界点的簇的个数
        # score = clf.inertia_

        # 每个样本所属的簇
        result = {}
        for text_idx, label_idx in enumerate(y):
            key = "cluster_{}".format(label_idx)
            if key not in result:
                result[key] = [text_idx]
            else:
                result[key].append(text_idx)
        return result

if __name__ == '__main__':
    Kmeans = KmeansClustering(stopwords_path='../data/stop_words.txt')
    result = Kmeans.kmeans('../data/test_data.txt', n_clusters=5)
    print(result)


# -*- coding: utf-8 -*-
"""

@author: fumagalli, alessio
"""
import numpy as np
import scipy.sparse as sps
from numpy.linalg import solve

import porepy as pp
from porepy.numerics.linalg.matrix_operations import sparse_array_to_row_col_data
from porepy.numerics.vem.dual_elliptic import DualElliptic

class HybridDualVEM:
    """Implementation of mixed virtual element method, using hybridization to
    arrive at a SPD system.

    WARNING: The implementation does not follow the newest formulations used
    in PorePy. Specifically, it does not support mixed-dimensional problems.
    This may or may not be improved in the future, depending on various factors.

    """

    def __init__(self, keyword="flow"):
        self.keyword = keyword

    def ndof(self, g):
        """
        Return the number of degrees of freedom associated to the method.
        In this case number of faces (hybrid dofs).

        Parameter
        ---------
        g: grid, or a subclass.

        Return
        ------
        dof: the number of degrees of freedom.

        """
        return g.num_faces

    def matrix_rhs(self, g, data):
        """
        Return the matrix and righ-hand side for a discretization of a second
        order elliptic equation using hybrid dual virtual element method.
        The name of data in the input dictionary (data) are:
        perm : tensor.SecondOrderTensor
            Permeability defined cell-wise. If not given a identity permeability
            is assumed and a warning arised.
        source : array (self.g.num_cells)
            Scalar source term defined cell-wise. If not given a zero source
            term is assumed and a warning arised.
        bc : boundary conditions (optional)
        bc_val : dictionary (optional)
            Values of the boundary conditions. The dictionary has at most the
            following keys: 'dir' and 'neu', for Dirichlet and Neumann boundary
            conditions, respectively.

        Parameters
        ----------
        g : grid, or a subclass, with geometry fields computed.
        data: dictionary to store the data.

        Return
        ------
        matrix: sparse csr (g.num_faces+g_num_cells, g.num_faces+g_num_cells)
            Saddle point matrix obtained from the discretization.
        rhs: array (g.num_faces+g_num_cells)
            Right-hand side which contains the boundary conditions and the scalar
            source term.

        Examples
        --------
        b_faces_neu = ... # id of the Neumann faces
        b_faces_dir = ... # id of the Dirichlet faces
        bnd = bc.BoundaryCondition(g, np.hstack((b_faces_dir, b_faces_neu)),
                                ['dir']*b_faces_dir.size + ['neu']*b_faces_neu.size)
        bnd_val = {'dir': fun_dir(g.face_centers[:, b_faces_dir]),
                   'neu': fun_neu(f.face_centers[:, b_faces_neu])}

        data = {'perm': perm, 'source': f, 'bc': bnd, 'bc_val': bnd_val}

        H, rhs = hybrid.matrix_rhs(g, data)
        l = sps.linalg.spsolve(H, rhs)
        u, p = hybrid.compute_up(g, l, data)
        P0u = dual.project_u(g, perm, u)

        """
        # pylint: disable=invalid-name

        # If a 0-d grid is given then we return an identity matrix
        if g.dim == 0:
            return sps.identity(self.ndof(g), format="csr"), np.zeros(1)

        parameter_dictionary = data[pp.PARAMETERS][self.keyword]
        k = parameter_dictionary["second_order_tensor"]
        f = parameter_dictionary["source"]
        bc = parameter_dictionary["bc"]
        bc_val = parameter_dictionary["bc_values"]
        a = parameter_dictionary["aperture"]

        faces, _, sgn = sparse_array_to_row_col_data(g.cell_faces)

        # Map the domain to a reference geometry (i.e. equivalent to compute
        # surface coordinates in 1d and 2d)
        c_centers, f_normals, f_centers, _, _, _ = pp.map_geometry.map_grid(g)

        # Weight for the stabilization term
        diams = g.cell_diameters()
        weight = np.power(diams, 2 - g.dim)

        # Allocate the data to store matrix entries, that's the most efficient
        # way to create a sparse matrix.
        size = np.sum(np.square(g.cell_faces.indptr[1:] - g.cell_faces.indptr[:-1]))
        row = np.empty(size, dtype=int)
        col = np.empty(size, dtype=int)
        data = np.empty(size)
        rhs = np.zeros(g.num_faces)

        idx = 0
        # Use a dummy keyword to trick the constructor of dualVEM.
        massHdiv = pp.MVEM("dummy").massHdiv

        # define the function to compute the inverse of the permeability matrix
        if g.dim == 1:
            inv_matrix = DualElliptic._inv_matrix_1d
        elif g.dim == 2:
            inv_matrix = DualElliptic._inv_matrix_2d
        elif g.dim == 3:
            inv_matrix = DualElliptic._inv_matrix_3d

        for c in np.arange(g.num_cells):
            # For the current cell retrieve its faces
            loc = slice(g.cell_faces.indptr[c], g.cell_faces.indptr[c + 1])
            faces_loc = faces[loc]
            ndof = faces_loc.size

            # Retrieve permeability and normals assumed outward to the cell.
            sgn_loc = sgn[loc].reshape((-1, 1))
            normals = np.multiply(
                np.tile(sgn_loc.T, (g.dim, 1)), f_normals[:, faces_loc]
            )

            # Compute the H_div-mass local matrix
            A = massHdiv(
                k.values[0 : g.dim, 0 : g.dim, c],
                inv_matrix(k.values[0 : g.dim, 0 : g.dim, c]),
                c_centers[:, c],
                a[c] * g.cell_volumes[c],
                f_centers[:, faces_loc],
                a[c] * normals,
                np.ones(ndof),
                diams[c],
                weight[c],
            )[0]
            # Compute the Div local matrix
            B = -np.ones((ndof, 1))
            # Compute the hybrid local matrix
            C = np.eye(ndof, ndof)

            # Perform the static condensation to compute the hybrid local matrix
            invA = np.linalg.inv(A)
            S = 1 / np.dot(B.T, np.dot(invA, B))
            L = np.dot(np.dot(invA, np.dot(B, np.dot(S, B.T))), invA)
            L = np.dot(np.dot(C.T, L - invA), C)

            # Compute the local hybrid right using the static condensation
            rhs[faces_loc] += np.dot(C.T, np.dot(invA, np.dot(B, np.dot(S, f[c]))))[
                :, 0
            ]

            # Save values for hybrid matrix
            indices = np.tile(faces_loc, (faces_loc.size, 1))
            loc_idx = slice(idx, idx + indices.size)
            row[loc_idx] = indices.T.ravel()
            col[loc_idx] = indices.ravel()
            data[loc_idx] = L.ravel()
            idx += indices.size

        # construct the global matrices
        H = sps.coo_matrix((data, (row, col))).tocsr()

        # Apply the boundary conditions
        if bc is not None:
            if np.any(bc.is_dir):
                norm = sps.linalg.norm(H, np.inf)
                is_dir = np.where(bc.is_dir)[0]

                H[is_dir, :] *= 0
                H[is_dir, is_dir] = norm
                rhs[is_dir] = norm * bc_val[is_dir]

            if np.any(bc.is_neu):
                faces, _, sgn = sparse_array_to_row_col_data(g.cell_faces)
                sgn = sgn[np.unique(faces, return_index=True)[1]]

                is_neu = np.where(bc.is_neu)[0]
                rhs[is_neu] += sgn[is_neu] * bc_val[is_neu] * g.face_areas[is_neu]

        return H, rhs

    def compute_up(self, g, solution, data):
        """
        Return the velocity and pressure computed from the hybrid variables.

        Parameters
        ----------
        g : grid, or a subclass, with geometry fields computed.
        solution : array (g.num_faces) Hybrid solution of the system.
        data: dictionary to store the data. See self.matrix_rhs for a detaild
            description.

        Return
        ------
        u : array (g.num_faces) Velocity at each face.
        p : array (g.num_cells) Pressure at each cell.

        """
        # pylint: disable=invalid-name

        if g.dim == 0:
            return 0, solution[0]

        param = data["param"]
        k = param.get_tensor(self)
        f = param.get_source(self)
        a = param.aperture

        faces, _, sgn = sparse_array_to_row_col_data(g.cell_faces)

        # Map the domain to a reference geometry (i.e. equivalent to compute
        # surface coordinates in 1d and 2d)
        c_centers, f_normals, f_centers, _, _, _ = pp.map_geometry.map_grid(g)

        # Weight for the stabilization term
        diams = g.cell_diameters()
        weight = np.power(diams, 2 - g.dim)

        # Allocation of the pressure and velocity vectors
        p = np.zeros(g.num_cells)
        u = np.zeros(g.num_faces)
        massHdiv = pp.DualVEM().massHdiv

        for c in np.arange(g.num_cells):
            # For the current cell retrieve its faces
            loc = slice(g.cell_faces.indptr[c], g.cell_faces.indptr[c + 1])
            faces_loc = faces[loc]
            ndof = faces_loc.size

            # Retrieve permeability and normals assumed outward to the cell.
            sgn_loc = sgn[loc].reshape((-1, 1))
            normals = np.multiply(
                np.tile(sgn_loc.T, (g.dim, 1)), f_normals[:, faces_loc]
            )

            # Compute the H_div-mass local matrix
            A = massHdiv(
                k.values[0 : g.dim, 0 : g.dim, c],
                c_centers[:, c],
                a[c] * g.cell_volumes[c],
                f_centers[:, faces_loc],
                a[c] * normals,
                np.ones(ndof),
                diams[c],
                weight[c],
            )[0]
            # Compute the Div local matrix
            B = -np.ones((ndof, 1))
            # Compute the hybrid local matrix
            C = np.eye(ndof, ndof)

            # Perform the static condensation to compute the pressure and velocity
            S = 1 / np.dot(B.T, solve(A, B))
            l_loc = solution[faces_loc].reshape((-1, 1))

            p[c] = np.dot(S, f[c] - np.dot(B.T, solve(A, np.dot(C, l_loc))))
            u[faces_loc] = -np.multiply(
                sgn_loc, solve(A, np.dot(B, p[c]) + np.dot(C, l_loc))
            )

        return u, p


from conf import conf
from sgfmill import sgf
import os
from collections import defaultdict
from math import sqrt

GAMES_DIR = conf['GAMES_DIR']

def analysis():
    directory = GAMES_DIR
    results = defaultdict(list)
    i = 0
    for model_directory in os.listdir(directory):
        for filename in os.listdir(os.path.join(directory, model_directory)):
            if filename.endswith('.sgf'): # Ignore sgf files.
                with open(os.path.join(directory, model_directory, filename), 'rb') as f:
                    game = sgf.Sgf_game.from_bytes(f.read())
                    i += 1

                    playerb = game.get_player_name('b')
                    playerw = game.get_player_name('w')
                    winner_is_black = game.get_winner() == 'b'
                    results[(playerb, playerw)].append( winner_is_black)
    return results

if __name__ == "__main__":
    results = analysis()
    total = 0
    n = 0
    for key, l in results.items():
        mean = sum(l) /  float(len(l))
        delta = 2./sqrt(len(l))
        interval = [mean - delta, mean + delta]

        n += len(l)
        total += sum(l)

    m = total / n
    delta = 1/sqrt(n)
    print("Average black winrate [", m - delta, ",", m + delta, "]")


import pandas as pd
from pybit import usdt_perpetual

import crypto_scraper as cs

extraction = True
updating = False

session_unauth = usdt_perpetual.HTTP(
    endpoint="https://api-testnet.bybit.com"
)

bybit_scr = cs.ByBit_Scraper(session_unauth)                                                            # Create a ByBit Scraper Object

# Data Extraction
if extraction:
    bybit_scr.extract_history("ETHUSDT", 15)                                                            # Extract Data
    print(bybit_scr.get_dataframe())                                                                    # Print the final dataframe
    bybit_scr.to_csv(r"C:\Users\axeli\Desktop\Github\crypto-trading\Data Analysis\ETHUSDT 15.csv")      # Save the data in a .csv file

# Data Updating
if updating:
    bybit_scr.set_dataframe(pd.read_csv(r"C:\Users\axeli\Desktop\Github\crypto-trading\Data Analysis\BTCUSDT 5.csv"))   # Read a .csv and set it as dataframe
    bybit_scr.update_history()                                                                                # Update the dataframe
    bybit_scr.to_csv(r"C:\Users\axeli\Desktop\Github\crypto-trading\Data Analysis\BTCUSDT 5.csv")                       # Save the data in a .csv file


import numpy as np
import math
import pygame
import sys
import copy

### CONSTANT
EDGE = 100
SHAPE = (8, 6)
WIDTH = SHAPE[0] * EDGE
HEIGHT = SHAPE[1] * EDGE
### COLOR
RED = (255, 0, 0)
BLUE = (0, 0, 255)
GREEN = (0, 255, 0)
YELLOW = (255, 255, 0)
BLACK = (0, 0, 0)

screen = pygame.display.set_mode((WIDTH, HEIGHT))
screen.fill(YELLOW)
board_game = np.ones((SHAPE[1], SHAPE[0])) * (-1)

INVALID = -1
EMPTY = 0
FIRE = 1
ICE = 2

POS = {'FIRE': [int((SHAPE[1] - 1) / 2),0],
        'ICE': [int((SHAPE[1] + 1) / 2), SHAPE[0] - 1],
        'SELECT': [0, 0]
} # [row,col]
board_game[POS['FIRE'][0]][POS['FIRE'][1]] = FIRE
board_game[POS['ICE'][0]][POS['ICE'][1]] = ICE
TURN = 0
### HELPER FUNCTION
def drawGrid():
    thickness = 2
    for r in range(0, SHAPE[1]):
        for c in range(0, SHAPE[0]):
            rect = pygame.Rect(c * EDGE, r * EDGE, EDGE, EDGE)
            if board_game[r][c] == INVALID:
                pygame.draw.rect(screen, YELLOW, rect)
            elif board_game[r][c] == EMPTY:
                pygame.draw.rect(screen, BLACK, rect)
            pygame.draw.rect(screen, BLACK, rect, thickness)
    rect = pygame.Rect(POS['SELECT'][1] * EDGE, POS['SELECT'][0] * EDGE, EDGE, EDGE)
    pygame.draw.rect(screen, GREEN, rect)
    pygame.draw.circle(screen, RED, (POS['FIRE'][1] * EDGE + int(EDGE / 2), POS['FIRE'][0] * EDGE + int(EDGE / 2)), int(EDGE / 2 - thickness))
    pygame.draw.circle(screen, BLUE, (POS['ICE'][1] * EDGE + int(EDGE / 2), POS['ICE'][0] * EDGE + int(EDGE / 2)), int(EDGE / 2 - thickness))
    # print(POS['SELECT'])

def getKeyboard():
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()
            sys.exit()
        if event.type == pygame.KEYDOWN:
            if event.key  == pygame.K_DOWN or event.key  == pygame.K_s:
                rect = pygame.Rect(POS['SELECT'][1] * EDGE, POS['SELECT'][0] * EDGE, EDGE, EDGE)
                pygame.draw.rect(screen, YELLOW, rect)
                prev = copy.deepcopy(POS['SELECT'][0])
                POS['SELECT'][0] = min(POS['SELECT'][0] + 1, SHAPE[1] - 1)
                if (board_game[POS['SELECT'][0]][POS['SELECT'][1]] == EMPTY):
                    POS['SELECT'][0] = prev
            if event.key  == pygame.K_UP or event.key  == pygame.K_w:
                rect = pygame.Rect(POS['SELECT'][1] * EDGE, POS['SELECT'][0] * EDGE, EDGE, EDGE)
                pygame.draw.rect(screen, YELLOW, rect)
                prev = copy.deepcopy(POS['SELECT'][0])
                POS['SELECT'][0] = max(POS['SELECT'][0] - 1, 0)
                if (board_game[POS['SELECT'][0]][POS['SELECT'][1]] == EMPTY):
                    POS['SELECT'][0] = prev
            if event.key  == pygame.K_RIGHT or event.key  == pygame.K_d:
                rect = pygame.Rect(POS['SELECT'][1] * EDGE, POS['SELECT'][0] * EDGE, EDGE, EDGE)
                pygame.draw.rect(screen, YELLOW, rect)
                prev = copy.deepcopy(POS['SELECT'][1])
                POS['SELECT'][1] = min(POS['SELECT'][1] + 1, SHAPE[0] - 1)
                if (board_game[POS['SELECT'][0]][POS['SELECT'][1]] == EMPTY):
                    POS['SELECT'][1] = prev
            if event.key  == pygame.K_LEFT or event.key  == pygame.K_a:
                rect = pygame.Rect(POS['SELECT'][1] * EDGE, POS['SELECT'][0] * EDGE, EDGE, EDGE)
                pygame.draw.rect(screen, YELLOW, rect)
                prev = copy.deepcopy(POS['SELECT'][1])
                POS['SELECT'][1] = max(POS['SELECT'][1] - 1, 0)
                if (board_game[POS['SELECT'][0]][POS['SELECT'][1]] == EMPTY):
                    POS['SELECT'][1] = prev
            if event.key  == pygame.K_SPACE:
                # TURN 0 (FIRE move) -> 1 (FIVE choose) -> 2 (ICE move) -> 3 (ICE choose)
                # print(1)
                rect = pygame.Rect(POS['SELECT'][1] * EDGE, POS['SELECT'][0] * EDGE, EDGE, EDGE)
                pygame.draw.rect(screen, YELLOW, rect)
                global TURN
                valid = False
                if TURN == 0:
                    if (abs(POS['SELECT'][0] - POS['FIRE'][0]) + abs(POS['SELECT'][1] - POS['FIRE'][1]) == 1):
                        board_game[POS['FIRE'][0]][POS['FIRE'][1]] = INVALID
                        POS['FIRE'] = copy.deepcopy(POS['SELECT'])
                        board_game[POS['FIRE'][0]][POS['FIRE'][1]] = FIRE
                        ### move selected quickly
                        POS['SELECT'] = copy.deepcopy(POS['ICE'])
                        #####
                        valid = True
                elif TURN == 1 or TURN == 3:
                    if (board_game[POS['SELECT'][0]][POS['SELECT'][1]] == INVALID):
                        board_game[POS['SELECT'][0]][POS['SELECT'][1]] = EMPTY
                        ### move selected quickly
                        if TURN == 1:
                            POS['SELECT'] = copy.deepcopy(POS['ICE'])
                        elif TURN == 3:
                            POS['SELECT'] = copy.deepcopy(POS['FIRE'])
                        #####
                        valid = True
                elif TURN == 2:
                    if (abs(POS['SELECT'][0] - POS['ICE'][0]) + abs(POS['SELECT'][1] - POS['ICE'][1]) == 1):
                        board_game[POS['ICE'][0]][POS['ICE'][1]] = INVALID
                        POS['ICE'] = copy.deepcopy(POS['SELECT'])
                        board_game[POS['ICE'][0]][POS['ICE'][1]] = ICE
                        valid = True
                        ### move selected quickly
                        POS['SELECT'] = copy.deepcopy(POS['FIRE'])
                        #####
                if valid:
                    TURN = (TURN + 1) % 4
            # print(board_game)

### MAIN
def main():
    # print(board_game)
    ### move selected quickly
    POS['SELECT'] = copy.deepcopy(POS['FIRE'])
    #####
    while True:
        drawGrid()
        getKeyboard()
        pygame.display.update()
main()


from django.urls import path
from django.conf import settings
from django.conf.urls.static import static
from . import views

app_name = 'members'

urlpatterns = [

    path('', views.all_members, name='all_members' ),
    path('add', views.Add_mumber, name='Add_mumber' ),

    path('<int:id>', views.detalis_members, name='detalis_members'),
    path('<int:id>edit', views.Editr_Mumber, name='Editr_Mumber'),
    path('<int:id>edit_sub', views.Editr_Mumber_usb, name='Editr_Mumber_usb'),

   # path('<str:slug>', views.job_detail, name='job_detail'),
]

urlpatterns +=  static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


from flask import Flask, jsonify, request
from flask_restful import Resource, Api, reqparse

import pandas as pd
import numpy as np

app = Flask(__name__)
api = Api(app)

# constructors ->

class add(Resource):
    def post(self):
        add_data = request.get_json()
        addition = sum(list(add_data.values()))
        return addition

# assign addition endpoint
api.add_resource(add, '/add')

class subtract(Resource):
    def post(self):
        sub_data = request.get_json()
        sub_list = list(sub_data.values())
        subtraction = []
        for number in range(len(sub_list)-1):
            temp_num = sub_list[number] - sub_list[number+1]
            subtraction.append(temp_num)
        return subtraction

# assign addition endpoint
api.add_resource(subtract, '/subtract')


from django.urls import path
from . import views

urlpatterns = [
    path('', views.get_posts, name='get_posts'),
    path('<int:pk>/', views.post_detail, name='post_detail'),
    path('add/', views.create_or_edit_post, name='add_post'),
    path('edit/<int:pk>/', views.create_or_edit_post, name='edit_post'),
    path('delete/<int:pk>/', views.delete_post, name='delete_post'),
    path('delete-comment/<int:comment_id>/', views.delete_comment,
         name='delete_comment'),
]


import os
from time import sleep
import platform
from tkinter import messagebox, Tk
import sys
import datetime
from checkInternet.Alerts import alert
from checkInternet import internet, AlertFactory
from checkInternet.Alerts import AlertTypes
# from internet import internet_fail, internet_on
# import Alerts
# import internet
import argparse

def reporter(f):

    def wrap(*args, **kwargs):
        x = f(*args, **kwargs)
        print("[{}] Internet up. Press \"Ctrl + c\" to quit".format(str(datetime.datetime.now().isoformat())))
        return x

    return wrap

@reporter
def report_internet():
    x = internet.multi_attempt()
    return x

def check_internet(fail_callback, sleeptime=60):
    try:
        while report_internet():
            sleep(sleeptime)
        sys.stdout.write('\a')
        sys.stdout.flush()
        fail_callback("Internet Down at {}".format(str(datetime.datetime.now().isoformat())))
    except KeyboardInterrupt:
        print()
        print("Exiting")
        pass

def check_internet2(alerter, sleeptime=60):
    try:
        while report_internet():
            sleep(sleeptime)
        # sys.stdout.write('\a')
        # sys.stdout.flush()
        alerter.message = "Internet Down at {}".format(str(datetime.datetime.now().isoformat()))
        alerter.execute()
    except KeyboardInterrupt:
        print()
        print("Exiting")
        pass

def get_alerter():
    alert_factory = AlertFactory()
    if platform.system() == "Darwin":
        return alert_factory.create_alerter(AlertTypes.COCO)
    else:
        return alert_factory.create_alerter(AlertTypes.TK)

def main():
    parser = argparse.ArgumentParser(description="Checks to see to let you know if the internet is still accessible")
    parser.add_argument("frequency", nargs="?", type=int, default=60,
                        help="How many seconds between each check. (Default is 60)")
    args = parser.parse_args()
    print()
    print("Starting to monitor Internet access\n")
    print("Checking every {} second(s)".format(args.frequency))
    alerter = get_alerter()
    check_internet2(alerter, sleeptime=args.frequency)
    # check_internet(alert.Alert, args.frequency)
    # check_internet(no_internet_tk, args.frequency)
    print("Goodbye")

if __name__ == '__main__':
    main()


#Embedded file name: overviewPresets\overviewPresetUtil.py
"""
    This file contains some utility functions for the overview presets, for storing them and sharing with others
"""
import hashlib
import yaml
MAX_SHARED_PRESETS = 15
MAX_TAB_NUM = 5

def GetDeterministicListFromDict(inputDict):
    outputList = [ (x, y) for x, y in inputDict.iteritems() ]
    outputList.sort()
    return outputList

def GetDictFromList(inputList):
    return {x:y for x, y in inputList}

def ReplaceInnerListsWithDicts(parentDict):
    for tabIdx, tabValue in parentDict.iteritems():
        newTabValue = GetDictFromList(tabValue)
        parentDict[tabIdx] = newTabValue

    return parentDict

def GetOrderedListFromDict(inputDict, orderedKeys):
    dictCopy = inputDict.copy()
    outputList = []
    for eachKey in orderedKeys:
        dictValue = dictCopy.pop(eachKey, None)
        if dictValue:
            outputList.append(dictValue)

    outputList += dictCopy.values()
    return outputList

def GetHashvalueFromDict(inputDict):
    inputList = GetDeterministicListFromDict(inputDict)
    hashvalue = GetHashvalueForList(inputList)
    return hashvalue

def GetYamlStringFromDict(inputDict):
    inputList = GetDeterministicListFromDict(inputDict)
    yamlString = GetYamlStringFromList(inputList)
    return yamlString

def GetYamlStringFromList(inputList):
    yamlString = yaml.safe_dump(inputList)
    return yamlString

def GetHashvalueForList(inputList):
    yamlString = GetYamlStringFromList(inputList)
    return GetHashValueFromString(yamlString)

def GetHashValueFromString(yamlString):
    hashvalue = hashlib.sha1(yamlString).hexdigest()
    return hashvalue

def EncodeKeyInDict(inputDict):
    outputDict = {}
    for key, value in inputDict.iteritems():
        newKey = '%s_%s' % (key[0], key[1])
        outputDict[newKey] = value

    return outputDict

def DecodeKeyInDict(inputDict):
    """ assumes first element in key is string, other int """
    outputDict = {}
    for key, value in inputDict.iteritems():
        parts = key.split('_')
        newKey = (parts[0], int(parts[1]))
        outputDict[newKey] = value

    return outputDict

def IsPresetTheSame(preset1, preset2):

    def CheckPreset(pA, pB):
        for pAKey, pAValue in pA.iteritems():
            pBValue = pB.get(pAKey, -1)
            if pBValue == -1 and not pAValue:
                continue
            if pBValue != pAValue:
                return False

        return True

    allP1InP2 = CheckPreset(preset1, preset2)
    allP2InP1 = CheckPreset(preset2, preset1)
    return allP1InP2 and allP2InP1

def ReorderList(parentDict, namesToReorder):
    for reorderName in namesToReorder:
        listToReorder = parentDict.get(reorderName, None)
        if listToReorder is None:
            continue
        listToReorder.sort()


from django.test import TestCase

from user.models import User
from user.factories import UserFactory
from .models import ChatRoom, Message

class ChatTestCase(TestCase):
    def setUp(self):
        self.user1 = UserFactory.create()
        self.user2 = UserFactory.create()
        self.chatroom = ChatRoom.objects.create(room_id='tets-room')
        self.chatroom.member.add(self.user1)
        self.chatroom.member.add(self.user2)
        self.chatroom.save()
        self.message = Message.objects.create(
            sender=self.user1,
            content='Hello, this is a test message',
            chatroom=self.chatroom
        )

    def test_create_message(self):
        message = Message.objects.create(
            sender=self.user1,
            content='Hello, this is another test message',
            chatroom=self.chatroom
        )
        self.assertEqual(Message.objects.count(), 2)
        self.assertEqual(message.sender, self.user1)
        self.assertEqual(message.content, 'Hello, this is another test message')
        self.assertEqual(message.chatroom, self.chatroom)

    def test_update_message(self):
        self.message.is_read = True
        self.message.save()
        self.assertTrue(Message.objects.get(id=self.message.id).is_read)

    def test_delete_message(self):
        self.message.delete()
        self.assertEqual(Message.objects.count(), 0)

    def test_create_chatroom(self):
        chatroom = ChatRoom.objects.create(room_id='Test Chat Room 2')
        self.assertEqual(ChatRoom.objects.count(), 2)
        self.assertTrue(chatroom.room_id ==  'Test Chat Room 2')

    def test_delete_chatroom(self):
        self.chatroom.delete()
        self.assertEqual(ChatRoom.objects.count(), 0)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import argparse
import logging
import json
import gc
import subprocess as sp

import matplotlib
matplotlib.use('Agg')

import churnr
from churnr import sample, extract, process, train, plot

logger = logging.getLogger('churnr.experiment')

def run(args):
    logger.info('Initializing experiment...')

    with open(args.exppath) as fi:
        expconf = json.load(fi)[args.experiment]

    datasets = expconf['datasets']
    models = expconf['models']
    plots = expconf['plots']
    expabspath = os.path.abspath(args.exppath)

    # sample stage
    if 'sample' in args.stages:
        # sort the datasets by size of observation or prediction window, and sample users only on
        # the largest one as to have the same set of user ids of across all datasets
        if 'preddays' not in datasets['global']:
            datasets_sorted = sorted(datasets.items(), key=lambda x: x[1].get('preddays', 0), reverse=True)
        else:
            datasets_sorted = sorted(datasets.items(), key=lambda x: x[1].get('obsdays', 0), reverse=True)

        sampleusers = True
        for dsname, _ in datasets_sorted:
            if dsname == 'global' or (len(args.datasets) and dsname not in args.datasets):
                continue

            sample.main(exppath=expabspath, experiment=args.experiment, dsname=dsname, sampleusers=sampleusers)
            sampleusers = False

    # play context parsing stage
    if 'parse' in args.stages:
        basepath = os.path.dirname(churnr.__file__)
        for dsname in datasets.keys():
            if dsname == 'global' or (len(args.datasets) and dsname not in args.datasets):
                continue

            bqdataset = datasets['global']['dataset']
            bqproject = datasets['global']['project']
            table = '{}.features_{}_{}_p'.format(bqdataset, args.experiment, dsname)
            sp.call([os.path.join(basepath,'scala', 'run.sh'), bqproject, table])

    # extract stage
    if 'extract' in args.stages:
        for dsname in datasets.keys():
            if dsname == 'global' or (len(args.datasets) and dsname not in args.datasets):
                continue

            extract.main(exppath=expabspath, experiment=args.experiment, dsname=dsname)

    # process stage
    if 'process' in args.stages:
        for dsname in datasets.keys():
            if dsname == 'global' or (len(args.datasets) and dsname not in args.datasets):
                continue

            process.main(exppath=expabspath, experiment=args.experiment, dsname=dsname)

    # train stage
    if 'train' in args.stages:
        for dsname in datasets.keys():
            if dsname == 'global' or (len(args.datasets) and dsname not in args.datasets):
                continue

            # remove any local files stored in another run
            dspath = os.path.join(datasets['global']['procpath'], args.experiment, dsname)
            if os.path.exists(dspath):
                import shutil
                shutil.rmtree(dspath)
            os.makedirs(dspath)

            for modelname in models.keys():
                if modelname == 'global' or (len(args.models) and modelname not in args.models):
                    continue

                args.modelname = modelname
                args.dsname = dsname
                train.main(args)

                # dispatches garbage collector after each training
                gc.collect()

    # plot stage
    if 'plot' in args.stages:
        for plotname in plots.keys():
            if plotname == 'global':
                continue

            plot.main(exppath=expabspath, experiment=args.experiment, plotname=plotname)

    logger.info('Done!')

def main():
    path = os.path.dirname(churnr.__file__)

    parser = argparse.ArgumentParser(description='Experiment dispatcher')
    parser.add_argument('--exppath', default=os.path.join(path, 'experiments.json'), help='Path to the experiments json file')
    parser.add_argument('--experiment', default='temporal_static', help='Name of the experiment being performed')
    parser.add_argument('--stages', default=['sample', 'parse', 'extract','process','train', 'plot'], help='Stages that will be executed', nargs='*')
    parser.add_argument('--models', default=[], help='Model that will be trained', nargs='*')
    parser.add_argument('--datasets', default=[], help='Datasets that will be processed', nargs='*')
    parser.add_argument('--debug', default=False, help='Debug flag that sped up some stages', action='store_true')

    args = parser.parse_args()

    run(args)

if __name__ == '__main__':
    main()


# =============================================================================
# Author: Shuo Zhou, shuo.zhou@sheffield.ac.uk
#         Haiping Lu, h.lu@sheffield.ac.uk or hplu@ieee.org
# =============================================================================

"""Implementation of MPCA->Feature Selection->Linear SVM/LogisticRegression Pipeline

References:
    [1] Swift, A. J., Lu, H., Uthoff, J., Garg, P., Cogliano, M., Taylor, J., ... & Kiely, D. G. (2020). A machine
    learning cardiac magnetic resonance approach to extract disease features and automate pulmonary arterial
    hypertension diagnosis. European Heart Journal-Cardiovascular Imaging.
    [2] Song, X., Meng, L., Shi, Q., & Lu, H. (2015, October). Learning tensor-based features for whole-brain fMRI
    classification. In International Conference on Medical Image Computing and Computer-Assisted Intervention
    (pp. 613-620). Springer, Cham.
    [3] Lu, H., Plataniotis, K. N., & Venetsanopoulos, A. N. (2008). MPCA: Multilinear principal component analysis of
    tensor objects. IEEE Transactions on Neural Networks, 19(1), 18-39.
"""

import logging

import numpy as np
from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.feature_selection import f_classif
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
from sklearn.svm import LinearSVC, SVC
from sklearn.utils.validation import check_is_fitted

from ..embed.factorization import MPCA

param_c_grids = list(np.logspace(-4, 2, 7))
classifiers = {
    "svc": [SVC, {"kernel": ["linear"], "C": param_c_grids, "max_iter": [50000]}],
    "linear_svc": [LinearSVC, {"C": param_c_grids}],
    "lr": [LogisticRegression, {"C": param_c_grids}],
}

# k-fold cross validation used for grid search, i.e. searching for optimal value of C
default_search_params = {"cv": 5}
default_mpca_params = {"var_ratio": 0.97, "vectorize": True}

class MPCATrainer(BaseEstimator, ClassifierMixin):
    """Trainer of pipeline: MPCA->Feature selection->Classifier

    Args:
        classifier (str, optional): Available classifier options: {"svc", "linear_svc", "lr"}, where "svc" trains a
            support vector classifier, supports both linear and non-linear kernels, optimizes with library "libsvm";
            "linear_svc" trains a support vector classifier with linear kernel only, and optimizes with library
            "liblinear", which suppose to be faster and better in handling large number of samples; and "lr" trains
            a classifier with logistic regression. Defaults to "svc".
        classifier_params (dict, optional): Parameters of classifier. Defaults to 'auto'.
        classifier_param_grid (dict, optional): Grids for searching the optimal hyper-parameters. Works only when
            classifier_params == "auto". Defaults to None by searching from the following hyper-parameter values:
            1. svc, {"kernel": ["linear"], "C": [0.0001, 0.001, 0.01, 0.1, 1, 10, 100], "max_iter": [50000]},
            2. linear_svc, {"C": [0.0001, 0.001, 0.01, 0.1, 1, 10, 100]},
            3. lr, {"C": [0.0001, 0.001, 0.01, 0.1, 1, 10, 100]}
        mpca_params (dict, optional): Parameters of MPCA, e.g., {"var_ratio": 0.8}. Defaults to None, i.e., using the
            default parameters (https://pykale.readthedocs.io/en/latest/kale.embed.html#module-kale.embed.mpca).
        n_features (int, optional): Number of features for feature selection. Defaults to None, i.e., all features
            after dimension reduction will be used.
        search_params (dict, optional): Parameters of grid search, for more detail please see
            https://scikit-learn.org/stable/modules/grid_search.html#grid-search . Defaults to None, i.e., using the
            default params: {"cv": 5}.
    """

    def __init__(
        self,
        classifier="svc",
        classifier_params="auto",
        classifier_param_grid=None,
        mpca_params=None,
        n_features=None,
        search_params=None,
    ):
        if classifier not in ["svc", "linear_svc", "lr"]:
            error_msg = "Valid classifier should be 'svc', 'linear_svc', or 'lr', but given %s" % classifier
            logging.error(error_msg)
            raise ValueError(error_msg)

        self.classifier = classifier
        # init mpca object
        if mpca_params is None:
            self.mpca_params = default_mpca_params
        else:
            self.mpca_params = mpca_params
        self.mpca = MPCA(**self.mpca_params)
        # init feature selection parameters
        self.n_features = n_features
        self.feature_order = None
        # init classifier object
        if search_params is None:
            self.search_params = default_search_params
        else:
            self.search_params = search_params
        self.classifier_param_grid = classifier_param_grid

        self.auto_classifier_param = False
        if classifier_params == "auto":
            self.auto_classifier_param = True
            if self.classifier_param_grid is None:
                self.classifier_param_grid = classifiers[classifier][1]
            self.grid_search = GridSearchCV(
                classifiers[classifier][0](), param_grid=self.classifier_param_grid, **self.search_params
            )
            self.clf = None
        elif isinstance(classifier_params, dict):
            self.clf = classifiers[classifier][0](**classifier_params)
        else:
            error_msg = "Invalid classifier parameter type"
            logging.error(error_msg)
            raise ValueError(error_msg)

        self.classifier_params = classifier_params

    def fit(self, x, y):
        """Fit a pipeline with the given data x and labels y

        Args:
            x (array-like tensor): input data, shape (n_samples, I_1, I_2, ..., I_N)
            y (array-like): data labels, shape (n_samples, )

        Returns:
            self
        """
        # fit mpca
        self.mpca.fit(x)
        self.mpca.set_params(**{"vectorize": True})
        x_transformed = self.mpca.transform(x)

        # feature selection
        if self.n_features is None:
            self.n_features = x_transformed.shape[1]
            self.feature_order = self.mpca.idx_order
        else:
            f_score, p_val = f_classif(x_transformed, y)
            self.feature_order = (-1 * f_score).argsort()
        x_transformed = x_transformed[:, self.feature_order][:, : self.n_features]

        # fit classifier
        if self.auto_classifier_param:
            self.grid_search.param_grid["C"].append(1 / x.shape[0])
            self.grid_search.fit(x_transformed, y)
            self.clf = self.grid_search.best_estimator_
        if self.classifier == "svc":
            self.clf.set_params(**{"probability": True})

        self.clf.fit(x_transformed, y)

    def predict(self, x):
        """Predict the labels for the given data x

        Args:
            x (array-like tensor): input data, shape (n_samples, I_1, I_2, ..., I_N)

        Returns:
            array-like: Predicted labels, shape (n_samples, )
        """
        return self.clf.predict(self._extract_feature(x))

    def decision_function(self, x):
        """Decision scores of each class for the given data x

        Args:
            x (array-like tensor): input data, shape (n_samples, I_1, I_2, ..., I_N)

        Returns:
            array-like: decision scores, shape (n_samples,) for binary case, else (n_samples, n_class)
        """
        return self.clf.decision_function(self._extract_feature(x))

    def predict_proba(self, x):
        """Probability of each class for the given data x. Not supported by "linear_svc".

        Args:
            x (array-like tensor): input data, shape (n_samples, I_1, I_2, ..., I_N)

        Returns:
            array-like: probabilities, shape (n_samples, n_class)
        """
        if self.classifier == "linear_svc":
            error_msg = "Linear SVC does not support computing probability."
            logging.error(error_msg)
            raise ValueError(error_msg)
        return self.clf.predict_proba(self._extract_feature(x))

    def _extract_feature(self, x):
        """Extracting features for the given data x with MPCA->Feature selection

        Args:
            x (array-like tensor): input data, shape (n_samples, I_1, I_2, ..., I_N)

        Returns:
            array-like: n_new, shape (n_samples, n_features)
        """
        check_is_fitted(self.clf)
        x_transformed = self.mpca.transform(x)

        return x_transformed[:, self.feature_order][:, : self.n_features]


import unittest
import time
from collections import defaultdict

def urlify_pythonic(text: str, length: int) -> str:
    return text[:length].replace(" ", "%20")

def urlify_algo(text: str, length: int) -> str:
    """
    Replaces the white spaces with "%20" and removes trailing spaces.
    """
    cropped_txt = text[:length]
    replaced_txt = str()
    for char in cropped_txt:
        if char == " ":
            replaced_txt += "%20"
        else:
            replaced_txt += char
    return replaced_txt

class Test_urlify(unittest.TestCase):
    test_cases = {
        ("much ado about nothing      ", 22): "much%20ado%20about%20nothing",
        ("Mr John Smith       ", 13): "Mr%20John%20Smith",
        (" a b    ", 4): "%20a%20b",
        (" a b       ", 5): "%20a%20b%20",
        (" a b ", 4): "%20a%20b"
    }

    test_functions = [
        urlify_pythonic,
        urlify_algo
    ]

    def test_urlify(self):
        number_runs = 1000
        runtimes = defaultdict(float)
        for _ in range(number_runs):
            for fctn in self.test_functions:
                for args, output in self.test_cases.items():
                    start = time.perf_counter()
                    res = fctn(*args)
                    self.assertEqual(res, output)
                    runtimes[fctn.__name__] += (time.perf_counter() - start) * 1000

        print(f"{number_runs} runs")
        for fctn_name, run_time in runtimes.items():
            print(f"{fctn_name}: {run_time: .1f}ms")

if __name__ == '__main__':
    unittest.main()


import sys
import random
from itertools import combinations
from typing import List

N = random.randint(99990, 100000)
points = [(random.randint(-100000, 100000), random.randint(-100000, 100000))
           for _ in range(N)]

# N = 4
# points = [(-6, 1), (-6, -2), (2, -3), (3, 2)]
points.sort(key=lambda x: x[0])

# print(N)
# print(points)

def len_point_square(point1, point2: tuple) -> float:
    return (point2[0] - point1[0])**2 + (point2[1] - point1[1])**2

def find_nearest_pair_len(N: int, points: List) -> int:
    if len(points) < 4:
        nearest_len_square = 800000000
        for point1, point2 in combinations(points, 2):
            tmp = len_point_square(point1, point2)
            if nearest_len_square > tmp:
                nearest_len_square = tmp

        return nearest_len_square

    nearest_len_square = min(find_nearest_pair_len(N//2, points[:N//2]), find_nearest_pair_len(N-N//2, points[N//2:]))
    nearest_len = nearest_len_square**(1/2)

    x_middle = (points[N//2-1][0] + points[N//2][0])/2
    left_points, right_points = [], []
    for point in reversed(points[:N//2]):
        if x_middle - point[0] < nearest_len:
            left_points.append(point)
        else:
            break

    if left_points:
        left_y_min = min(left_points, key=lambda x:x[1])
        left_y_max = max(left_points, key=lambda x:x[1])
    else:
        left_y_min = 800000000
        left_y_max = 0

    for point in points[N//2:]:
        if point[0] - x_middle < nearest_len and (abs(point[1] - left_y_min[1]) < nearest_len or abs(point[1] - left_y_max[1]) < nearest_len):
            right_points.append(point)
        else:
            break

    for lpoint in left_points:
        for rpoint in right_points:
            tmp = len_point_square(lpoint, rpoint)
            if nearest_len_square > tmp:
                nearest_len_square = tmp
    return nearest_len_square

print(find_nearest_pair_len(N, points))


import os, sys

import numpy as np
# from garl_gym.scenarios.simple_population_dynamics_rule_base import *
from tqdm import tqdm

def run_rulebase(env, args):
    model_dir = os.path.join('results', 'exp_{:d}'.format(args.experiment_id), 'models')
    bar = tqdm(range(args.test_step))
    try:
        os.makedirs(model_dir)
    except:
        shutil.rmtree(model_dir)
        os.makedirs(model_dir)
    rounds = 0

    img_dir = os.path.join('results', 'exp_{:d}'.format(args.experiment_id), 'images',str(rounds))
    log_dir = os.path.join('results', 'exp_{:d}'.format(args.experiment_id), 'logs', str(rounds))
    try:
        os.makedirs(img_dir)
    except:
        shutil.rmtree(img_dir)
        os.makedirs(img_dir)
    try:
        os.makedirs(log_dir)
    except:
        shutil.rmtree(log_dir)
        os.makedirs(log_dir)
    log = open(os.path.join(log_dir, 'log.txt'), 'w')

    for i in range(args.test_step):
        obs = get_obs(env, only_view=True)
        if args.video_flag:
            env.dump_image(os.path.join(img_dir, '{:d}.png'.format(i)))
        env.take_actions()

        next_view_batches, rewards, killed = get_obs(env)
        env.killed = killed
        episode_reward = np.sum(list(rewards.values()))

        killed = env.remove_dead_agents()

        view_batches = next_view_batches
        msg = "episode {:03d} episode step {:03d} reward:{:5.3f}".format(0, i, episode_reward/len(obs))
        bar.set_description(msg)
        bar.update(1)

        info = "Episode\t{:03d}\tStep\t{:03d}\tReward\t{:5.3f}\tnum_agents\t{:d}\tnum_preys\t{:d}\tnum_predators\t{:d}".format(0, i, episode_reward/len(obs), len(env.agents), len(self.env.preys), len(self.env.predators))
        log.write(info+'\n')
        log.flush()

        env.crossover_prey(self.args.crossover_scope, crossover_rate=self.args.prey_increase_prob)
        #env.increase_prey(self.args.prey_increase_prob)
        env.crossover_predator(self.args.crossover_scope, crossover_rate=self.args.predator_increase_prob)

    log.close()


#!/usr/bin/eny python
# -*- coding:utf-8 -*-
import urllib.request
import urllib.parse
from bs4 import BeautifulSoup
import re

def askURL(url):
    html = ""
    head = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/87.0.664.75"}
    request = urllib.request.Request(url,headers=head);
    # print (reponse.read().decode("utf-8"))
    try:
        reponse = urllib.request.urlopen(request)
        html = reponse.read()
    except urllib.error.URLError as e:
        print("error")

    return html

def getData():
    url = "https://www.baidu.com/s?ie=utf-8&medium=1&rtt=4&bsst=1&rsv_dl=news_b_pn&cl=2&wd=%E7%96%AB%E6%83%85&tn=news&rsv_bp=1&tfflag=0&x_bfe_rqs=03E80000001&x_bfe_tjscore=0.000000&tngroupname=organic_news&newVideo=12&pn=520"
    html = askURL(url)
    bs = BeautifulSoup(html,"lxml")
    # print(bs.title.string)

    for item in bs.find_all("div",class_='result-op c-container xpath-log new-pmd'):
        print(item.find_all("h3"))

getData()