import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

def my_softplus(x, tau=1., threshold=20.):
    truncate_mask = (x > threshold).type(torch.cuda.FloatTensor)
    return truncate_mask * x + (1. - truncate_mask) * (tau * torch.log(1 + torch.exp((1. - truncate_mask) * x / tau)))

def my_softplus_derivative(x, tau=1., threshold=20.):
    truncate_mask = (x > threshold).type(torch.cuda.FloatTensor)
    return truncate_mask + (1. - truncate_mask) / (1 + torch.exp(- (1. - truncate_mask) * x / tau) )

class Net(nn.Module):
    def __init__(self, \
            input_dim, \
            output_dim, \
            hidden_dim, \
            num_layer, \
            num_back_layer, \
            dense = False, \
            drop_type = 'none', \
            net_type = 'locally_constant', \
            approx = 'none'):
        super(Net, self).__init__()

        self.input_dim = input_dim
        self.output_dim = output_dim
        self.hidden_dim = hidden_dim
        self.num_layer = num_layer
        self.num_back_layer = num_back_layer
        self.dense = dense
        self.drop_type = drop_type
        self.net_type = net_type
        self.n_neuron = self.hidden_dim * self.num_layer
        self.approx = approx

        self.layer = nn.ModuleList()

        self.weights = dict()
        self.biases = dict()
        self.output_constants = dict()
        self.output_weights = dict()
        self.output_biases = dict()

        accu_dim = input_dim
        self.weight_masks = []
        for i in range(self.num_layer):
            self.layer.append(nn.Linear(accu_dim, hidden_dim))
            ite_mask = np.zeros((1, hidden_dim, accu_dim))
            pick = np.random.choice(accu_dim, int(np.sqrt(accu_dim)), replace=False)
            ite_mask[0, 0, pick] = 1
            assert(hidden_dim == 1)
            ite_mask = torch.tensor(ite_mask.astype(np.float32)).cuda()
            self.weight_masks.append(ite_mask)

            if self.dense:
                accu_dim += hidden_dim
            else:
                accu_dim = hidden_dim

        if self.net_type == 'locally_constant':
            self.backward_layer = nn.ModuleList()
            backward_dim = 256
            cur_dim = self.num_layer * self.hidden_dim * (1 + self.input_dim)
            for i in range(self.num_back_layer):
                self.backward_layer.append(nn.Linear(cur_dim, backward_dim))
                cur_dim = backward_dim
            self.backward_layer.append(nn.Linear(cur_dim, output_dim))

        elif self.net_type == 'locally_linear':
            self.output_fc = nn.Linear(accu_dim, self.output_dim)
        else:
            print('net_type', self.net_type, 'is not supported')
            exit(0)

    def normal_forward(self, init_layer, p=0, training=True):
        assert(self.net_type == 'locally_linear')
        relu_masks = []
        if len(init_layer.shape) == 4:
            bz = init_layer.shape[0]
            init_layer = init_layer.view(bz, -1)
        cur_embed = init_layer
        batch_size = init_layer.shape[0]

        for i in range(self.num_layer):
            if training == False:
                next_embed = self.layer[i](cur_embed)
            else:
                w = self.layer[i].weight
                w = w.view(1, w.shape[0], w.shape[1]).expand(batch_size, -1, -1)

                b = self.layer[i].bias
                b = b.view(1, b.shape[0]).expand(batch_size, -1)

                next_embed = torch.bmm(F.dropout(w, p=p, training=training), cur_embed.unsqueeze(-1)) + b.unsqueeze(-1)
                next_embed = next_embed.squeeze(-1)

            relu_masks.append( (next_embed > 0) )
            next_embed = F.relu(next_embed)
            if self.dense:
                cur_embed = torch.cat((cur_embed, next_embed), 1)
            else:
                cur_embed = next_embed
        return self.output_fc(cur_embed), relu_masks

    def forward(self, init_layer, p=0, training=True, alpha=None, anneal='none'):
        assert(self.net_type == 'locally_constant')
        relu_masks = []
        if len(init_layer.shape) == 4:
            bz = init_layer.shape[0]
            init_layer = init_layer.view(bz, -1)
        # cur_embed is used for forward computation.
        cur_embed = init_layer
        # x is used to compute Jacobian using dynamic programming.
        x = init_layer.unsqueeze(-1)

        batch_size = x.shape[0]
        patterns = []
        for i in range(self.num_layer):
            if self.drop_type != 'node_dropconnect':
                next_embed = self.layer[i](cur_embed)
            w = self.layer[i].weight
            w = w.view(1, w.shape[0], w.shape[1]).expand(batch_size, -1, -1)

            if self.drop_type == 'node_dropconnect':
                b = self.layer[i].bias
                b = b.view(1, b.shape[0]).expand(batch_size, -1)

                next_embed = torch.bmm(F.dropout(w, p=p, training=training), cur_embed.unsqueeze(-1)) + b.unsqueeze(-1)
                next_embed = next_embed.squeeze(-1)
            else:
                pass

            relu_masks.append( (next_embed > 0) )
            if self.approx == 'approx':
                neur_deriv = my_softplus_derivative(next_embed)
                next_embed = my_softplus(next_embed)
            elif anneal == 'interpolation':
                neur_deriv = alpha * (next_embed > 0).type(torch.cuda.FloatTensor) + (1 - alpha) * my_softplus_derivative(next_embed)
                next_embed = alpha * F.relu(next_embed) + (1 - alpha) * my_softplus(next_embed)
            elif anneal == 'none':
                neur_deriv = (next_embed > 0).type(torch.cuda.FloatTensor)
                next_embed = F.relu(next_embed)

            patterns.append(neur_deriv)
            neur_deriv = neur_deriv.unsqueeze(-1)

            if i == 0:
                jacobians = neur_deriv * w
                offsets = next_embed - torch.bmm(jacobians, x).squeeze(-1)
            else:
                if self.dense:
                    ite_jacobians = w[:, :, :self.input_dim] + torch.bmm(w[:, :, self.input_dim:], jacobians)
                else:
                    ite_jacobians = torch.bmm(w, jacobians[:, -self.hidden_dim:, :])

                ite_jacobians = neur_deriv * ite_jacobians
                ite_offsets = next_embed - torch.bmm(ite_jacobians, x).squeeze(-1)

                jacobians = torch.cat([jacobians, ite_jacobians], dim=1)
                offsets = torch.cat([offsets, ite_offsets], dim=1)

            if self.dense:
                cur_embed = torch.cat((cur_embed, next_embed), 1)
            else:
                cur_embed = next_embed

        leaf_input = torch.cat([jacobians, offsets.unsqueeze(-1)], dim=2).view(batch_size, -1)
        for i in range(self.num_back_layer):
            leaf_input = F.relu(self.backward_layer[i](leaf_input))
        leaf_output = self.backward_layer[-1](leaf_input)

        return leaf_output, relu_masks


import os
import pickle

import numpy as np
import pandas as pd
import torch
from dex_ycb_toolkit.factory import get_dataset
from manopth.manolayer import ManoLayer
from tqdm import tqdm

def build_frame_index(root_meta='../local_data/dex-ycb/meta', subfolder="s0_train", use_cache=True, cache_folder="../data/DexYCB/cache"):
    """
    root_meta: root of meta pkl file
    subfolder: one of  's0_train', 's0_val', 's0_test'
    """

    seqs = []
    for sub in os.listdir(os.path.join(root_meta,subfolder)):
        for sub1 in os.listdir(os.path.join(root_meta,subfolder,sub)):
            for sub2 in os.listdir(os.path.join(root_meta,subfolder,sub,sub1)):
                seqs.append(os.path.join(sub,sub1,sub2))

    cache_path = os.path.join(cache_folder, f"{subfolder}.pkl")
    if not os.path.exists(cache_path):
        os.makedirs(cache_folder, exist_ok=True)
    if use_cache and os.path.exists(cache_path):
        with open(cache_path, "rb") as p_f:
            pd_frame_index, annotations_map = pickle.load(p_f)
    else:
        with open(f'../local_data/dex-ycb/meta/Dex_{subfolder[3:]}_filter_1cm.pth', 'rb') as f:
          filter_index = pickle.load(f)
        annotations_map = {}
        annotations_list = []
        seq_lengths = {}
        # for seq in tqdm(sorted(seqs), desc="seq"):
        scene_id = 0
        for seq in tqdm(sorted(seqs)):
            # seq_folder = os.path.join(root_original, seq)
            meta_folder = os.path.join(root_meta,subfolder, seq)
            # rgb_folder = seq_folder

            img_nb = len(os.listdir(meta_folder))
            seq_lengths[seq] = img_nb
            # for frame_idx in tqdm(range(img_nb), desc="img"):
            for frame_idx in range(img_nb):
                meta_path = os.path.join(meta_folder, f"meta_{frame_idx:06d}.pkl")
                with open(meta_path, "rb") as p_f:
                    annot = pickle.load(p_f)
                # img_path = os.path.join(rgb_folder, f"color_{frame_idx:06d}.jpg")
                # annot["img"] = img_path
                annot["seq_idx"] = seq
                annot["frame_idx"] = frame_idx
                annot["frame_nb"] = img_nb
                # annot["scene_id"] = scene_id
                # annot["view_id"] = 0
                assert len(annot['mano_pose']) == 1
                #  and (seq, frame_idx) not in filter_index
                if not (annot['mano_pose'] == 0).all() and annot['mano_side'] == 'right' and (seq, frame_idx) not in filter_index:
                  scene_id += 1
                  annotations_map[(seq, frame_idx)] = annot
                  annotations_list.append(annot)
        pd_frame_index = pd.DataFrame(
                annotations_list).loc[:, ["img", "seq_idx", "frame_idx", "frame_nb"]]
        with open(cache_path, "wb") as p_f:
            pickle.dump((pd_frame_index, annotations_map), p_f)
    return pd_frame_index, annotations_map

def create_scene(sample, obj_file, name):
  """Creates the pyrender scene of an image sample.

  Args:
    sample: A dictionary holding an image sample.
    obj_file: A dictionary holding the paths to YCB OBJ files.

  Returns:
    A pyrender scene object.
  """

  ret = {}

  # Add camera.
  fx = sample['intrinsics']['fx']
  fy = sample['intrinsics']['fy']
  cx = sample['intrinsics']['ppx']
  cy = sample['intrinsics']['ppy']

  cam = np.eye(3)
  cam[0,0] = fx
  cam[1,1] = fy
  cam[0,-1] = cx
  cam[1,-1] = cy

  ret['camMat'] = cam

  # Load poses.
  label = np.load(sample['label_file'])
  pose_y = label['pose_y']
  pose_m = label['pose_m']

  # Load YCB meshes.
  i = sample['ycb_ids'][sample['ycb_grasp_ind']]
  obj_dir = obj_file[i]
  obj_pose = pose_y[sample['ycb_grasp_ind']]

  ret['objDir'] = obj_dir.replace('/home/wei/Documents/1T/','./')
  ret['objRt'] = obj_pose
  ret['img'] = sample['color_file'].replace('/home/wei/Documents/1T/','./')
  ret['mano_betas'] = sample['mano_betas']
  ret['mano_side'] = sample['mano_side']

  # Load MANO layer.
  mano_layer = ManoLayer(flat_hand_mean=False,
                         ncomps=45,
                         side=sample['mano_side'],
                         mano_root='/home/wei/Documents/projects/2022-hand-object/dex-ycb-toolkit/manopth/mano/models',
                         use_pca=True)
  mano_layer_ho3d = ManoLayer(
                              joint_rot_mode="axisang",
                              use_pca=False,
                              mano_root='/home/wei/Documents/projects/2022-hand-object/dex-ycb-toolkit/manopth/mano/models',
                              center_idx=None,
                              flat_hand_mean=True,
                              side=sample['mano_side'],
                              )
  faces = mano_layer.th_faces.numpy()
  betas = torch.tensor(sample['mano_betas'], dtype=torch.float32).unsqueeze(0)

  # Add MANO meshes.
  if not np.all(pose_m == 0.0):
    pose = torch.from_numpy(pose_m)
    trans = pose[:, 48:51]
    pose = pose[:, 0:48]
    ret['hand_trans'] = trans.cpu().data.numpy()
    th_hand_pose_coeffs = pose[:, mano_layer.rot:mano_layer.rot +mano_layer.ncomps]
    th_full_hand_pose = th_hand_pose_coeffs.mm(mano_layer.th_selected_comps)
    # Concatenate back global rot
    th_full_pose = torch.cat([
      pose[:, :mano_layer.rot],
      mano_layer.th_hands_mean + th_full_hand_pose
    ], 1)
    ret['mano_pose'] = th_full_pose.cpu().data.numpy()
    if np.random.rand(1)[0] < 0.01:
      vert, _, _ = mano_layer(pose, betas, trans)
      vert_ho3d, _, _ = mano_layer_ho3d(th_full_pose,betas,trans)
      assert (vert_ho3d-vert).abs().max() < 0.0001
  else:
    ret['mano_pose'] = np.zeros([1,48])
    ret['hand_trans'] = np.zeros([1,3])

  ret_dir = os.path.join(f'/home/wei/Documents/dex-ycb/{name}',*(ret['img'].split('/')[2:-1]))
  if not os.path.exists(ret_dir):
    os.makedirs(ret_dir)
  frame_id = int(ret['img'].split('/')[-1][:-4].split('_')[-1])
  ret_file = ret_dir + f'/meta_{frame_id:06d}.pkl'
  with open(ret_file, "wb") as p_f:
    pickle.dump(ret, p_f)

def main():
  names = ['s0_train','s0_val','s0_test']

  for name in names:
    dataset = get_dataset(name)

    for idx in tqdm(range(len(dataset))):
      # idx = 70
      sample = dataset[idx]

      create_scene(sample, dataset.obj_file, name)

if __name__ == '__main__':
  build_frame_index()
  # main()


import collections

import torch
import torch.nn as nn
import torchvision
import torchvision.transforms as transforms
import tqdm
import re
import random
import string
import os, time
from torch.utils.data import DataLoader, Subset
from const import PATIENCE_EPOCH
import numpy as np
from sklearn.datasets import fetch_20newsgroups

class CustomDataset:
    def __init__(self, dataset_name, root=None, transform=None, train=True, download=True, data=None):
        """
        Input:
        dataset_name
            string of desired dataset name
        root
            string of data download destination
        train
            bool to specify wheter the dataset is for train or test
        download
            bool to specify whether to download the dataset specified by dataset_name
        transform
            operations to perform transformation on the dataset
        data
            solely used during check interval
        """
        if dataset_name == "CIFAR10":
            self.dataset = torchvision.datasets.CIFAR10(root=root, train=train, download=download, transform=transform)
        elif dataset_name == "CIFAR100":
            self.dataset = torchvision.datasets.CIFAR100(root=root, train=train, download=download, transform=transform)
        elif dataset_name == "20news":
            self.dataset = fetch_20newsgroups(subset='all', remove=('headers', 'footers'))

            alphanumeric = lambda x: re.sub(r"""\w*\d\w*""", '', x)
            punc_lower = lambda x: re.sub('[%s]' % re.escape(string.punctuation), '', x.lower())

            texts = [alphanumeric(punc_lower(text)) for text in self.dataset.data]
            self.dataset = list(zip(texts, self.dataset.target))
        elif dataset_name == "check":
            self.dataset = data
        else:
            raise NotImplementedError

    def __getitem__(self, index):
        sample, label = self.dataset[index]
        return sample, label

    def __len__(self):
        return len(self.dataset)

def eval(model, test_dataloader, device):
    """
    Input:
    model
        pytorch model object to eval
    test_dataloader
        pytorch dataloader
    device
        string of either 'cuda' or 'cpu'

    Return:
        float accuracy on the validation set
    """
    model.eval()

    match_count, total_count = 0, 0
    for (X, y) in test_dataloader:
        with torch.no_grad():
            X, y = X.to(device), y.to(device)
            logits = model(X)
            match_count += torch.sum(torch.argmax(logits, dim=1) == y)
            total_count += len(X)

    return match_count / total_count

def train(model, epoch, train_dataset, test_dataloader, device, args, start = 0):
    """
    Input:
    model
        pytorch model object to train
    epoch
        int number of total training epochs
    train_dataset
        dataset object which satisfies pytorch dataset format
    test_dataloader
        pytorch dataloader
    device
        string of either 'cuda' or 'cpu'
    """

    criterion = nn.CrossEntropyLoss()
    opt = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=5e-4)

    best_val_acc = 0
    patience = 0

    # indices of examples that have been memorized by the model in a streak
    memorized_indices = set()
    # counter to record memory streak
    memory_streak_counter = collections.defaultdict(int)

    for e in range(epoch):
        model.train()

        train_loss = 0
        batch_size = args.batch_size

        # if in need of memorized fraction, we do not remove any data points from training process
        # else, we remove memorized ones
        if args.return_memorized_fraction:
            feed_indices = torch.randperm(len(train_dataset)).tolist()
        else:
            feed_indices = [i for i in torch.randperm(len(train_dataset)).tolist() if i not in memorized_indices]
        shuffled_dataset = Subset(train_dataset, feed_indices)
        dataloader = DataLoader(shuffled_dataset, batch_size, shuffle=False)

        with tqdm.tqdm(total=len(dataloader) * batch_size, unit='it', unit_scale=True, unit_divisor=1000) as pbar:
            sum_acc = 0
            total = 0
            for X, y in dataloader:
                if X.shape[0]==1: continue # avoid batch size = 1
                X, y = X.to(device), y.to(device)
                opt.zero_grad()
                logits = model(X)
                loss = criterion(logits, y)

                # if we are not evaluating baseline
                # then begin recording the memory streak
                # remove examples from training dataset if they met the streak
                if not args.baseline:
                    match = logits.argmax(dim=1) == y
                    batch_indices = feed_indices[total: total + X.shape[0]]
                    for idx, i in enumerate(batch_indices):
                        if memory_streak_counter[i] == -1: continue
                        if match[idx]:
                            memory_streak_counter[i] += 1
                            if memory_streak_counter[i] == args.streak:
                                memorized_indices.add(i)
                                memory_streak_counter[i] = 0
                        else:
                            # if we need fraction, set every wrongly predicted data point which has been correctly
                            # predicted before as -1
                            if args.return_memorized_fraction and memory_streak_counter[i] > 0:
                                memory_streak_counter[i] = -1
                            else:
                                memory_streak_counter[i] = 0

                acc = torch.sum(torch.argmax(logits, dim=1) == y).item()
                loss.backward()
                opt.step()

                # progress bar counter
                train_loss += loss.item()
                pbar.update(batch_size)
                sum_acc += acc
                total += X.shape[0]
                pbar.set_postfix(loss=loss.item(),
                                 acc=sum_acc / total)

        # TODO: feel free to remove this print
        if not args.baseline:
            print(f"Epoch {e} - size of memorized data {len(memorized_indices)}, size of training data {len(dataloader) * batch_size}" )

        validation_accuracy = eval(model, test_dataloader, device)
        print("Epoch {} - Training loss: {:.4f}, Validation Accuracy: {:.4f}, time {:.2f}".format(e, train_loss / len(dataloader),
                                                                                validation_accuracy, time.time()-start))

        # if we met the check interval, then check examples that are supposedly memorized
        # if the model forgot about them, then move them back to the training dataset
        # TODO: in fraction computation, do we check the memorized or not?
        if e > args.streak and e % args.check_interval == 0 and not args.return_memorized_fraction:
            print('Entering check interval')
            memorized_indices_list = list(memorized_indices)
            memorized_dataset = CustomDataset("check", data=[train_dataset[i] for i in memorized_indices_list])
            memorized_dataloader = DataLoader(memorized_dataset, batch_size, shuffle=False)
            memorized_total = 0
            forgot_count = 0
            with torch.no_grad():
                for X, y in memorized_dataloader:
                    X, y = X.to(device), y.to(device)
                    logits = model(X)
                    match = torch.argmax(logits, dim=1) == y #fix
                    batch_indices = memorized_indices_list[memorized_total: memorized_total + X.shape[0]]
                    for idx, i in enumerate(batch_indices):
                        if not match[idx]:
                            memorized_indices.remove(i)
                            forgot_count += 1
                    memorized_total += X.shape[0] #fix
            print("Model forgets {} examples out of {} total memorized ones".format(forgot_count,
                                                                                len(memorized_indices_list)))

        # Early stopping
        if args.early_stop:
            if validation_accuracy > best_val_acc:
                best_val_acc = validation_accuracy
                patience = 0
            else:
                patience += 1
                if patience > PATIENCE_EPOCH:
                    print("Training early stops at epoch {}".format(e))
                    break

    if args.return_memorized_fraction:
        truly_unforgettable = [k for k, v in memory_streak_counter.items() if v > 0]
        print(f"num unforgettable = {len(truly_unforgettable)}, num memorized indices = {len(memorized_indices)}  ")
        return len(memorized_indices) / len(truly_unforgettable), truly_unforgettable

    return memorized_indices

def SaveEnvironment():
    pass

def GenerateEnvironment(args):
    seeds = ''.join(random.choices(string.ascii_uppercase + string.digits, k=10))
    result_dir = './results'
    if not os.path.isdir(result_dir):
        os.mkdir(result_dir)
    expr_path = os.path.join(result_dir, seeds).replace("\\","/")
    args_path = os.path.join(expr_path, 'args.txt').replace("\\","/")
    os.mkdir(expr_path)
    print('Create enviornment at : {}'.format(expr_path))
    with open(args_path, 'w') as F:
        DumpOptionsToFile(args, F)
    return expr_path

def DumpOptionsToFile(args, fp):
    d = vars(args)
    for key,value in d.items():
        if type(value) == str:
            fp.write('{} = "{}"\n'.format(key, value))
        else:
            fp.write('{} = {}\n'.format(key, value))

def DumpIndicesToFile(noise, dataset_name, dir):
    dest = os.path.join(dir, dataset_name).replace("\\","/")
    with open(dest, 'w') as f:
        for n in noise:
            f.write(str(n) + '\n')
    return dest

# noisy
class ShuffledDataset():
    def __init__(self, dataset_name, root, shuffle_percentage, transform=None, train=True, download=True, noise_type='symmetric'):
        """
        Input:
        dataset_name
            string of desired dataset name
        root
            string of data download destination
        shuffle_percentage
            float between [0, 1] to specify what fraction of train data to have incorrect labels
        train
            bool to specify wheter the dataset is for train or test
        download
            bool to specify whether to download the dataset specified by dataset_name
        transform
            operations to perform transformation on the dataset
        """
        if dataset_name == "CIFAR10":
            self.dataset = torchvision.datasets.CIFAR10(root=root, train=train, download=download, transform=transform)
            self.label_len = 10
        elif dataset_name == "CIFAR100":
            self.dataset = torchvision.datasets.CIFAR100(root=root, train=train, download=download, transform=transform)
            self.label_len = 100
        else:
            raise NotImplementedError

        # Force the test set to be intact
        percentage = train * shuffle_percentage
        dataset_len = len(self.dataset)
        if noise_type=='asymmetric':
            self._create_shuffle_mapping_asymmetric_cifar10(self.dataset.targets, percentage)
        else: #uniform, symmetric noise
            indices_to_shuffle = random.sample(range(dataset_len), int(percentage * dataset_len))
            self._create_shuffle_mapping(indices_to_shuffle)

    def _create_shuffle_mapping(self, indices):
        """
        Input:
        indices
            list of int to specify which samples to shuffle label
        """
        self.mapping = {}
        for index in indices:
            _, label = self.dataset[index]
            label_range = list(range(self.label_len))
            label_range.remove(label)
            new_label = random.choice(label_range)
            self.mapping[index] = new_label

    def _create_shuffle_mapping_asymmetric_cifar10(self, y_train, noise, random_state=None, nb_classes=10):
        """mistakes:
            flip in the symmetric way
        """
        self.mapping = {}
        if noise == 0:
            return
        print("asymmetric shuffle")
        source_class = [9, 2, 3, 5, 4]
        target_class = [1, 0, 5, 3, 7]

        for s, t in zip(source_class, target_class):
            cls_idx = np.where(np.array(y_train) == s)[0]
            n_noisy = int(noise * cls_idx.shape[0])
            noisy_sample_index = np.random.choice(cls_idx, n_noisy, replace=False)
            for idx in noisy_sample_index:
                self.mapping[idx] = t

    def __getitem__(self, index):
        sample, label = self.dataset[index]
        label = self.mapping.get(index, label)
        return (sample, label)

    def get_shuffle_mapping(self):
        return self.mapping

    def __len__(self):
        return len(self.dataset)

    def cleanse(self,remove_list):
        mask = torch.ones(len(self.dataset), dtype=bool)
        mask[remove_list] = False
        self.dataset.data = self.dataset.data[mask]
        self.dataset.targets = (torch.tensor(self.dataset.targets)[mask]).tolist()
        self.mapping = {}


from django.shortcuts import render
from django.contrib.auth.decorators import login_required
from inventory.models import CustomerContact, EquipmentDetails, ElementalAnalysisTests, OilAnalysisTests
from oap.models import OAPReport, OilAnalysis, ElementalAnalysis, ProgressiveAnalysis
from base.models import Account
from django.utils import timezone
from itertools import zip_longest
from oap.utils import oap_make_json as make_json

@login_required
def create_new_oap(request):
    detect_user = Account.objects.get(id=request.user.id)
    if detect_user.is_sampler:
        if request.method == "POST":
            progressive_analysis_obj = ProgressiveAnalysis.objects.create()
            customer = CustomerContact.objects.get(customer_name=request.POST.get('customer_name'))
            equipment = EquipmentDetails.objects.get(equipment_number=request.POST.get('equipment_number'))
            oap_obj = OAPReport.objects.create(
                customer_contact=customer,
                equipment_details=equipment,
                progressive_analysis=progressive_analysis_obj,
                permission_status="Ini",
            )
            oil_analysis_tests = request.POST.getlist('oil_analysis_tests')
            elemental_analysis_tests = request.POST.getlist('elemental_analysis_tests')
            for test in oil_analysis_tests:
                OilAnalysis.objects.create(
                    oap_report=oap_obj,
                    test_name=test,
                    user_id=OilAnalysisTests.objects.get(test_name=test).user_id
                )
            for test in elemental_analysis_tests:
                ElementalAnalysis.objects.create(
                    oap_report=oap_obj,
                    test_name=test,
                    user_id=ElementalAnalysisTests.objects.get(test_name=test).user_id
                )
            request.method = 'GET'
            return view_oap_report(request, oap_obj.id)

    return render(request, 'oap/create_new_oap.html', {
        'customers': CustomerContact.objects.all(),
        'equipments':  EquipmentDetails.objects.all(),
        'eatests': ElementalAnalysisTests.objects.all(),
        'oatests': OilAnalysisTests.objects.all(),

    })

@login_required
def view_oap_reports(request):
    return render(request, 'oap/view_oap_reports.html', {
        'oap_reports': OAPReport.objects.all().prefetch_related(
            'progressive_analysis',
            'customer_contact',
            'equipment_details'
            ),
    })

@login_required
def view_oap_report(request, id):
    oap_obj = OAPReport.objects.get(id=id)
    customer = oap_obj.customer_contact
    equipment = oap_obj.equipment_details
    user = Account.objects.get(id=request.user.id)
    if user.is_sampler:
        if request.method == "POST":
            permission_status = request.POST.get('permission_status')
            progressive_analysis = ProgressiveAnalysis.objects.get(oapreport=oap_obj)
            progressive_analysis.date_sampled = None if request.POST.get('date_sampled') == '' \
                else request.POST.get('date_sampled')
            progressive_analysis.date_received = None if request.POST.get('date_received') == '' \
                else request.POST.get('date_received')
            progressive_analysis.lab_number = request.POST.get('lab_number')
            progressive_analysis.oil_life = request.POST.get('oil_life')
            progressive_analysis.component_life = request.POST.get('component_life')
            progressive_analysis.filter_life = request.POST.get('filter_life')
            progressive_analysis.daily_top_up = request.POST.get('daily_top_up')
            progressive_analysis.oil_changed = request.POST.get('oil_changed')
            progressive_analysis.date_of_oil_renewed = None if \
                request.POST.get('date_of_oil_renewed') == '' else request.POST.get('date_of_oil_renewed')
            progressive_analysis.sump_capacity = request.POST.get('sump_capacity')
            progressive_analysis.reservoir_temperature = request.POST.get('reservoir_temperature')
            progressive_analysis.oil_in = request.POST.get('oil_in')
            progressive_analysis.oil_out = request.POST.get('oil_out')
            progressive_analysis.coolant_in = request.POST.get('coolant_in')
            progressive_analysis.coolant_out = request.POST.get('coolant_out')
            progressive_analysis.save()
            oap_obj.permission_status = permission_status
            oap_obj.save()
        return render(request, 'oap/view_oap_sampler.html', {
        'customer': customer,
        'equipment': equipment,
        'oap_report': oap_obj,
        })
    elif user.is_verifier:
        oatests = OilAnalysis.objects.filter(oap_report=oap_obj, user_id=user.id)
        eatests = ElementalAnalysis.objects.filter(oap_report=oap_obj, user_id=user.id)
        if request.method == 'POST':
            results = []
            for test, result in zip(oatests, request.POST.getlist('oaresult')):
                test.result = result
                test.save()
            for test, result in zip(eatests, request.POST.getlist('earesult')):
                test.result = result
                test.save()

            for ob in OilAnalysis.objects.filter(oap_report=oap_obj):
                result = ob.result
                if result == 'None' or (result is None) or result == '':
                    results.append(result)
            for ob in ElementalAnalysis.objects.filter(oap_report=oap_obj):
                result = ob.result
                if result == 'None' or (result is None) or result == '':
                    results.append(result)
            if not results:
                oap_obj.permission_status = 'Ver'
                pa = ProgressiveAnalysis.objects.get(oapreport=oap_obj)
                pa.date_reported = timezone.now()
                pa.save()
            oap_obj.comment = request.POST.get('comment')
            oap_obj.save()
        return render(request, 'oap/view_oap_verifier.html', {
            'customer': customer,
            'equipment': equipment,
            'oap_report': oap_obj,
            "oatests": oatests,
            'eatests': eatests,
        })
    else:
        if request.method == 'POST':
            oap_obj.permission_status = request.POST.get('permission_status')
            oap_obj.save()
        oatests = OilAnalysis.objects.filter(oap_report=oap_obj)
        eatests = ElementalAnalysis.objects.filter(oap_report=oap_obj)
        permission_status = oap_obj.permission_status
        if permission_status == 'Apr':
            previous_samples = OAPReport.objects.filter(
                customer_contact=customer,
                equipment_details=equipment,
                permission_status='Apr',
                progressive_analysis__date_reported__lt=oap_obj.progressive_analysis.date_reported
            ).order_by('progressive_analysis__date_reported')[:4][::-1]
            ots = list()
            ots.append(list(OilAnalysis.objects.filter(oap_report=oap_obj)))
            ets = list()
            ets.append(list(ElementalAnalysis.objects.filter(oap_report=oap_obj)))
            temp = [ProgressiveAnalysis.objects.filter(oapreport=oap_obj).values()[0], {}, {}, {}, {}]
            for pos, obj in enumerate(previous_samples):
                temp[pos+1] = ProgressiveAnalysis.objects.filter(oapreport=obj).values()[0]
                ots.append(list(OilAnalysis.objects.filter(oap_report=obj)))
                ets.append(ElementalAnalysis.objects.filter(oap_report=obj))
            data = []
            for key, value in temp[0].items():
                if value is not None and value != "None":
                    data.append((key, value, temp[1].get(key), temp[2].get(key), temp[3].get(key), temp[4].get(key)))
            return render(request, 'oap/view_oap_report.html', {
                "oatests": zip_longest(*ots),
                'eatests': zip_longest(*ets),
                'customer': customer,
                'equipment': equipment,
                'data': data,
                'oap_report': oap_obj,
                'ea_chart_data': make_json(data, zip_longest(*ets)),
                'oa_chart_data': make_json(data, zip_longest(*ots)),

            })
        else:
            return render(request, 'oap/view_oap_report.html', {
                "oatests": oatests,
                'eatests': eatests,
                'customer': customer,
                'equipment': equipment,
                'oap_report': oap_obj,
            })


import data_processing as dp
import time
import numpy as np
from dtaidistance import dtw
import sys

if __name__ == '__main__':
    start_time = time.time()
    np.random.seed(0)

    '''get data from multiple csv files and merge them into one pandas dataframe'''
    path = r'data'
    data = dp.get_multiple_csvs(path)
    data.fillna(method='ffill')
    comp_symbols = data.Name.unique()

    print(data)

    '''convert DataFrame columns into numpy arrays'''
    adj_close_series = dp.col_as_ts(data, comp_symbols, col='Adj Close', normalization='minmax')
    dm = dtw.distance_matrix(adj_close_series)
    print(dm)

    print("--- DTW distance matrix %s seconds ---" % (time.time() - start_time))
    sys.exit()

    print("--- nparray transform in %s seconds ---\n" % (time.time() - start_time))

    linkage_types = ["single", "complete","ward"]
    for lt in linkage_types:
        plt.figure()
        hclust = AgglomerativeClustering(n_clusters=8,
                                        affinity='euclidean',
                                        compute_full_tree=True,
                                        linkage=lt).fit(adj_close_series)
        plt.title('%s Hierarchical Clustering Dendrogram' % lt)
        plot_dendrogram(hclust, labels=comp_symbols)
        plt.show()
    print("--- HIERARCHICAL in %s seconds ---" % (time.time() - start_time))
    print(hclust.labels_)


import argparse
import lib.interfaceUtils as interfaceUtil

def giveArgsAndParser():
    parser = argparse.ArgumentParser(description="Build a Minecraft Settlement, by Tsukuba Team (2021)")
    group = parser.add_mutually_exclusive_group(required=False)
    group.add_argument("-p", "--player",
                       help="Build the settlement around the player's current location",
                       action="store_true")

    group.add_argument("-c", "--coordinates", nargs=6, type=int,
                       metavar=('x0', 'y0', 'z0', 'x1', 'y1', 'z1'),
                       help="Build the settlement on the area defined by these coordinates")

    group.add_argument("-b", "--buildarea",
                       help="Build the settlement using a pre-existing buildArea, no -p, -c or -b argument do the same thing",
                       action="store_true")

    group.add_argument("-d", "--default",
                       help="Build the settlement using a the default buildArea, no -p, -c or -b argument do the same thing",
                       action="store_true")

    parser.add_argument("-a", "--radius", type=int, metavar="A",
                        help="Radius for building area, only meaningful with -p")

    parser.add_argument("-r", "--remove", type=str, metavar="R", nargs='?',
                        help="Remove all structure if debug was activated, temp.txt if r specified, elsewhere file name: -r temp_0.txt")

    args = parser.parse_args()
    return [args, parser]

def getBuildArea(args):
    size = 128

    if args.radius:
        size = args.radius

    if args.player:
        area = interfaceUtil.requestPlayerArea(size * 2, size * 2)
    elif args.coordinates:
        x0, y0, z0, x1, y1, z1 = args.coordinates
        area = interfaceUtil.setBuildArea(x0, y0, z0, x1, y1, z1)
    elif args.default:
        area = interfaceUtil.setBuildArea(0, 0, 0, size, 256, size)
    elif args.buildarea:
        area = interfaceUtil.requestBuildArea()
    else:
        area = interfaceUtil.requestBuildArea()

    print("AREA :" + str(area))
    return area


import asyncio
import os

import pytest

@pytest.fixture(scope="module")
def event_loop():
    loop = asyncio.new_event_loop()
    yield loop
    loop.close()

@pytest.fixture()
def google_api_key():
    key = os.getenv("GOOGLE_API_KEY")
    if not key:
        pytest.skip("Can not test this without GOOGLE_API_KEY envvar")
    return key

@pytest.fixture(scope="session")
def address() -> str:
    return "부천"

@pytest.fixture(scope="session")
def unavailable_address() -> str:
    return "WRONG"


from nose.tools import raises
from mock import MagicMock
from json import loads
from plugins.udp_proxy import UDPProxyPlugin
from . import TestPlugin

class TestUDDProxyPlugin(TestPlugin):
    def setUp(self):
        self.udp_proxy = UDPProxyPlugin()
        self.udp_proxy.log = MagicMock()
        self.checks = self._get_test_checks()
        self.contacts = self._get_test_contacts()

    @raises(Exception)
    def test_create_socket_raises_error(self):
        self.udp_proxy._create_docket = MagicMock(side_effect=Exception)
        self.udp_proxy.on_start()
        self.assertTrue(self.udp_proxy.log.called)

    @raises(Exception)
    def test_forward_raises_error(self):
        self.udp_proxy._create_docket = MagicMock(side_effect=Exception)
        self.assertTrue(self.udp_proxy.log.called)

    def test_forward_payload_format(self):
        self.udp_proxy._fd = MagicMock()
        payload = self.udp_proxy._forward('10.0.0.1', self.checks, self.contacts)
        self.assertEqual(type(loads(payload.strip('\n'))), dict)


from django.urls import path
from .views import (
    PiscinasListView,
    PiscinasCreateView,
    PiscinasUpdateView,
    PiscinasDeleteView,
    PiscinasDetailView,
    ZonasListView,
    ZonasCreateView,
    ZonasUpdateView,
    ZonasDeleteView,
    ZonasDetailView,
)

app_name = 'apps.piscinas'

urlpatterns = [
    path('piscinas/lista/',
         PiscinasListView.as_view(),
         name='lista-piscinas'),
    path('piscinas/<int:pk>/detalle/',
         PiscinasDetailView.as_view(extra_context={'title': 'Piscina'}),
         name='detalle-piscina'),
    path('nueva_piscina/',
         PiscinasCreateView.as_view(extra_context={'title': 'Nueva piscina'}),
         name='nueva-piscina'),
    path('piscinas/<int:pk>/editar/',
         PiscinasUpdateView.as_view(extra_context={'title': 'Modificar piscina'}),
         name='editar-piscina'),
    path('piscinas/<int:pk>/eliminar/',
         PiscinasDeleteView.as_view(extra_context={'title': 'Eliminar piscina'}),
         name='eliminar-piscina'),
    path('zonas/lista/',
         ZonasListView.as_view(), name='lista-zonas'),
    path('zonas/<int:pk>/detalle/',
         ZonasDetailView.as_view(extra_context={'title': 'Zona'}),
         name='detalle-zona'),
    path('nueva_zona/',
         ZonasCreateView.as_view(extra_context={'title': 'Nueva zona'}),
         name='nueva-zona'),
    path('zonas/<int:pk>/editar/',
         ZonasUpdateView.as_view(extra_context={'title': 'Modificar zona'}),
         name='editar-zona'),
    path('zonas/<int:pk>/eliminar/',
         ZonasDeleteView.as_view(extra_context={'title': 'Eliminar zona'}),
         name='eliminar-zona'),
]


import json

def load_candidates_from_json(path):
    with open(path, encoding='utf-8') as file:
        candidates = json.load(file)
    return candidates

def get_candidate(candidate_id):
    for candidate in load_candidates_from_json(data):
        if candidate['id'] == candidate_id:
            return candidate

def get_candidates_by_name(candidate_name):
    result = []
    for candidate in load_candidates_from_json(data):
        if candidate['name'] == candidate_name:
            result.append(candidate)
    return result

def get_candidates_by_skill(skill_name):
    result = []
    for candidate in load_candidates_from_json(data):
        if skill_name in candidate['skills'].lower().split(', '):
            result.append(candidate)
    return result

data = 'candidates.json'

print(get_candidates_by_skill('python'))


import os
import random
import numpy as np
import matplotlib.pyplot as plt
import torch
import imageio
import argparse
import yaml
from omegaconf import OmegaConf
from loader import get_dataset, get_dataloader
from models import get_model

def run(cfg):
    # Setup seeds
    seed = cfg.get("seed", 1)
    print(f"running with random seed : {seed}")
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    np.random.seed(seed)
    random.seed(seed)

    # Setup device
    device = cfg.device

    # Setup Dataloader
    d_datasets = {}
    d_dataloaders = {}
    for key, dataloader_cfg in cfg["data"].items():
        d_datasets[key] = get_dataset(dataloader_cfg)
        d_dataloaders[key] = get_dataloader(dataloader_cfg)

    # Save Figure of Generated Data
    training_data_fig = d_datasets['training'].visualize_data(
        d_datasets['training'].data,
        d_datasets['test'].data
    )

    graph_config = cfg["data"]["training"].get("graph", None)
    if graph_config is not None:
        graph_fig = d_datasets['training'].visualize_graph(
            d_datasets['training'].data,
            d_datasets['test'].data,
            d_datasets['training'].dist_mat_indices
        )

    # Setup Model
    model = get_model(cfg['model']).to(device)
    if graph_config is not None:
        model.dist_indices = d_datasets['training'].dist_mat_indices

    # Iterative Model Update
    params = {k: v for k, v in cfg['optimizer'].items() if k != "name"}
    optimizer = torch.optim.Adam(model.parameters(), **params)
    scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.9)

    list_of_images = []
    for epoch in range(cfg['training']['num_epochs']):
        training_loss = []
        if graph_config is not None:
            for x, x_nn in d_dataloaders['training']:
                train_dict = model.train_step(x.to(device), x_nn.to(device), optimizer)
                training_loss.append(train_dict["loss"])
        else:
            for x in d_dataloaders['training']:
                train_dict = model.train_step(x.to(device), optimizer)
                training_loss.append(train_dict["loss"])
        print(f"n_epoch: {epoch}, training_loss: {sum(training_loss)/len(training_loss)}")

        if epoch > 0.8*cfg['training']['num_epochs']:
            scheduler.step()

        if ((epoch)%40 == 0) or (epoch < 30):
            image_array = model.synthetic_visualize(
                epoch,
                sum(training_loss)/len(training_loss),
                d_datasets['training'].data,
                d_datasets['test'].data,
                device
            )
            list_of_images.append(image_array)

    # Convert Images to GIF
    f = plt.figure()
    model_name = cfg['model']['arch'].upper()
    plt.text(0.5, 0.5, f'{model_name} Training', size=24, ha='center', va='center')
    plt.axis('off')
    f.canvas.draw()
    f_arr = np.array(f.canvas.renderer._renderer)
    plt.close()
    list_figs = [f_arr]*10 + [training_data_fig]*10
    if graph_config is not None:
        list_figs = list_figs + [graph_fig]*10
    list_figs = list_figs + list_of_images
    list_figs = list_figs + [list_of_images[-1]]*20
    imageio.mimsave(
        os.path.join(cfg['logdir'], f'{model_name.upper()}_training.gif'),
        list_figs, duration=0.2)

def parse_arg_type(val):
    if val.isnumeric():
        return int(val)
    if (val == "True") or (val == 'true'):
        return True
    if (val == "False") or (val == 'false'):
        return False
    try:
        return float(val)
    except ValueError:
        return val

def parse_unknown_args(l_args):
    """convert the list of unknown args into dict
    this does similar stuff to OmegaConf.from_cli()
    I may have invented the wheel again..."""
    n_args = len(l_args) // 2
    kwargs = {}
    for i_args in range(n_args):
        key = l_args[i_args * 2]
        val = l_args[i_args * 2 + 1]
        assert "=" not in key, "optional arguments should be separated by space"
        kwargs[key.strip("-")] = parse_arg_type(val)
    return kwargs

def parse_nested_args(d_cmd_cfg):
    """produce a nested dictionary by parsing dot-separated keys
    e.g. {key1.key2 : 1}  --> {key1: {key2: 1}}"""
    d_new_cfg = {}
    for key, val in d_cmd_cfg.items():
        l_key = key.split(".")
        d = d_new_cfg
        for i_key, each_key in enumerate(l_key):
            if i_key == len(l_key) - 1:
                d[each_key] = val
            else:
                if each_key not in d:
                    d[each_key] = {}
                d = d[each_key]
    return d_new_cfg

def save_yaml(filename, text):
    """parse string as yaml then dump as a file"""
    with open(filename, "w") as f:
        yaml.dump(yaml.safe_load(text), f, default_flow_style=False)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--config", type=str)
    parser.add_argument("--logdir", default='results/')
    parser.add_argument("--device", default=0)
    args, unknown = parser.parse_known_args()
    d_cmd_cfg = parse_unknown_args(unknown)
    d_cmd_cfg = parse_nested_args(d_cmd_cfg)
    cfg = OmegaConf.load(args.config)
    cfg = OmegaConf.merge(cfg, d_cmd_cfg)
    if args.device == "cpu":
        cfg["device"] = "cpu"
    else:
        cfg["device"] = f"cuda:{args.device}"
    cfg['logdir'] = os.path.join(args.logdir, cfg['exp_name'])
    print(OmegaConf.to_yaml(cfg))

    # make save dir
    os.makedirs(cfg['logdir'], exist_ok=True)

    # copy config file
    copied_yml = os.path.join(cfg['logdir'], os.path.basename(args.config))
    save_yaml(copied_yml, OmegaConf.to_yaml(cfg))
    print(f"config saved as {copied_yml}")

    run(cfg)


import numpy as np
import pandas as pd
from sklearn import datasets
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from mpl_toolkits.mplot3d import Axes3D

# Read data
pdata = pd.read_csv("pdata.csv", sep=';')

# Plot data
fig = plt.figure()
ax = Axes3D(fig)
for index, row in pdata.iterrows():
    ax.scatter(row['X'], row['Y'], row['Z'], color='b')
    ax.text(row['X'], row['Y'], row['Z'], " x" + str(index + 1))
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')

# Apply principal component analysis
pca = PCA(n_components=2)
pca.fit(pdata)
print(pca.explained_variance_ratio_)

# Transform the data
pdata_transformed = pca.transform(pdata)
pdata_transformed = pd.DataFrame(data={'PC1': pdata_transformed[:, 0], 'PC2': pdata_transformed[:, 1]})

# Plot data
fig, ax = plt.subplots()
for index, row in pdata_transformed.iterrows():
    ax.scatter(row['PC1'], row['PC2'], color='b')
    ax.text(row['PC1'], row['PC2'], " x" + str(index + 1))
ax.set_xlabel('PC1')
ax.set_xlabel('PC2')
plt.show()


# coding: utf-8

# In[1]:

from tensorflow.keras.models import load_model
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.preprocessing.text import tokenizer_from_json
import jieba
import numpy as np

# In[2]:

# 载入tokenizer
json_file = open('token_config.json','r',encoding='utf-8')
token_config = json.load(json_file)
tokenizer = tokenizer_from_json(token_config)

# In[3]:

# 载入模型
model = load_model('cnn_model.h5')

# In[4]:

# 情感预测
def predict(text):
    # 对句子分词
    cw = list(jieba.cut(text))
    # list转字符串，元素之间用' '隔开
    texts = ' '.join(cw)
    # 把词转换为编号，编号大于30000的词会被过滤掉
    sequences = tokenizer.texts_to_sequences([texts])
    # model.input_shape为(None, 202)，202为训练模型时的序列长度
    # 把序列设定为202的长度，超过202的部分舍弃，不到202则补0
    sequences = pad_sequences(sequences, maxlen=model.input_shape[1], padding='pre')
    # 模型预测
    result = np.argmax(model.predict(sequences))
    if(result==1):
        print("正面情绪")
    else:
        print("负面情绪")

# In[5]:

predict("今天阳光明媚，手痒想打球了。")

# In[6]:

predict("一大屋子人，结果清早告停水了，我崩溃到现在[抓狂]")

程序数据整理/第16章-NLP任务项目实战/16-3 中文情感分类模型预测.py

16-3 中文情感分类模型预测.py


import numpy as np
import keras as K
import tensorflow as tf
import os

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
import matplotlib.pyplot as plt

def main():
    # starting
    print("\MNIST image recognition using Keras/TensorFlow ")
    np.random.seed(1)
    tf.set_random_seed(1)

if __name__ == "__main__":
    main()
    # loading data
print("Loading Keras version MNIST data into memory \n")
(train_x, train_y), (test_x, test_y) = K.datasets.mnist.load_data()
train_x = train_x.reshape(60000, 28, 28, 1).astype(np.float32)
test_x = test_x.reshape(10000, 28, 28, 1).astype(np.float32)
train_x /= 255;
test_x /= 255
train_y = K.utils.to_categorical(train_y, 10).astype(np.float32)
test_y = K.utils.to_categorical(test_y, 10).astype(np.float32)
# defining model
init = K.initializers.glorot_uniform(seed=1)
model = K.models.Sequential()
model.add(K.layers.Conv2D(filters=32, kernel_size=(3, 3),
                          strides=(1, 1), padding='valid', kernel_initializer=init,
                          activation='relu', input_shape=(28, 28, 1)))
model.add(K.layers.Conv2D(filters=64, kernel_size=(3, 3),
                          strides=(1, 1), padding='valid', kernel_initializer=init,
                          activation='tanh'))
model.add(K.layers.MaxPooling2D(pool_size=(2, 2)))
model.add(K.layers.Dropout(0.2))
model.add(K.layers.Flatten())
model.add(K.layers.Dense(units=100, kernel_initializer=init,
                         activation='sigmoid'))
model.add(K.layers.Dropout(0.2))
model.add(K.layers.Dense(units=10, kernel_initializer=init,
                         activation='softmax'))

model.compile(loss='categorical_crossentropy',
              optimizer='adadelta', metrics=['acc'])
# training model
bat_size = 64
max_epochs = 3
print("Starting training ")
model.fit(train_x, train_y, batch_size=bat_size,
          epochs=max_epochs, verbose=1)
print("Training complete")
# evaluating model
loss_acc = model.evaluate(test_x, test_y, verbose=0)
print("\nTest data loss = %0.4f  accuracy = %0.2f%%" % \
      (loss_acc[0], loss_acc[1] * 100))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Nov  6 12:03:06 2022

Dataset from Kaggle
https://www.kaggle.com/datasets/loovmj/tokyo-weather-data

@author: akshaypopat
"""

import pandas as pd

data = pd.read_csv("~/Documents/Data Science/tokyo_climate/tokyo_data.csv")

data["y/m/d"].isnull().sum()
data["high(C)"].isnull().sum()
data["low(C)"].isnull().sum()
data["daylight_hours(hours)"].isnull().sum()

import datetime

data["Date"] = data["y/m/d"].apply(lambda x: datetime.datetime(int(x.split("/")[0]), int(x.split("/")[1]), int(x.split("/")[2])))

data.to_csv("~/Documents/Data Science/tokyo_climate/data_clean.csv", index=False)


from django import forms
from django.db import models
from django.utils.text import Truncator
from modelcluster.contrib.taggit import ClusterTaggableManager
from taggit.models import TaggedItemBase, Tag as TaggitTag
from wagtail.admin.edit_handlers import FieldPanel, MultiFieldPanel, StreamFieldPanel, InlinePanel, FieldRowPanel
from wagtail.contrib.forms.models import AbstractFormField, AbstractEmailForm
from wagtail.core import blocks, fields
from wagtail.core.models import Page
from wagtail.embeds.blocks import EmbedBlock
from wagtail.images.blocks import ImageChooserBlock
from wagtail.images.edit_handlers import ImageChooserPanel
from wagtail.images.models import Image
from wagtail.core.models import Page
from wagtail.core.fields import RichTextField, StreamField

from django.db.models import TextField
# from wagtailmarkdown.blocks import MarkdownPanel
from wagtailmarkdown.edit_handlers import MarkdownPanel
from wagtailmarkdown.fields import MarkdownField
from wagtailmetadata.models import MetadataMixin

from wagtail.snippets.models import register_snippet
from modelcluster.fields import ParentalKey, ParentalManyToManyField
from wagtail.contrib.routable_page.models import RoutablePageMixin, route
from datetime import date, datetime
from wagtail.search import index

from .blocks import (
    ColumnBlock, TwoColumnBlock, VideoBlock
)
from wagtail_blocks.blocks import HeaderBlock, ListBlock, ImageTextOverlayBlock, CroppedImagesWithTextBlock, \
    ListWithImagesBlock, ThumbnailGalleryBlock, ChartBlock, MapBlock, ImageSliderBlock

from django.utils.dateformat import DateFormat
from django.utils.formats import date_format
from django.http import Http404

@register_snippet
class BlogCategory(models.Model):
    name = models.CharField(max_length=255)
    slug = models.SlugField(unique=True, max_length=80)

    panels = [
        FieldPanel('name'),
        FieldPanel('slug'),
    ]

    def __str__(self):
        return self.name

    class Meta:
        verbose_name = "Category"
        verbose_name_plural = "Categories"

class BlogPageTag(TaggedItemBase):
    content_object = ParentalKey('PostPage', related_name='post_tags')

@register_snippet
class Tag(TaggitTag):
    class Meta:
        proxy = True

class BlogPage(RoutablePageMixin, Page):
    description = models.CharField(max_length=255, blank=True, )

    content_panels = Page.content_panels + [
        FieldPanel('description', classname="full")
    ]

    # class Meta:
    #     unique_together = ("title", "body")

    # search_fields = Page.search_fields + [
    #     index.SearchField('title'),
    #     index.SearchField('body'),
    # ]

    def get_context(self, request, *args, **kwargs):
        context = super(BlogPage, self).get_context(request, *args, **kwargs)
        context['posts'] = self.posts
        context['blog_page'] = self
        context['search_type'] = getattr(self, 'search_type', "")
        context['search_term'] = getattr(self, 'search_term', "")
        context['menuitems'] = self.get_children().filter(
            live=True, show_in_menus=True
        )
        return context

    def get_posts(self):
        return PostPage.objects.descendant_of(self).live()

    @route(r'^(\d{4})/$')
    @route(r'^(\d{4})/(\d{2})/$')
    @route(r'^(\d{4})/(\d{2})/(\d{2})/$')
    def post_by_date(self, request, year, month=None, day=None, *args, **kwargs):
        self.posts = self.get_posts().filter(date__year=year)
        if month:
            self.posts = self.posts.filter(date__month=month)
            df = DateFormat(date(int(year), int(month), 1))
            self.search_term = df.format('F Y')
        if day:
            self.posts = self.posts.filter(date__day=day)
            self.search_term = date_format(
                date(int(year), int(month), int(day)))
        return Page.serve(self, request, *args, **kwargs)

    @route(r'^(\d{4})/(\d{2})/(\d{2})/(.+)$')
    def post_by_date_slug(self, request, year, month, day, slug, *args, **kwargs):
        post_page = self.get_posts().filter(slug=slug).first()
        if not post_page:
            raise Http404
        return Page.serve(post_page, request, *args, **kwargs)

    @route(r'^tag/(?P<tag>[-\w]+)/$')
    def post_by_tag(self, request, tag, *args, **kwargs):
        self.search_type = 'tag'
        self.search_term = tag
        self.posts = self.get_posts().filter(tags__slug=tag)
        return Page.serve(self, request, *args, **kwargs)

    @route(r'^category/(?P<category>[-\w]+)/$')
    def post_by_category(self, request, category, *args, **kwargs):
        self.search_type = 'category'
        self.search_term = category
        self.posts = self.get_posts().filter(categories__slug=category)
        return Page.serve(self, request, *args, **kwargs)

    @route(r'^$')
    def post_list(self, request, *args, **kwargs):
        self.posts = self.get_posts()
        return Page.serve(self, request, *args, **kwargs)

    @route(r'^search/$')
    def post_search(self, request, *args, **kwargs):
        from django.db.models import Q
        search_query = request.GET.get('q', None)
        self.posts = self.get_posts()
        if search_query:
            self.posts = self.posts.filter(
                Q(title__contains=search_query) | Q(body__contains=search_query))
            self.search_term = search_query
            self.search_type = 'search'
        return Page.serve(self, request, *args, **kwargs)

class PostPage(Page):
    # body = RichTextField(blank=True, null=True)
    ingredients = MarkdownField(verbose_name='ingredients', blank=True)
    body = StreamField([
        ('heading', blocks.CharBlock(classname="full title")),
        ('paragraph', blocks.RichTextBlock()),
        ('image', ImageChooserBlock(icon="image")),
        # ('two_columns', TwoColumnBlock()),
        # ('video', VideoBlock(icon="video")),
        ('embedded_video', EmbedBlock(icon="media")),
        # ('header', HeaderBlock()),
        # ('list', ListBlock()),
        # ('image_text_overlay', ImageTextOverlayBlock()),
        # ('cropped_images_with_text', CroppedImagesWithTextBlock()),
        # ('list_with_images', ListWithImagesBlock()),
        # ('thumbnail_gallery', ThumbnailGalleryBlock()),
        # ('chart', ChartBlock()),
        # ('image_slider', ImageSliderBlock()),
    ], null=True, blank=True)

    date = models.DateTimeField(
        verbose_name="Post date", default=datetime.today)
    excerpt = MarkdownField(verbose_name='excerpt', blank=True)
    header_image = models.ForeignKey(
        'wagtailimages.Image', null=True, blank=True, on_delete=models.SET_NULL, related_name='+')
    tags = ClusterTaggableManager(through=BlogPageTag, blank=True)
    categories = ParentalManyToManyField('piesecrets.BlogCategory', blank=True)
    content_panels = Page.content_panels + [
        ImageChooserPanel('header_image'),
        MarkdownPanel('ingredients'),
        StreamFieldPanel('body'),
        # FieldPanel('body', classname="full"),
        MarkdownPanel('excerpt'),
        FieldPanel('categories', widget=forms.CheckboxSelectMultiple),
        FieldPanel('tags'),
    ]

    settings_panels = Page.settings_panels + [
        FieldPanel('date'),
    ]

    @property
    def blog_page(self):
        return self.get_parent().specific

    def get_context(self, request, *args, **kwargs):
        context = super(PostPage, self).get_context(request, *args, **kwargs)
        context['blog_page'] = self.blog_page
        context['post'] = self
        return context

class LandingPage(Page):
    body = StreamField([
        ('heading', blocks.CharBlock(classname="full title")),
        ('paragraph', blocks.RichTextBlock()),
        ('image', ImageChooserBlock(icon="image")),
        ('two_columns', TwoColumnBlock()),
        ('embedded_video', EmbedBlock(icon="media")),
    ], null=True, blank=True)

    content_panels = Page.content_panels + [
        StreamFieldPanel('body'),
    ]

    @property
    def blog_page(self):
        return self.get_parent().specific

    def get_context(self, request, *args, **kwargs):
        context = super(LandingPage, self).get_context(
            request, *args, **kwargs)
        context['blog_page'] = self.blog_page
        return context

class FormField(AbstractFormField):
    page = ParentalKey('FormPage', related_name='custom_form_fields')

class FormPage(AbstractEmailForm):
    thank_you_text = RichTextField(blank=True)

    content_panels = AbstractEmailForm.content_panels + [
        InlinePanel('custom_form_fields', label="Form fields"),
        FieldPanel('thank_you_text', classname="full"),
        MultiFieldPanel([
            FieldRowPanel([
                FieldPanel('from_address', classname="col6"),
                FieldPanel('to_address', classname="col6"),
            ]),
            FieldPanel('subject'),
        ], "Email Notification Config"),
    ]

    def get_context(self, request, *args, **kwargs):
        context = super(FormPage, self).get_context(request, *args, **kwargs)
        context['blog_page'] = self.blog_page
        return context

    def get_form_fields(self):
        return self.custom_form_fields.all()

    @property
    def blog_page(self):
        return self.get_parent().specific

class AboutPage(Page):
    body = RichTextField(blank=True)
    header_image = models.ForeignKey(
        'wagtailimages.Image', null=True, blank=True, on_delete=models.SET_NULL, related_name='+')
    content_panels = Page.content_panels + [
        ImageChooserPanel('header_image'),
        MarkdownPanel('body', classname="full"),
    ]

    @property
    def about_page(self):
        return self.get_parent().specific

    def get_context(self, request, *args, **kwargs):
        context = super(AboutPage, self).get_context(request, *args, **kwargs)
        context['about_page'] = self.about_page
        return context


import itertools
import math

an = [1]
for i in range(2, 100):
    for zeros in itertools.count(0):
        binary = "1" + "0" * zeros + bin(an[-1])[2:]
        new_n = int(binary, 2)
        if all(math.gcd(new_n, past) == 1 for past in an):
            an.append(new_n)
            assert binary.endswith(bin(an[-1])[2:])
            print (f"{i} {new_n}")
            break


"""vogen.synth基础类库"""
import datetime
from vogen.synth import utils
from typing import List,Optional

def timeToFrame(timeSpan:datetime.timedelta)->float:
    return timeSpan/utils.Params.hopSize
def frameToTime(frames:float)->datetime.timedelta:
    return frames*utils.Params.hopSize

import json

class TPhoneme():
    """
    ph:音素名称
    on:开始时间，单位为0.25s，取整
    off:结束时间，单位为0.25s，取整
    """
    def __init__(self,ph:str="",on:int=0,off:int=0):
        self.ph=ph
        self.on=on
        self.off=off

class TNote():
    """
    pitch:音高
    on:开始时间，单位为0.25s，取整
    off:结束时间，单位为0.25s，取整
    """
    def __init__(self,pitch:int="",on:int=0,off:int=0):
        self.pitch=pitch
        self.on=on
        self.off=off

class TChar():
    """
    ch:汉字，对于空白为None
    rom:拼音，对于空白为None
    notes:音高列表
    ipa:（待处理）音素列表
    """
    def __init__(self,
                ch:Optional[str]=None,
                rom:Optional[str]=None,
                notes:Optional[List[TNote]]=None,
                ipa:Optional[List[TPhoneme]]=None):
        self.ch=ch
        self.rom=rom
        self.notes=notes
        self.ipa=ipa

class TUtt():
    def __init__(self,uttStartSec:float=0.0,uttDur:int=0,romScheme:str="",chars:List[TChar]=[]):
        self.uttStartSec=uttStartSec
        self.uttDur=uttDur
        self.romScheme=romScheme
        self.chars=chars


from PIL import Image
import numpy as np
import doors
import main
import random
import time
import sprites

MUUR = 1
TIMEDDEUR = 2
INTERACTABLEDEUR = 3
LEFT = 0
RIGHT = 1

TIPS = ["Watch out! At night the doors will close and monster will appear!",
        "If your hunger drops to 0 you will die a slow and painful death!",
        "By sprinting you can cover more ground but your hunger bar wil drop faster.",
        "Consumables and other items can be picked up and dropped using F and G.",
        "Certain doors can be opened by answering the question with wisdom and strength.",
        "Items can be crafted by combining specific items at the crafting station and pressing F",
        "Watch out to not get crushed by closing doors, they will not wait for you!",
        "To turn 180 degrees, press c on your keyboard.",
        "To move and look around, use the joystick on the NunChuk.",
        "To switch between items, use the green and blue button.",
        "To go to the resume screen, use the red button.",
        "Shake the controller to consume an item! But not to hard, some of the components might break!",
        "To sprint, press the c button on the NunChuk."
        ]

def generateWorld(afbeelding, factory, resources, textures, renderer, fontManager):
    input_image = Image.open(afbeelding)
    r_image_in, g_image_in, b_image_in = input_image.split()
    r_in = np.uint32(np.array(r_image_in))
    b_in = np.uint32(np.array(b_image_in))
    g_in = np.uint32(np.array(g_image_in))

    world_map = np.zeros((r_in.shape[0], r_in.shape[1]), int)
    door_map = np.empty((r_in.shape[0], r_in.shape[1]), object)
    wall_map = np.empty((r_in.shape[0], r_in.shape[1]), object)
    doorLocation = []
    spriteList = []
    timeIndex = random.randint(0, len(TIPS) - 1)
    starttime = time.time()
    for i in range(r_in.shape[0]):
        for j in range(r_in.shape[1]):
            # zwarte pixel ==> muur met texture[0]
            if r_in[i, j] == 0 and b_in[i, j] == 0 and g_in[i, j] == 0:
                world_map[i, j] = MUUR
                wall_map[i, j] = Wall(i, j, textures[0], "wall")

                # grijze pixel ==> craftingbench
            elif r_in[i, j] == 128 and b_in[i, j] == 128 and g_in[i, j] == 128:
                world_map[i, j] = MUUR
                wall_map[i, j] = Wall(i, j, textures[2], "crafting bench")

            # blauwwe pixel ==> timed deur die van links opent
            elif b_in[i, j] == 255 and r_in[i, j] == 0 and g_in[i, j] == 0:
                world_map[i, j] = TIMEDDEUR
                door_map[i, j] = doors.timedDoor((j, i), LEFT, textures[0])
                doorLocation.append((i, j))

            # groene pixel ==> timed deur die van rechts opent
            elif b_in[i, j] == 0 and r_in[i, j] == 0 and g_in[i, j] == 255:
                world_map[i, j] = TIMEDDEUR
                door_map[i, j] = doors.timedDoor((j, i), RIGHT, textures[0])
                doorLocation.append((i, j))

            elif b_in[i, j] == 0 and r_in[i, j] == 255 and g_in[i, j] == 0:
                world_map[i, j] = INTERACTABLEDEUR
                door_map[i, j] = doors.interactableDoor((j, i), RIGHT, textures[1])
                door_map[i, j].setPassCode(main.codeList, main.instructionsList)
                doorLocation.append((i, j))

            elif b_in[i, j] == 50 and r_in[i, j] == 117 and g_in[i, j] == 116:
                world_map[i, j] = 0
                spriteList.append(sprites.Sprite(j + 0.5, i + 0.5, 1, 0, "appel.png", 0.5, 0.5, 1, False, False, False, True, 25, 5, 0, resources, factory, None))

            elif b_in[i, j] == 50 and r_in[i, j] == 117 and g_in[i, j] == 50:
                world_map[i, j] = 0
                spriteList.append(
                    sprites.Sprite(j + 0.5, i + 0.5, 1, 0, "medkit.png", 0.5, 0.5, 1, False, False, False, True, 0, 25,
                                   0, resources, factory, None))

            elif r_in[i, j] == 78 and g_in[i, j] == 168 and b_in[i, j] == 153:
                world_map[i, j] = 0
                spriteList.append(
                    sprites.Sprite(j + 0.5, i + 0.5, 1, 0, "stick.png", 1.0, 1.0, 1, False, False, False, True, 10, 0,
                                   17, resources, factory, None))

            elif r_in[i, j] == 189 and g_in[i, j] == 17 and b_in[i, j] == 208:
                world_map[i, j] = 0
                spriteList.append(
                    sprites.Sprite(j + 0.5, i + 0.5, 1, 0, "kaart.png", 1.0, 1.0, 1, False, False, False, True, 0, 0, 0,
                                   resources, factory, None))

            elif r_in[i, j] == 25 and g_in[i, j] == 70 and b_in[i, j] == 153:
                world_map[i, j] = 0
                spriteList.append(
                    sprites.Sprite(j + 0.5, i + 0.5, 1, 0, "rock.png", 0.5, 0.5, 1, False, False, False, True, 0, 0, 0,
                                   resources, factory, None))

            elif r_in[i, j] == 80 and g_in[i, j] == 129 and b_in[i, j] == 61:
                world_map[i, j] = 10
            # witte pixel ==> openruimte
            else:
                world_map[i, j] = 0

        endtime = time.time()
        if (endtime - starttime) >= 2:
            starttime += 5
            timeIndex = random.randint(0, len(TIPS) - 1)
        renderLoadingScreen(fontManager, factory, renderer, i, r_in.shape[0], timeIndex)

    return (world_map, doorLocation, door_map, wall_map, spriteList)

class Wall:
    image = ""
    posWorldCoordinates = np.array([0.0, 0.0])
    type = "wall"

    def __init__(self, x, y, image, type):
        self.posWorldCoordinates = np.array([x, y])
        self.image = image
        self.type = type

def renderLoadingScreen(resources, factory, renderer, waarde, max, textindex):
    completion = int(100 * waarde / max)
    renderer.clear()
    renderer.copy(factory.from_image("resources/background.jpg"),
                  dstrect=(0, 0, main.BREEDTE, main.HOOGTE))
    renderer.fill((main.BREEDTE // 2 - 100, main.HOOGTE // 2 - 50, 2 * completion, 100), main.kleuren[2])
    text1 = "Loading: " + str(completion) + "%"
    text2 = TIPS[textindex]
    text = factory.from_text(text1, fontmanager=resources)
    renderer.copy(text, dstrect=(main.BREEDTE // 2 - 100, main.HOOGTE // 2 - 50, 200, 100))
    text = factory.from_text(text2, fontmanager=resources)
    renderer.copy(text, dstrect=(main.BREEDTE // 2 - len(text2) * 4, main.HOOGTE - 100, len(text2) * 8, 50))
    renderer.present()


import setuptools

with open("README.md") as fp:
    long_description = fp.read()

setuptools.setup(
    name="iac",
    version="0.0.1",

    description="An empty CDK Python app",
    long_description=long_description,
    long_description_content_type="text/markdown",

    author="author",

    package_dir={"": "iac"},
    packages=setuptools.find_packages(where="iac"),

    install_requires=[
        "aws-cdk.core==1.116.0",
        "aws-cdk.aws_iam==1.116.0",
        "aws-cdk.aws_codecommit==1.116.0",
        "aws-cdk.aws_codebuild==1.116.0",
        "aws_cdk.aws_codeartifact==1.116.0",
        "aws_cdk.aws_lambda_destinations==1.116.0",
    ],

    python_requires=">=3.6",

    classifiers=[
        "Development Status :: 4 - Beta",

        "Intended Audience :: Developers",

        "Programming Language :: JavaScript",
        "Programming Language :: Python :: 3 :: Only",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",

        "Topic :: Software Development :: Code Generators",
        "Topic :: Utilities",

        "Typing :: Typed",
    ],
)


# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html

# -- Path setup --------------------------------------------------------------

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))

import textwrap
# import exhale.utils
from exhale import utils

# -- Project information -----------------------------------------------------

project = 'exrt'
copyright = '2022, SVK'
author = 'SVK'

# -- General configuration ---------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ["breathe", 'exhale', "sphinx.ext.graphviz"]

# Breathe Configuration
breathe_default_project = "exrt"

graphviz_output_format = "svg"

graphviz_dot = "C:/Program Files/Graphviz/bin/dot.bat"
# graphviz_dot = "C:/Users/SVK/Desktop/aaa/dot.bat"

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# Tell sphinx what the primary language being documented is.
primary_domain = 'cpp'

# Tell sphinx what the pygments highlight language should be.
highlight_language = 'cpp'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = 'ru'

# somewhere in `conf.py`, *BERORE* declaring `exhale_args`
def specificationsForKind(kind):
    '''
    For a given input ``kind``, return the list of reStructuredText specifications
    for the associated Breathe directive.
    '''
    # Change the defaults for .. doxygenclass:: and .. doxygenstruct::
    if kind == "class" or kind == "struct":
        return [
            ":members:",
            ":protected-members:",
            ":private-members:",
            ":allow-dot-graphs:",
            ":undoc-members:"
        ]
    # Change the defaults for .. doxygenenum::
    elif kind == "enum":
        return [":no-link:"]
    # An empty list signals to Exhale to use the defaults
    else:
        return []

"""
    "exhaleDoxygenStdin": textwrap.dedent('''
        EXTRACT_ALL = YES
        SOURCE_BROWSER = YES
        EXTRACT_STATIC = YES
        OPTIMIZE_OUTPUT_FOR_C  = YES
        HIDE_SCOPE_NAMES = YES
        QUIET = YES
        INPUT = ../include ../src
        FILE_PATTERNS = *.c *.h
        EXAMPLE_RECURSIVE = YES
        GENERATE_TREEVIEW = YES
    '''),
"""

# Setup the exhale extension
exhale_args = {
    # These arguments are required
    "containmentFolder": "./api",
    "rootFileName": "library_root.rst",
    "doxygenStripFromPath": "..",
    # Heavily encouraged optional argument (see docs)
    "rootFileTitle": u"API библиотеки",

    # Suggested optional arguments
    "createTreeView": True,
    # TIP: if using the sphinx-bootstrap-theme, you need
    # "treeViewIsBootstrap": True,
    "exhaleExecutesDoxygen": True,
    "exhaleDoxygenStdin": "INPUT = ../include",
    "customSpecificationsMapping": utils.makeCustomSpecificationsMapping(
        specificationsForKind
    )

}

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']

# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']


import random
import mesa

class greenAgent(mesa.Agent):
    def __init__(self, unique_id, model, uncertainty_interval, opinion):
        super().__init__(unique_id, model)
        self.uncertainty_interval = uncertainty_interval
        self.uncertainty = random.randint(uncertainty_interval[0], uncertainty_interval[1])
        self.opinion = opinion

    # Talk to an agent, if their uncertainty is lower than ours we will change to their opinion.
    # If we change opinions, we will take halfway between our and their opinion (A node who is very certain will make us more certain than a less certain node)
    def talk(self, agent):
        if agent.uncertainty < self.uncertainty:
            self.opinion = agent.opinion
            self.uncertainty = (self.uncertainty + agent.uncertainty) / 2
        # elif agent.opinion == self.opinion:
            # self.uncertainty = max(self.uncertainty_interval[0], self.uncertainty * 0.05) # Likeminded people will reduce each others uncertainty, bound by the bottom of the interval.

    def step(self):
        # Get a list of adjacent node ids and iterate
        neighbors = self.model.grid.get_neighbors(self.pos, include_center=False)
        for neighbor in neighbors:
            # Get the agent at the node
            agent = self.model.grid.get_cell_list_contents([neighbor])[0]
            # Get a random int from 1 to 10, if its less or equal to the weight (which represents probability of interaction) we can talk to them.
            if (self.model.graph.edges[self.pos, neighbor]['weight'] >= random.randint(1,10)) and (isinstance(agent, greenAgent)): # we can only talk to other green agents
                # Talk to them.
                self.talk(agent)


import numpy as np
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_iris

if __name__ == '__main__':

    # Download the data of Iris Fischer flowers
    # The Fischer iris dataset contains three balanced classes of 50 observations,
    # each of which indicates the type of flower - iris bristly (Iris setosa),
    # iris virginica and iris multicolored (Iris versicolor).
    iris = load_iris()

    # Create a feature matrix
    features = iris.data

    # Create a vector of goals
    target = iris.target

    # Delete the first 40 observations
    features = features[40:,:]
    target = target[40:]

    # Create a binary vector of targets indicating whether the class is 0
    target = np.where((target == 0), 0, 1)

    # Take a look at the unbalanced vector of goals
    print(target)

    weights = {0: .9, 1: 0.1}

    RandomForestClassifier(class_weight=weights)
    RandomForestClassifier(bootstrap=True, class_weight={0: 0.9, 1: 0.1},
                           criterion='gini', max_depth=None,
                           max_features='auto',
                           max_leaf_nodes=None, min_impurity_decrease=0.0,
                           min_impurity_split=None, min_samples_leaf=l,
                           min_samples_split=2, min_weight_fraction_leaf=0.0,
                           n_estimators=10, n_jobs=l, oob_score=False,
                           random_state=None, verbose=0, warm_start=False)


import os
import lmdb
import numpy as np
from scipy import misc
import matplotlib.pyplot as plt
from PIL import Image
try:
	from BytesIO import BytesIO
except ImportError:
	from io import BytesIO

def extractAllFromDB(env):
	k = 0
	with env.begin(write=True) as txn:
		for root, dirs, files in os.walk("image"):
			root_ = os.path.basename(root)
			for f in files:
				try:
					key = "{0}\{1}".format(root_, f)
					print(key)
					buf = txn.get(key.encode('ascii'))
					location = "out/" + key
					directory = os.path.dirname(location)
					print(directory)
					if not os.path.exists(directory):
						os.makedirs(directory)
					with open(location,'wb') as out:
						out.write(buf)
					k += 1
				except:
					print("FAILED")
	return k

env = lmdb.open('nuswide', map_size= 8 * 1024 * 1024 * 1024)
k = extractAllFromDB(env)

print("Done {}", k)

#with env.begin(write=True) as txn:
#	for root, dirs, files in os.walk("image"):
#		root_ = os.path.basename(root)
#		for f in files:
#			key = "{0}\{1}".format(root_, f)
#			print(key)
#			buffer = BytesIO()
#			buf = txn.get(key.encode('ascii'))
#			buffer.write(buf)
#			image = misc.imread(buffer)
#			plt.imshow(image)
#			plt.show()


from astropy.time import Time
import astropy.units as u
from astropy.coordinates import SkyCoord, EarthLocation
import astroplan as ap
import warnings

def calc_vis(xo, loc, night='nautical', altMin=15, moonSep=30):
    """
        Calculate the visibility of the host star from the given site and a
        list of constraints and determine when the next visible transit occurs.

        Inputs
        ------

            xo: Exoplanet object
                Exoplanet object containing a time in the next_transit attribute

            loc: EarthLocation
                astropy.coordinates.EarthLocation object for the observer.

            night: str (Optional)
                Identifier of the kind of night to be used for checking visibility.
                Must be one of:
                    sunset: Sun 0 deg below horizon
                    civil: Sun 6 deg below horizon
                    nautical: Sun 12 deg below horizon
                    astronomical: Sun 18 deg below horizon

            altMin: float (Optional)
                Minimum altitude (in degrees) to be used for checking visibility.
                Must be in the range [0, 90].

            moonSep: float (Optional)
                Minimum angular separation from the Moon to be used for checking
                visibility. Must be greater than 0.

        Outputs
        -------
            True, if the current transit in xo.next_transits is visible
            False, if the star is never visible from this source
    """

    # Check visibility args
    if night.lower() not in ['sunset', 'civil', 'nautical', 'astronomical']:
        raise ValueError('{} is not a recognized definition of night time.'.format(night))
    if altMin < 0:
        warnings.warn('Minimum altitude cannot be less than 0 deg. Setting to 0 deg.')
        altMin = 0
    elif altMin >= 90:
        raise ValueError('Minimum altitude cannot be >= 90 deg.')
    if moonSep < 0:
        msg = 'Minimum Moon separation cannot be less than 0 deg. '
        msg += 'Setting to 0 deg.'
        warnings.warn(msg)
        moonSep = 0

    # Make sure Exoplanet object has a next_transit attribute computed
    if xo.next_transit is None:
        msg = 'Visibility is being evaluated before a transit '
        msg += 'time has been computed. Computing now...'
        warnings.warn(msg)
        xo.ephemeris_to_next_transit()

    # Build astroplan constraints list
    Constraints = []
    if night == 'sunset':
        Constraints.append(ap.AtNightConstraint(max_solar_altitude = 0 * u.deg))
    elif night == 'civil':
        Constraints.append(ap.AtNightConstraint.twilight_civil())
    elif night == 'nautical':
        Constraints.append(ap.AtNightConstraint.twilight_nautical())
    else:
        Constraints.append(ap.AtNightConstraint.twilight_astronomical())

    Constraints.append(ap.AltitudeConstraint(min=altMin * u.deg))
    Constraints.append(ap.MoonSeparationConstraint(min=moonSep * u.deg))

    Observer = ap.Observer(location=loc, name='Observer')

    # First check if this source is ever observable from this location
    all_year = Time(['2023-01-01T00:00:00', '2023-12-31T23:59:59'])
    ever_observable = ap.is_observable(Constraints, Observer, xo.st_coords,
                                       time_range=all_year)[0]
    if not ever_observable:
        return False

    # Continue to determine the next observable transit. Start by checking the
    # transit time already determined for the Exoplanet object
    vis = ap.is_observable(Constraints, Observer, xo.st_coords,
                           times=xo.next_transit)[0]

    if vis:
        return True

    while not vis:
        xo.ephemeris_to_next_transit(t=xo.next_transit + 1 * u.hour)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            vis = ap.is_observable(Constraints, Observer, xo.st_coords,
                                   times=xo.next_transit)[0]

        # Rip-cord for this while loop
        if xo.next_transit.byear > 10000:
            msg = 'No visibility found up to the year 10,000 (!!).'
            msg += ' Stopping analysis here.'
            raise RuntimeError(msg)

    return True


"""Tests for pandas_window_example_gen component."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import os

from mlp.components import transform_with_graph as component
from google.protobuf import json_format

from tfx.types import channel_utils
from tfx.types import standard_artifacts
from tfx.proto import pusher_pb2

class ComponentTest(tf.test.TestCase):
  def testConstruct(self):
    output_data_dir = os.path.join(
      os.environ.get('TEST_UNDECLARED_OUTPUTS_DIR', self.get_temp_dir()), self._testMethodName)
    examples_channel = channel_utils.as_channel([standard_artifacts.Examples()])
    schema_channel = channel_utils.as_channel([standard_artifacts.Schema()])
    transform_graph_channel = channel_utils.as_channel([standard_artifacts.TransformGraph()])
    component_instance = component.TransformWithGraph(
      examples=examples_channel,
      schema=schema_channel,
      transform_graph=transform_graph_channel,
    )
    self.assertEqual(
      'Schema',
      component_instance.inputs['schema'].type_name
    )
    self.assertEqual(
      'Examples',
      component_instance.inputs['examples'].type_name
    )
    self.assertEqual(
      'TransformGraph',
      component_instance.inputs['transform_graph'].type_name
    )
    self.assertEqual(
      'Examples',
      component_instance.outputs['transformed_examples'].type_name
    )

if __name__ == '__main__':
  tf.test.main()


import json

with open('szwb-data\json\szwb-dict-list.json', encoding='UTF-8') as f:

    dest_dict = {}
    source_dict = json.load(f)
    for date in source_dict:
        index_in_date = 0
        for record in source_dict[date]:
            index_all = record['date'].replace('-', '') + format(index_in_date, '03d')
            dest_dict[index_all] = record
            index_in_date = index_in_date + 1

with open('szwb-data\json\szwb-delist.json', 'w', encoding='UTF-8') as df:
    json.dump(dest_dict, df, ensure_ascii=False, indent=4, separators=(',', ':'))


import os
import sys

from flask import Flask, render_template, jsonify
from flask import request

from preprocessing import csvprocessor as csvproc
from shared.config import Config

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)) + '/')

app = Flask(__name__)

########################################################################################################################
# ROUTES
########################################################################################################################

@app.route('/')
def start():
    return render_template('index.html')

@app.route('/datasets')
def datasets():
    ls = []
    for filename in os.listdir(Config.PATH_DATA):
        if not filename.startswith('_') and filename.endswith(".csv"):
            ls.append(os.path.splitext(filename)[0])
    return jsonify(ls)

@app.route('/partynames')
def partynames():
    blocks = csvproc.get_blockDataSet(getParam("filename"))
    parties = csvproc.get_partyNames(blocks)
    return jsonify(parties)

@app.route('/election-results')
def electionResults():
    summary = csvproc.election_summary(getParam("filename"))
    return jsonify(summary)

@app.route('/compactness')
def compactness():
    measure = getParam('measure')
    fileName = getParam("filename")

    result = csvproc.compactness(fileName, measure)
    return jsonify(result)

def getParam(name):
    value = request.args.get(name, None)
    if not value:
        raise Exception("The parameter '{0}' was not provided.".format(name))
    return value

if __name__ == '__main__':
    app.debug = True
    app.run()