"""
This dataset assumes that you have a frozen camera rig.
All the scenes are shot using this rig.
The numeration of cameras is consistent across the scenes.
Therefore, the background for each camera (if known) is the same across all the scenes.
Every scene has images taken from all the available cameras.

The dataset files are expected to have the following structure:

| root/
|---- KRT.txt
|---- pose.txt
|---- background_0.jpg
|---- background_1.jpg
|---- scene_0/
     |---- time_0/
          |---- image_0.jpg
          |---- depth_0.jpg
          |---- image_1.jpg
          |---- depth_1.jpg
     |---- time_1/
     |---- time_2/
|---- scene_1/
     |---- time_0/
     |---- time_1/
     |---- time_2/

The file `roles.yaml` should contain the ids of cameras, used to extract the latent variable, and to reconstruct.
Example:

```

- scene_0:
  - time_0:
    - condition_train: [5, 7, 200]
    - reconstruction_train: [0, 2, 4]
    - condition_vis: [100, 19]
    - reconstruction_vis: [0, 1, 3, 11, 15, 17]
  - time_1:
    - condition_train: [6, 8, 201]
    - reconstruction_train: [2, 4, 7]
    - condition_vis: [100, 19, 58]
    - reconstruction_vis: [0, 1,, 11, 15, 17]
- scene_1:
  - time_2:
    - ...
  - time_1:
    - ...
- ...

```
"""

__all__ = ['CameraRigDataset']

import itertools
import logging
import os
import random

import numpy as np
import torch
import torch.nn.functional as F
import torch.utils.data

from lib.utils.io import load_krt, load_pose, load_roles
from .multiview_dataset import MultiviewDataset

logger = logging.getLogger(__name__)
ROLES_PATH = 'roles.yaml'

class CameraRigDataset(MultiviewDataset):
    def __init__(self,
                 root,
                 mode,  # train | val | vis | ...
                 roles_path=None,
                 background_type=None,
                 condition_sampling_type=None,  # random | deterministic
                 condition_sampling_number=None,  # 3
                 condition_data_types=None,  # [image, depth]
                 data_types=('image',),  # [image, background, depth]
                 world_scale=1.,
                 subsample_type=None,
                 subsample_size=0,
                 image_size=(1024, 667),
                 resize_scale=1,
                 fix_intrisics_coef=1,  # TODO: rewrite dryice KRT and get rid of this redundant parameter
                 change_orientation=False,
                 img_ext='jpg',
                 crop_size=None):

        super(torch.utils.data.Dataset, self).__init__()

        self.root = root
        self.img_ext = img_ext
        self.mode = mode
        if roles_path is None:
            roles_path = ROLES_PATH
        if os.path.isabs(roles_path):
            self.roles_path = roles_path
        else:
            self.roles_path = os.path.join(self.root, roles_path)

        self.background_type = background_type
        self.subsample_type = self.parse_subsample_type(subsample_type)
        self.subsample_size = subsample_size

        self.condition_sampling_type = condition_sampling_type
        self.condition_sampling_number = condition_sampling_number
        self.condition_data_types = condition_data_types
        self.data_types = data_types

        # firstly, the image is cropped. Secondly, the picked crop is resized.
        self.image_size = image_size
        self.crop_size = crop_size
        self.resize_scale = resize_scale
        if self.crop_size is not None:
            self.image_size = self.crop_size
        self.image_size = (int(self.image_size[0] / self.resize_scale),
                           int(self.image_size[1] / self.resize_scale))

        transf_path = os.path.join(self.root, 'pose.txt')
        transf = load_pose(transf_path)
        krt_path = os.path.join(self.root, 'KRT.txt')
        self.krt = load_krt(krt_path,
                            image_rescale_ratio=(1 / resize_scale) * fix_intrisics_coef,
                            world_center=transf[:, -1:],
                            world_scale=world_scale,
                            change_orientation=change_orientation,
                            image_size=image_size,
                            center_crop=crop_size,
                            )

        for val in self.krt.values():
            val.update({"image_size": np.array(self.image_size)})

        self.camera_ids = sorted(self.krt.keys())

        scene_prefix = 'scene_'
        time_prefix = 'time_'

        self.scene_time_links = dict()
        self.scene_time_list = []
        self.cumulated_number_of_views = []
        roles = load_roles(self.roles_path)

        for dirpath, subdirnames, filenames in os.walk(self.root, followlinks=True):
            possible_scene, possible_time = dirpath.split(os.sep)[-2:]
            if possible_scene.startswith(scene_prefix) and possible_time.startswith(time_prefix):
                scene_id = possible_scene[len(scene_prefix):]
                time_id = possible_time[len(time_prefix):]

                data = dict()
                try:
                    roles_data = roles[possible_scene][possible_time]
                except KeyError:
                    continue

                self.scene_time_list.append((scene_id, time_id))

                if self.condition_sampling_type not in {'none', None}:
                    data['condition_cameras'] = roles_data[f'condition_{self.mode}']
                else:
                    data['condition_cameras'] = []
                data['reconstruction_cameras'] = roles_data[f'reconstruction_{self.mode}']
                data['dirpath'] = dirpath
                self.scene_time_links[(scene_id, time_id)] = data
                self.cumulated_number_of_views.append(len(data['reconstruction_cameras']))
                self._check_constistency(data)

        self.cumulated_number_of_views = np.cumsum(self.cumulated_number_of_views)

    @property
    def all_cameras_krt(self):
        krt = []
        for index in self.camera_ids:
            camera = self.krt[index]
            camera['camera_name'] = index
            krt.append(camera)

        return {f'frozen_camera_rig': krt}

    def _check_constistency(self, data):
        if len(self.camera_ids) < len(data['condition_cameras']) + len(data['reconstruction_cameras']):
            intersecting_cameras = list(set(data['condition_cameras']) & set(data['reconstruction_cameras']))
            logger.warning(f'Some cameras are used both for conditioning and reconstruction in the folder '
                           f'{data["dirpath"]} -- '
                           + ', '.join(intersecting_cameras)
                           )

        for cam_id, dtype in itertools.product(data['condition_cameras'],
                                               self.condition_data_types,
                                               ):
            if dtype == 'background':
                filepath = os.path.join(self.root, f'{dtype}_{cam_id}.{self.img_ext}')
            else:
                filepath = os.path.join(data['dirpath'], f'{dtype}_{cam_id}.{self.img_ext}')
            if not os.path.exists(filepath):
                logger.warning(f'The condition camera {cam_id} is specified, however file {filepath} is missing')
            if cam_id not in self.krt:
                logger.warning(f'The condition camera {cam_id} is specified, however it is missing in KRT.txt')

        for cam_id, dtype in itertools.product(data['reconstruction_cameras'],
                                               self.data_types,
                                               ):
            if dtype == 'background':
                filepath = os.path.join(self.root, f'{dtype}_{cam_id}.{self.img_ext}')
            else:
                filepath = os.path.join(data['dirpath'], f'{dtype}_{cam_id}.{self.img_ext}')
            if not os.path.exists(filepath):
                logger.warning(f'The reconstruction camera {cam_id} is specified, however file {filepath} is missing')
            if cam_id not in self.krt:
                logger.warning(f'The reconstruction camera {cam_id} is specified, however it is missing in KRT.txt')

    def __getitem__(self, idx):
        position = np.searchsorted(self.cumulated_number_of_views, idx + 1)
        scene_id, time_id = self.scene_time_list[position]
        data = self.scene_time_links[(scene_id, time_id)]

        result = {'scene_id': scene_id,
                  'time_id': time_id,
                  'scene_time': 'frozen_camera_rig',
                  }

        # condition images and cameras
        if self.condition_sampling_type in {None, 'none'}:
            condition_ids = None
        elif self.condition_sampling_type == 'deterministic':
            condition_ids = data['condition_cameras']
        else:
            condition_ids = random.sample(data['condition_cameras'], self.condition_sampling_number)

        if condition_ids is not None:
            condition_keys = ['condition_cameras_ids',
                              'condition_extrinsics',
                              'condition_intrinsics',
                              'condition_images_sizes'
                              ]
            condition_keys.extend([f'condition_{dtype}' for dtype in self.condition_data_types])
            for key in condition_keys:
                result[key] = []
            for cond_id in condition_ids:
                result['condition_cameras_ids'].append(self.camera_ids.index(cond_id))
                result['condition_extrinsics'].append(self.krt[cond_id]['extrin'])
                result['condition_intrinsics'].append(self.krt[cond_id]['intrin'])
                result['condition_images_sizes'].append(self.krt[cond_id]['image_size'])

                for dtype in self.condition_data_types:
                    filepath = os.path.join(data['dirpath'], f'{dtype}_{cond_id}.{self.img_ext}')
                    image, _ = self._read_image(filepath)
                    result[f'condition_{dtype}'].append(self._preprocess_condition_image(image))
            for key in condition_keys:
                result[key] = np.stack(result[key], axis=0)

        # reconstruction camera, images and coords
        if position > 0:
            reconstruction_id = idx - self.cumulated_number_of_views[position - 1]
        else:
            reconstruction_id = idx
        reconstruction_id = data['reconstruction_cameras'][reconstruction_id]

        result['camera_id'] = self.camera_ids.index(reconstruction_id)
        result['extrinsics'] = self.krt[reconstruction_id]['extrin']
        result['intrinsics'] = self.krt[reconstruction_id]['intrin']
        result['images_sizes'] = self.krt[reconstruction_id]['image_size']

        for dtype in self.data_types:
            if dtype == 'background':
                filepath = os.path.join(self.root, f'{dtype}_{reconstruction_id}.{self.img_ext}')
            else:
                filepath = os.path.join(data['dirpath'], f'{dtype}_{reconstruction_id}.{self.img_ext}')
            image, _ = self._read_image(filepath)
            result[dtype] = image

        if self.background_type == 'real':
            assert 'background' in result, "There are no real background. " \
                                           "Probably your forgot add background option in " \
                                           "general_dataset_constants/data_types field in your config."
        elif self.background_type == 'white':
            result['background'] = np.ones_like(result['image']) * 255
        elif self.background_type == 'black':
            result['background'] = np.zeros_like(result['image'])
        elif self.background_type == 'self':
            result['background'] = result['image']

        height, width = result['image'].shape[1:3]
        result['pixel_coords'], result['relative_coords'] = self._sample_pixel_grid(height,
                                                                                width,
                                                                                image=result.get('image'),
                                                                                background=result.get('background'),
                                                                                )
        if result.get('image') is not None and result['pixel_coords'].shape[:2] != result['image'].shape[1:3]:
            result['image'] = F.grid_sample(torch.from_numpy(result['image']).float().unsqueeze(0),
                                            torch.from_numpy(result['relative_coords']).float().unsqueeze(0),
                                            ).squeeze(0)
            if 'background' in result:
                result['background'] = F.grid_sample(torch.from_numpy(result['background']).float().unsqueeze(0),
                                                    torch.from_numpy(result['relative_coords']).float().unsqueeze(0),
                                                    ).squeeze(0)

        return result


import streamlit as st
from send_email import send_email

st.header('Contact Me')
with st.form(key='form'):
    email = st.text_input('Enter your email address')
    text = st.text_area('Write your message here...')
    button = st.form_submit_button('Submit')
    if button:
        send_email(text)
        st.info('Your Email Was Sent Successfully!')


import os

from tornado import gen, testing

from . import base

os.environ['ASYNC_TEST_TIMEOUT'] = '10'

class ClusterTests(base.AsyncTestCase):

    CLUSTERING = True

    @property
    def redis_port(self):
        return int(os.environ['NODE1_PORT'])

    @testing.gen_test()
    def test_cluster_nodes_against_cluster(self):
        while not self.client.ready:
            yield gen.sleep(0.01)
        expectation = sorted([
            (os.environ['REDIS_HOST'], int(os.environ['NODE1_PORT'])),
            (os.environ['REDIS_HOST'], int(os.environ['NODE2_PORT'])),
            (os.environ['REDIS_HOST'], int(os.environ['NODE3_PORT'])),
        ])
        results = yield self.client.cluster_nodes()
        values = []
        for node in results:
            values.append((node.ip, node.port))
        self.assertListEqual(sorted(values), expectation)


#!/usr/bin/env pnpython4
#
# Merge Response_t from ph5 files and create corrected Das_t files
# Only creates corrected Response_t and Das_t files.
#
# Steve Azevedo, Mar 2017
#

import argparse
import logging
import os
import re
from ph5.core import ph5api

PROG_VERSION = '2019.14'
LOGGER = logging.getLogger(__name__)

ALL_FAMILIES = ['A', 'B', 'C', 'D', 'E', 'F', 'G',
                'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q']

def get_args():
    global ARGS, P5

    parser = argparse.ArgumentParser()
    parser.usage = "v{0}: set_n_i_response\
    (Run from top level families directory.)".format(
        PROG_VERSION)

    parser.add_argument("-F", dest="families_directory", required=False,
                        help="Directory that holds the family directories."
                             "Absolute path.")
    parser.add_argument("-N", dest="first_n_i", required=False, type=int,
                        default=0,
                        help="The n_i of the first entry. Defaults to zero.")

    ARGS = parser.parse_args()

    if ARGS.families_directory is None:
        ARGS.families_directory = os.getcwd()

def dump_kefs():
    '''   Dump each family Response_t into a single kef file.
          Label start of each Response_t with the family name '## A' as example
    '''
    global ORIG_RESPS
    miniRE = re.compile(r"miniPH5_\d{5}.ph5")
    here = os.getcwd()
    first = True
    if not os.path.exists("RESPONSE_T_N_I"):
        os.mkdir("RESPONSE_T_N_I")
    ORIG_RESPS = os.path.join(here, "RESPONSE_T_N_I",
                              "Response_t_by_family.kef")
    LOGGER.info("Creating: {0}".format(ORIG_RESPS))
    with open(ORIG_RESPS, 'w+') as fh:
        for fam in ALL_FAMILIES:
            if os.path.exists(os.path.join(fam, "master.ph5")):
                files = os.listdir(os.path.join('.', fam))
                matches = [f for f in files if miniRE.match(f)]
                if len(matches) == 0:
                    continue
                if first:
                    fh.write("## {0}\n".format(here))
                    first = False
                command = "ph5tokef -n {0}/master.ph5 -R".format(fam)
                fh.write("## {0}\n".format(fam))
                pipeh = os.popen(command)
                if pipeh:
                    while True:
                        line = pipeh.readline()
                        if not line:
                            break
                        fh.write(line)

#
# First line at start of family kef: "## A" (as an example)
#

def parse_kef():
    '''   Parse catted response file created by dump_kefs to a map:
          MAP[family][index]['n_i_all':n, 'n_i_family':n,
          [Response_t_dictionary]]
          family => Family name, 'A' as an example
          n_i_all => The n_i after all the Response_t are catted together
          n_i_family => The original n_i
          Response_t_dictionary => A dictionary for the line from the
          Response_t, keys are
          column names
    '''
    global ARGS
    ret = {}
    row = None
    n_i_all = 0
    with open(ORIG_RESPS) as fh:
        while True:
            line = fh.readline()
            if not line:
                break
            if line[0] == '#':
                # Comment with family
                family = line[3:].strip()
                if family not in ret:
                    ret[family] = []
                if len(line) == 5 and line[1] == '#':
                    # Save row, Just finished last family so save last row
                    if row is not None:
                        row['n_i_family'] = len(ret[family])
                        ret[family].append(row)

                    row = {}

            # We start a new row in the table here
            elif line[0] == '/':
                # Save row if there is one
                if row != {}:
                    row['n_i_family'] = len(ret[family])
                    ret[family].append(row)

                row = {}
                row['n_i_all'] = n_i_all + ARGS.first_n_i
                row['ppath'] = line.strip()
                n_i_all += 1
            elif line[0] == '\t':
                line = line.strip()
                key, value = line.split('=')
                if key == 'n_i':
                    last_n_i = int(value)
                if 'kv' not in row:
                    row['kv'] = {}
                row['kv'][key] = value

        row['n_i_family'] = last_n_i + ARGS.first_n_i
        ret[family].append(row)
    return ret

def print_new_Response_t(n_i_map):
    '''   Print the final Response_t.kef
    '''
    CORRECTED_RESPS = os.path.join(
        ARGS.families_directory, "RESPONSE_T_N_I", "Response_t_cor.kef")
    LOGGER.info("Creating: {0}".format(CORRECTED_RESPS))
    with open(CORRECTED_RESPS, 'w+') as fh:
        families = sorted(n_i_map.keys())
        for family in families:
            fh.write("#  {0}\n".format(family))
            n = 0
            for element in n_i_map[family]:
                fh.write("#  {0}\n".format(n))
                n += 1
                kv = element['kv']
                kv['n_i'] = element['n_i_all']
                keys = sorted(kv.keys())
                fh.write(element['ppath'] + '\n')
                for k in keys:
                    fh.write("\t{0}={1}\n".format(k, kv[k]))

def print_new_Das_t(P5, n_i_map, family):
    '''   Print Das_t corrected for Response_t n_i
    '''
    P5.read_das_g_names()
    for das_g in P5.Das_g_names:
        das = P5.read_das_t(das_g)
        MAP = n_i_map[family]
        if not das or das not in P5.Das_t:
            LOGGER.warning("#***\tMissing: {0}".format(das))
            continue

        if not os.path.exists(os.path.join(
                ARGS.families_directory, "RESPONSE_T_N_I")):
            os.mkdir(os.path.join(ARGS.families_directory, "RESPONSE_T_N_I"))

        DAS_KEF = os.path.join(
            ARGS.families_directory, "RESPONSE_T_N_I",
            "Das_t_response_n_i_{0}.kef".format(das))
        LOGGER.info("Creating: {0}\n".format(DAS_KEF))
        with open(DAS_KEF, 'w+') as fh:
            fh.write("#   PH5VERSION: {0}\n".format(ph5api.PH5VERSION))
            keys = sorted(P5.Das_t[das]['keys'])
            i = 1
            for das_t in P5.Das_t[das]['rows']:
                try:
                    M = MAP[das_t['response_table_n_i']]
                    das_t['response_table_n_i'] = M['n_i_all']
                    fh.write("#   {0}\n".format(i))
                    i += 1
                    fh.write(
                        "/Experiment_g/Receivers_g/{0}/Das_t\n".format(das_g))
                    for k in keys:
                        fh.write("\t{0}={1}\n".format(k, das_t[k]))
                except IndexError:
                    sr = ph5api.fepoch(
                        das_t['sample_rate_i'],
                        das_t['sample_rate_multiplier_i'])
                    LOGGER.warning(
                        "#   Index out of range for DAS: {0}, sample rate: {1}"
                        .format(das, sr))
                    LOGGER.warning("#   Entry unchanged! Suspect data.")
                    fh.write(
                        "#   {0} response_table_n_i entry suspect!\n".format(
                            i))
                    i += 1
                    fh.write(
                        "/Experiment_g/Receivers_g/{0}/Das_t\n".format(das_g))
                    for k in keys:
                        fh.write("\t{0}={1}\n".format(k, das_t[k]))

def main():
    get_args()
    os.chdir(ARGS.families_directory)
    dump_kefs()
    try:
        n_i_map = parse_kef()
    except BaseException:
        LOGGER.error("Cannot create n_i map. "
                     "Make sure the directory is correct using -F flag")
    else:
        for family in ALL_FAMILIES:
            ph5 = os.path.join(ARGS.families_directory, family)
            try:
                P5 = ph5api.PH5(path=ph5, nickname='master.ph5')
            except Exception as e:
                LOGGER.warning(e.msg)
                continue

            print_new_Das_t(P5, n_i_map, family)
            P5.close()
        print_new_Response_t(n_i_map)

if __name__ == '__main__':
    main()


#!/usr/bin/env python3

import sys
sys.path.append('../Lib/')

import torch
import torch.nn as nn

from transformers import get_linear_schedule_with_warmup
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score

import os, configparser, math, random
import datareader, tokenizer, utils

# deterministic determinism
torch.manual_seed(2020)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
random.seed(2020)

class TransformerClassifier(nn.Module):
  """A transformative experience"""

  def __init__(self, num_classes=2 ):
    """We have some of the best constructors in the world"""

    super(TransformerClassifier, self).__init__()

    self.embedding = nn.Embedding(
      num_embeddings=cfg.getint('data', 'vocab_size'),
      embedding_dim=cfg.getint('model', 'emb_dim'))

    encoder_layer = nn.TransformerEncoderLayer(
      d_model=cfg.getint('model', 'emb_dim'),
      nhead=cfg.getint('model', 'num_heads'),
      dim_feedforward=cfg.getint('model', 'feedforw_dim'))

    self.trans_encoder = nn.TransformerEncoder(
      encoder_layer=encoder_layer,
      num_layers=cfg.getint('model', 'num_layers'))

    self.dropout = nn.Dropout(cfg.getfloat('model', 'dropout'))

    self.linear = nn.Linear(
      in_features=cfg.getint('model', 'emb_dim'),
      out_features=num_classes)

  def forward(self, texts, attention_mask):
    """Moving forward"""

    output = self.embedding(texts) # * sqrtn

    # encoder input: (seq_len, batch_size, emb_dim)
    # encoder output: (seq_len, batch_size, emb_dim)
    output = output.permute(1, 0, 2)
    output = self.trans_encoder(output, attention_mask)

    # extract CLS token only
    # output = output[0, :, :]

    # average pooling
    output = torch.mean(output, dim=0)

    output = self.dropout(output)
    output = self.linear(output)

    return output

def fit(model, train_loader, val_loader, weights, n_epochs):
  """Training routine"""

  device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
  model.to(device)

  weights = weights.to(device)
  cross_entropy_loss = torch.nn.CrossEntropyLoss(weights)

  optimizer = torch.optim.AdamW(
    model.parameters(),
    lr=cfg.getfloat('model', 'lr'))

  scheduler = get_linear_schedule_with_warmup(
    optimizer,
    num_warmup_steps=100,
    num_training_steps=1000)

  best_roc_auc = -1
  optimal_epochs = -1

  for epoch in range(1, n_epochs + 1):
    model.train()
    train_loss, num_train_steps = 0, 0

    for batch in train_loader:
      batch = tuple(t.to(device) for t in batch)
      batch_inputs, batch_mask, batch_labels = batch
      batch_mask = batch_mask.repeat(cfg.getint('model', 'num_heads'), 1, 1)
      optimizer.zero_grad()

      logits = model(batch_inputs, batch_mask)
      loss = cross_entropy_loss(logits, batch_labels)
      loss.backward()

      torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
      optimizer.step()
      scheduler.step()

      train_loss += loss.item()
      num_train_steps += 1

    av_loss = train_loss / num_train_steps
    val_loss, roc_auc = evaluate(model, val_loader, weights)
    print('ep: %d, steps: %d, tr loss: %.3f, val loss: %.3f, val roc: %.3f' % \
          (epoch, num_train_steps, av_loss, val_loss, roc_auc))

    if roc_auc > best_roc_auc:
      best_roc_auc = roc_auc
      optimal_epochs = epoch

  return best_roc_auc, optimal_epochs

def evaluate(model, data_loader, weights):
  """Evaluation routine"""

  device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
  weights = weights.to(device)
  model.to(device)

  cross_entropy_loss = torch.nn.CrossEntropyLoss(weights)
  total_loss, num_steps = 0, 0

  model.eval()

  all_labels = []
  all_probs = []

  for batch in data_loader:
    batch = tuple(t.to(device) for t in batch)
    batch_inputs, batch_mask, batch_labels = batch
    batch_mask = batch_mask.repeat(cfg.getint('model', 'num_heads'), 1, 1)

    with torch.no_grad():
      logits = model(batch_inputs, batch_mask)
      loss = cross_entropy_loss(logits, batch_labels)

    batch_logits = logits.detach().to('cpu')
    batch_labels = batch_labels.to('cpu')
    batch_probs = torch.nn.functional.softmax(batch_logits, dim=1)[:, 1]

    all_labels.extend(batch_labels.tolist())
    all_probs.extend(batch_probs.tolist())

    total_loss += loss.item()
    num_steps += 1

  av_loss = total_loss / num_steps
  roc_auc = roc_auc_score(all_labels, all_probs)

  return av_loss, roc_auc

def main():
  """Fine-tune bert"""

  #
  # split train into train and validation and evaluate
  #

  tr_texts, tr_labels = datareader.DirDataReader.read(
    os.path.join(base, cfg.get('data', 'train')),
    {'no':0, 'yes':1})

  tr_texts, val_texts, tr_labels, val_labels = train_test_split(
    tr_texts, tr_labels, test_size=0.20, random_state=2020)

  tok = tokenizer.Tokenizer(cfg.getint('data', 'vocab_size'))
  tok.fit_on_texts(tr_texts)

  tr_texts = tok.texts_as_sets_to_seqs(tr_texts)
  val_texts = tok.texts_as_sets_to_seqs(val_texts)

  train_loader = utils.make_data_loader(
    tr_texts,
    tr_labels,
    cfg.getint('model', 'batch_size'),
    cfg.getint('data', 'max_len'),
    'train',
    utils.to_transformer_inputs)

  val_loader = utils.make_data_loader(
    val_texts,
    val_labels,
    cfg.getint('model', 'batch_size'),
    cfg.getint('data', 'max_len'),
    'dev',
    utils.to_transformer_inputs)

  print('loaded %d training and %d validation samples' % \
        (len(tr_texts), len(val_texts)))

  model = TransformerClassifier()

  label_counts = torch.bincount(torch.IntTensor(tr_labels))
  weights = len(tr_labels) / (2.0 * label_counts)

  best_roc, optimal_epochs = fit(
    model,
    train_loader,
    val_loader,
    weights,
    cfg.getint('model', 'num_epochs'))
  print('roc auc %.3f after %d epochs' % (best_roc, optimal_epochs))

  #
  # now retrain and evaluate on test
  #

  tr_texts, tr_labels = datareader.DirDataReader.read(
    os.path.join(base, cfg.get('data', 'train')),
    {'no':0, 'yes':1})

  test_texts, test_labels = datareader.DirDataReader.read(
    os.path.join(base, cfg.get('data', 'test')),
    {'no':0, 'yes':1})

  tok = tokenizer.Tokenizer(cfg.getint('data', 'vocab_size'))
  tok.fit_on_texts(tr_texts)

  tr_texts = tok.texts_as_sets_to_seqs(tr_texts)
  test_texts = tok.texts_as_sets_to_seqs(test_texts)

  train_loader = utils.make_data_loader(
    tr_texts,
    tr_labels,
    cfg.getint('model', 'batch_size'),
    cfg.getint('data', 'max_len'),
    'train',
    utils.to_transformer_inputs)

  test_loader = utils.make_data_loader(
    test_texts,
    test_labels,
    cfg.getint('model', 'batch_size'),
    cfg.getint('data', 'max_len'),
    'test',
    utils.to_transformer_inputs)

  print('loaded %d training and %d test samples' % \
        (len(tr_texts), len(test_texts)))

  model = TransformerClassifier()

  label_counts = torch.bincount(torch.IntTensor(tr_labels))
  weights = len(tr_labels) / (2.0 * label_counts)

  fit(model, train_loader, test_loader, weights, optimal_epochs)

if __name__ == "__main__":

  cfg = configparser.ConfigParser()
  cfg.read(sys.argv[1])
  base = os.environ['DATA_ROOT']

  main()


import datetime

import mock
from oslo.utils import timeutils
import six
import webtest.app

from ceilometer.publisher import utils
from ceilometer import sample
from ceilometer.tests.api import v2
from ceilometer.tests import db as tests_db

class TestListEvents(v2.FunctionalTest,
                     tests_db.MixinTestsWithBackendScenarios):

    def setUp(self):
        super(TestListEvents, self).setUp()
        patcher = mock.patch.object(timeutils, 'utcnow')
        self.addCleanup(patcher.stop)
        self.mock_utcnow = patcher.start()
        self.mock_utcnow.return_value = datetime.datetime(2014, 2, 11, 16, 42)
        self.sample1 = sample.Sample(
            'instance',
            'cumulative',
            '',
            1,
            'user-id',
            'project1',
            'resource-id',
            timestamp=datetime.datetime(2012, 7, 2, 10, 40),
            resource_metadata={'display_name': 'test-server',
                               'tag': 'self.sample',
                               'dict_properties': {'key': 'value'},
                               'not_ignored_list': ['returned'],
                               },
            source='test_source',
        )
        msg = utils.meter_message_from_counter(
            self.sample1,
            self.CONF.publisher.metering_secret,
        )
        self.conn.record_metering_data(msg)

        self.sample2 = sample.Sample(
            'instance',
            'cumulative',
            '',
            1,
            'user-id2',
            'project2',
            'resource-id-alternate',
            timestamp=datetime.datetime(2012, 7, 2, 10, 41),
            resource_metadata={'display_name': 'test-server',
                               'tag': 'self.sample2',
                               },
            source='source2',
        )
        msg2 = utils.meter_message_from_counter(
            self.sample2,
            self.CONF.publisher.metering_secret,
        )
        self.conn.record_metering_data(msg2)

    def test_all(self):
        data = self.get_json('/meters/instance')
        self.assertEqual(2, len(data))
        for s in data:
            self.assertEqual(timeutils.utcnow().isoformat(), s['recorded_at'])

    def test_all_trailing_slash(self):
        data = self.get_json('/meters/instance/')
        self.assertEqual(2, len(data))

    def test_all_limit(self):
        data = self.get_json('/meters/instance?limit=1')
        self.assertEqual(1, len(data))

    def test_all_limit_negative(self):
        self.assertRaises(webtest.app.AppError,
                          self.get_json,
                          '/meters/instance?limit=-2')

    def test_all_limit_bigger(self):
        data = self.get_json('/meters/instance?limit=42')
        self.assertEqual(2, len(data))

    def test_empty_project(self):
        data = self.get_json('/meters/instance',
                             q=[{'field': 'project_id',
                                 'value': 'no-such-project',
                                 }])
        self.assertEqual([], data)

    def test_by_project(self):
        data = self.get_json('/meters/instance',
                             q=[{'field': 'project_id',
                                 'value': 'project1',
                                 }])
        self.assertEqual(1, len(data))

    def test_empty_resource(self):
        data = self.get_json('/meters/instance',
                             q=[{'field': 'resource_id',
                                 'value': 'no-such-resource',
                                 }])
        self.assertEqual([], data)

    def test_by_resource(self):
        data = self.get_json('/meters/instance',
                             q=[{'field': 'resource_id',
                                 'value': 'resource-id',
                                 }])
        self.assertEqual(1, len(data))

    def test_empty_source(self):
        data = self.get_json('/meters/instance',
                             q=[{'field': 'source',
                                 'value': 'no-such-source',
                                 }])
        self.assertEqual(0, len(data))

    def test_by_source(self):
        data = self.get_json('/meters/instance',
                             q=[{'field': 'source',
                                 'value': 'test_source',
                                 }])
        self.assertEqual(1, len(data))

    def test_empty_user(self):
        data = self.get_json('/meters/instance',
                             q=[{'field': 'user_id',
                                 'value': 'no-such-user',
                                 }])
        self.assertEqual([], data)

    def test_by_user(self):
        data = self.get_json('/meters/instance',
                             q=[{'field': 'user_id',
                                 'value': 'user-id',
                                 }])
        self.assertEqual(1, len(data))

    def test_metadata(self):
        data = self.get_json('/meters/instance',
                             q=[{'field': 'resource_id',
                                 'value': 'resource-id',
                                 }])
        sample = data[0]
        self.assertIn('resource_metadata', sample)
        self.assertEqual(
            [('dict_properties.key', 'value'),
             ('display_name', 'test-server'),
             ('not_ignored_list', "['returned']"),
             ('tag', 'self.sample'),
             ],
            list(sorted(six.iteritems(sample['resource_metadata']))))


from .. import storage
import uuid
import os
import json
import time
import openai
import subprocess
from .types import Agent

max_runs = 10

class VisProgrammerAgent(Agent):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.preliminary_results_path = "Escher"
        self.final_results_path = "FinalResults"

    ##
    # Abstract implementations below
    ##
    def decide_next_job(self, jobs):
        if len(jobs) > 0:
            return jobs[0]
        else:
            return None

    def process_job(self, job_id):
        # Download the job_id
        job_json = storage.get_pending_job(job_id)
        job_type = job_json["type"]
        if job_type == "vis_request":
            new_jobs, new_files = self.gen_vis(job_json)
        else:
            print("Error in VisProgrammerAgent: Unknown job type")
            raise Exception(f"Unknown job type: {job_type}")

    ##
    # Helpers Below
    #
    def save_visualization(self, code, summary, image, job_id):
        new_files = {
            f"{self.id}_code.html": code["html"],
            f"{self.id}_code.js": code["js"],
            f"{self.id}_code.css": code["css"],
            f"{self.id}_summary.json": json.dumps(summary),
            f"{self.id}_image.png": image
        }

        for file_name, content in new_files.items():
            storage.save_and_upload_agent_result(
                self.repo_id, self.agent_id, job_id, content, file_name)

    def gen_vis(self, job_json):
        file = self.job_fn(job_json, f"{self.id}_image.png")

        new_jobs = []
        new_files = [file]

        return new_jobs, new_files

    def job_fn(self, job, ):
        job_id = job.job_id
        vis_request = job["vis_request"]

        openai.api_key = "your_openai_api_key"

        # Temp filename png file
        id = str(uuid.uuid4())
        png_file = f"output-{id}.png"

        filename = png_file

        prompt = f"""Create a three.js program that draws a mathematical scene related to topic {vis_request['topic']} , and goal: {vis_request['goal']} canvas it should then save that canvas in {filename}.
                  JS Code: """
        error = self.execute_code(code)

        code = self.generate_code_with_openai(prompt)

        runs = 0
        while error and runs < max_runs:
            runs = runs + 1
            prompt = f"The following error occurred while executing the code.\nError: {error}\nCode:{code}. Please provide an updated version of the code. Updated Code:"
            code = self.generate_code_with_openai(prompt)
            error = self.execute_code(code)

        # Save the visualization when error-free
        summary = {"topic": vis_request["topic"], "type": vis_request["type"]}
        self.save_visualization(code, summary, png_file, job_id)

        return png_file

    def generate_code_with_openai(self, prompt):
        response = openai.Completion.create(
            engine="davinci-codex",
            prompt=prompt,
            max_tokens=100,
            n=1,
            stop=None,
            temperature=0.5,
        )

        code = response.choices[0].text.strip()
        return code

    def execute_code(self, code):
        with open("temp_code.js", "w") as f:
            f.write(code)

        try:
            result = subprocess.run(
                ["node", "temp_code.js"], check=True, capture_output=True)
            return None
        except subprocess.CalledProcessError as e:
            return e.stderr.decode("utf-8").strip()

def main(agent_id=None, repo_id=None):
    if agent_id is None:
        agent_id = str(uuid.uuid4())
    if repo_id is None:
        repo_id = str(uuid.uuid4())

    agent = VisProgrammerAgent(agent_id=agent_id, repo_id=repo_id)
    agent.launch()

if __name__ == "__main__":
    # argparse first argument is repo id
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("--agent_id", type=str, default=None)
    parser.add_argument("--repo_id", type=str, default=None)
    args = parser.parse_args()

    main(args.agent_id, args.repo_id)


from django.conf.urls import url
from . import views
urlpatterns = [
    url(r'^$', views.index),
    url(r'^process_adduser$', views.process_adduser),
    url(r'^process_login$', views.process_login),
    url(r'^books$', views.books),
    url(r'^books/(?P<id>\d+)/$', views.process_addbook),
    url(r'^addnew$', views.addnew),
    url(r'^process_addbook$', views.process_addbook),
]


#coding:utf-8
import numpy as np
import cv2
import os
from os.path import join
from tqdm import tqdm
import numpy.random as npr

try:
    from .utils import IoU
except ImportError:
    from utils import IoU

import sys
sys.path.append('..')
from config import cfg

def GenExamplePnet(target='pnet'):
    pos_save_dir = join(cfg.path_output_files, target + '_pos')
    part_save_dir = join(cfg.path_output_files, target + '_part')
    neg_save_dir = join(cfg.path_output_files, target + '_neg')
    if not os.path.exists(pos_save_dir):
        os.mkdir(pos_save_dir)
    if not os.path.exists(part_save_dir):
        os.mkdir(part_save_dir)
    if not os.path.exists(neg_save_dir):
        os.mkdir(neg_save_dir)

    f1 = open(join(cfg.path_output_txt, target + '_pos.txt'), 'w')
    f2 = open(join(cfg.path_output_txt, target + '_neg.txt'), 'w')
    f3 = open(join(cfg.path_output_txt, target + '_part.txt'), 'w')
    with open(cfg.path_detection_labels, 'r') as f:
        annotations = f.readlines()

    if cfg.debug:
        annotations = annotations[:300]
    p_idx = 0 # positive
    n_idx = 0 # negative
    d_idx = 0 # dont care
    idx = 0
    box_idx = 0
    for annotation in tqdm(annotations):
        annotation = annotation.strip().split(' ')
        #image path
        im_path = annotation[0]
        #boxed change to float type
        boxes = annotation[1:]
        #gt
        boxes = np.array(boxes, dtype=np.float32).reshape(-1, 4)
        #load image
        img = cv2.imread(os.path.join(cfg.path_detection_imgs, im_path))
        print(os.path.join(cfg.path_detection_imgs, im_path))
        idx += 1
        height, width, channel = img.shape

        neg_num = 0
        #1---->50
        while neg_num < 50:
            #neg_num's size [40,min(width, height) / 2],min_size:40
            size = npr.randint(12, min(width, height) / 2)
            #top_left
            nx = npr.randint(0, width - size)
            ny = npr.randint(0, height - size)
            #random crop
            crop_box = np.array([nx, ny, nx + size, ny + size])
            #cal iou
            Iou = IoU(crop_box, boxes)

            cropped_im = img[ny : ny + size, nx : nx + size, :]
            resized_im = cv2.resize(cropped_im, (12, 12), interpolation=cv2.INTER_LINEAR)

            if np.max(Iou) < 0.3:
                # Iou with all gts must below 0.3
                save_file = os.path.join(neg_save_dir, "%s.jpg"%n_idx)
                name = join('%s_neg'%target, '%s.jpg'%n_idx)
                f2.write(name + ' 0\n')
                cv2.imwrite(save_file, resized_im)
                n_idx += 1
                neg_num += 1
        #as for 正 part样本
        for box in boxes:
            # box (x_left, y_top, x_right, y_bottom)
            x1, y1, x2, y2 = box
            #gt's width
            w = x2 - x1 + 1
            #gt's height
            h = y2 - y1 + 1

            # ignore small faces
            # in case the ground truth boxes of small faces are not accurate
            if max(w, h) < 40 or x1 < 0 or y1 < 0:
                continue
            for i in range(5):
                size = npr.randint(12, min(width, height) / 2)
                # delta_x and delta_y are offsets of (x1, y1)
                delta_x = npr.randint(max(-size, -x1), w)
                delta_y = npr.randint(max(-size, -y1), h)
                nx1 = int(max(0, x1 + delta_x))
                ny1 = int(max(0, y1 + delta_y))
                if nx1 + size > width or ny1 + size > height:
                    continue
                crop_box = np.array([nx1, ny1, nx1 + size, ny1 + size])
                Iou = IoU(crop_box, boxes)

                cropped_im = img[ny1: ny1 + size, nx1: nx1 + size, :]
                resized_im = cv2.resize(cropped_im, (12, 12), interpolation=cv2.INTER_LINEAR)

                if np.max(Iou) < 0.3:
                    # Iou with all gts must below 0.3
                    save_file = os.path.join(neg_save_dir, "%s.jpg" % n_idx)
                    name = join('%s_neg'%target, '%s.jpg'%n_idx)
                    f2.write(name + ' 0\n')
                    cv2.imwrite(save_file, resized_im)
                    n_idx += 1
            # generate positive examples and part faces
            for i in range(20):
                # pos and part face size [minsize*0.8,maxsize*1.25]
                size = npr.randint(int(min(w, h) * 0.8), np.ceil(1.25 * max(w, h)))

                # delta here is the offset of box center
                delta_x = npr.randint(-w * 0.2, w * 0.2)
                delta_y = npr.randint(-h * 0.2, h * 0.2)
                #show this way: nx1 = max(x1+w/2-size/2+delta_x)
                nx1 = int(max(x1 + w / 2 + delta_x - size / 2, 0))
                #show this way: ny1 = max(y1+h/2-size/2+delta_y)
                ny1 = int(max(y1 + h / 2 + delta_y - size / 2, 0))
                nx2 = nx1 + size
                ny2 = ny1 + size

                if nx2 > width or ny2 > height:
                    continue
                crop_box = np.array([nx1, ny1, nx2, ny2])
                #yu gt de offset
                offset_x1 = (x1 - nx1) / float(size)
                offset_y1 = (y1 - ny1) / float(size)
                offset_x2 = (x2 - nx2) / float(size)
                offset_y2 = (y2 - ny2) / float(size)
                #crop
                cropped_im = img[ny1 : ny2, nx1 : nx2, :]
                #resize
                resized_im = cv2.resize(cropped_im, (12, 12), interpolation=cv2.INTER_LINEAR)

                box_ = box.reshape(1, -1)
                if IoU(crop_box, box_) >= 0.65:
                    save_file = os.path.join(pos_save_dir, "%s.jpg"%p_idx)
                    name = join('%s_pos'%target, '%s.jpg'%p_idx)
                    f1.write(name + ' 1 %.2f %.2f %.2f %.2f\n'%(offset_x1, offset_y1, offset_x2, offset_y2))
                    cv2.imwrite(save_file, resized_im)
                    p_idx += 1
                elif IoU(crop_box, box_) >= 0.4:
                    save_file = os.path.join(part_save_dir, "%s.jpg"%d_idx)
                    name = join('%s_part'%target, '%s.jpg'%d_idx)
                    f3.write(name + ' -1 %.2f %.2f %.2f %.2f\n'%(offset_x1, offset_y1, offset_x2, offset_y2))
                    cv2.imwrite(save_file, resized_im)
                    d_idx += 1
            box_idx += 1
    f1.close()
    f2.close()
    f3.close()

if __name__=='__main__':
    pass
    # GenExamplePnet()


"""Add format to Pages

Revision ID: 07dfbe5e1edc
Revises: 75e8ab9a0014
Create Date: 2021-06-15 19:57:37.410152

"""
from alembic import op  # noqa: I001
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = "07dfbe5e1edc"
down_revision = "75e8ab9a0014"
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column("pages", sa.Column("format", sa.String(length=80), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column("pages", "format")
    # ### end Alembic commands ###


import time
import trimesh
import numpy as np

def rotate_objects(scene):

    # create an empty homogenous transformation
    matrix = np.eye(4)

    delta_change = time.time()
    # set Y as cos of time
    matrix[0][3] = np.sin(-delta_change*2)
    # set Z as sin of time
    matrix[2][3] = np.cos(-delta_change*2)

    # create a y axis rotation
    yaxis = [0,1,0]
    Ry = trimesh.transformations.rotation_matrix(delta_change*2, yaxis)

    node_sphere = s.graph.nodes_geometry[0]
    node_mesh = s.graph.nodes_geometry[1]

    # apply the transform to the node
    scene.graph.update(node_sphere, matrix=matrix)
    scene.graph.update(node_mesh, matrix=Ry)

if __name__ == '__main__':
    # create some spheres
    a = trimesh.primitives.Sphere(radius=0.1)

    # set some colors for the balls
    a.visual.face_colors = [0, 0, 255, 255]

    mesh_path = 'mesh/angelStatue_lp.obj'
    mesh = trimesh.load(mesh_path)

    # create a scene with the two balls
    s = trimesh.Scene([a,mesh])

    print(s.graph.nodes)

    # open the scene viewer and move a ball around
    s.show(callback=rotate_objects)

Python Libraries for Mesh, Point Cloud, and Data Visualization (Part 1 and Part 2)/trimesh_visualize.py


import glfw
import sys

class Controller:

    def __init__(self):
        self.camera = 1
        self.model = None

    def set_model(self, model):
        self.model = model

    def on_key(self, window, key, scancode, action, mods):

        if action != glfw.PRESS:
            return

        elif key == glfw.KEY_ESCAPE:
            sys.exit()

        elif self.model.state:

            if key == glfw.KEY_E:
                self.camera = 2

            elif key == glfw.KEY_R:
                self.camera = 1

            elif key == glfw.KEY_T:
                self.camera = 3


from typing import Optional
import argparse
from prefect.blocks.core import Block
from pydantic import SecretStr
from pathlib import Path
from prefect_gcp import GcpCredentials
from prefect_gcp.cloud_storage import GcsBucket

def main(params):
    """
    Create prefect blocks
    """
    service_account_file = params.service_account_file
    gcp_credentials_block_name = params.gcp_credentials_block_name
    gcs_bucket_name = params.gcs_bucket_name
    gcs_bucket_block_name = params.gcs_bucket_block_name

    gcp_credentials_block = GcpCredentials(service_account_file=service_account_file)
    gcp_credentials_block.save(gcp_credentials_block_name, overwrite=True)

    gcp_credentials = GcpCredentials.load(gcp_credentials_block_name)
    gcs_bucket_block = GcsBucket(bucket=gcs_bucket_name, gcp_credentials=gcp_credentials)
    gcs_bucket_block.save(gcs_bucket_block_name, overwrite=True)

    print('successfully created blocks')

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Create GcpCredentials Block and GcsBucket Block')

    parser.add_argument('--service_account_file', help='path for gcp_credentials_file')
    parser.add_argument('--gcp_credentials_block_name', help='name for gcp_credentials_block')
    parser.add_argument('--gcs_bucket_name', help='name for bucket_name')
    parser.add_argument('--gcs_bucket_block_name', help='name for gcs_bucket_block_name')

    args = parser.parse_args()
    main(args)


import requests
import csv

def fetch_data_from_api():
    """
    Fetches data from the given API endpoint.

    Returns:
        list: A list of dictionaries containing university data.
    """
    url = "http://universities.hipolabs.com/search?country=Canada"
    response = requests.get(url)
    response.raise_for_status()
    return response.json()

def write_data_to_csv(data, filename="universities.csv"):
    """
    Writes the provided data to a CSV file.

    Args:
        data (list): List of dictionaries containing university data.
        filename (str, optional): Name of the CSV file to write. Defaults to "universities.csv".
    """
    with open(filename, 'w', newline='') as csv_file:
        fieldnames = ['name', 'country', 'website']  # Add any other fields as required
        writer = csv.DictWriter(csv_file, fieldnames=fieldnames)

        writer.writeheader()
        for university in data:
            writer.writerow({
                'name': university['name'],
                'country': university['country'],
                'website': university.get('website', 'N/A')
            })

def main():
    data = fetch_data_from_api()
    write_data_to_csv(data)

if __name__ == "__main__":
    main()


#Class des map et Biome
from colorama import Fore
from colorama import Style
class biome:
    def __init__(self,image,effect=None):
        self.image = image
        self.effect = effect

mur= biome(Fore.BLACK+"#"+Style.RESET_ALL,"Stop")
rien = biome(" ")

class Map:
    def __init__(self,Longuer,Largeur,NextMap=None):
        self.NextMap = NextMap
        self.town =[]
        for i in range(Largeur+2):
            l=[]
            for j in range(Longuer+2):
                if i ==0 or j==0 or i==Largeur+1 or j == Longuer+1:
                    l.append(mur)
                else:
                    l.append(rien)
            self.town.append(l)
        self.List_of_Adversaire = []

    def append_biome(self,element,Point_de_depart_X,Point_de_depart_Y,Longuer_Y,Largeur_X):
        if Point_de_depart_X+Largeur_X > (len(self.town[0])-2) or Point_de_depart_Y+Longuer_Y >(len(self.town)-2):
            print("Dimmension pas correcte error Out-of-Range")
            return
        else:
            for i in range(Longuer_Y):
                for j in range(Largeur_X):
                    self.town[Point_de_depart_Y+i+1][Point_de_depart_X+j+1] = element

    def show(self):
        for i in self.town:
            for j in i:
                if type(j) == biome:
                    print(j.image,end="  ")
                else:
                    print(j.physique,end="  ")
            print()

    def Placement_Dresseur(self,Dresseur,X,Y):
        X+=1
        Y+=1
        if X<=0 or Y<=0 or X>=len(self.town[0])-1 or Y>=len(self.town)-1:
            print("Limite de Terrain vous ne pouvez pas le placer")
        else :
            if str(type(Dresseur)) == "<class 'Dresseur_Class.Joueur'>":
                for i in self.town:
                    for j in i:
                        if str(type(j)) == "<class 'Dresseur_Class.Joueur'>":
                            print("Il ne peut pas avoir Plus de 1 Joueur Par map")
                            return
            Dresseur.avant= self.town[Y][X]
            self.town[Y][X]=Dresseur
            Dresseur.AxeY=Y
            Dresseur.AxeX=X
            if str(type(Dresseur)) == "<class 'Dresseur_Class.PNJ_Adverse'>":
                self.List_of_Adversaire.append(Dresseur)
    def del_Dresseur(self,Dresseur):
            self.town[Dresseur.AxeY][Dresseur.AxeX] = Dresseur.avant
            Dresseur.avant= None
            Dresseur.AxeY=None
            Dresseur.AxeX=None


from pytket import Circuit
import random

def U(s, n_qubits):
    c = Circuit(n_qubits)
    for i in range(len(s)):
        if s[i] == "X":
            c.H(i)
        elif s[i] == "Y":
            c.Rx(i, 0.5)
        elif s[i] == "Z":
            pass
    return c

def phi(qubits, alpha, n_qubits):
    c = Circuit(n_qubits)
    for i in range(len(qubits) - 1):
        c.CX(qubits[i], qubits[i + 1])
    c.Rz(qubits[-1], alpha)
    for i in range(len(qubits) - 1):
        c.CX(qubits[-i - 2], qubits[-i - 1])
    return c

def pauli_gadget(alpha, s, n_qubits):
    c1 = U(s, n_qubits)
    c2 = phi([x for x in range(n_qubits) if s[x] != "I"], alpha, n_qubits)
    c = Circuit(n_qubits)
    c.add_circuit(c1, list(range(n_qubits)))
    c.add_circuit(c2, list(range(n_qubits)))
    c.add_circuit(c1.dagger(), list(range(n_qubits)))
    return c

def random_pauli_gadget(n_qubits):
    s = "".join(random.choices("XYZ", k=n_qubits))
    alpha = random.random() * 2
    return pauli_gadget(alpha, s, n_qubits)

def random_gadget_circuit(n_qubits, n_gadgets):
    c = Circuit(n_qubits)
    for _ in range(n_gadgets):
        c.add_circuit(random_pauli_gadget(n_qubits), list(range(n_qubits)))
    return c


from django.shortcuts import redirect, render, get_object_or_404
from django.contrib.auth.models import User, Permission, Group
from django.contrib.auth.decorators import login_required, permission_required
from django.db.models import Q

# Create your views here.
@login_required
@permission_required(['auth.add_usuario', 'auth.change_usuario', 'auth.delete_usuario'])
def cadastro(request):
    erros = []
    sucesso = []
    if request.method == 'POST':
        nome = request.POST['nome']
        sobrenome = request.POST['sobrenome']
        login = request.POST['login']
        email = request.POST['email']
        senha1 = request.POST['senha1']
        senha2 = request.POST['senha2']

        if User.objects.filter(username = login).exists():
            erros.append(f'Usuário {login} já existe!')
        else:
            if senha1 == senha2:
                user = User.objects.create_user(login, email, senha1)
                user.first_name = nome
                user.last_name = sobrenome
                user.save()
                sucesso.append(f'Usuário {nome} criado com sucesso!')
            else:
                erros.append('A senha e a confirmação devem ser iguais!')
    contexto = {
        'erros' : erros,
        'sucesso': sucesso
    }
    return render(request, 'usuarios/cadastro.html', contexto)

def login(request):
    return render(request, 'usuarios/login.html')

def logout(request):
    pass

@login_required
def dashboard(request):
    pass

@login_required
def usuarios(request):
    if 'busca' in request.GET:
        busca = request.GET['busca']
        usuarios = User.objects.filter(Q(first_name__icontains = busca) | Q(last_name__icontains = busca) | Q(username__icontains = busca) )
    else:
        usuarios = User.objects.all().order_by('first_name')

    contexto = {
        'usuarios':usuarios,
    }

    return render(request, 'usuarios/usuarios.html', contexto)

@login_required
@permission_required('auth.delete_usuario')
def exclui_usuario(request, id):
    user = User.objects.get(pk = id)

    user.is_active = False
    user.save()

    return redirect('usuarios')

@login_required
@permission_required('auth.change_usuario')
def edita_usuario(request, id):
    sucesso = []
    usuario = User.objects.get(pk = id)
    permissoes = Permission.objects.all()
    grupos = Group.objects.all()

    if request.method == 'POST':
        permissoes_selecionadas = request.POST.getlist('user_permissions')
        for perm in permissoes_selecionadas:
            usuario.user_permissions.add(perm)

        grupos_selecionados = request.POST.getlist('grupos')
        for grupo in grupos_selecionados:
            usuario.groups.add(grupo)

        first_name = request.POST['first_name']
        last_name = request.POST['last_name']
        email = request.POST['email']
        chaves = request.POST.keys()
        for c in chaves:
            if c == 'is_staff':
                usuario.is_staff = True
                break
            else:
                usuario.is_staff = False

        for c in chaves:
            if c == 'is_active':
                usuario.is_active = True
                break
            else:
                usuario.is_active = False

        usuario.first_name = first_name
        usuario.last_name = last_name
        usuario.email = email

        usuario.save()
        sucesso.append('Usuário editado com sucesso.')

    dados = {
        'grupos': grupos,
        'usuario': usuario,
        'permissoes' : permissoes,
    }
    return render(request, 'usuarios/edita_usuario.html', dados)

@login_required
def cria_grupo(request):
    permissoes = Permission.objects.all()
    sucesso = []
    erros = []

    if request.method == 'POST':
        nome = request.POST['nome']

        grupo_existe = Group.objects.filter(name = nome)
        if grupo_existe:
            erros.append(f'O Grupo {nome} já existe')
        else:
            permissoes_selecionadas = request.POST.getlist('permissions')
            grupo = Group.objects.create(name = nome)
            for perm in permissoes_selecionadas:
                print(perm)
                grupo.permissions.add(perm)
            sucesso.append('Grupo criado com sucesso.')

    context = {
        'erros': erros,
        'sucesso': sucesso,
        'permissoes': permissoes,
    }
    return render(request, 'usuarios/form_grupo.html', context)


import pytest
from schema import Schema

from cazart import bad_json, validation_error

def test_validates(cazart):
    result = None

    @cazart.route("/", schema=Schema({"name": str}))
    def root(res):
        nonlocal result
        result = res
        return ("ok", 200)

    with cazart.flask.test_client() as client:
        resp = client.get("/", json={"name": "foo"})
        assert resp.data == b"ok"
        assert resp.status_code == 200
        assert result == {"name": "foo"}

def test_does_not_validate(cazart):
    @cazart.route("/", schema=Schema({"name": str}))
    def root(res):
        return ("ok", 200)

    with cazart.flask.test_client() as client:
        resp = client.get("/", json={"name": 1234})
        assert resp.data == b"Key 'name' error:\n1234 should be instance of 'str'"
        assert resp.status_code == 400
        assert bad_json == {"name": 1234}
        assert validation_error == "Key 'name' error:\n1234 should be instance of 'str'"

def test_missing_json(cazart):
    @cazart.route("/", schema=Schema({"name": str}))
    def root(res):
        return ("ok", 200)

    with cazart.flask.test_client() as client:
        resp = client.get("/")
        assert resp.data == b"missing or malformed JSON"
        assert resp.status_code == 400

def test_malformed_json(cazart):
    @cazart.route("/", schema=Schema({"name": str}))
    def root(res):
        return ("ok", 200)

    with cazart.flask.test_client() as client:
        resp = client.get("/", data="{'bad json'}")
        assert resp.data == b"missing or malformed JSON"
        assert resp.status_code == 400

@pytest.mark.parametrize(
    ("method", "json", "exp_data", "status_code", "exp_result"),
    [
        ("GET", {"id": 1234}, b"ok", 200, {"id": 1234}),
        ("GET", {"name": "foo"}, b"Missing key: 'id'", 400, None),
        ("POST", {"name": "foo"}, b"ok", 200, {"name": "foo"}),
        ("POST", {"id": 1234}, b"Missing key: 'name'", 400, None),
    ],
)
def test_routes_methods(cazart, method, json, exp_data, status_code, exp_result):
    result = None

    @cazart.route("/", schema={"GET": Schema({"id": int}), "POST": Schema({"name": str})})
    def root(res):
        nonlocal result
        result = res
        return ("ok", 200)

    with cazart.flask.test_client() as client:
        resp = client.open(method=method, json=json)
        assert resp.data == exp_data
        assert resp.status_code == status_code

        if exp_result is not None:
            assert result == exp_result


import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
import time

plt.style.use('fivethirtyeight')

def animate(i):
    data = pd.read_csv('data.csv')
    plt.xlabel('Pressure')
    plt.ylabel('Time')
    x = data['x_value']
    p1 = data['pressure_1']
    p2 = data['pressure_2']
    p3 = data['pressure_3']

    plt.cla()
    plt.rc('lines', linewidth=1)
    plt.plot(x, p1, label="Pressure Sensor 1")
    plt.plot(x, p2, label="Pressure Sensor 2")
    plt.plot(x, p3, label="Pressure Sensor 3")

    plt.legend(loc='upper left')
    plt.tight_layout()

time.sleep(1)

anmt = FuncAnimation(plt.gcf(), animate, interval=20)

plt.tight_layout()
plt.show()


import os
import pickle

from matplotlib import pyplot

from model.transformer import Transformer, subsequent_mask
from util.config import Config
from util.dataloader import DataLoader
from tqdm import tqdm
import torch
from torch.autograd import Variable
from util.optimizer import LabelSmoothing, NoamOpt
from util.loss import SimpleLossCompute

conf = Config()
data = DataLoader(source_lang=conf.source_lang,
                  target_lang=conf.target_lang,
                  data_path=conf.dataset,
                  batch_size=conf.batch_size,
                  save_normal=conf.save_normal,
                  device=conf.device,
                  padding=conf.padding,
                  pad=conf.pad,
                  sos=conf.sos,
                  eos=conf.eos,
                  vocab_size=conf.vocab_size)

if conf.fp16:
    from torch.cuda.amp import GradScaler, autocast

    scaler = None  # GradScaler()
else:
    scaler = None

def run_batch(model, batch, loss_compute):
    out = model.forward(batch.source, batch.target[:, :-1])
    ntokens = (batch.target[:, 1:] != conf.pad_idx).data.sum()
    loss = loss_compute(out, batch.target[:, 1:], ntokens, conf.fp16, scaler)
    return out, loss, ntokens

def run_epoch(train_iter, model, loss_compute, desc):
    """Standard Training and Logging Function"""
    total_tokens = 0
    total_loss = 0
    progress = tqdm(enumerate(train_iter), desc=desc, total=len(train_iter))
    for i, batch in progress:
        if conf.fp16:
            with autocast():
                out, loss, ntokens = run_batch(model, batch, loss_compute)
        else:
            out, loss, ntokens = run_batch(model, batch, loss_compute)
        # trg = batch.target[:, :-1]
        # out = model.forward(batch.source, trg)
        # ntokens = (batch.target[:, 1:] != conf.pad_idx).data.sum()
        # loss = loss_compute(out, batch.target[:, 1:], ntokens)
        total_loss += loss
        total_tokens += ntokens
        progress.set_postfix_str(f"Loss: {(loss / ntokens).item():.5f}"+f"| lr: {loss_compute.opt.lr:.8f}" if loss_compute.opt is not None else "")
    return (total_loss / total_tokens).item()

if __name__ == '__main__':
    print("make dataset")
    train_iter, test_iter, valid_iter = data.make_data()
    print("make model")
    model = Transformer.make_model(data.source_vocab.length,
                                   data.target_vocab.length,
                                   conf.max_seq_len,
                                   conf.pad_idx,
                                   conf.n_layers,
                                   conf.dim_model,
                                   conf.feed_hidden,
                                   conf.n_head,
                                   conf.drop)
    if conf.load_model:
        print(f"load model {conf.load_model}")
        model.load_state_dict(torch.load(conf.load_model))

    model.to(conf.device)
    print("use device ", conf.device)
    print(f"The model has {sum(p.numel() for p in model.parameters() if p.requires_grad):,} trainable parameters")
    optimizer = NoamOpt(conf.dim_model, 1, 100, torch.optim.Adam(
        model.parameters(), lr=0, betas=(0.9, 0.98), eps=conf.eps
    ))
    criterion = LabelSmoothing(data.target_vocab.length, conf.pad_idx, smoothing=.1).to(conf.device)
    train_history = []
    valid_history = []
    best = float("inf")
    print("start train")
    for epoch in range(conf.epochs):
        model.train()
        train_loss = run_epoch(train_iter, model, SimpleLossCompute(
            model.generator, criterion, optimizer
        ), "Train")
        model.eval()
        with torch.no_grad():
            valid_loss = run_epoch(valid_iter, model, SimpleLossCompute(
                model.generator, criterion, None
            ), "Evaluate")
        print("Epoch: %d| Train Loss: %.5f| Evaluation Loss: %.5f" % (epoch, train_loss, valid_loss))
        train_history.append(train_loss)
        valid_history.append(valid_loss)
        if (epoch+1) % conf.save_interval == 0 and epoch:
            torch.save(model.state_dict(), os.path.join(conf.save_path, conf.model_name + f"_{epoch+1}.pt"))
        if valid_loss < best:
            print("save model for best loss:", valid_loss)
            best = valid_loss
            torch.save(model.state_dict(), os.path.join(conf.save_path, conf.model_name + "_best.pt"))
    pickle.dump(train_history, open("train_loss.pkl", "wb"))
    pickle.dump(valid_history, open("val_loss.pkl", "wb"))
    pyplot.plot(train_history, label="train")
    pyplot.plot(valid_history, label="val")
    pyplot.legend()
    pyplot.show()


#https://www.acmicpc.net/problem/14502
# 18:55 ~ 19:40

from collections import deque
from itertools import combinations

n, m = map(int, input().split())

data = [list(map(int, input().split())) for _ in range(n)]
virus = []
zero_list = []
move_r =[0, 1, 0, -1] #동, 남, 서, 북
move_c =[1, 0, -1, 0]
count = 0
answer = -1
def Virus(data, virus, count):
    N_map = [[0] * len(data[0]) for _ in range(len(data))]
    for r in range(len(data)):
        for c in range(len(data[0])):
            N_map[r][c] = data[r][c]
    Q = deque(virus)

    while Q:
        now_r, now_c = Q.popleft()
        for m_idx in range(4):
            next_r = now_r + move_r[m_idx]
            next_c = now_c + move_c[m_idx]
            if next_r>=0 and next_r < n and next_c>=0 and next_c <m:
                if N_map[next_r][next_c] ==0:
                    N_map[next_r][next_c] = 2
                    Q.append([next_r,next_c])
                    count -=1
    return count

for r in range(n):
    for c in range(m):
        if data[r][c] == 2:
            virus.append([r,c])
        elif data[r][c] == 0:
            zero_list.append([r,c])
            count +=1

wall_list = list(combinations(zero_list, 3))
for wall in wall_list:
    for n_wall in wall:
        data[n_wall[0]][n_wall[1]] = 1

    answer = max(answer, Virus(data, virus, count-3))
    for n_wall in wall:
        data[n_wall[0]][n_wall[1]] = 0

print(answer)


"""This is the test module for the env module."""

import unittest
from unittest.mock import patch

import env

class TestEnv(unittest.TestCase):
    """
    Test case for the env module.
    """

    @patch("os.getenv")
    def test_get_env_vars(self, mock_getenv):
        """
        Test the get_env_vars function when all environment variables are set correctly.
        """
        mock_getenv.side_effect = [
            "org",
            "repo,repo2",
            "token",
            "",
            "2022-01-01",
            "2022-12-31",
            "False",
        ]

        (
            organization,
            repository_list,
            token,
            ghe,
            start_date,
            end_date,
            sponsor_info,
        ) = env.get_env_vars()

        self.assertEqual(organization, "org")
        self.assertEqual(repository_list, ["repo", "repo2"])
        self.assertEqual(token, "token")
        self.assertEqual(ghe, "")
        self.assertEqual(start_date, "2022-01-01")
        self.assertEqual(end_date, "2022-12-31")
        self.assertEqual(sponsor_info, "false")

    @patch("os.getenv")
    def test_get_env_vars_missing_values(self, mock_getenv):
        """
        Test the get_env_vars function when none of the environment variables are set.
        Expect a ValueError to be raised.
        """
        mock_getenv.side_effect = [None, None, None, None, None, None, None]

        with self.assertRaises(ValueError):
            env.get_env_vars()

if __name__ == "__main__":
    unittest.main()


#!/usr/bin/env python3
'''
    block_media_replies
    ===================

    Block all accounts that reply to an account with media in their reply.

    Reads configuration settings from `config/block_media_replies.json`.
'''

import json
import os

try:
    import blockbot
except ImportError:
    # Script probably not installed, in scripts directory.
    import sys
    project_home = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
    sys.path.insert(0, project_home)
    import blockbot

from blockbot.path import config_dir

def load_config():
    '''Read configuration file.'''

    with open(os.path.join(config_dir(), 'block_media_replies.json')) as f:
        config = json.load(f)
        if not config['account_screen_name']:
            raise ValueError('Must provide screen name for account replied to.')
    return config

def main():
    config = load_config()
    blockbot.as_daemon(
        blockbot.block_media_replies,
        name='block_media_replies',
        account_screen_name=config['account_screen_name'],
        whitelist_screen_names=config['whitelist_screen_names'],
        **config['keywords'],
    )

if __name__ == '__main__':
    main()


"""add_index_on_offer_subcategoryId

Revision ID: a9a2e51e8e24
Revises: f8f80e3ca099
Create Date: 2021-06-30 15:26:01.922044

"""
from alembic import op

from pcapi import settings

# revision identifiers, used by Alembic.
revision = "a9a2e51e8e24"
down_revision = "f8f80e3ca099"
branch_labels = None
depends_on = None

def upgrade():
    op.execute("COMMIT")
    op.execute(
        """
        SET SESSION statement_timeout = '300s'
        """
    )
    op.execute(
        """
        CREATE INDEX CONCURRENTLY IF NOT EXISTS "ix_offer_subcategoryId" ON offer ("subcategoryId")
        """
    )
    op.execute(
        f"""
        SET SESSION statement_timeout={settings.DATABASE_STATEMENT_TIMEOUT}
        """
    )

def downgrade():
    op.execute("COMMIT")
    op.execute(
        """
        DROP INDEX CONCURRENTLY IF EXISTS "ix_offer_subcategoryId"
        """
    )


import os
import sys
import time
import logging
logging.getLogger("PIL.TiffImagePlugin").setLevel(logging.CRITICAL+1)
import spidev as SPI
sys.path.append("..")
sys.path.append("/home/gix/TECHIN514/Shinmood/")
from lib import LCD_1inch28
from PIL import Image,ImageDraw,ImageFont, TiffImagePlugin
TiffImagePlugin.DEBUG=False
Font3 = ImageFont.truetype("/home/gix/TECHIN514/Shinmood/Font/Font02.ttf",32)

# Reading the images into the memory
HAPPY_PATH = '/home/gix/TECHIN514/Shinmood/emoji/Happy 13FTS_Per Secend'
CRY_PATH = '/home/gix/TECHIN514/Shinmood/emoji/Cry 16FTS_Per Second'
LOADED_IMAGES = {
    'happy': [Image.open(os.path.join(HAPPY_PATH, f)) for f in os.listdir(HAPPY_PATH)],
    'cry': [Image.open(os.path.join(CRY_PATH, f)) for f in os.listdir(CRY_PATH)]
}

class LCD():
    def __init__(self, fps):
        # Raspberry Pi pin configuration:
        RST = 6
        DC = 5
        BL = 13
        bus = 0
        device = 0
        self.fps = fps
        self.delay = 1 / fps
        self.timer = 0
        self.arcmax = 135
        self.arcspeed = 270 # arc per second

        logging.basicConfig(level=logging.DEBUG)
        try:
            # display with hardware SPI:
            ''' Warning!!!Don't  creation of multiple displayer objects!!! '''
            self.disp = LCD_1inch28.LCD_1inch28(spi=SPI.SpiDev(bus, device),spi_freq=10000000,rst=RST,dc=DC,bl=BL)
            #self.disp = LCD_1inch28.LCD_1inch28()
            # Initialize library.
            self.disp.Init()
            # Clear display.
            self.disp.clear()
        except IOError as e:
            logging.info(e)
        except KeyboardInterrupt:
            self.disp.module_exit()
            logging.info("quit:")
            exit()

    def show_img(self, mode='happy', index=0, lux: float = 0):
        face = LOADED_IMAGES[mode][index]
        image = Image.new("RGB", (self.disp.width, self.disp.height), "BLACK")
        face_r = face.resize((120,120))
        image.paste(face_r, (60,60))
        # Create blank image for drawing.
        draw = ImageDraw.Draw(image)
        draw.text((74, 150),"%.2f" % lux, fill = "WHITE",font=Font3)
        # draw.text((74, 180),"%.2f" % patting_score, fill = "WHITE",font=Font3)
        image = image.rotate(180)
        draw = ImageDraw.Draw(image)
        # logging.info("draw point")
        # draw.rectangle((20,20,220,220), fill = "blue")
        # logging.info("draw circle")
        draw.arc((10,10,230,230),0, min(self.arcmax, self.delay * self.timer * self.arcspeed), fill =(0,0,255),width=5)
        image = image.rotate(180)
        self.disp.ShowImage(image)
        time.sleep(self.delay)
        self.timer += 1
    def end(self):
        self.disp.module_exit()
        logging.info("quit:")


import netCDF4 as nc
import requests
from bs4 import BeautifulSoup
import urllib
import numpy
import pandas as pd

#function to get max at a given latitude
def get_max(data,latitude):
    lat = latitude + 90
    maximum = 0
    for depth in range(len(data)):
        if data[depth][lat]>maximum:
            maximum = data[depth][lat]
        #maximum = max(maximum,data[depth][lat])
        #maximum = max(maximum, data[depth, lat].item
    return maximum

#function to access individual url and convert to usable netcdf object
def access_url(url):
    data = requests.get(url).content
    # Create a NetCDF Dataset object from the in-memory buffer.
    data_nc = nc.Dataset("anynamehere", memory=data)
    return data_nc

#function used to save extracted data into csv
def save_data(data_lists,titles,filename):#format:([x,y,...], [x title, y title, ...], filename)
    data={}
    # Combine the arrays using zip()
    for i in range(len(data_lists)):
        data[titles[i]] = data_lists[i]
    df = pd.DataFrame(data)
    df.to_csv(filename, index=False)
    print("data successfully saved as ", filename)

# URL of the website + preprocessing to find all files links
url = 'https://portal.nccs.nasa.gov/datashare/modelE_ocean/E213SSP245/'
box_url = urllib.request.urlopen(url).read()
box_soup = BeautifulSoup(box_url,'lxml')
folders = box_soup.find('tbody').find_all('a')[1:]
all_amocs = []
all_years = []
titles = ["Years"]
#print(folders)
for ensemble in folders:
    ensemble_name = ensemble.text
    box_url_temp = urllib.request.urlopen(url+ensemble_name).read()
    box_soup_temp = BeautifulSoup(box_url_temp,'lxml')
    #get links to individual files, split by type
    oijl_files=[]
    for element in box_soup_temp.find_all('td', class_='link')[1:][::2]:
        oijl_files.append(element.find('a').get('href'))

    ojl_files=[]
    for element in box_soup_temp.find_all('td', class_='link')[2:][::2]:
        ojl_files.append(element.find('a').get('href'))

    #iterate through list to get data from all files
    years = []
    max_amoc = []
    #pick latitude
    latitude = 48
    for file in ojl_files:
        #get year from file name
        year = file[3:7]
        years.append(year)
        #get complete file path from poral + file name
        complete_url = url + ensemble_name + file
        #print(complete_url)
        data = access_url(complete_url)
        yearly_max = get_max(data.variables["sf_Atl"],latitude)
        #append to results array
        max_amoc.append(yearly_max)

        print("year:",year,"ensemble:", ensemble_name)#checking progress, can be removed
    all_amocs.append(max_amoc)
    titles.append(ensemble_name[:-1])
    if all_years != years:
        print("ERROR")
    all_years = years

# File path for saving the CSV file
filename = 'AMOCvsTIME_lat48AllEnsembles.csv'

# next function used in testing to save data to avoid rerunning extraction
save_data([all_years] + all_amocs,titles,filename)


#!/usr/bin/env python

from icecube import icetray, phys_services, dataio
from icecube import dataclasses as dc
from I3Tray import I3Tray
import numpy as np
import argparse
from usefulFunctions import checkdir

class SaveSnowHeight(icetray.I3Module):
    def __init__(self, context):
        icetray.I3Module.__init__(self, context)
        self.AddOutBox('OutBox')
        self.AddParameter('outFile', 'Input name for output file', 'test')

    def Configure(self):
        self.outFile = self.GetParameter('outFile')
        pass

    def Geometry(self, frame):
    # Load I3Geometry to be used later
        self.geometry = frame['I3Geometry']
        self.PushFrame(frame)

    def DetectorStatus(self, frame):
    # Load I3DetectorStatus to gather positions and snow heights
    # for active tanks
        self.om2index = {}
        self.tankpositions = []
        self.snowheight = []
        self.status = frame['I3DetectorStatus']
        self.dom = self.status.dom_status
        index = 0
        for number, station in self.geometry.stationgeo:
            for tank in station:
                omList = tank.omkey_list
                if not any([om in self.dom.keys() for om in omList]):
                    continue
                for om in omList:
                    self.om2index[om] = index
                self.tankpositions.append(dc.i3position_to_tuple(tank.position))
                self.snowheight.append(tank.snowheight)
                index += 1

        if len(self.tankpositions) < 5: # or some arbitrary low number
            raise ValueError("Not enough tanks found!")

        self.snowheight = np.array(self.snowheight)
        self.PushFrame(frame)

    def Finish(self):
        if self.outFile:
            d = {}
            d['positions']   = self.tankpositions
            d['snowheights'] = self.snowheight
            checkdir(self.outFile)
            np.save(self.outFile, d)
        return

if __name__ == "__main__":

    # Global variables setup for path names
    # my.setupShowerLLH(verbose=False)

    p = argparse.ArgumentParser(
            description='Builds binned histograms for use with ShowerLLH')
    p.add_argument('-f', '--file', dest='file',
            help='Input filelist to run over')
    p.add_argument('-o', '--outFile', dest='outFile',
            help='Output filename')
    args = p.parse_args()

    # Execute
    tray = I3Tray()
    tray.Add('I3Reader', FileName=args.file)
    tray.Add(SaveSnowHeight,
        outFile=args.outFile)
    tray.Execute()
    tray.Finish()


import scrapy
import scrapy.spiders

from techradarscraper.items import TechLoader

class TechRadarSpider(scrapy.spiders.Spider):
    name = 'techradar'
    start_urls = ['https://www.thoughtworks.com/radar/a-z']

    def parse(self, response):
        links = response.css('.a-z-links > ul > li.blip.hit > a')
        for link in links:
            title = link.css('::text').extract_first()
            href = link.css('::attr(href)').extract_first()
            url = response.urljoin(href)
            yield scrapy.Request(url, self.parse_tech)

    def parse_tech(self, response):
        return TechLoader(response=response).load_item()

if __name__ == '__main__':
    from scrapy.utils.project import get_project_settings
    settings = get_project_settings()
    settings.setdict({
        'LOG_LEVEL': 'INFO',
        'OUTPUT_FILENAME': '../../out/techradar.json',
        'ITEM_PIPELINES': {
            'techradarscraper.pipelines.JsonWriterPipeline': 300,
        },
    })

    from scrapy.crawler import CrawlerProcess
    process = CrawlerProcess(settings)
    process.crawl(TechRadarSpider)
    process.start()


from __future__ import print_function
import matplotlib as mpl
mpl.use('agg')

import glob
import sys,os,shutil

sys.path.insert(0,'../')
import pyathena.yt_analysis.phase_plots_with_yt as ph
import yt

narg=len(sys.argv)
system='tigress'
base='/tigress/changgoo/'

# first argument to set system
system = sys.argv[1]
if system == 'pleiades':
    base='/u/ckim14/'
elif system =='tigress':
    base='/tigress/changgoo/'
elif system =='perseus':
    base='/perseus/scratch/gpfs/changgoo/'
elif system =='cori':
    base='/global/cscratch1/sd/changgoo/'
else:
    print('{} is not supported'.format(system))
    sys.exit()

# second argument to set problem ids
dirs=glob.glob('{}/{}'.format(base,sys.argv[2]))

# third and fourth arguments to set time range
t1 = eval(sys.argv[3])
t2 = eval(sys.argv[4])

overwrite=False
if narg > 5:
    overwrite=eval(sys.argv[5])

do_pickling = False
ids=[]
for dd in dirs:
    if os.path.isdir(dd):
        if os.path.isdir(dd+'/id0/'):
            ids.append(os.path.basename(dd))
            do_pickling = True
        else:
            do_pickling = False
            if os.path.isdir(dd+'/phase/'):
                ids.append(os.path.basename(dd))

for pid in ids:
    outdir='{}{}/phase/'.format(base,pid)
    if not os.path.isdir(outdir): os.mkdir(outdir)
    print(pid)
    if pid.startswith('R8') or pid.startswith('MHD') or pid.startswith('LGR8'):
        params=ph.phase_parameters(Sigma_factor=1.)
    elif pid.startswith('R4') or pid.startswith('LGR4'):
        params=ph.phase_parameters(Sigma_factor=10.)
        print(params.extrema['nH'])
    elif pid.startswith('R2') or pid.startswith('LGR2'):
        params=ph.phase_parameters(Sigma_factor=100.)
    elif pid.startswith('R16'):
        params=ph.phase_parameters(Sigma_factor=0.1)
    else:
        raise ValueError('need to set scale factor for {}'.format(pid))

    for i in range(t1,t2):
        filename='{}/{}/id0/{}.{:04d}.vtk'.format(base,pid,pid,i)
        if os.path.isfile(filename):
            print('*** Creating joint PDFs for {} ***'.format(os.path.basename(filename)))
            ph.phase_by_slab(filename,zmax=yt.YTQuantity(2048,'pc'),params=params,
                             verbose=60,overwrite=overwrite)
            for bf in params.bin_fields:
                print('*** Drawing for itime={} fields={bf[0]}-{bf[1]}'.format(i,bf=bf))
                for sidx in [[1,2],[8,9]]:
                    fig=ph.plot_joint_PDFs_slab(base,pid,i,bf,slab_index=sidx)
                    pngfname='{}{}/phase/{}.{:04d}-{b[0]}-{b[1]}-slab{s[0]}-{s[1]}.png'
                    pngfname=pngfname.format(base,pid,pid,i,b=bf,s=sidx)
                    fig.savefig(pngfname,bbox_inches='tight',dpi=150)


from dataHelper import get_dataset
import datasets
import logging
import torch
import os
import random
from dataclasses import dataclass, field
import numpy as np
import evaluate
import transformers
import wandb
from transformers import (
    AutoConfig,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    DataCollatorWithPadding,
    EvalPrediction,
    HfArgumentParser,
    PretrainedConfig,
    Trainer,
    TrainingArguments,
    default_data_collator,
    set_seed,
)
from peft import LoraConfig, get_peft_model
from transformers.trainer_utils import get_last_checkpoint
from transformers.utils import check_min_version, send_example_telemetry
from transformers.utils.versions import require_version

torch.cuda.set_per_process_memory_fraction(1.0, 0)
torch.cuda.empty_cache()

'''
settings
'''
lr = 1e-5
epoch_num = 5
batch_size = 16

classes_dic = {'restaurant_sup': 3, 'acl_sup': 6, 'agnews_sup': 4}

'''
Initialize metircs
'''
from datasets import load_metric
metric = load_metric("./metric.py")

# You can define your custom compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with a
# predictions and label_ids field) and has to return a dictionary string to float.
def compute_metrics(p: EvalPrediction):
    preds = p.predictions
    preds = np.argmax(preds, axis=1)
    result = metric.compute(predictions=preds, references=p.label_ids)
    return result

@dataclass
class Modelargs():
    model_name: str
    add_adaptor: bool

@dataclass
class Dataargs():
    dataset_name: str

logger = logging.getLogger(__name__)

def main():

        os.environ["HF_EVALUATE_OFFLINE"] = "1"

        '''
        initialize logging, seed, argparse...
        '''

        parser = HfArgumentParser((Modelargs, Dataargs, TrainingArguments))
        model_args, dataset_args, training_args = parser.parse_args_into_dataclasses()

        dataset_name = dataset_args.dataset_name
        model_name = model_args.model_name

        print("-----------------------------------------------------------------")
        print("Now we are running:")
        print("Dataset: "+ dataset_name)
        print("Model: " + model_name)
        print("Adaptor: " + str(model_args.add_adaptor))
        print("-----------------------------------------------------------------")

        training_args = TrainingArguments(
            output_dir=".\\results",
            learning_rate=lr,
            per_device_train_batch_size=batch_size,
            per_device_eval_batch_size=batch_size,
            num_train_epochs=epoch_num,
            logging_dir='./logs',            # directory for storing logs
            logging_steps=25,
            evaluation_strategy='steps',
            save_strategy='epoch',
            report_to='wandb',
        )

        logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')

        set_seed(training_args.seed)

        '''
            load datasets
        '''

        dataset = get_dataset(dataset_name, "<SEP>")

        '''
            load models
        '''

        config = AutoConfig.from_pretrained(model_name, num_labels=classes_dic[dataset_name])
        peft_config = LoraConfig(
            lora_alpha=16,
            lora_dropout=0.1,
            r=64,
            bias="none",
            task_type="SEQ_CLS",
        )

        tokenizer = AutoTokenizer.from_pretrained(model_name)
        tokenizer.pad_token = tokenizer.eos_token
        model = AutoModelForSequenceClassification.from_pretrained(model_name,config=config)
        if model_args.add_adaptor:
            model =  get_peft_model(model, peft_config)

        '''
            process datasets and build up datacollator
        '''

        def tokenize_dataset(dataset):
            return tokenizer(dataset["text"],truncation=True, padding=True, max_length=16, return_tensors="pt")

        dataset = dataset.map(tokenize_dataset, batched=True)

        data_collator = DataCollatorWithPadding(tokenizer=tokenizer)

        trainer = Trainer(
            model=model,
            args=training_args,
            train_dataset=dataset["train"],
            eval_dataset=dataset["test"],
            # compute_metrics=compute_metrics,
            tokenizer=tokenizer,
            data_collator=data_collator,
            compute_metrics=compute_metrics
        )

        wandb.init(project='Adaptor_v1.0', name=dataset_name+'-'+model_name, config=training_args)
        wandb.watch(model)
        # make the wandb obeserver to log the accuracy, micro_f1, macro_f1
        # wandb.log({'accuracy': trainer.compute_metrics['accuracy']})
        # define the name of the runs\

        trainer.train()

        trainer.evaluate()
        wandb.finish()

if __name__ == "__main__":
    main()


import filecmp
import os
import shutil
import sys
import tempfile
import unittest

TESTSCRIPT_DIR = os.path.dirname(__file__)
SCRIPT_DIR = os.path.realpath(os.path.join(TESTSCRIPT_DIR, os.pardir, os.pardir))
sys.path.append(SCRIPT_DIR)
import paypal2homebank # pylint: disable=import-error,wrong-import-position

class Testpaypal2homebank(unittest.TestCase):
    '''overall test'''

    @staticmethod
    def gen_tempfolder():
        '''Returns temporary folder name'''
        return tempfile.mkdtemp(prefix="tmp_paypal2homebank_tests")

    def test_01(self):
        '''Standard test'''

        temp_folder = self.gen_tempfolder()
        in_filepath = os.path.join(TESTSCRIPT_DIR, "paypal.csv")
        out_filepath = os.path.join(temp_folder, "homebank.csv")
        with open(out_filepath, mode="w", encoding="utf-8") as out_file:
            paypal2homebank.paypal2homebank(in_filepath, out_file)

        expected_out_filepath = os.path.join(TESTSCRIPT_DIR, "homebank_expected.csv")
        self.assertTrue(filecmp.cmp(out_filepath, expected_out_filepath, shallow=False))
        shutil.rmtree(temp_folder)

if __name__ == "__main__":
    unittest.main()


# -*- coding: utf-8 -*-
from django.shortcuts import render_to_response, get_object_or_404
from django.http import HttpResponseRedirect, get_host
from django.template import RequestContext
from django.db.models import Q
from django.http import Http404
from django.core.urlresolvers import reverse
from django.core.exceptions import ObjectDoesNotExist
from django.utils.translation import ugettext as _
from django.contrib.auth.models import User
from django.contrib.auth.decorators import login_required
from external.excel_response import ExcelResponse
from photos.models import PhotoSet

from events.models import Event, AnswerList
from events.forms import EventForm, QuestionsForm

def events_widget(request, template_name = "homepage.html"):

    events = Event.objects.filter(approved=True).order_by("-date_begin")

    return render_to_response(template_name, {
        "events": events,
        }, context_instance=RequestContext(request))

@login_required
def map(request, template_name="events/map.html"):
    return render_to_response(template_name)

@login_required
def destroy(request, id):
    """
    Delete event
    """
    event = get_object_or_404(Event, id=id)
    event.delete()

    redirect_to = reverse("events")

    return HttpResponseRedirect(redirect_to)

#@login_required
def details(request, id, template_name="events/details.html"):

    event = get_object_or_404(Event, id=id)

    if event.creator == request.user:
        is_me = True
    else:
        is_me = False

    is_member = event.user_is_member(request.user)

    if (not event.approved) and (not is_me):
        raise Http404
    photoset = get_object_or_404(PhotoSet, pk =   event.gallery.id)

    return render_to_response(template_name, {
    "event": event,
    "is_me": is_me,
    "is_member": is_member,
    "photoset": photoset,
    }, context_instance=RequestContext(request))

@login_required
def add_event(request, form_class=EventForm, template_name="events/add_event.html"):
    """
    Add new event
    """
    event_form = form_class(request.user)

    if request.method == 'POST' and request.POST.get("action") == "add":
        event_form = form_class(request.user, request.POST, request.FILES)

        if event_form.is_valid():
            event = event_form.save(commit=False)
            event.creator = request.user
            photoset = PhotoSet(name = event.title, user = request.user)
            photoset.save()

            event.gallery = photoset
            event.approved = request.user.is_staff
            # automatically approved if user is an administrator
            event.save()
            event.members.add(request.user)
            event.save()
            photoset.content_object = event
            photoset.save()

            if not event.approved:
                request.user.message_set.create(message=_(u"Адміністратор розгляне вашу заявку."))
            if request.POST.get("photoset") == "1":
                redirect_to = reverse('photos.views.editphotoset', kwargs={"id": photoset.pk})
            else:
                redirect_to = reverse("event_details", kwargs={"id": event.id})
            return HttpResponseRedirect(redirect_to)
#        else:
#            event_form = form_class()
    return render_to_response(template_name, {
        "event_form": event_form
    }, context_instance=RequestContext(request))

@login_required
def edit(request, id, form_class=EventForm, template_name="events/edit.html"):
    """
    Edit event
    """
    event = get_object_or_404(Event, id=id)

    event_form = form_class(request.user, instance=event)
    if request.method == "POST" and request.POST.get("action") == "update":
        event_form = form_class(request.user, request.POST, request.FILES, instance=event)
        if event_form.is_valid():
            event = event_form.save()

            include_kwargs = {"id": event.id}
            redirect_to = reverse("event_details", kwargs=include_kwargs)
            return HttpResponseRedirect(redirect_to)

    return render_to_response(template_name, {
        "event_form": event_form,
        "event": event,
    }, context_instance=RequestContext(request))

#@login_required
def events(request, template_name="events/latest.html"):
    """
    Latest ivents
    """
    events = Event.objects.filter(approved=True).order_by("-date_begin")
    photosets = PhotoSet.objects.all()

    photosets = photosets.order_by("-date_added")

    return render_to_response(template_name, {
        "events": events,
        "photosets": photosets,
        }, context_instance=RequestContext(request))

@login_required
def your_events(request, template_name="events/your_events.html"):
    return render_to_response(template_name, {
        "events": Event.objects.filter(creator=request.user),
    }, context_instance=RequestContext(request))

@login_required
def join(request, id, form_class=QuestionsForm, template_name="events/questions.html"):

    event = get_object_or_404(Event, id=id)

    if not event.approved:
        raise Http404

    include_kwargs = {"id": event.id}
    redirect_to = reverse("event_details", kwargs=include_kwargs)

    if event.user_is_member(request.user):
        request.user.message_set.create(message=_(u"Ви вже є учасником цієї події"))
    else:
        if event.questions == "":
            event.members.add(request.user)
            event.save()
        else:
            questions_form = form_class(event, request.user)
            questions_form.setQuestions(event.questions)

            if request.method == "POST":
                questions_form.setData(request.POST)
                if questions_form.is_valid():
                    questions_form.save()

                    event.members.add(request.user)
                    event.save()
                    return HttpResponseRedirect(redirect_to)

            return render_to_response(template_name, {
                "questions_form": questions_form,
                "event": event
            }, context_instance=RequestContext(request))

    return HttpResponseRedirect(redirect_to)

@login_required
def leave(request, id, template_name="events/details.html"):
    event = Event.objects.get(id = id)

    if event.creator != request.user:
        raise Http404

    if event.user_is_member(request.user):
        event.members.remove(request.user)
        try:
            answer_list = AnswerList.objects.get(event=event, user=request.user)
            answer_list.delete()
        except AnswerList.DoesNotExist:
            pass

    include_kwargs = {"id": event.id}
    redirect_to = reverse("event_details", kwargs=include_kwargs)
    return HttpResponseRedirect(redirect_to)

@login_required
def answers(request, id, template_name="events/answers.html"):
    event = get_object_or_404(Event, id=id)

    if event.creator != request.user:
        raise Http404

    if request.GET.get('action') == 'export':
        data = event.get_excel_data()
        return ExcelResponse(data, output_name='answers')

    table = event.get_table_data()

    return render_to_response(template_name, {
        "event": event,
        "table": table,
    }, context_instance=RequestContext(request))

@login_required
def members(request, id, template_name="events/members.html"):
    event = get_object_or_404(Event, id=id)

    if event.creator == request.user:
        is_me = True
    else:
        is_me = False

    if is_me and request.method == "POST" and request.POST.get('action') == "delete":
        try:
            member = User.objects.get(id = request.POST.get('member'))
            if event.user_is_member(member):
                event.members.remove(member)
                try:
                    answer_list = AnswerList.objects.get(event=event, user=member)
                    answer_list.delete()
                except AnswerList.DoesNotExist:
                    pass
                request.user.message_set.create(message=_(u"Учасника %s видалено з події.") % unicode(member.get_profile()))
        except User.DoesNotExist:
            pass

    return render_to_response(template_name, {
        "event": event,
        "is_me": is_me,
    }, context_instance=RequestContext(request))

def photoset(request, id):
    event = get_object_or_404(Event, id=id)
    pk = event.gallery.id
    photoset = get_object_or_404(PhotoSet, pk = pk)
    return render_to_response('events/details.html', {'photoset': photoset}, context_instance = RequestContext(request))