# augmenter.py
# augment data for disaster corpus
# clean text using cleaner
# create BOW's from POS tags for each class
# create simulated messages for class from its BOW's
# sample entries out of class
# augment sampled message with simulated message
# augment class flag to complement augmented simulation

# import libraries
import sys
import pandas as pd
import random
import re
import nltk
from nltk import pos_tag
from nltk.tokenize import sent_tokenize
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from collections import Counter
from itertools import chain
import cleaner

# text processing routine

def cleanmess(text):
  '''
  import message
  clean the message
  return the cleaned message
  '''
  sents = sent_tokenize(text)
  all_sents = []
  for sent in sents:
      clean = cleaner.CleanText(sent) + '.'
      all_sents.append(clean)

  return ' '.join(all_sents)

def tokenize(text):
    '''
    input clean message
    tokenize by sentences
    stem and remove stop words
    return cleaned tokens of text
    '''
    stop_words = stopwords.words("english") + ['.']
    lemmatizer = WordNetLemmatizer()

    sents = sent_tokenize(text)
    all_tokens = []
    for sent in sents:
        # convert previously cleaned sentence to tokens
        raw_tokens = nltk.word_tokenize(sent)
        pos_tags = pos_tag(raw_tokens)
        #process tokens
        stp_tokens = [(t,p) for t,p in pos_tags if t not in stop_words]
        tokens = [(lemmatizer.lemmatize(t),p) for t,p in stp_tokens]
        all_tokens += tokens

    return all_tokens

# utilities for message simulation

def BuildFreqDict(messages):
    '''
    Import series of messages
    Tokenize messages
    Build Token Count dictionaryt,p
    Return Token Count dictionary
    '''
    texts = messages.apply(tokenize).tolist()
    dwf = Counter(chain.from_iterable(texts))
    return dwf

def BuildBOWs(category, df_corpus):
    '''
    import category and corpus
    build frequency dictionary
    sort by part of speech
    build bag of words
    shuffle bag of words
    consolidate bags
    return list of bags
    '''
    # nltk POS notes:
    #
    # Here we see that 'and' is CC, a coordinating conjunction;
    # 'now' and 'completely' are RB, or adverbs;
    # 'for' is IN, a preposition;
    # 'something' is NN, a noun;
    # and 'different' is JJ, an adjective.
    #
    # https://www.nltk.org/book/ch05.html
    #
    # build frequency dictionary
    df = df_corpus.copy()
    dwf = BuildFreqDict(df[df[category] == 1]['message'])
    # build item lists
    nouns =[item for item in dwf if item[1] in ['NN','NNS','VBG','VBD']]
    verbs = [item for item in dwf if item[1] in ['VB','VBN','VBG','VBD']]
    adjectives = [item for item in dwf if item[1] in ['JJ']]
    adverbs = [item for item in dwf if item[1] in ['RB']]
    pos_items = [nouns,verbs,adjectives,adverbs]
    # build bags of words
    bow_list = []
    for item_list in pos_items:
        bow = []
        for item in item_list:
            bow += [item[0]]* dwf[item]
        random.shuffle(bow)
        bow_list.append(bow.copy())

    return bow_list

def BuildSimulatedMessage(bow_list):
    '''
    import bag of word lists
    random sample sentences
    random sample words
    return message
    '''
    # Basic sentence structures
    # Subject–Verb
    # Subject–Verb–ObjectS=
    # Subject–Verb–Adverb
    # Subject–Verb–Noun
    # https://www.wordy.com/writers-workshop/basic-english-sentence-structure/

    # sorted lists
    nouns,verbs,adjectives,adverbs = bow_list

    def RandomSent():
        '''
        generate a random sentence
        '''
        last_part = ['', random.choice(nouns), random.choice(adjectives),
                     random.choice(adverbs), random.choice(nouns)]
        sentence = '{} {} {}'.format(random.choice(nouns),
                                     random.choice(verbs),
                                     random.choice(last_part))
        return sentence.rstrip()

    min_length = random.randint(74,200)
    message = ''
    while len(message) < min_length:
        message += random.choice([ RandomSent() + '. ',
                                   RandomSent() + ' and ' + RandomSent() + '. ',
                                   RandomSent() + ' or ' + RandomSent() + '. '])
    return message

def CreateChildAloneMessage():
    '''
    Simulate a random child alone message.
    Return random message string.
    '''
    params =[['I','he','she','somebody','a stranger','authorities','helper','group'],
             ['found','lost','met','saw','lost track of', 'discovered','got'],
             ['got lost', 'abducted', 'was abandoned', 'got in an accident','is hurt and alone','is panicking'],
             ['child','boy','girl','son','daughter','kid','teenager'],
             ['Emma', 'Liam','Olivia','Noah','Ava','Elijah', 'Isabella', 'Logan'],
             ['four','five','six','seven','eight','nine','ten','eleven','twelve','thirteen'],
             ['black','brown','red','blond','bald','pink','blue'],
             ['swimsuit','white shirt','red shoes','green jacket','jeans','baseball hat'],
             ['wearing glasses','skinny','fat','loud','shy','tall','short'],
             ['coughing','sneezing','hyperactive','cold','sweating','hurt','healthy'],
             ['around noon','this morning','in the afternoon','after lunch', '3pm','10 a.m.','last night'],
             ['yesterday', 'day before yesterday', 'Friday', 'sat', 'today', 'now', 'this week'],
             ['grocery store','park','pool','school','beach','mall','street','sidewalk']]

    who_rept, obs_verb, event_verb, event_noun,\
    kid_name, kid_age, kid_hair, kid_outfit,\
    kid_feat, kid_health, event_time, event_day,\
    event_place = [random.choice(l) for l in params]

    sents = [['A {} {}. '.format(event_noun, event_verb),
              '{} {} a {}. '.format(who_rept,obs_verb,event_noun),
              'A {} {} named {}. '.format(event_noun, event_verb, kid_name)],
             ['{} and {} hair. '.format(kid_age, kid_hair),
              '{} hair and {}. '.format(kid_hair, kid_health),
              '{} wearing {} and {} years. '.format(kid_health, kid_outfit, kid_age)],
             ['Here {} at {}.'.format(event_time, event_place),
              'Was at the {}, {}. '. format(event_place, event_time),
              'Need help {}, please come to {}, {}. '.format(event_day, event_place, event_time)]]

    message = ('').join([random.choice(s) for s in sents])

    return message

def BuildCAs(dft):
    '''
    import train dataframe
    slice by category not true
    sample entries as copies
    update categories
    add simulated messages
    return piggy backed df
    '''
    # Simulate 'child_alone' message piggy-backed on other samples
    id_start = max(dft['id'].values) + 1
    df_ca = dft[(dft['related']==1) & (dft['offer']!=1)].sample(1000).copy()
    df_ca['id'] = [i for i in range(id_start, id_start+1000)]
    df_ca['aid_related'] = 1
    df_ca['child_alone'] = 1
    df_ca['message'] = df_ca['message'].map(lambda x: CreateChildAloneMessage() + x)
    df_ca = df_ca.reset_index(drop=True)

    return df_ca

def BuildSims(df_aug, dft):
    '''
    import augment and training df's
    itterate by low count categories
    slice by category not true
    sample entries as copies
    update categories
    add simulated messages
    return piggy backed df
    '''
    # work on copy
    dfa = df_aug.copy()
    # build dictionary of partner categories for low count categories
    partner = {'offer': ['aid_related'],
               'search_and_rescue':['aid_related'],
               'security':['aid_related','direct_report'],
               'military':['aid_related','direct_report'],
               'clothing':['aid_related'],
               'money':['request'],
               'missing_people':['aid_related'],
               'refugees':['aid_related'],
               'death':['aid_related'],
               'electricity':['request','aid_related','direct_report'],
               'tools':['aid_related'],
               'hospitals':['aid_related','infrastructure_related'],
               'shops':['aid_related','food'],
               'aid_centers':['aid_related','infrastructure_related'],
               'other_infrastructure':['aid_related','infrastructure_related','request'],
               'fire':['weather_related'],
               'cold':['weather_related']}

    # Simulate messages and augment them as piggy-backed on other samples
    for cat in list(partner.keys()):
        id_start = max(dfa['id'].values) + 1
        dfc = dft[(dft['related']==1) & (dft[cat]!=1)].sample(1000).copy()
        dfc['id'] = [i for i in range(id_start, id_start+1000)]
        dfc[cat] = 1
        for prdnr in partner[cat]:
            dfc[prdnr] = 1
        # tack on simulated messages
        bow = BuildBOWs(cat, dft)
        dfc['message'] = dfc['message'].map(lambda x: BuildSimulatedMessage(bow) + x)
        dfc = dfc.reset_index(drop=True)
        dfa = pd.concat([dfa, dfc], axis = 0, ignore_index=True)
    dfa.reset_index(drop=True, inplace=True)

    return dfa

def simulate(df_dirty):
    '''
    import formatted df
    clean messages
    split off validation set
    simulate data for empty categories
    return clean, validation, training, and simulated df's
    '''
    dfc = df_dirty.copy()
    # clean
    dfc['message'] = dfc.message.apply(cleanmess)

    # category column names
    category_colnames = dfc.columns[2:].tolist()

    # start with some related zero's
    dfv = dfc[dfc['related']==0].sample(36).copy()

    # add category entries to validation set
    idx_list = dfv.index.tolist()
    # add in some one's for each target category
    for cat in category_colnames:
        # manage child alone has no entries
        if cat != 'child_alone':
            df_slice = dfc[dfc[cat]==1].copy()
            samp = df_slice.sample()
            while samp.index[0] in dfv.index:
                samp = df_slice.sample()
            dfv = pd.concat([dfv, samp], axis=0)
        else:
            pass

    # independant train and validation data
    dft = dfc[~dfc.index.isin(dfv.index)].copy()

    # Simulate child alone Messages
    df_aug = BuildCAs(dft)

    # sample a 'child alone' entry for validation
    samp = df_aug.sample()
    dfv = pd.concat([dfv, samp], axis=0)

    # Simulate low count class messages
    dfa = BuildSims(df_aug, dft)

    return dfc, dfv, dft, dfa


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

import calendar
from datetime import timedelta
import logging
import json
import uuid

from django.conf import settings
from django.contrib.auth.models import Group, User
from django.contrib.gis.db import models
from django.contrib.gis.geos import GEOSGeometry
from django.contrib.postgres.fields import ArrayField
from django.db.models.fields import CharField
from django.db.models.signals import post_delete, post_save
from django.dispatch.dispatcher import receiver
from django.utils import timezone
from hdx.configuration import Configuration

from hdx_exports.hdx_export_set import HDXExportSet
from feature_selection.feature_selection import FeatureSelection
from utils import FORMAT_NAMES

HDX_URL_PREFIX = Configuration.create(
    hdx_site=settings.HDX_SITE,
    hdx_key=settings.HDX_API_KEY,
)
LOG = logging.getLogger(__name__)

# construct the upload path for export config files..

def get_upload_path(instance, filename):
    """
    Construct the path to where the uploaded config file is to be stored.
    """
    configtype = instance.config_type.lower()
    # sanitize the filename here..
    path = 'export/config/{0}/{1}'.format(configtype, instance.filename)
    return path

class LowerCaseCharField(CharField):
    """
    Defines a charfield which automatically converts all inputs to
    lowercase and saves.
    """

    def pre_save(self, model_instance, add):
        """
        Converts the string to lowercase before saving.
        """
        current_value = getattr(model_instance, self.attname)
        setattr(model_instance, self.attname, current_value.lower())
        return getattr(model_instance, self.attname)

class TimeStampedModelMixin(models.Model):
    """
    Mixin for timestamped models.
    """
    created_at = models.DateTimeField(default=timezone.now, editable=False)
    updated_at = models.DateTimeField(default=timezone.now, editable=False)

    class Meta:  # pragma: no cover
        abstract = True

class ExportConfig(TimeStampedModelMixin):
    """
    Model for export configuration.
    """
    PRESET = 'PRESET'
    CONFIG_TYPES = (
        (PRESET, 'Preset'),
    )
    id = models.AutoField(primary_key=True, editable=False)
    uid = models.UUIDField(unique=True, default=uuid.uuid4, editable=False)
    name = models.CharField(max_length=255, default='', db_index=True)
    user = models.ForeignKey(User, related_name='user')
    config_type = models.CharField(max_length=11, choices=CONFIG_TYPES, default=PRESET)
    filename = models.CharField(max_length=255)
    upload = models.FileField(max_length=255, upload_to=get_upload_path)
    content_type = models.CharField(max_length=30, editable=False)
    published = models.BooleanField(default=False)

    class Meta:  # pragma: no cover
        managed = True
        db_table = 'export_configurations'

class Job(TimeStampedModelMixin):
    """
    Model for a Job.
    """
    id = models.AutoField(primary_key=True, editable=False)
    uid = models.UUIDField(unique=True, default=uuid.uuid4, editable=False, db_index=True)
    user = models.ForeignKey(User, related_name='owner')
    name = models.CharField(max_length=100, db_index=True)
    description = models.CharField(max_length=1000, db_index=True)
    event = models.CharField(max_length=100, db_index=True, default='', blank=True)
    export_formats = ArrayField(models.CharField(max_length=10), default=list)
    config = models.ForeignKey(ExportConfig, related_name = 'config',null=True)
    published = models.BooleanField(default=False, db_index=True)  # publish export
    feature_save = models.BooleanField(default=False, db_index=True)  # save feature selections
    feature_pub = models.BooleanField(default=False, db_index=True)  # publish feature selections
    the_geom = models.GeometryField(verbose_name='Extent for export', srid=4326, default='')
    objects = models.GeoManager()
    feature_selection = models.TextField(blank=True)
    buffer_aoi = models.BooleanField(default=False)

    class Meta:  # pragma: no cover
        managed = True
        db_table = 'jobs'

    def save(self, *args, **kwargs):
        super(Job, self).save(*args, **kwargs)

    def __str__(self):
        return '{0}'.format(self.name)

    @property
    def feature_selection_object(self):
        """
        a valid FeatureSelection object based off the feature_selection column.
        """
        fs = FeatureSelection(self.feature_selection)
        # assert fs.valid, 'Feature selection is invalid'
        return fs

    @property
    def tag_dict(self,):
        """
        Return the unique set of Tag keys from this export
        with their associated geometry types.

        Used by Job.categorised_tags (below) to categorize tags
        according to their geometry types.
        """
        # get the unique keys from the tags for this export
        uniq_keys = list(self.tags.values('key').distinct('key'))
        tag_dict = {}  # mapping of tags to geom_types
        for entry in uniq_keys:
            key = entry['key']
            tag_dict['key'] = key
            geom_types = list(self.tags.filter(key=key).values('geom_types'))
            geom_type_list = []
            for geom_type in geom_types:
                geom_list = geom_type['geom_types']
                geom_type_list.extend([i for i in geom_list])
            tag_dict[key] = list(set(geom_type_list))  # get unique values for geomtypes
        return tag_dict

    @property
    def filters(self,):
        """
        Return key=value pairs for each tag in this export.

        Used in utils.overpass.filter to filter the export.
        """
        filters = []
        for tag in self.tags.all():
            kv = '{0}={1}'.format(tag.key, tag.value)
            filters.append(kv)
        return filters

    @property
    def categorised_tags(self,):
        """
        Return tags mapped according to their geometry types.
        """
        points = []
        lines = []
        polygons = []
        for tag in self.tag_dict:
            for geom in self.tag_dict[tag]:
                if geom == 'point':
                    points.append(tag)
                if geom == 'line':
                    lines.append(tag)
                if geom == 'polygon':
                    polygons.append(tag)
        return {'points': sorted(points), 'lines': sorted(lines), 'polygons': sorted(polygons)}

class Tag(models.Model):
    """
    Model to hold Export tag selections.

    Holds the data model (osm | hdm | preset)
    and the geom_type mapping.
    """
    id = models.AutoField(primary_key=True, editable=False)
    name = models.CharField(max_length=100, blank=False, default='', db_index=True)
    key = models.CharField(max_length=50, blank=False, default='', db_index=True)
    value = models.CharField(max_length=50, blank=False, default='', db_index=True)
    job = models.ForeignKey(Job, related_name='tags')
    data_model = models.CharField(max_length=10, blank=False, default='', db_index=True)
    geom_types = ArrayField(models.CharField(max_length=10, blank=True, default=''), default=[])
    groups = ArrayField(models.CharField(max_length=100, blank=True, default=''), default=[])

    class Meta:  # pragma: no cover
        managed = True
        db_table = 'tags'

    def __str__(self):  # pragma: no cover
        return '{0}:{1}'.format(self.key, self.value)

class HDXExportRegion(models.Model): # noqa
    PERIOD_CHOICES = (
        ('6hrs', 'Every 6 hours'),
        ('daily', 'Every day'),
        ('weekly', 'Every Sunday'),
        ('monthly', 'The 1st of every month'),
        ('disabled', 'Disabled'),
    )
    HOUR_CHOICES = zip(xrange(0, 24), xrange(0, 24))
    EXPORT_FORMAT_CHOICES = map(
        lambda name: (name, FORMAT_NAMES[name].description), FORMAT_NAMES)
    schedule_period = models.CharField(
        blank=False, max_length=10, default="disabled", choices=PERIOD_CHOICES)
    schedule_hour = models.IntegerField(
        blank=False, choices=HOUR_CHOICES, default=0)
    deleted = models.BooleanField(default=False)
    job = models.ForeignKey(Job, null=True,
        related_name='hdx_export_region_set')
    is_private = models.BooleanField(default=False)
    locations = ArrayField(
        models.CharField(blank=False, max_length=32), null=True)
    license = models.CharField(max_length=32, null=True)
    subnational = models.BooleanField(default=True)
    extra_notes = models.TextField(null=True)

    class Meta: # noqa
        db_table = 'hdx_export_regions'

    @property
    def delta(self): # noqa
        if self.schedule_period == '6hrs':
            return timedelta(hours=6)

        if self.schedule_period == 'daily':
            return timedelta(days=1)

        if self.schedule_period == 'weekly':
            return timedelta(days=7)

        if self.schedule_period == 'monthly':
            return timedelta(days=31)

    @property
    def next_run(self): # noqa
        now = timezone.now().replace(minute=0, second=0, microsecond=0)

        if self.schedule_period == '6hrs':
            delta = 6 - (self.schedule_hour + now.hour % 6)

            return now + timedelta(hours=delta)

        now = now.replace(hour=self.schedule_hour)

        if self.schedule_period == 'daily':
            anchor = now

            if timezone.now() < anchor:
                return anchor

            return anchor + timedelta(days=1)

        if self.schedule_period == 'weekly':
            # adjust so the week starts on Sunday
            anchor = now - timedelta((now.weekday() + 1) % 7)

            if timezone.now() < anchor:
                return anchor

            return anchor + timedelta(days=7)

        if self.schedule_period == 'monthly':
            (_, num_days) = calendar.monthrange(now.year, now.month)
            anchor = now.replace(day=1)

            if timezone.now() < anchor:
                return anchor

            return anchor + timedelta(days=num_days)

    @property
    def last_run(self): # noqa
        last = self.job.runs.last()
        if last is not None:
            return last.finished_at

    @property
    def buffer_aoi(self): # noqa
        return self.job.buffer_aoi

    @property
    def name(self): # noqa
        return self.job.description

    @property
    def dataset_prefix(self): # noqa
        return self.job.name

    @property
    def the_geom(self): # noqa
        return json.loads(GEOSGeometry(self.job.the_geom).geojson)

    @property
    def feature_selection(self): # noqa
        return self.job.feature_selection

    @property
    def export_formats(self): # noqa
        return self.job.export_formats

    @property
    def datasets(self): # noqa
        return map(lambda x: {
            'name': '{}_{}'.format(self.dataset_prefix, x),
            'url': '{}dataset/{}_{}'.format(
                HDX_URL_PREFIX, self.dataset_prefix, x),
        }, self.job.feature_selection_object.themes)

    @property
    def runs(self): # noqa
        return map(lambda run: {
            'elapsed_time': (run.finished_at or timezone.now()) - run.started_at,
            'run_at': run.started_at,
            'size': sum(map(
                lambda task: task.filesize_bytes or 0, run.tasks.all())),
            'status': run.status,
            'uid': run.uid,
        }, self.job.runs.order_by('-created_at').all())

    @property
    def hdx_dataset(self): # noqa
        """
        Initialize an HDXExportSet corresponding to this Model.
        """
#       # TODO make distinction between GOESGeom/GeoJSON better
        return HDXExportSet(
            data_update_frequency=self.update_frequency,
            dataset_date=timezone.now(),
            dataset_prefix=self.dataset_prefix,
            extent=self.job.the_geom,
            extra_notes=self.extra_notes,
            feature_selection=self.job.feature_selection_object,
            is_private=self.is_private,
            license=self.license,
            locations=self.locations,
            name=self.name,
            subnational=self.subnational,
        )

    @property
    def update_frequency(self):
        """Update frequencies in HDX form."""
        if self.schedule_period == '6hrs':
            return 1

        if self.schedule_period == 'daily':
            return 1

        if self.schedule_period == 'weekly':
            return 7

        if self.schedule_period == 'monthly':
            return 30

        return 0

    def sync_to_hdx(self): # noqa
        LOG.info("HDXExportRegion.sync_to_hdx called.")
        self.hdx_dataset.sync_datasets()

@receiver(post_delete, sender=ExportConfig)
def exportconfig_delete_upload(sender, instance, **kwargs):
    """
    Delete the associated file when the export config is deleted.
    """
    instance.upload.delete(False)
    # remove config from related jobs
    exports = Job.objects.filter(config__uid=instance.uid)
    for export in exports.all():
        export.config = None
        export.save()


# -*- coding: utf-8 -*-
"""
文 件 名: predict_result.py
文件描述: 预测结果
作    者: HanKin
创建日期: 2022.10.26
修改日期：2022.10.26

Copyright (c) 2022 HanKin. All rights reserved.
"""

from common import *

def load_model(model_path):
    """加载模型
    """

    with open(model_path, 'rb') as fd:
        buffer = fd.read()
    model = pickle.loads(buffer)
    return model

def voting_decision(df, rrname):
    """投票决定最终结果
    训练集标签结果中有138144个0，2149个1
    """

    tmp = df[df['domain'] == rrname]
    if tmp[tmp['label'] == 0].shape[0] >= tmp[tmp['label'] == 1].shape[0]:
        return {'domain': rrname, 'label': 0}
    else:
        return {'domain': rrname, 'label': 1}

def result_processing(df):
    """
    """

    label0_list = set(df[df['label'] == 0]['domain'])
    label1_list = set(df[df['label'] == 1]['domain'])
    intersection = list(label0_list & label1_list)
    logger.info('{}, {}, {}'.format(len(label0_list), len(label1_list), len(intersection)))

    domain = []
    label = []
    for rrname in label0_list:
        if rrname not in intersection:
            domain.append(rrname)
            label.append(0)

    for rrname in label1_list:
        if rrname not in intersection:
            domain.append(rrname)
            label.append(1)

    result = pd.DataFrame({
                'domain': domain,
                'label' : label
           })

    tasks = []
    with ThreadPoolExecutor(max_workers=THREAD_NUM) as pool:
        for rrname in intersection:
            task = pool.submit(voting_decision, df, rrname)
            tasks.append(task)

    for task in tasks:
        #result = result.append(task.result(), ignore_index=True)
        result = pd.concat([result, pd.DataFrame([task.result()])], ignore_index=True)
    logger.info('predict result shape: {}'.format(result.shape))
    result.to_csv(RESULT_PATH, index=False, header=False)

def predict(test_dataset_path, model_path):
    """加载模型进行测试集预测
    """

    # 获取数据集
    test_dataset = pd.read_csv(test_dataset_path)
    logger.info('test dataset shape: ({}, {}).'.format(test_dataset.shape[0], test_dataset.shape[1]))

    # 模型预测结果
    rrname = test_dataset['rrname']
    X = test_dataset.drop(['rrname'], axis=1, inplace=False).values
    #selector = load_model(MALICIOUS_SAMPLE_DETECTION_SELECTOR_PATH)
    #X = selector.transform(X)
    model = load_model(model_path)
    result = model.predict(X)

    # 存储结果
    df = pd.DataFrame({
                'domain': rrname,
                'label' : result
           })
    logger.info('predict result shape: {}'.format(df.shape))
    result_processing_v3(df)

def main():
    predict(TEST_DATASET_PATH, RFC_MODEL_PATH)

if __name__ == '__main__':
    #os.system('chcp 936 & cls')
    logger.info('******** starting ********')
    start_time = time.time()

    main()

    end_time = time.time()
    logger.info('process spend {} s.\n'.format(round(end_time - start_time, 3)))


#!/usr/bin/env python3

import os
import glob
import argparse
import random

FLAG = None

def parse_single_file(fn):
    feat_list = []
    with open(FLAG.dir_name + '/' + fn, 'r') as f:
        for line in f:
            line_sp = line.rstrip().split(',')
            line_sp.append(fn)
            feat_list.append(line_sp)

    return feat_list

def get_all_files():
    return_list = []
    for i in os.listdir(FLAG.dir_name):
        if i == '0':
            continue
        return_list.extend(parse_single_file(i))

    return return_list

def split_train_test(feat_list):
    shuf = list(range(len(feat_list)))
    random.shuffle(shuf)
    test_list = []
    train_list= []
    for i in range(len(feat_list)):
        if i < FLAG.test_num:
            test_list.append(feat_list[shuf[i]])
        else:
            train_list.append(feat_list[shuf[i]])

    return train_list, test_list

def write_file(fn, feat_list):
    with open(fn,'w') as f:
        for i, feat_lab in enumerate(feat_list):
            for j, feat in enumerate(feat_lab):
                if j != len(feat_lab)-1:
                    f.write(feat+' ')
                else:
                    f.write(feat+'\n')
    return

def main():
    all_feat_list = get_all_files()
    train, test = split_train_test(all_feat_list)
    write_file(FLAG.out_name+'_train', train)
    write_file(FLAG.out_name+'_test', test)

    return

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='a python script for dividing whole data into'
        ' training and testing set')
    parser.add_argument('dir_name',
        metavar='<directory name>',
        help='the directory that contains a lot of features in label file')
    parser.add_argument('out_name',
        metavar='<output name>',
        help='the output will be extensioned with train and test\n'
        'eg: tmp_train, tmp_test')
    parser.add_argument('--test_num',type=int,
        default=1000,
        metavar='<num of testing>',
        help='the number of testing number')
    FLAG = parser.parse_args()
    main()


from sys import argv, exit
import csv
import cs50

def main():
    if(len(argv) != 2):
        print("Usage: python roster.py house")
        exit(1)

    # Open databasefile.
    db = cs50.SQL("sqlite:///students.db")

    # Select all students from the house that was entered as an argument.
    result = db.execute("SELECT first, middle, last, birth FROM students WHERE house = ? ORDER BY last, first ASC", (argv[1],))

    # For each row, print
    for row in result:
        # Only print middlename if its not None.
        if row['middle'] == None:
            print(f"{row['first']} {row['last']}, born {row['birth']}")
        else:
            print(f"{row['first']} {row['middle']} {row['last']}, born {row['birth']}")

main()


#!/usr/bin/env python3
from random import randint
import re
from argparse import ArgumentParser

TRANSLATION_REGEX = r"([/ \t\w\-\(\)]{3,})=([/ \t\w\-\(\)]{3,})"
SKIP_PROBA = 90
REVERSE_PROBA = 50

def buildArgParser():
	parser = ArgumentParser()
	parser.add_argument("trans_file",metavar="translation file",
						help="a file containing translations (<word> = <translation>)")

	parser.add_argument("-p","--skipproba", metavar="skip probability",type=int, default=SKIP_PROBA,
						help="probability to skip the next question.")
	return parser

def main():
	parser = buildArgParser()
	args = parser.parse_args()

	try:
		f = open(args.trans_file)
	except Exception as e:
		print(e)
		exit(1)

	for line in f:

		if randint(0,100) < args.skipproba:
			continue

		match = re.match(TRANSLATION_REGEX, line)
		if match:
			ask_question(match.group(1).strip(), match.group(2).strip())
		else:
			print("error : %s" %line)
		print()
	f.close()

def ask_question(left_part, right_part):
	if randint(0,100) < REVERSE_PROBA:
		temp = right_part
		right_part = left_part
		left_part = temp

	print(left_part)
	answer = input().capitalize()

	if answer not in right_part.capitalize():
		print(right_part)

main()


import os

from nose import SkipTest, with_setup
from nose.plugins.attrib import attr
from numpy.testing import assert_equal, assert_raises

from brian2 import *
from brian2.core.functions import timestep
from brian2.parsing.sympytools import str_to_sympy, sympy_to_str
from brian2.utils.logger import catch_logs
from brian2.devices.device import reinit_devices
from brian2.tests.utils import assert_allclose
from brian2.codegen.generators import CodeGenerator
from brian2.codegen.codeobject import CodeObject

@attr('codegen-independent')
def test_constants_sympy():
    '''
    Make sure that symbolic constants are understood correctly by sympy
    '''
    assert sympy_to_str(str_to_sympy('1.0/inf')) == '0'
    assert sympy_to_str(str_to_sympy('sin(pi)')) == '0'
    assert sympy_to_str(str_to_sympy('log(e)')) == '1'

@attr('standalone-compatible')
@with_setup(teardown=reinit_devices)
def test_constants_values():
    '''
    Make sure that symbolic constants use the correct values in code
    '''
    G = NeuronGroup(3, 'v : 1')
    G.v[0] = 'pi'
    G.v[1] = 'e'
    G.v[2] = 'inf'
    run(0*ms)
    assert_allclose(G.v[:], [np.pi, np.e, np.inf])

def test_math_functions():
    '''
    Test that math functions give the same result, regardless of whether used
    directly or in generated Python or C++ code.
    '''
    default_dt = defaultclock.dt
    test_array = np.array([-1, -0.5, 0, 0.5, 1])
    def int_(x):
        return array(x, dtype=int)
    int_.__name__ = 'int'

    with catch_logs() as _:  # Let's suppress warnings about illegal values
        # Functions with a single argument
        for func in [cos, tan, sinh, cosh, tanh,
                     arcsin, arccos, arctan,
                     log, log10,
                     exp, np.sqrt,
                     np.ceil, np.floor, np.sign, int_]:

            # Calculate the result directly
            numpy_result = func(test_array)

            # Calculate the result in a somewhat complicated way by using a
            # subexpression in a NeuronGroup
            if func.__name__ == 'absolute':
                # we want to use the name abs instead of absolute
                func_name = 'abs'
            else:
                func_name = func.__name__
            G = NeuronGroup(len(test_array),
                            '''func = {func}(variable) : 1
                               variable : 1'''.format(func=func_name))
            G.variable = test_array
            mon = StateMonitor(G, 'func', record=True)
            net = Network(G, mon)
            net.run(default_dt)

            assert_allclose(numpy_result, mon.func_.flatten(),
                            err_msg='Function %s did not return the correct values' % func.__name__)

        # Functions/operators
        scalar = 3
        for func, operator in [(np.power, '**'), (np.mod, '%')]:

            # Calculate the result directly
            numpy_result = func(test_array, scalar)

            # Calculate the result in a somewhat complicated way by using a
            # subexpression in a NeuronGroup
            G = NeuronGroup(len(test_array),
                            '''func = variable {op} scalar : 1
                               variable : 1'''.format(op=operator))
            G.variable = test_array
            mon = StateMonitor(G, 'func', record=True)
            net = Network(G, mon)
            net.run(default_dt)

            assert_allclose(numpy_result, mon.func_.flatten(),
                            err_msg='Function %s did not return the correct values' % func.__name__)

@attr('standalone-compatible')
@with_setup(teardown=reinit_devices)
def test_clip():
    G = NeuronGroup(4, '''
                       clipexpr1 = clip(integer_var1, 0, 1) : integer
                       clipexpr2 = clip(integer_var2, -0.5, 1.5) : integer
                       clipexpr3 = clip(float_var1, 0, 1) : 1
                       clipexpr4 = clip(float_var2, -0.5, 1.5) : 1
                       integer_var1 : integer
                       integer_var2 : integer
                       float_var1 : 1
                       float_var2 : 1
                       ''')
    G.integer_var1 = [0, 1, -1, 2]
    G.integer_var2 = [0, 1, -1, 2]
    G.float_var1 = [0., 1., -1., 2.]
    G.float_var2 = [0., 1., -1., 2.]
    s_mon = StateMonitor(G, ['clipexpr1', 'clipexpr2',
                             'clipexpr3', 'clipexpr4'], record=True)
    run(defaultclock.dt)
    assert_equal(s_mon.clipexpr1.flatten(), [0, 1, 0, 1])
    assert_equal(s_mon.clipexpr2.flatten(), [0, 1, 0, 1])
    assert_allclose(s_mon.clipexpr3.flatten(), [0, 1, 0, 1])
    assert_allclose(s_mon.clipexpr4.flatten(), [0, 1, -0.5, 1.5])

@attr('standalone-compatible')
@with_setup(teardown=reinit_devices)
def test_bool_to_int():
    # Test that boolean expressions and variables are correctly converted into
    # integers
    G = NeuronGroup(2, '''
                       intexpr1 = int(bool_var) : integer
                       intexpr2 = int(float_var > 1.0) : integer
                       bool_var : boolean
                       float_var : 1
                       ''')
    G.bool_var = [True, False]
    G.float_var = [2.0, 0.5]
    s_mon = StateMonitor(G, ['intexpr1', 'intexpr2'], record=True)
    run(defaultclock.dt)
    assert_equal(s_mon.intexpr1.flatten(), [1, 0])
    assert_equal(s_mon.intexpr2.flatten(), [1, 0])

@attr('codegen-independent')
def test_timestep_function():
    dt = defaultclock.dt_
    # Check that multiples of dt end up in the correct time step
    t = np.arange(100000)*dt
    assert_equal(timestep(t, dt), np.arange(100000))

    # Scalar values should stay scalar
    ts = timestep(0.0005, 0.0001)
    assert np.isscalar(ts) and ts == 5

    # Length-1 arrays should stay arrays
    ts = timestep(np.array([0.0005]), 0.0001)
    assert ts.shape == (1,) and ts == 5

@attr('standalone-compatible')
@with_setup(teardown=reinit_devices)
def test_timestep_function_during_run():
    group = NeuronGroup(2, '''ref_t : second
                              ts = timestep(ref_t, dt) + timestep(t, dt) : integer''')
    group.ref_t = [-1e4*second, 5*defaultclock.dt]
    mon = StateMonitor(group, 'ts', record=True)
    run(5*defaultclock.dt)
    assert all(mon.ts[0] <= -1e4)
    assert_equal(mon.ts[1], [5, 6, 7, 8, 9])

@attr('standalone-compatible')
@with_setup(teardown=reinit_devices)
def test_user_defined_function():
    @implementation('cpp',"""
                inline double usersin(double x)
                {
                    return sin(x);
                }
                """)
    @implementation('cython', '''
            cdef double usersin(double x):
                return sin(x)
            ''')
    @check_units(x=1, result=1)
    def usersin(x):
        return np.sin(x)

    default_dt = defaultclock.dt
    test_array = np.array([0, 1, 2, 3])
    G = NeuronGroup(len(test_array),
                    '''func = usersin(variable) : 1
                              variable : 1''')
    G.variable = test_array
    mon = StateMonitor(G, 'func', record=True)
    run(default_dt)
    assert_allclose(np.sin(test_array), mon.func_.flatten())

@with_setup(teardown=reinit_devices)
def test_user_defined_function_units():
    '''
    Test the preparation of functions for use in code with check_units.
    '''
    if prefs.codegen.target != 'numpy':
        raise SkipTest('numpy-only test')

    def nothing_specified(x, y, z):
        return x*(y+z)

    no_result_unit = check_units(x=1, y=second, z=second)(nothing_specified)
    one_arg_missing = check_units(x=1, z=second, result=second)(nothing_specified)
    all_specified = check_units(x=1, y=second, z=second, result=second)(nothing_specified)

    G = NeuronGroup(1, '''a : 1
                          b : second
                          c : second''',
                    namespace={'nothing_specified': nothing_specified,
                               'no_result_unit': no_result_unit,
                               'one_arg_missing': one_arg_missing,
                               'all_specified': all_specified})
    net = Network(G)
    net.run(0*ms)  # make sure we have a clock and therefore a t
    G.c = 'all_specified(a, b, t)'
    assert_raises(ValueError,
                  lambda: setattr(G, 'c', 'one_arg_missing(a, b, t)'))
    assert_raises(ValueError,
                  lambda: setattr(G, 'c', 'no_result_unit(a, b, t)'))
    assert_raises(KeyError,
                  lambda: setattr(G, 'c', 'nothing_specified(a, b, t)'))
    assert_raises(DimensionMismatchError,
                  lambda: setattr(G, 'a', 'all_specified(a, b, t)'))
    assert_raises(DimensionMismatchError,
                  lambda: setattr(G, 'a', 'all_specified(b, a, t)'))

def test_simple_user_defined_function():
    # Make sure that it's possible to use a Python function directly, without
    # additional wrapping
    @check_units(x=1, result=1)
    def usersin(x):
        return np.sin(x)

    usersin.stateless = True

    default_dt = defaultclock.dt
    test_array = np.array([0, 1, 2, 3])
    G = NeuronGroup(len(test_array),
                    '''func = usersin(variable) : 1
                       variable : 1''',
                    codeobj_class=NumpyCodeObject)
    G.variable = test_array
    mon = StateMonitor(G, 'func', record=True, codeobj_class=NumpyCodeObject)
    net = Network(G, mon)
    net.run(default_dt)

    assert_allclose(np.sin(test_array), mon.func_.flatten())

    # Check that it raises an error for C++
    try:
        import scipy.weave
        G = NeuronGroup(len(test_array),
                        '''func = usersin(variable) : 1
                              variable : 1''',
                        codeobj_class=WeaveCodeObject)
        mon = StateMonitor(G, 'func', record=True,
                           codeobj_class=WeaveCodeObject)
        net = Network(G, mon)
        # This looks a bit odd -- we have to get usersin into the namespace of
        # the lambda expression
        assert_raises(NotImplementedError,
                      lambda usersin: net.run(0.1*ms), usersin)
    except ImportError:
        pass

def test_manual_user_defined_function():
    if prefs.codegen.target != 'numpy':
        raise SkipTest('numpy-only test')

    default_dt = defaultclock.dt

    # User defined function without any decorators
    def foo(x, y):
        return x + y + 3*volt
    orig_foo = foo
    # Since the function is not annotated with check units, we need to specify
    # both the units of the arguments and the return unit
    assert_raises(ValueError, lambda: Function(foo, return_unit=volt))
    assert_raises(ValueError, lambda: Function(foo, arg_units=[volt, volt]))
    foo = Function(foo, arg_units=[volt, volt], return_unit=volt)

    assert foo(1*volt, 2*volt) == 6*volt

    # Incorrect argument units
    group = NeuronGroup(1, '''
                       dv/dt = foo(x, y)/ms : volt
                       x : 1
                       y : 1''')
    net = Network(group)
    assert_raises(DimensionMismatchError,
                  lambda: net.run(0*ms, namespace={ 'foo': foo}))

    # Incorrect output unit
    group = NeuronGroup(1, '''
                       dv/dt = foo(x, y)/ms : 1
                       x : volt
                       y : volt''')
    net = Network(group)
    assert_raises(DimensionMismatchError,
                  lambda: net.run(0*ms, namespace={'foo': foo}))

    G = NeuronGroup(1, '''
                       func = foo(x, y) : volt
                       x : volt
                       y : volt''')
    G.x = 1*volt
    G.y = 2*volt
    mon = StateMonitor(G, 'func', record=True)
    net = Network(G, mon)
    net.run(default_dt)

    assert mon[0].func == [6] * volt

    # discard units
    foo.implementations.add_numpy_implementation(orig_foo,
                                                 discard_units=True)
    G = NeuronGroup(1, '''
                       func = foo(x, y) : volt
                       x : volt
                       y : volt''')
    G.x = 1*volt
    G.y = 2*volt
    mon = StateMonitor(G, 'func', record=True)
    net = Network(G, mon)
    net.run(default_dt)

    assert mon[0].func == [6] * volt

def test_manual_user_defined_function_weave():
    if prefs.codegen.target != 'weave':
        raise SkipTest('weave-only test')

    # User defined function without any decorators
    def foo(x, y):
        return x + y + 3*volt

    foo = Function(foo, arg_units=[volt, volt], return_unit=volt)

    code = {'support_code': '''
    inline double foo(const double x, const double y)
    {
        return x + y + 3;
    }
    '''}

    foo.implementations.add_implementation('cpp', code)

    G = NeuronGroup(1, '''
                       func = foo(x, y) : volt
                       x : volt
                       y : volt''')
    G.x = 1*volt
    G.y = 2*volt
    mon = StateMonitor(G, 'func', record=True)
    net = Network(G, mon)
    net.run(defaultclock.dt)
    assert mon[0].func == [6] * volt

@attr('cpp_standalone', 'standalone-only')
@with_setup(teardown=reinit_devices)
def test_manual_user_defined_function_cpp_standalone_compiler_args():
    set_device('cpp_standalone', directory=None)

    @implementation('cpp', '''
    static inline double foo(const double x, const double y)
    {
        return x + y + _THREE;
    }''',  # just check whether we can specify the supported compiler args,
           # only the define macro is actually used
        headers=[], sources=[], libraries=[], include_dirs=[],
        library_dirs=[], runtime_library_dirs=[],
        define_macros=[('_THREE', '3')])
    @check_units(x=volt, y=volt, result=volt)
    def foo(x, y):
        return x + y + 3*volt

    G = NeuronGroup(1, '''
                       func = foo(x, y) : volt
                       x : volt
                       y : volt''')
    G.x = 1*volt
    G.y = 2*volt
    mon = StateMonitor(G, 'func', record=True)
    net = Network(G, mon)
    net.run(defaultclock.dt)
    assert mon[0].func == [6] * volt

@attr('cpp_standalone', 'standalone-only')
@with_setup(teardown=reinit_devices)
def test_manual_user_defined_function_cpp_standalone_wrong_compiler_args1():
    set_device('cpp_standalone', directory=None)

    @implementation('cpp', '''
    static inline double foo(const double x, const double y)
    {
        return x + y + _THREE;
    }''',  some_arg=[])  # non-existing argument
    @check_units(x=volt, y=volt, result=volt)
    def foo(x, y):
        return x + y + 3*volt

    G = NeuronGroup(1, '''
                       func = foo(x, y) : volt
                       x : volt
                       y : volt''')
    mon = StateMonitor(G, 'func', record=True)
    net = Network(G, mon)
    assert_raises(ValueError, lambda: net.run(defaultclock.dt,
                                                 namespace={'foo': foo}))

@attr('cpp_standalone', 'standalone-only')
@with_setup(teardown=reinit_devices)
def test_manual_user_defined_function_cpp_standalone_wrong_compiler_args2():
    set_device('cpp_standalone', directory=None)

    @implementation('cpp', '''
    static inline double foo(const double x, const double y)
    {
        return x + y + _THREE;
    }''',  headers='<stdio.h>')  # existing argument, wrong value type
    @check_units(x=volt, y=volt, result=volt)
    def foo(x, y):
        return x + y + 3*volt

    G = NeuronGroup(1, '''
                       func = foo(x, y) : volt
                       x : volt
                       y : volt''')
    mon = StateMonitor(G, 'func', record=True)
    net = Network(G, mon)
    assert_raises(TypeError, lambda: net.run(defaultclock.dt,
                                                 namespace={'foo': foo}))

def test_manual_user_defined_function_weave_compiler_args():
    if prefs.codegen.target != 'weave':
        raise SkipTest('weave-only test')

    @implementation('cpp', '''
    static inline double foo(const double x, const double y)
    {
        return x + y + _THREE;
    }''',  # just check whether we can specify the supported compiler args,
           # only the define macro is actually used
        headers=[], sources=[], libraries=[], include_dirs=[],
        library_dirs=[], runtime_library_dirs=[],
                    define_macros=[('_THREE', '3')])
    @check_units(x=volt, y=volt, result=volt)
    def foo(x, y):
        return x + y + 3*volt

    G = NeuronGroup(1, '''
                       func = foo(x, y) : volt
                       x : volt
                       y : volt''')
    G.x = 1*volt
    G.y = 2*volt
    mon = StateMonitor(G, 'func', record=True)
    net = Network(G, mon)
    net.run(defaultclock.dt)
    assert mon[0].func == [6] * volt

def test_manual_user_defined_function_weave_wrong_compiler_args1():
    if prefs.codegen.target != 'weave':
        raise SkipTest('weave-only test')

    @implementation('cpp', '''
    static inline double foo(const double x, const double y)
    {
        return x + y + _THREE;
    }''',  some_arg=[])  # non-existing argument
    @check_units(x=volt, y=volt, result=volt)
    def foo(x, y):
        return x + y + 3*volt

    G = NeuronGroup(1, '''
                       func = foo(x, y) : volt
                       x : volt
                       y : volt''')
    mon = StateMonitor(G, 'func', record=True)
    net = Network(G, mon)
    assert_raises(ValueError, lambda: net.run(defaultclock.dt,
                                                 namespace={'foo': foo}))

def test_manual_user_defined_function_weave_wrong_compiler_args2():
    if prefs.codegen.target != 'weave':
        raise SkipTest('weave-only test')

    @implementation('cpp', '''
    static inline double foo(const double x, const double y)
    {
        return x + y + _THREE;
    }''',  headers='<stdio.h>')  # existing argument, wrong value type
    @check_units(x=volt, y=volt, result=volt)
    def foo(x, y):
        return x + y + 3*volt

    G = NeuronGroup(1, '''
                       func = foo(x, y) : volt
                       x : volt
                       y : volt''')
    mon = StateMonitor(G, 'func', record=True)
    net = Network(G, mon)
    assert_raises(TypeError, lambda: net.run(defaultclock.dt,
                                                 namespace={'foo': foo}))

def test_manual_user_defined_function_cython_compiler_args():
    if prefs.codegen.target != 'cython':
        raise SkipTest('Cython-only test')

    @implementation('cython', '''
    cdef double foo(double x, const double y):
        return x + y + 3
    ''',  # just check whether we can specify the supported compiler args,
        libraries=[], include_dirs=[], library_dirs=[], runtime_library_dirs=[])
    @check_units(x=volt, y=volt, result=volt)
    def foo(x, y):
        return x + y + 3*volt

    G = NeuronGroup(1, '''
                       func = foo(x, y) : volt
                       x : volt
                       y : volt''')
    G.x = 1*volt
    G.y = 2*volt
    mon = StateMonitor(G, 'func', record=True)
    net = Network(G, mon)
    net.run(defaultclock.dt)
    assert mon[0].func == [6] * volt

def test_manual_user_defined_function_cython_wrong_compiler_args1():
    if prefs.codegen.target != 'cython':
        raise SkipTest('Cython-only test')

    @implementation('cython', '''
    cdef double foo(double x, const double y):
        return x + y + 3
    ''',  some_arg=[])  # non-existing argument
    @check_units(x=volt, y=volt, result=volt)
    def foo(x, y):
        return x + y + 3*volt

    G = NeuronGroup(1, '''
                       func = foo(x, y) : volt
                       x : volt
                       y : volt''')
    mon = StateMonitor(G, 'func', record=True)
    net = Network(G, mon)
    assert_raises(ValueError, lambda: net.run(defaultclock.dt,
                                              namespace={'foo': foo}))

def test_manual_user_defined_function_cython_wrong_compiler_args2():
    if prefs.codegen.target != 'cython':
        raise SkipTest('Cython-only test')

    @implementation('cython', '''
    cdef double foo(double x, const double y):
        return x + y + 3
    ''',   libraries='cstdio')  # existing argument, wrong value type
    @check_units(x=volt, y=volt, result=volt)
    def foo(x, y):
        return x + y + 3*volt

    G = NeuronGroup(1, '''
                       func = foo(x, y) : volt
                       x : volt
                       y : volt''')
    mon = StateMonitor(G, 'func', record=True)
    net = Network(G, mon)
    assert_raises(TypeError, lambda: net.run(defaultclock.dt,
                                             namespace={'foo': foo}))

def test_external_function_cython():
    if prefs.codegen.target != 'cython':
        raise SkipTest('Cython-only test')

    this_dir = os.path.abspath(os.path.dirname(__file__))
    @implementation('cython', 'from func_def_cython cimport foo',
                    sources=[os.path.join(this_dir, 'func_def_cython.pyx')])
    @check_units(x=volt, y=volt, result=volt)
    def foo(x, y):
        return x + y + 3*volt

    G = NeuronGroup(1, '''
                       func = foo(x, y) : volt
                       x : volt
                       y : volt''')
    G.x = 1*volt
    G.y = 2*volt
    mon = StateMonitor(G, 'func', record=True)
    net = Network(G, mon)
    net.run(defaultclock.dt)
    assert mon[0].func == [6] * volt

def test_external_function_weave():
    if prefs.codegen.target != 'weave':
        raise SkipTest('weave-only test')

    this_dir = os.path.abspath(os.path.dirname(__file__))
    @implementation('cpp', '//all code in func_def_cpp.cpp',
                    headers=['"func_def_cpp.h"'],
                    include_dirs=[this_dir],
                    sources=[os.path.join(this_dir, 'func_def_cpp.cpp')])
    @check_units(x=volt, y=volt, result=volt)
    def foo(x, y):
        return x + y + 3*volt

    G = NeuronGroup(1, '''
                       func = foo(x, y) : volt
                       x : volt
                       y : volt''')
    G.x = 1*volt
    G.y = 2*volt
    mon = StateMonitor(G, 'func', record=True)
    net = Network(G, mon)
    net.run(defaultclock.dt)
    assert mon[0].func == [6] * volt

@attr('cpp_standalone', 'standalone-only')
@with_setup(teardown=reinit_devices)
def test_external_function_cpp_standalone():
    set_device('cpp_standalone', directory=None)
    this_dir = os.path.abspath(os.path.dirname(__file__))
    @implementation('cpp', '//all code in func_def_cpp.cpp',
                    headers=['"func_def_cpp.h"'],
                    include_dirs=[this_dir],
                    sources=[os.path.join(this_dir, 'func_def_cpp.cpp')])
    @check_units(x=volt, y=volt, result=volt)
    def foo(x, y):
        return x + y + 3*volt

    G = NeuronGroup(1, '''
                       func = foo(x, y) : volt
                       x : volt
                       y : volt''')
    G.x = 1*volt
    G.y = 2*volt
    mon = StateMonitor(G, 'func', record=True)
    net = Network(G, mon)
    net.run(defaultclock.dt)
    assert mon[0].func == [6] * volt

@attr('codegen-independent')
def test_user_defined_function_discarding_units():
    # A function with units that should discard units also inside the function
    @implementation('numpy', discard_units=True)
    @check_units(v=volt, result=volt)
    def foo(v):
        return v + 3*volt  # this normally raises an error for unitless v

    assert foo(5*volt) == 8*volt

    # Test the function that is used during a run
    assert foo.implementations[NumpyCodeObject].get_code(None)(5) == 8

@attr('codegen-independent')
def test_user_defined_function_discarding_units_2():
    # Add a numpy implementation explicitly (as in TimedArray)
    unit = volt
    @check_units(v=volt, result=unit)
    def foo(v):
        return v + 3*unit  # this normally raises an error for unitless v

    foo = Function(pyfunc=foo)
    def unitless_foo(v):
        return v + 3

    foo.implementations.add_implementation('numpy', code=unitless_foo)

    assert foo(5*volt) == 8*volt

    # Test the function that is used during a run
    assert foo.implementations[NumpyCodeObject].get_code(None)(5) == 8

@attr('codegen-independent')
def test_function_implementation_container():
    import brian2.codegen.targets as targets

    class ACodeGenerator(CodeGenerator):
        class_name = 'A Language'

    class BCodeGenerator(CodeGenerator):
        class_name = 'B Language'

    class ACodeObject(CodeObject):
        generator_class = ACodeGenerator
        class_name = 'A'

    class A2CodeObject(CodeObject):
        generator_class = ACodeGenerator
        class_name = 'A2'

    class BCodeObject(CodeObject):
        generator_class = BCodeGenerator
        class_name = 'B'

    # Register the code generation targets
    _previous_codegen_targets = set(targets.codegen_targets)
    targets.codegen_targets = {ACodeObject, BCodeObject}

    @check_units(x=volt, result=volt)
    def foo(x):
        return x
    f = Function(foo)

    container = f.implementations

    # inserting into the container with a CodeGenerator class
    container.add_implementation(BCodeGenerator, code='implementation B language')
    assert container[BCodeGenerator].get_code(None) == 'implementation B language'

    # inserting into the container with a CodeObject class
    container.add_implementation(ACodeObject, code='implementation A CodeObject')
    assert container[ACodeObject].get_code(None) == 'implementation A CodeObject'

    # inserting into the container with a name of a CodeGenerator
    container.add_implementation('A Language', 'implementation A Language')
    assert container['A Language'].get_code(None) == 'implementation A Language'
    assert container[ACodeGenerator].get_code(None) == 'implementation A Language'
    assert container[A2CodeObject].get_code(None) == 'implementation A Language'

    # inserting into the container with a name of a CodeObject
    container.add_implementation('B', 'implementation B CodeObject')
    assert container['B'].get_code(None) == 'implementation B CodeObject'
    assert container[BCodeObject].get_code(None) == 'implementation B CodeObject'

    assert_raises(KeyError, lambda: container['unknown'])

    # some basic dictionary properties
    assert len(container) == 4
    assert set((key for key in container)) == {'A Language', 'B', ACodeObject,
                                               BCodeGenerator}

    # Restore the previous codegeneration targets
    targets.codegen_targets = _previous_codegen_targets

def test_function_dependencies_weave():
    if prefs.codegen.target != 'weave':
        raise SkipTest('weave-only test')

    @implementation('cpp', '''
    float foo(float x)
    {
        return 42*0.001;
    }''')
    @check_units(x=volt, result=volt)
    def foo(x):
        return 42*mV

    # Second function with an independent implementation for numpy and an
    # implementation for C++ that makes use of the previous function.

    @implementation('cpp', '''
    float bar(float x)
    {
        return 2*foo(x);
    }''', dependencies={'foo': foo})
    @check_units(x=volt, result=volt)
    def bar(x):
        return 84*mV

    G = NeuronGroup(5, 'v : volt')
    G.run_regularly('v = bar(v)')
    net = Network(G)
    net.run(defaultclock.dt)

    assert_allclose(G.v_[:], 84*0.001)

def test_function_dependencies_cython():
    if prefs.codegen.target != 'cython':
        raise SkipTest('cython-only test')

    @implementation('cython', '''
    cdef float foo(float x):
        return 42*0.001
    ''')
    @check_units(x=volt, result=volt)
    def foo(x):
        return 42*mV

    # Second function with an independent implementation for numpy and an
    # implementation for C++ that makes use of the previous function.

    @implementation('cython', '''
    cdef float bar(float x):
        return 2*foo(x)
    ''', dependencies={'foo': foo})
    @check_units(x=volt, result=volt)
    def bar(x):
        return 84*mV

    G = NeuronGroup(5, 'v : volt')
    G.run_regularly('v = bar(v)')
    net = Network(G)
    net.run(defaultclock.dt)

    assert_allclose(G.v_[:], 84*0.001)

def test_function_dependencies_numpy():
    if prefs.codegen.target != 'numpy':
        raise SkipTest('numpy-only test')

    @implementation('cpp', '''
    float foo(float x)
    {
        return 42*0.001;
    }''')
    @check_units(x=volt, result=volt)
    def foo(x):
        return 42*mV

    # Second function with an independent implementation for C++ and an
    # implementation for numpy that makes use of the previous function.

    # Note that we don't need to use the explicit dependencies mechanism for
    # numpy, since the Python function stores a reference to the referenced
    # function directly

    @implementation('cpp', '''
    float bar(float x)
    {
        return 84*0.001;
    }''')
    @check_units(x=volt, result=volt)
    def bar(x):
        return 2*foo(x)

    G = NeuronGroup(5, 'v : volt')
    G.run_regularly('v = bar(v)')
    net = Network(G)
    net.run(defaultclock.dt)

    assert_allclose(G.v_[:], 84*0.001)

@attr('standalone-compatible')
@with_setup(teardown=reinit_devices)
def test_repeated_function_dependencies():
    # each of the binomial functions adds randn as a depency, see #988
    test_neuron = NeuronGroup(1, 'x : 1',
                              namespace={'bino_1': BinomialFunction(10, 0.5),
                                         'bino_2': BinomialFunction(10, 0.6)})
    test_neuron.x = 'bino_1()+bino_2()'

    run(0 * ms)

@attr('standalone-compatible')
@with_setup(teardown=reinit_devices)
def test_binomial():
    binomial_f_approximated = BinomialFunction(100, 0.1, approximate=True)
    binomial_f = BinomialFunction(100, 0.1, approximate=False)

    # Just check that it does not raise an error and that it produces some
    # values
    G = NeuronGroup(1, '''x : 1
                          y : 1''')
    G.run_regularly('''x = binomial_f_approximated()
                       y = binomial_f()''')
    mon = StateMonitor(G, ['x', 'y'], record=0)
    run(1*ms)
    assert np.var(mon[0].x) > 0
    assert np.var(mon[0].y) > 0

def test_declare_types():
    if prefs.codegen.target != 'numpy':
        raise SkipTest('numpy-only test')

    @declare_types(a='integer', b='float', result='highest')
    def f(a, b):
        return a*b
    assert f._arg_types==['integer', 'float']
    assert f._return_type == 'highest'

    @declare_types(b='float')
    def f(a, b, c):
        return a*b*c
    assert f._arg_types==['any', 'float', 'any']
    assert f._return_type == 'float'

    def bad_argtype():
        @declare_types(b='floating')
        def f(a, b, c):
            return a*b*c
    assert_raises(ValueError, bad_argtype)

    def bad_argname():
        @declare_types(d='floating')
        def f(a, b, c):
            return a*b*c
    assert_raises(ValueError, bad_argname)

    @check_units(a=volt, b=1)
    @declare_types(a='float', b='integer')
    def f(a, b):
        return a*b

    @declare_types(a='float', b='integer')
    @check_units(a=volt, b=1)
    def f(a, b):
        return a*b

    def bad_units():
        @declare_types(a='integer', b='float')
        @check_units(a=volt, b=1, result=volt)
        def f(a, b):
            return a*b
        eqs = '''
        dv/dt = f(v, 1)/second : 1
        '''
        G = NeuronGroup(1, eqs)
        Network(G).run(1*ms)
    assert_raises(TypeError, bad_units)

    def bad_type():
        @implementation('numpy', discard_units=True)
        @declare_types(a='float', result='float')
        @check_units(a=1, result=1)
        def f(a):
            return a
        eqs = '''
        a : integer
        dv/dt = f(a)*v/second : 1
        '''
        G = NeuronGroup(1, eqs)
        Network(G).run(1*ms)
    assert_raises(TypeError, bad_type)

def test_multiple_stateless_function_calls():
    # Check that expressions such as rand() + rand() (which might be incorrectly
    # simplified to 2*rand()) raise an error
    G = NeuronGroup(1, 'dv/dt = (rand() - rand())/second : 1')
    net = Network(G)
    assert_raises(NotImplementedError, lambda: net.run(0*ms))
    G2 = NeuronGroup(1, 'v:1', threshold='v>1', reset='v=rand() - rand()')
    net2 = Network(G2)
    assert_raises(NotImplementedError, lambda: net2.run(0*ms))
    G3 = NeuronGroup(1, 'v:1')
    G3.run_regularly('v = rand() - rand()')
    net3 = Network(G3)
    assert_raises(NotImplementedError, lambda: net3.run(0*ms))

if __name__ == '__main__':
    from brian2 import prefs
    # prefs.codegen.target = 'numpy'
    import time
    for f in [
            test_constants_sympy,
            test_constants_values,
            test_math_functions,
            test_clip,
            test_bool_to_int,
            test_timestep_function,
            test_timestep_function_during_run,
            test_user_defined_function,
            test_user_defined_function_units,
            test_simple_user_defined_function,
            test_manual_user_defined_function,
            test_manual_user_defined_function_weave,
            test_external_function_cython,
            test_external_function_weave,
            test_user_defined_function_discarding_units,
            test_user_defined_function_discarding_units_2,
            test_function_implementation_container,
            test_function_dependencies_numpy,
            test_function_dependencies_weave,
            test_function_dependencies_cython,
            test_repeated_function_dependencies,
            test_binomial,
            test_declare_types,
            test_multiple_stateless_function_calls,
            ]:
        try:
            start = time.time()
            f()
            print('Test', f.__name__, 'took', time.time()-start)
        except SkipTest as e:
            print('Skipping test', f.__name__, e)


import copy
import logging
from abc import ABC, abstractmethod
from time import time

import numpy as np
import torch

logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s', datefmt='%m/%d/%Y %H:%M:%S',
                    level=logging.INFO)
logger = logging.getLogger(__name__)

class Reranker(ABC):
    def __init__(self, rerank_weight, modify_original_score=True, rerank_score_field_name=None, cuda=False,
                 logging_level=logging.INFO):

        self.rerank_weight = rerank_weight
        self.emb_score_field_name = rerank_score_field_name
        self.modify_original_score = modify_original_score
        # Options
        logger.setLevel(logging_level)
        self.cuda = cuda
        if self.cuda:
            assert torch.cuda.is_available(), f"Cuda availability {torch.cuda.is_available()}"
            self.device = torch.device('cuda')
        else:
            self.device = torch.device("cpu")

    @abstractmethod
    def get_extra_embed(self, result):
        pass

    def rerank(self, query, results):
        if self.rerank_weight == 0.0:
            logger.debug("No reranking as weight is 0.0")
            return results
        start_time = time()
        # Construct start and end queries
        num_queries = query.shape[0]
        results_per_query = len(results[0])
        query = np.tile(np.expand_dims(query, 1), [1, results_per_query, 1])  # batch, results , 2 * size
        query_start, query_end = np.split(query, 2, axis=2)  # batch , results, size

        # Construct start/end embedding vectors and intial score vectors of embedding array and results
        start_extra_emb, end_extra_emb, mips_scores = [], [], []
        for out in results:
            extra_emb_list = []
            scores = []
            for each in out:
                extra_emb_list.append(self.get_extra_embed(each))

                scores.append(each['score'])
            start_extra_emb_per, end_extra_emb_per = np.split(np.stack(extra_emb_list), 2,
                                                              axis=1)  # results, size
            start_extra_emb.append(start_extra_emb_per)
            end_extra_emb.append(end_extra_emb_per)
            mips_scores.append(np.stack(scores))
        start_extra_emb, end_extra_emb, mips_scores = np.stack(start_extra_emb), np.stack(end_extra_emb), np.stack(
            mips_scores)
        assert start_extra_emb.shape == end_extra_emb.shape == query_start.shape == query_end.shape
        assert start_extra_emb.shape[0] == end_extra_emb.shape[0] == mips_scores.shape[0] == num_queries

        logger.debug(f'1) {time() - start_time:.3f}s: formed extra embed vector')

        # Compute start and end scores
        with torch.no_grad():
            start_extra_emb = torch.FloatTensor(start_extra_emb).to(self.device)
            end_extra_emb = torch.FloatTensor(end_extra_emb).to(self.device)
            query_start_t = torch.FloatTensor(query_start).to(self.device)
            query_end_t = torch.FloatTensor(query_end).to(self.device)
            mips_scores_t = torch.FloatTensor(mips_scores).to(self.device)
            start_emb_scores = (query_start_t * start_extra_emb).sum(2)
            end_emb_scores = (query_end_t * end_extra_emb).sum(2)
            rerank_start_scores = (1 - self.rerank_weight) * mips_scores_t + self.rerank_weight * start_emb_scores
            rerank_end_scores = (1 - self.rerank_weight) * mips_scores_t + self.rerank_weight * end_emb_scores
            rerank_start_scores = rerank_start_scores.cpu().numpy()
            rerank_end_scores = rerank_end_scores.cpu().numpy()
            # Average the two out to get final rerank scores
            rerank_scores = np.mean(np.array([rerank_start_scores, rerank_end_scores]), axis=0)
            assert rerank_scores.shape == rerank_start_scores.shape == rerank_end_scores.shape

        logger.debug(f'2) {time() - start_time:.3f}s: Computed rerank scores')

        for i in range(num_queries):
            for j in range(len(results[i])):
                if self.modify_original_score:
                    results[i][j]['score'] = rerank_scores[i][j]
                else:
                    results[i][j][self.emb_score_field_name] = rerank_scores[i][j]

        # Sort output
        for i in range(num_queries):
            results[i] = sorted(results[i],
                                key=lambda each_out: -each_out['score'] if self.modify_original_score else -each_out[
                                    self.emb_score_field_name])

        logger.debug(f'3) {time() - start_time:.3f}s: Reranking done')
        return results


import FWCore.ParameterSet.Config as cms

GTTFileWriter = cms.EDAnalyzer('GTTFileWriter',
  tracks = cms.untracked.InputTag("TTTracksFromTrackletEmulation", "Level1TTTracks"),
  convertedTracks = cms.untracked.InputTag("L1GTTInputProducer","Level1TTTracksConverted"),
  vertices = cms.untracked.InputTag("VertexProducer", "l1verticesEmulation"),
  inputFilename = cms.untracked.string("L1GTTInputFile"),
  inputConvertedFilename = cms.untracked.string("L1GTTInputConvertedFile"),
  outputFilename = cms.untracked.string("L1GTTOutputToCorrelatorFile"),
  format = cms.untracked.string("APx")
)


import re

def headerType(header,datatype,onto):
	headertype=[]
	headerId=[i['typeId'] for i in header]

	for i in header:
		for j in datatype:
			if i['typeId']==j['id']:
				i["type"]=j["name"]
				i["typeValue"]=j["value"]
		for k in onto:
			if i['ontoId']==k['id']:
				i["ontoType"]=k["label"]
				i["ontoValue"]=k["value"]
	return header

def procTable(header,datatype,onto,table,entity,subj_prefix,pred_prefix,encode):
	header=headerType(header,datatype,onto)
	# print header
	lines_new=table
	result=[]
	for i in range(len(table)):
		lines_new[i]=['a <'+entity+'>']+lines_new[i]
		lines_new[i]=['<'+subj_prefix+str(i+1)+'>']+lines_new[i]
		for j in range(2,len(lines_new[i])-1):
			count=0
			if lines_new[i][j]!='':
				if str(header[j-2]["ontoId"])!="1":
					lines_new[i][j]='<'+str(header[j-2]["ontoValue"])+'> "'+lines_new[i][j].encode("utf-8")+'"'+str(header[j-2]["typeValue"])+' ;'
				else:
					lines_new[i][j]='<'+pred_prefix+str(header[j-2]["name"].encode(encode))+'> "'+lines_new[i][j].encode(encode)+'"'+str(header[j-2]["typeValue"])+' ;'
			else:
				count+=1

		if lines_new[i][-1]!='':
			if str(header[-1]["ontoId"])!="1":
				lines_new[i][-1]='<'+str(header[-1]["ontoValue"])+'> "'+lines_new[i][-1].encode("utf-8")+'"'+str(header[-1]["typeValue"])+''
			else:
				lines_new[i][-1]='<'+pred_prefix+str(header[-1]["name"].encode(encode))+'> "'+lines_new[i][-1].encode(encode)+'"'+str(header[-1]["typeValue"])+''

    #delete all the empty element in lines_new
	for i in range(len(lines_new)):
		lines_new[i]=filter(None,lines_new[i])

	#prcoess list to a large string
	for i in lines_new:
		result.append(i[0]+'\n')
		for j in i[1:len(i)-1]:
			result.append('\t'+j+'\n')
		result.append('\t'+i[-1]+' .'+'\n')
	return " ".join(result)

def prefix_handler(text,namespace_text):
	if re.match(r'@prefix[^:].+<.+>',namespace_text) is not None:
		namespace=namespace_text.replace('@prefix','')
		prefixlis=namespace.split("\n")
		prefixdic={}
		for prefix in prefixlis:
			if prefix.find(':')!=1:
				prefixdic[prefix.split(":",1)[0].strip()+":"]=prefix.split(":",1)[1].strip()
		# print prefixdic
		# re.sub(r'%s'% value,re.match(r'%s'% value,text).group(1),"text")
		# re.sub("Date\((.+?)\)",r"\1",'{"date":Date(12455),"out_date":Date(45677)}')
		for key in prefixdic:
			# text=text.replace(prefixdic[key][:-1],key)
			pattern="\<"+prefixdic[key][1:-1]+"(.+?)\>"
			text=re.sub(pattern,key+r"\1",text)
	text=namespace_text+'\n'+text
	return text


import twitter
import config_
import time,sys,json

def del_tweet(status_id):
    s = config_.sessions["saga_kana"]["apps"]
    print(s.post("https://api.twitter.com/1.1/statuses/destroy.json",params={"id":status_id}).status_code)

def del_retweet(status_id):
    s = config_.sessions["saga_kana"]["apps"]
    _id = json.loads(s.get("https://api.twitter.com/1.1/statuses/show.json",params={"id":status_id,"include_my_retweet":1}).text)
    if "current_user_retweet" in _id.keys():
        _id = _id["current_user_retweet"]
        del_tweet(_id["id"])
        return
    print("DEL RT")
    #ids = json.loads(s.get("https://api.twitter.com/1.1/statuses/retweets.json",params={"id":status_id,"trim_user":False,"count":100}).text)
    r = s.get("https://api.twitter.com/1.1/statuses/retweets.json",params={"id":status_id,"trim_user":False,"count":100})
    print("WAIT LIMIT RESET")
    twitter.wait_limit_reset(r)
    ids = json.loads(r.text)
    #print(ids)
    if "errors" in ids:
        print("error")
        print(ids)
        sys.exit()
    for i in ids:
        print(i["user"]["screen_name"])
        if i["user"]["id"] == 485621951: # saga_kana
            del_tweet(i["id"])
            return
    print("NOT RT")

if __name__ == '__main__':
    r = config_.sessions["saga_kana"]["apps"]
    with open(sys.argv[1], "r") as f:
        txt = f.read().strip("\n")
        lines = txt.split("\n")
        for line in lines:
            status_id = line.split("/")[-1]
            print(line)
            r = twitter.db.tweets.find_one({"tweet_id_str":status_id})
            #if r != None and "full_text" in r.keys():
            if False and r != None and "full_text" in r.keys():
                print(r["full_text"])
            else:
                try:
                    r = twitter.get_status(status_id)
                    #twitter.process_tweet(res)
                    if "errors" in r.keys():
                        print(r)
                    print(r["full_text"])
                    #if "7mb_yut" in r["full_text"]:
                    #    del_tweet(status_id)
                    #continue
                    if not "@" in r["full_text"] or "popopooch" in r["full_text"]:
                        continue
                except Exception as e:
                    print("ERROR:{0}".format(e))
                    continue
            screen_name = line.split("/")[-3]
            print(screen_name)
            input_test_word = input("delete? y/[n]")
            if input_test_word == "y":
                if screen_name == "saga_kana":
                    del_tweet(status_id)
                else:
                    del_retweet(status_id)


import sys
import logging
import argparse
from PyQt5 import QtWidgets
from PyQt5 import QtGui
import mb_global
import mb_main_window

ICON_FILE_PATH_STR = "icon.png"

if __name__ == "__main__":

    argument_parser = argparse.ArgumentParser()
    argument_parser.add_argument("--persistent", "-p", help="Persistent db storage", action="store_true")
    # -for info about "store_true" please search here: https://docs.python.org/3/howto/argparse.html
    args = argument_parser.parse_args()
    if args.persistent:
        mb_global.persistent_bool = True
    else:
        mb_global.persistent_bool = False

    logging.basicConfig(level=logging.DEBUG)  # -by default only warnings and higher are shown
    app = QtWidgets.QApplication(sys.argv)
    main_window = mb_main_window.MbMainWindow()

    # System tray
    main_window.tray_icon = QtWidgets.QSystemTrayIcon(QtGui.QIcon(ICON_FILE_PATH_STR), app)
    main_window.tray_icon.show()

    tray_menu = QtWidgets.QMenu(main_window)
    tray_restore_action = QtWidgets.QAction("Restore")
    tray_restore_action.triggered.connect(main_window.showNormal)
    tray_menu.addAction(tray_restore_action)
    tray_maximize_action = QtWidgets.QAction("Maximize")
    tray_maximize_action.triggered.connect(main_window.showMaximized)
    tray_menu.addAction(tray_maximize_action)
    tray_quit_action = QtWidgets.QAction("Quit")
    tray_quit_action.triggered.connect(main_window.exit_application)
    tray_menu.addAction(tray_quit_action)

    main_window.tray_icon.setContextMenu(tray_menu)
    if not main_window.tray_icon.supportsMessages():
        logging.warning("Your system doesn't support notifications. If you are using MacOS please install growl")

    sys.exit(app.exec_())


""" Module for parsing FMS timings """

import tarfile as tf
import io
import pandas as pd
from gfdlvitals.util import gmeantools

__all__ = ["timing"]

def timing(ascii_file, fyear, outdir, label):
    """Extracts FMS timings

    Parameters
    ----------
    ascii_file : str, path-like
        Path to ascii tar file
    fyear : str
        Year to process (YYYY)
    outdir : str
        Path for output SQLite file
    label : str
        Name of output SQLite file
    """

    def ascii_tar_to_stats_df(ascii_file):
        """Subroutine to read fms.out from ascii tar file,
        extract the clock timings, and return a pandas dataframe

        Parameters
        ----------
        ascii_file : str, path-like
            Path to ascii tar file

        Returns
        -------
        pandas.DataFrame of timings
        """

        member = None
        tar = tf.open(ascii_file)
        for member in tar.getnames():
            if "fms.out" in member:
                break

        txtfile = tar.extractfile(member)
        content = txtfile.readlines()
        x = -1
        for x, line in enumerate(content):
            if "Total runtime" in line.decode("utf-8"):
                break
        tar.close()
        content = content[x::]

        output = io.BytesIO()
        output.write(str.encode("clock,min,max,mean,std\n"))
        for line in content:
            line = line.decode("utf-8")
            if not line.startswith(" "):
                line = line.split()
                line = line[-1::-1]
                label = " ".join(line[8::][-1::-1])
                label = label.replace(" ", "_")
                label = label.replace("-", "_")
                label = label.replace("&", "_")
                for x in ["(", ")", "*", "/", ":"]:
                    label = label.replace(x, "")
                line = [label] + line[0:8][-1::-1][0:4]
                line = ",".join(line) + "\n"
                output.write(str.encode(line))
        output.seek(0)

        df = pd.read_csv(output, delimiter=",")
        # df = df.set_index('clock')

        return df

    df = ascii_tar_to_stats_df(ascii_file)

    clocks = sorted(df["clock"].to_list())
    for clock in clocks:
        for attr in ["mean", "min", "max"]:
            val = df[df["clock"] == clock][attr].values[0]

            # -- Write to sqlite
            gmeantools.write_sqlite_data(
                outdir + "/" + fyear + ".globalAve" + label + ".db",
                clock + "_" + attr,
                fyear[:4],
                val,
            )


import discord
from discord.ext import commands
import asyncio
import datetime
from math import floor
import redis

strikesdb = redis.StrictRedis(host='localhost', port=6379, db=0)
punishments = redis.StrictRedis(host='localhost', port=6379, db=1)

bot = commands.Bot(command_prefix='?', description="""Every time someone is jailed using strikesy, they gain a strike. At certain numbers of strikes, punishments automatically occur.
PLEASE send ALL complaints and suggestions to me, through DM.""", guild_subscriptions=True, intents=discord.Intents.all())

# Message sent whenever
# jail_response = "To be clear, we did not jail this user due to financial fraud related to GameStop or other stocks."
jail_response = "This user has been found to have violated the most recently updated Terms of Service to Life as outlined by the " + \
                "Communist Party of the People's Republic of China, and have been dealt with as was appropriate."

##############################################################
# #THIS CODE IS LESS THAN WHOLLY STOLEN FROM ROWBOAT. THANKS #
##############################################################
UNITS = {
    's': 1,
    'm': 60,
    'h': 60 * 60,
    'd': 60 * 60 * 24,
    'w': 60 * 60 * 24 * 7,
}

def parse_duration(timespec, source=None, negative=False):
    if timespec == '-':
        # It looks like "None" time is handled by strikesy as an
        # indefinite period.
        return None

    # Parse sequences of digits followed by a unit specifier, so you
    # could compound them like "5d10m15s" for 5 days, 10 minutes, and
    # 15 seconds. I don't know specifically how it's useful for a jail
    # duration, but it *do* be sounding neat.
    value = 0
    digits = ''
    for char in timespec:
        if char.isdigit():
            digits += char
            continue

        if char not in UNITS or not digits:
            raise ValueError(f"Invalid time specifier: '{timespec}'")

        value += int(digits) * UNITS[char]
        digits = ''

    # NOTE Why would the value be negative?
    if negative:
        value = value * -1

    # NOTE what's the +1 for?
    return datetime.timedelta(seconds=value+1)

#########################################
# NO LONGER STOLEN FROM ROWBOAT. THANKS #
#########################################

async def check_punishments(member, mesg=""):
    if strikesdb.get(member.id) == b'3':
        await dayjail(member, 1, mesg)
    elif strikesdb.get(member.id) == b'4':
        await dayjail(member, 2, mesg)
    elif strikesdb.get(member.id) == b'5':
        await permjail(member, mesg)
    elif strikesdb.get(member.id) == b'6':
        await weekban(member, mesg)
    elif strikesdb.get(member.id) == b'7':
        await permban(member, mesg)
    else:
        await reports.send(mesg + ".")

def add_punishment(ptype, person, timedelta):
    unpunish_date = floor((datetime.datetime.utcnow() + timedelta).timestamp())
    punishments.zadd(ptype, {person.id: unpunish_date})

async def get_member(person):
    try:
        return server.get_member(person)
    except Exception:
        return discord.utils.get([i.user for i in server.bans()], id=person)

async def dayjail(person, days, mesg):
    await reports.send(f"{mesg} but gained too many strikes and is jailed for {days} days.")
    try:
        await person.remove_roles(jail)
    finally:
        pass
    await person.add_roles(jail)
    await person.remove_roles(memer)
    add_punishment("unjail", person, datetime.timedelta(days=1))

async def permjail(person, mesg):
    await reports.send(f"{mesg} but gained too many strikes and is jailed until they write an essay to get out.")
    await person.add_roles(jail)
    await person.remove_roles(memer)
    await punishments.zrem("unjail", person.id)

async def unjail(person):
    await reports.send(f"{person.name}(`{person.id}`) is unjailed!")
    await person.remove_roles(jail)
    await person.add_roles(memer)

async def unsolitary(person):
    await reports.send(f"{person.name}(`{person.id}`) is released from solitary confinement!")
    await person.remove_roles(solitary)
    await person.add_roles(memer)

async def unban(person):
    await reports.send(f"{person.name}(`{person.id}`) is unbanned!")
    await server.unban(person)

async def weekban(person, mesg):
    await reports.send(f"{mesg} but gained too many strikes and is banned for a week.")
    await server.ban(person, delete_message_days=0)
    add_punishment("unban", person, datetime.timedelta(weeks=1))

async def permban(person, mesg):
    await reports.send(f"{mesg} but gained too many strikes and is permanently banned.")
    await server.ban(person, delete_message_days=0)

async def strike_decay(person):
    await reports.send(f"{person.name}(`{person.id}`) has lost a strike due to strike decay!")
    if strikesdb.get(person.id) != b'0':
        strikesdb.decr(person.id)
        add_punishment("strike_decay", person, datetime.timedelta(weeks=1))

def check_action(actionbytes):
    if actionbytes == b'unjail':
        return unjail
    elif actionbytes == b'unsolitary':
        return unsolitary
    elif actionbytes == b'strike_decay':
        return strike_decay
    elif actionbytes == b'unban':
        return unban
    else:
        raise ValueError(f"Unknown action '{actionbytes}'")

@bot.command(name="strike")
@commands.has_role(431368741197053953)
async def command_strike(ctx, member: discord.Member, *, reason=""):
    """Strikes a member."""
    if member.top_role >= ctx.author.top_role:
        return
    await ctx.send('striking...')
    await strike(member, f"{member.name}(`{member.id}`) was striked by {ctx.author.name}(`{ctx.author.id}`) for: {reason}")

async def strike(member, mesg):
    if member.id == bot.user.id:
        return
    strikesdb.incr(member.id)
    add_punishment("strike_decay", member, datetime.timedelta(weeks=1))
    await check_punishments(member, mesg)

@bot.command(name="unstrike")
@commands.has_role(431368741197053953)
async def command_unstrike(ctx, member: discord.Member, *, reason=""):
    """Unstrikes a member. You can't go beneath 0 strikes, so don't even try, OK?"""
    if member.top_role >= ctx.author.top_role:
        return
    await ctx.send('destriking...')
    await unstrike(member)
    await reports.send(f"{member.name}(`{member.id}`) has been unstriked by {ctx.author.name}(`{ctx.author.id}`) for: {reason}")

async def unstrike(member):
    current_strikes = strikesdb.get(member.id)
    if current_strikes != b'0' and current_strikes is not None:
        strikesdb.decr(member.id)

@bot.command(name="strikes")
async def strikes(ctx, member: discord.Member = None):
    """Tells you the strikes of a member, or of yourself if no member is supplied."""
    if member is None:
        member = ctx.author
    a = strikesdb.get(member.id)
    if a is None:
        await ctx.send("0")
    else:
        await ctx.send(a.decode("utf-8"))

@bot.command(name="jail")
@commands.has_role(431368741197053953)
async def jail(ctx, member: discord.Member, duration: str, *, reason=""):
    """Jails the member given for the duration given (in the familiar M0DBOT format). Optionally, add a reason which goes in #police_reports.
    One interesting thing is that consecutive jails override each other, allowing you to extend sentences."""
    if member.top_role >= ctx.author.top_role:
        return
    await member.add_roles(jail)
    await member.remove_roles(memer)
    durat = parse_duration(duration)
    if durat is not None:
        add_punishment("unjail", member, parse_duration(duration))
    await ctx.send(jail_response)
    await strike(member, f"{member.name}(`{member.id}`) was jailed for {duration} by: {ctx.author.name}(`{ctx.author.id}`) because: {reason}")

@bot.command(name="jale", aliases=["softjail", "jaiI"])
@commands.has_role(431368741197053953)
async def jale(ctx, member: discord.Member, duration: str, *, reason=""):
    """Jails the member given for the duration given without a strike. Optionally, add a reason which goes in #police_reports.
    One interesting thing is that consecutive jails override each other, allowing you to extend sentences."""
    if member.top_role >= ctx.author.top_role:
        return
    await member.add_roles(jail)
    await member.remove_roles(memer)
    durat = parse_duration(duration)
    if durat is not None:
        add_punishment("unjail", member, parse_duration(duration))
    await ctx.send(jail_response)
    await reports.send(f"{member.name}(`{member.id}`) was jailed for {duration} by: {ctx.author.name}(`{ctx.author.id}`) because: {reason}")

@bot.command(name="pardon", aliases=["unjail"])
@commands.has_role(431368741197053953)
async def pardon(ctx, member: discord.Member):
    """Unjails no matter what."""
    if member.top_role >= ctx.author.top_role:
        return
    await unjail(member)
    await ctx.send("Releasing the prisoner.")

@bot.command(name="murder", aliases=["ban"])
@commands.has_role(431368741197053953)
async def murder(ctx, member: discord.Member, *, reason="unspecified reasons"):
    """Bans no matter what, say `yes` to confirm the ban."""

    def verify_user(message):
        if message.author == ctx.message.author and message.channel == ctx.message.channel:
            return True
        else:
            return False

    if member.top_role >= ctx.author.top_role:
        return
    try:
        await ctx.send(f"You are about to ban {member.name}(`{member.id}`) for {reason}, are you sure? Say `yes` to confirm.")
        choice = await bot.wait_for('message', check=verify_user, timeout=30)

    except asyncio.TimeoutError:
        await ctx.send("No input found, not banning.")
        return
    if choice.content == "yes":
        await ctx.send("Removing their privilege to life.")
        await reports.send(f"{member.name}(`{member.id}`) did an oopsie woopsie and has been banned for {reason}. Forever.")
        await server.ban(member, reason=reason, delete_message_days=0)
    else:
        await ctx.send(f"You did not say 'yes', you said '{choice.content}' - they live to see another day.")

@bot.command(name="solitary")
@commands.has_role(431368741197053953)
async def solitary(ctx, member: discord.Member, duration: str, *, reason=""):
    """Puts the member given in solitary for the duration given (in the familiar M0DBOT format). Optionally, add a reason which goes in #police_reports.
    One interesting thing is that consecutive solitaries override each other, allowing you to extend sentences."""
    if member.top_role >= ctx.author.top_role:
        return
    try:
        await member.remove_roles(jail)
    finally:
        pass
    await member.add_roles(solitary)
    await member.remove_roles(memer)
    durat = parse_duration(duration)
    await ctx.send('confined inside a void from which no screams can escape. thanks')
    if durat is not None:
        add_punishment("unsolitary", member, parse_duration(duration))
    await strike(member, f"{member.name}(`{member.id}`) was put in solitary confinement {duration} by: {ctx.author.name}(`{ctx.author.id}`) because: {reason}")

async def unpunish_loop():
    global reports
    global server
    global jail
    global solitary
    while 1:
        if reports is None:
            server = bot.get_guild(302953345105002496)
            # server = bot.get_guild(330518039961403393)
            reports = bot.get_channel(267150859605901314)
            # reports = bot.get_channel(450122812435202048)
            jail = discord.utils.get(server.roles, id=285615006442192896)
            # jail = discord.utils.get(server.roles, id =450377477949227018)
            solitary = discord.utils.get(server.roles, id=394608676276535296)
            # authorised = discord.utils.get(server.roles, id=431368741197053953)
        for i in punishments.keys():
            action = check_action(i)
            timey = floor(datetime.datetime.utcnow().timestamp())
            unpunish_set = punishments.zrangebyscore(i, min=0, max=timey)
            punishments.zremrangebyscore(i, min=0, max=timey)
            for i in unpunish_set:
                member = await get_member(int(i))
                if member is not None:
                    await action(member)
        await asyncio.sleep(10)

@bot.event
async def on_ready():
    print('We have logged in as {0.user}'.format(bot))
    global server
    global reports
    global jail
    global memer
    global solitary
    global authorised
    global moderator
    server = bot.get_guild(231084230808043522)
    reports = bot.get_channel(267150859605901314)
    jail = discord.utils.get(server.roles, id=285615006442192896)
    memer = discord.utils.get(server.roles, id=590791241034366986)
    solitary = discord.utils.get(server.roles, id=394608676276535296)
    authorised = discord.utils.get(server.roles, id=431368741197053953)
    moderator = discord.utils.get(server.roles, id=431368741197053953)
    asyncio.ensure_future(unpunish_loop())

@bot.event
async def on_message(message):
    if message.content.startswith("!!tempmute"):
        await message.channel.send("Use me, not M0DBOT for jails!")
    # delete message on sticker use :trollface:
    if message.stickers != []:
        await message.delete()
        await message.channel.send(f"{message.author.mention}, this is a no-sticker server.", delete_after=5)
        return
    await bot.process_commands(message)

@bot.event
async def on_command_error(ctx, error):
    if isinstance(error, commands.errors.CommandNotFound):
        pass
    elif isinstance(error, commands.errors.CheckFailure):
        await ctx.send("Not authorised. thanks")
    else:
        await ctx.send("Something went wrong. Please try again, doing it a different way this time. The problem: " + str(error))
        print(error)

@bot.event
async def on_member_join(person):
    if strikesdb.get(person.id) > b'0':
        add_punishment("strike_decay", person, datetime.timedelta(weeks=1))

@bot.event
async def on_member_remove(person):
    for i in punishments.keys():
        member = await get_member(int(i))
        if member is None:
            await punishments.zrem("strike_decay", person.id)

token = open("token.txt", 'r')
token = token.read()
token = token.strip()
bot.run(token, bot=True, reconnect=True)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Oct 30 12:42:40 2021

@author: maryamali
"""

import random
import operator
import matplotlib.pyplot
import agentframework
import csv

# Install environment as a list of list
environment = []
with open('in.txt', newline='') as f:
    reader = csv.reader(f, quoting=csv.QUOTE_NONNUMERIC)
    for row in reader:
        rowlist = []
        for value in row:
            #print(value)
            rowlist.append(value)
        environment.append(rowlist)

# To check if it works, print enviroment
print(environment)

rows = len(environment)
cols = len(environment[0])
print("rows", rows)
print("cols", cols)

# To check if each of the rows have the same number columns
for rows in range(rows):
    if (len(environment[rows]) != cols):
        print("unhappy")
    # else :
    #     print("unhappy")

# #visualisation
# matplotlib.pyplot.imshow(environment)
# matplotlib.pyplot.show()

def distance_between(agents_row_a, agents_row_b):
     return (((agents_row_a.x - agents_row_b.x)**2) +
     ((agents_row_a.y - agents_row_b.y)**2))**0.5

num_of_agents = 10
num_of_iterations = 10
agents = []

# Make the agents.
for i in range(num_of_agents):
    agents.append(agentframework.Agent(environment))

# Move the agents.
for j in range(num_of_iterations):
     for i in range(num_of_agents):
          print(i, "before move", agents[i].x, agents[i].y)
          agents[i].move()
          print(i, "after move", agents[i].x, agents [i].y)
          print("store before eat", agents[i].store)
          agents[i].eat()
          print("store after eat", agents[i].store)

 # #visualisation of agents plots environment (nibbling)
matplotlib.pyplot.imshow(environment)
matplotlib.pyplot.xlim(0, cols )
matplotlib.pyplot.ylim(0, rows)
for i in range(num_of_agents):
     matplotlib.pyplot.scatter(agents[i].x, agents[i].y)
matplotlib.pyplot.show()

for agents_row_a in agents:
     for agents_row_b in agents:
        distance = distance_between(agents_row_a, agents_row_b)

#Can you override __str__(self) in the agents

#print agents
for i in range(num_of_agents):
    print(agents[i])


import pygame, sys, time

#Initialize pygame so that we can use all of the built in functions in the pygame library

pygame.init()

windowSize = (800, 600)

screen = pygame.display.set_mode(windowSize)

myriadProFont = pygame.font.SysFont('myriadProFont', 48)

helloWorldText = myriadProFont.render('hello world', 1,(100,100,255), (255,255,255))

x,y = 0,0

while 1:
	for event in pygame.event.get():
		if event.type == pygame.QUIT: sys.exit()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# @file: finder/runner.py

import os

from celery import Celery
from celery.utils.log import get_task_logger
from . import config

from index import Indexer

# initialize the runner application
app = Celery(__name__)
app.config_from_object(config)
logger = get_task_logger(__name__)

# define the indexing task
@app.task
def index_site(site):
    if site not in config.SITE_FEEDS:
        logger.warning('Site "%s" is required to be indexed but not exist.' %
                       site)
        return
    try:
        indexer = Indexer(os.path.join(config.DB_PATH, 'index_%s' % site))
        for feed in config.SITE_FEEDS[site]:
            indexer.index_feed(feed)
    except Exception:
        logger.exception('Index site "%s" failed.' % site)
    else:
        logger.info('%d posts indexed for site "%s".' % (indexer.ncount, site))


import evdev
from Classes.Movement import *
from Classes.Ramp import *
import RPi.GPIO as IO
m = Movement()
r = Ramp()
speed = int(input('Speed: ')) % 101
deadzone = 10
buttonMap = {'312':[m.forward,(100,0,1)],'310':[m.turn,('l',0,speed)],'311':[m.turn,('r',0,speed)],'316':[m.finish,()],'304':[m.stop,()],'544':[r.angleup,()],'545':[r.angledown,()]}
trigThresh = int(input('Trigger Thresholds: '))
for i in buttonMap.values():
    i.append(False)

def buttonPressed(button):
    a = buttonMap[button]
    print(a)
    a[2] = not a[2]
    try:
        if a[2]:
            a[0](*a[1])
        elif not a[2]:
            m.stop()
            r.anglestop()
    except:
        pass

def deadZones(conInput,cutoff):
    a = conInput
    if abs(a) < cutoff: return 0
    else: return a

def mapper(coord,scaler,dir1):
    l,r=100,100
    if coord>0:
        l= 100 - (100/128)*abs(coord)
    elif coord<0:
        r= 100 - (100/128)*abs(coord)
    l *= scaler
    r *= scaler
    return int(abs(l)),int(abs(r)),dir1,dir1
x = 0
R2 = 0
#L2 = 0
devices = [evdev.InputDevice(path) for path in evdev.list_devices()]
for device in devices:
    print(device.path, device.name, device.phys)
number = input('Event No: ')
device = evdev.InputDevice('/dev/input/event'+number)
print(device)
pressed = False
for event in device.read_loop():
    if event.type == evdev.ecodes.EV_KEY:
        print(evdev.categorize(event))
        button = str(evdev.categorize(event)).split()[4]
        if button not in ['313']:
            buttonPressed(button)

    elif event.type == evdev.ecodes.EV_ABS:
        if event.code == evdev.ecodes.ABS_X:
            x = deadZones(event.value-127,deadzone)
        if event.code == 5:
            R2 = event.value
            if R2 > 0:
                    out = mapper(x,R2/255,0)
                    m.set(*out)
        """
        if event.code == 0:
            L2  = event.value
            if L2 > trigThresh:
                    out = mapper(x, L2 / 255, 1)
                    m.set(*out)
        """