# from matplotlib import pyplot as plt
import math
import cv2
import numpy as np
import image_preprocessing

class hog_descriptor:
    """class of calculating hog and drawing gradient functions"""
    def __init__(self, img, cell_size, block_size, overlap_num, bin_num):
        self.img = img
        self.cell_size = cell_size
        self.block_size = block_size
        self.overlap_num = overlap_num # = block stride
        self.bin_num = bin_num
        # If overlap==0, then block_size would equal to cell_size.
    def hog(self):
        """
        Calulate gradients and magnitude of each pixels. Classify them by 8*8 cell.
        Parameters:
            self
        Returns:
            histogram with 9 bins of each cells.
        Raises:
        """
        img = self.img
        cell_size = self.cell_size
        gx = cv2.Sobel(img, cv2.CV_32F, 1, 0)
        gy = cv2.Sobel(img, cv2.CV_32F, 0, 1)
        mag, ang = cv2.cartToPolar(gx, gy, angleInDegrees=True)
        ang = ang % 180 # the range of angle = (0,179)
        histogram = []
        for i in range(0, int(img.shape[0]/cell_size)): #0~15
            for j in range(0, int(img.shape[1]/cell_size)): #0~8
                cell_angle = []
                cell_magnitude = []
                cell_angle.append(ang[i*cell_size : i*cell_size+cell_size, j*cell_size : j*cell_size+cell_size])
                cell_magnitude.append(mag[i*cell_size : i*cell_size+cell_size, j*cell_size : j*cell_size+cell_size])
                hist = self.cal_histogram(cell_angle[0], cell_magnitude[0])
                histogram.append(hist)
        histogram = np.array(histogram)
        return histogram

    def cal_histogram(self, cell_angle, cell_magnitude):
        """
        Calculate histogram.
        Parameters:
            param1 - cell_angle vector of the cell from hog function
            param2 - cell_magnitude vector of the cell from hog function
        Returns:
            return histogram of that cell
        Raises:
        """
        bin_num = self.bin_num
        orientation_centers = [0] * bin_num
        bin_size = int(180/bin_num)
        for k in range(cell_magnitude.shape[0]):
            for l in range(cell_magnitude.shape[1]):
                gradient_strength = cell_magnitude[k][l]
                gradient_angle = cell_angle[k][l]
                min_angle = int(gradient_angle / bin_size) # find the bin to add into
                max_angle = (int(gradient_angle / bin_size)+1) % bin_num # find the bin to add into
                mod = (gradient_angle-min_angle*bin_size)/bin_size # calculate the ratio of gradient_strength(=magnitude)
                orientation_centers[min_angle] += (gradient_strength * (1 - mod)) # ex: 25 degree will have 75% put into bin1
                orientation_centers[max_angle] += (gradient_strength * mod)       # and 25% put into bin2
        return orientation_centers

    def draw_gradient(self, histogram, hog_img):
        """
        Draw gradient on the original img.
        Parameters:
            param1 - All cells' histogram vector.
            param2 - original img
        Returns:
            img with graient lines
        Raises:
        """
        block_size = self.block_size
        hog_img = self.img
        overlap = self.overlap_num
        bin_num = self.bin_num
        cell_x_num = int((hog_img.shape[1]-overlap) / (block_size-overlap))
        cell_y_num = int((hog_img.shape[0]-overlap) / (block_size-overlap))
        x_center = 0
        y_center = 0
        for j in range(0, cell_y_num): #0~15 cell_y_num 0~14
            for i in range(0, cell_x_num): #0~7 cell_x_num 0~6
                x_center = int(block_size/2+i*(block_size-overlap))
                y_center = int(block_size/2+j*(block_size-overlap))
                for line in range(0, bin_num):
                    line_length = histogram[i+j*cell_x_num][line]
                    # if line_length<0.8: # Use it when calculating histograms by cv2
                    #     line_length = 0
                    # else:line_length = line_length*2
                    line_length = line_length*0.3 # Use it when calculating histograms by hog_descriptor.hog()
                                                       # bolt:*0.3, cat1:*0.0015
                    angle_rad = (line*20*np.pi) / 180
                    x_end = int(x_center - (line_length)*math.sin(angle_rad)) # Lines at left hand side
                    y_end = int(y_center - (line_length)*math.cos(angle_rad)) # Lines at left hand side
                    x_end_opposite = int(x_center + (line_length)*math.sin(angle_rad)) # Lines at right hand side
                    y_end_opposite = int(y_center + (line_length)*math.cos(angle_rad)) # Lines at right hand side
                    cv2.line(hog_img, (x_end, y_end), (x_end_opposite, y_end_opposite), (0, 0, 255), 1)
                    # 0 degree points to the top, and add degrees counterclockwise.
        cv2.imshow("hog_img", hog_img)
        cv2.waitKey(0)
        cv2.destroyAllWindows()
        return hog_img

if __name__ == '__main__':
    img_bolt = cv2.imread('./img/bolt2.png', 0)
    img_bolt = np.float32(img_bolt) / 255.0 # a way to normalize to 0~1
    img_bolt = image_preprocessing.gamma_transform(0.5, img_bolt)

    img_bolt = img_bolt[100:300, 200:300]
    img_bolt = cv2.resize(img_bolt, (64, 128))
    cv2.imshow("img_bolt", img_bolt)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

    bolt_hog = hog_descriptor(img_bolt, 8, 8, 0, 9)
    bolt_hist = bolt_hog.hog()
    new_img = bolt_hog.draw_gradient(bolt_hist, img_bolt)

    new_img = cv2.normalize(new_img, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F)
    # Since imwrite will output all black img with pixels in range 0~1,
    # we need to normalize it back to range 0~255
    cv2.imwrite('./img/gradient_output/bolt_calculate_myself.jpg', new_img)


import pathlib
import subprocess

recipe_default = "recipe"
experiment_config_default = pathlib.Path("config/experiment.yaml")
options_config_default = pathlib.Path("config/options.yaml")

def weld_pipeline(
    batch_id,
    recipe_folder=recipe_default,
    experiment_config_file=experiment_config_default,
    options_config_file=options_config_default,
    force=False,
):
    """
    Send a series of subprocess jobs to execute the full weld
    """

    payload = [
        "--batch_id",
        str(batch_id),
        "--experiment_config_file",
        experiment_config_file,
        "--options_config_file",
        options_config_default,
    ]

    if force:
        payload += ["--force"]

    # Module 0 - Preprocess Sites
    preprocess_dir = str(pathlib.Path(f"{recipe_folder}/0.preprocess-sites"))

    preprocess_scripts = [
        "0.prefilter-features.py",
        "1.process-spots.py",
        "2.process-cells.py",
        "3.visualize-cell-summary.py",
        "4.image-and-segmentation-qc.py",
    ]

    split_step = ["--split_step", "qc"]
    for script in preprocess_scripts:
        full_script = str(pathlib.Path(f"{preprocess_dir}/{script}"))
        p = subprocess.Popen(
            ["python", full_script] + payload + split_step, shell=False
        )
        p.communicate()

    # Module 1 - Generate Profiles
    profile_dir = str(pathlib.Path(f"{recipe_folder}/1.generate-profiles"))

    profile_scripts = [
        "0.merge-single-cells.py",
        "1.aggregate.py",
        "2.normalize.py",
        "3.feature-select.py",
    ]

    split_step = ["--split_step", "profile"]
    for script in profile_scripts:
        full_script = str(pathlib.Path(f"{profile_dir}/{script}"))
        p = subprocess.Popen(
            ["python", full_script] + payload + split_step, shell=False
        )
        p.communicate()


# -*- coding: utf-8 -*-
from setuptools import setup, find_packages

import sys
import os

__version__ = "0.0.1"

# The main interface is through Pybind11Extension.
# * You can add cxx_std=11/14/17, and then build_ext can be removed.
# * You can set include_pybind11=false to add the include directory yourself,
#   say from a submodule.
#
# Note:
#   Sort input source files if you glob sources to ensure bit-for-bit
#   reproducible builds (https://github.com/pybind/python_example/pull/53)

def resolve_requirements(file):
    requirements = []
    with open(file) as f:
        req = f.read().splitlines()
        for r in req:
            if r.startswith("-r"):
                requirements += resolve_requirements(
                    os.path.join(os.path.dirname(file), r.split(" ")[1]))
            else:
                requirements.append(r)
    return requirements

requirements = resolve_requirements(
    os.path.join(os.path.dirname(__file__), "requirements.txt"))

setup(
    name='uroc',
    version='0.1',
    author='Eva-Maria Walz',
    author_email='evamaria.walz@gmx.de',
    description='Generalization of ROC curves and AUC to real-valued prediction problems',
    long_description=open('README.md').read(),
    license='MIT',
    install_requires=requirements,
    url='https://github.com/evwalz/isodisreg',
    packages=find_packages('src'),
    package_dir={'':'src'},
)


#!/usr/bin/env python3

import kubernetes
from kubernetes import client, config, watch
from kubernetes.client.rest import ApiException
import datetime
import yaml
import os

AVAILABLE_FIELDS = ["level", "namespace", "component", "ob", "name", "reason", "message"]
RULES_FILE = "eventRules.yaml"

class Event(object):
  _fields = []
  def __init__(self, *args):
    for name, val in zip(self._fields, args):
      setattr(self, name, val)

class EventObject(Event):
  _fields = AVAILABLE_FIELDS

  def return_formated_event(self):
    f_event = "[%s] [%s] [%s] [%s] [%s] [%s] [%s] %s" % (timestamp(), self.level, self.namespace,
                       self.component, self.ob, self.name, self.reason, self.message)
    return f_event

def timestamp():
  return datetime.datetime.now().strftime("%d-%m-%Y %H:%M:%S UTC")

def api():
  '''
  Initialize connection to Kubernetes
  '''
  try:
    config.load_incluster_config()
  except:
    config.load_kube_config()

  v1 = client.CoreV1Api()
  return v1

def parseyaml(event, attr):
  conditions = []
  events = set()
  with open("eventRules.yaml", "r") as f:
      rules = yaml.safe_load(f)
  for key, value in rules.items():
    if "include" not in rules.keys():
      events.add(event)
    if key == "include":
        for k, v in rules["include"].items():
                if k in AVAILABLE_FIELDS:
                    if isinstance(v, str):
                        if attr[k] != v:
                          conditions.append("False")
                    if isinstance(v, list):
                        for i in v:
                            if attr[k] != i:
                              conditions.append("False")
        if "False" not in conditions:
                events.add(event)

    if key == "exclude":
        for k, v in rules["exclude"].items():
            if k in AVAILABLE_FIELDS:
                if isinstance(v, str):
                    if attr[k] == v:
                      if event in events:
                        events.remove(event)
                if isinstance(v, list):
                    for i in v:
                        if attr[k] == i:
                          if event in events:
                            events.remove(event)
  for e in events:
    print(e)

def main():

  watcher = kubernetes.watch.Watch()
  stream = watcher.stream(api().list_event_for_all_namespaces)
  for event in stream:
    try:
      namespace = event['object'].involved_object.namespace
      ob = event['object'].involved_object.name
      message = event['object'].message
      name = event['object'].metadata.name
      reason = event['object'].reason
      level = event['object'].type
      component = event['object'].source.component
      event_obj = EventObject(level, namespace, component, ob, name, reason, message)
      event_obj_dict = event_obj.__dict__
      if os.path.exists("eventRules.yaml") and os.path.getsize("eventRules.yaml") > 0:
        parseyaml(event_obj.return_formated_event(), event_obj_dict)
      else:
        print(event_obj.return_formated_event())

    except ApiException as e:
      print ("Exception when calling CoreV1Api->create_namespaced_binding: %s\n" % e)

if __name__ == '__main__':
  main()


import requests
import json
from bs4 import BeautifulSoup
import json
import os
import datetime
import time

data = requests.get('https://www.worldometers.info/coronavirus/country/india/')
time.sleep(2)
soup = BeautifulSoup(data.text,'html.parser')

totalcase = soup.find('div',{'class':'maincounter-number'}).text.strip()
time.sleep(2)
deaths = soup.find_all('div',{'class':'maincounter-number'})[1].text.strip()
time.sleep(2)
recovered = soup.find_all('div',{'class':'maincounter-number'})[2].text.strip()

information = 'Country: India \n\nTotal Cases: {}\nDeaths: {}\nRecovered: {}'.format(totalcase,deaths,recovered)
# print(information)

class telegram_chatbot():

    def __init__(self):
        # self.token = self.read_token_from_config_file(config)
        self.token = '1196686551:AAHg8FCKNLZKePl7sIk0E2hMdBW1sLp3rcU'
        self.base = "https://api.telegram.org/bot{}/".format(self.token)

    def get_updates(self, offset=None):
        url = self.base + "getUpdates?timeout=100"
        if offset:
            url = url + "&offset={}".format(offset + 1)
        r = requests.get(url)
        return json.loads(r.content)

    def send_message(self, msg, chat_id):
        url = self.base + "sendMessage?chat_id={}&text={}".format(chat_id, msg)
        if msg is not None:
            requests.get(url)

    def read_token_from_config_file(self, config):
        parser = cfg.ConfigParser()
        parser.read(config)
        return parser.get('creds', 'token')

bot = telegram_chatbot()

def make_reply(msg):
    # reply = None
    if msg =="/new":
        reply = information
    else:
        reply = "Check your input"
    return reply

update_id = None
while True:
    updates = bot.get_updates(offset=update_id)
    updates = updates["result"]
    if updates:
        for item in updates:
            update_id = item["update_id"]
            try:
                message = str(item["message"]["text"])
            except:
                message = None
            from_ = item["message"]["from"]["id"]
            reply = make_reply(message)
            bot.send_message(reply, from_)


import os
import sys
import pathlib
import argparse
import configparser
import numpy as np
from collections import OrderedDict
from meta_cli import log

CONFIG_FILE_NAME = os.path.join(str(pathlib.Path.home()), 'meta.conf')

SECTIONS = OrderedDict()

SECTIONS['general'] = {
    'config': {
        'default': CONFIG_FILE_NAME,
        'type': str,
        'help': "File name of configuration",
        'metavar': 'FILE'},
    'verbose': {
        'default': True,
        'help': 'Verbose output',
        'action': 'store_true'},
    'key': {
        'type': str,
        'default': '',
        'help': "when set only tags containing key are shown"},
    'value': {
        'type': float,
        'default': None,
        'help': "value used by the set function to replace the original key entry"},
        }

SECTIONS['meta'] = {
    'doc-dir': {
        'type': str,
        'default': '.',
        'help': "sphinx/readthedocs documentation directory where the meta data table extracted from the hdf5 file should be saved, e.g. docs/source/..."},
    'doc-file-prefix': {
        'type': str,
        'default': 'log',
        'help': "the sphinx/readthedocs compatible table file name prefix"},
    'file-name': {
        'type': str,
        'default': '.',
        'help': "hdf5 file or directory containing multiple hdf5 files, e.g. /data/sample.h5 or /data/"},
        }

meta_PARAMS = ('meta', )

NICE_NAMES = ('General', 'Meta HDF5')

def get_config_name():
    """Get the command line --config option."""
    name = CONFIG_FILE_NAME
    for i, arg in enumerate(sys.argv):
        if arg.startswith('--config'):
            if arg == '--config':
                return sys.argv[i + 1]
            else:
                name = sys.argv[i].split('--config')[1]
                if name[0] == '=':
                    name = name[1:]
                return name

    return name

def parse_known_args(parser, subparser=False):
    """
    Parse arguments from file and then override by the ones specified on the
    command line. Use *parser* for parsing and is *subparser* is True take into
    account that there is a value on the command line specifying the subparser.
    """
    if len(sys.argv) > 1:
        subparser_value = [sys.argv[1]] if subparser else []
        config_values = config_to_list(config_name=get_config_name())
        values = subparser_value + config_values + sys.argv[1:]
        #print(subparser_value, config_values, values)
    else:
        values = ""

    return parser.parse_known_args(values)[0]

def config_to_list(config_name=CONFIG_FILE_NAME):
    """
    Read arguments from config file and convert them to a list of keys and
    values as sys.argv does when they are specified on the command line.
    *config_name* is the file name of the config file.
    """
    result = []
    config = configparser.ConfigParser()

    if not config.read([config_name]):
        return []

    for section in SECTIONS:
        for name, opts in ((n, o) for n, o in SECTIONS[section].items() if config.has_option(section, n)):
            value = config.get(section, name)

            if value != '' and value != 'None':
                action = opts.get('action', None)

                if action == 'store_true' and value == 'True':
                    # Only the key is on the command line for this action
                    result.append('--{}'.format(name))

                if not action == 'store_true':
                    if opts.get('nargs', None) == '+':
                        result.append('--{}'.format(name))
                        result.extend((v.strip() for v in value.split(',')))
                    else:
                        result.append('--{}={}'.format(name, value))

    return result

class Params(object):
    def __init__(self, sections=()):
        self.sections = sections + ('general', )

    def add_parser_args(self, parser):
        for section in self.sections:
            for name in sorted(SECTIONS[section]):
                opts = SECTIONS[section][name]
                parser.add_argument('--{}'.format(name), **opts)

    def add_arguments(self, parser):
        self.add_parser_args(parser)
        return parser

    def get_defaults(self):
        parser = argparse.ArgumentParser()
        self.add_arguments(parser)

        return parser.parse_args('')

def write(config_file, args=None, sections=None):
    """
    Write *config_file* with values from *args* if they are specified,
    otherwise use the defaults. If *sections* are specified, write values from
    *args* only to those sections, use the defaults on the remaining ones.
    """
    config = configparser.ConfigParser()

    for section in SECTIONS:
        config.add_section(section)
        for name, opts in SECTIONS[section].items():
            if args and sections and section in sections and hasattr(args, name.replace('-', '_')):
                value = getattr(args, name.replace('-', '_'))
                if isinstance(value, list):
                    # print(type(value), value)
                    value = ', '.join(value)
            else:
                value = opts['default'] if opts['default'] is not None else ''

            prefix = '# ' if value == '' else ''

            if name != 'config':
                config.set(section, prefix + name, str(value))
    with open(config_file, 'w') as f:
        config.write(f)

def log_values(args):
    """Log all values set in the args namespace.

    Arguments are grouped according to their section and logged alphabetically
    using the DEBUG log level thus --verbose is required.
    """
    args = args.__dict__

    for section, name in zip(SECTIONS, NICE_NAMES):
        entries = sorted((k for k in args.keys() if k in SECTIONS[section]))
        if entries:
            log.info(name)
            for entry in entries:
                value = args[entry] if args[entry] is not None else "-"
                log.info("  {:<16} {}".format(entry, value))

def show_config(args):
    """Log all values set in the args namespace.

    Arguments are grouped according to their section and logged alphabetically
    using the DEBUG log level thus --verbose is required.
    """
    args = args.__dict__

    log.warning('meta status start')
    for section, name in zip(SECTIONS, NICE_NAMES):
        entries = sorted((k for k in args.keys() if k.replace('_', '-') in SECTIONS[section]))
        if entries:
            for entry in entries:
                value = args[entry] if args[entry] != None else "-"
                log.info("  {:<16} {}".format(entry, value))

    log.warning('meta status end')


from uuid import UUID

class JobCancelledError(Exception):
    def __init__(self, job: UUID) -> None:
        self.job = job
        super().__init__(f'The job "{job}" was canceled while it was being processed')

class JobDoesNotExist(Exception):
    def __init__(self, job: UUID) -> None:
        self.job = job
        super().__init__(f'The job "{job}" does not exist')

class QueueDoesNotExist(Exception):
    def __init__(self, queue_name: str) -> None:
        self.queue_name = queue_name
        super().__init__(f'The queue "{queue_name}" does not exist')

class ReceiptHandleExpired(Exception):
    def __init__(self, receipt_handle: int) -> None:
        self.receipt_handle = receipt_handle
        msg = (
            f'The receipt handle "{receipt_handle}" expired.'
            " This likely means this client is failing to send back heartbeats for in-flight jobs"
        )
        super().__init__(msg)


from django.conf import settings
from django.conf.urls.defaults import patterns, include, url

# Uncomment the next two lines to enable the admin:
from django.contrib import admin

admin.autodiscover( )

urlpatterns = patterns( '',
    # Serve static
    url( r'^static/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.STATIC_ROOT, 'show_indexes': True} ),

    url( r'^', include( 'limited.core.urls' ) ),

    url( r'^login/$', 'django.contrib.auth.views.login', {'template_name': 'limited/login.html'}, name='login' ),
    url( r'^logout/$', 'django.contrib.auth.views.logout_then_login', name='logout' ),

    # Uncomment the next line to enable the admin:
    url( r'^admin/doc/', include( 'django.contrib.admindocs.urls' ) ),
    url( r'^admin/', include( admin.site.urls ) ),
)


import pandas as pd
import requests
import time
import logging

# current template : midcap+ and ipo years >= 10
finviz_url_template = "https://finviz.com/screener.ashx?v=152&f=cap_smallover,fa_pe_u50,geo_usa,ind_stocksonly,ipodate_more10,sh_curvol_o50,sh_price_o2&ft=4&o=-marketcap&r={}&c=0,1,2,3,4,6,7,8,11,16,17,21,39,52,53,54,57,58,62,63,65,67,69,70"

finviz_numeric_cols = ['Market Cap',
                       'P/E',
                       'Fwd P/E',
                       'P/B',
                       'EPS',
                       'EPS this Y',
                       'Sales past 5Y', 'Gross M', 'SMA20', 'SMA50', 'SMA200',
                       '52W High', '52W Low', 'Recom',
                       'Avg Volume',
                       'Price',
                       'Volume',
                       'Target Price', ]

def finviz_features(url_template, start_i=1):
    url = url_template.format(str(start_i))
    header = {
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.75 Safari/537.36",
        "X-Requested-With": "XMLHttpRequest"
    }
    dfs = pd.read_html((requests.get(url, headers=header)).text)
    valid_df_i = 7  # among downloaded dataframes, which dataframe has relevant data
    result = dfs[valid_df_i]
    first_row = result.loc[0, :].values.tolist()
    result.columns = first_row
    result = result[1:]
    return result

def convert_numeric(download_result, numeric_cols):
    def to_numeric(x):
        if isinstance(x, str):
            if x == "-":
                result = float(-1000)
            else:
                result = float(
                    eval(x.replace("%", "").replace("K", "*1000").replace("M", "*1000000").replace("B", "*1000000000")))
        else:
            result = x
        return result

    for a in numeric_cols:
        download_result[a] = download_result[a].map(lambda x: to_numeric(x))
    return download_result

def build_universe(url_template, num_cols, page_i_max=None) -> pd.DataFrame:
    next_flag = True
    page_i = 1
    result_dfs = []
    while next_flag:
        print("page :{}".format(str(page_i)))
        time.sleep(0.5)
        df = finviz_features(url_template, page_i)
        df = df[df['Industry'].map(lambda x: x != "Shell Companies")]  # excluding shell companies SPAC
        df = df[df['Company'].map(lambda x: "fund" not in x.lower())]  # excluding fund
        df = convert_numeric(df, num_cols)
        if df.shape[0] > 0:
            result_dfs.append(df)
            if df.shape[0] < 20:
                next_flag = False
            page_i = page_i + 20
            if page_i_max is not None:
                if page_i >= page_i_max:
                    next_flag = False
        else:
            next_flag = False
    result = pd.concat(result_dfs, ignore_index=True)
    return result

def test_finviz_features():
    actual = finviz_features(finviz_url_template)
    logging.info(actual.dtypes)
    actual_numeric = convert_numeric(actual, finviz_numeric_cols)
    logging.info(actual_numeric.dtypes)
    multiple_pages = build_universe(finviz_url_template, finviz_numeric_cols, 60)
    logging.info(multiple_pages.dtypes)
    logging.info(multiple_pages.head(5).to_dict("records"))
    logging.info(multiple_pages.shape)
    logging.info(multiple_pages.Company.unique())


# -*- coding: utf-8 -*-
"""
Created on Wed Jul  8 23:55:31 2020

@author: Bhageerath
"""

import csv
import re
from nltk.corpus import stopwords
from string import punctuation
import nltk
import pandas as pd
nltk.download('stopwords')
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import train_test_split
from scipy.sparse import csr_matrix
import numpy as np
from sklearn.model_selection import cross_val_score
from wordcloud import WordCloud
import matplotlib.pyplot as plt

def buildTrainingData(twitterDataFile):
    training_data = []
    with open(twitterDataFile, "rt") as f:
        lineReader = csv.reader(f,delimiter=',', quotechar="\"")
        for row in lineReader:
            training_data.append({"company":row[0], "label":row[1], "text":row[2]})
    return training_data

def buildTestingData(testcsvDataFile):
    testing_data = []
    with open(testcsvDataFile, "rt", encoding="utf-8") as f:
        lineReader = csv.reader(f,delimiter=',', quotechar="\"")
        for row in lineReader:
            if row != []:
                testing_data.append({"text":row[0], "label":"dummy"})
    return testing_data

class PreProcessTweets:
    def __init__(self):
        self._stopwords = set(stopwords.words('english') + list(punctuation) + ['AT_USER','URL'])

    def processTweets(self, list_of_tweets):
        processedTweets=[]
        for tweet in list_of_tweets:
            processedTweets.append((self._processTweet(tweet["text"]),tweet["label"]))
            text, label = tuple(zip(*processedTweets))
        return pd.DataFrame({'text': text, 'label': label})

    def _processTweet(self, tweet):
        tweet = tweet.lower() # convert text to lower-case
        tweet = re.sub('((www\.[^\s]+)|(https?://[^\s]+))', 'URL', tweet) # remove URLs
        tweet = re.sub('@[^\s]+', 'AT_USER', tweet) # remove usernames
        tweet = re.sub(r'#([^\s]+)', r'\1', tweet) # remove the # in #hashtag
        #tweet = word_tokenize(tweet) # remove repeated characters (helloooooooo into hello)
        return tweet

def toUnigramTfDf(train_data, test_data):
    unigram_vectorizer = CountVectorizer(ngram_range=(1, 1))
    unigram_vectorizer.fit(train_data)
    X_train_unigram = unigram_vectorizer.transform(train_data)
    Y_train_unigram = unigram_vectorizer.transform(test_data)
    unigram_tf_idf_transformer = TfidfTransformer()
    unigram_tf_idf_transformer.fit(X_train_unigram)
    X_train_tf_idf = unigram_tf_idf_transformer.transform(X_train_unigram)
    Y_train_tf_idf = unigram_tf_idf_transformer.transform(Y_train_unigram)
    return (X_train_tf_idf, Y_train_tf_idf)

def train_and_show_scores(X: csr_matrix, y: np.array, title: str):
    X_train, X_valid, y_train, y_valid = train_test_split(
        X, y, train_size=0.95, stratify=y
    )
    clf = SGDClassifier()
    clf.fit(X_train, y_train)
    train_score = clf.score(X_train, y_train)
    valid_score = clf.score(X_valid, y_valid)
    print(f'{title}\nTrain score: {round(train_score, 2)} ; Validation score: {round(valid_score, 2)}\n')
    return clf

training_data_raw = buildTrainingData("C:/Users/bvulu/Documents/AIT-664/full-corpus.csv")
training_data_raw = training_data_raw[1:-1]
tweetProcessor = PreProcessTweets()
training_data = tweetProcessor.processTweets(training_data_raw)
#training_features = toUnigramTfDf(training_data['text'].values)
training_labels = training_data['label'].values
testing_data_raw = buildTestingData("C:/Users/bvulu/Documents/AIT-664/test123457.csv")
testing_data = tweetProcessor.processTweets(testing_data_raw)
training_features,testing_features = toUnigramTfDf(training_data['text'].values,testing_data['text'].values)

clf = train_and_show_scores(training_features, training_labels, 'Unigram Counts')
scores = cross_val_score(clf, training_features, training_labels, cv=5)
scores1 = clf.predict(testing_features)

testing_data_str = " ".join(str(e) for e in testing_data['text'].values)
wordcloud = WordCloud().generate(testing_data_str)
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")

k = pd.DataFrame(scores1)
print(k[0].value_counts())
print("Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2))
print(scores)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct  4 21:18:36 2023

@author: feer
"""

import numpy  as np
#import pandas as pd
from pandas import DataFrame
from IPython.display import HTML
from matplotlib import pyplot as plt
from scipy import signal
#from astropy.stats import mad_std

def periodogram(x_t, fs, N, k):
    df  = fs/N
    f   = np.linspace(0, (N-1)*df, N)
    spec = np.fft.fft( x_t, axis = 0 )
    X_f = 10 * np.log10(np.abs(k/N * spec)**2)
    return f, X_f

def periodogram_welch(x_t, fs, window):
    N = len(x_t)
    L = N//5
    D = L//2 # 50% overlap

    f,Pxx = signal.welch(x_t,
                         fs=fs,
                         window=window,
                         nperseg=L,
                         noverlap=D,
                         nfft=N,
                         average='median',
                         axis=0
                        )
    Pxx[ 0] = Pxx[ 1]
    Pxx[-1] = Pxx[-2]
    return f, Pxx

# def periodogram_smooth(X, fs, window, M, n1=0, n2=None):
#     N = len(X)
#     delta_f = fs/N
#     X = np.reshape(X, -1)
#     if n2 is None:
#         n2 = N

#     R = np.cov(X[n1:n2])
#     r = [np.fliplr(R[0,1:M-1]), R[0,0], R[0,1:M-1]]
#     M = 2*M-1
#     w = np.ones((M,1))
#     match window:
#         case 'hamming':
#             w = signal.windows.hamming(M)
#         case 'hann':
#             w = signal.windows.hann(M)
#         case 'barlett':
#             w = signal.windows.bartlett(M)
#         case 'blackman':
#             w = signal.windows.blackman(M)
#         case _:
#             w = signal.windows.boxcar(M)

#     r = np.transpose(r) * w
#     Px = np.abs(np.fft.fft(r, 1024))
#     Px[0] = Px[1]
#     f = np.arange(start = 0, stop = N, step = delta_f)

#     return f, Px

# ver spectrum psdcorrelogram
# w: ventana que va de -M a +M, con M < N-1
def blackman_tukey(x, fs, window, M = None):
    N,R,Z = x.shape
    delta_f = fs/N

    if M is None:
        M = N//5

    r_len = 2*M-1

    w = signal.windows.get_window(window, r_len)

    Px = np.zeros_like(x)

    for j in range(Z):
        for i in range(R):
            # hay que aplanar los arrays por np.correlate.
            # usaremos el modo same que simplifica el tratamiento
            # de la autocorr
            xx = x[:,i,j].ravel()[:r_len];

            r = np.correlate(xx, xx, mode='same') / r_len

            Px[:,i,j] = np.abs(np.fft.fft(r * w, n = N) ) / N

    f  = np.arange(start = 0, stop = N, step = delta_f)

    return f, Px;

def graficar_espectro(ax, f, X_f, fs, *args, **kwargs):
    bfrec = (f <= fs/2)
#    bfrec.resize(max(len(f), len(X_f)), refcheck=False)
    X_f = 20 * np.log10(np.abs(X_f)) # Lo paso a dB
    return ax.plot(f[bfrec], X_f[bfrec], *args, **kwargs);

# def graficar_espectro(x_t, fs, N, k, *args, **kwargs):
#     df  = fs/N
#     f   = np.linspace(0, (N-1)*df, N)
#     bfrec = (f <= fs/2)
#     X_f = 10 * np.log10(np.abs(k/N * np.fft.fft( x_t, axis = 0 ))**2)

#     plt.plot(f[bfrec], X_f[bfrec], *args, **kwargs)

def vertical_flaten(x):
    return x.reshape(x.shape[0], 1)

#%%
R = 200  # 200 repeticiones o realizaciones
N = 1000 # nº de muestras

# frec. de sampling normalizada
fs = N
delta_f = fs/N

# Desplazamiento en frecuencia de cada realización
fr = np.random.uniform(-1/2, 1/2, size=(1,R,1))

# Frecuencia central normalizada: pi/2 - es equivalente a fs/4
W0 = fs/4
W1 = W0 + fr

snr_dB = np.array([ 3., 10. ], dtype=np.float64)
var_noise  = 10 ** (snr_dB / (-10)) # varianza (potencia)
noise = np.random.normal(loc=0.0, scale=np.sqrt(var_noise), size=(N,R,2))

a1 = np.sqrt(2) # amplitud normalizada

end = N / fs

n = np.linspace(start=0, stop=end, num = N, endpoint = False).reshape((N,1,1))

x  = a1 * np.sin(2*np.pi * W1 * n) + noise

# Espectros
window='bartlett'
f,  X_welch_f = periodogram_welch(x, fs, window=window)
f2, X_BT_f = blackman_tukey(x, fs, window=window)
#f2, X_BT_f = periodogram_smooth(x, fs, window=window, M=N//5)

#%%
#-------------------------------------------------------------------------------

fig = plt.figure(1)
fig.clf()
axis = fig.subplots(2,1, sharex=True)

for i, ax in enumerate(axis):
    #fig.xticks(np.linspace(0, 500, 5), minor=False)
    ax.set_xticks(np.arange(W0-10, W0+10+1), minor=True)
    ax.grid(which='major', alpha=0.7)
    ax.grid(which='minor', alpha=0.3)

    *line, = graficar_espectro(ax, f, X_welch_f[:,:,i], fs, 'o--g', linewidth=1)
    line[0].set_label('welch')

    graficar_espectro(ax, f, np.median(X_welch_f[:,:,i], axis=1), fs, 'b--',
                      linewidth=1, label='welch median')

    *line, = graficar_espectro(ax, f2, X_BT_f[:,:,i], fs, 'x-r', linewidth=1)
    line[0].set_label('blackman tukey')

    ax.set_title('Periodogramas (window='+window+', SNR='+str(int(snr_dB[i]))+')')
    ax.set_ylabel('Amplitud [dB]')
    ax.legend()
    ax.set_ylim([-100,  0])

axis[0].set_xlim([W0-20, W0+20])
axis[1].set_xlabel('f [Hz]')
fig.show()

#-------------------------------------------------------------------------------

#%%
# Estimacion de frecuencia W1
W1_est_welch = f[  np.argmax(X_welch_f [f  <= fs/2], axis=0) ]
W1_est_BT    = f2[ np.argmax(X_BT_f    [f2 <= fs/2], axis=0) ]

W1_est_welch_median = np.median(W1_est_welch, axis=0)
W1_est_welch_bias   = W1_est_welch_median - np.median(W1)
W1_est_welch_var    = np.var(W1_est_welch, axis=0)

W1_est_BT_median = np.median(W1_est_BT, axis=0)
W1_est_BT_bias   = W1_est_BT_median - np.median(W1)
W1_est_BT_var    = np.var(W1_est_BT, axis=0)

nbins = 5; rang = (W0-2.5,W0+2.5); alpha = 0.5

fig = plt.figure(2)
fig.clf()
axis = fig.subplots(2,1, sharex=True)

for i, ax in enumerate(axis):
    ax.hist(W1_est_welch[:,i], bins=nbins, range=rang, alpha=alpha, label='welch', align='mid')
    ax.hist(W1_est_BT[:,i],    bins=nbins, range=rang, alpha=alpha, label='BT',    align='mid')
    ax.axvline(W0, linestyle='--', color='black', label='real')
    ax.set_title('Histograma para frecuencia, SNR='+str(int(snr_dB[i])))
    ax.set_ylabel('n° de muestras')
    ax.legend();

ax.set_xlabel('Frecuencia [Hz]')
#fig.set_figheight(6)
#fig.set_figwidth(10)
fig.show()

#%%

variability_welch_est = W1_est_welch_var / (W1_est_welch_median**2)
variability_BT_est = W1_est_BT_var / (W1_est_BT_median**2)

variability = np.array((variability_welch_est, variability_BT_est)).T

# L = N//5
# D = L//2
# K = 2*N/L-1
# variability_welch = 9/(8*K)

df = DataFrame(variability, columns=['$i_{W}$', '$i_{BT}$'],
                index=[
                          '3 dB',
                          '10 dB',
                      ])

# # Dict used to center the table headers
# d = dict(selector="th",
#     props=[('text-align', 'center')])

# # Style
# s = df.style.set_properties(**{'width':'10em', 'text-align':'center'})\
#       .set_table_styles([d])

HTML(df.to_html(classes="table",justify="center"))


#!/usr/bin/env python3
"""
Provides utility methods (i.e. methods not directly related to ``nevis``).
"""
import timeit

import nevis

class Timer(object):
    """
    Provides accurate timing.

    Example
    -------
    ::

        timer = nevis.Timer()
        print(timer.format(timer.time()))

    """

    def __init__(self, output=None):
        self._start = timeit.default_timer()
        self._methods = {}

    def format(self, time=None):
        """
        Formats a (non-integer) number of seconds, returns a string like
        "5 weeks, 3 days, 1 hour, 4 minutes, 9 seconds", or "0.0019 seconds".
        """
        if time is None:
            time = self.time()
        if time < 1e-2:
            return str(time) + ' seconds'
        elif time < 60:
            return str(round(time, 2)) + ' seconds'
        output = []
        time = int(round(time))
        units = [
            (604800, 'week'),
            (86400, 'day'),
            (3600, 'hour'),
            (60, 'minute'),
        ]
        for k, name in units:
            f = time // k
            if f > 0 or output:
                output.append(str(f) + ' ' + (name if f == 1 else name + 's'))
            time -= f * k
        output.append('1 second' if time == 1 else str(time) + ' seconds')
        return ', '.join(output)

    def reset(self):
        """
        Resets this timer's start time.
        """
        self._start = timeit.default_timer()

    def time(self):
        """
        Returns the time (in seconds) since this timer was created, or since
        meth:`reset()` was last called.
        """
        return timeit.default_timer() - self._start

#
# Version-related methods
#
def howdy(name='version'):
    """ Say hi the old fashioned way. """
    print('')
    print('                |>          ')
    print(' Starting Ben   |   Nevis   ')
    print('               / \    ' + name)
    print('            /\/---\     ' + nevis.__version__)
    print('           /---    \/\      ')
    print('        /\/   /\   /  \     ')
    print('     /\/  \  /  \_/    \    ')
    print('    /      \/           \   ')

def print_result(x, y, z):
    """
    Print information about an optimisation result.

    Arguments:

    ``x``
        The x-coordinate of the result.
    ``y``
        The y-coordinate of the result.
    ``z``
        The z-coordinate of the result.
    """

    coords = nevis.Coords(gridx=x, gridy=y)
    hill, distance = nevis.Hill.nearest(coords)
    print('Congratulations!' if distance < 100 else (
        'Good job!' if distance < 1000 else 'Interesting!'))
    print(f'You landed at an altitude of {round(z)}m.')
    print(f'  {coords.opentopomap}')
    dm = (
        f'{round(distance)}m' if distance < 1000
        else f'{round(distance / 1000, 1)}km'
    )
    print(f'You are {dm} from the nearest named hill top, "{hill.name}",')
    print(f'  ranked the {hill.ranked} heighest in GB.')
    photo = hill.photo()
    if photo:
        print('  ' + photo)

def generate_kml(path, labels=None, trajectory=None, points=None,):
    """
    Generate a KML (Keyhole Markup Language, used to display geographic data
    in an Earth browser such as Google Earth) file from given data.

    For a description of the KML format, see
    https://developers.google.com/kml/documentation/kml_tut

    Arguments:

    ``path``
        The path of the file to write to.
    ``labels``
        An optional dictionary mapping string labels to points (tuples in
        meters or Coords objects) that will be marked on the map. The points
        will be shown as green pinpoints with text labels beside them.
    ``trajectory``
        An optional array of shape ``(n_points, 2)`` indicating the trajectory
        to be plotted (points specified in meters). All the points along the
        trajectory will be shown as small orange pinpoints with labels of
        their index prefixed with "T" (e.g. "T3"). The trajectory itself will
        be shown as a red curve extended down to the ground connecting
        adjacent points.
    ``points``
        An optional array of shape ``(n_points, 2)`` indicating points on the
        map (points specified in meters). All the points will be shown as small
        blue pinpoints with labels of their index prefixed with "P" (e.g.
        "P3").

    ``lables``, ``trajectory``, and ``points`` can be used simultaneously.
    """
    from pykml.factory import KML_ElementMaker as KML
    from lxml import etree

    def make_coords(x, y):
        """
        Convert a pair of coordinates (in meters) to a KML coordinates string.
        """
        lat, lon = nevis.Coords(x, y).latlong
        return f'{lon},{lat}'

    marks = []
    if labels is not None:
        for label, p in labels.items():
            if isinstance(p, nevis.Coords):
                p = p.grid
            x, y = p

            marks.append(
                KML.Placemark(
                    KML.name(label),
                    KML.Point(
                        KML.coordinates(make_coords(x, y))
                    ),
                    KML.styleUrl('#label_icon_style'),
                )
            )

    if points is not None:
        for i, (x, y) in enumerate(points):
            marks.append(
                KML.Placemark(
                    KML.name(f'P{i}'),
                    KML.Point(
                        KML.coordinates(make_coords(x, y))
                    ),
                    KML.styleUrl('#points_icon_style'),
                )
            )

    if trajectory is not None:
        for i, (x, y) in enumerate(trajectory):
            marks.append(
                KML.Placemark(
                    KML.name(f'T{i}'),
                    KML.Point(
                        KML.coordinates(make_coords(x, y))
                    ),
                    KML.styleUrl('#trajectory_icon_style'),
                )
            )

        marks.append(
            KML.Placemark(
                KML.name('Trajectory'),
                KML.styleUrl('#line_style'),
                KML.LineString(
                    KML.coordinates(
                        ' '.join([make_coords(x, y) for x, y in trajectory])
                    ),
                    KML.extrude(1),
                    KML.tessellate(1),
                )
            )
        )

    icon_url = 'http://maps.google.com/mapfiles/kml/paddle/wht-blank.png'

    doc = KML.kml(
        KML.Document(
            KML.name('Nevis Project'),
            KML.Style(
                KML.LineStyle(
                    KML.color('ff1400FF'),
                    KML.width(3),
                ),
                id="line_style"
            ),
            KML.Style(
                KML.IconStyle(
                    KML.Icon(
                        KML.href(icon_url),
                    ),
                    KML.color('ff00FF14'),
                ),
                id="label_icon_style"
            ),
            KML.Style(
                KML.IconStyle(
                    KML.Icon(
                        KML.href(icon_url),
                    ),
                    KML.scale(0.5),
                    KML.color('ffFF7800'),
                ),
                KML.LabelStyle(
                    KML.scale(0.5),
                ),
                id="points_icon_style"
            ),
            KML.Style(
                KML.IconStyle(
                    KML.Icon(
                        KML.href(icon_url),
                    ),
                    KML.scale(0.5),
                    KML.color('ff1e90ff'),
                ),
                KML.LabelStyle(
                    KML.scale(0.5),
                ),
                id="trajectory_icon_style"
            ),
            *marks,
        )
    )

    with open(path, 'w') as fobj:
        fobj.write(
            etree.tostring(doc, pretty_print=True).decode('utf-8'))


import torch
from torch import nn

# from quantized_ops import zs_quantized_ops
# conv_clamp_val=0.1
# fc_clamp_val=0.1

class RandomNet(nn.Module):
    def __init__(self, input_channels, classes, precision):
        super(RandomNet, self).__init__()
        #        if (precision < 0):
        self.conv1 = nn.Conv2d(input_channels, 32, 3, 1)
        #        else:
        #            self.conv1 = zs_quantized_ops.nnConv2dSymQuant_op(
        #            input_channels, 32, kernel_size=3, stride=1,
        #            padding=0, bias=True, precision=precision,
        #            clamp_val=conv_clamp_val)
        #        if (precision < 0):
        self.fc1 = nn.Linear(28800, 128)
        self.fc2 = nn.Linear(128, classes)
        #        else:
        #            self.fc1 = zs_quantized_ops.nnLinearSymQuant_op(
        #            28800,128,precision, fc_clamp_val)
        #            self.fc2 = zs_quantized_ops.nnLinearSymQuant_op(
        #            128,classes,precision, conv_clamp_val)
        self.relu1 = nn.ReLU(True)
        self.relu2 = nn.ReLU(True)

    def forward(self, x):
        x = self.conv1(x)
        x = self.relu1(x)
        # x = self.conv2(x)
        # x = self.relu2(x)
        #    x = F.max_pool2d(x, 2)
        x = torch.flatten(x, 1)
        x = self.fc1(x)
        x = self.relu2(x)
        output = self.fc2(x)
        # output = F.log_softmax(x, dim=1)
        return output

def randomnet(input_channels, classes, precision):
    model = RandomNet(input_channels, classes, precision)
    return model

def test():
    net = randomnet(3, 10, -1)
    print(net)
    for name, layer in net.named_modules():
        print(name, layer)
    y = net(torch.randn(1, 3, 32, 32))
    print(y.size())


from kazoo.exceptions import NoNodeError
from sys import maxsize
from .mutex import Mutex
from .internals import LockDriver
from .utils import lazyproperty

READ_LOCK_NAME = "__READ__"
WRITE_LOCK_NAME = "__WRIT__"

class _LockDriver(LockDriver):
    def sort_key(self, string, _lock_name):
        string = super(_LockDriver, self).sort_key(string, READ_LOCK_NAME)
        string = super(_LockDriver, self).sort_key(string, WRITE_LOCK_NAME)
        return string

class _ReadLockDriver(_LockDriver):
    def __init__(self, predicate):
        super(_ReadLockDriver, self).__init__()
        self._predicate = predicate

    def is_acquirable(self, children, sequence_node_name, max_leases):
        return self._predicate(children, sequence_node_name)

class _Mutex(Mutex):
    def __init__(self, client, path, name, max_leases, driver, timeout):
        super(_Mutex, self).__init__(
            client,
            path,
            max_leases,
            name=name,
            driver=driver,
            timeout=timeout
        )

    def get_participant_nodes(self):
        nodes = super(_Mutex, self).get_participant_nodes()
        return list(filter(lambda node: self.name in node, nodes))

class ReadWriteLock(object):
    def __init__(self, client, path, timeout=None):
        self._client = client
        self._path = path
        self._timeout = timeout

    @property
    def path(self):
        return self._path

    @property
    def timeout(self):
        return self._timeout

    @timeout.setter
    def timeout(self, value):
        self._timeout = value
        self.read_lock.timeout = value
        self.write_lock.timeout = value

    @lazyproperty
    def read_lock(self):
        def predicate(children, sequence_node_name):
            return self._read_is_acquirable_predicate(
                children,
                sequence_node_name
            )

        return _Mutex(
            self._client,
            self.path,
            READ_LOCK_NAME,
            maxsize,
            _ReadLockDriver(predicate),
            self.timeout
        )

    @lazyproperty
    def write_lock(self):
        return _Mutex(
            self._client,
            self.path,
            WRITE_LOCK_NAME,
            1,
            _LockDriver(),
            self.timeout
        )

    def get_participant_nodes(self):
        nodes = self.read_lock.get_participant_nodes()
        nodes.extend(self.write_lock.get_participant_nodes())
        return nodes

    def _read_is_acquirable_predicate(self, children, sequence_node_name):
        if self.write_lock.is_owned_by_current_thread:
            return (None, True)

        index = 0
        write_index = maxsize
        our_index = -1

        for node in children:
            if WRITE_LOCK_NAME in node:
                write_index = min(index, write_index)
            elif node.startswith(sequence_node_name):
                our_index = index
                break

            index += 1

        if our_index < 0:
            raise NoNodeError

        acquirable = our_index < write_index
        path = None if acquirable else children[write_index]
        return (path, acquirable)


__version__ = '$Rev$'

import logging
import re

from datetime import date

from core.constants import TIME_MINUTE
from core.config import Config
from objects.calendar import Event

from components.calendar import InvalidObjectId
from interaction.irc.module import InteractiveModule, InteractiveModuleCommand, InteractiveModuleRequest, InteractiveModuleResponse, ModuleError, Location

#-------------------------------------------------------------------------------
# Constants
#-------------------------------------------------------------------------------
REGEX_DATE = re.compile('^(\d{1,2})\.(\d{1,2})\.(\d{4})$')

#-------------------------------------------------------------------------------
# Exceptions
#-------------------------------------------------------------------------------
class CalendarModuleError(ModuleError): pass
class EventError(CalendarModuleError): pass
class EventNotFound(EventError): pass
class NoEventsFound(EventError): pass
class AmbiguousEventsFound(EventError): pass
class DateError(CalendarModuleError): pass
class DateFormatInvalid(DateError): pass
class DateRangeInvalid(DateError): pass

#-------------------------------------------------------------------------------
# Business Logic
#-------------------------------------------------------------------------------
class Calendar(InteractiveModule):
    """
    This module provides calendaring functions.
    """

    #---------------------------------------------------------------------------
    # Module implementation
    #---------------------------------------------------------------------------
    def initialize(self):
        bot = self.client.bot;

        bot.register_config(CalendarConfig)

        self.logger = logging.getLogger('interaction.irc.calendar')
        self.config = bot.get_config('interaction.irc.calendar')
        self.component = bot.get_subsystem('calendar-component');

        bot.register_timer(
            'interaction.irc.calendar.reminder',
            'daily',
            self.config.get('reminderInterval'),
            self._display_reminder
        )

    # --------------------------------------------------------------------------
    # Lifecycle
    # --------------------------------------------------------------------------
    def start(self):
        self.client.bot.get_timer('interaction.irc.calendar.reminder').start()

    def stop(self):
        self.client.bot.get_timer('interaction.irc.calendar.reminder').stop()

    #---------------------------------------------------------------------------
    # InteractiveModule implementation
    #---------------------------------------------------------------------------
    def module_identifier(self):
        return 'Kalender'

    def init_commands(self):
        return [
            InteractiveModuleCommand(
                 keyword='kalender',
                 callback=self.display_calendar_address,
            ),
            InteractiveModuleCommand(
                 keyword='listtoday',
                 callback=self.display_events_today,
            ),
            InteractiveModuleCommand(
                 keyword='listdeleted',
                 callback=self.display_deleted_objects,
                 pattern=r'^(.*)$',
                 syntaxhint='[kalendar|event|kontakt]'
            ),
            InteractiveModuleCommand(
                 keyword='restore',
                 callback=self.restore_deleted_object,
                 pattern=r'^[\d]+$',
                 syntaxhint='<id>'
            ),

            InteractiveModuleCommand(
                 keyword='addevent',
                 callback=self.insert_event,
                 pattern=r'^(\d{1,2}\.\d{1,2}\.\d{4}|\d{1,2}\.\d{1,2}\.\d{4}-\d{1,2}\.\d{1,2}\.\d{4})\s(.+)$',
                 syntaxhint='<datumvon>[-datumbis] <titel>'
            ),
            InteractiveModuleCommand(
                 keyword='editevent',
                 callback=self.change_event,
                 pattern=r'^([\d]+)\s(.+?)\s(.+)$',
                 syntaxhint='<id> <start|ende|titel|beschreibung|ort> <wert>'
            ),
            InteractiveModuleCommand(
                 keyword='delevent',
                 callback=self.delete_event,
                 pattern=r'^([\d]+|\d{1,2}\.\d{1,2}\.\d{4})$',
                 syntaxhint='<id|datum>'
            ),
            InteractiveModuleCommand(
                 keyword='searchevent',
                 callback=self.search_event,
                 pattern=r'^(.+)$',
                 syntaxhint='<text>'
            ),
            InteractiveModuleCommand(
                 keyword='topicevent',
                 callback=self.topic_event,
                 pattern=r'^[\d]+$',
                 syntaxhint='<id>'
            ),

            InteractiveModuleCommand(
                 keyword='addcontact',
                 callback=self.insert_contact,
                 pattern=r'^(.+)\s(\d{1,2}\.\d{1,2}\.\d{4})$',
                 syntaxhint='<nickname> <geburtsdatum>'
            ),
            InteractiveModuleCommand(
                 keyword='editcontact',
                 callback=self.change_contact,
                 pattern=r'^[\d]+\s(.+)\s(.+)$',
                 syntaxhint='<id> <vorname|nachname|nickname|geburtsdatum> <wert>'
            ),
            InteractiveModuleCommand(
                 keyword='delcontact',
                 callback=self.delete_contact,
                 pattern=r'^([\d]+|(.+))$',
                 syntaxhint='<id>'
            ),
            InteractiveModuleCommand(
                 keyword='searchcontact',
                 callback=self.search_contact,
                 pattern=r'^(.+)$',
                 syntaxhint='<text>'
            )
        ]

    #---------------------------------------------------------------------------
    # Internal module commands
    #---------------------------------------------------------------------------
    def _get_date(self, date_string):
        """
        Check if the input_string contains a valid date.

        The input string is first converted using REGEX_DATE and then
        checked.

        @param date_string: The input string. See REGEX_DATE for the format.

        @return A date object.

        @raise DateFormatInvalid If no properly formatted date was given.
        @raise DateRangeInvalid  If the date values are out of range.
        """

        matchlist = REGEX_DATE.search(date_string)

        if not matchlist:
            raise DateFormatInvalid

        date_tokens = matchlist.group(1, 2, 3)

        try:
            return date(int(date_tokens[2]),int(date_tokens[1]),int(date_tokens[0]))

        except ValueError as e:
            raise DateRangeInvalid(e)

    def _display_reminder(self, date):
        """
        Timer callback function.
        """

        request = InteractiveModuleRequest(
        )

        response = self.display_events_by_date(None, Location.CHANNEL, None, [date])

        for channel in self.usermgmt.chanlist.get_channels().values():
            self.send_response(channel, response)

    #---------------------------------------------------------------------------
    # InteractiveModule commands - display
    #---------------------------------------------------------------------------
    def display_calendar_address(self, request):
        """
        Display the web calendar's address.
        """

        response = InteractiveModuleResponse(self.config.get('calendarUrl'))

        return response

    def display_events_today(self, request):
        """
        Display today's events, if any.
        """

        request.parameter = [date.today()]

        return self.display_events_by_date(request)

    def display_events_by_date(self, request):
        """
        Display events occuring at the given date.
        """

        response = InteractiveModuleResponse()

        date = request.parameter[0]

        try:
            calendar_events = self.component.find_events_by_date(date)

            if len(calendar_events) == 0:
                raise NoEventsFound

            for calendar_event in calendar_events:
                response.add_line("ID {0}: {1}".format(calendar_event.id, calendar_event.title))

        except NoEventsFound:
            response.add_line('Keine Termine gefunden.')

        return response

    def display_deleted_objects(self, event, location, command, parameter):
        """
        """

        return InteractiveModuleResponse('not implemented')

    def restore_deleted_object(self, request):
        """
        """

        return InteractiveModuleResponse('not implemented')

    #---------------------------------------------------------------------------
    # InteractiveModule commands - events
    #---------------------------------------------------------------------------
    def insert_event(self, request):
        """
        Create a new event.

        .addevent <startdate>[-<enddate>] title
        """

        response = InteractiveModuleResponse()

        date = request.parameter[0]
        title = request.parameter[1]

        try:
            if '-' not in date:
                dateFrom = self._get_date(date)
                dateTo = dateFrom

            else:
                dates = date.split('-')

                dateFrom = self._get_date(dates[0])
                dateTo = self._get_date(dates[1])

            calendar = self.component.find_default_calendar()

            event = Event(calendar=calendar, start=dateFrom, end=dateTo, title=title)
            event = self.component.insert_object(event)

            response.add_line("Eintrag erfolgreich eingefügt! ID: {0}".format(event.id))

        except DateFormatInvalid:
            response.add_line("Datum muss im Format [d]d.[m]m.yyyy sein. Bsp: 12.5.2010")

        except DateRangeInvalid as e:
            response.add_line('Ungültiges Datum: {0}.'.format(e))

        return response

    def change_event(self, event, location, command, parameter):
        """
        Change an existing event.
        The event is identified by its ID.

        Syntax: <id> <start|ende|titel|beschreibung|ort> <wert>
        """

        response = InteractiveModuleResponse()

        eventId = parameter[0]
        attribute = parameter[1]
        value = parameter[2]

        try:
            map = {
                'start' : 'start',
                'ende' : 'end',
                'titel' : 'title',
                'beschreibung' : 'description',
                'ort' : 'location'
            }

            if attribute in ['start', 'ende']:
                value = self._get_date(value)

            event = self.component.find_event_by_id(eventId)

            setattr(event, map[attribute], value)

            self.component.update_object(event)

            response.add_line('{0} fuer Eintrag {1} wurde auf {2} gesetzt'.format(
                attribute.capitalize(),
                event.id,
                value
            ))

        except DateFormatInvalid:
            response.add_line("Datum muss im Format [d]d.[m]m.yyyy sein. Bsp: 12.5.2010")

        except EventNotFound:
            response.add_line('Kein Event mit ID {0} gefunden'.format(eventId))

        return response

    def delete_event(self, request):
        """
        .delevent [<id>|<datum>]
        """

        response = InteractiveModuleResponse()

        # when only one parameter is defined a string is returned
        id_or_date = request.parameter[0]

        try:
            try:
                date = self._get_date(id_or_date)

                #---------------------------------------------------------------
                # delete by date
                #---------------------------------------------------------------
                events = self.component.find_events_by_date(date)

                count = len(events)

                if count == 0:
                    raise NoEventsFound

                if count > 1:
                    raise AmbiguousEventsFound(events)

                event = events[0]

            except DateFormatInvalid:
                #---------------------------------------------------------------
                # delete by id
                #---------------------------------------------------------------
                eventId = int(id_or_date)

                event = self.component.find_event_by_id(eventId)

                if not event:
                    raise NoEventsFound

            self.component.delete_object(event)

            response.add_line("Eintrag {0} wurde geloescht.".format(id_or_date))

        except InvalidObjectId:
            response.add_line("Kein Eintrag zu dieser ID gefunden.")

        except NoEventsFound:
            response.add_line("Kein Eintrag zu diesem Datum gefunden.")

        except AmbiguousEventsFound as error:
            response.add_line('Mehrere Einträge gefunden:')
            [response.add_line("(ID={0}) {1}".format(event.id, event.title)) for event in error.message]

        return response

    def search_event(self, request):
        """
        Search for an event

        .searchevent <text>
        """

        response = InteractiveModuleResponse()

        query = request.parameter[0]

        try:
            events = self.component.find_events_by_substring(query)

            [response.add_line("(ID={0}) {1}".format(event.id, event.title)) for event in events]

        except NoEventsFound:
            response.add_line("Keine Einträge mit diesem Inhalt gefunden.")

        return response

    def topic_event(self, request):
        """
        Post the given event to the current topic.
        .topicevent

        TODO: implement
        """

        response = InteractiveModuleResponse()

        eventId = int(request.parameter[0])

        try:
            result = self.component.find_event_by_id(eventId)

            if result is None:
                raise EventNotFound

            response.add_line('currently not implemented');

        except EventNotFound:
            response.add_line('Kein Event mit ID {0} gefunden'.format(eventId))

        return response

    #---------------------------------------------------------------------------
    # InteractiveModule commands - contacts
    #---------------------------------------------------------------------------
    def insert_contact(self, request):
        """
        """

        response = InteractiveModuleResponse("not implemented")

        return response

    def change_contact(self, request):
        """
        """

        response = InteractiveModuleResponse("not implemented")

        return response

    def delete_contact(self, request):
        """
        """

        response = InteractiveModuleResponse("not implemented")

        return response

    def search_contact(self, request):
        """
        """

        response = InteractiveModuleResponse("not implemented")

        return response

class CalendarConfig(Config):
    identifier = 'interaction.irc.calendar'

    def valid_keys(self):
        return [
            'reminderInterval',
            'calendarUrl'
        ]

    def default_values(self):
        return {
            'reminderInterval' : TIME_MINUTE * 5,
            'calendarUrl'      : ''
        }


import turtle
import numpy as np
from sympy import Point
from turtle import Turtle, Screen
from matplotlib import pyplot as plt
from utils import read_perimeter_from_file, rearrange_points
from douglas_peucker import douglas_peucker

screen = Screen()
screen.bgpic("field.gif")

t = Turtle()

t.pencolor("red")
t.width(4)

t.speed(-1)
points = list()

def dragging(x, y):
    point = Point(x, y)
    points.append(point)
    t.ondrag(None)
    t.setheading(t.towards(x,y))
    t.goto(x, y)
    t.ondrag(dragging)

def draw_points(pinpoints, color="red"):
    if not pinpoints:
        raise Exception("Empty point set!")
    for i in range(len(pinpoints)-1):
        draw_line(pinpoints[i], pinpoints[i+1], color=color)
    draw_line(pinpoints[0], pinpoints[-1])

def click_right(curr_x, curr_y):
    global points
    # points = read_perimeter_from_file("points.in")
    epsilon = 15
    # draw_points(points, color="red")
    print(f"Detected {len(points)} points")
    # points = rearrange_points(points)
    points = douglas_peucker(points, epsilon)
    print(f"Polygon {len(points)} points found")
    t.clear()
    draw_points(points, color="blue")

def move_pen(x, y):
    t.penup()
    t.goto(x, y)
    t.pendown()

def draw_line(point1, point2, color="red"):
    t.pencolor(color)
    t.penup()
    t.goto(point1.x, point1.y)
    t.pendown()
    t.goto(point2.x, point2.y)
    t.penup()

def main():
    turtle.listen()

    t.ondrag(dragging)

    turtle.onscreenclick(click_right, 3)
    turtle.onscreenclick(move_pen, 2)
    screen.mainloop()

main()


import socket
import os
import sys
import argparse
import random
import yaml
import errno
import time

YAML_FILE = './dictionary.yml'
NUM_QUES = 2

questions = [
		'./questions/question1.txt',
		'./questions/question2.txt'
	]

connection_table = {}

def handle_connection(cfd, cli_address):
	(host, port) = cli_address
	key = host + "@" + str(port)
	qno = random.randrange(1, NUM_QUES+1, 1)
	f = open(questions[qno-1], "r")
	contents = f.read()

	cfd.send(key.encode())
	time.sleep(0.05)
	cfd.send(str(qno).encode())
	time.sleep(0.05)
	cfd.send(contents.encode())

	connection_table[key] = qno
	with open(YAML_FILE, 'w') as ymlf:
		yaml.dump(connection_table, stream=ymlf, default_flow_style=False)

def connection_loop(port):
	sfd = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	sfd.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
	sfd.bind(('localhost', port))
	sfd.listen(200)

	while True:
		cfd, cli_address = sfd.accept()

		pid = os.fork()
		if pid == 0:
			sfd.close()
			handle_connection(cfd, cli_address)
			cfd.close()

		cfd.close()

if __name__ == '__main__':
	parser = argparse.ArgumentParser()
	parser.add_argument('port', type=int, help='server port')
	args = parser.parse_args()
	connection_loop(args.port)


import requests
from bs4 import BeautifulSoup
import csv
import json
import xml.dom.minidom

def getHTMLtext(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        bs = BeautifulSoup(r.text, "html.parser")  # 创建BeautifulSoup对象
        body = bs.body
        if (body):
            print("成功访问")
            return r.text
        else:
            print("空包")
            return 1
    except:
        print("访问错误")
        return 0

def get_content(html, area):
    bs = BeautifulSoup(html, "html.parser")  # 创建BeautifulSoup对象
    body = bs.body
    # 下面爬取当天的数据

    data2 = body.find_all('div', {'class': 'left-div'})
    text = data2[2].find('script').string
    text = text[text.index('=') + 1:-2]  # 移除改var data=将其变为json数据
    jd = json.loads(text)
    dayone = jd['od']['od2']
    final_day = []  # 存放当天的数据
    count = 0
    for i in dayone:
        temp = []
        if count == 0:
            count = count + 1
            continue
        if count <= 24:
            temp.append(area)  # 添加地区名称
            temp.append(i['od21'])  # 添加时间
            temp.append(i['od22'])  # 添加当前时刻温度
            temp.append(i['od25'])  # 添加当前时刻风级
            temp.append(i['od26'])  # 添加当前时刻降水量
            temp.append(i['od27'])  # 添加当前时刻相对湿度
            temp.append(i['od28'])  # 添加当前时刻控制质量
            final_day.append(temp)
        count = count + 1
    return final_day

def write_to_csv(file_name, data):
    with open(file_name, 'a', errors='ignore', newline='') as f:
        header = ['地区', '小时', '温度', '风级', '降水量', '相对湿度', '空气质量']
        f_csv = csv.writer(f)
        f_csv.writerow(header)
        f_csv.writerows(data)

def main():  # url = 'http://www.weather.com.cn/weather/101280701.shtml'  # 7天天气中国天气网
    dom = xml.dom.minidom.parse('id.xml')
    root = dom.documentElement
    county = root.getElementsByTagName('county')
    base = 'http://www.weather.com.cn/weather/'
    a = 0;
    for eachcounty in county:
        name = eachcounty.getAttribute('name')
        idcode = eachcounty.getAttribute('weatherCode')
        url = base + idcode + '.shtml'
        html = getHTMLtext(url)
        if html != 1 and html != 0:
            data = get_content(html, name)
            write_to_csv('weather_'+ str(a).zfill(3) + '.csv', data)
            a = a + 1
if __name__ == '__main__':
    main()


import json
import pandas
import random

# INSTRUCTIONS FOR CREATING MOCK STUDENTS
# BACKGROUND: this python script allows for the creation of realistic mock students,
# potentially to be used in populating a database.
# The methods of this script use results from a survey given to the UBC student body.
# By doing so, the script is able to produce students that somewhat resemble the surveyed students.

# IF YOU WISH TO CREATE MOCK STUDENTS...,
# scroll down to line "320" to the function call "manufactureStudents(integer)".
# Uncomment this line by deleting the "#" symbol at the start of the line,
# and replace "integer" with the number of students you wish to create.

# then, if you own a Mac, open a terminal at the directory you are storing this file,
# and copy and paste the following command into the terminal: python3 CreateStudents.py
# The terminal should then output a list of student objects.
# You may wish to copy and paste this list into a JSON file.
# NOTE: you must have python downloaded in your system. You may find the latest version
# may be found here: https://www.python.org/downloads/mac-osx/

excel_data_df = pandas.read_excel('SurveyResponses.xlsx', sheet_name='Form Responses 1')

academicYear = pandas.DataFrame(data=excel_data_df, columns=["Academic Year"])
coop = pandas.DataFrame(data=excel_data_df, columns=["Co-op"])
gpa = pandas.DataFrame(data=excel_data_df, columns=["GPA"])
genCSSkills = pandas.DataFrame(data=excel_data_df, columns=["General CS Skills"])
workExperience = pandas.DataFrame(data=excel_data_df, columns=["work experience"])

languages = pandas.DataFrame(data=excel_data_df,
	columns=[
	"Java",
	"Python",
	"C++",
	"C",
	"C#",
	"Javascript",
	"Typescript",
	"HTML",
	"CSS",
	"R",
	"MATLAB",
	"Mathematica",
	"SQL",
	"Other"
	])

frameworks = pandas.DataFrame(data=excel_data_df,
	columns=[
	"React",
	"Node",
	"Angular",
	"Bootstrap",
	"MongoDB",
	"Ruby on Rails",
	"Google Cloud",
	"AWS",
	"Docker",
	"Linux",
	"Unix",
	"Other"
	])

workflowTools = pandas.DataFrame(data=excel_data_df,
	columns= [
	"GitHub",
	"Azure",
	"Jira",
	"Jupyter",
	"Other",
	])

languagesDict = languages.to_dict('index')
frameworksDict = frameworks.to_dict('index')
workflowToolsDict = workflowTools.to_dict('index')
academicYearDict = academicYear.to_dict('index')
coopDict = coop.to_dict('index')
gpaDict = gpa.to_dict('index')
genCSSkillsDict = genCSSkills.to_dict('index')
workExperienceDict = workExperience.to_dict('index')

## Methods from lines 87 through to 275 are used for purposes of synthetically populating fields from realistically weighted pools

cssConcepts = ['Data Structures',
 'Algorithms',
 'Asynchronous programming',
 'Agile Development',
 'Object-Oriented Programming',
 'Functional Programming',
 'web APIs',
 'RESTify services',
 'Computer Networks']

def cssConceptsPopulator():
	studentCSSConcepts = []
	for concept in cssConcepts:
		if (bool(random.getrandbits(1))):
			studentCSSConcepts.append(concept)
	return studentCSSConcepts

def reformatDictionary(dictionary):
	for key in dictionary:
		dictionary[key] = reformatElement(dictionary[key])
	return(dictionary)

def reformatElement(content):
	if (content == '≥ 4 months'):
		return 4
	elif (content == '≥ 8 months'):
		return 8
	elif (content == '≥ 12 months'):
		return 12
	elif (content == '≥ 2 years'):
		return 24
	elif (content == 'None'):
		return 0
	elif (content == 'Below 60%'):
		return random.randint(50, 59)
	elif (content == '60-69%'):
		return random.randint(60, 69)
	elif (content == '70-79%'):
		return random.randint(70, 79)
	elif (content == '80-89%'):
		return random.randint(80, 89)
	elif (content == '90%+'):
		return random.randint(90, 100)
	elif (content == '4th Year+'):
		return 4
	elif (content == '3rd Year'):
		return 3
	elif (content == '2nd Year'):
		return 2
	elif (content == '1st Year'):
		return 1
	elif (content == 'Yes'):
		return True
	elif (content == 'No'):
		return False

def gpaRandomizer(figure):
	if (figure == 50):
		return random.randint(50, 59)
	elif (figure == 60):
		return random.randint(60, 69)
	elif (figure == 70):
		return random.randint(70, 79)
	elif (figure == 80):
		return random.randint(80, 89)
	elif (figure == 90):
		return random.randint(90, 100)

seeking = [True, True, True, True, True, True, True, False, False, False]
coop = [True, True, True, True, True, True, False, False, False, False]
workExperience = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 12, 12, 24]
aYear = [1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4]
sGPA = [50, 60, 60, 60, 70, 70, 70, 70, 70, 70, 80, 80, 80, 80, 80, 80, 80, 90, 90, 90]
techExperience = [0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 8, 8, 8, 8, 8, 12, 12, 12, 12, 12, 24, 24, 24, 24, 24]
javaExperience = [0, 4, 4, 4, 4, 4, 8, 8, 8, 12, 12, 12, 12, 12, 12, 24, 24, 24, 24, 24]
pythonExperience = [0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 8, 8, 8, 12, 12, 12, 12, 24, 24, 24]
cppExperience = [0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 8, 8, 8, 8, 8, 12, 12, 12, 24]
cExperience = [0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 8, 8, 12, 12, 12, 24, 24]
cppppExperience = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 12, 24]
javascriptExperience = [0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 12, 12, 12, 12, 12, 24, 24, 24, 24, 24]
typescriptExperience = [0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 8, 8, 12, 12, 12, 12, 12, 24]
htmlExperience = [0, 4, 4, 4, 4, 4, 8, 8, 8, 12, 12, 12, 12, 12, 12, 24, 24, 24, 24, 24]
cssExperience = [0, 4, 4, 4, 4, 4, 4, 4, 8, 12, 12, 12, 12, 12, 12, 24, 24, 24, 24, 24]
rExperience = [0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 8, 8, 8, 12, 24]
matlabExperience = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 8, 12, 24]
mathematicaExperience = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 12, 24]
sqlExperience = [0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 24]
otherExperience = [0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12, 24, 24, 24, 24]
reactExperience = [0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 8, 8, 8, 12, 12, 12, 12, 24, 24, 24]
nodeExperience = [0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 8, 8, 8, 12, 12, 12, 12, 12, 24, 24]
angularExperience = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 12, 24]
bootstrapExperience = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 8, 8, 12, 12, 24]
mongodbExperience = [0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 8, 8, 8, 12, 12, 24, 24]
rubyonrailsExperience = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 8, 12]
googlecloudExperience = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 8, 8, 8, 12, 12, 24]
awsExperience = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 8, 8, 8, 12, 12, 24, 24]
dockerExperience = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 8, 8, 8, 12, 12, 24]
linuxExperience = [0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12, 24, 24, 24, 24]
unixExperience = [0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 24, 24]
githubExperience = [0, 4, 8, 8, 12, 12, 12, 12, 12, 12, 12, 24, 24, 24, 24, 24, 24, 24, 24, 24]
azureExperience = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 8, 8, 12, 12, 24]
jiraExperience = [0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 8, 8, 8, 12, 12, 12, 24, 24, 24]
jupyterExperience = [0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 24, 24]
asanaExperience = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 12 ,12, 24]
confluenceExperience = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 12 ,12, 24]
notionExperience = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 12 ,12, 24]
kubernetesExperience = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 12 ,12, 24]
materialuiExperience = [0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12, 24, 24, 24, 24]

location = [
'Canada', 'Canada', 'Canada', 'Canada', 'Canada',
'Canada', 'Canada', 'Canada', 'Canada', 'Canada',
'Canada', 'Canada', 'Canada', 'Canada', 'Canada',
'Canada', 'Canada', 'France', 'China', 'Japan',
'Singapore', 'Russia', 'India', 'Turkey', 'Germany']

degree = [
"BSc", "BSc", "BSc", "BSc", "BSc",
"BSc", "BSc", "BSc", "BSc", "BSc",
"BSc", "BSc", "BSc", "BSc", "BSc",
"MSc", "MSc", "MSc", "MSc", "PhD"]

degreeLocation = [
'University of British Columbia', 'University of British Columbia', 'University of British Columbia',
'University of British Columbia', 'University of British Columbia', 'University of British Columbia',
'University of British Columbia', 'University of British Columbia', 'University of British Columbia',
'University of British Columbia', 'University of British Columbia', 'University of British Columbia',
'University of British Columbia', 'University of British Columbia', 'University of British Columbia',
'University of Waterloo', 'University of Alberta', 'University of Toronto', 'McGill University', 'University of Victoria']

citizenship = ['Citizen', 'Citizen', 'Citizen', 'Citizen', 'Citizen',
'Citizen', 'Citizen', 'International', 'International', 'Permanent Residency']

# Use this method to create student objects from the survey results (62 objects in total)
def createStudents():
	i = 0
	students = []
	print('starting to assemble students')
	for entry in frameworksDict:
		student = {
		'gpaValue': reformatDictionary(gpaDict[i])['GPA'],
		'year': reformatDictionary(academicYearDict[i])['Academic Year'],
		'coOp': reformatDictionary(coopDict[i])['Co-op'],
		'seeking': seeking[random.randint(0, 9)],
		'candidate': citizenship[random.randint(0, 9)],
		'experience': reformatDictionary(workExperienceDict[i])['work experience'],
		'languages': reformatDictionary(languagesDict[i]),
		'frameworks': reformatDictionary(frameworksDict[i]),
		'tools': reformatDictionary(workflowToolsDict[i]),
		'concepts': genCSSkillsDict[i]['General CS Skills'].split(", "),
		'location': location[random.randint(0, 24)],
		'academicReq': {degree[random.randint(0, 19)] : degreeLocation[random.randint(0, 19)]}
		}
		students.append(student)
		i+=1
	print(json.dumps(students))
	print('finished assembling students!!')

# Use this method to synthesize a new, realistic, group of students
def manufactureStudents(numStudents):
	i = 0
	students = []
	print('starting to manufacture students')
	while i < numStudents:
		student = {
		'gpaValue': gpaRandomizer(sGPA[random.randint(0, 19)]),
		'year': aYear[random.randint(0, 19)],
		'coOp': coop[random.randint(0,9)],
		'seeking': seeking[random.randint(0, 9)],
		'candidate': citizenship[random.randint(0, 9)],
		'experience': workExperience[random.randint(0,19)],
		'languages': {
			'Java': javaExperience[random.randint(0,19)],
			'Python': pythonExperience[random.randint(0,19)],
			'C++': cppExperience[random.randint(0,19)],
			'C': cExperience[random.randint(0,19)],
			'C#': cppppExperience[random.randint(0,19)],
			'JavaScript': javascriptExperience[random.randint(0,19)],
			'TypeScript': typescriptExperience[random.randint(0,19)],
			'HTML': htmlExperience[random.randint(0,19)],
			'CSS': cssExperience[random.randint(0,19)],
			'R': rExperience[random.randint(0,19)],
			'MATLAB': matlabExperience[random.randint(0,19)],
			'Mathematica': mathematicaExperience[random.randint(0,19)],
			'SQL': sqlExperience[random.randint(0,19)],
			'Other': otherExperience[random.randint(0,19)]

		},
		'frameworks': {
			'React': reactExperience[random.randint(0,19)],
			'Node': nodeExperience[random.randint(0,19)],
			'Angular': angularExperience[random.randint(0,19)],
			'Bootstrap': bootstrapExperience[random.randint(0,19)],
			'MongoDB': mongodbExperience[random.randint(0,19)],
			'Ruby on Rails': rubyonrailsExperience[random.randint(0,19)],
			'Google Cloud': googlecloudExperience[random.randint(0,19)],
			'AWS': awsExperience[random.randint(0,19)],
			'Docker': dockerExperience[random.randint(0,19)],
			'Linux': linuxExperience[random.randint(0,19)],
			'Unix': unixExperience[random.randint(0,19)],
			'MaterialUI': materialuiExperience[random.randint(0,19)],
			'Other': otherExperience[random.randint(0,19)]

		},
		'tools': {
			'GitHub': githubExperience[random.randint(0,19)],
			'Azure': azureExperience[random.randint(0,19)],
			'Jira': jiraExperience[random.randint(0,19)],
			'Jupyter': jupyterExperience[random.randint(0,19)],
			'Asana': asanaExperience[random.randint(0,19)],
			'Confluence': confluenceExperience[random.randint(0,19)],
			'Notion': notionExperience[random.randint(0,19)],
			'Kubernetes': kubernetesExperience[random.randint(0,19)],
			'Other': otherExperience[random.randint(0,19)]

		},
		'concepts': cssConceptsPopulator(),
		'location': location[random.randint(0, 24)],
		'academicReq': {degree[random.randint(0, 19)] : degreeLocation[random.randint(0, 19)]}

		}
		student['languages']['CSS'] = min(student['languages']['CSS'], student['languages']['HTML'])
		student['frameworks']['React'] = min(student['frameworks']['React'], student['languages']['JavaScript'])
		student['frameworks']['Node'] = min(student['frameworks']['Node'], student['languages']['JavaScript'])
		student['frameworks']['Angular'] = min(student['frameworks']['Angular'], student['languages']['JavaScript'])
		student['frameworks']['MaterialUI'] = min(student['frameworks']['MaterialUI'], student['frameworks']['React'])
		students.append(student)
		i+=1
	print(json.dumps(students))
	print('completed manufacturing students')

# manufactureStudents(integer)


#User function Template for python3

from typing import List
from queue import Queue
class Solution:
    #Function to return Breadth First Traversal of given graph.
    def bfsOfGraph(self, V: int, adj: List[List[int]]) -> List[int]:
        # code here
        bfsTraversal = []
        visited = [False] * V  # To keep track of visited nodes
        queue = Queue()  # Queue to store nodes to be visited

        # Start BFS from node 0
        queue.put(0)
        visited[0] = True

        while not queue.empty():
            currentNode = queue.get()  # Get the front node from the queue
            bfsTraversal.append(currentNode)  # Add the current node to the result

            # Visit all adjacent nodes of the current node
            for neighbor in adj[currentNode]:
                if not visited[neighbor]:
                    queue.put(neighbor)
                    visited[neighbor] = True

        return bfsTraversal

#{
 # Driver Code Starts

if __name__ == '__main__':
	T=int(input())
	for i in range(T):
		V, E = map(int, input().split())
		adj = [[] for i in range(V)]
		for _ in range(E):
			u, v = map(int, input().split())
			adj[u].append(v)
		ob = Solution()
		ans = ob.bfsOfGraph(V, adj)
		for i in range(len(ans)):
		    print(ans[i], end = " ")
		print()

# } Driver Code Ends


# Ambient から定期的にデータを取得する

from AmbientDataHandler import AmbientDataHandler
import datetime
from get_filepath import get_filepath
import pandas as pd
import ambient
import os
from recieve_username_auth import recieve_username_auth

def main():
    adh = AmbientDataHandler()
    read_config_df = adh.fetch_ambient_read_config()
    chID_readKey_df = read_config_df[["chID", "readKey"]]
    # chID_set =  set(chID_readKey_df["chID"])

    all_read_data_df = pd.DataFrame(columns=["created","chID"] + list(set(read_config_df["dataID"])))
    datetime_now = datetime.datetime.now()

    # 最後に読み込みを行った日時を読み込む。
    # そこを起点として新しいデータを読み込む。
    file = "../resource/latest_read_datetime.txt"
    filepath = get_filepath(file)
    try:
        with open(filepath, encoding="shift-jis", mode = "r") as f:
            latest_read_datetime = f.read()
    except FileNotFoundError:
        # 最終読み込み時刻を保存したファイルが無ければ、読み込み間隔を取得し、その分を引いた時刻を起点とする
        # それが見つからなければエラーを返す
        file = "../resource/cron_setting.csv"
        filepath = get_filepath(file)
        try:
            cron_setting_df = pd.read_csv(filepath, encoding="shift-jis", index_col=0)
        except FileNotFoundError:
            message = "最後に読み込みを行った時刻を保存したファイルが存在しません"
            file = "../stderr.log"
            filepath = get_filepath(file)
            with open(filepath, mode = "a") as f:
                f.write(message + "\n")
            return
        data_getting_run_span = int(cron_setting_df.loc["data_getting", "run_span"])
        latest_read_datetime = (datetime_now - datetime.timedelta(minutes=data_getting_run_span)).strftime("%Y-%m-%d %H:%M:%S")

    start_date_time = latest_read_datetime
    end_date_time = datetime_now.strftime("%Y-%m-%d %H:%M:%S")

    for chID in set(chID_readKey_df["chID"]):
        # chIDが同じなら一意にreadKeyが決まるはず。
        readKey = set(chID_readKey_df.loc[chID_readKey_df["chID"] == chID, "readKey"]).pop()

        # WriteKeyは使わないので ""でよい。
        am = ambient.Ambient(chID,"", readKey)
        read_data = am.read(start = start_date_time, end = end_date_time)
        read_data_df = pd.DataFrame(read_data)
        read_data_df["chID"] = chID
        all_read_data_df = pd.concat([all_read_data_df, read_data_df], axis = 0)

    # all_read_data_dfが空 = 更新するデータが無い.
    # よって、以後の処理を行わない。
    if all_read_data_df.empty:
        print("No data to read")
        return

    all_read_data_df = all_read_data_df.rename(columns={"created": "datetime"})
    # Ambient から帰ってくる日付はUTC
    all_read_data_df["datetime"] = pd.to_datetime(all_read_data_df["datetime"]).dt.tz_convert('Asia/Tokyo')
    # ここから後でtimezoneの情報はいらないと思うので消す。
    all_read_data_df["datetime"] = list(map(lambda x: x.replace(tzinfo = None).strftime("%Y-%m-%d %H:%M:%S"), all_read_data_df["datetime"]))

    melted_df = all_read_data_df.melt(id_vars=["datetime", "chID"], var_name="dataID")
    # chIDとdataIDをキーとしてdatatyoeとhouse_nameの情報を列に追加する。
    update_dict = {
        (row["chID"], row["dataID"]): (row["datatype"], row["house_name"])
        for _, row in read_config_df.iterrows()
    }
    melted_df[["datatype", "house_name"]] = melted_df.apply(lambda x: update_dict.get((x["chID"], x["dataID"]), (None, None)), axis=1, result_type="expand")
    melted_df = melted_df.drop(columns=["chID", "dataID"])
    melted_df = melted_df.reindex(columns=["datetime", "house_name", "datatype", "value"])

    file = "../resource/I2C_data_period.csv"
    filepath = get_filepath(file)
    melted_df.to_csv(filepath, mode="a", encoding="shift-jis", header = not os.path.exists(filepath), index=False)

    # 読み込んだデータの中で最新のものを抜きだし、定期通知用に記録する。
    latest_df = pd.DataFrame()
    house_datatype_tuple_list = melted_df[["house_name", "datatype"]].apply(tuple, axis=1).drop_duplicates().tolist()
    for house_datatype_tuple in house_datatype_tuple_list:
        house_datatype_df = melted_df.loc[(melted_df["house_name"] == house_datatype_tuple[0]) & (melted_df["datatype"] == house_datatype_tuple[1]), :]
        latest_datetime = max(house_datatype_df["datetime"])
        concat_df = house_datatype_df.loc[house_datatype_df["datetime"] == latest_datetime, :]
        latest_df = pd.concat([latest_df, concat_df], axis = 0)

    file = "../resource/I2C_data_latest_period.csv"
    filepath = get_filepath(file)
    latest_df.to_csv(filepath, mode="w", encoding="shift-jis", header = True, index=False)

    # データの保存が無事終了してから、最終読み込み日時を書き込む
    # 読み込み日時が更新されない＝どこかでエラー発生
    # 次に読み込む時に、保存できなかった分のデータも一緒に読み込む
    file = "../resource/latest_read_datetime.txt"
    filepath = get_filepath(file)
    with open(filepath, encoding="shift-jis", mode = "w") as f:
        f.write(end_date_time)

if recieve_username_auth():
    main()


import unittest, json
from classifurlr import run
from classifurlr.classifiers import *

FIXTURE_DIR = 'tests/fixtures/'
def test_result(session_filename):
    with open(FIXTURE_DIR + session_filename, 'r') as f:
        result = run(json.load(f))
    return result

class DifferingDomainTest(unittest.TestCase):
    def test_is_ip(self):
        d = DifferingDomainClassifier()
        self.assertTrue(d.is_ip('https://192.128.0.1:80'))
        self.assertFalse(d.is_ip('https://www.google.com'))

class StatusCodeTest(unittest.TestCase):
    def test_non_200(self):
        filename = '403.json'
        result = test_result(filename)
        self.assertTrue(result.is_down())
        self.assertTrue(result.get_constituents()[0].get_constituent_from(StatusCodeClassifier).is_down())

class InconclusiveFilterTest(unittest.TestCase):
    def test_seized_domain(self):
        filename = 'kickass.json'
        result = test_result(filename)
        self.assertTrue(result.is_inconclusive())

class BlockpageSignatureTest(unittest.TestCase):
    def test_redirects_to_different_domain(self):
        filename = 'lesbiansubmission.json'
        result = test_result(filename)
        self.assertTrue(result.is_blocked())

    def test_blocked_embed(self):
        filename = 'samurpress.json'
        result = test_result(filename)
        self.assertFalse(result.is_blocked())

if __name__ == '__main__':
    unittest.main()


import os
import argparse
import torch
import random
from torch import optim
from math import exp

from model import Copynet, Normal, MoG, Vamp, GP_Full
from training import train, evaluate
from utils import reader
from decode import BeamSearch

def parse_arguments():
    p = argparse.ArgumentParser(description='Hyperparams')
    # Training config
    p.add_argument('-t', '--task', type=str, default="train",
                   help="specify the task to do: (train)ing, (eval)uation, decode")
    p.add_argument('-epochs', type=int, default=200,
                   help='number of epochs for train')
    p.add_argument('-bs', '--batch_size', type=int, default=64,
                   help='number of epochs for train')
    p.add_argument('-optim', type=int, default=2,
                   help="optimization method. 0: SGD; 1: Adagrad, 2: Adam")
    p.add_argument('-lr', type=float, default=0.001,
                   help='initial learning rate')
    p.add_argument('-grad_clip', type=float, default=5.0,
                   help='in case of gradient explosion')
    p.add_argument('-val_step', type=int, default=1,
                   help='the number of epoches between two validations')
    p.add_argument('-print_step', type=int, default=100,
                   help='the number of batches between two tf.summary')
    p.add_argument('-kw', type=float, default=0.,
                   help="scaler for kl annealing function")
    p.add_argument('-x0', type=int, default=25000,
                   help="mean of kl annealing function")
    # Model config
    p.add_argument('-mt', '--model_type', type=str, default='gp_full',
                   help='type of model: copynet/normal/mog/vamp/gp_full')
    p.add_argument('-e', '--embed_size', type=int, default=300,
                   help='embedding size')
    p.add_argument('-hs', '--hidden_size', type=int, default=512,
                   help='output dimension of the recognition network')
    p.add_argument('-k', '--latent_size', type=int, default=256,
                   help="dimension of the latent variable z")
    p.add_argument('-wd', '--word_dropout', type=float, default=0,
                   help='decoder input sentence dropout rate')
    p.add_argument('--embed_dropout', type=float, default=0.5,
                   help='word embedding dropout rate')
    p.add_argument('-c', '--components', type=int, default=10,
                   help="number of components for mixture gaussian")
    p.add_argument('-kld_sampled', type=int, default=0,
                   help="how to compute kld. 1: sampled kld; 0: analytical kld")
    p.add_argument('-v', '--kernel_v', type=float, default=65.0,
                   help="Hyper-parameter for prior kernel,  control the signal variance")
    p.add_argument('-r', '--kernel_r', type=float, default=0.0001,
                   help="Hyper-parameter for prior kernel.")
    p.add_argument('-vocab', '--vocab_size', type=int, default=20000,
                   help="vocab size")
    p.add_argument('-max_len', type=int, default=30,
                   help="max sequence length")
    p.add_argument('-using_cuda', type=bool, default=True,
                   help="whether to use cuda")
    # Generation - decoding strategy
    p.add_argument('-samn', '--sample_num', type=int, default=10,
                   help="Number of samples from latent codes.")
    p.add_argument('-bms', '--beam_size', type=int, default=10,
                   help="beam size, if 0, use greedy search; otherwise, using beam search.")
    p.add_argument('-df', '--data_file', type=str, default='../../data/GYAFC/em',
                   help='path to dataset')
    p.add_argument('-mf', '--model_file', type=str, default=None,
                   help='path to a pretrained model')
    # Generation - sampling latent codes
    p.add_argument('-topn', type=int, default=10,
                   help="number of generated samples for the decoder")
    p.add_argument('-std', type=float, default=1.,
                   help="Standard deviation for sampling z.")
    p.add_argument('-decf', '--decode_from', type=str, default='sample',
                   help="sample: decode from sampled latent codes; mean: decode from mean")
    p.add_argument('--seed', type=int, default=123)
    return p.parse_args()

def main():
    # --------------------------------------------
    # print("*** Parsing arguments ***")
    args = parse_arguments()
    args.using_cuda = args.using_cuda and torch.cuda.is_available()

    # --------------------------------------------
    print("*** Preparing dataset ... ***")
    train_iter, val_iter, test_iter, txtfield = reader(suffix=".tsv",
                                                       rpath=args.data_file,
                                                       batch_size=args.batch_size)
    print("[TRAIN]:%d (dataset:%d)\t[VAL]:%d (dataset:%d)\t[TEST]:%d (dataset:%d)"
          % (len(train_iter), len(train_iter.dataset),
             len(val_iter), len(val_iter.dataset),
             len(test_iter), len(test_iter.dataset)))
    print("[vocab]:%d" % (len(txtfield.vocab)))

    # --------------------------------------------
    print("*** Initializing the model ... ***")
    if args.model_type == "copynet":
        model = Copynet(args, txtfield)
    elif args.model_type == "normal":
        model = Normal(args, txtfield)
    elif args.model_type == "mog":
        model = MoG(args, txtfield)
    elif args.model_type == "vamp":
        model = Vamp(args, txtfield)
    elif args.model_type == "gp_full":
        model = GP_Full(args, txtfield)
    else:
        raise ValueError("Unrecognized model type: %s" % args.model_type)

    # --------------------------------------------
    if (args.model_file is not None) and os.path.isfile(args.model_file):
        print("Loading the pretrained model from: {}".format(args.model_file))
        if args.using_cuda:
            model.load_state_dict(torch.load(args.model_file))
        else:
            model.load_state_dict(torch.load(args.model_file, map_location='cpu'))
    else:
        print("Couldn't find the pretrained model from: {}".format(args.model_file))
    if args.using_cuda: model.cuda()

    # --------------------------------------------
    optimizer = None
    params = model.parameters()
    if args.optim == 0:
        print("*** Choosing SGD as the optimization method ... ***")
        optimizer = optim.SGD(params, lr=args.lr)
    elif args.optim == 1:
        print("*** Choosing Adagrad as the optimization method ... ***")
        optimizer = optim.Adagrad(params, lr=args.lr)
    elif args.optim == 2:
        print("*** Choosing Adam as the optimization method ... ***")
        optimizer = optim.Adam(params, lr=args.lr)
    else:
        raise ValueError("Unspecified optimization method")

    # --------------------------------------------
    if args.task == "train":
        print("*** Training with validation ... ***")
        train(model, optimizer, train_iter, val_iter, txtfield, args)
    elif args.task == "eval":
        print("*** Testing ... ***")
        test_nll, test_kld = evaluate(model, test_iter, txtfield, args)
        print("[TEST] NLL: %5.4f KLD: %5.4f PPLx: %5.4f" % (test_nll, test_kld, exp(test_nll+test_kld)))
    elif args.task == "decode":
        print("*** Decode ... ***")
        # set random seed
        random.seed(args.seed)
        torch.manual_seed(args.seed)
        if torch.cuda.is_available():
            torch.cuda.manual_seed_all(args.seed)
        beam_searcher = BeamSearch(model, txtfield, args)
        if args.decode_from == 'sample':
            beam_searcher.decode_sample(test_iter, txtfield, args, num_samples=30000)
        else:
            beam_searcher.decode_beam(test_iter, txtfield, args, num_samples=30000)
    else:
        raise ValueError("Unrecognized task: {}".format(args.task))

if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt as e:
        print("[STOP]", e)