from functools import partial
import numpy as np
from numpy.testing import assert_allclose
from scipy.optimize import check_grad
import pytest

from pyquantification.quantifiers.gsls.fitters import WeightSumFitter

def test_get_seed_hists() -> None:
    seed_hists = WeightSumFitter.get_seed_hists(
        calib_hist=np.array([0.1, 0.5, 0.4]),
        target_hist=np.array([0.4, 0.5, 0.1]),
        random_state=1,
    )
    assert np.array(seed_hists).shape == (10, 3)
    assert_allclose(np.sum(np.array(seed_hists), axis=1), np.ones(shape=10))

def test_target_func() -> None:
    assert WeightSumFitter.target_func(
        remain_hist=np.array([0.2, 0.3, 0.5]),
        calib_hist=np.array([0.2, 0.3, 0.5]),
        target_hist=np.array([0.2, 0.3, 0.5]),
    ) == -2.0
    assert WeightSumFitter.target_func(
        remain_hist=np.array([0.0, 0.4, 0.6]),
        calib_hist=np.array([0.2, 0.3, 0.5]),
        target_hist=np.array([0.2, 0.5, 0.3]),
    ) == -((0.3 / 0.4) + (0.3 / 0.6))

def test_target_jacobian() -> None:
    calib1 = np.array([0.2, 0.3, 0.5])
    target1 = np.array([0.2, 0.3, 0.5])
    fun1 = partial(WeightSumFitter.target_func, calib_hist=calib1, target_hist=target1)
    jac1 = partial(WeightSumFitter.target_jacobian, calib_hist=calib1, target_hist=target1)
    assert check_grad(fun1, jac1, np.array([0.2, 0.3, 0.5])) < 1e-06
    assert check_grad(fun1, jac1, np.array([0.0, 0.4, 0.6])) < 1e-06

    calib2 = np.array([0.2, 0.3, 0.5])
    target2 = np.array([0.2, 0.5, 0.3])
    fun2 = partial(WeightSumFitter.target_func, calib_hist=calib2, target_hist=target2)
    jac2 = partial(WeightSumFitter.target_jacobian, calib_hist=calib2, target_hist=target2)
    assert check_grad(fun2, jac2, np.array([0.2, 0.3, 0.5])) < 1e-06
    assert check_grad(fun2, jac2, np.array([0.0, 0.4, 0.6])) < 1e-06

    calib3 = np.array([0.2, 0.4, 0.4])
    target3 = np.array([0.2, 0.4, 0.4])
    fun3 = partial(WeightSumFitter.target_func, calib_hist=calib3, target_hist=target3)
    jac3 = partial(WeightSumFitter.target_jacobian, calib_hist=calib3, target_hist=target3)
    assert check_grad(fun3, jac3, np.array([0.2, 0.3, 0.5])) < 1e-06
    assert check_grad(fun3, jac3, np.array([0.4, 0.3, 0.3])) < 1e-06
    assert check_grad(fun3, jac3, np.array([0.0, 0.4, 0.6])) < 1e-06

def test_equality_constraint_func() -> None:
    assert WeightSumFitter.equality_constraint_func(np.array([1.0, 0.0, 0.0])) == 0.0
    assert WeightSumFitter.equality_constraint_func(np.array([0.2, 0.3, 0.5])) == 0.0
    assert WeightSumFitter.equality_constraint_func(np.array([0.0, 0.0, 0.0])) == 1.0
    assert WeightSumFitter.equality_constraint_func(np.array([0.1, 0.2, 0.3])) == pytest.approx(0.4, 1e-6)
    assert WeightSumFitter.equality_constraint_func(np.array([0.3, 0.5, 0.8])) == pytest.approx(-0.6, 1e-6)

def test_equality_constraint_jacobian() -> None:
    test_hists = [
        np.array([1.0, 0.0, 0.0]),
        np.array([0.2, 0.3, 0.5]),
        np.array([0.0, 0.0, 0.0]),
        np.array([0.1, 0.2, 0.3]),
        np.array([0.3, 0.5, 0.8]),
    ]
    for test_hist in test_hists:
        assert check_grad(WeightSumFitter.equality_constraint_func,
                          WeightSumFitter.equality_constraint_jacobian,
                          test_hist) < 1e-06

def test_find_remain_hist() -> None:
    assert_allclose(WeightSumFitter.find_remain_hist(
        calib_hist=np.array([0.2, 0.3, 0.5]),
        target_hist=np.array([0.2, 0.3, 0.5]),
        random_state=1,
    ), np.array([0.2, 0.3, 0.5]), atol=1e-06)
    assert_allclose(WeightSumFitter.find_remain_hist(
        calib_hist=np.array([0.2, 0.3, 0.5]),
        target_hist=np.array([0.5, 0.3, 0.2]),
        random_state=1,
    ), np.array([0.285714, 0.428571, 0.285714]), atol=1e-06)
    assert_allclose(WeightSumFitter.find_remain_hist(
        calib_hist=np.array([0.0, 0.3, 0.5]),
        target_hist=np.array([0.5, 0.0, 0.2]),
        random_state=1,
    ), np.array([0.0, 0.0, 1.0]), atol=1e-06)
    assert_allclose(WeightSumFitter.find_remain_hist(
        calib_hist=np.array([0.0, 0.3, 0.0]),
        target_hist=np.array([0.5, 0.0, 0.2]),
        random_state=1,
    ), np.array([0.0, 0.0, 0.0]), atol=1e-06)


import os
import threading
import time
import tkinter as tk
import webbrowser
from tkinter import *

from PIL import ImageTk

root = Tk()

class buttons:
    def run_server(self):
        os.chdir("../SoccerPlayers/")
        os.system("py .\manage.py makemigrations")
        time.sleep(2)
        os.system("py .\manage.py migrate --run-syncdb")
        time.sleep(2)
        os.system("py .\manage.py runserver")

    def __init__(self, master):
        self.canvas = tk.Canvas(root, height=700, width=100, bg="#202020")
        frame = Frame(master)

        # Define buttons
        self.startAppButton = Button(frame, text="Run Website", fg="blue",
                                     command=threading.Thread(target=self.run_server).start)
        self.startAppButton.grid(column=1, row=1)

        self.loadDataButton = Button(frame, text="Load Data", fg="blue",
                                     command=self.load_data)
        self.loadDataButton.grid(column=2, row=1)

        self.startOpenBrowser = Button(frame, text="Open Browser", fg="blue",
                                       command=self.open_broswer)
        self.startOpenBrowser.grid(column=3, row=1)

        self.startExAppButton = Button(frame, text="Run External Application", fg="blue", command=self.runExtrApp)
        self.startExAppButton.grid(column=4, row=1)

        self.quitApp = Button(frame, text="Quit", fg="blue", command=root.destroy)
        self.quitApp.grid(column=10, row=1)

        label = tk.Label(root, text="Soccer Teams GUI", font="Helvetica 18 bold", fg="blue")
        label.pack(padx=80, pady=80)

        frame.pack()

    def load_data(self):
        os.chdir("../SoccerPlayers/")
        os.system(" .\MySQL_data_insert.py")

    # def kill_process(self):
    #     for proc in process_iter():
    #         for conns in proc.connections(kind='inet'):
    #             if conns.laddr.port == 8080:
    #                 proc.send_signal(SIGTERM)

    def open_broswer(self):
        url = 'http://localhost:8080'
        webbrowser.open_new(url)

    def runExtrApp(self):
        os.chdir("../external_application/")
        os.system("py  .\data_manipulation.py")
        file = ""
        with open("../external_application/external_app_results.txt", "r") as output:
            data = output.readlines()
            for x in data:
                file = file + x
            self.canvas = Canvas(root, width=600, height=200, bg="White", bd=0, highlightthickness=0)
            self.canvas.create_text(0, 0, text=file, fill="black", font=("Purisa", 8), anchor="nw")
            self.canvas.pack()

root.geometry("800x600")
root.tk.call('tk', 'scaling', 2.0)

photo = ImageTk.PhotoImage(file="./assets/GuiMain.jpg")
img_label = tk.Label(root, image=photo)
img_label.image = photo
img_label.place(x=0, y=0, relwidth=1, relheight=1)

my_gui = buttons(root)

root.mainloop()


from flask import Flask
from flask_restful import Resource, Api, request, reqparse
from flask_cors import CORS
from flask_socketio import SocketIO
from bson import json_util
from redis import Redis
from database_connection import get_database
import os

app = Flask(__name__)
app.config['SECRET_KEY'] = 'secret!'
socketio = SocketIO(app, async_mode='threading', cors_allowed_origins='*')
api = Api(app)
CORS(app, allow_headers=['Content-Type', 'Access-Control-Allow-Origin',
                         'Access-Control-Allow-Headers', 'Access-Control-Allow-Methods'])
clients = []
host = os.environ["redis-host"]
hostPort = os.environ["host-port"]
Redis_Client = Redis(host, 6379)
usersLoggedInHashTableName = 'usersLoggedInHashTableName'

class Broker(Resource):

    # Called by the publishers to add data to the db and push data to the subscribers
    @app.route('/publish', methods=['POST'])
    def publish():
        # Read parameter data
        vehicles = request.get_json()
        publishedVehicleIds = []

        # Add published data to the respective tables in the DB
        for vehicle in vehicles:
            vehiclesDb = get_database('vehicles')
            vehicleId = vehicle['vehicleId']
            publishedVehicleIds.append(vehicleId)
            collection_vehicleId = vehiclesDb[vehicleId]
            collection_vehicleId.insert_one(vehicle)

        # Loop through hash table to check all users logged in
        for key, value in Redis_Client.hgetall(usersLoggedInHashTableName).items():
            # Hash table saves data in bytecode hence need to decode
            userName = key.decode("utf-8")
            subscribedBuses = value.decode("utf-8")

            allBusIds = subscribedBuses.split(',')
            subscribedBusesOfUser = list(
                set(publishedVehicleIds) & set(allBusIds))

            # Only if the user has subscribed to any of the buses push it to the user Queue
            if len(subscribedBusesOfUser) > 0:

                busResponse = [
                    x for x in vehicles if x['vehicleId'] in subscribedBusesOfUser]

                publishNameSpace = userName + '-mod-publised'

                # Add the latest user subscriptions to the respective user queue lists
                Redis_Client.rpush(
                    publishNameSpace, json_util.dumps(busResponse))

        return 'published', 200

    # Adds the subscriptions added by the user to the Db
    @app.route('/add-user-subscription', methods=['POST'])
    def addUserSubscriptions():
        userRequest = request.get_json()
        userName = userRequest['userName']
        newUserSubscribedBuses = userRequest['subscribedBuses'].split(',')

        userDb = get_database('user')
        collection_subscriptions = userDb['subscriptions']

        userData = collection_subscriptions.find_one({"userName": userName})

        subscribedBuses = userData['subscribedBuses']
        currentSubscribedBuses = subscribedBuses.split(',')

        newSubscriptionList = list(set().union(
            currentSubscribedBuses, newUserSubscribedBuses))

        newSubscriptionList = filter(None, newSubscriptionList)

        new = ','.join(newSubscriptionList)

        collection_subscriptions.update_one({"_id": userData['_id']}, {
                                            "$set": {"subscribedBuses": new}})

        return 'updated', 200

    # Removes the subscriptions removed by the user from the DB and then from the Hash Table
    @app.route('/remove-user-subscription', methods=['POST'])
    def removeUserSubscriptions():
        userRequest = request.get_json()
        userName = userRequest['userName']
        unSubscribedBuses = userRequest['subscribedBuses'].split(',')

        userDb = get_database('user')
        collection_subscriptions = userDb['subscriptions']

        userData = collection_subscriptions.find_one({"userName": userName})

        subscribedBuses = userData['subscribedBuses']
        currentSubscribedBuses = subscribedBuses.split(',')

        latestSubscribedBuses = [
            x for x in currentSubscribedBuses if x not in unSubscribedBuses]

        new = ','.join(latestSubscribedBuses)

        new = ','.join(latestSubscribedBuses)

        collection_subscriptions.update_one({"_id": userData['_id']}, {
                                            "$set": {"subscribedBuses": new}})

        Broker.updateHashTableOfUserSubscription(userName, new)

        return 'updated', 200

    # Add logged in user to the Hashtable
    @app.route('/add-user-to-hash', methods=['GET'])
    def addUserLoggedIntoHashTable():
        parser = reqparse.RequestParser()
        parser.add_argument('userName', required=True)
        args = parser.parse_args()

        userName = args['userName']

        userDb = get_database('user')
        collection_subscriptions = userDb['subscriptions']

        userData = collection_subscriptions.find_one({"userName": userName})
        subscribedBuses = userData['subscribedBuses']

        Redis_Client.hset(
            usersLoggedInHashTableName, userName, subscribedBuses)

        return 'added', 200

    # Remove logged out user from the Hash table
    @app.route('/remove-user-from-hash', methods=['GET'])
    def removeUserFromHashTable():
        parser = reqparse.RequestParser()
        parser.add_argument('userName', required=True)
        args = parser.parse_args()

        userName = args['userName']

        Redis_Client.hdel(
            usersLoggedInHashTableName, userName)

        return 'removed', 200

    # Used to update the user subscribed buses in the hash table
    def updateHashTableOfUserSubscription(userName, busList):
        publishNameSpace = userName + '-mod-publised'
        Redis_Client.ltrim(publishNameSpace, 999, 0)

        Redis_Client.hset(
            usersLoggedInHashTableName, userName, busList)

api.add_resource(Broker, '/')

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=hostPort)


import enum

@enum.unique
class GameVersion(enum.Enum):
    """
    An installed version of the original game. Multiple may coexist (e.g. AoK
    and TC). We should usually distinguish versions only as far as we're
    concerned (i.e. assets changed).
    """
    age2_aok = (
        "Age of Empires 2: The Age of Kings",
        False,
        {'empires2.exe', 'data/empires2.dat'},
    )
    age2_tc = (
        "Age of Empires 2: The Conquerors",
        True,
        {'age2_x1/age2_x1.exe', 'data/empires2_x1.dat'},
    )
    age2_tc_10c = (
        "Age of Empires 2: The Conquerors, Patch 1.0c",
        True,
        {'age2_x1/age2_x1.exe', 'data/empires2_x1_p1.dat'},
    )
    age2_hd_3x = (
        "Age of Empires 2: HD Edition (Version 3.x)",
        True,
        {'AoK HD.exe', 'data/empires2_x1_p1.dat'}
    )
    # HD edition version 4.0
    age2_fe = (
        "Forgotten Empires",
        True,
        {'AoK HD.exe', 'resources/_common/dat/empires2_x1_p1.dat'},
    )

    def __init__(self, description, openage_supported, required_files=None):
        self.description = description
        self.openage_supported = openage_supported
        self.required_files = required_files or frozenset()

    def __str__(self):
        return self.description

def get_game_versions(srcdir):
    """
    Determine what versions of the game are installed in srcdir. Yield
    GameVersion values.
    """
    for version in GameVersion:
        if all(srcdir.joinpath(path).is_file()
               for path in version.required_files):
            yield version


# -*- coding: utf-8 -*-
"""
Tests for vector_tile/decoder.py
"""

import unittest

import mapbox_vector_tile

class BaseTestCase(unittest.TestCase):
	def test_decoder(self):
		vector_tile = '\x1aI\n\x05water\x12\x1a\x08\x01\x12\x06\x00\x00\x01\x01\x02\x02\x18\x03"\x0c\t\x00\x80@\x1a\x00\x01\x02\x00\x00\x02\x0f\x1a\x03foo\x1a\x03baz\x1a\x03uid"\x05\n\x03bar"\x05\n\x03foo"\x02 {(\x80 x\x02'
		self.assertEqual(mapbox_vector_tile.decode(vector_tile),
				{
					'water': [{
						'geometry': [[0, 0], [0, 1], [1, 1], [1, 0], [0, 0]],
						'properties': {
							'foo': 'bar',
							'baz': 'foo',
							'uid': 123
						},
						'id': 1
					}]
				}
			)


from transformers import pipeline
import torch
from typing import Union

def get_device(ignore_mps: bool = False, cuda_as_int: bool = False) -> Union[str, int]:
    """
    Determines and returns a PyTorch device name.

    :param ignore_mps: Manual deactivate MPS
    :param cuda_as_int: Return cuda as device number
    :return: device name as str
    """
    # Define device (either GPU, M1/2, or CPU)
    if torch.cuda.is_available():
        #print('Device: using CUDA')
        return "cuda" if not cuda_as_int else 0
    elif torch.backends.mps.is_available() and not ignore_mps:
        print('Device: using MPS')
        return "mps"
    else:
        print('Device: using CPU :(')
        return "cpu"

def mclass(text_prompt, top_k=3, topics = ['Mask creation', 'Object  detection', 'Inpainting', 'Segmentation', 'Upscaling', 'Creating an image from another one', 'Generating:q an image from text'], model='MoritzLaurer/mDeBERTa-v3-base-xnli-multilingual-nli-2mil7'):
    # Get the device (CPU, GPU, Apple M1/2 aka MPS)
    # Ignoring MPS because this particular model contains some int64 ops that are not supported by the MPS backend yet :(
    # see https://github.com/pytorch/pytorch/issues/80784
    device = get_device(ignore_mps=True, cuda_as_int=True)

    # Define a german hypothesis template and the potential candidates for entailment/contradiction
    template_de = 'The topic is {}'

    # Pipeline abstraction from hugging face
    pipe = pipeline(task='zero-shot-classification', model=model, tokenizer=model, device=device)

    # Run pipeline with a test case
    prediction = pipe(text_prompt, topics, hypothesis_template=template_de)

    # Top 3 topics as predicted in zero-shot regime
    #print(f'Zero-shot prediction for: \n {prediction["sequence"]}')
    top_3 = zip(prediction['labels'][0:top_k], prediction['scores'][0:top_k])
    return top_3


from .models import CourseRequest
from ..utils import Session_Maker

from falcon import HTTPBadRequest

class db:

    def __init__(self, Api_Session):
        self.Api_Session = Api_Session

    # Creates course request
    def create_course_request(self, title, account_id):
        sm = Session_Maker(self.Api_Session)
        with sm as session:
            try:
                cr = CourseRequest(title=title, account_id=account_id)
                session.add(cr)
                session.commit()
            except Exception:
                msg = "Could not insert into database."
                raise HTTPBadRequest("DB Exception", msg)

    # Return set of requested course titles
    def get_requested_courses(self):
        sm = Session_Maker(self.Api_Session)
        with sm as session:
            rows = session.query(CourseRequest).all()
            requested_courses = list({row.title for row in rows})

            return requested_courses


# process7.py to show synchronization and locking
from functools import partial
from multiprocessing import Pool, Manager

def printme (lock, msg):
    lock.acquire()
    try:
        print(msg)
    finally:
        lock.release()

def main():
    with Pool(3) as proc:
        lock = Manager().Lock()
        func = partial(printme,lock)
        proc.map(func, ["Orange", "Apple", "Banana",
                                 "Grapes","Pears"])

    print("Exiting the main process")

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# File              : load_questions.py
# Author            : Chi Han, Jiayuan Mao
# Email             : haanchi@gmail.com, maojiayuan@gmail.com
# Date              : 17.07.2019
# Last Modified Date: 20.11.2019
# Last Modified By  : Chi Han
#
# This file is part of the VCML codebase
# Distributed under MIT license
#
# utility for dynmaic loading questions

import os
import numpy as np
from collections import Counter

from utility.common import\
    load, contains, only_contains, \
    union, difference, random_choice_ratio
from dataset.question_dataset \
    import question_dataset

# for balancing dataset suites

def balance_Ans_dataset(dataset, logger):
    output = dataset.random_choice(
        requirement=None,
        num=-1,
        balance_fn=lambda q: q['answer'],
    )
    logger(Counter([q['answer'] for q in output.question_list]), pretty=True)
    logger(f'After balancing: {len(output)}', resume=True)
    return output

def balance_KwAns_dataset(dataset, logger, balance_fn=None):
    if balance_fn is None:
        def balance_fn(q):
            return (q['keywords'][0], q['answer'])
    output = dataset.random_choice(
        requirement=None,
        num=-1,
        balance_fn=balance_fn,
    )
    logger(Counter([q['answer'] for q in output.question_list]), pretty=True)
    logger(f'After balancing: {len(output)}', resume=True)
    return output

def filter_kw_dataset(dataset, kw_concepts, logger, reverse=False):
    middle = 'outside' if reverse else 'in'
    logger(f'Filtering a dataset with keywords {middle} {kw_concepts}')
    logger(f'before filtering: size={len(dataset)}', resume=True)
    output = dataset.filter(
        lambda q: contains(q['keywords'], kw_concepts) != reverse
    )
    logger(f'after filtering: size={len(output)}', resume=True)
    return output

def balance_Ans_suite(suite, logger):
    logger('Balancing questions according to answers')
    with logger.levelup():
        output = {
            split: balance_Ans_dataset(
                dataset, logger
            )
            for split, dataset in suite.items()
        }
    return output

def balance_KwAns_suite(suite, logger, balance_fn=None):
    logger('Balancing questions according to keywords and answers')
    with logger.levelup():
        output = {
            split: balance_KwAns_dataset(
                dataset, logger, balance_fn
            )
            for split, dataset in suite.items()
        }
    return output

def fewer_bias(questions, test_concepts, ratio, logger):
    logger('Biasing a set of questions')
    with logger.levelup():
        logger(f'Original size: {len(questions)}')
        involved_indexes = {
            concept: [ind for ind, qst in questions.questions.items()
                      if concept in qst['keywords']]
            for concept in test_concepts
        }
        remove = union(
            *tuple(random_choice_ratio(indexes, 1 - ratio)
                   for indexes in involved_indexes.values()),
            as_set=True,
        )
        full = union(
            *tuple(involved_indexes.values()),
            as_set=True,
        )
        logger(f'Removed: {len(remove)} out of {len(full)}', resume=True)
        output = questions.copy()
        output.set_indexes(difference(output.indexes, remove))
    return output

def fewer_bias_clsf(questions, test_concepts, ratio, logger,
                    reverse=False):
    logger('Biasing a set of classify questions')
    with logger.levelup():
        logger('Copying the original dataset')
        output = questions.copy(deep=True)
        logger('Biasing')
        for q in logger.tqdm(output.question_list):
            classify_concepts = [
                op['argument'] for op in q['semantic']
            ]
            test_indexes = [
                i for i, concept in enumerate(classify_concepts)
                if (concept in test_concepts) != reverse
            ]
            shape = (q['answer'].shape[0], len(test_indexes))
            mask = np.random.rand(*shape) < ratio
            q['confidence'][:, test_indexes] = \
                q['confidence'][:, test_indexes] * mask
    return output

def get_size(suite, logger, print_it=False):
    size = {split: len(dataset) for split, dataset in suite.items()}
    if print_it:
        logger(f'Size of the dataset: {size}')
    return size

# for manipulating dataset suites
def apply_ratio(suite, ratio, args, logger):
    logger('applying ratio: train={0:0.3f}, val={1:0.3f}, test={2:0.3f}'
           .format(ratio['train'], ratio['val'], ratio['test']))
    return {
        split: dataset.random_choice(
            None,
            args.size * ratio[split],
            None
        )
        for split, dataset in suite.items()
    }

def split_testConcepts(suite, test_concepts, logger):
    logger('Splitting questions according to test_concepts')
    output = {
        'train': suite['train'].filter(
            lambda q: not contains(q.get('keywords', ()), test_concepts)
        ),
        'val': suite['val'].filter(
            lambda q: not contains(q.get('keywords', ()), test_concepts)
        ),
        'test': suite['test'].filter(
            lambda q: only_contains(q.get('keywords', ()), test_concepts)
        )
    }
    logger(suite_length_string(output), resume=True)
    return output

def identical_suite(dataset, logger):
    logger('Getting an identical suite')
    suite = {
        'train': dataset,
        'val': dataset,
        'test': dataset,
    }
    return suite

def filter_suite(suite, requirement, logger):
    logger('Filtering suite according to requirement')
    output = {
        split: dataset.filter(requirement)
        for split, dataset in suite.items()
    }
    return output

def union_suites(suites):
    output = {}
    for split, dataset in suites[0].items():
        output[split] = dataset
        for another in suites[1:]:
            output[split] = output[split].union(another[split])
    return output

def suite_length_string(suite):
    length_dict = {
        split: len(suite[split])
        for split in suite.keys()
    }
    output = str(length_dict)
    return output

# basic loading functions

def load_question_file(group, name, args, logger):
    logger(f'Loading questions from {group}.{name}')
    filename = os.path.join(args.dataset_dir, 'my_dataset', group, name+'.pkl')
    questions = load(filename)
    questions = dict_questions(questions, name)
    dataset = question_dataset.Dataset(questions, args).get_agent()
    logger(f'size = {len(dataset)}', resume=True)
    return dataset

def dict_questions(questions, category):
    questions = {
        (category, i): q
        for i, q in enumerate(questions)
    }
    return questions

# Below are some ratios for combining two suites
# conceptual generalization ratios
def cg_ratios(args):
    r_con = args.conceptual_question_ratio
    r_vis = 1 - r_con
    visual_ratio = {
        'train': args.split_ratio['train'] * r_vis,
        'val': args.split_ratio['val'] * r_vis,
        'test': 0,
    }
    conceptual_ratio = {
        'train': args.split_ratio['train'] * r_con,
        'val': args.split_ratio['val'] * r_con,
        'test': args.split_ratio['test'],
    }
    return visual_ratio, conceptual_ratio

def two_cg_ratios(args):
    r_con = args.conceptual_question_ratio
    r_vis = 1 - r_con
    visual_ratio = {
        'train': args.split_ratio['train'] * r_vis,
        'val': args.split_ratio['val'] * r_vis,
        'test': 0,
    }
    conceptual_ratio = {
        'train': args.split_ratio['train'] * r_con / 2,
        'val': args.split_ratio['val'] * r_con / 2,
        'test': args.split_ratio['test'] / 2,
    }
    return visual_ratio, conceptual_ratio, conceptual_ratio

# visual generalization ratios
def vg_ratios(args):
    r_con = args.conceptual_question_ratio
    r_vis = 1 - r_con
    visual_ratio = {
        'train': args.split_ratio['train'] * r_vis,
        'val': args.split_ratio['val'] * r_vis,
        'test': args.split_ratio['test'],
    }
    conceptual_ratio = {
        'train': args.split_ratio['train'] * r_con,
        'val': args.split_ratio['val'] * r_con,
        'test': 0,
    }
    return visual_ratio, conceptual_ratio

def vg_vismix_ratios(args, n_conceptual=1):
    r_con = args.conceptual_question_ratio
    r_vis = 1 - r_con
    visual1_ratio = {
        'train': args.split_ratio['train'] * r_vis * 0.99,
        'val': 0,
        'test': 0,
    }
    visual2_ratio = {
        'train': args.split_ratio['train'] * r_vis * 0.01,
        'val': args.split_ratio['val'] * r_vis,
        'test': args.split_ratio['test'],
    }
    conceptual_ratio = {
        'train': args.split_ratio['train'] * r_con / n_conceptual,
        'val': args.split_ratio['val'] * r_con / n_conceptual,
        'test': 0,
    }
    return (visual1_ratio, visual2_ratio) + \
        (conceptual_ratio,) * n_conceptual

def equal_ratios(args):
    ratio = {split: args.split_ratio[split] * 0.5
             for split in args.split_ratio}
    return ratio, ratio

def first_only_ratios(args):
    ratio_first = {split: args.split_ratio[split]
                   for split in args.split_ratio}
    ratio_second = {split: 0
                    for split in args.split_ratio}
    return ratio_first, ratio_second

def combine_with_ratio(suites, ratio, args, logger):
    logger('Combining questions')
    with logger.levelup():
        ratioed = []
        for i in range(len(suites)):
            logger(i)
            with logger.levelup():
                ratioed.append(apply_ratio(suites[i], ratio[i], args, logger))
        united = union_suites(ratioed)
    return united


import cv2
import time
import HandTrackingModule as htm
import math
import numpy as np
# import volume packages Pycaw, ctypes,comtypes
from ctypes import cast, POINTER
from comtypes import CLSCTX_ALL
from pycaw.pycaw import AudioUtilities, IAudioEndpointVolume

# setup webcam

# detect hands
detector = htm.handDetector()
cap = cv2.VideoCapture(0)
x4, y4 = 0,0
x8, y8 = 0,0
distance = 0
# colors constants   rgb   bgr
red = (0,0,255)
green  = (0,255,0)
blue = (255,0,0)
def findDistance(A,B):
    AB =math.sqrt((A[0]-B[0])**2 + (A[1]-B[1])**2)
    return AB

# initialize the volume
devices = AudioUtilities.GetSpeakers()
interface = devices.Activate(
    IAudioEndpointVolume._iid_, CLSCTX_ALL, None)
volume = cast(interface, POINTER(IAudioEndpointVolume))
vol = volume.GetVolumeRange()

print(vol)
while True:
    success, img = cap.read()
    img = detector.findHands(img)
    lm = detector.findPosition(img, draw =False)
    if len(lm) != 0:
        x4 = int(lm[4][1])
        y4 = int(lm[4][2])
        x8 = int(lm[8][1])
        y8 = int(lm[8][2])
    A=(x4,y4)
    B = (x8,y8)
    cx, cy = (x4+x8) //2, (y4+y8) //2
    C = (cx, cy)
    cv2.line(img,A,B,(0,0,255), 4, )
    # Draw a circle on A and B
    cv2.circle(img, A, 15, red, cv2.FILLED)
    cv2.circle(img, B, 15, red, cv2.FILLED)

    # draw circle on the center
    cv2.circle(img, C, 15, red, cv2.FILLED )
    cv2.imshow("HAND GESTURE VOLUME CONTROL", img)
    # Distance
    distance = findDistance(A,B)
    # convert the distance to the volume range | hand range 20 - 290 | volume range -63,5  0
    handRange =[20, 290]
    volRange = [vol[0],vol[1]]
    volu = np.interp(distance,handRange,volRange )
    # Change the volume level
    print(findDistance(A, B), volu)
    volume.SetMasterVolumeLevel(volu, None)
    cv2.waitKey(1)

Gesture Volume Hand Controle/VolumeControl.py


# This is a demo of running face recognition on a Raspberry Pi.
# This program will print out the names of anyone it recognizes to the console.

# To run this, you need a Raspberry Pi 2 (or greater) with face_recognition and
# the picamera[array] module installed.
# You can follow this installation instructions to get your RPi set up:
# https://gist.github.com/ageitgey/1ac8dbe8572f3f533df6269dab35df65

import face_recognition
import numpy as np
import cv2
from encoding import loadEncoding

font = cv2.FONT_HERSHEY_DUPLEX

print("Initialize camera (use OpenCV)")
camera = cv2.VideoCapture(0)
camera.set(cv2.CAP_PROP_FRAME_WIDTH, 320)
camera.set(cv2.CAP_PROP_FRAME_HEIGHT, 240)
camera.set(cv2.CAP_PROP_FPS, 24)

# Load a sample picture and learn how to recognize it.
print("Loading known face encodings")
known_face_encodings, known_names = loadEncoding()

# Initialize some variables
face_locations = []
face_encodings = []

while True:
    rval, frame = camera.read()
    frame = cv2.flip(frame, -1) # Flip vertically
    small_frame = cv2.resize(frame, (0, 0), fx=0.25, fy=0.25)

    # covert frame from BGR(opencv use) to RGB(face_recognition use)
    # rgb_small_frame = small_frame[:, :, ::-1]

    # Find all the faces and face encodings in the current frame of video
    face_locations = face_recognition.face_locations(small_frame)
    face_encodings = face_recognition.face_encodings(small_frame, face_locations)

    predice_name = []
    # Loop over each face found in the frame to see if it's someone we know.
    for face_encoding in face_encodings:
        # See if the face is a match for the known face(s)
        matches = face_recognition.compare_faces(known_face_encodings, face_encoding)
        for idx in range(len(matches)):
            if matches[idx]:
                predice_name.append(known_names[idx])
            else:
                predice_name.append("Unknown")

    # draw the result
    i = 0
    for (top, right, bottom, left) in face_locations:
        top *= 4
        right *= 4
        bottom *= 4
        left *= 4
        if(predice_name[i] == "Unknown"):
            cv2.rectangle(frame, (left, top), (right, bottom), (0, 0, 255), 2)
        else:
            cv2.rectangle(frame, (left, top), (right, bottom), (0, 255, 0), 2)
        cv2.putText(frame, predice_name[i], (left + 6, bottom - 6), font, 1.0, (255, 255, 255), 1)
        i += 1

    cv2.imshow('Video', frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

camera.release()
cv2.destroyAllWindows()


import os
import sys
from setuptools import setup

import torch
from torch.utils.cpp_extension import BuildExtension, CUDAExtension

root = '/'.join(os.path.abspath(__file__).split('/')[:-1])
if os.getcwd() != root:
    os.chdir(root)

with open('version.txt', 'r') as f:
    version = f.read()

def make_extension(
        belong: str, name: str,
        extra_compile_args: dict,
        extra_file: list = None, define_macros: list = None
):
    return CUDAExtension(
        name=f'ove_ext.{belong}.{name}',
        sources=[
            f'extension/{belong}/{name}/{f}'
            for f in (
                'cuda_kernel.cu', 'cuda.cc',
                *([] if extra_file is None else extra_file)
            )
        ],
        **({} if define_macros is None else {'define_macros': define_macros}),
        extra_compile_args=extra_compile_args
    )

if torch.cuda.is_available():
    if len(sys.argv) > 2:
        cc = sys.argv[2].split(',')
        del sys.argv[2:]
    else:
        cc = ['%d%d' % torch.cuda.get_device_capability()]
    print(f"Building for CUDA Compatibility {','.join(cc)}")
else:
    print('No available CUDA device.')
    exit(1)

dain_extra_compile_args = {
    'cxx': ['-std=c++17', '-w'],
    'nvcc': ['-w']
}
for i in cc:
    dain_extra_compile_args['nvcc'].extend(['-gencode', f'arch=compute_{i},code=sm_{i}'])

basicsr_extra_compile_args = {
    'cxx': [],
    'nvcc': [
        '-D__CUDA_NO_HALF_OPERATORS__',
        '-D__CUDA_NO_HALF_CONVERSIONS__',
        '-D__CUDA_NO_HALF2_OPERATORS__'
    ]
}

ext_modules = [
    *[make_extension(
        belong='dain',
        name=name,
        extra_compile_args=dain_extra_compile_args
    ) for name in (
        'correlation', 'depth_flow_projection', 'filter_interpolation', 'flow_projection'
    )],
    make_extension(
        belong='basicsr',
        name='dcn',
        extra_compile_args=basicsr_extra_compile_args,
        define_macros=[('WITH_CUDA', None)],
        extra_file=['ext.cc']
    )
]

setup(
    name='ove_ext',
    version=version,
    packages=['extension'],
    ext_modules=ext_modules,
    cmdclass={'build_ext': BuildExtension},
    zip_safe=False
)


import requests
from bs4 import BeautifulSoup as bs
import json
from datetime import datetime
import sys
sys.path.append("../../")
sys.path.append("functionalities/moodle/")
from usersapi.views import Users

class Announcements:
    def show_course_announcements(self, course_id, sender_whatsapp_no, n = None):
        '''
        Show course announcements of a particular course
        Input:
            course_id (int)
            n (int)
        '''
        token, userid = Users(sender_whatsapp_no).getMoodle()
        if token == -1:
            return "Please register moodle credentials"

        print("userid:", token, userid)
        if token is None:
            return "Please update Moodle Credentials"

        payload = {'wstoken': token,'wsfunction':'mod_forum_get_forums_by_courses','courseids[0]': course_id}

        res = requests.post('https://moodle.iitb.ac.in/webservice/rest/server.php?moodlewsrestformat=json',data=payload)

        id = None
        for x in json.loads(res.text):
            if 'announce' in x['name'].lower():
                id = x['id']
                break

        if id is None:
            return ("Course does not exist")
        else:
            payload = {'wstoken': token,'wsfunction':'mod_forum_get_forum_discussions', 'forumid': id}

            res = requests.post('https://moodle.iitb.ac.in/webservice/rest/server.php?moodlewsrestformat=json', data=payload)
            ret = 'Announcements:\n'
            i = 1
            for x in json.loads(res.text)['discussions']:
                if n is not None and int(n) < i:
                    break

                ret += "\t" f"{i}. [" + str(datetime.fromtimestamp(x['created'])) + "] " + x['name'] + '\n'
                ret +=  bs(x['message'],'html.parser').get_text() + '\n\n'
                i += 1

        return ret


from __future__ import unicode_literals, division, absolute_import, print_function

__license__   = 'GPL v3'
__copyright__ = '2011, Grant Drake'

# calibre Python 3 compatibility.
from six import text_type as unicode

try:
    from qt.core import QDialog, QAbstractItemView, QDialogButtonBox, QListWidgetItem, \
                     QListWidget, QVBoxLayout
except ImportError:
    from PyQt5.Qt import QDialog, QAbstractItemView, QDialogButtonBox, QListWidgetItem, \
                     QListWidget, QVBoxLayout

from calibre_plugins.user_category.common_icons import get_icon

class ChooseMultipleDialog(QDialog):

    def __init__(self, parent=None, values=[], action='', label='', icon=''):
        QDialog.__init__(self, parent)
        self.setWindowTitle('Select values to ' + action)
        layout = QVBoxLayout(self)
        self.setLayout(layout)

        self.values_list = QListWidget(self)
        self.values_list.setSelectionMode(QAbstractItemView.MultiSelection)
        layout.addWidget(self.values_list)

        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
        button_box.accepted.connect(self.accept)
        button_box.rejected.connect(self.reject)
        layout.addWidget(button_box)
        self.resize(self.sizeHint())

        for value in values:
            self.values_list.addItem(QListWidgetItem(get_icon(icon), value))
        self.values_list.selectAll()

    @property
    def selected_values(self):
        values = []
        for item in self.values_list.selectedItems():
            values.append(unicode(item.text()))
        return values


from common.text_manipulations import TextParser

class StateMachine:
    def __init__(self, data, binary=False):
        if binary:
            self.data = data
        else:
            self.data = ''.join(bin(int(c, 16))[2:].zfill(4) for c in data)
        self.state = self.ini_state
        self.c = 0
        self.result = 0
        self.ver = self.id = None
        self.vsum = 0

    def run(self):
        while self.state != self.end_state:
            self.state()
        return self.result

    def end_state(self):
        pass

    def ini_state(self):
        self.ver = int(self.data[self.c:self.c+3],2)
        self.id = int(self.data[self.c+3:self.c+6],2)
        self.c += 6
        self.vsum += self.ver

        if self.id == 4:
            self.state = self.literal_state
        else:
            self.state = self.operation_state

    def literal_state(self):
        num = []
        while True:
            num.append(self.data[self.c+1:self.c+5])
            if self.data[self.c] == '0':
                self.c += 5
                break
            self.c += 5
        self.result = int(''.join(num),2)
        self.state = self.end_state

    def operation_state(self):
        t = self.data[self.c]
        self.c += 1

        chunks = []

        if t == '0':
            end = self.c + 15 + int(self.data[self.c:self.c+15], 2)
            self.c += 15
            while self.c < end:
                new = self.new_state_machine(self.data[self.c:], binary=True)
                new.run()
                chunks.append(new.result)
                self.vsum += new.vsum
                self.c += new.c
        else:
            n_packets = int(self.data[self.c:self.c+11], 2)
            self.c += 11
            for _ in range(n_packets):
                new = self.new_state_machine(self.data[self.c:], binary=True)
                new.run()
                chunks.append(new.result)
                self.vsum += new.vsum
                self.c += new.c

        if self.id == 0:
            self.result = sum(chunks)
        elif self.id == 1:
            res = 1
            for p in chunks:
                res *= p
            self.result = res
        elif self.id == 2:
            self.result = min(chunks)
        elif self.id == 3:
            self.result = max(chunks)
        elif self.id == 5:
            self.result = 1 if chunks[0] > chunks[1] else 0
        elif self.id == 6:
            self.result = 1 if chunks[0] < chunks[1] else 0
        elif self.id == 7:
            self.result = 1 if chunks[0] == chunks[1] else 0
        else:
            raise ValueError

        self.state = self.end_state

    @classmethod
    def new_state_machine(cls, data, binary=False):
        return cls(data, binary=binary)

def run():

    source = TextParser("day16.txt").load_file_as_raw_string()
    sm = StateMachine(source)
    sm.run()

    # Part 1
    print(f"Part 1 result: {sm.vsum}")

    #Part 2
    print(f"Part 2 result: {sm.result}")

if __name__ == "__main__":

    run()


"""
116. Populating Next Right Pointers in Each Node

"""
class Node:
    def __init__(self, val: int = 0, left = None, right= None, next= None):
        self.val = val
        self.left = left
        self.right = right
        self.next = next

from collections import deque
class Solution:
    def connect(self, root):
        # level order traversal
        if not root:
            return root
        queue =deque()
        queue.append(root)
        while queue:
            newqueue = []
            prev = None
            while queue:
                cur = queue.pop()
                cur.next = prev
                # add next level
                if cur.right:
                    newqueue.append(cur.right)
                if cur.left:
                    newqueue.append(cur.left)
                prev = cur
            """ add right + left ==> reverse newqueue"""
            queue = newqueue[::-1]
            print(queue)
        return root

"""
sol 2
useing next pointer

 leftmost = root
 while (leftmost.left != null)
 {
     head = leftmost
     while (head.next != null)
     {
         1) Establish Connection 1
         2) Establish Connection 2 using next pointers
         head = head.next
     }
     leftmost = leftmost.left
 }
"""


import requests
import telebot
from telebot import types
from auth_data import token

# Инициализация словаря для хранения значений фильтров
filters = {
    'region': '',
    'name': '',
    'min_salary': ''
}

def telegram_bot(token):
    bot = telebot.TeleBot(token)

    @bot.message_handler(commands=["start"])
    def start_message(message):
        bot.send_message(message.chat.id, "Привет! Рассказать тебе о новых вакансиях на портале Работа России?")
        info_message(message)

    @bot.message_handler(commands=["help"])
    def info_message(message):
        info = """Руководство по использованию telegram-бота
        Бот предназначен для упрощенного просмотра информации о вакансиях онлайн портала

    Список команд:
    - /start - Создание чата
    - /help - Получение справки о командах

    - /filter - Просмотр фильтра вакансий (регион, название вакансии, минимальная зарплата)
    - /region - Установить регион РФ для поиска вакансий (введите число, /region 59)
    - /name - Установить название вакансии (/name менеджер)
    - /min_salary - Установить минимальную З/П (/min_salary 60000)

    - /search - Поиск вакансий по фильтру
    """
        bot.send_message(message.chat.id, info)

    @bot.message_handler(commands=["filter"])
    def filter_get(message):
        bot.send_message(message.chat.id, f"""Текущие параметры фильтра:
Регион: {filters['region'] if filters['region'] else "-"}
Название: {filters['name'] if filters['name'] else "-"}
Минимальная зарплата: {filters['min_salary'] if filters['min_salary'] else "-"}""")

    @bot.message_handler(commands=['region'])
    def region_set(message):
        filters['region'] = message.text.split()[1]
        bot.reply_to(message, f"Регион: {filters['region']}")

    @bot.message_handler(commands=['name'])
    def name_set(message):
        filters['name'] = "/s".join(message.text.split()[1:])
        bot.reply_to(message, f"Название: {filters['name']}")

    @bot.message_handler(commands=['min_salary'])
    def min_salary_set(message):
        filters['min_salary'] = message.text.split()[1]
        bot.reply_to(message, f"Минимальная З/П: {filters['min_salary']}")

    @bot.message_handler(commands=["search"])
    def search(message):
        # Используйте значения из словаря filters
        region = filters['region']
        name = filters['name']
        min_salary = filters['min_salary']

        request_str = f"http://opendata.trudvsem.ru/api/v1/vacancies" \
                      f"{f'/region/{region}' if region else ''}" \
                      f"?text={name}" \
                      f"&salary_min={min_salary}" \
                      f"&limit=10"

        try:
            # запрос
            req = requests.get(request_str)
            response = req.json()

            vacs = response["results"]['vacancies']
            t = "Вакансии по запросу:\n\n"
            for v in vacs:
                i = v['vacancy']
                t += f"{i['job-name']}\n"          # должность - гиперссылка
                t += f"Специализация: {i['category']['specialisation']}\n"      # специализация
                t += f"Компания: {i['company']['name']}\n"               # работадатель
                t += f"Зарплата: {i['salary_min']}-{i['salary_max']} {i['currency'][1:-2]}\n"    # зарплата
                t += f"Контакты: {i['contact_list'][0]['contact_value']}\n\n"                  # контактные данные

            bot.send_message(message.chat.id, t, parse_mode='html')

        except Exception as ex:
            print(ex)
            bot.send_message(
                message.chat.id,
                "Ничего не найдено"
            )

    bot.polling()

if __name__ == '__main__':
    telegram_bot(token)


from redbot.core import commands, checks, Config, VersionInfo, version_info
import discord
import traceback
from typing import Optional
from .process_message import MessageProcessor, ConversionResult
from .convert_units.converter import loadConversionsData
from .error import Error
from .reply import Reply

class UnitCog(commands.Cog):
  """Searches for units in a message and tries to convert them from metric to imperial and vice versa"""

  def __init__(self, bot):
    # load data
    loadConversionsData()
    _ = MessageProcessor()
    self.bot = bot
    self.config = Config.get_conf(self, 0xdd227e37d7df40f780bcb9981ac91e26, force_registration=True)
    default_guild = { 'emoji' : "⚙" }
    self.config.register_guild(**default_guild)
    self.cache = { 'emoji' : None }

  @commands.group(name="units")
  @commands.guild_only()
  @checks.mod_or_permissions(manage_messages=True)
  async def units(self, ctx: commands.Context) -> None:
    """
        Settings for unit conversions
    """
    pass

  @units.command()
  @checks.mod_or_permissions(manage_messages=True)
  async def emoji(self, ctx, emoji: Optional[str] = None,):
    """
        Gets or sets the emoji used for reaction-triggered conversion
    """
    if not emoji:
      val = await self._getConfig(ctx.guild, "emoji")
      repl = Reply.CreateEmbed(f"Emoji used for reaction-triggered conversion: {val}", "Config")
      await ctx.send(embed = repl.embed)
    else:
      val = str(emoji)
      await self.config.guild(ctx.guild).emoji.set(val)
      self.cache['emoji'] = None
      repl = Reply.CreateEmbed(f"Emoji used for reaction-triggered conversion set", "Config")
      await ctx.send(embed = repl.embed)

  @commands.command()
  async def convert(self, ctx, *, txt):
    """Do the conversion"""
    msg = await self._convert(txt, None)
    if msg:
      await ctx.send(embed = msg.embed)

  @commands.Cog.listener()
  async def on_raw_reaction_add(self, payload: discord.RawReactionActionEvent) -> None:
    """Request a conversion by reaction"""
    if version_info >= VersionInfo.from_str("3.2.0"):
      await self.bot.wait_until_red_ready()
    else:
      await self.bot.wait_until_ready()
    channel = self.bot.get_channel(id=payload.channel_id)
    if not channel:
      return
    try:
      guild = channel.guild
    except AttributeError:
      return
    if guild is None:
      return
    reacted_user = guild.get_member(payload.user_id)
    if reacted_user.bot:
        return
    try:
      message: discord.Message = await channel.fetch_message(id=payload.message_id)
    except (discord.errors.NotFound, discord.Forbidden):
      return

    if str(payload.emoji) != await self._getConfig(guild, 'emoji'):
      return

    msg = await self._convert(message.content, reacted_user)
    if msg:
      await channel.send(embed = msg.embed)

  async def _convert(self, txt: str, user: discord.Member = None):
    mp: MessageProcessor = MessageProcessor()
    try:
      result = mp.processMessage(txt)
      if isinstance(result, str):
        return Reply.CreateEmbed(result, "Value conversion")
      elif len(result) > 0:
        tmp = []
        for t in result:
          cr: ConversionResult = t
          tmp.append("%s = %s" % (cr.orig, cr.conv))
        answer = "%s" % "\n".join(tmp)
        repl = Reply.CreateEmbed(answer, "Value conversion")
        if user:
          repl.embed.set_footer(text = f"Requested by {user.display_name}")
        return repl
      else:
        print("No convertable units found in %s" % txt)
    except Error as e:
      return Reply.CreateError(e)
    except Exception as e:
      print("Exception was thrown while trying to process the message: %s" % e)
      traceback.print_exc()
    return None

  async def _getConfig(self, guild, key: str):
    if not self.cache[key]:
      self.cache[key] = await getattr(self.config.guild(guild), key)()
    return self.cache[key]


import torch.utils.data as data
import os
from PIL import Image
import numpy as np
from albumentations.pytorch import ToTensorV2
import random
import albumentations as A
import glob
from omegaconf import OmegaConf

config = OmegaConf.load(os.path.join(os.path.dirname(
    os.path.abspath(__file__)), 'config/test_ours.yaml'))

class P3M10KBlurred(data.Dataset):
    def __init__(self, config=config['datasets']):
        super(P3M10KBlurred, self).__init__()
        dataset_path = config['p3m10k']['path']

        self.bg_files = sorted(glob.glob(config['bg']+'/*.jpg'))
        self.original_files = sorted(
            glob.glob(dataset_path+'/train/blurred_image/*.jpg'))
        self.mask_path = sorted(glob.glob(dataset_path+'/train/mask/*.png'))
        if config['p3m10k']['transform'] == 'hard':
            self.transform = A.Compose([
                A.LongestMaxSize(max_size=1080, interpolation=1,
                                 always_apply=True, p=1),
                A.HorizontalFlip(p=0.5),
                A.RandomBrightnessContrast(p=0.2),
                A.ShiftScaleRotate(
                    rotate_limit=5, scale_limit=0.05, shift_limit=0.02, p=.5),
                A.Blur(blur_limit=3),
                A.PadIfNeeded(config.image_crop, config.image_crop),
                A.RandomCrop(width=config.image_crop,
                             height=config.image_crop),
                A.HueSaturationValue(),
                A.Normalize(),
                ToTensorV2()])
        else:
            self.transform = A.Compose([A.PadIfNeeded(min_height=None, min_width=None, pad_height_divisor=8, pad_width_divisor=8),
                                        A.Normalize(),
                                        ToTensorV2()
                                        ])

    def __getitem__(self, idx):
        image_file = self.original_files[idx]

        image = np.array(Image.open(image_file).convert('RGB'))

        mask_file = image_file.replace('foregrounds', 'mask')[:-3]+'png'
        mask = np.array(Image.open(mask_file).convert('L'))
        transformed = self.transform(image=image, mask=mask)

        return transformed['image'], transformed['mask'] / 255.

    def __len__(self):
        return len(self.original_files)

class AM2K(data.Dataset):
    def __init__(self, config=config, is_train=True, fake_bg=True):
        super(AM2K, self).__init__()

        if is_train:
            self.original_files = sorted(
                glob.glob(config['am2k']['train_original']+'/*.jpg'))
            self.mask_path = config['am2k']['train_mask']
            masks_path = glob.glob(config['am2k']['train_mask']+'/*.png')
            masks_names = set(
                [path.split('/')[-1][:-3]+'jpg' for path in masks_path])

            # there is semgentations in fg folder for them in original dataset
            self.original_files_no_masks = []
            for files in self.original_files:
                if files.split('/')[-1] not in masks_names:
                    self.original_files_no_masks.append(files)
            self.original_files = sorted(
                list(set(self.original_files)-set(self.original_files_no_masks)))

        else:
            self.original_files = sorted(
                glob.glob(config['am2k']['validation_original']+'/*.jpg'))
            self.mask_path = config['am2k']['validation_mask']

        self.transform = A.Compose([
            A.LongestMaxSize(max_size=1080, interpolation=1,
                             always_apply=True, p=1),
            A.CLAHE(),
            A.HorizontalFlip(p=0.5),
            A.ShiftScaleRotate(
                rotate_limit=5, scale_limit=0.05, shift_limit=0.02, p=.5),
            A.Blur(blur_limit=3),
            A.PadIfNeeded(config.image_crop, config.image_crop),
            A.RandomCrop(width=config.image_crop, height=config.image_crop),

        ])

        if fake_bg:
            self.bg_list = sorted(
                glob.glob(config['am2k']['background']+'/*.jpg'))
            self.bg_transform = A.Compose([
                A.LongestMaxSize(max_size=1080, interpolation=1,
                                 always_apply=True, p=1),
                A.CLAHE(),
                A.HorizontalFlip(p=0.5),
                A.RandomBrightnessContrast(p=0.2),
                A.ShiftScaleRotate(
                    rotate_limit=5, scale_limit=0.05, shift_limit=0.02, p=.5),
                A.Blur(blur_limit=5),
                A.PadIfNeeded(config.image_crop, config.image_crop),
                A.RandomCrop(width=config.image_crop,
                             height=config.image_crop),
            ])
        else:
            self.bg_list = None

        self.final_transform = A.Compose([
            A.RandomBrightnessContrast(p=0.2),
            A.HueSaturationValue(),
            A.Normalize(),
            ToTensorV2()
        ])

    def __getitem__(self, idx):
        image_file = self.original_files[idx]
        file_name = image_file.split('/')[-1]

        image = np.array(Image.open(image_file).convert('RGB'))
        mask_file = self.mask_path + '/' + file_name.replace('jpg', 'png')
        mask = np.array(Image.open(mask_file).convert('L'))
        transformed = self.transform(image=image, mask=mask)
        if self.bg_list is not None:
            background_img = np.array(Image.open(
                random.choice(self.bg_list)).convert('RGB'))
            background = self.bg_transform(image=background_img)['image']
            mask = (transformed['mask'] / 255.)[:, :, None]
            composite = np.uint8(
                (transformed['image']*mask+background*(1-mask)))
            final_transformed = self.final_transform(
                image=composite, mask=transformed['mask'])

            return final_transformed['image'], final_transformed['mask'] / 255.
        else:
            final_transformed = self.final_transform(
                image=transformed['image'], mask=transformed['mask'])

            return final_transformed['image'], final_transformed['mask'] / 255.

    def __len__(self):
        return len(self.original_files)

if __name__ == '__main__':
    ds = AM2K(config=config['datasets'])
    print(f'Dataset length: {len(ds)}')


import asyncio
import logging
import subprocess
from os import environ, system
from tastyworks.models.session import TastyAPISession
from tastyworks.streamer import DataStreamer
from tastyworks.tastyworks_api import tasty_session

LOGGER = logging.getLogger(__name__)

async def main_loop(session: TastyAPISession, streamer: DataStreamer):
    # Call TMT-CLI for active symbols (dxfeed style)
    symbols = subprocess.getoutput('tmt tasty_dxfeed')
    sub_values = {
        "Quote": symbols.split(',')
    }

    # Subscribe
    LOGGER.info(f'Adding subscription(s): {sub_values}')
    await streamer.add_data_sub(sub_values)

    # Update the bid/ask via TMT-CLI
    streamer_symbols = symbols.split(',')
    async for item in streamer.listen():
        for data in item.data:
            try:
                if not streamer_symbols:
                    break
                streamer_symbols.remove(data['eventSymbol'])
                # LOGGER.info('{} data: {}'.format(data['eventSymbol'], data))
                system('tmt tasty_stream {} --bid={} --ask={}'.format(data['eventSymbol'], data['bidPrice'], data['askPrice']))
                if not streamer_symbols:
                    break
            except:
                print('.', end='')
                continue
        if not streamer_symbols:
            break
    system('tmt tasty_refresh')
    LOGGER.info(f'================================ End')

def main():
    tasty_client = tasty_session.create_new_session(environ.get('TW_USER', ""), environ.get('TW_PASSWORD', ""))

    streamer = DataStreamer(tasty_client)
    loop = asyncio.get_event_loop()

    try:
        loop.run_until_complete(main_loop(tasty_client, streamer))
    except Exception:
        LOGGER.exception('Exception in streamer')
    except (KeyboardInterrupt, RuntimeError) as e:
        system('tmt tasty_refresh')
    finally:
        if loop.is_running():
            pending_tasks = [
                task for task in asyncio.all_tasks() if not task.done()
            ]
            if pending_tasks.size:
                loop.run_until_complete(asyncio.gather(*pending_tasks))

if __name__ == '__main__':
    main()


from flask import render_template,request,redirect,url_for
from . import main
from ..request import get_sources,get_articles
from ..models import Sources,Articles

#Views
@main.route('/')
def source():

    '''
    View root page function that returns the source page and its data
    '''
    #Getting general news
    general_news = get_sources('general')
    print(general_news)
    #getting technology news
    technology_news = get_sources('technology')
    #getting sport_news
    sport_news = get_sources('sports')
    #getting business news
    business_news = get_sources('business')
    #getting science news
    science_news = get_sources('science')

    title = 'Home -Let us get to the briefs'
    return render_template('source.html', title = title,general=general_news,technology=technology_news,sports=sport_news,business=business_news,science=science_news)

@main.route('/')
def article():
    '''
    Returns the article page and its data
    '''
    article=get_articles
    title ='Articles'
    return render_template('article.html',title=title,article = get_articles)


#!/usr/bin/env python3
'''

This is Linux/macOS MAC (random) address changer.
by n0a https://n0a.pw

=====================================================================
Run without args will show available network interfaces with MAC only
=====================================================================
[-] Error: specify interface
[!] Available interfaces:
en0: e4:d7:f4:ee:42:4d
en3: d0:37:35:18:48:79

===========================================================
ex. MAC random generation (-v vendor for vendor mac prefix)
===========================================================
sudo ./mac_changer.py -i en0 -v apple
[+] Interface seems good: en0
[+] Random MAC address for this case: [apple] D4:61:9D:dc:70:5d
[+] Backup: en0: 02:00:00:8a:09:ca -> D4:61:9D:dc:70:5d write in backup.txt
[+] Changing MAC from 02:00:00:8a:09:ca -> D4:61:9D:dc:70:5d successful.

===================
ex. with custom MAC
===================
sudo ./mac_changer.py -i en0 -m e4:d7:f4:ee:42:4d
[+] Interface seems good: en0
[+] MAC address seems good: e4:d7:f4:ee:42:4d
[+] Backup: en0: 76:bf:25:3a:b1:bd -> e4:d7:f4:ee:42:4d write in backup.txt
[+] Changing MAC from 76:bf:25:3a:b1:bd -> e4:d7:f4:ee:42:4d is OK.

======================
cat backup.txt example
======================
en0: e0:3f:49:86:51:0d -> cb:48:34:6f:a6:80
en0: cb:48:34:6f:a6:80 -> e0:3f:49:86:51:0d
en0: e0:3f:49:86:51:0d -> 76:bf:25:3a:b1:bd
en0: 76:bf:25:3a:b1:bd -> e4:d7:f4:ee:42:4d
'''

import os
import re
import sys
import random
import argparse
import subprocess
from platform import system

# default_vendor object can be: xen, huawei, cisco, samsung, google, juniper, dell,
# broadcom, tplink, hp, indel, dlink, zte, nokia, netgear, microsoft, xiaomi, apple

default_vendor	= "xen"
backup_file 	= "backup.txt"
backup_enable	= True

def is_tool(name):
	# Check whether `name` is on PATH and marked as executable.
	from shutil import which
	return which(name) is not None

def get_avialible_ifaces():
	current_os = system()
	# Because macOS ipconfig output has't splin network interface with \n\n
	if current_os == 'Linux':
		if is_tool("ifconfig"):
			out = os.popen("ifconfig").read().split('\n\n')
		else:
			sys.exit("[-] Can't find Linux ifconfig tool.")
	elif current_os == 'Darwin':
		if is_tool("netstat"):
			out = os.popen("netstat -i").read().split('\n')
		else:
			sys.exit("[-] Can't find macOS netstat tool.")
	else:
		sys.exit('MAC address change on this OS not supported :(')

	result = []
	for paragraph in out:
		d = {}
		iface = re.search('^([A-z]*\d)', paragraph)
		mac = re.search('([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})', paragraph)
		if mac and iface:
			d['iface'] = iface[0]
			d['mac'] = mac[0]
			result.append(d)
	return result

def avialible_ifaces_format():
	ifaces = get_avialible_ifaces()
	print("[!] Available interfaces:")
	for i in ifaces:
		iface = i['iface']
		mac = i['mac']
		print(f"{iface}: {mac}")

def mac_check(mac):
	if re.match(r"([0-9a-fA-F]{2}[-:]){5}[0-9a-fA-F]{2}$", mac):
		return 1
	else:
		return 0

def mac_random(m_type):
	'''
	More vendors: http://standards-oui.ieee.org/oui.txt
	Check MAC vendor: https://www.ipchecktool.com/tool/macfinder
	'''

	d = {'vendor': 'apple', 'mac': 'D4:61:9D'}, {'vendor': 'xen', 'mac': '02:00:00'}, {'vendor': 'huawei', 'mac': 'CC:05:77'}, {'vendor': 'cisco', 'mac': '30:8B:B2'}, {'vendor': 'samsung', 'mac': '8C:B8:4A'}, {'vendor': 'google', 'mac': '30:FD:38'}, {'vendor': 'juniper', 'mac': '20:D8:0B'}, {'vendor': 'dell', 'mac': 'E0:D8:48'}, {'vendor': 'broadcom', 'mac': 'BC:97:E1'}, {'vendor': 'tplink', 'mac': '54:A7:03'}, {'vendor': 'hp', 'mac': '40:B0:34'}, {'vendor': 'indel', 'mac': '00:E1:8C'}, {'vendor': 'dlink', 'mac': '10:62:EB'}, {'vendor': 'zte', 'mac': 'D0:71:C4'}, {'vendor': 'nokia', 'mac': '40:7C:7D'}, {'vendor': 'netgear', 'mac': '08:BD:43'}, {'vendor': 'microsoft', 'mac': '6C:5D:3A'}, {'vendor': 'xiaomi', 'mac': '2C:D0:66'}

	for i in d:
		if i["vendor"] == m_type:
			mac_part = i["mac"]

	try:
		mac_part
	except NameError:
		print(f"[-] Incorrect vendor: {m_type}")
		print(f"[!] Possible vendors: xen, huawei, cisco, samsung, google, juniper, dell, broadcom, tplink, hp, indel, dlink, zte, nokia, netgear, microsoft, xiaomi")
		sys.exit(0)
	else:
		random_mac = mac_part + ":%02x:%02x:%02x" % (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
		return random_mac

def do_backup(interface, old_mac, new_mac):
	# without checking permission because ifconfig needs root in all ways
	print(f"[+] Backup: {interface}: {old_mac} -> {new_mac} write in {backup_file}")
	file_to_write = open(backup_file, "a")
	file_to_write.write(interface + ": " + old_mac + " -> " + new_mac)
	file_to_write.write("\n")
	file_to_write.close()

def change_mac(interface, old_mac, new_mac):
	current_os = system()

	if is_tool("ifconfig"):
		if current_os == 'Linux':
			subprocess.call(["ifconfig", interface, "down"])
			subprocess.call(["ifconfig", interface, "hw", "ether", new_mac])
			subprocess.call(["ifconfig", interface, "up"])

		if current_os == 'Darwin': # a little bit different way on macOS
			subprocess.call(["ifconfig", interface, "ether", new_mac])
	else:
		print("[-] Error: ifconfig isn't installed. Try apt/yum install -y net-tools first.")
		sys.exit(0)

	ifaces = get_avialible_ifaces()

	for i in ifaces:
		if i["iface"] == interface:
			current_mac = i["mac"]

	if old_mac != current_mac:
		return 1
	else:
		return 0

def main():
	parser = argparse.ArgumentParser("Linux/macOS MAC address changer by n0a")
	parser.add_argument("-i", "--interface", help="set network interface")
	parser.add_argument("-m", "--mac", help="set new MAC. If not set MAC will be randomly generated.")
	parser.add_argument("-v", "--vendor", help="Set vendor for MAC: xen (default), huawei, cisco, samsung, google, juniper, dell, broadcom, tplink, hp, indel, dlink, zte, nokia, netgear, microsoft, xiaomi")

	options = parser.parse_args()

	iface_is_good	= False
	mac_is_good		= False
	random_enable	= False

	if options.interface is not None:
		interface 	= options.interface
		newmac 		= options.mac

		ifaces = get_avialible_ifaces()
		for i in ifaces:
			if i["iface"] == interface:
				old_mac = i["mac"]
				iface_is_good = True

		if iface_is_good:
			print(f"[+] Interface seems good: {interface}")
		else:
			print(f"[-] Interface {interface} can't do this shit.")
			avialible_ifaces_format()
			sys.exit(0)

		if options.mac is not None:
			if mac_check(newmac):
				mac_is_good = True
				print(f"[+] MAC address seems good: {newmac}")
			else:
				print(f"[+] But yours MAC address looks like shit: {newmac}")
				sys.exit(0)
		else:
			random_enable = True
			if options.vendor is not None:
				vendor = options.vendor
				random_mac = mac_random(options.vendor)
			else:
				vendor = default_vendor
				random_mac = mac_random(default_vendor)

			print(f"[+] Random MAC address for this case: [{vendor}] {random_mac}")

		if iface_is_good and mac_is_good:
			working_mac = newmac

		if iface_is_good and random_enable:
			working_mac = random_mac

		run = change_mac(interface, old_mac, working_mac)

		if run:
			if backup_enable:
				do_backup(interface, old_mac, working_mac)
			print(f"[+] Changing MAC from {old_mac} -> {working_mac} successful.")
		else:
			print(f"[-] Something wrong. Try run with sudo.")

	else:
		print("[-] Error: what about set network interface?")
		avialible_ifaces_format()
		print("")
		parser.print_help()
		sys.exit(0)

if __name__== "__main__":
	main()


#!/usr/bin/python3
"""prints the State object with name passed as argument from hbtn_0e_6_usa"""
import sqlalchemy
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sys import argv
from model_state import Base, State

if __name__ == "__main__":
    connector = 'mysql+mysqldb://{}:{}@localhost/{}'
    engine = create_engine(connector.format(
        argv[1], argv[2], argv[3]), pool_pre_ping=True)
    Base.metadata.create_all(engine)
    Base.metadata.bind = engine
    Session = sessionmaker()
    Session.bind = engine
    session = Session()

    state = session.query(State).filter(State.name == argv[4]).first()
    print("Not found" if not state else state.id)
    session.close()


import numpy as np
import array
import time
import re
import urllib.request, urllib.error # .request
from bs4 import BeautifulSoup, Comment
import csv
import os
import sqlite3
from itertools import islice
from datetime import datetime
import sys

time_var = 0
EMA_var = 0

def Playoffs_Parser(base_url):

	with sqlite3.connect('Games.db') as conn:  # open('Playoffs_list_ByYear.csv', 'w', newline='') as csvfile, sqlite3.connect('Games.db') as conn:
		#  conn.execute("DROP TABLE IF EXISTS games_table")
		# conn.execute('''CREATE TABLE IF NOT EXISTS games_table(
		# GAME_TAG TEXT NOT NULL PRIMARY KEY,
		# SEASON INT,
		# YEAR INT,
		# MONTH INT,
		# DAY INT,
		# HOME_TEAM TEXT NOT NULL,
		# AWAY_TEAM TEXT NOT NULL,
		# HOME_SCORE INT NOT NULL,
		# AWAY_SCORE INT NOT NULL,
		# OT INT,
		# PBP_URL TEXT NOT NULL
		# );
		# ''')
		insert_rows = [(None,)*11] * 2000
		row_num = 0
		row_values = [None] * 11
		# game_tag_regex = re.compile("(?:.*)/([^/]+)\.html")
		# ot_regex = re.compile("(\d*)OT")

		# writer = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
		# dict_writer = csv.DictWriter(csvfile, ['GAME_TAG', 'Year', 'Month', 'Day', 'Home Name', 'Away Name', 'Home score', 'Away score', 'OT', 'PbP URL'])
		# dict_writer.writeheader()
		global time_var

		for i in range(2018, 2017, -1):
			cur_url = base_url + '/leagues/NBA_' + str(i) + '_games.html'  # https://www.basketball-reference.com/leagues/NBA_2017_games.html
			start_t = time.time()
			r = urllib.request.urlopen(cur_url)
			html = r.read()
			time_var += time.time() - start_t
			soup = BeautifulSoup(html, "html5lib")

			months_list = soup.find("div", id='content').find("div", attrs={'class': 'filter'}).find_all('a', href=re.compile("leagues/NBA_\d+_games-.+\.html"))

			row_values[1] = i
			for month in months_list:
				start_t = time.time()
				soup2 = BeautifulSoup(urllib.request.urlopen(base_url + month.get('href')).read(), "html5lib")
				time_var += time.time() - start_t
				games_table = soup2.find("table", id="schedule").find("tbody").find_all("tr")  # , attrs={"class": re.compile("toggleable.*$")})

				for game_row in games_table:
					cols = game_row.find_all(['th', 'td'])
					if len(cols) < 10:
						continue
					row_values[0] = game_tag_regex.search(base_url + cols[6].a.get('href')).group(1)
					# tmp_date = re.split("(?: |,)+", cols[0].text)
					tmp_date = datetime.strptime(cols[0].text, '%a, %b %d, %Y')
					row_values[2:5] = [tmp_date.year, tmp_date.month, tmp_date.day]
					row_values[5:7] = [cols[4].text, cols[2].text]
					row_values[7:9] = [int(cols[5].text), int(cols[3].text)]
					ot_num = ot_regex.search(cols[7].text)
					if ot_num is not None:
						row_values[9] = int(ot_num.group(1)) if ot_num.group(1) != '' else 1
					else:
						row_values[9] = None
					row_values[10] = 'https://www.basketball-reference.com/boxscores/pbp/' + row_values[0] + '.html'
					insert_rows[row_num] = tuple(row_values)
					row_num += 1
					# insert_rows.append(tuple(row_values))
					# prev_tr = game_row.previous_sibling.previous_sibling
					# prev_td = prev_tr.td.next_sibling.next_sibling
					# team_names = [a_text.text for a_text in game.find_all("a", href=re.compile("teams"), limit=2)]
					# games_links = game_row.find_all("a", href=re.compile("/boxscores/[0-9]+.*$"))
					# games_urls = [base_url + "/boxscores/pbp/" + str.split(a_text['href'], "/")[2] for a_text in games_links]
					# dict_writer.writerow({'Year': i, 'Team1 Name': team_names[0], 'Team2 Name': team_names[1], 'URLs': ' '.join(games_urls)})

				# writer.writerow([])
			conn.executemany("INSERT INTO games_table VALUES (?,?,?,?,?,?,?,?,?,?,?)", insert_rows[0:row_num])
			row_num = 0
			print("finished season {}-{}".format(i-1, i))
		conn.commit()

def playoffs_series_parser(base_url):
	with sqlite3.connect('Games.db') as conn:  # open('Playoffs_list_ByYear.csv', 'w', newline='') as csvfile, sqlite3.connect('Games.db') as conn:
		conn.execute("DROP TABLE IF EXISTS series_table")
		conn.execute("""CREATE TABLE IF NOT EXISTS series_table(
							GAME_TAG TEXT NOT NULL PRIMARY KEY,
							LEAGUE TEXT NOT NULL,
							SEASON INT,
							SERIES_ID INT,
							GAME_NUM INT,
							HOME_TEAM TEXT NOT NULL,
							AWAY_TEAM TEXT NOT NULL,
							HOME_SCORE INT NOT NULL,
							AWAY_SCORE INT NOT NULL
		);
		""")
		insert_rows = [(None,) * 9] * 2000
		row_values = [None] * 9
		row_num = 0
		# game_tag_regex = re.compile("(?:.*)/([^/]+)\.html")
		# ot_regex = re.compile("(\d*)OT")

		# writer = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
		# dict_writer = csv.DictWriter(csvfile, ['GAME_TAG', 'Year', 'Month', 'Day', 'Home Name', 'Away Name', 'Home score', 'Away score', 'OT', 'PbP URL'])
		# dict_writer.writeheader()

		for i in range(2017, 1946, -1):
			start_t = time.time()
			for league_name in ['NBA', 'ABA', 'BAA']:
				cur_url = base_url + '/playoffs/' + league_name + '_' + str(i) + '.html'  # ex: https://www.basketball-reference.com/playoffs/NBA_1965.html
				try:
					r = urllib.request.urlopen(cur_url)
				except urllib.error.HTTPError as err:
					if err.code == 404:
						continue
				html = r.read()
				soup = BeautifulSoup(html, "html5lib")

				series_table = soup.find("table", id='all_playoffs')  # .find("div", attrs={'class': 'filter'}).find_all('a', href=re.compile("leagues/NBA_\d+_games-.+\.html"))
				if series_table is None:
					print(i, league_name)
					continue

				series_list = series_table.find("tbody").find_all("tr", id=re.compile("s(\d+)"))
				row_values[1] = league_name
				row_values[2] = i
				for series in series_list:
					row_values[3] = int(series.get('id')[1:])
					games_table = series.find("table").find("tbody").find_all("tr")  # , attrs={"class": re.compile("toggleable.*$")})

					for game_row in games_table:
						cols = game_row.find_all(['th', 'td'])
						if len(cols) < 6:
							continue
						row_values[0] = game_tag_regex.search(base_url + cols[0].a.get('href')).group(1)
						row_values[4] = int(cols[0].text[5:])
						row_values[5:7] = [cols[4].text[2:], cols[2].text]
						row_values[7:9] = [int(cols[5].text), int(cols[3].text)]
						insert_rows[row_num] = tuple(row_values)
						row_num += 1
					# insert_rows.append(tuple(row_values))
					# prev_tr = game_row.previous_sibling.previous_sibling
					# prev_td = prev_tr.td.next_sibling.next_sibling
					# team_names = [a_text.text for a_text in game.find_all("a", href=re.compile("teams"), limit=2)]
					# games_links = game_row.find_all("a", href=re.compile("/boxscores/[0-9]+.*$"))
					# games_urls = [base_url + "/boxscores/pbp/" + str.split(a_text['href'], "/")[2] for a_text in games_links]
					# dict_writer.writerow({'Year': i, 'Team1 Name': team_names[0], 'Team2 Name': team_names[1], 'URLs': ' '.join(games_urls)})

					# writer.writerow([])
			conn.executemany("INSERT INTO series_table VALUES (?,?,?,?,?,?,?,?,?)", insert_rows[0:row_num])
			row_num = 0
			print("finished season {} in {} seconds".format(i, time.time() - start_t))
		conn.commit()

def games_quarter_scores(base_url):
	with sqlite3.connect('Games.db') as conn:
		# conn.execute("DROP TABLE IF EXISTS games_scores")
		# conn.execute("""
		# CREATE TABLE IF NOT EXISTS games_scores(
		# 	GAME_TAG TEXT NOT NULL PRIMARY KEY,
		# 	HOME_SCORE INT NOT NULL,
		# 	AWAY_SCORE INT NOT NULL,
		# 	QT1_H_A TEXT,
		# 	QT2_H_A TEXT,
		# 	QT3_H_A TEXT,
		# 	QT4_H_A TEXT,
		# 	OT1_H_A TEXT,
		# 	OT2_H_A TEXT,
		# 	OT3_H_A TEXT,
		# 	OT4_H_A TEXT
		# );
		# """)
		insert_rows = [(None,) * 14] * 2000
		row_values = [None] * 14

		for i in range(2018, 2017, -1):
			result = conn.execute("""
			SELECT COUNT(*)
			FROM games_table
			WHERE SEASON=?
			""", (i,))
			num_of_rows = result.fetchone()[0]
			games_stats = conn.execute("""
			SELECT games_table.GAME_TAG, games_table.OT
			FROM games_table
			WHERE SEASON=?
			""", (i,))

			games_data = games_stats.fetchall()
			row_index = 0

			for k, game in enumerate(games_data):
				start_t = time.time()
				game_tag = game[0]
				ot_count = game[1] if game[1] is not None else 0

				try:
					cur_url = base_url + "/boxscores/" + game_tag + ".html"  # https://www.basketball-reference.com/boxscores/201611230PHI.html
					r = urllib.request.urlopen(cur_url)
				except urllib.error.HTTPError as err:
					if err.code == 404:
						print("Error:\t", game_tag)
						continue
				html = r.read(70000).decode('utf-8')
				table_search = re.search("<table.{0,50}?id=\"line_score\".{0,50}?>.+?</table>", html, flags=re.I | re.DOTALL)
				table_text = html[table_search.start():table_search.end()]
				# soup = BeautifulSoup(html, "html5lib")
				# scores_table2 = BeautifulSoup(str(soup.find(text=lambda text: isinstance(text, Comment) and re.search("<table .+ id=\"line_score\" .+>", text))), "html5lib")
				scores_table2 = BeautifulSoup(table_text, "html5lib")
				# print(scores_table2.find("table", id='line_score'))
				# print(scores_table2.find("table", id='line_score').find("tbody"))
				scores_table = scores_table2.find("table", id='line_score').find("tbody").find_all("tr")
				away_scores = scores_table[2].find_all("td")
				home_scores = scores_table[3].find_all("td")

				if len(away_scores) - 6 != ot_count or len(home_scores) - 6 != ot_count:
					print("OT count error:\t", game_tag, len(away_scores), ot_count)
					continue

				row_values[0] = game_tag
				row_values[1], row_values[2] = int(home_scores[-1].text), int(away_scores[-1].text)

				for j in range(8):
					if j >= 4 + ot_count:
						row_values[j + 3] = None
					else:
						row_values[j + 3] = "{}-{}".format(home_scores[j+1].text, away_scores[j+1].text)

				insert_rows[row_index] = tuple(row_values)
				row_index += 1
				start_t = time.time() - start_t
				if k%100 == 0:
					print("{} - {}".format(k, start_t))

			conn.executemany("INSERT INTO games_scores VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)", insert_rows[0:row_index])
			row_index = 0
			print("finished season {} in {} seconds avg".format(i, time.time() - start_t))

		conn.commit()

def Game_Parser(game_id, game_season, game_url):

	dir_name = "Games/{}-{}/".format(game_season-1, game_season)
	if not os.path.exists(dir_name):
		os.makedirs(dir_name)

	#  game_id = str.split(game_url, '/')[5].split('.')[0]
	#  game_id = game_tag_regex.search(game_url).group(1)

	start_t = time.time()
	for k in range(5):
		try:
			r = urllib.request.urlopen(game_url)
			html = r.read()
		except:
			print("\rfailed urlopen {} out of 5".format(k))
			if k == 4:
				raise
			time.sleep(10)
		else:
			break
	global time_var
	global EMA_var
	EMA_var = time.time() - start_t
	time_var += EMA_var
	soup = BeautifulSoup(html, "html5lib")

	title = names_regex.search(soup.find("div", id="content").find("h1").string)
	home_name = title.group(2)
	away_name = title.group(1)

	pbp_table = soup.find("table", id="pbp").find("tbody").find_all("tr")

	with open(dir_name + game_id + ".csv", 'w', newline='') as csvfile:
		writer = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_ALL)
		writer.writerow([title.group(1), title.group(2), title.group(3), title.group(4)])
		writer.writerow('')
		writer.writerow(['Time', 'H/A action', 'Home: ' + home_name, 'Away: ' + away_name, 'score change', 'description'])
		arr2csv = [''] * 6

		for table_row in pbp_table:
			row_cells = table_row.find_all("td")

			if len(row_cells) == 2:
				arr2csv[0] = row_cells[0].get_text()
				arr2csv[1] = row_cells[1].get_text()
				writer.writerow([arr2csv[0], '-', arr2csv[1]])

			elif len(row_cells) == 6:
				arr2csv[0] = row_cells[0].get_text()
				is_home_action = (row_cells[1].get_text() == '\xa0')
				cur_score = row_cells[3].get_text().split('-')
				arr2csv[1] = 'H' if is_home_action else 'A'
				arr2csv[2] = cur_score[1]
				arr2csv[3] = cur_score[0]
				arr2csv[4] = row_cells[4 if is_home_action else 2].get_text()
				arr2csv[5] = row_cells[5 if is_home_action else 1].get_text()
				writer.writerow(arr2csv)

def Games_Parser():

	global EMA_var
	EMA_time = 0

	with sqlite3.connect('Games.db') as conn:

		for i in range(2018, 2017, -1):
			result = conn.execute("SELECT COUNT(*) FROM games_table WHERE SEASON=?", (i,))
			num_of_rows = result.fetchone()[0]
			games_list = conn.execute("SELECT GAME_TAG, PBP_URL FROM games_table WHERE SEASON=?", (i,))

			for index, game in enumerate(games_list, start=1):
				Game_Parser(game[0], i, game[1])
				EMA_time = (1/6)*EMA_time + (5/6)*EMA_var
				sys.stdout.write('\r')
				if (index % (num_of_rows//10)) == 0:
					sys.stdout.write("sleep.\tindex={},\tnum_of_rows={}".format(index, num_of_rows))
					time.sleep(10)  # sleep for 10 seconds
				else:
					sys.stdout.write("{}% finished ({} avg sec for game:\t{} min remains)".format(index/num_of_rows*100, EMA_time, EMA_time*(num_of_rows - index)/60))
				sys.stdout.flush()

			print("\nFinished season {}-{}".format(i-1, i))

names_regex = re.compile("(.+) at (.+) Play-By-Play, (.+), (.+)$")
game_tag_regex = re.compile("(?:.*)/([^/]+)\.html")
ot_regex = re.compile("(\d*)OT")
game_tag_from_file_name = re.compile("/?(?:.*/)?(.+)\.csv")
start_of_nth_q_regex = re.compile("Start of (\d+)(\S+) (quarter|overtime)", flags=re.I)
jump_ball_regex = re.compile("Jump ball: (.+) vs\. (.+)(?: \((.+)\))?", flags=re.I)
game_pt_regex = re.compile("(.+) (makes|misses) .*(\d)-pt shot (.+)(?:\( assist by (.+)\))?", flags=re.I)
free_throw_regex = re.compile("(.+) (makes|misses) .*free throw(?: (.*)(?:(\d) of (\d))?)?", flags=re.I)
rebound_regex = re.compile("(Defensive|Offensive) rebound by (.+)", flags=re.I)
foul_regex = re.compile("(.*) ?foul by (.+)(?: \(drawn by (.+)\))?", flags=re.I)
turnover_regex = re.compile("Turnover by (.+)(?: \((.+)\))", flags=re.I)
timeout_regex = re.compile("(.+) timeout.*", flags=re.I)
player_substitution_regex = re.compile("(.+) enters the game for (.+)", flags=re.I)

def pbp_to_possessions(game_file):

	with open(game_file, "r", newline='') as csvfile:  # , sqlite3.connect('Games.db') as conn:
		reader = csv.reader(csvfile)
		# conn.execute("DROP TABLE IF EXISTS pbp_table")
		# db_cursor = conn.cursor()
		# db_cursor.execute('''
		# CREATE TABLE IF NOT EXISTS pbp_table(
		# GAME_TAG TEXT NOT NULL,
		# HOME_SCORE INT,
		# AWAY_SCORE INT,
		# TIME_MIN INT,
		# TIME_SEC INT,
		# TIME_MILLI INT,
		# QUARTER TEXT,
		# HorA_ACTION TEXT NOT NULL,
		# SCORE_CHANGE INT,
		# DESCRIPTION TEXT
		# );
		# ''')

		# insert_arr = [(None, ) * 10] * 1000  # max number of rows to insert
		possessions_arr = ["-"] + [0] * 1000  # max number of possessions
		row_values = [None] * 10  # [None] * 10
		row_num = 0

		possessions_index = 1
		cur_time = 0
		cur_quarter = '1'
		last_writen_possession = ''
		score_A, score_H = 0, 0
		team_complement = {'H': 'A', 'A': 'H'}  # used to turn possession

		row_values[0] = game_tag_from_file_name.search(game_file).group(1)  # GAME TAG

		for row in islice(reader, 3, None):
			cur_time = re.split("\D+", row[0])
			row_values[3:6] = map(int, cur_time)  # TIME
			row_values[6] = cur_quarter  # QUARTER
			row_values[8] = None

			if row[1] != '-':  # team action
				row_values[7] = row[1]  # HorA_ACTION
				row_values[9] = row[5]  # DESCRIPTION
				score_A, score_B = int(row[2]), int(row[3])
				row_values[1], row_values[2] = row[2], row[3]  # TEAM SCORES

				res = game_pt_regex.search(row[5])
				if res is not None:  # game point
					if last_writen_possession == row[1]:
						possessions_index -= 1
					elif last_writen_possession == row[1] + ' ':
						possessions_arr[possessions_index] = '0'  # place holder
						possessions_index += 1

					if res.group(2) == 'makes':
						possessions_arr[possessions_index] += int(row[4])  # append new team possession's score
						possessions_index += 1
						last_writen_possession = row[1]
						row_values[8] = int(row[4])  # SCORE CHANGE
					else:  # misses
						# possessions_arr[possessions_index] += 0  # append new team possession
						possessions_index += 1
						last_writen_possession = row[1]

				else:
					res = free_throw_regex.search(row[5])
					if res is not None:  # free throw
						# if res.group(3) is not None and res.group(4) is not None:
						# 	free_throws_count = int(res.group(4))
						# 	cur_free_throw = int(res.group(3))
						#
						# 	if last_writen_possession != row[1] and cur_free_throw > 1:
						# 		print(" {}: Error in game possessions (free throw)".format(row_values[0]))
						# 		raise Exception('Error in possessions')

						if last_writen_possession == row[1]:
							possessions_index -= 1
						elif last_writen_possession == row[1] + ' ':
							possessions_arr[possessions_index] = '0'  # place holder
							possessions_index += 1

						if res.group(2) == 'makes':
							possessions_arr[possessions_index] += int(row[4])  # append new team possession's score
							possessions_index += 1
							last_writen_possession = row[1]
							row_values[8] = int(row[4])  # SCORE CHANGE
						else:  # misses
							# possessions_arr[possessions_index] += 0  # append new team possession
							possessions_index += 1
							last_writen_possession = row[1]

					else:
						res = rebound_regex.search(row[5])
						if res is not None:
							continue
						else:
							res = foul_regex.search(row[5])
							if res is not None: # foul
								continue
							else:
								res = turnover_regex.search(row[5])
								if res is not None:  # turnover
									if last_writen_possession == row[1]:
										possessions_index -= 1
									elif last_writen_possession == row[1] + ' ':
										possessions_arr[possessions_index] = '0'  # place holder
										possessions_index += 1

									# possessions_arr[possessions_index] += 0
									possessions_index += 1
									last_writen_possession = row[1]
								else:
									res = timeout_regex.search(row[5])
									if res is not None:
										continue
									else:
										res = player_substitution_regex.search(row[5])
										if res is not None:
											continue
				if possessions_arr[0] == '-' and possessions_index == 3:  # who starts the
					possessions_arr[0] = row[1]

			else:  # game comment
				row_values[1:3] = [None]*2
				row_values[7] = '-'  # HorA_ACTION
				row_values[9] = row[2]  # DESCRIPTION
				res = start_of_nth_q_regex.search(row[2])

				if res is not None:  # start of a quarter
					cur_quarter = res.group(1)
					last_writen_possession = last_writen_possession + ' '
					if res.group(3) == 'overtime':
						cur_quarter += 'OT'
					possessions_arr[possessions_index] = cur_quarter
					possessions_index += 1
				else:
					res = jump_ball_regex.search(row[2])
					if res is not None:  # jump ball
						continue

			# insert_arr[row_num] = tuple(row_values)  # "('{0[0]}', {0[1]}, {0[2]}, {0[3]}, {0[4]}, {0[5]}, '{0[6]}', '{0[7]}', {0[8]},
			# '{0[9]}')".format(row_values)
			row_num += 1

		#  print(sum(possessions_arr[0:possessions_index:2]), sum(possessions_arr[1:possessions_index:2]))
		# conn.executemany("INSERT INTO pbp_table VALUES (?,?,?,?,?,?,?,?,?,?)", insert_arr[0:row_num])
		# conn.commit()
		# db_cursor.close()
		# conn.close()
	return possessions_arr[0:possessions_index]

def check_pbp(game_tag, season):
	game_possessions = pbp_to_possessions("Games/{}-{}/{}.csv".format(season-1, season, game_tag))
	a_poss = [i for i in game_possessions[0::2] if type(i) == int]
	b_poss = [i for i in game_possessions[1::2] if type(i) == int]
	print(sum(a_poss), sum(b_poss))

# if __name__ == '__main__':

	# check_pbp('200710310DEN', 2008)

	# start_time = time.time()
	# Playoffs_Parser('https://www.basketball-reference.com')
	# playoffs_series_parser('https://www.basketball-reference.com')
	# games_quarter_scores('https://www.basketball-reference.com')
	# print(time.time() - start_time, time_var)

	# start_time = time.time()
	# Games_Parser()
	# print(time.time() - start_time, time_var)

	# Game_Parser('https://www.basketball-reference.com/boxscores/pbp/201706010GSW.html')
	# print(pbp_to_possessions('Games/201706010GSW.csv'))
	# Game_Parser('https://www.basketball-reference.com/boxscores/pbp/201210310DET.html')
	# start_time = time.time()
	# print(pbp_to_possessions('Games/201210310DET.csv'))
	# print(time.time() - start_time)
	# Games_Parser()


"""Ensemble backend that combines results from multiple projects"""
from __future__ import annotations

from typing import TYPE_CHECKING, Any

import annif.eval
import annif.parallel
import annif.util
from annif.exception import NotSupportedException
from annif.suggestion import SuggestionBatch

from . import backend, hyperopt

if TYPE_CHECKING:
    from datetime import datetime

    from optuna.study.study import Study
    from optuna.trial import Trial

    from annif.backend.hyperopt import HPRecommendation
    from annif.corpus.document import DocumentCorpus

class BaseEnsembleBackend(backend.AnnifBackend):
    """Base class for ensemble backends"""

    def _get_sources_attribute(self, attr: str) -> list[bool | None]:
        params = self._get_backend_params(None)
        sources = annif.util.parse_sources(params["sources"])
        return [
            getattr(self.project.registry.get_project(project_id), attr)
            for project_id, _ in sources
        ]

    def initialize(self, parallel: bool = False) -> None:
        # initialize all the source projects
        params = self._get_backend_params(None)
        for project_id, _ in annif.util.parse_sources(params["sources"]):
            project = self.project.registry.get_project(project_id)
            project.initialize(parallel)

    def _suggest_with_sources(
        self, texts: list[str], sources: list[tuple[str, float]]
    ) -> dict[str, SuggestionBatch]:
        return {
            project_id: self.project.registry.get_project(project_id).suggest(texts)
            for project_id, _ in sources
        }

    def _merge_source_batches(
        self,
        batch_by_source: dict[str, SuggestionBatch],
        sources: list[tuple[str, float]],
        params: dict[str, Any],
    ) -> SuggestionBatch:
        """Merge the given SuggestionBatches from each source into a single
        SuggestionBatch. The default implementation computes a weighted
        average based on the weights given in the sources tuple. Intended
        to be overridden in subclasses."""

        batches = [batch_by_source[project_id] for project_id, _ in sources]
        weights = [weight for _, weight in sources]
        return SuggestionBatch.from_averaged(batches, weights).filter(
            limit=int(params["limit"])
        )

    def _suggest_batch(
        self, texts: list[str], params: dict[str, Any]
    ) -> SuggestionBatch:
        sources = annif.util.parse_sources(params["sources"])
        batch_by_source = self._suggest_with_sources(texts, sources)
        return self._merge_source_batches(batch_by_source, sources, params)

class EnsembleOptimizer(hyperopt.HyperparameterOptimizer):
    """Hyperparameter optimizer for the ensemble backend"""

    def __init__(
        self, backend: EnsembleBackend, corpus: DocumentCorpus, metric: str
    ) -> None:
        super().__init__(backend, corpus, metric)
        self._sources = [
            project_id
            for project_id, _ in annif.util.parse_sources(
                backend.config_params["sources"]
            )
        ]

    def _prepare(self, n_jobs: int = 1) -> None:
        self._gold_batches = []
        self._source_batches = []

        for project_id in self._sources:
            project = self._backend.project.registry.get_project(project_id)
            project.initialize()

        psmap = annif.parallel.ProjectSuggestMap(
            self._backend.project.registry,
            self._sources,
            backend_params=None,
            limit=int(self._backend.params["limit"]),
            threshold=0.0,
        )

        jobs, pool_class = annif.parallel.get_pool(n_jobs)

        with pool_class(jobs) as pool:
            for suggestions, gold_batch in pool.imap_unordered(
                psmap.suggest_batch, self._corpus.doc_batches
            ):
                self._source_batches.append(suggestions)
                self._gold_batches.append(gold_batch)

    def _normalize(self, hps: dict[str, float]) -> dict[str, float]:
        total = sum(hps.values())
        return {source: hps[source] / total for source in hps}

    def _format_cfg_line(self, hps: dict[str, float]) -> str:
        return "sources=" + ",".join(
            [f"{src}:{weight:.4f}" for src, weight in hps.items()]
        )

    def _objective(self, trial: Trial) -> float:
        eval_batch = annif.eval.EvaluationBatch(self._backend.project.subjects)
        proj_weights = {
            project_id: trial.suggest_float(project_id, 0.0, 1.0)
            for project_id in self._sources
        }
        for gold_batch, src_batches in zip(self._gold_batches, self._source_batches):
            batches = [src_batches[project_id] for project_id in self._sources]
            weights = [proj_weights[project_id] for project_id in self._sources]
            avg_batch = SuggestionBatch.from_averaged(batches, weights).filter(
                limit=int(self._backend.params["limit"])
            )
            eval_batch.evaluate_many(avg_batch, gold_batch)
        results = eval_batch.results(metrics=[self._metric])
        return results[self._metric]

    def _postprocess(self, study: Study) -> HPRecommendation:
        line = self._format_cfg_line(self._normalize(study.best_params))
        return hyperopt.HPRecommendation(lines=[line], score=study.best_value)

class EnsembleBackend(BaseEnsembleBackend, hyperopt.AnnifHyperoptBackend):
    """Ensemble backend that combines results from multiple projects"""

    name = "ensemble"

    @property
    def is_trained(self) -> bool:
        sources_trained = self._get_sources_attribute("is_trained")
        return all(sources_trained)

    @property
    def modification_time(self) -> datetime | None:
        mtimes = self._get_sources_attribute("modification_time")
        return max(filter(None, mtimes), default=None)

    def get_hp_optimizer(
        self, corpus: DocumentCorpus, metric: str
    ) -> EnsembleOptimizer:
        return EnsembleOptimizer(self, corpus, metric)

    def _train(self, corpus: DocumentCorpus, params: dict[str, Any], jobs: int = 0):
        raise NotSupportedException("Training ensemble backend is not possible.")


#!/usr/bin/env python3
from collections import defaultdict
import time
import re
import os
import sys
import textwrap
import argparse
import urllib.request
import ssl
from urllib.error import HTTPError
gcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1)

parser = argparse.ArgumentParser(
    prog="silva-survey.py",
    formatter_class=argparse.RawDescriptionHelpFormatter,
    description=textwrap.dedent('''
    Developed by Arkadiy Garber; University of Delaware, Geological Sciences
    Please send comments and inquiries to arkg@udel.edu
    '''))

parser.add_argument('-taxa', help='taxanomic affiliation of organism that you would like to investigate')
parser.add_argument('-silva_DB', help="location of SILVA database")
parser.add_argument('-out_dir', help="directory to which the output file will be written")

if len(sys.argv) == 1:
    parser.print_help(sys.stderr)
    sys.exit(0)

args = parser.parse_known_args()[0]

def replace(stringOrlist, list, item):
    emptyList = []
    for i in stringOrlist:
        if i not in list:
            emptyList.append(i)
        else:
            emptyList.append(item)
    outString = "".join(emptyList)
    return outString

def remove(stringOrlist, list):
    emptyList = []
    for i in stringOrlist:
        if i not in list:
            emptyList.append(i)
        else:
            pass
    outString = "".join(emptyList)
    return outString

def fasta(fasta_file):
    seq = ''
    header = ''
    Dict = defaultdict(lambda: defaultdict(lambda: 'EMPTY'))
    for i in fasta_file:
        i = i.rstrip()
        if re.match(r'^>', i):
            if len(seq) > 0:
                Dict[header] = seq
                header = i[1:]
                seq = ''
            else:
                header = i[1:]
                seq = ''
        else:
            seq += i
    Dict[header] = seq
    return Dict

silva = open(args.silva_DB, "r")
silva =fasta(silva)
outfile = open(args.out_dir + "/" + args.taxa + "_SILVA.csv", "w")
outfile.write(args.taxa + "_strain" + "," + "Source" + "," + "Reference" + "," + "16S_seq" + "\n")

for i in silva.keys():
    if re.findall(args.taxa, i):
        print(i)
        time.sleep(2)
        outfile.write(i + ",")
        id = i.split(" ")[0]
        id = id.split(".")[0]
        fp = urllib.request.urlopen("https://www.ebi.ac.uk/ena/data/view/%s&display=text" % id, context=gcontext)
        mybytes = fp.read()
        mystr = mybytes.decode("utf8")
        fp.close()
        lines = mystr.split('\n')
        string = ''
        count = 0
        ref = ''
        for j in lines:
            if re.findall(r'isolation_source', j) or re.findall(r'tissue_type', j) or \
                    re.findall(r'Source of Isolate', j):
                count += 1
                try:
                    string = j.split("\"")[1]
                    string = replace(string, [","], ";")
                except IndexError:
                    string = j
                    string = replace(j, [","], ";")
                count += 1
            if re.match(r'^RT', j) and len(j) > 6:
                ref += j
        if count == 0:
            outfile.write("No_data" + ",")
        else:
            outfile.write(replace(string, [","], ";") + ",")
        ref = (replace(ref, [","], ";"))
        ref = remove(ref, ["RT"])
        outfile.write(ref + ",")
        outfile.write(silva[i] + "\n")