"""
Crawler to get results from specific query from github.com.

__External modules__: `requests`, `random`, `random`, `bs4`, `json`

`Compatible with Python3.8 or higher`
"""
import requests
from random import randint
from bs4 import BeautifulSoup
import json

def get_random_proxy(*args):
	"""Function to wait process thread responsible for calculation the solutions.

	Args:
		proxies (array): _Optional_ array of `str` with proxies

	Returns:
		proxy (str): a random proxy
	"""
	if not args:
		http_proxies = [
			'200.68.49.184:8080',
			'119.252.160.165:3128',
			'94.180.249.187:38051',
			'62.210.69.176:5566' ]
	else:
		http_proxies = args[0]

	n = randint(0,len(http_proxies)-1)

	return http_proxies[n]

def process_input(crawler_input):
	"""Function process the input, making sure it is valid.

	Args:
		crawler_input (dict): `dict` with `keywords`, `proxies` and `type`

	Returns:
		input_dict (dict): a sanatised `dict` with the inputs for the crawler
	"""
	try:
		keywords = crawler_input['keywords']
	except KeyError:
		print('No keywords found')
		quit()

	try:
		proxy = get_random_proxy(crawler_input['proxies'])
	except KeyError:
		print('No proxies found, using existing ones')
		proxy = get_random_proxy()

	try:
		query_type = crawler_input['type']
	except KeyError:
		print('No type found')
		quit()

	if query_type not in ['Repositories','Issues', 'Wikis']:
		print('Invalid type')
		quit()

	input_dict = {
		'keywords': keywords,
		'proxy': proxy,
		'type': query_type
	}

	return input_dict

def parse_keywords(keywords):
	"""Function to convert the keywords `array` in to a `str` that will be used for the crawler query

	Args:
		keywords (array): `array` with `keywords`

	Returns:
		formated_keyword (str): `str` with the concatenated keywords using + between them
	"""
	q = ''
	for i in range(len(keywords)):
		q += f'{keywords[i]}+'
	return q[0:-1]

def get_results_page(input_dict):
	"""Function to retrieve the first 10 results for the specified input

	Args:
		input_dict (dict): `dict` with crawler input

	Returns:
		page (str): `str` with the html content of the page
	"""
	q = parse_keywords(input_dict['keywords'])
	proxy_dict = {'http': f'http://{input_dict["proxy"]}'}
	search_type = input_dict['type']

	r = requests.get(f'https://github.com/search?q={q}&type={search_type}', proxies=proxy_dict)

	return r.text

def results_parser(page):
	"""Function to parse the result page

	Args:
		page (str): `str` with html content

	Returns:
		results_dict (dict): `dict` with the results from the crawler
	"""
	results_dict = {}

	soup_page = BeautifulSoup(page, 'html.parser')
	results = soup_page.findAll('div', {'class':'f4 text-normal'})

	for result in results:
		temp = result.find('a').get('data-hydro-click')
		json_input = json.loads(temp)
		payload = json_input['payload']
		results_dict[payload['result_position']] = payload

	return results_dict

def languages_parser(page):
	"""Function to parse the languages in a repository page

	Args:
		page (str): `str` with html content

	Returns:
		language_stats (dict): `dict` with the language status of the repository
	"""
	language_stats = {}

	soup_page = BeautifulSoup(page, 'html.parser')
	languages = soup_page.findAll('li', {'class': 'd-inline'})
	for language in languages:
		temp = language.find('span',{'class': 'text-gray-dark text-bold mr-1'})
		lang = temp.get_text()
		value = temp.findNext('span').get_text()
		language_stats[lang] = float(value.strip('%'))

	return language_stats

def get_urls(results_dict):
	"""Function to get the url of each result in the results `dict`

	Args:
		results_dict (dict): `dict` with all the results

	Returns:
		url (array): `array` with the urls of returned by the crawler
	"""
	urls = []
	for result in results_dict:
		url = results_dict[result]['result']['url']
		urls.append({"url":url})
	return urls

def get_repo_details(url,proxy):
	"""Function to get the details of a specific repository

	Args:
		url (dict): `dict` with all the results format {'url': url_str}\n
		proxy(str): `str` with the proxy to be used

	Returns:
		repo_details (dict): `dict` with the repository details
	"""
	temp = url['url'].replace('https://github.com/','')
	owner = temp[0:temp.find("/")]

	proxies = {'http': f'http://{proxy}'}

	r = requests.get(url['url'],proxies=proxies)

	language_stats = languages_parser(r.text)
	return {'owner': owner, 'language_stats': language_stats }

def crawler(crawler_input):
	"""Function to crawl the webpage using the inputs given by the user

	Args:
		crawler_input (dict): `dict` with the crawler input

	Returns:
		details (json): an indented json object with all the crawler information
	"""
	c_input = process_input(crawler_input)
	text = get_results_page(c_input)
	results_dict = results_parser(text)
	urls = get_urls(results_dict)

	if c_input['type'] == 'Repositories':
		for url in urls:
			url['extra'] = get_repo_details(url,c_input['proxy'])

	return json.dumps(urls,indent=1)

def memory_usage(method,args):
	"""Function to check the memory usage

	Args:
		method (function): `function` to be evaluated\n
		args (args): `args` to be used on the evalutated method

	Returns:
		__Nonen__
	"""
	import tracemalloc
	tracemalloc.start()

	method(args)

	current, peak = tracemalloc.get_traced_memory()
	print(f"Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB")
	tracemalloc.stop()

if __name__ == '__main__':
	crawler_input = {
		'keywords': [
			'recursion',
			# 'rickfernandes',
			# 'css',
			# 'rick',
			# 'otrisus'
		],
		'proxies': [
			'194.126.37.94:8080',
			'13.78.125.167:8080'
		],
		'type': 'Repositories'
	}

	print(crawler(crawler_input))

	# memory_usage(crawler,crawler_input)


"""An empty class used to throw helpful exceptions when a search client isn't configured."""

from django.core.exceptions import ImproperlyConfigured

class Client(object):
    """Throws exceptions on attempts to use this class."""

    def __getattr__(self, _):
        raise ImproperlyConfigured(
            "Search client not configured. Start with settings.ENABLE_SEARCHIFY")


# -*-coding: utf-8 -*-

"""
甩手工具从淘宝网上爬下来的宝贝图片，是用数字命名的。如1, 10, 12, 2, 3。
这样图片放置顺序是不对的。
解决方法是，将数字转化为字母，如1-->a，2-->b，10-->j。
"""

import os
import re
from functools import partial

pattern = re.compile(r'(\d+)')
for path, dirs, files in os.walk(os.getcwd()):
    join = partial(os.path.join, path)
    for oldname in files:
        newname = pattern.sub(lambda r: chr(96+int(r.group(1))), oldname)
        if oldname != newname:
            os.rename(join(oldname), join(newname))


"""The Epsilon module."""

import time
import random
import logging

log = logging.getLogger(__name__)

class Worker(object):
    """The Worker class."""

    def __init__(self):
        log.info("Initializing the Worker object")

        self.data = {}
        self.data["fibonacci"] = None
        self.data["collatz"] = []

    def _fib(self, n):
        a, b = 0, 1
        log.debug("a is %d, b is %d", a, b)
        for _ in range(n):
            time.sleep(0.1)
            a, b = b, a + b
            log.debug("a is %d, b is %d", a, b)
        return a

    def _collatz(self, n):
        log.debug("n is %d", n)
        while True:
            time.sleep(0.1)
            if n == 1:
                log.debug("n is %d", n)
                break
            if n % 2 == 0:
                n = n // 2
                log.debug("n is %d", n)
                yield n
            else:
                n = n * 3 + 1
                log.debug("n is %d", n)
                yield n
        yield n

    def compute(self):
        """Perform some computation."""
        log.info("Started computing ...")

        seed = random.randint(10, 19)
        log.info("The seed is %d", seed)

        fib = self._fib(seed)
        log.info("The %d-th Fibonacci number is %d", seed, fib)
        self.data["fibonacci"] = fib

        for c in self._collatz(fib):
            self.data["collatz"].append(c)


from pelican import signals
from datetime import datetime
import pytz

def hide_events(generator):

    for event in generator.context["filenames"].values():
        if hasattr(event, "event_start"):
            mytz = pytz.timezone(generator.settings["TIMEZONE"])
            dtnow = pytz.utc.localize(datetime.utcnow()).astimezone(mytz)
            delta = generator.settings.get("SHOW_FUTURE_EVENTS_IN_BLOG_UNTIL")
            if (delta and mytz.localize(event.event_start) - dtnow > delta) or\
                    mytz.localize(event.event_end) < dtnow:
                event.status = "draft"
                if event in generator.articles:
                    generator.articles.remove(event)
                if event not in generator.drafts:
                    generator.drafts.append(event)

def register():
    signals.article_generator_pretaxonomy.connect(hide_events)


import pickle
import base64
from django_redis import get_redis_connection

def merge_cart_cookie_to_redis(request, user, response):
    """
    合并请求用户的购物车数据，将未登录保存在cookie里的保存到redis中
    :param request: 用户的请求对象
    :param user: 当前登录的用户
    :param response: 响应对象，用于清楚购物车cookie
    :return:
    """
    cookie_cart = request.COOKIES.get('carts')
    if cookie_cart is not None:
        return response

    cookie_cart = pickle.loads(base64.b64decode(cookie_cart.encode()))
    redis_conn = get_redis_connection('carts')
    pl = redis_conn.pipeline()
    for electron_id, data in cookie_cart.items():
        # 修改商品数量
        pl.hset("cart_%s" % user.id, electron_id, data['count'])
    pl.execute()

    # 4. 清除cookie中的购物车
    response.delete_cookie('carts')

    return response


#!/usr/bin/python
#-----------------------------------------------------------
#  Extracts GB Users with point totals for 0-1000 range
#-----------------------------------------------------------

import csv
import requests
from bs4 import BeautifulSoup
import time
import lxml

timestr = time.strftime("%Y-%m-%d-%H"+"00")  # Sets date string used in filename

import sys			# This section corrects for
#reload(sys)			# UNICODE characters and converts
#sys.setdefaultencoding('utf-8') # to ASCII

url = 'http://www.kcgasprices.com/Member_pts_leader.aspx?range=2000'
response = requests.get(url, timeout=5)
html = response.content

#soup = BeautifulSoup(lxml)
soup = BeautifulSoup(html, 'html.parser')

table = soup.find('div', attrs={'class': 'Wrapper'})

table = soup.find('table', attrs={'style': 'width: 100%;'})

# <table style="width: 100%;">

list_of_rows = []
for row in table.findAll('tr'):
    list_of_cells = []
    for cell in row.findAll('td'):
        text = cell.text.replace(' ', '')
        list_of_cells.append(text)
    list_of_rows.append(list_of_cells)

filename = "/home/juren/Projects/GB/Extended/GB-2K-"+timestr+".csv"
outfile = open(filename, "w")
writer = csv.writer(outfile)
writer.writerow(["Rank", "Name", "Location", "Points"])
writer.writerows(list_of_rows)

outfile.close()

# =============================================================

##import csv
##with open(filename, 'rt') as inp, open('corrected.csv', 'wt') as out:
##    writer = csv.writer(out)
##    for row in csv.reader(inp):
##        if row[1] != "2001":
##            writer.writerow(row)

import pandas
import locale

print (filename)

locale.setlocale(locale.LC_NUMERIC, '')

df = pandas.read_csv(filename)

df1 = df['Points'].apply(locale.atoi)

del df['Points']

Name = df['Name']

Rank = df['Rank']

Location = df['Location']

df2 = pandas.concat([Name,Rank,Location,df1],axis=1)

df2.columns = ['Name','Rank','Location','Points']

df2 = df2.set_index('Name')

df2.to_csv(filename)


import matplotlib.pyplot as plt
import matplotlib.animation as ani
fig=plt.figure()
ax1=fig.add_subplot(1,1,1)
def animate(i):
    file=open("sample.txt","r+").read()
    print(file)
    ldata=file.split("\n")
    xdata=[]
    ydata=[]
    for x in ldata:
        if len(x)>1:
            temp=x.split(" ")
            xdata.append(temp[0])
            ydata.append(temp[1])
    ax1.clear()
    ax1.plot(xdata,ydata)
anim=ani.FuncAnimation(fig,animate,interval=1000)


from conll_reader import DependencyStructure, DependencyEdge, conll_reader
from collections import defaultdict
import copy
import sys

import numpy as np
import keras

from extract_training_data import FeatureExtractor, State

class Parser(object):

    def __init__(self, extractor, modelfile):
        self.model = keras.models.load_model(modelfile)
        self.extractor = extractor

        # The following dictionary from indices to output actions will be useful
        self.output_labels = dict([(index, action) for (action, index) in extractor.output_labels.items()])

    def parse_sentence(self, words, pos):
        state = State(range(1,len(words)))
        state.stack.append(0)

        while state.buffer:
            pass
            # TODO: Write the body of this loop for part 4

            dep_relations = ['tmod', 'vmod', 'csubjpass', 'rcmod', 'ccomp', 'poss', 'parataxis', 'appos', 'dep', 'iobj', 'pobj', 'mwe', 'quantmod', 'acomp', 'number', 'csubj', 'root', 'auxpass', 'prep', 'mark', 'expl', 'cc', 'npadvmod', 'prt', 'nsubj', 'advmod', 'conj', 'advcl', 'punct', 'aux', 'pcomp', 'discourse', 'nsubjpass', 'predet', 'cop', 'possessive', 'nn', 'xcomp', 'preconj', 'num', 'amod', 'dobj', 'neg','dt','det']
            output_labels_2 = {}
            output_labels_2[0] = ('shift',None)
            count3 = 1
            for dep in dep_relations:
                output_labels_2[count3] = ('left_arc',dep)
                count3 = count3 + 1

            count4 = 46
            for dep in dep_relations:
                output_labels_2[count4] = ('right_arc',dep)
                count4 = count4 + 1

            # citation: https://docs.scipy.org/doc/numpy/reference/generated/numpy.reshape.html
            in_rep = np.reshape(self.extractor.get_input_representation(words,pos,state),(-1,6))
            probs = self.model.predict(in_rep)
            # citation: https://docs.scipy.org/doc/numpy/reference/generated/numpy.ndarray.flatten.html
            probs = probs.flatten()

            count = 0
            probs_new = []
            for prob in probs:
                probs_new.append((prob,count))
                count = count + 1
            probs_new.sort()

            count2 = -1
            proceed = True
            while proceed:
                highest_prob = probs_new[count2][1]

                if len(state.stack) == 0 and (output_labels_2[highest_prob][0] == 'left_arc' or output_labels_2[highest_prob][0] == 'right_arc'):
                    count2 = count2 - 1
                elif len(state.stack) == 1 and output_labels_2[highest_prob][0] == 'left_arc':
                    count2 = count2 - 1
                elif len(state.stack) > 0 and len(state.buffer) == 1 and output_labels_2[highest_prob][0] == 'shift':
                    count2 = count2 - 1
                elif output_labels_2[highest_prob][0] == 'left_arc' and highest_prob > 45:
                    count2 = count2 - 1
                elif output_labels_2[highest_prob][0] == 'right_arc' and highest_prob < 46:
                    count2 = count2 - 1
                elif output_labels_2[highest_prob][0] == 'shift' and highest_prob > 0:
                    count2 = count2 - 1
                else:
                    if output_labels_2[highest_prob][0] == 'left_arc':
                        state.left_arc(output_labels_2[highest_prob][1])
                        proceed = False
                    elif output_labels_2[highest_prob][0] == 'right_arc':
                        state.right_arc(output_labels_2[highest_prob][1])
                        proceed = False
                    elif output_labels_2[highest_prob][0] == 'shift':
                        state.shift()
                        proceed = False

        result = DependencyStructure()
        for p,c,r in state.deps:
            result.add_deprel(DependencyEdge(c,words[c],pos[c],p, r))
        return result

if __name__ == "__main__":

    WORD_VOCAB_FILE = 'data/words.vocab'
    POS_VOCAB_FILE = 'data/pos.vocab'

    try:
        word_vocab_f = open(WORD_VOCAB_FILE,'r')
        pos_vocab_f = open(POS_VOCAB_FILE,'r')
    except FileNotFoundError:
        print("Could not find vocabulary files {} and {}".format(WORD_VOCAB_FILE, POS_VOCAB_FILE))
        sys.exit(1)

    extractor = FeatureExtractor(word_vocab_f, pos_vocab_f)
    parser = Parser(extractor, sys.argv[1])

    with open(sys.argv[2],'r') as in_file:
        for dtree in conll_reader(in_file):
            words = dtree.words()
            pos = dtree.pos()
            deps = parser.parse_sentence(words, pos)
            print(deps.print_conll())
            print()


"""
Configuration file for infotaxis simulation.
"""

import numpy as np
from fluids import logprob_odor
from infotaxis import sampling_functions

# SIMULATION DESCRIPTORS
simulation_id = '2D-demo-collimated'
simulation_type = 'demo'
simulation_description = 'This simulation runs the infotaxis algorithm once \
through a collimated plume in a 2D environment. It is for demonstration \
purposes.'

# SPATIOTEMPORAL DISCRETIZATION PARAMETERS
XRBINS = np.linspace(-.3, 1.0, 91)
YRBINS = np.linspace(-.15, .15, 31)
ZRBINS = np.linspace(-.15, .15, 2)
DX = XRBINS[1] - XRBINS[0]
DY = YRBINS[1] - YRBINS[0]
DZ = ZRBINS[1] - ZRBINS[0]
DT = .1 # timestep (s)

# PLUME PARMETERS TO LOOP OVER
#PLUME_TYPEs = ['collimated']
#PLUME_SRC_POS = 'uw_center'
#PLUME_PARAMSs = [{'src_xr':'full',
#                  'src_yr':'full',
#                  'src_zr':'full',
#                  'wind': .4, # m/s
#                  }]
#PLUME_AUX_PARAMSs = [{'width': .0005, # m
#                      'peak': 70, # conc
#                      }]
PLUME_TYPEs = ['empty']
PLUME_SRC_POS = 'none'
PLUME_PARAMSs = [{'src_xr':'full',
                  'src_yr':'full',
                  'src_zr':'full',
                  'wind': .4, # m/s
                  }]
PLUME_AUX_PARAMSs = [{}]

# SIMULATION PARAMETERS
MAX_TIMESTEP = 10000
NTRIALS = 1

# INSECT PARAMETERS TO LOOP OVER
VRs = [.01,] # visual radius
Rs = [100,] # source rate (conc*m^2/s)
Ds = [.001] # diffusion coeff (m^2/s)
As = [.00000001] # particle size (m)
TAUs = [25.] # particle lifetime (s)
THs = [50.] # threshold (conc)

# INSECT FUNCTIONS
LOGLIKE = logprob_odor.binary_advec_diff_tavg
SAMPLING_FUNCTION = sampling_functions.threshold

# INSECT APPROXIMATION PARAMETERS
APPROX_PARAMS = {}


#!/usr/bin/env python3

from sched import scheduler
import time
import yaml
import tweepy
from postgres import Postgres
from os import environ
from telegram.ext import Updater

import logging
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO)
logger = logging.getLogger(__name__)

with open('config.yml', 'r') as ymlfile:
    cfg = yaml.load(ymlfile)

db = Postgres(environ.get("DATABASE_URL"))

twitter_auth = tweepy.OAuthHandler(environ.get("TWITTER_CONSUMER_KEY"), environ.get("TWITTER_CONSUMER_SECRET"))
twitter_auth.set_access_token(environ.get("TWITTER_ACCESS_TOKEN"), environ.get("TWITTER_ACCESS_TOKEN_SECRET"))
twitter = tweepy.API(twitter_auth)

def read_last_twit_from_db() -> str:
    return db.one("SELECT last_twit_id FROM twitlegram WHERE id=1")

def write_last_twit_to_db(twit: str):
    db.run("INSERT INTO twitlegram (id, last_twit_id)"
           " VALUES (1, %(last_twit_id)s)"
           " ON CONFLICT (id) DO"
           " UPDATE SET last_twit_id = excluded.last_twit_id;", {"last_twit_id": twit})

def post_twit(bot, job):
    bot.send_message(chat_id=cfg['chat_id'], text=job.context)

def fetch_latest_twits(queue):
    logger.info('Fetching...')
    last_twit_id: str = read_last_twit_from_db()

    if last_twit_id:
        twits = twitter.user_timeline(screen_name=cfg['screen_name'], since_id=last_twit_id)
    else:
        twits = twitter.user_timeline(screen_name=cfg['screen_name'], count=1)

    if twits:
        write_last_twit_to_db(twits[0].id_str)

    when: int = 0
    for twit in reversed(twits):
        logger.info(twit.text)
        queue.run_once(post_twit, ++when, twit.text)

def periodic(sched, interval, action, actionargs=()):
    sched.enter(interval, 1, periodic, (sched, interval, action, actionargs))
    action(*actionargs)

def main():
    db.run("CREATE TABLE IF NOT EXISTS twitlegram ("
           "id INTEGER NOT NULL UNIQUE,"
           "last_twit_id VARCHAR(100));")

    upd = Updater(environ.get("TELEGRAM_BOT_TOKEN"))
    queue = upd.job_queue
    upd.start_polling()
    sched = scheduler(time.time, time.sleep)
    periodic(sched, cfg['fetch_interval'], fetch_latest_twits, [queue])
    sched.run()

if __name__ == '__main__':
    main()


import socket
import pyaudio
import numpy as np
import wave
import math
import struct

# Socket
HOST = "192.168.1.218"
PORT = 10010

# Audio
p = pyaudio.PyAudio()
CHUNK = 1024 * 4
FORMAT = pyaudio.paInt16
CHANNELS = 4
RATE = 44100
cRATE = 22050

#record channel
fm, f0, f1, f2, f3 = [], [], [], [], []

def save(name, channels, rate, frames):
    wf = wave.open(name, 'wb')
    wf.setnchannels(channels)
    wf.setsampwidth(2)
    wf.setframerate(rate)
    wf.writeframes(b''.join(frames))
    wf.close()

def sig(num):
    return 1/(1 + math.exp(-10 * num))

def amplitude(block):
    count = len(block)/2
    count = count if count != 0 else 1
    format = "%dh"%(count)
    shorts = struct.unpack(format, block)

    sum_squares = 0.0
    for sample in shorts:
        n = sample * (1.0/32768.0)
        sum_squares += n**2

    # return 100 * (sum_squares/count) ** 0.5
    return int(2 * (100 * sig((sum_squares/count) ** 0.5) - 50))

def avgfreq(block):
        result = np.fromstring(block, dtype=np.int16)
        w = np.fft.fft(result)
        freqs = np.abs((np.fft.fftfreq(len(w))*44100))
        w = np.abs(w)
        indices = np.argsort(w)[int(len(w)*0.99):]
        print(indices)
        bin_const = 10
        indices = indices[len(indices) % bin_const:]
        w = w[indices]
        freqs = freqs[indices]
        w = w[np.argsort(freqs)]
        freqs = np.sort(freqs)
        w = np.reshape(w, (-1, bin_const)).sum(axis=1)

        if len(freqs) == 0:
            return None

        freqs = np.average(np.reshape(freqs, (-1, bin_const)), axis=1)
        w = w / np.sum(w)
        freqdict = dict(zip(freqs, w))
        wsum = sum(w)
        wsum = wsum if wsum != 0 else 1
        out = 0
        for i in range(len(w)):
            out += w[i] * freqs[i] / wsum
        return 0 if math.isnan(float(out)) else int(out)

with socket.socket() as server_socket:
    server_socket.bind((HOST, PORT))
    server_socket.listen(1)
    conn, address = server_socket.accept()
    print("Connection from " + address[0] + ":" + str(address[1]))
    data = conn.recv(8192)

    while data != "":
        try:
            data = conn.recv(8192)

            channels = np.fromstring(data, dtype='int16')
            c0 = channels[0::8]
            c1 = channels[1::8]
            c2 = channels[2::8]
            c3 = channels[3::8]

            fm.append(data)
            f0.append(c0.tostring())
            f1.append(c1.tostring())
            f2.append(c2.tostring())
            f3.append(c3.tostring())

            print(amplitude(f0[-1]), avgfreq(f0[-1]), "\t", amplitude(f1[-1]), avgfreq(f1[-1]), "\t", amplitude(f2[-1]), avgfreq(f2[-1]), "\t", amplitude(f3[-1]), avgfreq(f3[-1]))
        except (socket.error, KeyboardInterrupt) as e:
            print("Client Disconnected")

            save("combinedpost.wav", 4, RATE, fm)

            save("post0.wav", 1, cRATE, f0)
            save("post1.wav", 1, cRATE, f1)
            save("post2.wav", 1, cRATE, f2)
            save("post3.wav", 1, cRATE, f3)

            print("Done Saving")
            break


import matplotlib.pyplot as plt

def plt_result(inputs, predictions, target, directory):
    fig = plt.figure()
    ax=fig.add_axes([0,0,1,1])
    ax.scatter(predictions, target, color='b')
    ax.plot(predictions, predictions, color='r')
    ax.set_xlabel('Grades Range')
    ax.set_ylabel('Grades Scored')
    ax.set_title('scatter plot')
    plt.xlabel('True Energy')
    plt.xlabel('Calibrated energy')
    fig.savefig(directory + "/figures.png")
    #plt.show()


#!/usr/bin/env python3.3
# -*- coding: utf8 -*-

from pprint import pprint as prn # < DEBUG
import re
import logging
import inspect
import threading
import os.path
from abc import ABCMeta
from abc import abstractmethod
from collections import deque
from collections import Sequence
from collections import MutableSequence
from multiprocessing import cpu_count
from concurrent import futures
from urllib import request
from urllib.error import URLError
from lxml import etree

_MAX_THREADS = 8

def threads_count ():
    try:
        return cpu_count()
    except NotImplementedError:
        return _MAX_THREADS

class Poller (object):
    __metaclass__ = ABCMeta

    @abstractmethod
    def __init__ (self, pool):
        self.__pool = pool
        self.__lock = threading.RLock()
        self.__pendings = set()

    @property
    def pendings (self):
        with self.__lock:
            return self.__pendings

    def __discard_pending (self, future):
        with self.__lock:
            self.__pendings.discard(future)

    @abstractmethod
    def _actor (self, *args, **kwargs):
        pass

    def run (self, *args, **kwargs):
        with self.__lock:
            future = self.__pool.submit(self._actor, *args, **kwargs)
            self.__pendings.add(future)
            future.add_done_callback(self.__discard_pending)
            return future

    @abstractmethod
    def result (self, *args, **kwargs):
        pass

class Fetcher (Poller):
    _log = logging.getLogger('Fetcher')

    def __init__ (self, pool):
        super().__init__(pool)
        self.__out = {}

    def _actor (self, url, path):
        """path can be a single value or a sequence of values."""
        # Create XPath objects.
        if isinstance(path, str):
            path = (path,)
        if isinstance(path, (Sequence, MutableSequence)):
            try:
                xpath = [etree.XPath(p) for p in path]
            except etree.XPathError:
                self._log.error('XPath error: "%s"', path)
                return False
        else:
            return False
        # Open URL.
        try:
            response = request.urlopen(url)
        except URLError:
            self._log.error('Can\'t open URL: "%s"', url)
            return False
        # Get elements.
        tree = etree.parse(response, etree.HTMLParser())
        h = hash((url, tuple(path)))
        self.__out[h] = [xp(tree) for xp in xpath]
        return h

    def result (self, h=None):
        if h:
            return self.__out[h]
        else:
            return self.__out

    def clear (self):
        self.__out.clear()

class App (object):
    _log = logging.getLogger('App')

    def __init__ (self, root=None, categories=None, resolutions=None,
                  output='.'):
        self.__pool = futures.ThreadPoolExecutor(threads_count())
        self.__fetcher = Fetcher(self.__pool)
        self.root = root
        self.categories = categories
        self.resolutions = resolutions
        self.output = output

    def __verify_input (self, tpl, regex):
        if isinstance(tpl, tuple):
            rx = re.compile(regex)
            return all((isinstance(s, str) and rx.match(s) for s in tpl))
        return False

    def __getter (self, caller):
        return getattr(self, '_{0}__{1}'
                       .format(self.__class__.__name__, caller),
                       None)
    def __setter (self, caller, tpl, regex=r'^.*$'):
        if self.__verify_input(tpl, regex):
            setattr(self, '_{0}__{1}'.format(self.__class__.__name__, caller),
                    tpl)
            return True
        return False

    @property
    def root (self):
        return self.__getter(inspect.stack()[0][3])
    @root.setter
    def root (self, root):
        if isinstance(root, str) and len(root.strip()) > 0:
            self.__root = root
        else:
            self._log.error('URL must be a non-empty string')

    @property
    def categories (self):
        return self.__getter(inspect.stack()[0][3])
    @categories.setter
    def categories (self, ctgs):
        """ "*" passed as a category name tells to fetch all subcategories for
        all of the categories that available on site."""
        if not self.__setter(inspect.stack()[0][3], ctgs,
                             r'^(?:\*|(?:\w+|\w+\s+)+)$'):
            self._log.error('Categories must be represented as a tuple of '
                            'string(s)')

    @property
    def resolutions (self):
        return self.__getter(inspect.stack()[0][3])
    @resolutions.setter
    def resolutions (self, resl):
        if not self.__setter(inspect.stack()[0][3], resl, r'^\d+[xX]\d+$'):
            self._log.error('Resolutions must be represented as a tuple of '
                            'string(s)')

    @property
    def output (self):
        return self.__getter(inspect.stack()[0][3])
    @output.setter
    def output (self, path):
        if isinstance(path, str) and os.path.exists(path):
            self.__output = path
        else:
            self._log.error('Given path "%s" didn\'t exists', path)

    @staticmethod
    def __pretty (iterable):
        return (x.rsplit(' ', 1)[0] for x in iterable)

    @staticmethod
    def __max_page (iterable):
        return max([int(p.rsplit('/', 1)[-1]) for p in iterable])

    @staticmethod
    def __updkey ():
        pass

    def __send_to_pool (self, mapping, path, dct, *functors):
        i = 1
        ff = {}
        page = 1
        total = 0
        is_url = False
        for c in mapping:
            if isinstance(mapping[c], str):
                ff[self.__fetcher.run(self.root + mapping[c], path)] = c
            elif isinstance(mapping[c], (Sequence, MutableSequence)):
                is_url = True
                total += len(mapping[c])
                for url in mapping[c]:
                    ff[self.__fetcher.run(self.root + url, path)] = c
            else:
                return None
        for f in futures.as_completed(ff):
            if not is_url:
                self._log.info('"%s" was successfully fetched', mapping[ff[f]])
            else:
                self._log.info('Page was successfully fetched (%d/%d)', page,
                               total)
                page += 1
            results = self.__fetcher.result(f.result())
            if 1 == len(functors):
                if self.__updkey is functors[0]:
                    dct[(url, i)] = results[0]
                    i += 1
                else:
                    dct[ff[f]] = functors[0](results[0]) if results[0] else 1
            else:
                if 2 == len(functors):
                    _ = dict
                if len(functors) > 2:
                    _ = list
                for i in range(len(functors)):
                    results[i] = tuple(functors[i](results[i]))
                    x = _(zip(*results))
                    if not is_url:
                        x = tuple([x, mapping[ff[f]]])
                    if is_url and ff[f] in dct:
                        dct[ff[f]].update(x)
                    else:
                        dct[ff[f]] = x

    def get_categories (self, show=False):
        """Fetch all of the categories and its subcategories."""
        self.__fetcher.clear()
        path = '//ul[@class="side-panel categories"]/li{0}/a'
        # Root categories.
        self._log.info('Fetching root categories for "%s"...', self.root)
        fut = self.__fetcher.run(self.root, [path.format('') + '/@title',
                                             path.format('') + '/@href'])
        futures.wait(self.__fetcher.pendings)
        res = self.__fetcher.result(fut.result())
        kategories = dict(zip(self.__pretty(res[0]), res[1]))
        # Subcategories.
        if '*' in self.categories:
            self.categories = tuple(kategories.keys())
            sel_ctgs = kategories
        if not all(c in kategories for c in self.categories):
            self._log.error('Unknown category(ies)')
            return None
        else:
            sel_ctgs = {k: kategories[k] for k in self.categories}
        path = path.format('[@style="padding-left:5px;"]')
        subs = {}
        self._log.info('Fetching subcategories...')
        self.__send_to_pool(sel_ctgs, [path + '/@title', path + '/@href'],
                            subs, self.__pretty, lambda x: x)
        if show:
            for cat in subs:
                if len(subs[cat][0]) > 0:
                    self._log.info('Subcategories for category "%s" ==>', cat)
                    for subcat in subs[cat][0]:
                        self._log.info('Sub: "%s"', subcat)
                    self._log.info('"%s" |', cat)
                else:
                    self._log.info('Category: "%s" ==>|', cat)
        return subs

    def get_wallpapers (self, subcategories):
        """Download wallpapers according to its resolution and category."""
        if not subcategories or len(subcategories) < 1:
            return None
        s = set(subcategories)
        flat_ctgs = {}
        for cat in self.categories:
            if cat in s:
                flat_ctgs[cat] = subcategories[cat][1]
                s.discard(cat)
        ss = {}
        for c in s:
            ss.update(subcategories[c][0])
        for cat in self.categories:
            if cat in ss:
                flat_ctgs[cat] = ss[cat]
        self.__fetcher.clear()
        path = '//div[@class="pagination"]/a[contains(@href, "page/")]/@href'
        pages = {}
        self._log.info('Fetching pages for an each category...')
        self.__send_to_pool(flat_ctgs, path, pages, self.__max_page)
        wallpapers = {k: [flat_ctgs[k].rsplit('.', 1)[0] + '/page/' + str(p)
            if p > 1 else flat_ctgs[k].rsplit('.', 1)[0]
            for p in range(1, pages[k] + 1)] for k in flat_ctgs}
        path = ('//li[@class="wall"]/div[@class="thumb"]/div[@class="mini-hud"]'
                '/a')
        walls = {}
        self._log.info('Fetching wallpapers by pages...')
        self.__send_to_pool(wallpapers, [path + '/@href', path + '/@title'],
                            walls, lambda x: x, lambda x: x)
        flat_walls = {}
        for c in walls:
            flat_walls[c] = list(walls[c])
        r = deque(self.resolutions)
        res = ' and ('
        try:
            while True:
                res += 'contains(@href, "-{0}.jpg")'.format(r.popleft())
                if len(r):
                    res += ' or '
        except IndexError:
            res += ')'
        path = ('//div[@class="wallpaper-resolutions"]/a[@target="_self"{0}]'
                '/@href'.format(res))
        resolutions = {}
        self._log.info('Fetching wallpaper resolutions...')
        self.__send_to_pool(flat_walls, path, resolutions, self.__updkey)
        return resolutions

def main ():
    logging.basicConfig(level=logging.INFO)

    app = App('http://wallpaperswide.com/', ('Charity', 'Motors',), ('2560x1440',))

    # DEBUG ==>
    subs = app.get_categories(True)
    # app.categories = ('ATV',)
    prn(app.get_wallpapers(subs))

if '__main__' == __name__:
    main()


import uncertainpy as un
import chaospy as cp
import numpy as np
import scipy.integrate as integrate
import time
from tqdm import tqdm
from tqdm import tqdm_gui
import matplotlib.pyplot as plt

from uqtools import *

class MethodOfLines:

    def __init__(self,N,tmax):
        #variaveis de inicializacao
        self.N = N # number of ages
        self.xmin = 0
        self.xmax  = 50 # number of classes of ages (50)
        #self.agePt = np.linspace(0, ageFim, ageNpts)
        #self.ageCont = 0
        self.tmax = tmax # number of days for simulation (31)
        self.dt   = 0.01 # time step
        #self.tempoPt = np.linspace(0, tempoFim, tempoNpts)
        #self.tempoCont = 0
        self.Virus = []
        self.v0 = 6.4799*10**5
        self.Target = []
        self.t0 = 1.3*10**5
        #self.RNAt = 1
        self.RNAp = 1
        #self.RNAn = 0
        self.delta = 0.07
        self.beta = 5*10**-8
        self.theta = 1.2
        self.s     = 1.3*10**5
        self.d     = 0.01
        self.beta  = 5*10**-8
        self.c     = 19.0
        self.rho   = 12.0
        self.prm_alpha = 22
        self.mu    = 1
        self.initializeDomain()
        self.initializeVariables()
        self.initializeParams()

    def initializeDomain(self):
        self.dx = (self.xmax-self.xmin)/self.N
        self.x = np.arange(self.xmin-self.dx, self.xmax+(2*self.dx), self.dx)
        self.time = np.arange(0,self.tmax, self.dt)

    def Infected(self,x,args):
        v,t=args
        var = v*t*self.beta*np.exp(-self.delta*x)
        return var

    def Target(self):
        return self.s - self.d*self.target - self.v*self.target*self.beta

    def initializeVariables(self):
        # target cell
        self.t0 = self.t0+(self.s - self.d*self.t0 - self.v0*self.t0*self.beta)*self.dt
        self.target = self.t0
        # infected cell
        self.u0 = self.v0*self.t0*self.beta*np.exp(-self.delta*self.x)
        self.u = self.u0.copy()
        self.unp1 = self.u0.copy()
        # rna
        self.rna_0 = self.RNAp*np.exp(self.x)
        self.rna = self.rna_0.copy()
        self.unp2 = self.rna_0.copy()
        # virus
        res = integrate.quad(self.Infected,self.xmin,self.xmax,[self.v0,self.target])
        self.v0 = self.v0+(self.rho*res[0]-self.v0*self.c)*self.dt
        self.v = self.v0

    def initializeParams(self):
        self.nsteps = round(self.tmax/self.dt)
        self.alpha = self.dt/(2*self.dx)
        self.MEAN_PATS = [6.47991433, 6.42897983, 5.857277, 4.63766183, 4.08108333, 3.38275467, 3.18124267, 2.88121, 2.66053917]#, 2.54078967, 2.487822, 2.21939417, 1.90103167, 1.7158415]
        self.t_exp = [0.04, 0.09, 0.17, 0.34, 0.50, 1.00, 1.50, 2.98, 4.91]#, 6.92, 10.94, 13.96, 20.91, 27.98]

    def solve_and_plot(self):
        tc = 0

        for i in range(self.nsteps):
            plt.clf()

            # The Lax-Friedrichs scheme
            #for j in range(self.N):
            #    self.unp1[j] = self.u[j]-self.alpha*(self.u[j+1]-self.u[j-1])+(1/2)*(self.u[j+1]-2*self.u[j]+self.u[j-1])

            # finite differences
            for j in range(self.N):
                self.unp1 =  self.u[j]+(-self.delta*self.u[j]+self.alpha*self.u*(self.u[j]-self.u[j-1]))
                self.unp2 =  self.rna[j]+(self.prm_alpha*self.rna[j]-(self.mu+self.rho)*self.rna[j]+self.alpha*self.rna*(self.rna[j]-self.rna[j-1]))

            # infected cells
            self.u = self.unp1.copy()

            # rna
            self.rna = self.unp2.copy()

            # virus
            exported_virus = integrate.quad(self.Infected,self.xmin,self.xmax,[self.v,self.target])
            self.v = self.v+(self.dt*(self.rho*exported_virus[0]-self.c*self.v))
            self.Virus.append(self.v)
            # target cells
            self.target = self.target + self.dt*(self.s - self.d*self.target - self.v*self.target*self.beta)
            self.Target.append(self.target)

            #plt.plot(self.x,self.u,'bo-', label= "")
            #plt.ylabel('Infected cells')
            #plt.ylim(0.0,8.0)
            plt.plot(self.x,self.rna,'bo-', label= "")
            plt.xlabel('day')
            plt.ylabel('RNA')
            plt.legend(loc='upper left', prop={'size':13})
            #plt.pause(0.01)
            plt.grid(True)
            tc += self.dt

        plt.figure()
        plt.plot(self.time,self.Virus,'bo-', label= "")
        plt.plot(self.t_exp, self.MEAN_PATS, 'r', label='data', linewidth=4)
        plt.xlabel('day')
        plt.ylabel('HVC RNA log_{10}')
        plt.yscale('log')
        plt.grid(True)

        plt.figure()
        plt.plot(self.time,self.Target,'bo-', label= "")
        plt.xlabel('day')
        plt.ylabel('Target Cells')
        plt.yscale('log')
        plt.grid(True)

    '''
    def calcIntegral(I,Rp,Rt):
        soma = 0.0
        for a in range(0, ageNpts):
            soma = soma + I[a]*Rp[a]*Rt[a]
        return soma/float(ageNpts)
    '''

    #recebe como parametro os parametros estocasticos, individuos, e os valores experimentais
    def viralmodelfit(prms):

        [alpha, r, rho, epsilon_alpha, epsilon_r, delta] = prms
        #Inicializacao
        #parametros
        s     = 1.3*10**5
        d     = 0.01
        beta  = 5*10**-8
        c     = 19.0

        k     = 0.80 # coeficiente da funcao exponencial de atraso na exportacao de RNA positivo
        tau   = 0.50 # tempo de atraso para a exportacao de RNA positivo
        Rmax  = 50.0 # numero maximo de RNA negativo / complexo de replicacao (Rn)
        sigma = 4.0  # taxa de formacao de complexos de replicacao
        mu_t  = 0.8  # 0.8 no artigo # decaimento natural de Rt
        theta = 1.20 # taxa de disponibilidade para traducao
        mu_c  = 2.8  # 0.89 no artigo # decaimento natural de Rc e Rn
        epsilon_s = 0.998

        #variaveis
        I = np.zeros((tempoNpts, ageNpts))
        Rp = np.zeros((tempoNpts, ageNpts))
        Rt = np.zeros((tempoNpts, ageNpts))
        Rn = np.zeros((tempoNpts, ageNpts))
        V = np.zeros(tempoNpts)
        T = np.zeros(tempoNpts)

        def diferencasfinitas():
            #Solve
            kappa_t       = 1.0 #poi[3]
            kappa_c       = 1.0 #poi[4]

            for t in range(1, tempoNpts):

                T[t] = (s - d*T[t-1] - beta*V[t-1]*T[t-1])*deltaT + T[t-1]

                V[t] =  deltaT*((1 - epsilon_s)*rho*calcIntegral(I[t-1],Rp[t-1],Rt[t-1])
                - c*V[t-1]) + V[t-1]

                rho1 = 0.0

                Rp[t][0] = 0
                Rn[t][0] = 0
                Rt[t][0] = 1
                I[t][0]  = beta*V[t]*T[t]

                for a in range(1, ageNpts):

                    if(float(a*deltaA) < tau):
                        rho1 = 0
                    else:
                        rho1 = rho*(1 - np.exp(-k*(float(a*deltaA) - tau)))

                    I[t][a] = (-delta*I[t-1][a] - (I[t-1][a] - I[t-1][a-1])/deltaA)*deltaT + I[t-1][a]

                    Rn[t][a] = ((1 - epsilon_r)*r*Rp[t-1][a]*(1 - (Rn[t-1][a]/Rmax)) - kappa_c*mu_c*Rn[t-1][a]
                                - (Rn[t-1][a] - Rn[t-1][a-1])/(deltaA ))*deltaT + Rn[t-1][a]

                    Rp[t][a] = ((1 - epsilon_alpha)*alpha*Rn[t-1][a] + sigma*Rt[t-1][a] - theta*Rp[t-1][a]
                                - (1- epsilon_s)*rho1*Rp[t-1][a] - kappa_c*mu_c*Rp[t-1][a]
                                - (Rp[t-1][a] - Rp[t-1][a-1])/(deltaA))*deltaT + Rp[t-1][a]

                    Rt[t][a] = (theta*Rp[t-1][a] - sigma*Rt[t-1][a] - (1 - epsilon_s)*rho1*Rt[t-1][a] - kappa_t*mu_t*Rt[t-1][a]
                                - (Rt[t-1][a] - Rt[t-1][a-1])/(deltaA))*deltaT + Rt[t-1][a]
            return V

        #condicoes iniciais
        T0 = 1.3*10**5 #PAT83:
        V0 = 6.4799*10**5
        I0_t0 = beta*T0*V0

        Rt0_t0 = 1
        Rp0_t0 = 0
        Rn0_t0 = 0

        T[0] = T0
        V[0] = V0
        Rt[0][0] = Rt0_t0
        Rp[0][0] = Rp0_t0
        Rn[0][0] = Rn0_t0
        I[0][0] = I0_t0

        rho1 = 0.00
        # Condicao inicial para as EDPs
        for ageCont in range(1, ageNpts):
            I[0][ageCont] = (beta*T[0]*V[0]*np.exp(-deltaA*ageCont))*deltaA + I[0][ageCont-1]
            if ageCont*deltaA < tau:
                rho1 = 0
            else:
                rho1 = (1 - np.exp(-k*((ageCont*deltaA) - tau)))*rho
            Rn[0][ageCont] = (r*Rp[0][ageCont-1] - r*Rp[0][ageCont-1]*(Rn[0][ageCont-1]/Rmax)- mu_c*Rn[0][ageCont-1])*deltaA + Rn[0][ageCont-1]
            Rp[0][ageCont] = (alpha*Rn[0][ageCont-1]+sigma*Rt[0][ageCont-1]-theta*Rp[0][ageCont-1]-rho1*Rp[0][ageCont-1]- mu_c*Rp[0][ageCont-1])*deltaA + Rp[0][ageCont-1]
            Rt[0][ageCont] = (theta*Rp[0][ageCont-1] - sigma*Rt[0][ageCont-1]- rho1*Rt[0][ageCont-1] - mu_t*Rt[0][ageCont-1])*deltaA + Rt[0][ageCont-1]
        #Fim das condicoess iniciais----------------------------------------------+

        # Executa o metodo numerico para resolver
        # Precisa passar como parametro as condicoes iniciais
        V = diferencasfinitas()

        # Passa para a base log o resultado
        V_log = np.log10(V)

        return tempoPt, V_log

    ##############FIM MODELO################

    '''

    model = un.Model(
        run = viralmodelfit,
        labels=["Tempo (dias)",
            "Carga viral (log10)"]
    )

    epsilon_r = 0.3
    epsilon_alpha = 0.9
    delta = 0.07
    alpha = 22
    r = 11.1
    rho = 12.0

    # create pdf distributions
    min_bound = 0.9
    max_bound = 1.1

    pdf_alpha = cp.Uniform(alpha*min_bound,alpha*max_bound)
    pdf_r = cp.Uniform(r*min_bound,r*max_bound)
    pdf_rho = cp.Uniform(rho*min_bound,rho*max_bound)
    pdf_epsilon_alpha = cp.Uniform(epsilon_alpha*min_bound,epsilon_alpha*max_bound)
    pdf_epsilon_r = cp.Uniform(epsilon_r*min_bound,epsilon_r*max_bound)
    pdf_delta = cp.Uniform(delta*min_bound,delta*max_bound)

    #print('Criando distribuicoes normais')
    ##epsilon_r = cp.Normal(0.3,0.1)
    #epsilon_r = cp.Normal(0.3,0.001)
    ##epsilon_s = cp.Normal(0.998,0.001)
    ##epsilon_alpha = cp.Normal(0.92,0.002)
    #epsilon_alpha = cp.Normal(0.92,0.001)
    #delta = cp.Normal(0.07,0.01)
    #alpha = cp.Normal(22,0.001)
    #r = cp.Normal(11.1,0.1)
    #rho = cp.Normal(12.0,1.0)

    print('Distribuicoes criadas')

    dist = cp.J(pdf_alpha, pdf_r, pdf_rho, pdf_epsilon_alpha, pdf_epsilon_r)

    dist = cp.J(dist, pdf_delta)

    npar = len(dist)

    #grau da aproximacao polinomial
    degpc = 2
    ns = 10#3*P(degpc,npar)
    print("number of input parameters %d" % npar)
    print("polynomial degree %d" % degpc)
    print("min number of samples %d" % ns)

    evals_virus = []

    samples = dist.sample(ns,"L")
    k = 0
    lpts = range(ns)
    samp = samples.T
    print("evaluating samples: ")
    for i in tqdm(lpts,bar_format='{l_bar}{bar:20}{r_bar}{bar:-20b}'):
    #for s in samples.T:
        s = samp[k]
        k = k+1
        [sim_time,virus] = viralmodelfit(s)
        evals_virus.append(virus)

    evals_virus = np.array(evals_virus)

    plt.style.use('../estilo/PlotStyle.mplstyle')
    plt.close('all')

    output_path = './output_monte_carlo/'
    ext = '.png'
    itvl = 6
    caso = 'caso.'

    plt.figure('HCV RNA')
    #dadosViremiaLog10.plot.scatter(x='Day',y='Viral_load',color='m',label='Dados experimentais')
    plt.plot(t_exp, MEAN_PATS, 'o', label='data', linewidth=4)
    plt.xlim(0.0,tempoFim)
    #plt.ylim(0.0,8.0)
    plt.xlabel('day')
    plt.ylabel('HVC RNA log_{10}')
    plt.legend(loc='upper left', prop={'size':13})
    plt.grid(True)
    plt.title("Solution Virus")
    mean_virus= plot_confidence_interval(plt, sim_time, evals_virus, 'red', 'Modelo: Virus')
    plt.xticks(rotation=45)
    plt.tight_layout()
    plt.savefig(output_path+caso+'_output_Virus'+ext)
    plt.show()
    '''

def main():
    #sim = LaxFriedrichs(100, 1.5)
    sim = MethodOfLines(50,5)
    sim.solve_and_plot()
    plt.show()

if __name__ == "__main__":
    main()

# alpha, r, rho, epsilon_alpha, epsilon_r, delta

'''

# define parameter dictionary
parameters = {"alpha": pdf_alpha,
        "r": pdf_r,
        "rho": pdf_rho,
        "epsilon_alpha": pdf_epsilon_alpha,
        "epsilon_r": pdf_epsilon_r,
        "delta": pdf_delta
            }

# set up UQ
UQ = un.UncertaintyQuantification(
    model=model,
    parameters=parameters
)

print('Inicio UQ --- ')
begin = time.perf_counter()
data = UQ.monte_carlo(nr_samples=1000)
end = time.perf_counter()
print(' --- Fim UQ')
tempo = end-begin
with open('tempo.txt', 'w') as file:
    file.write("Tempo de execução: "+str(tempo)+" segundos")

'''


import win32com.client as wincl
import speech_recognition as sr
import webbrowser as wb

speak = wincl.Dispatch ("SAPI.SpVoice")

r = sr.Recognizer ()

with sr.Microphone()as source:
    speak.Speak("hello leader, what should I do for you")
    print ("shut up")

    audio = r.listen(source)
    print ("stop talking")

try:
    words = r.recognize_google (audio)
    speak.Speak ("ok,lets look.")
    wb.open ("https://www.youtube.com/results?search_query=" + words)

except sr.UnknownValueError:
    print("google Speach Recognition could not understand audio")
except sr.RequestError as e:
    print("Could not request results from Google Speech Recognitionservice;")


import numpy as np
import matplotlib.pyplot as plt
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from sklearn.datasets import load_digits
from sklearn.model_selection import learning_curve
from sklearn.model_selection import ShuffleSplit
import numpy as np
from numpy import array
from sklearn import svm
from sklearn import preprocessing
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn import datasets
from AA_dictionary import AA_seq_dict
from structure_dict import structure_dict
from structure_decode_dict import structure_decode_dict
from sklearn.preprocessing import OneHotEncoder
from sklearn.externals import joblib

testfile = "../datasets/membrane-beta_4state.3line.txt"
smalltestfile = "../datasets/parsetest"

###################################################################################
# Convert into SVM input vector (numerical input) - Steps to follow
###################################################################################

###################################################################################
# Convert into SVM input vector: Sliding Window Input
###################################################################################

listID = []
listaa = []
listTop = []
final_AAlist = []
final_Toplist = []
listaa_window = []

with open(testfile) as pf:
    lines = [line.strip() for line in pf]
listID = lines[0::3]
listaa = lines[1::3]
listTop = lines[2::3]

#window_input = 0
#while window_input%2==0:
    #window_input = int(input("Window size (must be odd number): "))
window_input=17
x = window_input//2

###################################################################################
# Convert into SVM input vector: Sliding Window Input for AA sequence
###################################################################################

for zeroseq in listaa:
    zeroseq = ((x)*'0')+zeroseq+((x)*'0')

    for aa in range(0,len(zeroseq)):
        window=zeroseq[aa:aa+window_input]
        if len(window)==window_input:
            listaa_window.append(window)

###################################################################################
# Convert into SVM input vector: AA sequence to binary
###################################################################################

for aa in listaa_window:
    AAlist = []
    for ch in aa:

        if ch in AA_seq_dict.keys():

            AAlist.extend(AA_seq_dict[ch])
        if ch == '0':
            AAlist.extend(AA_seq_dict['X'])

    final_AAlist.append(AAlist)
#print (final_AAlist)
###################################################################################
# Convert into SVM input vector: topology to numerical
###################################################################################
for ch in listTop:
    for x in ch:
        final_Toplist.extend(structure_dict[x])

x, y = final_AAlist, final_Toplist
X_train, X_test, Y_train, Y_test = train_test_split(x, y, test_size=0.30, random_state=15)
clf_model = svm.SVC(gamma=0.01, kernel = 'rbf', C=1.0, class_weight = "balanced").fit(X_train, Y_train)
prediction = clf_model.predict(X_test)

def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
                        n_jobs=1, train_sizes=np.linspace(.1, 1.0, 5)):

    plt.figure()
    plt.title(title)
    if ylim is not None:
        plt.ylim(*ylim)
    plt.xlabel("Training examples")
    plt.ylabel("Score")
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    plt.grid()

    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                     train_scores_mean + train_scores_std, alpha=0.1,
                     color="r")
    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                     test_scores_mean + test_scores_std, alpha=0.1, color="g")
    plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
             label="Training score")
    plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
             label="Cross-validation score")

    plt.legend(loc="best")
    return plt

X, y = final_AAlist, final_Toplist

title = "Learning Curves (SVM, RBF kernel, $\gamma=0.01$)"
cv = ShuffleSplit(n_splits=10, test_size=0.2, random_state=0)
estimator = SVC(gamma=0.01)
plot_learning_curve(estimator, title, X, y, cv=cv, n_jobs=4)
'''
title = "Learning Curves (SVM, linear kernel, $\gamma=0.01$)"
cv = ShuffleSplit(n_splits=10, test_size=0.2, random_state=0)
estimator = SVC(gamma=0.01)
plot_learning_curve(estimator, title, X, y, cv=cv, n_jobs=4)

title = "Learning Curves (SVM, RBF kernel, $\gamma=0.01$)" # PSSM as input
cv = ShuffleSplit(n_splits=10, test_size=0.2, random_state=0)
estimator = SVC(gamma=0.01)
plot_learning_curve(estimator, title, X, y, cv=cv, n_jobs=4)'''

plt.show()


import importlib
import inspect
import os
import re
import sys

from modules.data_module.data_manager import DataManager
from modules.data_module.parsing.i_behavioral_parser import IBehavioralParser
from modules.data_module.parsing.i_eye_parser import IEyeParser
from modules.ml_module.i_extraction import IExtraction
from modules.ml_module.i_predict import IPredict
from modules.ml_module.ml_manager import MLManager
from modules.visualization_module.i_visualization import IVisualization
from modules.visualization_module.visualization_manager import *

class ProgramManager:
    """
    Manage the app. Intermediary between the gui_module and the user requests, and the program modules.
    """

    def __init__(self):
        self.data_manager = DataManager()
        self.visualization_manager = VisualizationManager()
        self.ml_manager = MLManager()
        self.behavioral_data = None
        self.dh = None  # data handler
        self.stop_predict = False

        # dynamically load all interface implementations into the relevant manager instance.
        self.__load_interface_implementation("data_module/parsing/eye_tracking_parsers")
        self.__load_interface_implementation("data_module/parsing/behavioral_parsers")
        self.__load_interface_implementation("visualization_module/visualizations")
        self.__load_interface_implementation("ml_module/extractors")
        self.__load_interface_implementation("ml_module/models")

# Data section
    def load(self, path):
        """
        Asks the DataManager to load experiment info file, and updates accordingly the screen resolution in VisuManager.
        :param path: Full path to an experiment_ifo file to load.
        :return: Return_status, behavioral data DataFrame (None in case of failure)
        """
        try:
            # Load experiment info
            status_string, self.dh = self.data_manager.load(path)
            if not status_string == "Success":
                self.behavioral_data = None
                self.dh = None
                return status_string, None

            # Update screen resolution in VisualizationManager
            resolution = [int(self.dh.exp_info['ScreenResolution']['Width']), int(self.dh.exp_info['ScreenResolution']['Height'])]
            self.visualization_manager.set_screen_resolution(resolution)
            return status_string, self.dh.behavioral_data
        except:
            # Un-expected error
            return "Error accord.", None

    def filter_data(self, query):
        """
        Asks DataManager to filter the loaded data according to a query.
        :param query: Filtering data query.
        :return: Return_status, filtered behavioral data DataFrame (None in case of failure)
        """
        try:
            status_string = self.data_manager.filter_data(query)
            return status_string, self.dh.behavioral_data
        except:
            # Un-expected error
            return "Error accord.", None

# Visualization section
    def visualize(self, visu_type, query, multi_visu_option):
        """
        Asks VisualizationManager to create & present the visualization the user choose.
        :param visu_type: Type of visualization to produce.
        :param query: Optional. Data filtering query, to choose what trials to visualize out-of the trials loaded.
        :param multi_visu_option: How to handel multiple trials visualization, if supported.
        :return: Return status string.
        """
        if self.dh is None:
            return "Please browse an experiment info file."

        # set multi_visu_option
        if multi_visu_option == 0:
            multi_visu_option = ProcessingMethod.MultiSimultaneous
        elif multi_visu_option == 2:
            multi_visu_option = ProcessingMethod.MultiMean

        # get trials and visualize
        try:
            status_string, trials_to_visualize = self.dh.get_trials_by_query(query)
            if not status_string == "Success":
                return status_string
            if len(trials_to_visualize) < 1:
                return "No trial was chosen"
            if len(trials_to_visualize) < 2:
                multi_visu_option = ProcessingMethod.SingleTrial

            return self.visualization_manager.visualize(visu_type, trials_to_visualize, multi_visu_option)
        except:
            # Un-expected error
            return "Error accord."

    def get_visu_types(self):
        try:
            return self.visualization_manager.get_visu_types()
        except:
            return None

# prediction section
    ### Getters & Setters for Predict tab's lists ###
    def get_available_extractors(self):
        try:
            return self.ml_manager.get_available_extractors()
        except:
            return None

    def set_extractor(self, extractor):
        try:
            self.ml_manager.set_current_extractor(extractor)
        except:
            pass

    def get_available_models(self):
        try:
            return self.ml_manager.get_available_models()
        except:
            return None

    def set_ml_model(self, model):
        try:
            self.ml_manager.set_current_model(model)
        except:
            pass

    def get_exclusion_criteria(self):
        try:
            return self.data_manager.get_behavioral_labels()
        except:
            return None

    def get_different_items_amount_by_label(self, label):
        try:
            return self.data_manager.get_different_items_amount_by_label(label)
        except:
            return 0

    def get_ml_features(self):
        try:
            return self.ml_manager.get_extractable_features()
        except:
            pass
    ###################################################

    def load_ml_model(self, file_path):
        return self.ml_manager.load_model(file_path)

    def save_ml_model(self, file_path):
        return self.ml_manager.save_model(file_path)

    def predict(self, features, exclusion_category, num_to_exclude, iterations, action):
        """
        This function receives the chosen user parameters from the GUI 'predict' tab and calls the predict method in the MLManager.
        :param features: Chosen features to extract
        :param exclusion_category: column label from the behavioral data, will be used for trial grouping during learning.
        :param num_to_exclude: The number of groups that will be excluded during training (int).
        :param iterations: Maximum iterations number for the training session.
        :param action: The prediction action to process - Cross Validate, Fit Model, or Predict.
        :return:
        """
        if self.dh is None:
            return "Please browse an experiment info file.", None

        try:
            status, trial_list = self.dh.get_eyetracking_trials(self.dh.behavioral_data)
            if status != "Success":
                return status, None
            if trial_list is None:
                return "Could not extract any trails.", None
            if len(trial_list) < 1:
                return "No trials chosen.", None

            if self.stop_predict:
                return "Canceled.", None

            output_folder = self.dh.exp_info['Output']['OutputFolder']
            status, prediction_string = self.ml_manager.predict(trial_list, self.dh.behavioral_data, features,
                                                                exclusion_category, num_to_exclude, iterations,
                                                                output_folder, action)
            if status != "Success":
                return status, None

            return status, prediction_string
        except:
            # Un-expected error
            return "Error accord.", None

    def stop_prediction(self):
        """
        Stop Prediction process when the user clicks "Cancel"
        :return: void
        """
        if self.dh is None or self.ml_manager is None:
            return
        self.stop_predict = True
        self.dh.stop_trials_extraction()
        self.ml_manager.stop_thread()

    def reset_stop_flags(self):
        """
        Reset prediction stop flags.
        :return: void
        """
        if self.dh is None or self.ml_manager is None:
            return
        self.stop_predict = False
        self.dh.reset_stop_flags()
        self.ml_manager.reset_stop_flags()

# Dynamic load section
    def __load_interface_implementation(self, folder_path):
        """
        This method searches the given folder for the file implementing any of the programs interfaces, and stores them
        in the relevant manager's dictionary.
        :param: package_name (String) - RELATIVE path to the location from which files will be loaded.
                example: "visualization_module/visualizations"
        :return: None
        """
        # find .py files in the given folder.
        package_name = folder_path.replace("/", ".").replace("\\", ".")
        pysearchre = re.compile('.py$', re.IGNORECASE)
        pluginfiles = filter(pysearchre.search, os.listdir(os.path.join(os.path.dirname(__file__), folder_path)))
        form_module = lambda fp: '.' + os.path.splitext(fp)[0]
        plugins = map(form_module, pluginfiles)

        # import parent module / namespace
        importlib.import_module(package_name)

        # import each file as a module using import lib.
        modules = []
        for plugin in plugins:
            if not plugin.startswith('.__'):
                modules.append(importlib.import_module(plugin, package=package_name))

        # for each imported file, make a class list of the classes in it.
        for mod in modules:
            classes = inspect.getmembers(sys.modules[mod.__name__],
                                         lambda cls: inspect.isclass(cls) and cls.__module__ == mod.__name__)

            # for each class in the list, add a key value pair (class.get_name() : class_instance).
            # Load only classes that actually implement an interface: IExtraction, IPredict, IVisualization, IParser.
            for cls in classes:
                if issubclass(cls[1], IExtraction):
                    instance = self.__get_instance(cls[1])
                    if instance is not None:
                        self.ml_manager.available_extractors[cls[1].get_name()] = instance
                elif issubclass(cls[1], IPredict):
                    instance = self.__get_instance(cls[1])
                    if instance is not None:
                        self.ml_manager.available_models[cls[1].get_name()] = instance
                elif issubclass(cls[1], IVisualization):
                    instance = self.__get_instance(cls[1])
                    if instance is not None:
                        self.visualization_manager.available_visualizations[cls[1].get_name()] = instance
                elif issubclass(cls[1], IEyeParser):
                    instance = self.__get_instance(cls[1])
                    if instance is not None:
                        self.data_manager.available_eye_tracking_parsers[cls[1].get_name()] = instance
                elif issubclass(cls[1], IBehavioralParser):
                    instance = self.__get_instance(cls[1])
                    if instance is not None:
                        self.data_manager.available_behavioral_parsers[cls[1].get_name()] = instance

    def __get_instance(self, cls):
        """
        A method used by the dynamic loading mechanism.
        This method returns an instance of the requested class.
        :param class
        :return: instance of this class.
        """
        try:
            instance = cls.__new__(cls)
            instance.__init__()
        except:
            return None
        return instance


import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.patheffects as mpe
import pandas as pd
# styling
LINEWIDTH=4
EDGEWIDTH=1
CAPSTYLE="projecting"
COLORMAP="viridis_r"
ALPHA=1
FIRSTDAY=6 # 0=Mon, 6=Sun

# load dataset and parse timestamps
df = pd.read_csv('Trips.csv')
df[['trip_start', 'trip_stop']] = df[['trip_start', 'trip_stop']].apply(pd.to_datetime)

# set origin at the first FIRSTDAY before the first trip, midnight
first_trip = df['trip_start'].min()
origin = (first_trip - pd.to_timedelta(first_trip.weekday() - FIRSTDAY, unit='d')).replace(hour=0, minute=0, second=0)
weekdays = pd.date_range(origin, origin + np.timedelta64(1, 'W')).strftime("%a").tolist()

# # convert trip timestamps to week fractions
df['start'] = (df['trip_start'] - origin) / np.timedelta64(1, 'W')
df['stop']  = (df['trip_stop']  - origin) / np.timedelta64(1, 'W')

# sort dataset so shortest trips are plotted last
# should prevent longer events to cover shorter ones, still suboptimal
df = df.sort_values('dist', ascending=False).reset_index()
fig = plt.figure(figsize=(8, 6))
ax = fig.gca(projection="polar")

for idx, event in df.iterrows():

    # sample normalized distance from colormap
    ndist = event['dist'] / df['dist'].max()
    color = plt.cm.get_cmap(COLORMAP)(ndist)
    tstart, tstop = event.loc[['start', 'stop']]
    # timestamps are in week fractions, 2pi is one week

    nsamples = int(1000. * (tstop - tstart))
    t = np.linspace(tstart, tstop, nsamples)
    theta = 2 * np.pi * t
    arc, = ax.plot(theta, t, lw=LINEWIDTH, color=color, solid_capstyle=CAPSTYLE, alpha=ALPHA)
    if EDGEWIDTH > 0:
        arc.set_path_effects([mpe.Stroke(linewidth=LINEWIDTH+EDGEWIDTH, foreground='black'), mpe.Normal()])

# grid and labels
ax.set_rticks([])
ax.set_theta_zero_location("N")
ax.set_theta_direction(-1)
ax.set_xticks(np.linspace(0, 2*np.pi, 7, endpoint=False))
ax.set_xticklabels(weekdays)
ax.tick_params('x', pad=2)
ax.grid(True)

# setup a custom colorbar, everything's always a bit tricky with mpl colorbars
vmin = df['dist'].min()
vmax = df['dist'].max()
norm = mpl.colors.Normalize(vmin=vmin, vmax=vmax)
sm = plt.cm.ScalarMappable(cmap=COLORMAP, norm=norm)
sm.set_array([])
plt.colorbar(sm, ticks=np.linspace(vmin, vmax, 10), fraction=0.04, aspect=60, pad=0.1, label="distance", ax=ax)
plt.savefig("spiral.png", pad_inches=0, bbox_inches="tight")
plt.show()
#https://stackoverflow.com/questions/46575723/creating-a-temporal-range-time-series-spiral-plot


#!/usr/bin/env python
#
# License: BSD
#   https://raw.githubusercontent.com/stonier/py_trees_ros/devel/LICENSE
#
##############################################################################
# Documentation
##############################################################################

"""
Assorted utility functions.
"""

##############################################################################
# Imports
##############################################################################

import py_trees.console as console
import rospy
import std_msgs.msg as std_msgs

##############################################################################
# Methods
##############################################################################

def basename(name):
    """
    Generate the basename from a ros name.

    Args:
        name (:obj:`str`): ros name

    Returns:
        :obj:`str`: name stripped up until the last slash or tilde character.
    Examples:

        .. code-block:: python

           basename("~dude")
           # 'dude'
           basename("/gang/dude")
           # 'dude'
    """
    return name.rsplit('/', 1)[-1].rsplit('~', 1)[-1]

def publish_resolved_names(publisher, ros_communication_handles):
    """
    Worker that provides a string representation of all the resolved names
    and publishes it so we can use it as an introspection topic in runtime.

    Args:
        publisher (:obj:`rospy.Publisher`): use this object to publish with
        ros_communication_handles ([]): list of handles with their resolved names to to publish
    """
    s = console.bold + "\nResolved Names\n\n" + console.reset
    for handle in ros_communication_handles:
        s += console.yellow + "  " + handle.resolved_name + "\n" + console.reset
        publisher.publish(std_msgs.String("%s" % s))

##############################################################################
# Classes
##############################################################################

class Publishers(object):
    """
    Utility class that groups the publishers together in one convenient structure.

    Args:
        publishers (obj:`tuple`): list of (str, str, bool, int) tuples representing (topic_name, publisher_type, latched, queue_size) specifications to create publishers with

    Examples:
        Convert the incoming list of publisher name, type, latched, queue_size specifications into proper variables of this class.

        .. code-block:: python

           publishers = rocon_python_comms.utils.Publishers(
               [
                   ('~foo', std_msgs.String, True, 5),
                   ('/foo/bar', std_msgs.String, False, 5),
                   ('foobar', '/foo/bar', std_msgs.String, False, 5),
               ]
           )

        Note: '~/introspection/dude' will become just 'dude' unless you prepend a field for the name
        as in the third example above.
    """
    def __init__(self, publishers, introspection_topic_name="publishers"):
        publisher_details = []
        for info in publishers:
            if len(info) == 4:
                publisher_details.append((basename(info[0]), info[0], info[1], info[2], info[3]))
            else:
                # naively assume the user got it right and added exactly 5 fields
                publisher_details.append(info)
        self.__dict__ = {name: rospy.Publisher(topic_name, publisher_type, latch=latched, queue_size=queue_size) for (name, topic_name, publisher_type, latched, queue_size) in publisher_details}
        publisher = rospy.Publisher("~introspection/" + introspection_topic_name, std_msgs.String, latch=True, queue_size=1)
        publish_resolved_names(publisher, self.__dict__.values())
        self.introspection_publisher = publisher

class Subscribers(object):
    """
    Converts the incoming list of subscriber name, msg type, callback triples into proper
    variables of this class. Optionally you can prefix an arg that forces the name of
    the variable created.

    Args:
        subscribers (obj:`tuple`): list of (str, str, bool, int) tuples representing (topic_name, subscriber_type, latched, queue_size) specifications to create subscribers with

    Examples:

        .. code-block:: python

           subscribers = rocon_python_comms.utils.Subscribers(
               [
                   ('~dudette', std_msgs.String, subscriber_callback),
                   ('/dudette/jane', std_msgs.String, subscriber_callback),
                   ('jane', /dudette/jane', std_msgs.String, subscriber_callback),
               ]
           )

        Note: '~/introspection/dude' will become just 'dude' unless you prepend a field for the name
        as in the third example above.
    """
    def __init__(self, subscribers, introspection_topic_name="subscribers"):
        subscriber_details = []
        for info in subscribers:
            if len(info) == 3:
                subscriber_details.append((basename(info[0]), info[0], info[1], info[2]))
            else:
                # naively assume the user got it right and added exactly 4 fields
                subscriber_details.append(info)
        self.__dict__ = {name: rospy.Subscriber(topic_name, subscriber_type, callback) for (name, topic_name, subscriber_type, callback) in subscriber_details}
        publisher = rospy.Publisher("~introspection/" + introspection_topic_name, std_msgs.String, latch=True, queue_size=1)
        publish_resolved_names(publisher, self.__dict__.values())
        self.introspection_publisher = publisher


import rasterio
from rasterio.warp import transform_bounds
from rasterio.io import DatasetReader
import statistics
import math
import random
from rio_tiler.errors import TileOutsideBounds
from cw_tiler import utils
import numpy as np

def tile_utm_source(src, ll_x, ll_y, ur_x, ur_y, indexes=None, tilesize=256, nodata=None, alpha=None,
                    dst_crs='epsg:4326'):
    """
    Create UTM tile from any images.

    Attributes
    ----------
    source : rasterio.Dataset
    tile_x : int
        Mercator tile X index.
    tile_y : int
        Mercator tile Y index.
    tile_z : int
        Mercator tile ZOOM level.
    indexes : tuple, int, optional (default: (1, 2, 3))
        Bands indexes for the RGB combination.
    tilesize : int, optional (default: 256)
        Output image size.
    nodata: int or float, optional
        Overwrite nodata value for mask creation.
    alpha: int, optional
        Overwrite alpha band index for mask creation.

    Returns
    -------
    data : numpy ndarray
    mask: numpy array

    """

    wgs_bounds = transform_bounds(
        *[src.crs, dst_crs] + list(src.bounds), densify_pts=21)

    indexes = indexes if indexes is not None else src.indexes
    tile_bounds = (ll_x, ll_y, ur_x, ur_y)
    if not utils.tile_exists_utm(wgs_bounds, tile_bounds):
        raise TileOutsideBounds(
            'Tile {}/{}/{}/{} is outside image bounds'.format(ll_x, ll_y, ur_x, ur_y))

    return utils.tile_read_utm(src,
                               tile_bounds,
                               tilesize,
                               indexes=indexes,
                               nodata=nodata,
                               alpha=alpha,
                               dst_crs=dst_crs)

def tile_utm(source, ll_x, ll_y, ur_x, ur_y, indexes=None, tilesize=256, nodata=None, alpha=None, dst_crs='epsg:4326'):
    """
    Create UTM tile from any images.

    Attributes
    ----------
    address : str
        file url or rasterio.Dataset.
    tile_x : int
        Mercator tile X index.
    tile_y : int
        Mercator tile Y index.
    tile_z : int
        Mercator tile ZOOM level.
    indexes : tuple, int, optional (default: (1, 2, 3))
        Bands indexes for the RGB combination.
    tilesize : int, optional (default: 256)
        Output image size.
    nodata: int or float, optional
        Overwrite nodata value for mask creation.
    alpha: int, optional
        Overwrite alpha band index for mask creation.

    Returns
    -------
    data : numpy ndarray
    mask: numpy array

    """

    if isinstance(source, DatasetReader):

        return tile_utm_source(source,
                               ll_x,
                               ll_y,
                               ur_x,
                               ur_y,
                               indexes=indexes,
                               tilesize=tilesize,
                               nodata=nodata,
                               alpha=alpha,
                               dst_crs=dst_crs)

    else:
        with rasterio.open(source) as src:

            return tile_utm_source(src,
                                   ll_x,
                                   ll_y,
                                   ur_x,
                                   ur_y,
                                   indexes=indexes,
                                   tilesize=tilesize,
                                   nodata=nodata,
                                   alpha=alpha,
                                   dst_crs=dst_crs)

def get_chip(source, ll_x, ll_y, gsd,
             utm_crs=[],
             indexes=None,
             tile_size_pixels=512,
             nodata=None,
             alpha=None):
    """ Get Chip From Image Address"

    :param address:
    :param ll_x:
    :param ll_y:
    :param xres:
    :param yres:

    :return:

    """

    ur_x = ll_x + gsd * tile_size_pixels
    ur_y = ll_y + gsd * tile_size_pixels

    if isinstance(source, DatasetReader):

        if not utm_crs:
            wgs_bounds = utils.get_wgs84_bounds(source)
            utm_crs = utils.calculate_UTM_crs(wgs_bounds)

        return tile_utm(source, ll_x, ll_y, ur_x, ur_y,
                        indexes=indexes,
                        tilesize=tile_size_pixels,
                        nodata=nodata,
                        alpha=alpha,
                        dst_crs=utm_crs)

    else:
        with rasterio.open(source) as src:

            wgs_bounds = utils.get_wgs84_bounds(src)
            utm_crs = utils.calculate_UTM_crs(wgs_bounds)

            return tile_utm(source, ll_x, ll_y, ur_x, ur_y,
                            indexes=indexes,
                            tilesize=tile_size_pixels,
                            nodata=nodata,
                            alpha=alpha,
                            dst_crs=utm_crs)

def calculate_anchor_points(utm_bounds, stride_size_meters=400, extend=False, quad_space=False):
    if extend:
        min_x = math.floor(utm_bounds[0])
        min_y = math.floor(utm_bounds[1])
        max_x = math.ceil(utm_bounds[2])
        max_y = math.ceil(utm_bounds[3])
    else:
        print("NoExtend")
        print('UTM_Bounds: {}'.format(utm_bounds))
        min_x = math.ceil(utm_bounds[0])
        min_y = math.ceil(utm_bounds[1])
        max_x = math.floor(utm_bounds[2])
        max_y = math.floor(utm_bounds[3])

    anchor_point_list = []
    if quad_space:
        print("quad_space")
        row_cell = np.asarray([[0, 1],[2,3]])
        anchor_point_list_dict={0: [],
                               1: [],
                               2:[],
                               3:[]
                               }
    else:
        anchor_point_list_dict={0:[]}

    for rowidx, x in enumerate(np.arange(min_x, max_x, stride_size_meters)):
        for colidx, y in enumerate(np.arange(min_y, max_y, stride_size_meters)):

            if quad_space:
                anchor_point_list_dict[row_cell[rowidx % 2, colidx % 2]].append([x,y])
            else:
                anchor_point_list_dict[0].append([x, y])

    return anchor_point_list_dict

def calculate_cells(anchor_point_list_dict, cell_size_meters, utm_bounds=[]):
    """ calculate Cells for splitting based on anchor points

    :param anchor_point_list:
    :param cell_size_meters:
    :return:
    cells_list [(minx, miny, maxx, maxy),
                ...]
    """
    cells_list_dict = {}
    for anchor_point_list_id, anchor_point_list in anchor_point_list_dict.items():
        cells_list = []
        for anchor_point in anchor_point_list:

            if utm_bounds:
                if (anchor_point[0] + cell_size_meters < utm_bounds[2]) and (anchor_point[1] + cell_size_meters < utm_bounds[3]):
                    cells_list.append(
                        [anchor_point[0], anchor_point[1], anchor_point[0] + cell_size_meters, anchor_point[1] + cell_size_meters])

                else:
                    pass

        cells_list_dict[anchor_point_list_id] = cells_list

    return cells_list_dict

def calculate_analysis_grid(utm_bounds, stride_size_meters=300, cell_size_meters=400, quad_space=False, snapToGrid=False):

    anchor_point_list_dict = calculate_anchor_points(utm_bounds, stride_size_meters=stride_size_meters, quad_space=quad_space)

    cells_list_dict = calculate_cells(anchor_point_list_dict, cell_size_meters, utm_bounds=utm_bounds)

    return cells_list_dict

if __name__ == '__main__':
    utmX, utmY = 658029, 4006947
    cll_x = utmX
    cur_x = utmX + 500
    cll_y = utmY
    cur_y = utmY + 500
    stride_size_meters = 300
    cell_size_meters = 400
    ctile_size_pixels = 1600
    spacenetPath = "s3://spacenet-dataset/AOI_2_Vegas/srcData/rasterData/AOI_2_Vegas_MUL-PanSharpen_Cloud.tif"
    address = spacenetPath

    with rasterio.open(address) as src:
        cwgs_bounds = utils.get_wgs84_bounds(src)
        cutm_crs = utils.calculate_UTM_crs(cwgs_bounds)
        cutm_bounds = utils.get_utm_bounds(src, cutm_crs)

        #ccells_list = calculate_analysis_grid(cutm_bounds, stride_size_meters=stride_size_meters,
        #                                     cell_size_meters=cell_size_meters)

        #random_cell = random.choice(ccells_list)
        #cll_x, cll_y, cur_x, cur_y = random_cell
        tile, mask, window, window_transform = tile_utm(src, cll_x, cll_y, cur_x, cur_y, indexes=None, tilesize=ctile_size_pixels, nodata=None, alpha=None,
                        dst_crs=cutm_crs)


import pygame

# import all the constants from the constants package
from checkers.constants import *

# import the class "Board"
from checkers.board import Board
from checkers.game import Game

# Frames per second => for rendering and drawing the game
FPS = 60
WIN = pygame.display.set_mode((WIDTH, HEIGHT))

# Name of the game displayed on bar
pygame.display.set_caption("Checkers")

# Take position of our mouse and based on that position will tell us which row/column we are in
def get_row_col_from_mouse(pos):
    x, y = pos
    row = y//SQUARE_SIZE
    col = x//SQUARE_SIZE
    return row, col

# function to actually run the game
def main():
    run = True

    # Make sure game doesn't run too fast or too slow
    clock = pygame.time.Clock()

    # Create a Game object which will control the board for us
    game = Game(WIN)

    # Create an event loop while the game is running
    while run:
        clock.tick(FPS)

        if game.winner() != None:
            print(game.winner())
            run = False

        # Check for any events happening at current time
        for event in pygame.event.get():

            # End the game and get rid of window by clicking the cross
            if event.type == pygame.QUIT:
                run = False

            # If we press any button on mouse, we first get the row, column we are in
            # Then we will select the piece on that location and move that to wherever we want to move
            if event.type == pygame.MOUSEBUTTONDOWN:
                pos = pygame.mouse.get_pos()
                row, col = get_row_col_from_mouse(pos)
                game.select(row, col)

        game.update()

    pygame.quit()

main()


import requests
import json
import main_page.data_modules.functions
import main_page.data_modules.match
import main_page.data_modules.team

class League:
    def __init__(self, league_id):
        self.base_url = 'http://api.eslgaming.com/play/v1/leagues/'
        self.league_id = str(league_id)

    def get_league_mode(self):
        """Returns a league mode ('1on1', '5on5', etc.)."""
        r = requests.get(self.base_url + self.league_id)
        response = json.loads(r.text)
        if response['mode'] is None:
            return response['name']['full']
        return response['mode']

    def get_league_description(self):
        """Returns full league description."""
        r = requests.get(self.base_url + self.league_id)
        response = json.loads(r.text)
        if main_page.data_modules.functions.Functions.is_error(response):
            return "error"
        else:
            return response

    def get_league_results(self):
        """Returns league results."""
        r = requests.get(self.base_url + self.league_id + "/results")
        response = json.loads(r.text)
        results = []
        for match_info in response:
            match_id = match_info['id']
            match = main_page.data_modules.match.Match(match_id)
            if match.get_match_status() == "closed":
                result = match.get_match_results()
                results.append(result)
            else:
                if "1on1" in League.get_league_mode(self) or "Single Elimination" in League.get_league_mode(self):
                    player_a_id = match.get_match_info()['contestants'][0]['user']['id']
                    player_b_id = match.get_match_info()['contestants'][1]['user']['id']
                    if player_a_id is None or player_b_id is None:
                        result = {'Status': 'Match is  still open', 'Match ID': match_id,
                                  'Link': main_page.data_modules.functions.Functions.get_match_link(match_id),
                                  'Contestants': 'There are no 2 contestants yet'}
                        results.append(result)
                    else:
                        player_a_name = match.get_match_info()['contestants'][0]['user']['nickname']
                        player_b_name = match.get_match_info()['contestants'][1]['user']['nickname']
                        result = {'Status': 'Match is  still open', 'Match ID': match_id,
                                  'Link': main_page.data_modules.functions.Functions.get_match_link(match_id),
                                  main_page.data_modules.functions.Functions.get_user_link(player_a_id): player_a_id,
                                  main_page.data_modules.functions.Functions.get_user_link(player_b_id): player_b_id}
                        results.append(result)
                else:
                    team_a_id = match.get_match_info()['contestants'][0]['team']['id']
                    team_b_id = match.get_match_info()['contestants'][1]['team']['id']
                    if team_a_id is None or team_b_id is None:
                        result = {'Status': 'Match is  still open', 'Match ID': match_id,
                                  'Link': main_page.data_modules.functions.Functions.get_match_link(match_id),
                                  'Contestants': 'There are no 2 contestants yet'}
                        results.append(result)
                    else:
                        team_a_name = match.get_match_info()['contestants'][0]['team']['name']
                        team_b_name = match.get_match_info()['contestants'][1]['team']['name']
                        result = {'Status': 'Match is  still open', 'Match ID': match_id,
                                  'Link': main_page.data_modules.functions.Functions.get_match_link(match_id),
                                  main_page.data_modules.functions.Functions.get_team_link(team_a_id): team_a_id,
                                  main_page.data_modules.functions.Functions.get_team_link(team_b_id): team_b_id}
                        results.append(result)
        return results

    def get_league_contestants(self):
        """Returns all contestants in the tournament."""
        r = requests.get(self.base_url + self.league_id + "/contestants")
        response = json.loads(r.text)
        return response

    def get_league_members_info(self):  # have to think about optimisation
        """Returns info about users, participating in the tournament."""
        r = requests.get(self.base_url + self.league_id + "/contestants")
        response = json.loads(r.text)
        league_members_info = []
        for contestant in response:
            team = main_page.data_modules.team.Team(contestant['id'])
            league_members_info.append(team.get_useful_members_info())
        return league_members_info

    def get_matches_and_players(self):
        """Returns Matches IDs and Players Gameaccounts there."""
        r = requests.get(self.base_url + self.league_id + "/matches")
        response = json.loads(r.text)
        gameaccounts = []
        for match in response:
            match_id = match['id']
            match_id_dict = {"match id": match_id,
                             "Match link": main_page.data_modules.functions.Functions.get_match_link(match_id)}
            gameaccounts_dict = match['gameaccounts']
            for key, value in gameaccounts_dict.items():
                gameaccounts_dict[key] = ' {1} '.format(key, value)
            match_id_dict.update(gameaccounts_dict)
            gameaccounts.append(match_id_dict)
        return gameaccounts

    def check_tickets(self):
        """Returns opened protests, if any."""
        r = requests.get(self.base_url + self.league_id + "/matches")
        response = json.loads(r.text)
        tickets_amount = 0
        protests = []
        matches_id = []
        if "1on1" in League.get_league_mode(self) or "Single Elimination" in League.get_league_mode(self):
            for match in response:
                match_status = match['status']
                if match_status == "protest":

                    tickets_amount += 1

                    match_id = match['id']
                    player_a_id = match['contestants'][0]['user']['id']
                    player_a_name = match['contestants'][0]['user']['nickname']
                    player_b_id = match['contestants'][1]['user']['id']
                    player_b_name = match['contestants'][1]['user']['nickname']

                    new_r = requests.get(self.base_url + self.league_id)
                    new_response = json.loads(new_r.text)

                    league_name = new_response['uri']
                    base_link = "https://play.eslgaming.com"
                    match_link = "match/" + str(match_id) + "/"
                    protest_link = "admin_tickets/all"
                    admin_protests_link = base_link + league_name + protest_link
                    admin_protests_link_url = ' {1} '.format(admin_protests_link, "Open protests")
                    match_protest_link = base_link + league_name + match_link
                    match_protest_link_url = ' {1} '.format(match_protest_link[:-1], "Match page")
                    protest_dict = {"ticket number": tickets_amount, "match_id": match_id,
                                    player_a_id: player_a_name, player_b_id: player_b_name,
                                    "Check your tickets": admin_protests_link_url,
                                    "Check match": match_protest_link_url}
                    protests.append(protest_dict)
                    matches_id.append(match_id)
        else:
            for match in response:
                match_status = match['status']
                if match_status == "protest":
                    tickets_amount += 1

                    match_id = match['id']
                    team_a_id = match['contestants'][0]['team']['id']
                    team_a_name = match['contestants'][0]['team']['name']
                    team_b_id = match['contestants'][1]['team']['id']
                    team_b_name = match['contestants'][1]['team']['name']

                    new_r = requests.get(self.base_url + self.league_id)
                    new_response = json.loads(new_r.text)

                    league_name = new_response['uri']
                    base_link = "https://play.eslgaming.com"
                    match_link = "match/" + str(match_id) + "/"
                    protest_link = "admin_tickets/all"
                    admin_protests_link = base_link + league_name + protest_link
                    admin_protests_link_url = ' {1} '.format(admin_protests_link, "Open protests")
                    match_protest_link = base_link + league_name + match_link
                    match_protest_link_url = ' {1} '.format(match_protest_link[:-1], "Match page")
                    protest_dict = {"ticket number": tickets_amount, "match_id": match_id, team_a_id: team_a_name,
                                    team_b_id: team_b_name, "Check your tickets": admin_protests_link_url,
                                    "Check match": match_protest_link_url}
                    protests.append(protest_dict)
                    matches_id.append(match_id)
                else:
                    continue
        return protests, tickets_amount, matches_id


import argparse

import tensorflow as tf

from convert_torch_to_tf import get_tf_pretrained_model
from wav2vec2 import Wav2Vec2Config

def get_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--hf_model_id",
        default="facebook/wav2vec2-base",
        type=str,
        help="Model ID of HuggingFace wav2vec2 which needs to be converted into TensorFlow",
    )
    parser.add_argument(
        "--with_lm_head",
        action="store_true",
        help="Whether to use `Wav2Vec2Model` or `Wav2Vec2ForCTC` from `wav2vec2/modeling.py`",
    )
    parser.add_argument(
        "--verbose",
        default=False,
        type=bool,
        help="Whether to display specific layers conversion while running conversion script",
    )
    parser.add_argument(
        "--saved_model_dir",
        default="saved-model/",
        type=str,
        help="Where to save the obtained `saved-model`",
    )
    parser.add_argument(
        "--seqlen",
        default=246000,
        type=int,
        help="With what sequence length should model be exported",
    )
    return parser

if __name__ == "__main__":

    args = get_parser().parse_args()

    # spec_augmentation can't be supported with saved_model for now
    config = Wav2Vec2Config(apply_spec_augment=False)
    model, _ = get_tf_pretrained_model(
        config, args.hf_model_id, verbose=args.verbose, with_lm_head=args.with_lm_head
    )

    input_signature = [tf.TensorSpec((None, args.seqlen), tf.float32, name="speech")]
    model.__call__ = tf.function(model.__call__, input_signature=input_signature)
    tf.saved_model.save(model, args.saved_model_dir)

    print(f"Your `TF SavedModel ({model})` saved in {args.saved_model_dir}")


# -*- coding: utf-8 -*-

from numba import vectorize

import numpy as np
from numba import jit, void, double

from numba_funcs.multi_thread_vectorizer import mvectorize
from numba_funcs.timer import Timer

import unittest

"""
def fn( x ):
    result = 0.0
    for i in range( 100 ):
        result += (1+i+x)/(1+x)
    return result
"""

def fn( x ):
    result = 0.0
    for i in range( 10000 ):
        result+=i*x
    return result

nb_fn = jit( double(double, ), nopython=True )( fn )

class Test(unittest.TestCase):

    def test_mvectorize(self):

        x = np.linspace( 1, 1000, 10000 )

        mf_fn = mvectorize( nb_fn, ( double[:], double[:] ), num_threads=8 )

        result = mf_fn( x )
        expected = np.vectorize( fn )( x )

        np.testing.assert_array_equal( expected, result )

    def test_mvectorize_performance(self):

        x = np.linspace( 0, 1000 )

        mf_fn = mvectorize( nb_fn, ( double[:], double[:] ), num_threads=8 )
        # Call once to initialize
        mf_fn( x )
        num_tests = 100

        with Timer( 'mf_nb' ) as nbtimer:
            for _ in range( num_tests ):
                mf_fn( x )

        np_fn = np.vectorize( fn )
        with Timer( 'np' ) as nptimer:
            for _ in range( num_tests ):
                np_fn( x )

        ratio = nbtimer.interval / nptimer.interval
        print( 'Numba version took %s as long as numpy'%ratio)


# Imports
import os
import argparse
import pandas as pd
from imblearn.under_sampling import RandomUnderSampler

def main():

    # Argument parsing
    parser = argparse.ArgumentParser()
    parser.add_argument('--input_folder', default = None, type = str,
                        required = True, help = 'input folder')
    parser.add_argument('--min_maj_ratio', default = None, type = float,
                        required = True, help = 'Minority to majority class ratio')
    args = parser.parse_args()

    # Path definition
    input_path = args.input_folder

    # Global initialization
    seed = 1234

    # Data load
    data = pd.read_pickle(input_path)

    # EDA before resampling
    num_neg = pd.value_counts(data.label)[0]
    num_pos = pd.value_counts(data.label)[1]
    print(f'% neg before resampling = {num_neg/(num_pos+num_neg)*100:.2f}')
    print(f'% pos before resampling = {num_pos/(num_pos+num_neg)*100:.2f}')

    # Undersampling
    x_train = data[list(data.columns)[:-1]]
    y_train = data[list(data.columns)[-1:]]
    random_undersampler = RandomUnderSampler(sampling_strategy = args.min_maj_ratio,
                                             random_state = seed)
    x_train_res, y_train_res = random_undersampler.fit_resample(x_train, y_train)
    res_data = pd.concat([x_train_res, y_train_res], axis = 1)

    # EDA before resampling
    num_neg = pd.value_counts(res_data.label)[0]
    num_pos = pd.value_counts(res_data.label)[1]
    print(f'% neg before resampling = {num_neg/(num_pos+num_neg)*100:.2f}')
    print(f'% pos before resampling = {num_pos/(num_pos+num_neg)*100:.2f}')

    # Save resampled data
    file_name, file_ext = os.path.splitext(os.path.basename(input_path))
    output_filename = file_name + '_resampled_' + str(args.min_maj_ratio) + file_ext
    output_folder = os.path.dirname(input_path)
    output_path = os.path.join(output_folder, output_filename)
    pd.to_pickle(res_data, output_path)

#%% Main
if __name__ == "__main__":
    main()