import torch
from torch import Tensor, nn
import torchvision
import os
import numpy as np

class Normalize:
    def __init__(self, n_channels, expected_values, variance):
        self.n_channels = n_channels
        self.expected_values = expected_values
        self.variance = variance
        assert self.n_channels == len(self.expected_values)

    def __call__(self, x):
        x_clone = x.clone()
        for channel in range(self.n_channels):
            x_clone[:, channel] = (x[:, channel] - self.expected_values[channel]) / self.variance[channel]
        return x_clone

class Denormalize:
    def __init__(self, n_channels, expected_values, variance):
        self.n_channels = n_channels
        self.expected_values = expected_values
        self.variance = variance
        assert self.n_channels == len(self.expected_values)

    def __call__(self, x):
        x_clone = x.clone()
        for channel in range(self.n_channels):
            x_clone[:, channel] = x[:, channel] * self.variance[channel] + self.expected_values[channel]
        return x_clone

class RegressionModel(nn.Module):
    def __init__(self, task, model, init_mask, init_pattern):
        self._EPSILON = 1e-7
        super(RegressionModel, self).__init__()
        self.mask_tanh = nn.Parameter(torch.tensor(init_mask))
        self.pattern_tanh = nn.Parameter(torch.tensor(init_pattern))

        self.classifier = self._get_classifier(model)
        self.normalizer = self._get_normalize(task)
        self.denormalizer = self._get_denormalize(task)

    def forward(self, x):
        mask = self.get_raw_mask()
        pattern = self.get_raw_pattern()
        if self.normalizer:
            pattern = self.normalizer(self.get_raw_pattern())
        x = (1 - mask) * x + mask * pattern
        return self.classifier(x)

    def get_raw_mask(self):
        mask = nn.Tanh()(self.mask_tanh)
        return mask / (2 + self._EPSILON) + 0.5

    def get_raw_pattern(self):
        pattern = nn.Tanh()(self.pattern_tanh)
        return pattern / (2 + self._EPSILON) + 0.5

    def _get_classifier(self, model):
        classifier = model
        for param in classifier.parameters():
            param.requires_grad = False
        classifier.eval()
        return classifier.to('cuda')

    def _get_denormalize(self, task):
        if task == 'cifar10':
            denormalizer = Denormalize(3, [0.4914, 0.4822, 0.4465], [0.247, 0.243, 0.261])
        elif task == 'mnist':
            denormalizer = Denormalize(1, [0.5], [0.5])
        elif task == 'imageNet':
            denormalizer = Denormalize(3,[0.485,0.456,0.406],[0.229,0.224,0.225])
        elif task == 'gtsrb':
            denormalizer = None
        else:
            raise Exception("Invalid dataset")
        return denormalizer

    def _get_normalize(self, task):
        if task == 'cifar10':
            normalizer = Normalize(3, [0.4914, 0.4822, 0.4465], [0.247, 0.243, 0.261])
        elif task == 'mnist':
            normalizer = Normalize(1, [0.5], [0.5])
        elif task == 'imageNet':
            normalizer = Denormalize(3,[0.485,0.456,0.406],[0.229,0.224,0.225])
        elif task == 'gtsrb':
            normalizer = None
        else:
            raise Exception("Invalid dataset")
        return normalizer

class Recorder:
    def __init__(self,target_label,task):
        super().__init__()

        # Best optimization results
        self.mask_best = None
        self.pattern_best = None
        self.reg_best = float('inf')
        self.target_label = target_label

        # Logs and counters for adjusting balance cost
        self.logs = []
        self.cost_set_counter = 0
        self.cost_up_counter = 0
        self.cost_down_counter = 0
        self.cost_up_flag = False
        self.cost_down_flag = False

        # Counter for early stop
        self.early_stop_counter = 0
        self.early_stop_reg_best = self.reg_best

        # Cost
        self.cost = 1e-3
        self.cost_multiplier_up = 2
        self.cost_multiplier_down = 2 ** 1.5

        self.task = task

    def reset_state(self):
        self.cost = 1e-3
        self.cost_up_counter = 0
        self.cost_down_counter = 0
        self.cost_up_flag = False
        self.cost_down_flag = False
        print("Initialize cost to {:f}".format(self.cost))

    def save_result_to_dir(self):
        result_dir = '%s/u_t_%s'%(self.task,self.task)
        if not os.path.exists(result_dir):
            os.makedirs(result_dir)
        result_dir = os.path.join(result_dir, str(self.target_label))
        if not os.path.exists(result_dir):
            os.makedirs(result_dir)

        pattern_best = self.pattern_best
        mask_best = self.mask_best
        trigger = pattern_best * mask_best

        path_mask = os.path.join(result_dir, 'mask.png')
        path_pattern = os.path.join(result_dir, 'pattern.png')
        path_trigger = os.path.join(result_dir, 'trigger.png')

        torchvision.utils.save_image(mask_best, path_mask, normalize=True)
        torchvision.utils.save_image(pattern_best, path_pattern, normalize=True)
        torchvision.utils.save_image(trigger, path_trigger, normalize=True)

class UniversalTrigger:
    def __init__(self):
        self.universal_trigger_dict = {}

def train(task, model, target_label, init_mask, init_pattern, test_loader):
    # Build regression model
    regression_model = RegressionModel(task, model, init_mask, init_pattern).to('cuda')

    # Set optimizer
    optimizerR = torch.optim.Adam(regression_model.parameters(), lr=1e-1, betas=(0.5, 0.9))

    # Set recorder (for recording best result)
    recorder = Recorder(target_label,task)

    for epoch in range(50):
        early_stop = train_step(regression_model, optimizerR, test_loader, recorder, epoch, target_label)
        if early_stop:
            break

    # Save result to dir
    recorder.save_result_to_dir()

    return recorder

def train_step(regression_model, optimizerR, dataloader, recorder, epoch, target_label,early_stop=True):
    print("Epoch {} - Label: {}".format(epoch, target_label))
    # Set losses
    cross_entropy = nn.CrossEntropyLoss()
    total_pred = 0
    true_pred = 0

    # Record loss for all mini-batches
    loss_ce_list = []
    loss_reg_list = []
    loss_list = []
    loss_acc_list = []

    # Set inner early stop flag
    inner_early_stop_flag = False
    for batch_idx, (inputs, labels) in enumerate(dataloader):
        # Forwarding and update model
        optimizerR.zero_grad()

        inputs = inputs.to('cuda')
        sample_num = inputs.shape[0]
        total_pred += sample_num
        target_labels = torch.ones((sample_num), dtype=torch.int64).to('cuda') * target_label
        predictions = regression_model(inputs)

        loss_ce = cross_entropy(predictions, target_labels)
        loss_reg = torch.norm(regression_model.get_raw_mask(), 2)
        total_loss = loss_ce + recorder.cost * loss_reg
        total_loss.backward()
        optimizerR.step()

        # Record minibatch information to list
        minibatch_accuracy = torch.sum(torch.argmax(predictions, dim=1) == target_labels).detach() * 100. / sample_num
        loss_ce_list.append(loss_ce.detach())
        loss_reg_list.append(loss_reg.detach())
        loss_list.append(total_loss.detach())
        loss_acc_list.append(minibatch_accuracy)

        true_pred += torch.sum(torch.argmax(predictions, dim=1) == target_labels).detach()

    loss_ce_list = torch.stack(loss_ce_list)
    loss_reg_list = torch.stack(loss_reg_list)
    loss_list = torch.stack(loss_list)
    loss_acc_list = torch.stack(loss_acc_list)

    avg_loss_ce = torch.mean(loss_ce_list)
    avg_loss_reg = torch.mean(loss_reg_list)
    avg_loss = torch.mean(loss_list)
    avg_loss_acc = torch.mean(loss_acc_list)

    # Check to save best mask or not
    if avg_loss_acc >= 99. and avg_loss_reg < recorder.reg_best:
        recorder.mask_best = regression_model.get_raw_mask().detach()
        recorder.pattern_best = regression_model.get_raw_pattern().detach()
        recorder.reg_best = avg_loss_reg
        recorder.save_result_to_dir()
        print(" Updated !!!")

    # Show information
    print('  Result: Accuracy: {:.3f} | Cross Entropy Loss: {:.6f} | Reg Loss: {:.6f} | Reg best: {:.6f}'.format(
        true_pred * 100. / total_pred,
        avg_loss_ce,
        avg_loss_reg,
        recorder.reg_best))

    # Check early stop
    if early_stop:
        if recorder.reg_best < float('inf'):
            if recorder.reg_best >= 99. * recorder.early_stop_reg_best:
                recorder.early_stop_counter += 1
            else:
                recorder.early_stop_counter = 0

        recorder.early_stop_reg_best = min(recorder.early_stop_reg_best, recorder.reg_best)

        if (
                recorder.cost_down_flag and recorder.cost_up_flag and recorder.early_stop_counter >= 25):
            print('Early_stop !!!')
            inner_early_stop_flag = True

    if not inner_early_stop_flag:
        # Check cost modification
        if recorder.cost == 0 and avg_loss_acc >= 99.:
            recorder.cost_set_counter += 1
            if recorder.cost_set_counter >= 5:
                recorder.reset_state()
        else:
            recorder.cost_set_counter = 0

        if avg_loss_acc >= 99.:
            recorder.cost_up_counter += 1
            recorder.cost_down_counter = 0
        else:
            recorder.cost_up_counter = 0
            recorder.cost_down_counter += 1

        if recorder.cost_up_counter >= 5:
            recorder.cost_up_counter = 0
            print("Up cost from {} to {}".format(recorder.cost, recorder.cost * recorder.cost_multiplier_up))
            recorder.cost *= recorder.cost_multiplier_up
            recorder.cost_up_flag = True

        elif recorder.cost_down_counter >= 5:
            recorder.cost_down_counter = 0
            print("Down cost from {} to {}".format(recorder.cost, recorder.cost / recorder.cost_multiplier_down))
            recorder.cost /= recorder.cost_multiplier_down
            recorder.cost_down_flag = True

        # Save the final version
        if recorder.mask_best is None:
            recorder.mask_best = regression_model.get_raw_mask().detach()
            recorder.pattern_best = regression_model.get_raw_pattern().detach()

    return inner_early_stop_flag


class Solution:
    # @param A : string
    # @return an integer
    def atoi(self, s):
        s = s.strip() # strips all spaces on left and right
        if not s: return 0
        sign = -1 if s[0] == '-' else 1
        val, index = 0, 0
        if s[0] in ['+', '-']: index = 1
        while index < len(s) and s[index].isdigit():
            val = val*10 + ord(s[index]) - ord('0') # assumes there're no invalid chars in given string
            index += 1
        #return sign*val
        return max(-2**31, min(sign * val,2**31-1))


import requests
import json
import re
import time
import csv
import MySQLdb as mdb
from bs4 import BeautifulSoup
import numpy as np
import pandas as pd
from helpers import *

"""
This script scrapes and stores the fantasy points achieved by each player for the 2013 season.
The data is stored in a .csv file.
"""
def main():
  #open .csv file to store data and writes col names
  wfile = open("espn-actual2.csv", "wb")
  field_names = ['game_id', 'plyr_id', 'tot_pts','week']
  writer = csv.writer(wfile)
  writer.writerow(field_names)

  #for each week in the 2013 season, the player id, game id, and points scored are scraped and stored in the .csv file
  for w in range(1,3):
    for pg in range(0, 300, 50):
      pts_url = "http://games.espn.go.com/ffl/leaders?&scoringPeriodId=%s&seasonId=2013&startIndex=%s" % (w, pg)
      pts_res = requests.get(pts_url)
      soup = BeautifulSoup(pts_res.content)
      for tr in soup.find_all('tr', class_="pncPlayerRow"):
        id_match = re.search(r'plyr(\d+)',tr['id'])
        id = int(id_match.group(1))
        td_pts = tr.find('td', class_="playertableStat appliedPoints appliedPointsProGameFinal")
        projpts = td_pts.contents[0].encode('ascii', 'ignore')
        td_game = tr.find('td', class_="gameStatusDiv")
        href = td_game.find('a')
        href_str = str(href)
        game_match = re.search(r'gameId=(\d+)', href_str)
        game_id = game_match.group(1)
        if projpts == '--':
          projpts = 0
        data = [game_id, id, projpts, w]
        writer.writerow(data)
  wfile.close()
if __name__ == '__main__':
  main()


#!/usr/bin/python

import json
import utils
import logging
import os
import subprocess
import smt_encoding
import pysmt.shortcuts
import re
import requests

import core_data

import hyportage_pattern
import hyportage_db

from pysmt.smtlib.parser import SmtLib20Parser
import cStringIO

"""
This file contains all the functions related to solving the constraints generated from a set of spls,
in order to compute a new configuration of the system
"""

__author__ = "Michael Lienhardt and Jacopo Mauro"
__copyright__ = "Copyright 2017, Michael Lienhardt and Jacopo Mauro"
__license__ = "GPL3"
__version__ = "0.5"
__maintainer__ = "Michael Lienhardt and Jacopo Mauro"
__email__ = "michael lienhardt@laposte.net & mauro.jacopo@gmail.com"
__status__ = "Prototype"

##########################################################################
# UTILITIES TO CALL THE HYVAR-REC SOLVER
##########################################################################

def run_local_hyvar(json_data, explain_modality, cmd, par_cores):
	"""
	Run hyvar locally assuming that there is a command hyvar-rec
	"""
	file_name = utils.get_new_temp_file(".json")
	with open(file_name, "w") as f:
		json.dump(json_data, f)
	cmd.extend(["--constraints-minimization", "--features-as-boolean", "--no-default-preferences"])
	if explain_modality: cmd.append("--explain")
	if par_cores > 1: cmd.extend(["-p", unicode(par_cores)])
	cmd.append(file_name)

	# executing the solver
	utils.phase_start("Running: " + unicode(cmd))
	process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
	out, err = process.communicate()
	if process.returncode != 0:
		logging.error("command ended with an error code: " + str(process.returncode))
		return None
	logging.debug("Stderr of the command")
	logging.debug(err)
	utils.phase_end("Execution ended")
	res = json.loads(out)
	return res

def run_remote_hyvar(json_data, explain_modality, url):
	"""
	Run hyvar
	"""
	if explain_modality: url += "/explain"
	else: url += "/process"
	utils.phase_start("Invoking url: " + url)
	response = requests.post(url, data=json.dumps(json_data), headers={'content-type': 'application/json'})
	utils.phase_end("Execution ended")
	if response.status_code != requests.codes.ok:
		logging.error("server answered with an error code: " + str(response.status_code))
		return None
	res = response.json()
	if 'error' in res:
		logging.error("server answered with an error message: " + json.dumps(res))
		return None
	return res

run_hyvar = None

##########################################################################
# 1. INITIALIZE THE DATA (COMPUTE REQUEST AND UPDATE THE DATABASE)
##########################################################################

def process_request(pattern_repository, id_repository, config, atoms):
	"""
	This function simply takes the user request and his USE configuration from the environment,
	and translate them in relevant data
	:param pattern_repository: the pattern repository of hyportage
	:param id_repository: the id repository of hyportage
	:param mspl: the mspl of hyportage
	:param spl_groups: the spl_groups of hyportage
	:param config: the config object
	:param atoms: the user request
	:return: the list of spls involved in the request (extended with the ones in the config),
		with the corresponding SMT constraint.
		Additionally, the pattern repository is extended with the new patterns,
		and the config object have been updated with th USE user configuration
	"""
	requested_patterns = set([hyportage_pattern.pattern_create_from_atom(atom) for atom in atoms])
	requested_patterns.update(config.pattern_required_flat)

	config.set_use_manipulation_env(os.environ.get("USE", "").split())

	root_spls, constraint = smt_encoding.convert_patterns(pattern_repository, id_repository, requested_patterns)
	return root_spls, constraint

##########################################################################
# 2. SOLVER WRAPPER
##########################################################################

def get_preferences_core(id_repository, mspl, installed_spls, spl_name_set):
	"""
	the priority of preferences is decided as follows:
		- remove less packages installed as possible,
		- minimize number of new packages to install
	:param id_repository: the repository of hyportage
	:param mspl: the mspl of hyportage
	:param installed_spls: the currently installed spls with their configuration
	:param spl_name_set: the names of the spls considered in the reconfiguration process
	:return: an equation encoding the preference as described previously
	"""
	installed_spls = {
		spl_name
		for spl_name in installed_spls.iterkeys()
		if (spl_name in spl_name_set) and (not mspl[spl_name].is_deprecated)}
	uninstalled_spls = spl_name_set - installed_spls
	# preference for removing less packages installed and not deprecated as possible
	res = '0'
	if installed_spls:
		res = " + ".join([
			smt_encoding.get_spl_hyvarrec(id_repository, spl_name) for spl_name in installed_spls])
	# preference to minimize number of new packages to install
	if uninstalled_spls:
		res = res + " - (" + (" + ".join([
			smt_encoding.get_spl_hyvarrec(id_repository, spl_name) for spl_name in uninstalled_spls])) + ")"
	return [res]

def get_preferences_use_flags(id_repository, mspl, spl_names):
	"""
	This function translates the use flag default selection of the spl in parameter into a preferences
	TODO: implement this function. It requires an API change in the SPL class to do it efficiently.
	:param id_repository: the id repository of hyportage
	:param mspl: the mspl of hyportage
	:param spl_names: the names of the spls to include in the preferences construction
	:return: an equation encoding the default configuration of the spls in parameter
	"""
	use_flags_positive = set()
	use_flag_negative = set()
	return 0

def installed_spls_to_solver(id_repository, installed_spls, spl_names):
	res = []
	for spl_name, use_selection in installed_spls.iteritems():
		if spl_name in spl_names:
			res.append(smt_encoding.get_spl_hyvarrec(id_repository, spl_name))
	return res

def get_better_constraint_visualization(id_repository, mspl, constraints):
	"""
	function that manipulates the constraints in a more readable form for analysing them.
	Useful for debugging or error reporting
	:param id_repository: the id repository of hyportage
	:param mspl: the mspl of hyportage
	:param constraints: the constraints to manipulate
	:return: the manipulated constraints
	"""
	ls = []
	parser = SmtLib20Parser()
	for i in constraints:
		f = cStringIO.StringIO(i)
		script = parser.get_script(f)
		f.close()
		formula = script.get_last_formula()
		formula = pysmt.shortcuts.to_smtlib(formula, daggify=False)
		# translate packages
		where_declared = "user-required: "
		spl_ids = set(re.findall('(p[0-9]+)', formula))
		for spl_id in spl_ids:
			name = id_repository.ids[spl_id][1]
			formula = re.sub(spl_id, name, formula)
			if i in mspl[name].smt:
				where_declared = name + ": "

		# translate uses
		use_ids = set(re.findall('(u[0-9]+)', formula))
		for use_id in use_ids:
			formula = re.sub(use_id, id_repository.ids[use_id][2] + "#" + id_repository.ids[use_id][1], formula)
		ls.append(where_declared + formula)
	return ls

def generate_to_install_spls(id_repository, mspl, feature_list):
	"""
	translate the output of the solver into a system to install (mapping from spl names to use flag selection)
	:param id_repository: the id repository of hyportage
	:param mspl: the mspl of hyportage
	:param feature_list: the solution found by the solver
	:return: a dictionary spl_name -> use flag selection
	"""
	# 1. translate the computed solution into a configuration
	res_core = core_data.dictSet()
	use_flags = []
	for feature in feature_list:
		el = id_repository.data_from_id(feature)
		if el[0] == "package":  # el = ("package", spl_name)
			res_core.add_key(el[1])
		else:  # el = ("use", use, spl_name)
			use_flags.append((el[1], el[2]))
	for use_flag, spl_name in use_flags:
		if spl_name in res_core:
			res_core.add(spl_name, use_flag)

	# 2. compares the computed solution to the actual spl use flag configuration, and generate the final configuration
	res = core_data.dictSet()
	for spl_name, use_selection_core in res_core.iteritems():
		spl = mspl[spl_name]
		if spl.use_selection_core == use_selection_core:
			res[spl_name] = spl.use_selection_full
		else:
			annex_use_flags = spl.use_selection_full - spl.iuses_core
			res[spl_name] = use_selection_core | annex_use_flags
	return res

def solve_spls(
		id_repository, config, mspl, spl_groups,
		spls, annex_constraint, exploration_use, exploration_mask, exploration_keywords, explain_modality=False):
	"""
	Solves the spls in input locally assuming that there is a command hyvar-rec
	:param id_repository: the id repository of hyportage
	:param config: the config of hyportage
	:param mspl: the mspl of hyportage
	:param spl_groups: the spl groups of hyportage
	:param spls: the spls to solve
	:param annex_constraint: the additional constraint to add in the solver input
	:param exploration_use: boolean saying if the solver can change the use flag default selection
	:param exploration_mask: boolean saying if the solver can change the use mask status of the packages
	:param exploration_keywords: boolean saying if the solver can change the keywords of the packages
	:param explain_modality: boolean saying if a problem should be explained (by default: False)
	:return: the solution found by the solver, if it exists
	"""
	# 1. construct the input data for the solver
	# 1.1. construct the constraint
	constraint = annex_constraint[:]
	spl_group_names = core_data.dictSet()
	#tmp = 0
	for spl in spls:
		spl_group_names.add(core_data.spl_core_get_spl_group_name(spl.core), spl)
		included = (spl.unmasked or exploration_mask) and (spl.unmasked_keyword or exploration_keywords)
		if included:
			#tmp = tmp + 1
			constraint.extend(spl.smt)
			if exploration_use:
				constraint.extend(spl.smt_use_exploration)
			else:
				constraint.extend(spl.smt_use_selection)
		else:
			#logging.info("spl \"" + spl.name + "\" is not scheduled for possible installation")
			constraint.extend(spl.smt_false)
	for spl_group_name, spls_tmp in spl_group_names.iteritems():
		spl_group = spl_groups[spl_group_name]
		constraint.extend(spl_group.smt)
		for spl in spl_group:
			if spl not in spls_tmp: constraint.append(smt_encoding.smt_to_string(smt_encoding.get_spl_smt_not(id_repository, spl.name)))
	#logging.info("included spl: " + str(tmp))
	logging.debug("number of constraints to solve: " + str(len(constraint)))
	# 1.2. construct the preferences
	spl_names = {spl.name for spl in spls}
	preferences = get_preferences_core(id_repository, mspl, config.installed_packages, spl_names)
	# 1.3. construct the current system
	current_system = [] #installed_spls_to_solver(id_repository, installed_spls, spl_names)
	data_configuration = {"selectedFeatures": current_system, "attribute_values": [], "context_values": []} # current configuration
	data_smt_constraints = {"formulas": constraint, "features": [], "other_int_symbols": []}
	data = {
		"attributes": [],  # attributes of the features (empty in our case)
		"contexts": [],  # contexts to consider (empty in our case)
		"configuration": data_configuration,
		"constraints": [], # constraints to fill in hyvarrec format (empty in our case for efficiency)
		"preferences": preferences, # preferences in hyvarrec format
		"smt_constraints": data_smt_constraints,
		"hyvar_options": ["--features-as-boolean", "--constraints-minimization", "--no-default-preferences"]
	}

	# 2. run hyvar-rec
	res = run_hyvar(data)
	if res is None: return None
	logging.debug("HyVarRec output: " + json.dumps(res))

	# 4. managing the solver output
	if res["result"] != "sat":
		if explain_modality:
			# todo handle explain modality when the answer is unsat
			# try to print a better explanation of the constraints
			constraints = get_better_constraint_visualization(id_repository, mspl, res["constraints"])
			logging.error("Conflict detected. Explanation:\n" + "\n".join(constraints) + '\n')
		return None

	return generate_to_install_spls(id_repository, mspl, res['features'])

def generate_installation_files(
		mspl,
		path_emerge_script, path_use_flag_configuration, path_mask_configuration, path_keywords_configuration,
		old_installation, new_installation):
	"""
	This function generates two files:
	1. the script file to execute to install and uninstall the spls found by the solver
	2. spl configuration file (usually package.use) from the solution found by the solver
	:param mspl: the mspl of hyportage
	:param path_emerge_script: path to the script file to generate
	:param path_use_flag_configuration: path to the configuration file to generate
	:param path_mask_configuration: the path to the unmask file to generate
	:param path_keywords_configuration: the path to the accept_keywords file to generate
	:param old_installation: the currently installed spls
	:param new_installation: the spls to install, found by the solver
	:return: None (but the script file has been generated)
	"""

	# the spls to emerge are the ones that are not present in the old installation, or that have a new configuration
	added_spl_names = []
	for spl_name, product in new_installation.iteritems():
		if spl_name in old_installation:
			if old_installation[spl_name] != product:
				added_spl_names.append(spl_name)
		else: added_spl_names.append(spl_name)

	# the spls to remove are the ones that are not in the new configuration and that are not replaced by a new version
	removed_spl_names = []
	new_spl_goups_info = core_data.dictSet()
	for spl_name in new_installation.iterkeys():
		spl = mspl[spl_name]
		new_spl_goups_info.add(core_data.spl_core_get_spl_group_name(spl.core), spl)
	for spl_name in old_installation.iterkeys():
		spl = mspl[spl_name]
		new_versions = new_spl_goups_info.get(core_data.spl_core_get_spl_group_name(spl.core))
		if new_versions is None:
			removed_spl_names.append(spl_name)
		else:
			replaced = False
			for new_spl in new_versions:
				if (spl.slot == new_spl.slot) and (spl.name != new_spl.name):
					replaced = True
					break
			if replaced: removed_spl_names.append(spl_name)

	# write the files
	added_spl_names.sort()
	with open(path_emerge_script, 'w') as f:
		f.write("#!/bin/bash\n")
		f.write("\n")
		f.write("# File auto-generated by the hyportage tool\n")
		f.write("# Do not update, any modification on this file will will overwritten by the tool\n")
		f.write("\n")
		if added_spl_names:
			f.write("emerge -p --newuse " + " ".join(["=" + spl_name for spl_name in added_spl_names]) + "\n")
		if removed_spl_names:
			f.write("emerge -p --unmerge " + " ".join(["=" + spl_name for spl_name in removed_spl_names]) + "\n")
		f.write("\n")

	with open(path_use_flag_configuration, 'w') as f:
		f.write("# File auto-generated by the hyportage tool\n")
		f.write("# Do not update, any modification on this file will will overwritten by the tool\n")
		f.write("\n")
		for spl_name in added_spl_names:
			use_selection = new_installation[spl_name]
			string = "=" + spl_name + " "
			string = string + " ".join(use_selection)
			use_unselection = mspl[spl_name].iuses_full - use_selection
			if use_unselection:
				string = string + " -" + " -".join(use_unselection) + "\n"
			f.write(string)
		f.write("\n")

	with open(path_mask_configuration, 'w') as f:
		f.write("# File auto-generated by the hyportage tool\n")
		f.write("# Do not update, any modification on this file will will overwritten by the tool\n")
		f.write("\n")
		for spl_name in added_spl_names:
			f.write("=" + spl_name + "\n")
			#if not mspl[spl_name].unmasked:
			#	f.write("=" + spl_name)
		f.write("\n")

	with open(path_keywords_configuration, 'w') as f:
		f.write("# File auto-generated by the hyportage tool\n")
		f.write("# Do not update, any modification on this file will will overwritten by the tool\n")
		f.write("\n")
		for spl_name in added_spl_names:
			#f.write("=" + spl_name + " ~" + hyportage_db.mspl_config.arch + "\n")
			f.write("=" + spl_name + " **\n")
			#if not mspl[spl_name].unmasked_keyword:
			#	f.write("=" + spl_name + " ~" + hyportage_db.mspl_config.arch)
		f.write("\n")

##########################################################################
# 3. SPL SET COMPUTATION
##########################################################################

def next_spls(pattern_repository, spl):
	res = set()
	pattern_iterator = spl.dependencies.iterkeys()
	for pattern in pattern_iterator:
		res.update(pattern_repository.get_with_default(pattern).matched_spls)
	return res

def get_dependency_transitive_closure(pattern_repository, mspl, spls):
	for spl in mspl.itervalues():
		spl.visited = False
	nexts = spls
	res = set()
	while len(nexts) > 0:
		accu = set()
		for spl in nexts:
			spl.visited = True
			res.add(spl)
			accu.update(next_spls(pattern_repository, spl))
		nexts = filter(lambda spl: not spl.visited, accu)

	return res


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('siteScrape', '0001_initial'),
    ]

    operations = [
        migrations.AddField(
            model_name='teacher',
            name='averageGrade',
            field=models.CharField(default=b'', max_length=3),
        ),
        migrations.AddField(
            model_name='teacher',
            name='responseRate',
            field=models.DecimalField(default=0, max_digits=3, decimal_places=2),
        ),
        migrations.AlterField(
            model_name='teacher',
            name='quarters',
            field=models.IntegerField(default=0),
        ),
    ]


from Christiaan.csvLoading.CSVLoader import read_csv_tableau
from Christiaan.dataCleaning.dataCleaner import cleanTechColumn
import pandas as pd
import numpy as np

def readTechCSV(filename):
    df = read_csv_tableau(filename, filename)
    df = df[['Employee Number', 'Firstname Firstname', 'Lastname Lastname', 'Level Level', 'Practice Practice', 'Suggested Daily Rate', 'Name Name']]
    df.columns =['Employee Number', 'Firstname', 'Lastname', 'Level', 'Practice', 'Suggested Daily Rate', 'Technology']
    df.sort_values('Employee Number', inplace=True)
    return df

def fixLevels(dataFrame):
    emp_first_names = ['Jean-Francois',
                       'Arnaud',
                       'Vasilij',
                       'Pieter',
                       'Koen',
                       'Faisal',
                       'Gunther',
                       'Philip',
                       'Geoffrey']
    emp_last_names = ['Gigot',
                      'Deflorenne',
                      'Nevlev',
                      'Vandamme',
                      'Dils',
                      'Orakzai',
                      'Hellebaut',
                      'Allegaert',
                      'Moerenhoudt']

    for i in range(len(emp_first_names)):
        f = emp_first_names[i]
        l = emp_last_names[i]
        if f == 'Jean-Francois' and l == 'Gigot':
            dataFrame.loc[np.logical_and(dataFrame.Firstname == f, dataFrame.Lastname == l),'Level'] = 'Director'
        elif f == 'Arnaud' and l == 'Deflorenne':
            dataFrame.loc[np.logical_and(dataFrame.Firstname == f, dataFrame.Lastname == l),'Level'] = 'Partner'
        elif f == 'Vasilij' and l == 'Nevlev':
            dataFrame.drop(dataFrame[np.logical_and(dataFrame.Firstname == f, dataFrame.Lastname == l)].index, axis=0, inplace=True)
        elif f == 'Pieter' and l == 'Vandamme':
            dataFrame.loc[np.logical_and(dataFrame.Firstname == f, dataFrame.Lastname == l),'Level'] = 'Director'
        elif f == 'Koen' and l == 'Dils':
            dataFrame.loc[np.logical_and(dataFrame.Firstname == f, dataFrame.Lastname == l),'Level'] = 'Director'
        elif f == 'Faisal' and l == 'Orakzai':
            dataFrame.loc[np.logical_and(dataFrame.Firstname == f, dataFrame.Lastname == l),'Level'] = 'Senior'
        elif f == 'Gunther' and l == 'Hellebaut':
            dataFrame.loc[np.logical_and(dataFrame.Firstname == f, dataFrame.Lastname == l),'Level'] = 'Senior'
        elif f == 'Philip' and l == 'Allegaert':
            dataFrame.loc[np.logical_and(dataFrame.Firstname == f, dataFrame.Lastname == l),'Level'] = 'Director'
        elif f == 'Geoffrey' and l == 'Moerenhoudt':
            dataFrame.loc[np.logical_and(dataFrame.Firstname == f, dataFrame.Lastname == l),'Level'] = 'Consultant'

    dataFrame.loc[np.logical_or(dataFrame.Level=='Senior 13', dataFrame.Level=='Senior 8'),'Level'] = 'Senior'

    return dataFrame

def cleanTech(dataFrame, techColumnName):
    df = cleanTechColumn(dataFrame, techColumnName)
    df.drop_duplicates(inplace=True)
    df = fixLevels(df)
    return df

def exportDF(dataFrame, fileName, path=''):
    dataFrame.to_csv(path_or_buf=path+fileName, sep=',', index=False)

# Read Tableau csv export
file = r'C:\Users\adebola.oshomoji\PycharmProjects\case-keyrus\tableau_technology_export.csv'
df = readTechCSV(file)

# Clean Technology column values and drop duplicates
df = cleanTech(df, 'Technology')

# Write DataFrame to csv
exportDF(df, 'EmployeeTechnologyData.csv')

#print(df.Level.unique())


from argparse import ArgumentParser
from inference import Infer

parser = ArgumentParser()

parser.add_argument("modelname", help="name of model to use")
parser.add_argument("imagepath", help="relative path to image")
parser.add_argument("--use_gpu", help="use gpu or not", nargs="?", default=False, const=True, type = bool)
args = parser.parse_args()

infer = Infer(args.use_gpu)

try:
    infer.infer(args.imagepath, args.modelname)
except:
    print("Something BAD happened!!!")


import sys
from collections import deque

dx = [-1, 1, 0, 0]
dy = [0, 0, -1, 1]
INF = 1e9

N, M = map(int, sys.stdin.readline().rstrip().split())
maps = [list(map(int, sys.stdin.readline().rstrip().split())) for _ in range(N)]
hospital_comb = []
answer = INF

def dfs(hospital_list, pick_list, idx):
    if idx == len(hospital_list):
        if len(pick_list) == M:
            hospital_comb.append(pick_list[:])
        return

    pick_list.append(hospital_list[idx])
    dfs(hospital_list, pick_list, idx + 1)
    pick_list.pop()
    dfs(hospital_list, pick_list, idx + 1)

def bfs(hospital_list):
    global answer
    q = deque([])
    visited = [[False] * N for _ in range(N)]
    time_maps = [[0] * N for _ in range(N)]

    for h in hospital_list:
        q.append((h[0], h[1], 0))
        visited[h[0]][h[1]] = True

    while q:
        x, y, cnt = q.popleft()
        for i in range(4):
            nx, ny = x + dx[i], y + dy[i]
            if 0 <= nx < N and 0 <= ny < N and not visited[nx][ny]:
                if maps[nx][ny] == 0 or maps[nx][ny] == -2:
                    q.append((nx, ny, cnt + 1))
                    visited[nx][ny] = True
                    time_maps[nx][ny] = cnt + 1

    time = 0
    for i in range(N):
        for j in range(N):
            if maps[i][j] == 0 and time_maps[i][j] == 0:
                return
            if maps[i][j] == 0:
                time = max(time, time_maps[i][j])
    answer = min(answer, time)

hospital = []
for i in range(N):
    for j in range(N):
        if maps[i][j] == 2:
            hospital.append((i, j))
            maps[i][j] = -2
        if maps[i][j] == 1:
            maps[i][j] = -1

dfs(hospital, [], 0)

for i in range(len(hospital_comb)):
    bfs(hospital_comb[i])

print(-1) if answer == INF else print(answer)


from django.core import validators
from rest_framework import serializers
from django.utils.translation import gettext_lazy as _

from degvabank.apps.account.models import Account
from degvabank.apps.card.models import CreditCard
from degvabank.apps.transaction.utils import is_our_number

from .models import Transaction

class TransactionSerializer(serializers.ModelSerializer):
    class Meta:
        model = Transaction
        fields = "__all__"

class UserTransactionSerializer(serializers.ModelSerializer):
    acc = card = dst = dst_not_our = None
    document_id = serializers.CharField(
        write_only=True,
        max_length=15,
        validators=[
            validators.RegexValidator(
                regex=r"^[eEvVjJ]\d+$",
                message=_("your document id is not well formatted"),
            ),
        ],
    )

    def validate_source(self, value):
        user = self.context["request"].user
        self.acc = user.accounts.filter(id=value, is_active=True).first()
        self.card = user.credit_cards.filter(number=value, is_active=True).first()
        if not (self.acc or self.card):
            raise serializers.ValidationError(_("Invalid source account or card"))
        return value

    def validate_target(self, value):
        # TODO: if value not ours: return value
        if not is_our_number(value):
            self.dst = value
            self.dst_not_our = True
            return value

        dst_acc = Account.objects.filter(id=value, is_active=True).first()
        dst_card = CreditCard.objects.filter(number=value, is_active=True).first()
        if not (dst_acc or dst_card):
            raise serializers.ValidationError(
                _("Target account or card does not exists")
            )
        self.dst = dst_card or dst_acc
        return value

    def validate_document_id(self, value):
        if not self.dst_not_our and self.dst and self.dst.user.document_id.lower() != str(value).lower():
            raise serializers.ValidationError(
                _("Target account or card is not associated with that document id")
            )
        return value

    def validate_amount(self, value):
        if self.acc and self.acc.balance < value:
            raise serializers.ValidationError(_("Insufficent balance"))
        if self.card and self.card.credit < value:
            raise serializers.ValidationError(_("Insufficent balance"))
        return value

    class Meta:
        model = Transaction
        fields = [
            "id",
            "source",
            "target",
            "document_id",
            "amount",
            "type",
            "status",
            "reason",
            "date",
        ]
        read_only_fields = ("type", "status", "date", "id")

    def create(self, validated_data):
        field_names = [field.name for field in self.Meta.model._meta.get_fields()]
        data = {a: b for a, b in validated_data.items() if a in field_names}
        kwargs = {
            "amount": data["amount"],
            "reason": data["reason"],
            "source": {
                "number": data["source"]
            },
            "target": {
                "number": data["target"],
                "document_id": validated_data["document_id"],
            }
        }
        return self.Meta.model.objects.create_any_transaction(**kwargs)

class TransactionCardSerializer(serializers.Serializer):
    number = serializers.CharField()
    security_code = serializers.CharField()
    expiration_date = serializers.DateTimeField()
    document_id = serializers.CharField(
        required=False,
        write_only=True,
        max_length=15,
        validators=[
            validators.RegexValidator(
                regex=r"^[eEvVjJ]\d+$",
                message=_("your document id is not well formatted"),
            ),
        ],
    )

class TransactionAccountSerializer(serializers.Serializer):
    number = serializers.CharField()
    document_id = serializers.CharField(
        required=True,
        write_only=True,
        max_length=15,
        validators=[
            validators.RegexValidator(
                regex=r"^[eEvVjJ]\d+$",
                message=_("your document id is not well formatted"),
            ),
        ],
    )

class ForeignTransactionSerializer(serializers.ModelSerializer):

    acc_src = TransactionAccountSerializer(required=False)
    acc_dst = TransactionAccountSerializer(required=False)
    card_src = TransactionCardSerializer(required=False)
    card_dst = TransactionCardSerializer(required=False)

    class Meta:
        model = Transaction
        fields = [
            "id",
            "acc_src",
            "acc_dst",
            "card_src",
            "card_dst",
            "amount",
            "type",
            "status",
            "reason",
            "date",
        ]
        read_only_fields = ("type", "status", "date", "id")

    def create(self, validated_data):
        field_names = [field.name for field in self.Meta.model._meta.get_fields()]
        data = {a: b for a, b in validated_data.items() if a in field_names}
        kwargs = {
            "amount": data["amount"],
            "reason": data["reason"],
            "source": validated_data.get("acc_src") or validated_data.get("card_src"),
            "target": validated_data.get("acc_dst") or validated_data.get("card_dst")
        }
        return self.Meta.model.objects.create_any_transaction(from_foreign=True, **kwargs)


import os
import json
import numpy as np
import preprocessing as preprocessing
from tensorflow import keras

def init():
    global model
    global vocab
    global max_len
    model = keras.models.load_model(os.path.join(os.getenv('AZUREML_MODEL_DIR'), 'model'), compile=False)
    with open(os.path.join(os.getenv('AZUREML_MODEL_DIR'), 'model', 'vocab.json')) as json_file:
        vocab = json.load(json_file)
    with open(os.path.join(os.getenv('AZUREML_MODEL_DIR'), 'model', 'params.json')) as json_file:
        params = json.load(json_file)
    max_len = params['max_len']

def run(raw_data):
    tweets = np.array(json.loads(raw_data)["data"])
    processed_tweets = preprocessing.process_tweets(tweets, vocab, max_len)
    result = model.predict(processed_tweets).ravel()
    return result.tolist()


import requests
from bs4 import BeautifulSoup as bs
import csv
from itertools import chain

def get_urls():
    """
    Skilar lista sem inniheldur slóðir á allar undirsíður
    með kosningaúrslitum
    """
    main_page = requests.get("http://www.kosningastofnun.in/")
    page_soup = bs(main_page.content, "html.parser")

    urls = [title.a.get("href") for title in page_soup.find_all(class_="blog-post-title")]

    return urls

def scrape(urls):
    """
    Tekur inn lista af slóðum og skrapar fyrstu töfluna sem finnst á hverri slóð.
    Skilar lista af dicts þar sem hvert dict er með töfluhausa sem lykla og
    töfluhólf sem gildi
    """
    all_dicts = []

    for url in urls:
        elections_page = requests.get(url)
        elections_soup = bs(elections_page.content, "html.parser")

        # Finnum alla töfluhausa og setjum í lista:
        headers = [header.string.strip() for header in elections_soup.find_all("th")]

        # Förum svo í gegnum hverja röð og setjum gildin í lista:
        for row in elections_soup.find_all("tr"):
            results = [cell.string.strip() for cell in row.find_all("td")]

            # Athugum hvort þetta er gild niðurstöðulína, annars gerum við ekkert:
            if results:
                # Nú getum við sett þessar niðurstöður í dict með hausunum:
                elections = dict(zip(headers, results))

                # Ef dagsetning var ekki með í þessum niðurstöðum þurfum við
                # að skrapa hana úr titlinum og setja inn í dictið:
                if "Dagsetning" not in elections:
                    full_title = elections_soup.h2.a.string
                    # Splittum titlinum á fyrsta bili og tökum seinni hlutann:
                    elections["Dagsetning"] = full_title.split(" ", 1)[1]

                # Og setjum svo dictið í stóra listann:
                all_dicts.append(elections)

    return all_dicts

def save_csv(list_of_dicts):
    """
    Tekur inn lista af dicts og skrifar út í CSV-skrána "kosningar.csv" í möppunni
    sem skriftan er keyrð úr
    """
    with open("kosningar.csv", "w") as csv_file:
        fieldnames = set([key for key in chain(*list_of_dicts)])

        # Röðum dálkaheitunum, fyrst í stafrófsröð og svo aftur til að fá
        # dagsetningu fremst (ekki nauðsynlegt):
        fieldnames = sorted(fieldnames)
        fieldnames = sorted(fieldnames, key=lambda x: x == "Dagsetning", reverse=True)

        writer = csv.DictWriter(csv_file, fieldnames)
        writer.writeheader()
        writer.writerows(list_of_dicts)

# Keyrum heila klabbið:
if __name__ == "__main__":
    save_csv(scrape(get_urls()))


from numpy.testing import *
import numpy
import numpy.random

from adolc import *
from adolc.cgraph import *
from adolc.tangent import *

class TangentOperationsTests(TestCase):
    def test_constructor(self):
        t1 = Tangent(1,2)
        t2 = Tangent(adouble(1),2)

    def test_float_tangent_float_tangent(self):
        tx = Tangent(2.,3.)
        ty = Tangent(5.,7.)

        tz = tx + ty
        assert_array_almost_equal([tz.x,tz.xdot], [2+5, 3+7])

        tz = tx - ty
        assert_array_almost_equal([tz.x,tz.xdot], [2-5, 3-7])

        tz = tx * ty
        assert_array_almost_equal([tz.x,tz.xdot], [2*5, 3*5 + 2*7])

        tz = tx / ty
        assert_array_almost_equal([tz.x,tz.xdot], [2./5., (3*5 - 2*7.)/5**2])

    def test_double_tangent_adouble(self):
        tx = Tangent(2,3)
        ay = adouble(5)

        tz = tx + ay
        assert_array_almost_equal([tz.x.val,tz.xdot], [2+5, 3])

        tz = tx * ay
        assert_array_almost_equal([tz.x.val,tz.xdot.val], [2*5, 3*5])

    def test_adouble_tangent_adouble_addition(self):
        tx = Tangent(adouble(2), 1)
        ty = Tangent(adouble(3), 0)
        tz = tx + ty
        assert_array_almost_equal([tz.x.val,tz.xdot], [5, 1])

class SemiImplicitOdeLhsTest(TestCase):
    """
    This is a test example taken from PYSOLVIND

    In chemical engineering, semi-implicit ODEs of the type::

        d/dt g(t,y(t)) = f(t,y(t))
        y(0) = y_0

    have to be solved. PYSOLVIND requires a function afcn that computes::

        d/dy g(t,y) d/dt y

        where d/dt y = xdd
                   y = xd
    """

    def test_differentiation_of_gfcn(self):
        def gfcn(a):
            print('called gfcn')
            ty = [Tangent(a.xd[0], a.xdd[0]),Tangent(a.xd[1], a.xdd[1]), Tangent(a.xd[2], a.xdd[2])]
            tlhs = [ty[0] * ty[2], ty[1] * ty[2], ty[2]]

            a.lhs[0] = tlhs[0].xdot
            a.lhs[1] = tlhs[1].xdot
            a.lhs[2] = tlhs[2].xdot

        def afcn(a):
            a.lhs[0] = a.xd[2] * a.xdd[0] + a.xd[0] * a.xdd[2]
            a.lhs[1] = a.xd[2] * a.xdd[1] + a.xd[1] * a.xdd[2]
            a.lhs[2] = a.xdd[2]

        class Args:
            def __init__(self):
                self.xd  = numpy.random.rand(3)
                self.xdd = numpy.random.rand(3)
                self.lhs = numpy.zeros(3)

        args = Args()

        gfcn(args)
        result1 = args.lhs.copy()

        afcn(args)
        result2 = args.lhs.copy()

        assert_array_almost_equal(result1, result2)

# class FunctionExampleTests(TestCase):
    # def test_utps_on_jacobian(self):

        # def f(x,p):
            # print p
            # print p[0] + p[1]
            # return (p[0] + p[1]) *  x**2

        # AP = AdolcProgram()
        # AP.trace_on(1)
        # ax = adouble(3.)
        # ap = adouble([5.,7.])
        # AP.independent(ax)
        # AP.independent(ap)

        # tp = [Tangent(ap[0],1),Tangent(ap[1],0)]
        # tf = f(ax,tp)

        # aJ = tf.xdot

        # print aJ

        # AP.dependent(aJ)
        # AP.trace_off()

        # g = gradient(1, [1,2,3])

        # print g

if __name__ == '__main__':
    try:
        import nose
    except:
        print('Please install nose for unit testing')
    nose.runmodule()


## ~~~~~~~~~~~~~~~~~~
#  Deep Willy Network
## ~~~~~~~~~~~~~~~~~~

import numpy as np, json, sys, os
sys.path.append(os.path.dirname(__file__))

from willies import *

class DeepWilly(object):

    _willy_classes = {'connected': ConnectedWilly,
                      'dropout': DropoutWilly,
                      'convolutional': ConvolutionalWilly,
                      'pooling': PoolingWilly,
                      'stacking': StackingWilly}

    def __init__(self, cost_func = 'cross entropy'):

        self.willies = []
        self.n_willies = 0

        self.cost_func = cost_func
        if cost_func == 'cross entropy':
            self.cost  = lambda a, y: -np.mean(y * np.log(a + (a == 0)) + (1-y) * np.log(1-a + (a == 1)))
            self.dcost = lambda a, y: (1/a.shape[0]) * (a - y) / (a * (1 - a) + 1e-99)

        elif cost_func == 'quadratic':
            self.cost  = lambda a, y: 0.5 * np.mean((a - y)**2)
            self.dcost = lambda a, y: (1/a.shape[0]) * (a - y)

        else:
            raise Exception('Unsupported cost function: ' + cost_func)

    def add(self, willy):
        ''' Add a willy to the network and set it up based on the previous willy's output (if not the first one). '''

        if len(self.willies) == 0:
            if not willy.is_set_up:
                raise Exception('Input shape or number must be provided to first Willy.')

        else:
            willy.set_up(self.willies[-1].out_shape)

        self.willies.append(willy)
        self.n_willies += 1

    def forward_prop(self, X):
        ''' Forward propagates X through the willies. '''

        for willy in self.willies:
            X = willy.forward_prop(X)

        return X

    def backward_prop(self, X, y,
                      learn_rate, mom_rate, reg_rate):
        ''' Backward propagates errors while simultaneously updating weights in each layer. '''

        # Output layer dC/da
        batch_size = X.shape[0]
        dA = self.dcost(self.willies[-1].A, y)

        # Backpropagate, updating weights
        XA = [X] + [willy.A for willy in self.willies[:-1]]
        XA = XA[::-1]

        for w, willy in enumerate(reversed(self.willies)):
            willy.update_weights(dA, XA[w],
                                 learn_rate, mom_rate, reg_rate)
            dA = willy.backward_prop()

    @staticmethod
    def get_batches(X, y, batch_size):
        ''' Shuffles training data and gets random batch of desired size. '''

        if batch_size == -1 or batch_size >= X.shape[0]:
            return [X], [y]

        # Shuffle data
        shuffled_indices = np.random.permutation(len(X))
        shuffled_X = X[shuffled_indices]
        shuffled_y = y[shuffled_indices]

        # Get batch of desired size
        X_batches = []
        y_batches = []
        for i in range(X.shape[0]//batch_size):
            X_batches.append(shuffled_X[int(batch_size*i):int(batch_size*(i+1))])
            y_batches.append(shuffled_y[int(batch_size*i):int(batch_size*(i+1))])

        return X_batches, y_batches

    def train(self, X, y,
              num_iterations, batch_size,
              learn_rate, reg_rate, mom_rate = 0,
              verbose = False):
        ''' Builds network, trains using given data and training parameters. '''

        # Change dtypes
        X = X.astype(np.float32)
        y = y.astype(np.float32)

        # Initialise momenta to 0
        for willy in self.willies:
            willy.reset_momenta()

        # Train network
        for iteration in range(num_iterations):

            # Get batches:
            X_batches, y_batches = self.get_batches(X, y, batch_size)

            for batchX, batchy in zip(X_batches, y_batches):

                # Forward propagate
                self.forward_prop(batchX)

                # Backward propagate & update weights
                self.backward_prop(batchX, batchy,
                                   learn_rate, mom_rate, reg_rate)

            # Print progress
            if verbose:
                if iteration % verbose == 0:
                    print("Training cost on last batch: ", self.cost(self.willies[-1].A, batchy))

    def predict(self, X, pred_type = 'as is'):

        self.yhat = self.forward_prop(X)

        if pred_type == 'binary':
            self.yhat = 1 * (self.yhat > 0.5)

        elif pred_type == 'argmax':
            self.yhat = np.argmax(self.yhat, axis = 1).reshape(-1, 1)

        else:
            assert pred_type == 'as is', \
                "Provided argument pred_type (" + pred_type + ") not supported."

        return self.yhat

    def accuracy(self, X, y, pred_type = 'as is'):
        ''' Gets accuracy of predictions. '''

        return np.mean(self.predict(X, pred_type) == y)

    def save(self, filename):
        ''' Saves deep willy to file \filename. '''

        willy_data = []
        for willy in self.willies:
            willy_data.append(willy.save())

        data = {'cost': self.cost_func,
                'willies': willy_data}

        file = open(filename, "w")
        json.dump(data, file)
        file.close()

    @classmethod
    def load(cls, filename):
        ''' Loads deep willy from file \filename. '''

        file = open(filename, "r")
        data = json.load(file)
        file.close()

        deep_willy = cls(data['cost_func'])
        for willy in data['willies']:
            willy_class = DeepWilly._willy_classes[willy['willy']]
            deep_willy.add(willy_class.load(willy))

        return deep_willy

    def copy(self):
        ''' Replicates this deep willy using its attributes. '''

        copy = DeepWilly(cost_func = self.cost_func)

        for willy in self.willies:
            copy.willies.append(willy.copy())
            copy.n_willies += 1

        return copy


from typing import *

# 一个较长的字符串肯定不会是一个较短字符串的子序列，那么只需要从长到短判断，每一个字符串是否为其他长度不小于它的字符串的子序列就行了
# When we add a letter Y to our candidate longest uncommon subsequence answer of X, it only makes it strictly harder to find a common subsequence.
# Thus our candidate longest uncommon subsequences will be chosen from the group of words itself.
class Solution:
    def findLUSlength(self, strs: List[str]) -> int:
        def isSubSeq(w1, w2):
            if len(w1) > len(w2):
                return False
            l, r = 0, 0
            while l < len(w1) and r < len(w2):
                if w1[l] == w2[r]:
                    l += 1
                r += 1
            return l == len(w1)
        strs.sort(key = len, reverse = True)
        for i, w1 in enumerate(strs):
            if all(not isSubSeq(w1, w2) for j, w2 in enumerate(strs) if i != j):
                return len(w1)
        return -1

if __name__ == "__main__":
    s = Solution()

    assert s.findLUSlength(["aba","cdc","eae"]) == 3
    assert s.findLUSlength(["aaa","aaa","aa"]) == -1


# coding:utf-8
from bs4 import BeautifulSoup
import urllib.request as req
import sqlite3
from contextlib import closing

url="http://su-gi-rx.com/2017/07/16/python_4/"
dbname='database.db'
conn=sqlite3.connect(dbname)
c=conn.cursor()
table_name = 'test'

def get_html():
    #urlopen()でデータ取得
    res=req.urlopen(url)

    #BeautifulSoup()で解析
    soup=BeautifulSoup(res,'html.parser')

    #任意のデータを抽出
    title1=soup.find("h1").string
    #print("title=",title1)

    p_list=soup.find_all("p")
    #print("text=",p_list)

    return [(str(title1),str(p_list))]

def create_table(tname):

    #executeメソッドでＳＯＬ文を実行する
    create_table='''create table if NOT EXISTS {0} (title varchar(64),p_list varchar(32))'''.format(tname)
    c.execute(create_table)

def insert_data(tname,data):
    insert_sql='insert into {0} (title,p_list) values(?,?)'.format(tname)
    c.executemany(insert_sql,test)
    conn.commit()

if __name__=='__main__':
    create_table(table_name)
    test=get_html()
    insert_data(table_name,test)

    select_sql = 'select * from {0}'.format(table_name)

    for row in c.execute(select_sql):
        print(row)

    conn.close()


def convert_rank(num):
    rank = 6 - num + 1
    if rank >= 6 :
        rank = 6
    return rank

def solution(lottos, win_nums):
    match, zero = 0, 0
    for num in lottos :
        if not num :
            zero += 1
        if num in win_nums:
            match += 1
    return [convert_rank(match+zero), convert_rank(match)]

test_case = [
    [[44, 1, 0, 0, 31, 25],	[31, 10, 45, 1, 6, 19]],
    [[0, 0, 0, 0, 0, 0],	[38, 19, 20, 40, 15, 25]],
    [[45, 4, 35, 20, 3, 9],	[20, 9, 3, 45, 4, 35]]
]

for lottos, win_nums in test_case:
    print(solution(lottos, win_nums))


model_cfg = {
    "high_level_parameters": {  # parameters common to all models, e.g. how much data to get, debugging parameters etc.
        "data_period": ("2020-01-01", "2023-01-01"),
        "parameter2": 60,
        "parameter3": 20,
    },
    "low_level_parameters": {
        "model1_parameters": {
            "learning_rate": 1,
            "hidden_layers": 2,
        },
        "model2_parameters": {
            "ewma_alpha": 0.05,
            "sigma": 3,
        },
    },
}


# from game import random
# from game import np
from game import Game
from agent import Agent
import pandas as pd

class GameAnalysis:
    def __init__(self, imported_game: Game):
        self.game = imported_game

    def print_data(self):
        rows = []
        for rob in self.game.robots:
            rows.append([rob.name, rob.logs["food_found"]])
        robdf = pd.DataFrame(rows, columns=["name", "food found"]).set_index(["name"])
        return robdf

    @staticmethod
    def vocab_to_df(rob: Agent):
        rows = []
        columns = [f"{rob.name}_input", f"{rob.name}_action", f"{rob.name}_features", f"{rob.name}_inertia",
                   f"{rob.name}_mistakes"]
        for word, action_dict in rob.vocab.items():
            for action, action_data in action_dict.items():
                row = [str(word), str(action), str(action_data[0]), action_data[1][0], action_data[1][1]]
                rows.append(row)
        return pd.DataFrame(rows, columns=columns).set_index([f"{rob.name}_input"])

    def print_vocabs(self):
        for rob in self.game.robots:
            df = self.vocab_to_df(rob)
            if "merged_df" not in locals():
                merged_df = df
            else:
                merged_df = merged_df.join(df, how="outer")
        return merged_df

    def print_vocab_chain(self, requester: Agent, fetcher: Agent):
        dfr = self.vocab_to_df(requester).reset_index()
        dff = self.vocab_to_df(fetcher).reset_index()
        return pd.merge(dfr, dff, left_on="A_action", right_on="B_input", how="outer")

    # def plot_robot_paths(self, l_slice=None, r_slice=None, reverse=True):
    #     fig, ax = plt.subplots()
    #     empty_plot = True
    #     for rob in self.game.robots:
    #         col = (np.random.random(), np.random.random(), np.random.random())
    #         if l_slice and r_slice:
    #             paths_to_plot = rob.logs["taken_paths"][l_slice:r_slice]
    #         elif l_slice or r_slice:
    #             paths_to_plot = rob.logs["taken_paths"][l_slice:]
    #         else:
    #             paths_to_plot = rob.logs["taken_paths"]
    #         if reverse:
    #             paths_to_plot.reverse()
    #         for path in paths_to_plot:
    #             final_location = path[-1]
    #             unzipped = self.unzip_locations(path)
    #             x = unzipped[0]
    #             y = unzipped[1]
    #             # if self.test_for_movement(x, y):
    #             #     empty_plot = False
    #             #     touching_food = False
    #             #     for food_type in self.game.food_types:
    #             #         if self.game.maze.is_player_colliding_with_food(final_location,food_type):
    #             #             touching_food = True
    #             #             print(touching_food)
    #             ax.plot(x, y, c=col)
    #     if not empty_plot:
    #         for food_type in self.game.food_types:
    #             ax.plot(*food_type[1], 'o')
    #             ax.plot(*food_type[1], 'o', markersize=int(self.game.close_enough * 12))
    #         plt.xlim([-self.game.arena_size, self.game.arena_size])
    #         plt.ylim([-self.game.arena_size, self.game.arena_size])
    #         plt.show()
    #     else:
    #         plt.close()

    # @staticmethod
    # def test_for_movement(x, y):
    #     if math.isclose(np.average(x), x[0]) and math.isclose(np.average(y), y[0]):
    #         return False
    #     return True

    def show_instructions(self):
        for rob in self.game.robots:
            if rob.role == "fetcher":
                df = pd.DataFrame(rob.logs["given_sentences"])
                return df

    # @staticmethod
    # def unzip_locations(entries):
    #     x = []
    #     y = []
    #     for entry in entries:
    #         x.append(entry[0])
    #         y.append(entry[1])
    #     return x, y


# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import pymysql

class ProxyPoolCrawlerPipeline(object):
    def process_item(self, item, spider):
        return item

# 存储到mysql数据库
class ProxyPoolCrawler2mysql(object):
    # # 测试ip是否有效，有效再插入数据库

    # def test_alive(proxy):
    #     http_url = "http://www.baidu.com"
    #     proxy_url = "http://{0}".format(proxy)
    #     try:
    #         proxy_dict = {
    #             "http": proxy_url,
    #         }
    #         response = requests.get(http_url, proxies=proxy_dict, timeout=5)
    #     except Exception as e:
    #         # print("invalid ip and port")
    #         return False
    #     else:
    #         code = response.status_code
    #         if code >= 200 and code < 300:
    #             # print("effective ip")
    #             return True
    #         else:
    #             # print("invalid ip and port")
    #             return False

    def process_item(self, item, spider):
        address = item['address']

        connection = pymysql.connect(
            host='localhost',  # 连接的是本地数据库
            user='root',  # 自己的mysql用户名
            passwd='',  # 自己的密码
            db='proxypool',  # 数据库的名字
            charset='utf8mb4',  # 默认的编码方式：
            cursorclass=pymysql.cursors.DictCursor)

        try:
            with connection.cursor() as cursor:
                # 创建更新值的sql语句
                sql = """INSERT INTO proxy(address)
                            VALUES (%s)"""
                # 执行sql语句
                # excute 的第二个参数可以将sql缺省语句补全，一般以元组的格式
                cursor.execute(
                    sql, (address))

            # 提交本次插入的记录
            connection.commit()
        finally:
            # 关闭连接
            connection.close()

        return item


import copy, random, datetime
suppliesDataSets = [
    [
        {
            "labels": [], #food
            "data": []
        },
        {
            "labels": [], #drink
            "data": []
        },
        {
            "labels": [], #medicine
            "data": []
        }
    ]
]
now = datetime.datetime.now()
for i in range(3):
    for j in range(42):
        t_ = (now + datetime.timedelta(days=-j)).strftime('%Y-%m-%d')
        suppliesDataSets[0][i]["labels"].append(t_)
        k = random.randint(0, 200)
        suppliesDataSets[0][i]["data"].append(k)
for i in range(8):
    cloned_ = copy.deepcopy(suppliesDataSets[0])
    suppliesDataSets.append(cloned_)


from __future__ import print_function
from __future__ import absolute_import
import sys
import math
import heapq
import gzip
import warnings
from xml.sax import handler, parse
from copy import copy
from collections import defaultdict
from itertools import chain

import sumolib
from . import lane, edge, netshiftadaptor, node, connection, roundabout  # noqa
from .connection import Connection

class TLS:

    """Traffic Light Signal for a sumo network"""

    def __init__(self, id):
        self._id = id
        self._connections = []
        self._maxConnectionNo = -1
        self._programs = {}

    def addConnection(self, inLane, outLane, linkNo):
        self._connections.append([inLane, outLane, linkNo])
        if linkNo > self._maxConnectionNo:
            self._maxConnectionNo = linkNo

    def getConnections(self):
        return self._connections

    def getID(self):
        return self._id

    def getLinks(self):
        links = {}
        for the_connection in self._connections:
            if the_connection[2] not in links:
                links[the_connection[2]] = []
            links[the_connection[2]].append(the_connection)
        return links

    def getEdges(self):
        edges = set()
        for c in self._connections:
            edges.add(c[0].getEdge())
        return edges

    def addProgram(self, program):
        self._programs[program._id] = program

    def removePrograms(self):
        self._programs.clear()

    def toXML(self):
        ret = ""
        for p in self._programs:
            ret = ret + self._programs[p].toXML(self._id)
        return ret

    def getPrograms(self):
        return self._programs

class Phase:

    def __init__(self, duration, state, minDur=-1, maxDur=-1, next=None, name=""):
        self.duration = duration
        self.state = state
        self.minDur = minDur  # minimum duration (only for actuated tls)
        self.maxDur = maxDur  # maximum duration (only for actuated tls)
        self.next = [] if next is None else next
        self.name = name

    def __repr__(self):
        name = "" if self.name == "" else ", name='%s'" % self.name
        next = "" if len(self.next) == 0 else ", next='%s'" % self.next
        return ("Phase(duration=%s, state='%s', minDur=%s, maxDur=%s%s%s" %
                (self.duration, self.state, self.minDur, self.maxDur, name, next))

class TLSProgram:

    def __init__(self, id, offset, type):
        self._id = id
        self._type = type
        self._offset = offset
        self._phases = []
        self._params = {}

    def addPhase(self, state, duration, minDur=-1, maxDur=-1, next=None, name=""):
        self._phases.append(Phase(duration, state, minDur, maxDur, next, name))

    def toXML(self, tlsID):
        ret = '  <tlLogic id="%s" type="%s" programID="%s" offset="%s">\n' % (
            tlsID, self._type, self._id, self._offset)
        for p in self._phases:
            minDur = '' if p.minDur < 0 else ' minDur="%s"' % p.minDur
            maxDur = '' if p.maxDur < 0 else ' maxDur="%s"' % p.maxDur
            name = '' if p.name == '' else ' name="%s"' % p.name
            next = '' if len(p.next) == 0 else ' next="%s"' % ' '.join(map(str, p.next))
            ret += '    <phase duration="%s" state="%s"%s%s%s%s/>\n' % (
                p.duration, p.state, minDur, maxDur, name, next)
        for k, v in self._params.items():
            ret += '    <param key="%s" value="%s"/>\n' % (k, v)
        ret += '  </tlLogic>\n'
        return ret

    def getPhases(self):
        return self._phases

    def getType(self):
        return self._type

    def setParam(self, key, value):
        self._params[key] = value

    def getParam(self, key, default=None):
        return self._params.get(key, default)

    def getParams(self):
        return self._params

class Net:

    """The whole sumo network."""

    def __init__(self):
        self._location = {}
        self._id2node = {}
        self._id2edge = {}
        self._crossings_and_walkingAreas = set()
        self._macroConnectors = set()
        self._id2tls = {}
        self._nodes = []
        self._edges = []
        self._tlss = []
        self._ranges = [[10000, -10000], [10000, -10000]]
        self._roundabouts = []
        self._rtreeEdges = None
        self._rtreeLanes = None
        self._allLanes = []
        self._origIdx = None
        self._proj = None
        self.hasInternal = False
        # store dijsktra heap for reuse if the same origin is used repeatedly
        self._shortestPathCache = None

    def setLocation(self, netOffset, convBoundary, origBoundary, projParameter):
        self._location["netOffset"] = netOffset
        self._location["convBoundary"] = convBoundary
        self._location["origBoundary"] = origBoundary
        self._location["projParameter"] = projParameter

    def addNode(self, id, type=None, coord=None, incLanes=None, intLanes=None):
        if id is None:
            return None
        if id not in self._id2node:
            n = node.Node(id, type, coord, incLanes, intLanes)
            self._nodes.append(n)
            self._id2node[id] = n
        self.setAdditionalNodeInfo(
            self._id2node[id], type, coord, incLanes, intLanes)
        return self._id2node[id]

    def setAdditionalNodeInfo(self, node, type, coord, incLanes, intLanes=None):
        if coord is not None and node._coord is None:
            node._coord = coord
            self._ranges[0][0] = min(self._ranges[0][0], coord[0])
            self._ranges[0][1] = max(self._ranges[0][1], coord[0])
            self._ranges[1][0] = min(self._ranges[1][0], coord[1])
            self._ranges[1][1] = max(self._ranges[1][1], coord[1])
        if incLanes is not None and node._incLanes is None:
            node._incLanes = incLanes
        if intLanes is not None and node._intLanes is None:
            node._intLanes = intLanes
        if type is not None and node._type is None:
            node._type = type

    def addEdge(self, id, fromID, toID, prio, function, name, edgeType=''):
        if id not in self._id2edge:
            fromN = self.addNode(fromID)
            toN = self.addNode(toID)
            e = edge.Edge(id, fromN, toN, prio, function, name, edgeType)
            self._edges.append(e)
            self._id2edge[id] = e
            if function:
                self.hasInternal = True
        return self._id2edge[id]

    def addLane(self, edge, speed, length, width, allow=None, disallow=None):
        return lane.Lane(edge, speed, length, width, allow, disallow)

    def addRoundabout(self, nodes, edges=None):
        r = roundabout.Roundabout(nodes, edges)
        self._roundabouts.append(r)
        return r

    def addConnection(self, fromEdge, toEdge, fromlane, tolane, direction, tls, tllink, state, viaLaneID=None):
        conn = connection.Connection(
            fromEdge, toEdge, fromlane, tolane, direction, tls, tllink, state, viaLaneID)
        fromEdge.addOutgoing(conn)
        fromlane.addOutgoing(conn)
        toEdge._addIncoming(conn)
        if viaLaneID:
            try:
                # internal lanes are only available when building with option withInternal=True
                viaLane = self.getLane(viaLaneID)
                viaEdge = viaLane.getEdge()
                viaEdge._addIncoming(connection.Connection(
                    fromEdge, viaEdge, fromlane, viaLane, direction, tls,
                    tllink, state, ''))
            except Exception:
                pass
        return conn

    def getEdges(self, withInternal=True):
        if not withInternal:
            return [e for e in self._edges if e.getFunction() == '']
        else:
            return self._edges

    def getRoundabouts(self):
        return self._roundabouts

    def hasEdge(self, id):
        return id in self._id2edge

    def getEdge(self, id):
        return self._id2edge[id]

    def getLane(self, laneID):
        edge_id, lane_index = laneID.rsplit("_", 1)
        return self.getEdge(edge_id).getLane(int(lane_index))

    def _initRTree(self, shapeList, includeJunctions=True):
        import rtree  # noqa
        result = rtree.index.Index()
        result.interleaved = True
        for ri, shape in enumerate(shapeList):
            result.add(ri, shape.getBoundingBox(includeJunctions))
        return result

    # Please be aware that the resulting list of edges is NOT sorted
    def getNeighboringEdges(self, x, y, r=0.1, includeJunctions=True, allowFallback=True):
        edges = []
        try:
            if self._rtreeEdges is None:
                self._rtreeEdges = self._initRTree(self._edges, includeJunctions)
            for i in self._rtreeEdges.intersection((x - r, y - r, x + r, y + r)):
                e = self._edges[i]
                d = sumolib.geomhelper.distancePointToPolygon(
                    (x, y), e.getShape(includeJunctions))
                if d < r:
                    edges.append((e, d))
        except ImportError:
            if not allowFallback:
                raise
            warnings.warn("Module 'rtree' not available. Using brute-force fallback.")
            for the_edge in self._edges:
                d = sumolib.geomhelper.distancePointToPolygon((x, y), the_edge.getShape(includeJunctions))
                if d < r:
                    edges.append((the_edge, d))
        return edges

    def getNeighboringLanes(self, x, y, r=0.1, includeJunctions=True, allowFallback=True):
        lanes = []
        try:
            if self._rtreeLanes is None:
                for the_edge in self._edges:
                    self._allLanes += the_edge.getLanes()
                self._rtreeLanes = self._initRTree(self._allLanes, includeJunctions)
            for i in self._rtreeLanes.intersection((x - r, y - r, x + r, y + r)):
                the_lane = self._allLanes[i]
                d = sumolib.geomhelper.distancePointToPolygon((x, y), the_lane.getShape(includeJunctions))
                if d < r:
                    lanes.append((the_lane, d))
        except ImportError:
            if not allowFallback:
                raise
            warnings.warn("Module 'rtree' not available. Using brute-force fallback.")
            for the_edge in self._edges:
                for the_lane in the_edge.getLanes():
                    d = sumolib.geomhelper.distancePointToPolygon((x, y), the_lane.getShape(includeJunctions))
                    if d < r:
                        lanes.append((the_lane, d))
        return lanes

    def hasNode(self, id):
        return id in self._id2node

    def getNode(self, id):
        return self._id2node[id]

    def getNodes(self):
        return self._nodes

    def getTLS(self, tlid):
        return self._id2tls[tlid]

    def getTLSSecure(self, tlid):
        if tlid in self._id2tls:
            tls = self._id2tls[tlid]
        else:
            tls = TLS(tlid)
            self._id2tls[tlid] = tls
            self._tlss.append(tls)
        return tls

    def getTrafficLights(self):
        return self._tlss

    def addTLS(self, tlid, inLane, outLane, linkNo):
        tls = self.getTLSSecure(tlid)
        tls.addConnection(inLane, outLane, linkNo)
        return tls

    def addTLSProgram(self, tlid, programID, offset, type, removeOthers):
        tls = self.getTLSSecure(tlid)
        program = TLSProgram(programID, offset, type)
        if removeOthers:
            tls.removePrograms()
        tls.addProgram(program)
        return program

    def setFoes(self, junctionID, index, foes, prohibits):
        self._id2node[junctionID].setFoes(index, foes, prohibits)

    def forbids(self, possProhibitor, possProhibited):
        return possProhibitor.getFrom().getToNode().forbids(possProhibitor, possProhibited)

    def getDownstreamEdges(self, edge, distance, stopOnTLS, stopOnTurnaround):
        """return a list of lists of the form
           [[firstEdge, pos, [edge_0, edge_1, ..., edge_k], aborted], ...]
           where
             firstEdge: is the downstream edge furthest away from the intersection,
             [edge_0, ..., edge_k]: is the list of edges from the intersection downstream to firstEdge
             pos: is the position on firstEdge with distance to the end of the input edge
             aborted: a flag indicating whether the downstream
                 search stopped at a TLS or a node without incoming edges before reaching the distance threshold
        """
        ret = []
        seen = set()
        toProc = []
        toProc.append([edge, 0, []])
        while not len(toProc) == 0:
            ie = toProc.pop()
            if ie[0] in seen:
                continue
            seen.add(ie[0])
            if ie[1] + ie[0].getLength() >= distance:
                ret.append(
                    [ie[0], ie[0].getLength() + ie[1] - distance, ie[2], False])
                continue
            if len(ie[0]._incoming) == 0:
                ret.append([ie[0], ie[0].getLength() + ie[1], ie[2], True])
                continue
            mn = []
            stop = False
            for ci in ie[0]._incoming:
                if ci not in seen:
                    prev = copy(ie[2])
                    if stopOnTLS and ci._tls and ci != edge and not stop:
                        ret.append([ie[0], ie[1], prev, True])
                        stop = True
                    elif (stopOnTurnaround and ie[0]._incoming[ci][0].getDirection() == Connection.LINKDIR_TURN and
                          not stop):
                        ret.append([ie[0], ie[1], prev, True])
                        stop = True
                    else:
                        prev.append(ie[0])
                        mn.append([ci, ie[0].getLength() + ie[1], prev])
            if not stop:
                toProc.extend(mn)
        return ret

    def getEdgesByOrigID(self, origID):
        if self._origIdx is None:
            self._origIdx = defaultdict(set)
            for the_edge in self._edges:
                for the_lane in the_edge.getLanes():
                    for oID in the_lane.getParam("origId", "").split():
                        self._origIdx[oID].add(the_edge)
        return self._origIdx[origID]

    def getBBoxXY(self):
        """
        Get the bounding box (bottom left and top right coordinates) for a net;
        Coordinates are in X and Y (not Lat and Lon)

        :return [(bottom_left_X, bottom_left_Y), (top_right_X, top_right_Y)]
        """
        return [(self._ranges[0][0], self._ranges[1][0]),
                (self._ranges[0][1], self._ranges[1][1])]

    # the diagonal of the bounding box of all nodes
    def getBBoxDiameter(self):
        return math.sqrt(
            (self._ranges[0][0] - self._ranges[0][1]) ** 2 +
            (self._ranges[1][0] - self._ranges[1][1]) ** 2)

    def getGeoProj(self):
        if self._proj is None:
            import pyproj
            try:
                self._proj = pyproj.Proj(projparams=self._location["projParameter"])
            except RuntimeError:
                if hasattr(pyproj.datadir, 'set_data_dir'):
                    pyproj.datadir.set_data_dir('/usr/share/proj')
                    self._proj = pyproj.Proj(projparams=self._location["projParameter"])
                raise
        return self._proj

    def getLocationOffset(self):
        """ offset to be added after converting from geo-coordinates to UTM"""
        return list(map(float, self._location["netOffset"].split(",")))

    def getBoundary(self):
        """ return xmin,ymin,xmax,ymax network coordinates"""
        return list(map(float, self._location["convBoundary"].split(",")))

    def convertLonLat2XY(self, lon, lat, rawUTM=False):
        x, y = self.getGeoProj()(lon, lat)
        if rawUTM:
            return x, y
        else:
            x_off, y_off = self.getLocationOffset()
            return x + x_off, y + y_off

    def convertXY2LonLat(self, x, y, rawUTM=False):
        if not rawUTM:
            x_off, y_off = self.getLocationOffset()
            x -= x_off
            y -= y_off
        return self.getGeoProj()(x, y, inverse=True)

    def move(self, dx, dy, dz=0):
        for n in self._nodes:
            n._coord = (n._coord[0] + dx, n._coord[1] + dy, n._coord[2] + dz)
        for e in self._edges:
            for l in e._lanes:
                l._shape = [(p[0] + dx, p[1] + dy, p[2] + dz)
                            for p in l.getShape3D()]
            e.rebuildShape()

    def getInternalPath(self, conn, fastest=False):
        minInternalCost = 1e400
        minPath = None
        for c in conn:
            if c.getViaLaneID() != "":
                viaCost = 0
                viaID = c.getViaLaneID()
                viaPath = []
                while viaID != "":
                    viaLane = self.getLane(viaID)
                    viaCost += viaLane.getLength() if not fastest else viaLane.getLength() / viaLane.getSpeed()
                    viaID = viaLane.getOutgoing()[0].getViaLaneID()
                    viaPath.append(viaLane.getEdge())
                if viaCost < minInternalCost:
                    minInternalCost = viaCost
                    minPath = viaPath
        return minPath, minInternalCost

    def getOptimalPath(self, fromEdge, toEdge, fastest=False, maxCost=1e400, vClass=None, reversalPenalty=0,
                       includeFromToCost=True, withInternal=False, ignoreDirection=False,
                       fromPos=0, toPos=0):
        """
        Finds the optimal (shortest or fastest) path for vClass from fromEdge to toEdge
        by using using Dijkstra's algorithm.
        It returns a pair of a tuple of edges and the cost.
        If no path is found the first element is None.
        The cost for the returned path is equal to the sum of all edge costs in the path,
        including the internal connectors, if they are present in the network.
        The path itself does not include internal edges except for the case
        when the start or end edge are internal edges.
        The search may be limited using the given threshold.
        """

        def speedFunc(edge):
            return edge.getSpeed() if fastest else 1.0

        if self.hasInternal:
            appendix = ()
            appendixCost = 0.
            while toEdge.getFunction() == "internal":
                appendix = (toEdge,) + appendix
                appendixCost += toEdge.getLength() / speedFunc(toEdge)
                toEdge = list(toEdge.getIncoming().keys())[0]
        fromCost = fromEdge.getLength() / speedFunc(fromEdge) if includeFromToCost else 0
        q = [(fromCost, fromEdge.getID(), (fromEdge, ), ())]
        if fromEdge == toEdge and fromPos > toPos and not ignoreDirection:
            # start search on successors of fromEdge
            q = []
            startCost = (fromEdge.getLength() - fromPos) / speedFunc(fromEdge) if includeFromToCost else 0
            for e2, conn in fromEdge.getAllowedOutgoing(vClass).items():
                q.append((startCost + e2.getLength() / speedFunc(e2), e2.getID(), (fromEdge, e2), ()))

        seen = set()
        dist = {fromEdge: fromEdge.getLength() / speedFunc(fromEdge)}
        while q:
            cost, _, e1via, path = heapq.heappop(q)
            e1 = e1via[-1]
            if e1 in seen:
                continue
            seen.add(e1)
            path += e1via
            if e1 == toEdge:
                if self.hasInternal:
                    return path + appendix, cost + appendixCost
                if includeFromToCost and toPos == 0:
                    # assume toPos=0 is the default value
                    return path, cost
                return path, cost + (-toEdge.getLength() + toPos) / speedFunc(toEdge)
            if cost > maxCost:
                return None, cost

            for e2, conn in chain(e1.getAllowedOutgoing(vClass).items(),
                                  e1.getIncoming().items() if ignoreDirection else []):
                # print(cost, e1.getID(), e2.getID(), e2 in seen)
                if e2 not in seen:
                    newCost = cost + e2.getLength() / speedFunc(e2)
                    if e2 == e1.getBidi():
                        newCost += reversalPenalty
                    minPath = (e2,)
                    if self.hasInternal:
                        viaPath, minInternalCost = self.getInternalPath(conn, fastest=fastest)
                        if viaPath is not None:
                            newCost += minInternalCost
                            if withInternal:
                                minPath = tuple(viaPath + [e2])
                    if e2 not in dist or newCost < dist[e2]:
                        dist[e2] = newCost
                        heapq.heappush(q, (newCost, e2.getID(), minPath, path))
        return None, 1e400

    def getShortestPath(self, fromEdge, toEdge, maxCost=1e400, vClass=None, reversalPenalty=0,
                        includeFromToCost=True, withInternal=False, ignoreDirection=False,
                        fromPos=0, toPos=0):
        """
        Finds the shortest path from fromEdge to toEdge respecting vClass, using Dijkstra's algorithm.
        It returns a pair of a tuple of edges and the cost. If no path is found the first element is None.
        The cost for the returned path is equal to the sum of all edge lengths in the path,
        including the internal connectors, if they are present in the network.
        The path itself does not include internal edges except for the case
        when the start or end edge are internal edges.
        The search may be limited using the given threshold.
        """

        return self.getOptimalPath(fromEdge, toEdge, False, maxCost, vClass, reversalPenalty,
                                   includeFromToCost, withInternal, ignoreDirection, fromPos, toPos)

    def getFastestPath(self, fromEdge, toEdge, maxCost=1e400, vClass=None, reversalPenalty=0,
                       includeFromToCost=True, withInternal=False, ignoreDirection=False,
                       fromPos=0, toPos=0):
        """
        Finds the fastest path from fromEdge to toEdge respecting vClass, using Dijkstra's algorithm.
        It returns a pair of a tuple of edges and the cost. If no path is found the first element is None.
        The cost for the returned path is equal to the sum of all edge costs in the path,
        including the internal connectors, if they are present in the network.
        The path itself does not include internal edges except for the case
        when the start or end edge are internal edges.
        The search may be limited using the given threshold.
        """

        return self.getOptimalPath(fromEdge, toEdge, True, maxCost, vClass, reversalPenalty,
                                   includeFromToCost, withInternal, ignoreDirection, fromPos, toPos)

class NetReader(handler.ContentHandler):

    """Reads a network, storing the edge geometries, lane numbers and max. speeds"""

    def __init__(self, **others):
        self._net = others.get('net', Net())
        self._currentEdge = None
        self._currentNode = None
        self._currentConnection = None
        self._currentLane = None
        self._crossingID2edgeIDs = {}
        self._withPhases = others.get('withPrograms', False)
        self._latestProgram = others.get('withLatestPrograms', False)
        if self._latestProgram:
            self._withPhases = True
        self._withConnections = others.get('withConnections', True)
        self._withFoes = others.get('withFoes', True)
        self._withPedestrianConnections = others.get('withPedestrianConnections', False)
        self._withMacroConnectors = others.get('withMacroConnectors', False)
        self._withInternal = others.get('withInternal', self._withPedestrianConnections)
        if self._withPedestrianConnections and not self._withInternal:
            sys.stderr.write("Warning: Option withPedestrianConnections requires withInternal\n")
            self._withInternal = True
        self._bidiEdgeIDs = {}

    def startElement(self, name, attrs):
        if name == 'location':
            self._net.setLocation(attrs["netOffset"], attrs["convBoundary"], attrs[
                                  "origBoundary"], attrs["projParameter"])
        if name == 'edge':
            function = attrs.get('function', '')
            if (function == ''
                    or (self._withInternal and function in ['internal', 'crossing', 'walkingarea'])
                    or (self._withMacroConnectors and function == 'connector')):
                prio = -1
                if 'priority' in attrs:
                    prio = int(attrs['priority'])

                # get the  ids
                edgeID = attrs['id']
                fromNodeID = attrs.get('from', None)
                toNodeID = attrs.get('to', None)

                # for internal junctions use the junction's id for from and to node
                if function == 'internal':
                    fromNodeID = toNodeID = edgeID[1:edgeID.rfind('_')]

                # remember edges crossed by pedestrians to link them later to the crossing objects
                if function == 'crossing':
                    self._crossingID2edgeIDs[edgeID] = attrs.get('crossingEdges').split(' ')

                self._currentEdge = self._net.addEdge(edgeID, fromNodeID, toNodeID, prio, function,
                                                      attrs.get('name', ''), attrs.get('type', ''))

                self._currentEdge.setRawShape(convertShape(attrs.get('shape', '')))

                bidi = attrs.get('bidi', '')
                if bidi:
                    self._bidiEdgeIDs[edgeID] = bidi
            else:
                if function in ['crossing', 'walkingarea']:
                    self._net._crossings_and_walkingAreas.add(attrs['id'])
                elif function == 'connector':
                    self._net._macroConnectors.add(attrs['id'])
                self._currentEdge = None
        if name == 'lane' and self._currentEdge is not None:
            self._currentLane = self._net.addLane(
                self._currentEdge,
                float(attrs['speed']),
                float(attrs['length']),
                float(attrs.get('width', 3.2)),
                attrs.get('allow'),
                attrs.get('disallow'))
            self._currentLane.setShape(convertShape(attrs.get('shape', '')))
        if name == 'neigh' and self._currentLane is not None:
            self._currentLane.setNeigh(attrs['lane'])
        if name == 'junction':
            if attrs['id'][0] != ':':
                intLanes = None
                if self._withInternal:
                    intLanes = attrs["intLanes"].split(" ")
                self._currentNode = self._net.addNode(attrs['id'], attrs['type'],
                                                      tuple(
                                                          map(float, [attrs['x'], attrs['y'],
                                                                      attrs['z'] if 'z' in attrs else '0'])),
                                                      attrs['incLanes'].split(" "), intLanes)
                self._currentNode.setShape(
                    convertShape(attrs.get('shape', '')))
                if 'fringe' in attrs:
                    self._currentNode._fringe = attrs['fringe']

        if name == 'succ' and self._withConnections:  # deprecated
            if attrs['edge'][0] != ':':
                self._currentEdge = self._net.getEdge(attrs['edge'])
                self._currentLane = attrs['lane']
                self._currentLane = int(
                    self._currentLane[self._currentLane.rfind('_') + 1:])
            else:
                self._currentEdge = None
        if name == 'succlane' and self._withConnections:  # deprecated
            lid = attrs['lane']
            if lid[0] != ':' and lid != "SUMO_NO_DESTINATION" and self._currentEdge:
                connected = self._net.getEdge(lid[:lid.rfind('_')])
                tolane = int(lid[lid.rfind('_') + 1:])
                if 'tl' in attrs and attrs['tl'] != "":
                    tl = attrs['tl']
                    tllink = int(attrs['linkIdx'])
                    tlid = attrs['tl']
                    toEdge = self._net.getEdge(lid[:lid.rfind('_')])
                    tolane2 = toEdge._lanes[tolane]
                    tls = self._net.addTLS(
                        tlid, self._currentEdge._lanes[self._currentLane], tolane2, tllink)
                    self._currentEdge.setTLS(tls)
                else:
                    tl = ""
                    tllink = -1
                toEdge = self._net.getEdge(lid[:lid.rfind('_')])
                tolane = toEdge._lanes[tolane]
                viaLaneID = attrs['via']
                self._net.addConnection(self._currentEdge, connected, self._currentEdge._lanes[
                                        self._currentLane], tolane,
                                        attrs['dir'], tl, tllink, attrs['state'], viaLaneID)
        if name == 'connection' and self._withConnections and (attrs['from'][0] != ":" or self._withInternal):
            fromEdgeID = attrs['from']
            toEdgeID = attrs['to']
            if ((self._withPedestrianConnections or not (fromEdgeID in self._net._crossings_and_walkingAreas or
                                                         toEdgeID in self._net._crossings_and_walkingAreas))
                and (self._withMacroConnectors or not (fromEdgeID in self._net._macroConnectors or toEdgeID in
                                                       self._net._macroConnectors))):
                fromEdge = self._net.getEdge(fromEdgeID)
                toEdge = self._net.getEdge(toEdgeID)
                fromLane = fromEdge.getLane(int(attrs['fromLane']))
                toLane = toEdge.getLane(int(attrs['toLane']))
                if 'tl' in attrs and attrs['tl'] != "":
                    tl = attrs['tl']
                    tllink = int(attrs['linkIndex'])
                    tls = self._net.addTLS(tl, fromLane, toLane, tllink)
                    fromEdge.setTLS(tls)
                else:
                    tl = ""
                    tllink = -1
                try:
                    viaLaneID = attrs['via']
                except KeyError:
                    viaLaneID = ''

                self._currentConnection = self._net.addConnection(
                    fromEdge, toEdge, fromLane, toLane, attrs['dir'], tl,
                    tllink, attrs['state'], viaLaneID)

        # 'row-logic' is deprecated!!!
        if self._withFoes and name == 'ROWLogic':
            self._currentNode = attrs['id']
        if name == 'logicitem' and self._withFoes:  # deprecated
            self._net.setFoes(
                self._currentNode, int(attrs['request']), attrs["foes"], attrs["response"])
        if name == 'request' and self._withFoes:
            self._currentNode.setFoes(
                int(attrs['index']), attrs["foes"], attrs["response"])
        # tl-logic is deprecated!!! NOTE: nevertheless, this is still used by
        # netconvert... (Leo)
        if self._withPhases and name == 'tlLogic':
            self._currentProgram = self._net.addTLSProgram(
                attrs['id'], attrs['programID'], float(attrs['offset']), attrs['type'], self._latestProgram)
        if self._withPhases and name == 'phase':
            self._currentProgram.addPhase(
                attrs['state'], int(attrs['duration']),
                int(attrs['minDur']) if 'minDur' in attrs else -1,
                int(attrs['maxDur']) if 'maxDur' in attrs else -1,
                list(map(int, attrs['next'].split())) if 'next' in attrs else [],
                attrs['name'] if 'name' in attrs else ""
            )
        if name == 'roundabout':
            self._net.addRoundabout(
                attrs['nodes'].split(), attrs['edges'].split())
        if name == 'param':
            if self._currentLane is not None:
                self._currentLane.setParam(attrs['key'], attrs['value'])
            elif self._currentEdge is not None:
                self._currentEdge.setParam(attrs['key'], attrs['value'])
            elif self._currentNode is not None:
                self._currentNode.setParam(attrs['key'], attrs['value'])
            elif self._currentConnection is not None:
                self._currentConnection.setParam(attrs['key'], attrs['value'])
            elif self._withPhases and self._currentProgram is not None:
                self._currentProgram.setParam(attrs['key'], attrs['value'])

    def endElement(self, name):
        if name == 'lane':
            self._currentLane = None
        if name == 'edge':
            self._currentEdge = None
        if name == 'junction':
            self._currentNode = None
        if name == 'connection':
            self._currentConnection = None
        # 'row-logic' is deprecated!!!
        if name == 'ROWLogic' or name == 'row-logic':
            self._haveROWLogic = False
        # tl-logic is deprecated!!!
        if self._withPhases and (name == 'tlLogic' or name == 'tl-logic'):
            self._currentProgram = None
        if name == 'net':
            for edgeID, bidiID in self._bidiEdgeIDs.items():
                self._net.getEdge(edgeID)._bidi = self._net.getEdge(bidiID)

    def endDocument(self):
        # set crossed edges of pedestrian crossings
        for crossingID, crossedEdgeIDs in self._crossingID2edgeIDs.items():
            pedCrossing = self._net.getEdge(crossingID)
            for crossedEdgeID in crossedEdgeIDs:
                pedCrossing._addCrossingEdge(self._net.getEdge(crossedEdgeID))

    def getNet(self):
        return self._net

def convertShape(shapeString):
    """ Convert xml shape string into float tuples.

    This method converts the 2d or 3d shape string from SUMO's xml file
    into a list containing 3d float-tuples. Non existant z coordinates default
    to zero. If shapeString is empty, an empty list will be returned.
    """

    cshape = []
    for pointString in shapeString.split():
        p = [float(e) for e in pointString.split(",")]
        if len(p) == 2:
            cshape.append((p[0], p[1], 0.))
        elif len(p) == 3:
            cshape.append(tuple(p))
        else:
            raise ValueError(
                'Invalid shape point "%s", should be either 2d or 3d' % pointString)
    return cshape

def readNet(filename, **others):
    """ load a .net.xml file
    The following named options are supported:

        'net' : initialize data structurs with an existing net object (default Net())
        'withPrograms' : import all traffic light programs (default False)
        'withLatestPrograms' : import only the last program for each traffic light.
                               This is the program that would be active in sumo by default.
                               (default False)
        'withConnections' : import all connections (default True)
        'withFoes' : import right-of-way information (default True)
        'withInternal' : import internal edges and lanes (default False)
        'withPedestrianConnections' : import connections between sidewalks, crossings (default False)
    """
    netreader = NetReader(**others)
    try:
        parse(gzip.open(filename), netreader)
    except IOError:
        parse(filename, netreader)
    return netreader.getNet()


data = []
count = 0
with open ('reviews.txt', 'r') as f:
    for line in f:
        data.append(line)
        count += 1
        if count % 1000 == 0:
            print(len(data))
print('Files finished reading, we have total', len(data), 'reviews')

sum_len = 0
for d in data:
    sum_len = sum_len + len(d)

print('The average number of the reviews', sum_len/len(data))