import argparse

import numpy as np

import ratings_adapter
import ratings_provider

# Compute the Euclidean distance score between user1 and user2
def euclidean_score(dataset, user1, user2):
    if user1 not in dataset:
        raise TypeError('Cannot find ' + user1 + ' in the dataset')

    if user2 not in dataset:
        raise TypeError('Cannot find ' + user2 + ' in the dataset')

    # Movies rated by both user1 and user2
    common_movies = {}

    for item in dataset[user1]:
        if item in dataset[user2]:
            common_movies[item] = 1

    # If there are no common movies between the users,
    # then the score is 0
    if len(common_movies) == 0:
        return 0

    squared_diff = []

    for item in dataset[user1]:
        if item in dataset[user2]:
            squared_diff.append(np.square(dataset[user1][item] - dataset[user2][item]))

    return 1 / (1 + np.sqrt(np.sum(squared_diff)))

# Compute the Pearson correlation score between user1 and user2
def pearson_score(dataset, user1, user2):
    if user1 not in dataset:
        raise TypeError('Cannot find ' + user1 + ' in the dataset')

    if user2 not in dataset:
        raise TypeError('Cannot find ' + user2 + ' in the dataset')

    # Movies rated by both user1 and user2
    common_movies = {}

    for item in dataset[user1]:
        if item in dataset[user2]:
            common_movies[item] = 1

    num_ratings = len(common_movies)

    # If there are no common movies between user1 and user2, then the score is 0
    if num_ratings == 0:
        return 0

    # Calculate the sum of ratings of all the common movies
    user1_sum = np.sum([dataset[user1][item] for item in common_movies])
    user2_sum = np.sum([dataset[user2][item] for item in common_movies])

    # Calculate the sum of squares of ratings of all the common movies
    user1_squared_sum = np.sum([np.square(dataset[user1][item]) for item in common_movies])
    user2_squared_sum = np.sum([np.square(dataset[user2][item]) for item in common_movies])

    # Calculate the sum of products of the ratings of the common movies
    sum_of_products = np.sum([dataset[user1][item] * dataset[user2][item] for item in common_movies])

    # Calculate the Pearson correlation score
    Sxy = sum_of_products - (user1_sum * user2_sum / num_ratings)
    Sxx = user1_squared_sum - np.square(user1_sum) / num_ratings
    Syy = user2_squared_sum - np.square(user2_sum) / num_ratings

    if Sxx * Syy == 0:
        return 0

    return Sxy / np.sqrt(Sxx * Syy)

# Finds users in the dataset that are similar to the input user
def find_similar_users(dataset, user, num_users):
    if user not in dataset:
        raise TypeError('Cannot find ' + user + ' in the dataset')

    # Compute Pearson score between input user
    # and all the users in the dataset
    scores = np.array([[x, pearson_score(dataset, user,
                                         x)] for x in dataset if x != user])

    # Sort the scores in decreasing order
    scores_sorted = np.argsort(scores[:, 1])[::-1]

    # Extract the top 'num_users' scores
    top_users = scores_sorted[:num_users]

    return scores[top_users]

def get_user_content_from_data(user, data):
    if isinstance(user, str):
        return data[user]
    elif isinstance(user, np.ndarray):
        return data[user[0]]

def create_pseudo_user(similar_users, data, from_worst=False):
    # revert so if a movie is repeated, the score given by more similar user is kept
    similar_users = similar_users[::-1]

    pseudo_user = {}
    for u in similar_users:
        pseudo_user.update(get_user_content_from_data(u, data))

    ret = sorted((value,key) for (key,value) in pseudo_user.items())
    if not from_worst:
        ret.reverse()
    return ret

def was_watched(user, data, movie):
    watched = get_user_content_from_data(user, data)
    if movie in watched:
        return True
    return False

def get_new_films(user, pseudo_user, data, amount):
    movies = []
    for movie in pseudo_user:
        if not was_watched(user, data, movie):
            if len(movies) < amount:
                movies.append(movie)
            else:
                break

    return movies

def find_best_unwatched_movies(user, similar_users, data, amount):
    pseudo_user = create_pseudo_user(similar_users, data)
    return get_new_films(user, pseudo_user, data, amount)

def find_worst_unwatched_movies(user, similar_users, data, amount):
    pseudo_user = create_pseudo_user(similar_users, data, from_worst=True)
    return get_new_films(user, pseudo_user, data, amount)

def build_arg_parser():
    parser = argparse.ArgumentParser(description='Find users who are similar to the input user')
    parser.add_argument('--user', dest='user', required=True,
                        help='Input user')
    return parser

if __name__ == '__main__':
    args = build_arg_parser().parse_args()
    user = args.user

    sheet_data = ratings_provider.get_ratings()
    data = ratings_adapter.process_ratings(sheet_data)

    similar_users = find_similar_users(data, user, 3)
    best = find_best_unwatched_movies(user, similar_users, data, 10)
    worst = find_worst_unwatched_movies(user, similar_users, data, 10)

    print('Recommended movies:\n')
    for movie in best:
        print(movie[1])
    print('\n')

    print('Best-to-avoid movies:\n')
    for movie in worst:
        print(movie[1])


import re
import pprint as pp
import Stemmer
import psutil
import time
import psutil
import indexer as Indexer
import document
import sys
import gc
import os
import merge
import random
import calculations

# Document id
docid = 0
# Collection of Documents, for future purposes
#collection = []						#Collection of existing documents
indexer = Indexer.Indexer()			#Indexer of tokenizer

indexBlock = 0
# Begin the timer.
start = time.time()

def memory_usage_psutil():
    # return the memory usage in percentage like top
    process = psutil.Process(os.getpid())
    mem = process.memory_percent()
    return mem

#Now loads all files from input folder
def getFiles(path):
	files = os.listdir(path)
	output = []
	for file in files:
		#print(file.split("-"))
		if os.path.isdir(file):
			print(file)
			continue
		if path[-1] == "/":

			output += [path+file]
		else:
			output += [path+"/"+file]
	return output

print(sys.argv)
# Paramenter filename
if len(sys.argv) < 4:
	print("Error: no parameters found, usage: python r1_1.py [tokenizer=(1 | 2)] [number (0-1 activate prob of block creation after each doc)| (>=1 activates fixed size)] [folder of documents]")
	exit(1)
inputFiles = getFiles(sys.argv[-1])
limite = float(sys.argv[2])
if limite <= 0:
	print("Number needs to be greater than 0")
	exit(1)
for i in inputFiles:
	with open(i,"r",errors="ignore") as f:
		indexer.setPath("result/")
		text = ""
		pmid = 0
		for line in f:
			#Saving PMID for future use, I am not 100% certain
			if "PMID-" in line:
				pmid = line.split(" ")[1]
			#Extracting the titles of each document
			if "TI  -" in line:
				text += (line.strip().split("-")[1]) if len(line.strip().split("-")) > 1 else ""
			#Assuming that the document structure is the same for all documents, each title ends when "PG -""  appears
			if "PG  -" in line:
				temp = document.Document(docid,pmid,text)
				#Token the title using both tokenizers
				# Adding the tokens to the indexeres,
				if sys.argv[1] == "1":
					temp.tokenize1()
					for token in temp.getTokens1():
						indexer.addIndex(token,temp.getDocID(),temp.getTokV1()[token])
				elif sys.argv[1] == "2":
					temp.tokenize2()
					for token in temp.getTokens2():
						indexer.addIndex(token,temp.getDocID(),temp.getTokV2()[token])

				del temp

				#Saving the document for future purposes
				#collection+=[temp]
				#Next Document id, trivial
				docid+=1
				text = ""
				#print(memory_usage_psutil())
				if memory_usage_psutil() > limite:
					#print("Before "+str(memory_usage_psutil()))
					indexer.orderTokens()
					indexer.writeIndexes(indexBlock)
					#print(indexBlock)
					indexBlock += 1
					del indexer
					gc.collect()

					indexer = Indexer.Indexer()			#Indexer of tokenizer
					indexer.setPath("result/")
					#print("After "+str(memory_usage_psutil()))

			#There are scenarios where are titles with \n
			elif "TI" not in line and len(text) > 0:		#We are concatenating text
				text += " "+line.strip()

indexer.orderTokens()
indexer.writeIndexes(indexBlock)
indexer.clear()

del indexer
gc.collect()

print("--- %s seconds ---" % (time.time() - start))
print("Indexing finished")

merge.merge("merge/merged.txt")
print("--- %s seconds ---" % (time.time() - start))
print("Merge finished")

calculations.calculateIndexes(docid,"merge/merged.txt",limite)
print("--- %s seconds ---" % (time.time() - start))
print("Calculations finished")

'''
TODO LIST:
'''


# Created by: Storm Shadow http://www.techbliss.org
import os, sys
import ida_idaapi, ida_kernwin, ida_idaapi, ida_loader
import idc
from idc import *
from ida_diskio import *
from ida_idaapi import *
from ida_loader import *
import sys
sys.path.insert(0 , ida_diskio.idadir("plugins\\Code editor\\icons"))
import ico
from ico import *

PLUGIN_VERSION = "1.6"
IDAVERISONS    = "IDA PRO 7.2+"
AUTHORS        = "Storm Shadow"
DATE           = "2019"
TWITTER        = "Twitter @zadow28"

def banner():
    banner_options = (PLUGIN_VERSION, AUTHORS, DATE, TWITTER, IDAVERISONS)
    banner_titles = "Python Editor v%s - (c) %s - %s - %s - %s" % banner_options

# print plugin banner
    print("---[" + banner_titles + "]---\n")

banner()

# 1) Create the handler class
class MyEditorHandler(ida_kernwin.action_handler_t):
    def __init__(self):
        ida_kernwin.action_handler_t.__init__(self)

    # Run editor when invoked.
    def activate(self, ctx):
        g = globals()
        idahome = ida_diskio.idadir("plugins\\Code editor")
        tupac = str(idahome + "\\pyeditor.py")
        ida_kernwin.load_custom_icon(":/ico/python.png")
        load_and_run_plugin(tupac, True)
    def update(self, ctx):
        return ida_kernwin.AST_ENABLE_ALWAYS

class ripeye(ida_idaapi.plugin_t):
    flags = ida_idaapi.PLUGIN_FIX
    comment = "Run me"
    help = "Python Editor"
    wanted_name = "Python Editor"
    wanted_hotkey = "" #the tooltip horkey goes away when setting it here DONT DO it! and only is shown in File/Plugins menu

    def editor_menuaction(self):
        action_desc = ida_kernwin.action_desc_t(
            'my:editoraction',  # The action name. This acts like an ID and must be unique
            'Python Editor!',  # The action text.
            MyEditorHandler(),  # The action handler.
            'Ctrl+H',  # Optional: the action shortcut DO IT  HERE!
            'Script editor',  # Optional: the action tooltip (available in menus/toolbar)
            ida_kernwin.load_custom_icon(":/ico/python.png")  # hackish load action icon , if no custom icon use number from 1-150 from internal ida
        )

        # 3) Register the action
        ida_kernwin.register_action(action_desc)

        ida_kernwin.attach_action_to_menu(
            'Edit/Editor...',  # The relative path of where to add the action
            'my:editoraction',  # The action ID (see above)
            ida_kernwin.SETMENU_APP)  # We want to append the action after the 'Manual instruction...

        ida_kernwin.get_current_widget()
        ida_kernwin.attach_action_to_popup(form, None, "my:editoraction", None)

    def init(self):
        """
        This is called by IDA when it is loading the plugin.
        """
        #self._icon_id_file = ida_diskio.BADADDR
        # attempt plugin initialization
        try:
            self._install_plugin()

        # failed to initialize or integrate the plugin, log and skip loading
        except Exception as e:
            form = ida_kernwin.get_current_widget()
            pass

        return PLUGIN_KEEP

    def _install_plugin(self):
        """
        Initialize & integrate the plugin into IDA.
        """
        self.editor_menuaction()
        self._init()

    def term(self):
        pass

    def run(self, arg = 0):
        #we need the calls again if we wanna load it via File/Plugins/editor
        ida_kernwin.msg("Python Editor Loaded to menu \n use Alt+E hot key to quick load ")
        hackish = MyEditorHandler()
        hackish.activate(self)

def PLUGIN_ENTRY():
    return ripeye()

7.5/Add to plugins folder/Python_editor.py


def square_and_multiply(b, e, m):
    """
    Returns the reduction of b raised to e
    modulo m using the square and multiply algorithm.

    Args:
        b: the base of the number to be reduced
        e: the exponent of the number to be reduced
        m: modulus which to reduce b raised to e

    Returns:
        the reduction of n modulo m

    Requires:
        e,m > 0
    """
    squared_reductions = _squaring(b, e, m)
    return _multiplying(squared_reductions, m)

def _squaring(b, e, m):
    binary_repr = "{0:b}".format(e)
    reductions = [b % m]
    iterations = len(binary_repr)

    for i in range(1, iterations):
        prev = reductions[i-1]
        reductions.append((prev**2) % m)

    required_reductions = []
    num_reductions = len(reductions)

    for j in range(0, num_reductions):
        if binary_repr[j] == '1':
            required_reductions.append(reductions[num_reductions-j-1])

    return required_reductions

def _multiplying(reductions, m):
    n = len(reductions)
    assert(n > 0)
    out = reductions[0]
    for i in range(1, n):
        out = (out * reductions[i]) % m
    return out


from os.path import join, dirname
from dotenv import load_dotenv
import os
import discord
from utils.config import PREFIX, HOST_CHANNEL, ROLES
from utils.roles import add_role, reaction_to_role, remove_role, remove_all_roles
from utils.emojis import get_emoji_from_reaction, is_clearing_emoji, is_listed_emoji
import commands

dotenv_path = join(dirname(__file__), '.env')
load_dotenv(dotenv_path)

TOKEN = os.environ.get('TOKEN')

bot = discord.Client();

@bot.event
async def on_ready():
    print("Logged in as")
    print(bot.user.name)
    print(bot.user.id)
    print("------")

@bot.event
async def on_message(message):
    if message.author.bot:
        return

    if message.content.startswith(PREFIX) \
            and message.channel.name == HOST_CHANNEL:
        command = message.content[1:]
        if command == 'help':
            await commands.help_cmd(bot, message)
        elif command == 'init':
            await commands.init(bot, message)
    else:
        return

@bot.event
async def on_reaction_add(reaction, user):
    if user == bot.user:
        return

    channel_reacted_in = reaction.message.channel.name
    if channel_reacted_in != HOST_CHANNEL:
        return

    emoji = get_emoji_from_reaction(reaction)

    # if a clearing emoji was clicked, remove all roles
    if is_clearing_emoji(emoji):
        await remove_all_roles(bot, user)

    # if emoji was not already listed, remove
    elif not is_listed_emoji(emoji) and channel_reacted_in == HOST_CHANNEL:
        await bot.remove_reaction(reaction.message, reaction.emoji, user)

    else:
        role = reaction_to_role(reaction)
        await add_role(bot, user, role)

@bot.event
async def on_reaction_remove(reaction, user):
    if user == bot.user:
        return
    role = reaction_to_role(reaction)
    await remove_role(bot, user, role)

bot.run(TOKEN)


from src import app
import mysql.connector

def db_get_periods():
    mycursor = app.config["DATABASE"].cursor()
    try:
        sql = "SELECT * FROM period"
        mycursor.execute(sql)
        periods = mycursor.fetchall()
        return periods
    except mysql.connector.Error as err:
        print("Error: {}".format(err.msg))
    finally:
        mycursor.close()

def db_get_times(period):
    mycursor=app.config["DATABASE"].cursor()
    try:
        sql = "SELECT timeid,TIME_FORMAT(time,'%H:%i') FROM time_period WHERE period=%s"
        mycursor.execute(sql, (str(period),))
        times = mycursor.fetchall()
        return times
    except mysql.connector.Error as err:
        print("Error: {}".format(err.msg))
    finally:
        mycursor.close()

def db_get_unavailable_tables(rid, time, date):
    timeid = db_get_timeid(time)
    mycursor = app.config["DATABASE"].cursor()
    try:
        sql = "SELECT tid FROM rest_book WHERE rid=%s AND timeid=%s AND date=%s"
        mycursor.execute(sql, (rid, timeid, date))
        unvTables = mycursor.fetchall()
        return unvTables
    except mysql.connector.Error as err:
        print("Error: {}".format(err.msg))
    finally:
        mycursor.close()

def db_get_timeid(time):
    mycursor = app.config["DATABASE"].cursor()
    try:
        sql = "SELECT timeid FROM time_period WHERE time=%s"
        mycursor.execute(sql, (time,))
        timeid = mycursor.fetchall()
        return timeid[0][0]
    except mysql.connector.Error as err:
        print("Error: {}".format(err.msg))
    finally:
        mycursor.close()

def db_get_unavailable_tables(rid, time, date):
    timeid = db_get_timeid(time)
    mycursor = app.config["DATABASE"].cursor()
    try:
        sql = "SELECT tid FROM rest_book WHERE rid=%s AND timeid=%s AND date=%s"
        mycursor.execute(sql, (rid, timeid, date))
        unvTables = mycursor.fetchall()
    except mysql.connector.Error as err:
        print("Error: {}".format(err.msg))
    finally:
        mycursor.close()
    return unvTables

def db_insert_booking(rid, tid, date, timeid, cid,people, add_info):
    mycursor = app.config["DATABASE"].cursor()
    try:
        sql = "INSERT INTO booking_info (cid,additional_info) VALUES (%s, %s)"
        mycursor.execute(sql, (str(cid), add_info))

        bid = db_get_bid(cid)

        sql = "INSERT INTO rest_book VALUES (%s, %s, %s, %s, %s,%s)"
        mycursor.execute(sql, (str(rid), str(bid), str(tid), str(date), str(timeid),str(people)))

        app.config["DATABASE"].commit()
    except mysql.connector.Error as err:
        print("Error: {}".format(err.msg))
    finally:
        mycursor.close()

def db_get_bid(cid):
    mycursor = app.config["DATABASE"].cursor()
    try:
        sql = "SELECT bid FROM booking_info WHERE cid=%s;"
        mycursor.execute(sql, (str(cid),))
        bid = mycursor.fetchall()
        return bid[len(bid) - 1][0] # the last inserted booking for that customer
    except mysql.connector.Error as err:
        print("Error: {}".format(err.msg))
    finally:
        mycursor.close()

def db_get_new_cid():
    mycursor = app.config['DATABASE'].cursor()

    try:
        sql = "SELECT cid FROM booking_info ORDER BY cid DESC LIMIT 1;"
        mycursor.execute(sql)
        last_cid = mycursor.fetchall()
        try:
            # if the table is not empty we return id + 1, otherwise 0
            return last_cid[0][0] + 1
        except:
            return 0
    except mysql.connector.Error as err:
        print("Error: {}".format(err.msg))
    finally:
        mycursor.close()

def db_insert_full_day(): # Default fullDay
    mycursor = app.config['DATABASE'].cursor()
    try:
        mycursor.execute(
            "INSERT INTO booking_info VALUES(30,1,'01'),(31,2,'02'),(32,3,'03'),(33,3,'04'),(34,4,'05'),(50,5,'05'),(51,6,'05'),(52,7,'05'),(53,8,'05'),(54,5,'05'),(55,5,'05'),(56,5,'05'),(57,5,'05'),(58,5,'05'),(59,5,'05'),(60,5,'05'),(61,5,'05'),(62,5,'05')")
        app.config["DATABASE"].commit()
        mycursor.execute(
            "INSERT INTO rest_book VALUES(1,30,'01','2018-12-01',1,2),(1,31,'02','2018-12-01',1,2),(1,32,'03','2018-12-01',1,3),(1,33,'04','2018-12-01',1,3),(1,34,'05','2018-12-01',1,3),(1,50,'05','2018-12-02',1,6),(1,51,'05','2018-12-02',1,4),(1,52,'05','2018-12-02',1,4),(1,53,'05','2018-12-02',1,6),(1,54,'05','2018-12-02',1,3),(1,55,'05','2018-12-02',1,3),(1,56,'05','2018-12-02',1,3),(1,57,'05','2018-12-03',1,8),(1,58,'05','2018-12-03',1,8),(1,59,'05','2018-12-03',1,8),(1,60,'05','2018-12-03',1,8),(1,61,'05','2018-12-03',1,8),(1,62,'05','2018-12-03',1,3)")
        app.config["DATABASE"].commit()
    except mysql.connector.Error as err:
        print("Error: {}".format(err.msg))
    finally:
        mycursor.close()

def db_delete_full_day():  # Remove default fullDay
    mycursor = app.config['DATABASE'].cursor()
    try:
        mycursor.execute("DELETE FROM rest_book WHERE rest_book.bid IN(30,31,32,33,34,50,51,52,53,54,55,56,57,58,59,60,61,62)")
        mycursor.execute("DELETE FROM booking_info WHERE booking_info.bid IN(30,31,32,33,34,50,51,52,53,54,55,56,57,58,59,60,61,62)")
        app.config["DATABASE"].commit()
    except mysql.connector.Error as err:
        print("Error: {}".format(err.msg))
    finally:
        mycursor.close()

def db_get_times_from_period(period):
    mycursor = app.config['DATABASE'].cursor()
    try:
        sql = "SELECT time_period.timeid FROM time_period WHERE period=%s"
        mycursor.execute(sql, (str(period),))
        return mycursor.fetchall()
    except mysql.connector.Error as err:
        print("Error: {}".format(err.msg))
    finally:
        mycursor.close()

def db_get_attendance(date,period):
    mycursor = app.config['DATABASE'].cursor()
    try:
        sql = "SELECT SUM(people) FROM rest_book JOIN time_period on rest_book.timeid=time_period.timeid WHERE time_period.period=%s AND rest_book.date=%s"
        mycursor.execute(sql, (str(period),str(date)))
        sum=mycursor.fetchall()[0][0]
        if str(sum) !="None":
            return int(sum)
        return 0
    except mysql.connector.Error as err:
        print("Error: {}".format(err.msg))
    finally:
        mycursor.close()


'''
Apy dataset| single FC layer| no activation function| w2v semantic space| embedding space-semantic space
hinge ranking loss| no regularization | attribute-300 normalization is necessary|
'''
import tensorflow as tf
import numpy as np
import sys
import time
from datahelpers import datahelpersCUB
import matplotlib.pyplot as plt
import random

n_iteration = 1500  #10 epochs with random shuffling after each epoch
batch_size = 64
n_output = 312
learning_rate = 1e-2  # less the learning rate more close it will reach the global minimum
alpha = 0.0
margin_value = 5.0
summary_dir = '/tmp/zsl-apy'

def weight_variable(shape):
	initial = tf.truncated_normal(shape=shape, mean=0.0, stddev=1.0, dtype=tf.float32)
	return tf.Variable(initial)

def bias_variable(shape, value=0.5):
	initial = tf.constant(value, shape=shape)
	return tf.Variable(initial)

def get_next_batch(step, n_fullbatch=4, n_trainingExamples=500):
	step = step%(n_fullbatch+1)
	if step%n_fullbatch == 0 and step!=0:
		lst = range(step*batch_size, n_trainingExamples)
		return lst+range(0, batch_size-len(lst)), True
	return range(step*batch_size, (step+1)*batch_size), False
'''
def get_next_batch_new(step, n_fullbatch):
	flag = False
	step = step%n_fullbatch
	if step == n_fullbatch-1:
		flag = True
	return range(step*batch_size, (step+1)*batch_size), flag
'''
def training(train_x, train_y, train_class, test_x, test_y, test_class, attributes):
	start_time = time.time()
	train_class_len = len(train_class)
	test_class_len = len(test_class)
	n_trainingExamples = train_x.shape[0]
	n_testExamples = test_x.shape[0]
	n_input = train_x.shape[1]

	# for selecting same number of samples per class while training
	'''
	class_dict = {}
	for label in train_class:
		class_dict[label] = 0
	for ex in range(n_trainingExamples):
		class_dict[train_y[ex]] += 1
	class_dict_val = np.array(class_dict.values(), np.float32)
	class_dict_key = np.array(class_dict.keys())
	'''
	input_placeholder = tf.placeholder(tf.float32, shape=[None,n_input], name='deepFeatures') #[batch_size,4096]
	label_placeholder = tf.placeholder(tf.float32, shape=[None], name='actualLabels')   # [batch_size]
	attr_placeholder = tf.placeholder(tf.float32, shape=[n_output, None], name='attributes') #[n_output, train_class_len]
	minNNindex = tf.placeholder(tf.float32, shape=[None], name='predictedLabels')  # [batch_size]
	index = tf.placeholder(tf.int32, shape=[None], name='index')
	#weights = tf.placeholder(tf.float32, shape=[None])
	print('Placeholders added')

	# Fully connected layer
	'''
	w1 = weight_variable([n_input, n_hidden1])
	b1 = bias_variable([n_hidden1])
	out_hidden1 = tf.nn.relu(tf.add(tf.matmul(input_placeholder, w1), b1))
	w2 = weight_variable([n_hidden1, n_output])
	b2 = bias_variable([n_output])
	output = tf.add(tf.matmul(out_hidden1, w2),b2, name='output')   # [batch_size, n_output]
	'''
	w1 = weight_variable([n_input, n_output])
	b1 = bias_variable([n_output])
	output = tf.add(tf.matmul(input_placeholder, w1),b1, name='output')  #[batch_size, n_output]
	print('Model addition done')

	# Hinge loss implementation
	matrix = tf.matmul(output, attr_placeholder)  #[batch_size,len(train_class)]
	op2 = [[(-1*matrix[i][index[i]] + matrix[i][j])
				for j in range(train_class_len)] for i in range(batch_size)]
	generate_exp = tf.add(margin_value, op2)

	loss1 = tf.reduce_sum(tf.maximum(0.0, generate_exp), reduction_indices=1)-margin_value
	regularizers = tf.nn.l2_loss(w1)
	loss = tf.reduce_mean(loss1) + (alpha*regularizers)
	print('Loss function added to the tensorflow graph')

	train = tf.train.AdamOptimizer(learning_rate).minimize(loss)
	accuracy = tf.reduce_mean(tf.cast(tf.equal(minNNindex,label_placeholder), tf.float32),name='accuracy')
	with tf.Session() as sess:
		#loss_weights = sess.run(tf.nn.l2_normalize(tf.reciprocal(class_dict_val), dim=0))
		#new_dict = dict(zip(class_dict_key, loss_weights))
		#loss = tf.reduce_sum(tf.multiply(tf.reduce_sum(
		#								tf.maximum(zero, generate_exp), reduction_indices=1), weights))
		#loss = tf.reduce_mean(loss+alpha*regularizers)
		attributes = sess.run(tf.nn.l2_normalize(attributes, dim=1))
		print('Normalizing Done')
		sess.run(tf.global_variables_initializer())

		# for hinge loss
		train_class_attr=sess.run(tf.cast(tf.transpose(attributes[train_class-1]), tf.float32)) #[n_output,len(train_class)]
		test_class_attr =sess.run(tf.cast(tf.transpose(attributes[test_class-1]),tf.float32))

		#x_axis = []
		#y_axis = []
		n_fullbatch = int(n_trainingExamples/batch_size)
		lst = range(n_trainingExamples)
		random.shuffle(lst)
		train_x = train_x[lst]
		train_y = train_y[lst]

		for step in range(n_iteration):
			'''
			sample_per_class = batch_size/train_class_len
			indices = list([])
			start_idx = 0
			end_idx = 0
			for label in train_class:
				end_idx = start_idx + class_dict[label]
				indices.extend(random.sample(range(start_idx, end_idx), sample_per_class))
				start_idx = end_idx
			random.shuffle(indices)
			'''
			indices, flag = get_next_batch(step, n_fullbatch, n_trainingExamples)
			train_batch_x = train_x[indices]             # [batch_size, 4096]
			train_batch_y = train_y[indices]              # [batch_size]
			dontknow = np.reshape([np.where(train_class==item) for item in train_batch_y], (batch_size))
			#w = np.array([new_dict[label] for label in train_batch_y])
			sess.run(train, feed_dict={
				input_placeholder:train_batch_x,
				attr_placeholder:train_class_attr,
				index:dontknow
				#weights:w
				})
			loss_val, check_matrix = sess.run([loss,op2], feed_dict={
				input_placeholder:train_batch_x,
				attr_placeholder: train_class_attr,
				index:dontknow
				#weights:w
				})
			if (step+1)%100 == 0:
				# for hinge loss
				# Loss metric minimization
				out= sess.run(output, feed_dict={
					input_placeholder:train_batch_x,
					})
				matrix = np.matmul(out, train_class_attr)  #[batch_size, train_class_len]
				loss_per_image = np.zeros([train_class_len, batch_size])  # [batch_size]
				for count in range(train_class_len):
					generate_exp = np.add(margin_value, [[(-1*matrix[i][count] + matrix[i][j])
								for j in range(train_class_len) if j!=count]  for i in range(batch_size)])
					loss_per_image[count] = np.sum(np.maximum(0.0, generate_exp), axis=1)
				temp_list = np.argmin(loss_per_image, axis=0)
				# dot product maximization
				'''
				dot_product = tf.matmul(output, train_class_attr)  # [batch_size, len(train_class)]
				temp_list = np.array(sess.run(tf.argmax(dot_product, axis=1), feed_dict={
					input_placeholder:train_batch_x
					}))
				'''
				pred_result = train_class[temp_list]
				#print('Predicted labels:\n{},\n Actual labels:\n{}'.format(pred_result, train_batch_y))
				train_accuracy= (sess.run(accuracy, feed_dict={
					label_placeholder:train_batch_y,
					minNNindex:pred_result
					}))*100
				print ('Step:{}, Training Accuracy:{} % \n check_matrix:{}'.format(step, train_accuracy, check_matrix))
			print('Step:{}, Loss:{}'.format(step, loss_val))

			if flag:
				random.shuffle(lst)
				train_x = train_x[lst]
				train_y = train_y[lst]
			#y_axis.append(loss_val)

		# Finding the learned model accuracy on entire training dataset
		print('Calculating the training accuracy on {} classes'.format(train_class_len))
		# loss function minimization
		out = sess.run(output, feed_dict={
			input_placeholder:train_x
		})
		matrix = np.matmul(out, train_class_attr)  #[batch_size, train_class_len]
		loss_per_image = np.zeros([train_class_len, n_trainingExamples])
		for count in range(train_class_len):
			generate_exp = np.add(margin_value, [[(-1*matrix[i][count] + matrix[i][j])
						for j in range(train_class_len) if j!=count]  for i in range(n_trainingExamples)])
			loss_per_image[count] = np.sum(np.maximum(0.0, generate_exp), axis=1)
		temp_list = np.argmin(loss_per_image, axis=0)
		# dot product maximization
		'''
		dot_product = tf.matmul(output, train_class_attr)  # [batch_size, len(train_class)]
		temp_list = np.array(sess.run(tf.argmax(dot_product, axis=1), feed_dict={
			input_placeholder:train_x
			}))
		'''
		pred_result = train_class[temp_list]
		final_train_accuracy = (sess.run(accuracy, feed_dict={
			label_placeholder:train_y,
			minNNindex:pred_result
			}))*100
		time_taken = (time.time() - start_time)/60.0
		print('Training Ended with training time:{} mins\n'.format(time_taken))
		# loss function minimization
		print('Calculating the test accuracy on {} classes'.format(test_class_len))
		out = sess.run(output, feed_dict={
			input_placeholder:test_x
		})
		matrix = np.matmul(out, test_class_attr)  #[n_testExamples, test_class_len]
		loss_per_image = np.zeros([test_class_len, n_testExamples])
		for count in range(test_class_len):
			generate_exp = np.add(margin_value, [[(-1*matrix[i][count] + matrix[i][j])
						for j in range(test_class_len) if j!=count]  for i in range(n_testExamples)])
			loss_per_image[count] = np.sum(np.maximum(0.0, generate_exp), axis=1)
		temp_list = np.argmin(loss_per_image, axis=0)
		# dot product maximization
		'''
		dot_product = tf.matmul(output, test_class_attr)  # [batch_size, len(train_class)]
		temp_list = np.array(sess.run(tf.argmax(dot_product, axis=1), feed_dict={
					input_placeholder:test_x
			}))
		'''
		pred_result = test_class[temp_list]
		final_test_accuracy = (sess.run(accuracy, feed_dict={
			label_placeholder:test_y,
			minNNindex:pred_result
			}))*100
		# Plotting the loss function
		'''
		x_axis = np.array(range(n_iteration))
		y_axis = np.array(y_axis)
		fig = plt.figure()
		ax = plt.subplot(111)
		ax.plot(x_axis, y_axis)
		plt.title('Loss vs iterations')
		#fig.savefig('/Documents/zero-shot-learning-Saqib/plots/img8.png')
		#line, = plt.plot(x_axis, y_axis, lw=2.0)
		plt.legend()
		plt.show()
		'''
		return final_train_accuracy, final_test_accuracy

def main(train=True, cv=False, test=False):
	dataset = datahelpersCUB()
	print('Data loaded into dataset')

	n_class = dataset['attr312'].shape[0]
	dataset['defaultTestClassLabels'] = np.array(random.sample(range(1, n_class+1), 50))  # 50 unseen class
	dataset['defaultCVClassLabels'] = np.array(random.sample([label for label in range(1, n_class+1)
										if label not in dataset['defaultTestClassLabels']], 50))
	test_class = dataset['defaultTestClassLabels']
	cv_class = dataset['defaultCVClassLabels']
	train_class = np.array([label for label in range(1, n_class+1)
				if label not in np.concatenate((test_class, cv_class))])
	extra_class = np.sort(np.concatenate((train_class, cv_class)))
	datasetlabels = dataset['datasetLabels']

	features = dataset['X']      	        # [n_totalExamples, n_input]
	attributes = dataset['attr312']     	# [n_class, n_output]
	n_totalExamples = datasetlabels.shape[0]
	train_x = list([])
	test_x = list([])
	train_y = list([])
	test_y = list([])
	cv_x = list([])
	cv_y = list([])

	for ex in range(n_totalExamples):
		if datasetlabels[ex] in train_class:
			train_x.append(features[ex])
			train_y.append(datasetlabels[ex])
		elif datasetlabels[ex] in test_class:
			test_x.append(features[ex])
			test_y.append(datasetlabels[ex])
		else:
			cv_x.append(features[ex])
			cv_y.append(datasetlabels[ex])

	train_x = np.array(train_x)
	train_y = np.array(train_y)
	test_x = np.array(test_x)
	test_y = np.array(test_y)
	cv_x = np.array(cv_x)
	cv_y = np.array(cv_y)
	extra_x = np.concatenate((train_x, cv_x))
	extra_y = np.concatenate((train_y, cv_y))
	print('Data separated into train and test classes')
	# cross-validation
	if (cv == True):
		'''
		cv_accuracy_dict = {}
		train_accuracy_dict = {}
		margin_list = list([0.1, 1.0, 5.0, 10.0, 15.0, 20.0])
		#margin_value = 5.0
		alpha_list = list([0.0, 0.001, 0.01, 0.1, 1.0, 5.0])

		n_iteration = 1500
		for alpha in alpha_list:
			for margin in margin_list:
				#model2save = 'zsl-awa-w2v-model-{}'.format(item)
				item = (alpha, margin)
				print ('Trying out with this item:{}'.format(item))
				train_accuracy_dict[item], cv_accuracy_dict[item]  = training(train_x, train_y, train_class,
												    cv_x, cv_y, cv_class, attributes, item, n_iteration)
				with open('hyperfile.md', 'a') as file:
					file.write('{}\t{}\n'.format(train_accuracy_dict[item], cv_accuracy_dict[item]))
				#saved_model = 'zsl-awa-w2v-model-{}.meta'.format(item)
				#cv_accuracy[item] = testing(cv_x, cv_y, cv_class, attributes, saved_model)
		print('Training Accuracy on all the hyper-parameters:{}'.format(train_accuracy_dict))
		print('Cross validation accuracy on all the hyper-parameters:{}'.format(cv_accuracy_dict))
		# margin value for which cv accuracy is maximum
		alpha_value, margin_value = cv_accuracy_dict.keys()[np.argmax(cv_accuracy_dict.values())]
		# Now using this margin value train and test on the 40 classes and 10 classes resp.
		#test_model = 'zsl-awa-w2v-model-{}.meta'.format(margin_value)
		#test_accuracy = testing(test_x, test_y, test_class, attributes, test_model)
		item = (alpha_value, margin_value)
		n_iteration=5650
		'''
		final_train_accuracy, final_test_accuracy = training(extra_x, extra_y, extra_class, test_x, test_y,
												test_class, attributes)
		print('Final training and test accuracy:{} % , {} %'.format(final_train_accuracy, final_test_accuracy))

	# logging useful information in a file
	log_dict = {
		#'alpha_list':alpha_list,
		#'margin-list':margin_list,
		#'alpha-optimized':alpha_value,
		#'margin-optimized':margin_value,
		'metric':'loss-minimization',
		'alpha':alpha,
		'margin':margin_value,
		'sample':'serialwise-with-random-shuffling-after-each-epoch',
		#'sample':'sample-per-class-same',
		'description':'linear-model-without-activation',
		'attributes':'attr300+norm',
		'dataset':'apy',
		'feature':'vgg',
		'learning rate': learning_rate,
		'n_iterations':n_iteration,
		'batch size':batch_size,
		'number of layers':2,
		#'train_accuracy in cv step':train_accuracy_dict,
		#'cv_accuracy in cv step':cv_accuracy_dict,
		'final train accuracy':final_train_accuracy,
		'final test accuracy':final_test_accuracy
	}
	with open('logfile.md', 'a') as file:
		file.write('\n{}\n'.format(log_dict))

if __name__ == '__main__':
  	train=False
  	cv=True
  	test=False
  	main(train, cv, test)


import wx
import wx.lib.dialogs
import wx.stc as stc
import os
from xml.dom.minidom import parse
import xml.dom.minidom
import text_handler
import checkReferences
import getSummaries

if wx.Platform == '__WXMSW__':
	faces = { 'times': 'Times New Roman',
			  'mono': 'Courier New',
			  'helv': 'Arial',
			  'other': 'Comic Sans MS',
			  'size': 10,
			  'size2': 8}
elif wx.Platform == '__WXMAC__':
	faces = { 'times': 'Times New Roman',
			  'mono': 'Monaco',
			  'helv': 'Arial',
			  'other': 'Comic Sans MS',
			  'size': 12,
			  'size2': 10}

class MainWindow(wx.Frame):
	def __init__(self, parent, title):
		wx.Frame.__init__(self, parent, title=title, size = (800, 600))

		self.frameSettings()
		self.createMenus()
		self.addBinds()

	def frameSettings(self):
		self.leftMarginWidth = 20
		self.dirname = ''
		self.filename = ''
		self.sentenceIndexStart = 0
		self.sentenceIndexEnd = 0
		self.lineNumbersEnabled = True
		self.normalStylesFore = dict()
		self.normalStylesBack = dict()
		self.listControlLIndex = 0
		self.currentFilesRefs = {}
		self.currentSelectedWord = ""
		self.filesList = {}
		self.wordList = []
		self.CreateStatusBar()
		self.StatusBar.SetBackgroundColour((192, 192, 192))
		self.txt = stc.StyledTextCtrl(self, wx.TOP, style=wx.TE_MULTILINE | wx.TE_WORDWRAP | wx.BORDER_NONE)
		self.mentionsInFile = wx.ListCtrl(self, wx.ID_ANY, style=wx.LC_REPORT | wx.BORDER_NONE)
		self.mentionsOtherFileLocations = wx.ListCtrl(self, wx.ID_ANY, style=wx.LC_REPORT | wx.BORDER_NONE)

	def createMenus(self):
		self.menuBar = wx.MenuBar()
		self.SetMenuBar(self.menuBar)
		self.setFileMenu()
		self.setEditMenu()
		self.setPreferencesMenu()
		self.setHelpMenu()
		self.setNLPMenu()

	def addBinds(self):
		self.fileMenuBinds()
		self.editMenuBinds()
		self.prefMenuBinds()
		self.helpMenuBinds()
		self.nlpMenuBinds()

	## Setting up drop down menus
	def setFileMenu(self):
		filemenu = wx.Menu()
		self.menuNew = filemenu.Append(wx.ID_NEW, "&New", " Create a new document")
		self.menuOpen = filemenu.Append(wx.ID_OPEN, "&Open", " Open an existing document")
		self.menuSave = filemenu.Append(wx.ID_SAVE, "&Save", " Save the current document")
		self.menuSaveAs = filemenu.Append(wx.ID_SAVEAS, "Save &As", " Save a new document")
		filemenu.AppendSeparator()
		self.menuClose = filemenu.Append(wx.ID_EXIT, "&Close", " Close nlpTextEditor")
		self.menuBar.Append(filemenu, "&File")

	def setEditMenu(self):
		editmenu = wx.Menu()
		self.menuUndo = editmenu.Append(wx.ID_UNDO, "&Undo", " Undo last action")
		self.menuRedo = editmenu.Append(wx.ID_REDO, "&Redo", " Redo last action")
		editmenu.AppendSeparator()
		self.menuSelectAll = editmenu.Append(wx.ID_SELECTALL, "&Select All", " Select all text")
		self.menuCopy = editmenu.Append(wx.ID_COPY, "&Copy", " Copy selected text")
		self.menuCut = editmenu.Append(wx.ID_CUT, "C&ut", " Cut selected text")
		self.menuPaste = editmenu.Append(wx.ID_PASTE, "&Paste", " Paste text from the clipboard")
		self.menuBar.Append(editmenu, "&Edit")

	def setPreferencesMenu(self):
		preferencesMenu = wx.Menu()
		self.menuLineNumbers = preferencesMenu.Append(wx.ID_ANY, "Toggle &Line Numbers", " Show/Hide line numbers")
		self.menuBar.Append(preferencesMenu, "&Preferences")

	def setHelpMenu(self):
		helpMenu = wx.Menu()
		self.menuHelp = helpMenu.Append(wx.ID_ANY, "&Help", " How to use nlpTextEditor")
		helpMenu.AppendSeparator()
		self.menuAbout = helpMenu.Append(wx.ID_ABOUT, "&About", " About nlp and your note taking process")
		self.menuBar.Append(helpMenu, "&Help")

	def setNLPMenu(self):
		nlpMenu = wx.Menu()
		# menuSearchInCurrentFile = nlpMenu.Append(wx.ID_ANY, "Search in &Current", " Search for a topic in current file")
		# menuSearchElswhere = nlpMenu.Append(wx.ID_ANY, "Search &Elsewhere", " Search for a topic in other files")
		self.menuRefreshTopics = nlpMenu.Append(wx.ID_ANY, "&Refresh Topics", "Refresh the topic list")
		nlpMenu.AppendSeparator()
		self.menuViewAllTopics = nlpMenu.Append(wx.ID_ANY, "&View all Topics", "View all topics everywhere")
		nlpMenu.AppendSeparator()
		self.menuGetSummary = nlpMenu.Append(wx.ID_ANY, "&Get Wiki Facts", "Get facts on your topics pulled from Wikipedia")
		self.menuBar.Append(nlpMenu, "&NLP")

	## Sets up all events: menu events, list control click events,
	def fileMenuBinds(self):
		self.Bind(wx.EVT_MENU, self.onNew, self.menuNew)
		self.Bind(wx.EVT_MENU, self.onOpen, self.menuOpen)
		self.Bind(wx.EVT_MENU, self.onSave, self.menuSave)
		self.Bind(wx.EVT_MENU, self.onSaveAs, self.menuSaveAs)
		self.Bind(wx.EVT_MENU, self.onClose, self.menuClose)
		self.Bind(wx.EVT_CLOSE, self.onCloseWindow)

	def editMenuBinds(self):
		self.Bind(wx.EVT_MENU, self.onUndo, self.menuUndo)
		self.Bind(wx.EVT_MENU, self.onRedo, self.menuRedo)
		self.Bind(wx.EVT_MENU, self.onSelectAll, self.menuSelectAll)
		self.Bind(wx.EVT_MENU, self.onCopy, self.menuCopy)
		self.Bind(wx.EVT_MENU, self.onPaste, self.menuPaste)
		self.Bind(wx.EVT_MENU, self.onCut, self.menuCut)

	def prefMenuBinds(self):
		self.Bind(wx.EVT_MENU, self.onToggleLineNums, self.menuLineNumbers)

	def helpMenuBinds(self):
		self.Bind(wx.EVT_MENU, self.onHowTo, self.menuHelp)
		self.Bind(wx.EVT_MENU, self.onAbout, self.menuAbout)

	def nlpMenuBinds(self):
		self.Bind(wx.EVT_LIST_ITEM_ACTIVATED, self.onColClickTopics, self.mentionsInFile)
		self.Bind(wx.EVT_LIST_ITEM_ACTIVATED, self.onColClickElsewhere, self.mentionsOtherFileLocations)
		################# Need to add an event for on column click for mentions to open that file
		self.Bind(wx.EVT_MENU, self.refreshAllTopics, self.menuRefreshTopics)
		self.Bind(wx.EVT_MENU, self.addSummary, self.menuGetSummary)
		## These functions are called to set up the theme and the frames/sizes/their properties.
		self.setProperties()
		self.setLayout()

		## Lets the main window be visible
		self.Show()

	def setProperties(self):
		self.mentionsInFile.InsertColumn(0, 'Topics')
		self.mentionsInFile.SetColumnWidth(1, wx.LIST_AUTOSIZE_USEHEADER|wx.LC_SORT_ASCENDING)
		self.mentionsInFile.SetForegroundColour("black")

		self.mentionsOtherFileLocations.InsertColumn(0, 'Elsewhere')
		self.mentionsOtherFileLocations.SetColumnWidth(1, wx.LIST_AUTOSIZE_USEHEADER|wx.LC_SORT_ASCENDING)
		self.mentionsOtherFileLocations.SetForegroundColour("black")

		self.ParseSettings("theme.xml")
		self.txt.StyleSetSpec(stc.STC_STYLE_LINENUMBER, "back:#c0c0c0, face:%(helv)s, size:%(size2)d" % faces)
		self.txt.Bind(wx.EVT_KEY_UP, self.getLineColumn)

		self.txt.Bind(wx.EVT_KEY_DOWN, self.onKeyPress)

		self.txt.CmdKeyAssign(ord('='), stc.STC_SCMOD_CTRL, stc.STC_CMD_ZOOMIN)
		self.txt.CmdKeyAssign(ord('-'), stc.STC_SCMOD_CTRL, stc.STC_CMD_ZOOMOUT)

		self.txt.SetViewWhiteSpace(False)
		self.txt.SetMargins(5, 0)
		self.txt.SetMarginType(1, stc.STC_MARGIN_NUMBER)
		self.txt.SetMarginWidth(1, self.leftMarginWidth)
		self.txt.SetUseHorizontalScrollBar(False)

		nSFore = self.normalStylesFore
		nSBack = self.normalStylesBack

		self.txt.StyleSetBackground(stc.STC_STYLE_DEFAULT, nSBack["Main"])
		self.txt.SetSelBackground(True, "#9C9C9C")

		self.txt.StyleSetSpec(stc.STC_P_DEFAULT, "fore:%s,back:%s" % (nSFore["Default"], nSBack["Default"]))
		self.txt.StyleSetSpec(stc.STC_P_DEFAULT, "face:%(helv)s,size:%(size)d" % faces)

		self.mentionsInFile.SetBackgroundColour(nSBack["Main"])
		self.mentionsOtherFileLocations.SetBackgroundColour(nSBack["Main"])

	def setLayout(self):
		sizer = wx.BoxSizer(wx.HORIZONTAL)

		sizer.Add(self.txt, 6, wx.EXPAND, border=0)
		sizer.Add(self.mentionsInFile, 0, wx.EXPAND, border=0)
		sizer.Add(self.mentionsOtherFileLocations, 0, wx.EXPAND, border=0)

		self.SetSizer(sizer)
		self.Layout()

	def ParseSettings(self, settings_file):
		DOMTree = xml.dom.minidom.parse(settings_file)
		collection = DOMTree.documentElement

		styles = collection.getElementsByTagName("style")
		for s in styles:
			item = s.getElementsByTagName("item")[0].childNodes[0].data
			color = s.getElementsByTagName("color")[0].childNodes[0].data
			side = s.getElementsByTagName("side")[0].childNodes[0].data
			sType = s.getAttribute("type")
			if side == "Back":  # background
				self.normalStylesBack[str(item)] = str(color)
			else:
				self.normalStylesFore[str(item)] = str(color)

	def GetListCtrl(self):
		return self.mentionsInFile, self.mentionsOtherFileLocations

	def onNew(self, e):
		frame = MainWindow(title=' Elsewhere - Untitled', parent=None)

	def onOpen(self, e):
		if self.txt.IsEmpty():
			try:
				dlg = wx.FileDialog(self, "Choose a file", self.dirname, "", "*.*", wx.FD_OPEN)
				if(dlg.ShowModal() == wx.ID_OK):
					self.filename = dlg.GetFilename()
					self.dirname = dlg.GetDirectory()
					f = open(os.path.join(self.dirname, self.filename), 'r')
					self.txt.SetValue(f.read())
					self.mentionsInFile.DeleteAllItems()
					self.mentionsOtherFileLocations.DeleteAllItems()

					self.refreshAllTopics(e)
					self.sentenceIndexStart = self.txt.GetLength() - 1
					############# This needs to be changed to whatever function we decide to read in the file and its own words
					#self.populateMentionsInFile(items)

					self.SetTitle(' Elsewhere - ' + self.filename)

					f.close()

				dlg.Destroy()
			except:
				dlg = wx.MessageDialog(self, "Failed to open file", "Error", wx.ICON_ERROR)
		else:
			try:
				dlg = wx.FileDialog(self, "Choose a file", self.dirname, "", "*.*", wx.FD_OPEN)
				if(dlg.ShowModal() == wx.ID_OK):
					frame2 = MainWindow(title=' Elsewhere - Untitiled', parent=None)
					self.filename = dlg.GetFilename()
					self.dirname = dlg.GetDirectory()
					f = open(os.path.join(self.dirname, self.filename), 'r')
					frame2.txt.SetValue(f.read())

					self.refreshAllTopics(e)
					self.sentenceIndexStart = self.txt.GetLength() - 1
					############# This needs to be changed to whatever function we decide to read in the file and its own words
					#self.populateMentionsInFile(items)

					frame2.SetTitle(' Elsewhere - ' + self.filename)

					f.close()

				dlg.Destroy()
			except:
				dlg = wx.MessageDialog(self, "Failed to open file", "Error", wx.ICON_ERROR)

	def onSave(self, e):
		try:
			self.save()
		except:
			try:
				self.saveDialogue()
			except:
				pass

	def onSaveAs(self, e):
		try:
			self.saveDialogue()
		except:
			pass

	def save(self):
		f = open(os.path.join(self.dirname, self.filename), 'w')
		f.write(self.txt.GetValue())
		self.SetTitle(' Elsewhere - ' + self.filename)
		f.close()

	def saveDialogue(self):
		dlg = wx.FileDialog(self, "Save file as", self.dirname, "Untitled", "*.txt",
							wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT)
		if (dlg.ShowModal() == wx.ID_OK):
			self.filename = dlg.GetFilename()
			self.dirname = dlg.GetDirectory()
			f = open(os.path.join(self.dirname, self.filename), 'w')
			f.write(self.txt.GetValue())
			self.SetTitle(' Elsewhere - ' + self.filename)
			f.close()
		dlg.Destroy()

	def onClose(self, e):
		try:
			f = open(os.path.join(self.dirname, self.filename), 'w')
			f.write(self.txt.GetValue())
			f.close()
		except:
			try:
				dlg = wx.FileDialog(self, "Save file as", self.dirname, "Untitled", "*.txt", wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT)
				if (dlg.ShowModal() == wx.ID_OK):
					self.filename = dlg.GetFilename()
					self.dirname = dlg.GetDirectory()
					f = open(os.path.join(self.dirname, self.filename), 'w')
					f.write(self.txt.GetValue())
					f.close()
				dlg.Destroy()
			except:
				pass
		text_handler.jsonDump(self.currentFilesRefs, self.filename, self.dirname)
		self.Close(True)

	def onCloseWindow(self, event):
		# dialog to verify exit (including menuExit)
		dlg = wx.MessageDialog(self, "Are you sure you want to exit", "Exit", wx.YES_NO | wx.ICON_QUESTION)
		if dlg.ShowModal() == wx.ID_YES:
			if self.filename == "":
				dlg2 = wx.MessageDialog(self, "Would you like to save?", "Save?", wx.YES_NO | wx.ICON_QUESTION)
				if dlg2.ShowModal() == wx.ID_YES:
					self.saveDialogue()
					text_handler.jsonDump(self.wordList, self.filename, self.dirname)
					dlg2.Destroy()
					self.Destroy()
				else:
					dlg2.Destroy()
					self.Destroy()
			else:
				self.save()
				text_handler.jsonDump(self.wordList, self.filename, self.dirname)
				self.Destroy()  # frame
		else:
			pass

	def onUndo(self, e):
		self.txt.Undo()

	def onRedo(self, e):
		self.txt.Redo()

	def onSelectAll(self, e):
		self.txt.SelectAll()

	def onCopy(self, e):
		self.txt.Copy()

	def onPaste(self, e):
		self.txt.Paste()

	def onCut(self, e):
		self.txt.Cut()

	def onToggleLineNums(self, e):
		if(self.lineNumbersEnabled):
			self.txt.SetMarginWidth(1, 0)
			self.lineNumbersEnabled = False
		else:
			self.txt.SetMarginWidth(1, self.leftMarginWidth)
			self.lineNumbersEnabled = True

	def onHowTo(self, e):
		dlg = wx.lib.dialogs.scrolledMessageDialog(self, "This is how to use our nlpTextEditor", size=(200,200))
		dlg.ShowModal()
		dlg.Destroy()

	def onAbout(self, e):
		dlg = wx.MessageDialog(self, "Text editor that implements nlp and wxPython. Made by Elizabeth Mena and Andrei Butuc", wx.OK)
		dlg.ShowModal()
		dlg.Destroy()

	## Gets line and column numbers to populate status bar
	def getLineColumn(self, e):
		line = self.txt.GetCurrentLine() + 1
		column = self.txt.GetColumn(self.txt.GetCurrentPos())
		stat = "Line: %s, Column %s" % (line, column)
		self.StatusBar.SetStatusText(stat, 0)

	## When user presses space, punctuation, etc, word gets sent over to text handler to be filtered
	def onKeyPress(self, event):
		keycode = event.GetKeyCode()
		## 13 is enter, 46 is .
		keycode_list = [13, 46]
		if keycode_list.__contains__(keycode):
			self.sentenceIndexEnd = self.txt.GetInsertionPoint()
			sentence = self.txt.GetRange(self.sentenceIndexStart, self.sentenceIndexEnd)
			if sentence.strip() != "":
			## True: word.strip(), add to word list, send word list to populate fn
				toBeAdded = text_handler.text_filtering(sentence)
				for word in toBeAdded:
					if word not in self.wordList:
						self.wordList.append(word)
				self.populateTopics(self.wordList)
				self.sentenceIndexStart = self.sentenceIndexEnd + 1
		event.Skip()
	## Preconditions: Event when user clicks a word in the Topics list control
	## Postcondition: word is sent over to populateElsewhere, and currentSelectedWord is reset.
	def onColClickTopics(self, event):
		self.mentionsOtherFileLocations.DeleteAllItems()
		word = event.GetText()
		self.populateElsewhere(word)
		self.currentSelectedWord = word

	## Precondition: Event when user clicks a word in the Elsewhere list control
	## Postcondition: filename and currentSelected word are sent to check references, return value is sent to open
	def onColClickElsewhere(self, event):
		fn = event.GetText()
		fp = checkReferences.getDirName(fn, self.currentSelectedWord)
		f = open(os.path.join(fp, fn), 'r')
		newFrame = MainWindow(title=' Elsewhere - ' + fn, parent = None)
		newFrame.txt.SetValue(f.read())
		self.refreshAllTopics(event)

	## Clears list control to add the new words and alphabetize them.
	## Precondition: wordList, every time a new word is added
	## Postcondition: Topics list control populated with sorted topic references from the current file
	def populateTopics(self, wordList):
		self.mentionsInFile.DeleteAllItems()
		wordList.sort(reverse=True)
		## We initiate index as 0, because this determines which spot in the listctrl that the reference is put in
		index = 0

		for i in wordList:
			if i != " ":
				self.mentionsInFile.InsertItem(self.listControlLIndex, i)
				self.mentionsInFile.SetItem(self.listControlLIndex, 0, i)
				index += 1

	## Clears list control every time a new word is clicked and populates with alphabetized file names
	## Precondition: word that was clicked in the topic list control
	## Postcondition: a populated list control containing file names in alphabetical order
	def populateElsewhere(self, word):
		## We initiate index as 0, because this determines which spot in the listctrl that the reference is put in
		## create file_list: we send word to checkReferences.getFileName which returns a list of file names, then iterate
		## through this list and populate the list ctrl
		index = 0
		file_list = checkReferences.getFileName(word)
		if len(file_list) == 0:
			return
		file_list.sort()
		for i in file_list:
			self.mentionsOtherFileLocations.InsertItem(index, i)
			self.mentionsOtherFileLocations.SetItem(index, 0, i)

			index += 1

	## ALL NEW FUNCTIONS GET ADDED BELOW HERE

	def refreshAllTopics(self, event):
		for d in range(0, len(self.wordList)):
			self.mentionsInFile.DeleteItem(d)
		temp = text_handler.text_filtering(self.txt.GetText())
		self.populateTopics(temp)
		self.wordList = temp
		return

	## Add a function that pulls all topics, searches wikipedia, appends statements to end of textarea.
	def addSummary(self, e):
		summaries = getSummaries.getWikiSummary(self.wordList)
		self.txt.AppendText("\n\n\n##WikiSummaries##\n" + summaries)

app = wx.App()
frame = MainWindow(None, " Elsewhere - Untitled")
app.MainLoop()


from deap import base, creator, tools, algorithms
from PIL import Image, ImageDraw
from numpy import ceil, sqrt, square, sum
from numpy.random import randint
from scoop import futures
import numpy

PIC = Image.open('mountain-small.jpg')
WIDTH, HIGHT = PIC.size
WINDOW = WIDTH if WIDTH < HIGHT else HIGHT

NUMBER_OF_TRIANGLES = 100
POPULATION = 3
NGEN = 10000
POLY = 3
GEN = 0

pix1 = numpy.matrix( PIC.getdata() )

def gen_one_poly():
    return (tuple([(randint(0, WIDTH), randint(0, HIGHT)) for i in xrange(POLY)]), # axis point
            randint(0,255), randint(0,255), randint(0,255), randint(0,30)) # color, transparency
            #0, 0, 0, 0)

def poly_to_image(triangles):
    im = Image.new('RGB', (WIDTH, HIGHT), (255, 255, 255))
    for tri in triangles:
        mask = Image.new('RGBA', (WIDTH, HIGHT))
        draw = ImageDraw.Draw(mask)
        draw.polygon(tri[0], fill=tri[1:])
        im.paste(mask, mask=mask)
        del mask, draw
    return im

def evaluate(t2):
    im2 = poly_to_image(t2)
    pix2 = numpy.matrix(im2.getdata())
    def err(m, n):
        return sum( square( m - n ) )
    ans = sum( err(pix1.H, pix2.H) )
    return 1 - (1. * sqrt(ans) / sqrt(WIDTH * HIGHT * POLY * 255 * 255)),

def mutate(triangles):

    e0 = evaluate(triangles)
    tid = randint( 0, NUMBER_OF_TRIANGLES - 1 )
    oldt = triangles[tid]

    def evolv( mutants ): # mark it as an advantage?
        for t in mutants:
            triangles[tid] = tuple(t)
            if evaluate(triangles) < e0:
                triangles[tid] = oldt

    fil = lambda oldt: [ list(x) for x in oldt[0] ] + [ x for x in oldt[1:] ]
    tpl = lambda t: [tuple( [tuple(x) for x in t[:POLY]] )] + t[POLY:]
    # mut = lambda x: [randint(WIDTH), randint(HIGHT)] # we can improve the mutation method!
    def mut(x):
        wb = max(0, x[0]-WINDOW), min(x[0]+WINDOW, WIDTH)
        hb = max(0, x[1]-WINDOW), min(x[1]+WINDOW, HIGHT)
        return [randint(wb[0], wb[1]), randint(hb[0], hb[1])] # find better axis point in a window

    mutants = []
    for i in xrange(POLY):
        t = fil( oldt ); t[i] = mut( t[i] ); mutants.append( tpl(t) ) # change an axis point
    evolv( mutants )

    # mut = randint(255)
    mut = lambda x: randint( max(0, x-WINDOW), min(x+WINDOW, 255) ) # find better color in a window

    mutants = []
    for i in xrange(POLY,len( fil(oldt) )):
        t = fil( oldt ); t[ i ] = mut( t[i] ); mutants.append( tpl(t) ) # change colors
        t = fil( oldt ); t[-1 ] = randint(100); mutants.append( tpl(t) ) # change transparency
    evolv( mutants )

    global GEN, WINDOW

    if GEN % 200 == 0:
        poly_to_image(triangles).save('genetic-pics/result'+str(GEN)+'.jpg')
        WINDOW = int( ceil(WINDOW * .9) ) # shrinks window
    GEN += 1

    return triangles,

creator.create("Fitness", base.Fitness, weights=(1.0,))
creator.create("Individual", list, fitness=creator.Fitness)

toolbox = base.Toolbox()
toolbox.register("attr", gen_one_poly)
toolbox.register("individual", tools.initRepeat,
                 creator.Individual, toolbox.attr, NUMBER_OF_TRIANGLES)
toolbox.register("population", tools.initRepeat,
                 list, toolbox.individual)

toolbox.register("evaluate", evaluate)
toolbox.register("mate", tools.cxTwoPoint)
toolbox.register("mutate", mutate)
toolbox.register("select", tools.selTournament, tournsize=3)

toolbox.register("map", futures.map)

def main():
    pop = toolbox.population(n=POPULATION)
    hof = tools.HallOfFame(1)

    stats = tools.Statistics(lambda ind: ind.fitness.values)
    stats.register("std", numpy.std)
    stats.register("max", numpy.max)
    stats.register("avg", numpy.mean)
    stats.register("min", numpy.min)

    try:
        pop, log = algorithms.eaSimple(
            pop, toolbox, cxpb=0.5, mutpb=0.2, ngen=NGEN, stats=stats,
            halloffame=hof, verbose=True)
    finally:
        open('result.txt', 'w').write(repr(hof[0])) # best suvivor
        poly_to_image(hof[0]).save('result.jpg')

if __name__ == '__main__':
    main()


import numpy as np
import glob
import os
import matplotlib.pyplot as plt

import numpy as np
import glob
import os
import matplotlib.pyplot as plt

# hod model number
model_no = 1

z_in = 0.800#0.576

# HOD directory on alan
hod_dir = "/mnt/gosling1/boryanah/light_cone_catalog/AbacusSummit_base_c000_ph006/HOD/z%.3f/model_%d_rsd/"%(z_in,model_no)

sats_fns = sorted(glob.glob(hod_dir+"*sats*"))
cent_fns = sorted(glob.glob(hod_dir+"*cent*"))

print(len(cent_fns),len(sats_fns))

def load_gals(fns,dim):

    for fn in fns:
        tmp_arr = np.fromfile(fn).reshape(-1,dim)
        try:
            gal_arr = np.vstack((gal_arr,tmp_arr))
        except:
            gal_arr = tmp_arr

    return gal_arr

sats_arr = load_gals(sats_fns,dim=9)
cent_arr = load_gals(cent_fns,dim=8)

# first three columns of file are positions, next are velocities, then halo index and finally halo mass
sats_pos = sats_arr[:,0:3]
cent_pos = cent_arr[:,0:3]
sats_mass = sats_arr[:,-1]
cent_mass = cent_arr[:,-1]

print(sats_pos.shape)
print(cent_pos.shape)

print(cent_pos.min())
print(cent_pos.max())
print(sats_pos.min())
print(sats_pos.max())

x_min = 0
x_max = x_min+40.
i = 1
j = 2
k = 0
sel_cent = (cent_pos[:,k] > x_min) & (cent_pos[:,k] < x_max)
sel_sats = (sats_pos[:,k] > x_min) & (sats_pos[:,k] < x_max)

print("number of centrals in cross section = ",np.sum(sel_cent))
print("number of satellites in cross section = ",np.sum(sel_sats))

plt.title("Cross-section of the simulation")
plt.scatter(cent_pos[sel_cent,i],cent_pos[sel_cent,j],color='dodgerblue',s=1,alpha=0.8,label='centrals')
plt.scatter(sats_pos[sel_sats,i],sats_pos[sel_sats,j],color='orangered',s=1,alpha=0.8,label='satellites')
plt.axis('equal')
plt.xlabel('Y [Mpc]')
plt.ylabel('Z [Mpc]')
plt.savefig("scatter.png")
plt.show()


from getting_information import get_params
from getting_field import get_field
import numpy
import copy

field = [[".", ".", ".", ".", "#", ".", ".", ".", "."],
         [".", ".", ".", ".", ".", ".", ".", ".", "."],
         [".", ".", ".", ".", "#", ".", "#", "#", "#"],
         [".", ".", ".", ".", "#", ".", "#", ".", "."],
         [".", "#", ".", ".", ".", ".", "#", ".", "."],
         [".", "#", ".", ".", ".", ".", ".", ".", "."],
         [".", "#", ".", "#", "#", "#", ".", ".", "."]]

def has_no_wall(field, a, b, x, y):
    for w in range(min(a, x), max(a, x) + 1):
        for v in range(min(b, y), max(b, y) + 1):
            if field[w][v] == "#":
                return False

    return True

def get_around_area(field, needed_info):
    print(needed_info)
    rows, cols, R = needed_info[0], needed_info[1], needed_info[2]
    a, b, list_of_X = needed_info[3][0], needed_info[3][1], []

    for x in range(rows+1):
        for y in range(cols+1):
            if abs(a-x) <= R and abs(b-y) <= R and \
                    has_no_wall(field, a, b, x, y) and (x, y) != (a, b):
                field[x][y] = "X"
                list_of_X.append((x, y))

    field[a][b] = "r"

    return numpy.array(field), (a, b), list_of_X
    # return (a, b), list_of_X

print(get_around_area(field, [6, 8, 2, (0, 0)]))

#
# def all_rs_possible(field, info):
#     needed_info = [info["rows"], info["columns"], info["radius"], ()]
#     r_dictionary = dict()
#
#     for line in range(info["rows"]):
#         for symb in range(info["columns"]):
#             if field[line][symb] != "#" and field[line][symb] != "-":
#                 needed_info[3] = (line, symb)
#                 key, value = get_around_area(field, needed_info)
#                 r_dictionary[key] = value
#
#     return sorted(r_dictionary.items(), key=lambda kv: len(kv[1]), reverse=True)
#
#
# def only_needed_rs(field, info_dict):
#     all_points_dict, final_list_of_rs = all_rs_possible(field, info_dict), []
#
#     for i in range(len(all_points_dict)):
#         for j in range(len(all_points_dict)):
#             if len(all_points_dict[i][1]) > len(all_points_dict[j][1]) and \
#                bool(set(all_points_dict[i][1]) & set(all_points_dict[j][1])):
#                 all_points_dict[j] = ((0, 0), [])
#
#     for i in all_points_dict:
#         if i != ((0, 0), []):
#             final_list_of_rs.append(i)
#
#     return final_list_of_rs
#
#
# def filling_new_field(new_field, coords_list):
#     for i in coords_list:
#         r_coord1, r_coord2 = i[0][0], i[0][1]
#         new_field[r_coord1][r_coord2] = "R"
#         for coords in i[1]:
#             new_field[coords[0]][coords[1]] = "x"
#
#     return numpy.array(new_field)
#
#
# def write_file(file_name, write_in_file):
#     field = get_field(file_name)
#     new_field = copy.deepcopy(field)
#     info_dict = get_params(file_name)
#
#     with open(write_in_file, "w") as file:
#         res = filling_new_field(new_field, only_needed_rs(field, info_dict))
#
#         final_str = ""
#         for i in res:
#             final_str += " ".join(i) + "\n"
#
#         file.write(final_str)
#
#
# if __name__ == "__main__":
#     given_files = "charleston_road.in"
#     output_files = "charleston_res.txt"

    # for i in range(len(given_files)):
    #     write_file(given_files[i], output_files[i])


from flask import Flask, render_template, request, redirect, url_for, send_from_directory
import pandas as pd
import numpy as np
from flask_bootstrap import Bootstrap

use_col = "     -----     "
use_key = "     -----     "

data_xlsx = pd.ExcelFile('data/200218_antiDoping.xlsx').parse()
data_dic_original = {ind: [v for v in data_xlsx.loc[ind].values] for ind in data_xlsx.index}
data_col = list(data_xlsx.columns)
search_dic = {}
for c, col in enumerate(data_col):
    data_temp = data_xlsx.iloc[:, c]
    key_list = [v for v in set(data_temp) if v != "-"]
    search_dic[col] = sorted(key_list)

"""
box = []
for ind in data_xlsx.index:
    for No, v in enumerate(data_xlsx.loc[ind].values):
        if No == 4:
            if (str(v) != "nan") and ("(" in v):
                name = v.split("(")[0]
                hospital = v.split("(")[1].split(")")[0]
            else:
                name = v
                hospital = ""

            print(name, hospital)
            box += [name]

temp = pd.DataFrame(box)
temp.to_csv("a.csv", encoding="cp932")
"""

app = Flask(__name__, static_url_path="")
bootstrap = Bootstrap(app)

DATA_DIR = "./data"
@app.route('/data/<path:path>')
def send_js(path):
    return send_from_directory(DATA_DIR, path)

@app.route('/')
def index():
    app.config['data_dic'] = data_dic_original
    app.config['search_dic'] = search_dic
    app.config['use_col'] = use_col
    app.config['use_key'] = use_key

    return render_template('index_bootStrap.html',
                           data_col=data_col, data_dic=app.config['data_dic'], search_dic=app.config['search_dic'],
                           use_col=app.config['use_col'], use_key=app.config['use_key'])

@app.route('/data_refine', methods=['POST', 'GET'])
def data_refine():
    print("A", request.form["btn"])
    if request.form["btn"] == "全て表示":
        app.config['data_dic'] = data_dic_original
        app.config['use_col'] = use_col
        app.config['use_key'] = use_key

    elif request.form["btn"] == "絞り込む":
        app.config['use_col'] = request.form.get('select_refine1')
        app.config['use_key'] = request.form.get('select_refine2')
        index_use = [ind for ind in list(data_xlsx.index) if data_xlsx.loc[ind, app.config['use_col']] == app.config['use_key']]
        app.config['data_dic'] = {ind: [v for v in data_xlsx.loc[ind].values] for ind in index_use}

    return render_template('index_bootStrap.html',
                           data_col=data_col, data_dic=app.config['data_dic'], search_dic=app.config['search_dic'],
                           use_col=app.config['use_col'], use_key=app.config['use_key'])

if __name__ == '__main__':
    app.run(debug=True)


import subprocess
from Bio.Align.Applications import MuscleCommandline

#muscle = "muscle3.8.31_i86linux64"
class alinear():
    def __init__(self):

        self.cline = MuscleCommandline(input ="alinear.fasta",out="arbol.aln",clw=True)
        self.string = str(self.cline) + "&"
        subprocess.call("gedit alinear.fasta", shell=True)
        subprocess.call(self.string, shell=True)


import click
import urllib2
import urllib
import json
import os.path
from functools import update_wrapper

API_ROOT = 'http://api.private.civitaslearning.com/state/v1'

#this are from the click imagepipe example code
#https://github.com/mitsuhiko/click/blob/1c3dfacddcb73bb256d0e604e239cfd4c75f71f2/examples/imagepipe/imagepipe.py
def processor(f):
    """Helper decorator to rewrite a function so that it returns another
    function from it.
    """
    def new_func(*args, **kwargs):
        def processor(stream):
            return f(stream, *args, **kwargs)
        return processor
    return update_wrapper(new_func, f)

def slate_get_request(endpoint, params):
    params = urllib.urlencode(params)
    url_path = '{}?{}'.format(os.path.join(API_ROOT, endpoint), params)

    click.echo(click.style('Fetching URL: ', fg='cyan') + click.style(url_path, fg='red'))

    url_request = urllib2.Request(url_path)
    url_obj = urllib2.urlopen(url_request)
    results = json.loads((url_obj.read()))
    return results

@click.group(chain=True, invoke_without_command=True)
@click.option('-f', '--out-file', type=click.File('w'), is_flag=True)
def cli(out_file):
    pass

@cli.resultcallback()
def process_lookup_pipeline(processors, out_file):
    """
    example call

    deslater Institution -natural_key arizona Resource

    should get you all Resource under all institutions with the name arizona

    :param processors: list of functions ordered as called on the cmd line
    :param out_file: optional out file for more verbose output
    :return: None
    """
    # Start with an empty iterable.
    stream = ({},)

    # Pipe it through all stream processors.
    for processor in processors:
        stream = show(processor(stream))

    # Evaluate the stream and throw away the items.
    for _ in stream:
        pass

def show(streams):
    for stream in streams:
        out_str = json.dumps(stream[stream['last']], indent=2)
        click.echo(click.style('JSON for {} Request'.format(click.style(stream['last'], fg='red')), fg='cyan'))
        click.echo(click.style(out_str, fg='yellow', bg='blue'))
        yield stream

@cli.command('Institutions',
               help='Query the Slate API for institutions.')
@click.option('-n', '--natural-key', type=click.STRING, help="The people friendly name of a institution.")
@processor
def get_institutions(streams, natural_key):
    """
    The sub command for looking up institutions. Institutions are top level keys.
    """
    endpoint = 'institutions'

    for stream in streams:
        params = {}
        if natural_key:
            params['natural_key'] = natural_key
        results = slate_get_request(endpoint, params)

        for result in results['results']:
            stream['last'] = 'institution'
            stream['institution'] = result
            yield stream

@cli.command('Namespaces')
@click.option('-n', '--natural-key', type=click.STRING, help="The descriptive name of a namesapce.")
@click.option('-i', '--institution', type=click.INT, help="The integer key for a institution record.")
@click.option('-c', '--created-by-user', type=click.STRING, help="The integer key of creating user")
@processor
def get_namespaces(streams, natural_key, institution, created_by_user):
    """
    Sub command for looking up namespaces via the slate API.
    """
    endpoint = 'namespaces'

    for stream in streams:
        #Both the stream and the function params can have the same data.
        #We want to choose cmd line params over stream data.
        params = {}
        if natural_key:
            params['natrual_key'] = natural_key
        if institution:
            params['instituion'] = institution
        elif 'institution' in stream:
            params['institution'] = stream['institution']['_id']
        if created_by_user:
            params['created_by_user'] = created_by_user
        elif 'user' in stream:
            params['created_by_user'] = stream['user']['_id']
        results = slate_get_request(endpoint, params)

        for result in results['results']:
            stream['last'] = 'namespace'
            stream['namespace'] = result
            yield stream

@cli.command('Resources')
@click.option('-s', '--namespace', type=click.INT, help='The integer key of the resources namespace.')
@click.option('-l', '--label', type=click.STRING, help="The resources label.")
@click.option('-t', '--type', type=click.STRING, help="The resource type")
@click.option('-n', '--name', type=click.STRING, help="The resource name")
@processor
def get_resources(streams, namespace, label, type, name):
    """
    Query the resources endpoint of the slate API
    """
    endpoint = 'resources'

    for stream in streams:
        params = {}
        if label:
            params['label'] = label
        if type:
            params['type'] = type
        if name:
            params['name'] = name
        if namespace:
            params['namespace'] = namespace
        elif 'namespace' in stream:
            params['namespace'] = stream['namespace']['_id']

        # namespace is required so lets try to see if we have some sort of upstream data to guess at them
        # we could go as wide as pulling them all,
        # but that seems like it will just make long running useless requests
        if 'institution' in stream:
            ns_params = {'institution': stream['institution']['_id']}

            results = slate_get_request('namespaces', ns_params)
            for result in results['results']:
                params['namespace'] = result['_id']

                results = slate_get_request(endpoint, params)
                for result in results['results']:
                    stream['last'] = 'resource'
                    stream['resource'] = result
                    yield stream
        else:
            results = slate_get_request(endpoint, params)
            for result in results['results']:
                stream['last'] = 'resource'
                stream['resource'] = result
                yield stream


import itertools
from operator import itemgetter
import plotly.graph_objs as go
from django.db.models.functions import (
    TruncYear, TruncQuarter, TruncMonth, TruncWeek, TruncDay)

# TODO : use classes instead of methods (queryset class, plotly_data class...)

def trunc_date_attribute(queryset, attribute_name, trunc_period='day'):
    """Add truncated attribute to queryset

    Parameters
    ----------
    queryset : queryset
        Query object with datetime attribute.
    attribute_name : str
        Name of datetime attribute in the query.
    trunc_period : str
        Possible options are: 'year', 'quarter', 'month', 'week', 'day', by default 'day'

    Returns
    -------
    queryset
        Annotate a new value to queryset having the name `<attribute_name>_truncated` with the truncated date.
    """
    truncated_attribute_name = attribute_name + '_truncated'
    annotate_kwargs = {}

    if trunc_period == 'year':
        annotate_kwargs[truncated_attribute_name] = TruncYear(
            attribute_name)
        pass
    elif trunc_period == 'quarter':
        annotate_kwargs[truncated_attribute_name] = TruncQuarter(
            attribute_name)

    elif trunc_period == 'month':
        annotate_kwargs[truncated_attribute_name] = TruncMonth(
            attribute_name)

    elif trunc_period == 'week':
        annotate_kwargs[truncated_attribute_name] = TruncWeek(
            attribute_name)

    elif trunc_period == 'day':
        annotate_kwargs[truncated_attribute_name] = TruncDay(
            attribute_name)
    else:
        # Value by default is `day`
        annotate_kwargs[truncated_attribute_name] = TruncDay(
            attribute_name)

    qs = queryset.annotate(**annotate_kwargs)

    return qs

def convert_group_by_field_to_attribute(group_by_field, model):
    if model == 'StockControl':
        if group_by_field == 'product':
            return 'stock__product'
        elif group_by_field == 'category':
            return 'stock__product__category'
    elif model == 'DeliveryDetail':
        if group_by_field == 'product':
            return 'stock__product'
        elif group_by_field == 'category':
            return 'stock__product__category'
    else:
        if group_by_field == 'product':
            return 'product'
        elif group_by_field == 'category':
            return 'product__category'

def group_data_by_category_2(queryset, x_key, y_keys):
    '''
    input = queryset([
        {'x_key': 2020, 'y_keys[0]': 0, 'y_keys[1]': 2},
        {'x_key': 2021, 'y_keys[0]': 1, 'y_keys[1]': 0},
        {'x_key': 2022, 'y_keys[0]': 1, 'y_keys[1]': 2},
    ])
    output = dict({
        'y_keys[0]': [
            {'x': [2020, 2021, 2022]},
            {'y': [0, 1 ,1]}
        ],
        'y_keys[1]': [
            {'x': [2020, 2021, 2022]},
            {'y': [2, 0, 2]}
        ]
    })
    '''
    chartdata = []
    data = queryset.values(x_key, *y_keys, named=True)
    for y_key in y_keys:
        chartdata.append(
            go.Bar(
                x=data.values_list(x_key, flat=True),
                y=data.values_list(y_keys, flat=True),
                name=y_key,
            )
        )
    return chartdata

def build_scatter_chart(grouped_data, x_key='x', y_key='y'):
    '''
    This is used for building chart scatter
    '''
    from django.contrib.admin.utils import flatten

    chartdata = []
    x = []
    y = []

    # # Plot data grouped by category or product
    # for key, value in grouped_data.items():
    #     chartdata.append(
    #         go.Scatter(
    #             x=value[x_key],
    #             y=value[y_key],
    #             name=key,
    #             mode='markers',
    #         ),
    #     )

    for _, value in grouped_data.items():
        x.append(value[x_key])
        y.append(value[y_key])

    chartdata.append(
        go.Scatter(
            x=flatten(x),
            y=flatten(y),
            # name=key,
            mode='markers',
        ),
    )
    return chartdata

def count_dio_average_by_level_range(queryset, dio_level_low, dio_level_high, category_key, x_key):
    '''
    If we suppose:
    dio_level_low = 2
    dio_level_high = 4
    category_key = 'product__reference'
    x_key = 'avg_dio'

    dio_occurence_lev1 = count(avg_dio) where (0 <= avg_dio < dio_level_low)
    dio_occurence_lev2 = count(avg_dio) where (dio_level_low <= avg_dio < dio_level_high)
    dio_occurence_lev3 = count(avg_dio) where (dio_level_high <= avg_dio)

    input = queryset([
        {'product__reference': 'Sku001', 'avg_dio': 1},
        {'product__reference': 'Sku001', 'avg_dio': 4},
        {'product__reference': 'Sku001', 'avg_dio': 3},
        {'product__reference': 'Sku002', 'avg_dio': 0.5},
        {'product__reference': 'Sku002', 'avg_dio': 8},
    ])

    output = dict({
        'Sku001': [
            {'dio_occurence_lev1': 1},
            {'dio_occurence_lev2': 1},
            {'dio_occurence_lev3': 1},
        ],
        'Sku002': [
            {'dio_occurence_lev1': 1},
            {'dio_occurence_lev2': 0},
            {'dio_occurence_lev3': 1},
        ]
    })
    '''
    lev1 = 'dio_occurence_lev1'
    lev2 = 'dio_occurence_lev2'
    lev3 = 'dio_occurence_lev3'
    none = 'dio_occurence_none'
    # Sort queryset data by category key
    dataset = sorted(list(queryset), key=itemgetter(category_key))
    # print(queryset) # NOTE : print alerte
    # Return data grouped by category
    dataset_grouped_by_category = {}
    for key, value in itertools.groupby(dataset, key=itemgetter(category_key)):
        dataset_grouped_by_category[key] = {}
        dataset_grouped_by_category[key][lev1] = 0
        dataset_grouped_by_category[key][lev2] = 0
        dataset_grouped_by_category[key][lev3] = 0
        dataset_grouped_by_category[key][none] = 0
        for v in value:
            if (float(v.get(x_key)) >= 0) and (float(v.get(x_key)) < float(dio_level_low)):
                dataset_grouped_by_category[key][lev1] += 1
            elif (float(v.get(x_key)) >= float(dio_level_low)) and (float(v.get(x_key)) < float(dio_level_high)):
                dataset_grouped_by_category[key][lev2] += 1
            elif (float(v.get(x_key)) >= float(dio_level_high)):
                dataset_grouped_by_category[key][lev3] += 1
            else:
                dataset_grouped_by_category[key][none] += 1
    # print(dataset_grouped_by_category) # NOTE  print() alert
    return dataset_grouped_by_category

def build_dio_chart_series(grouped_data, dio_level_low, dio_level_high):
    '''
    Build data to plot in chart
    input = grouped data returned by function `group_data_by_category`
    output = data ready for the chart
    '''
    chartdata = []
    X_cat = []
    Y_lev1 = []
    Y_lev2 = []
    Y_lev3 = []
    for key, value in grouped_data.items():
        X_cat.append(key)
        Y_lev1.append(value['dio_occurence_lev1'])
        Y_lev2.append(value['dio_occurence_lev2'])
        Y_lev3.append(value['dio_occurence_lev3'])

    chartdata = [
        go.Bar(
            x=X_cat,
            y=Y_lev1,
            name='DIO less than {} (days)'.format(dio_level_low),
        ),
        go.Bar(
            x=X_cat,
            y=Y_lev2,
            name='DIO between [{}, {}[ (days)'.format(
                dio_level_low, dio_level_high),
        ),
        go.Bar(
            x=X_cat,
            y=Y_lev3,
            name='DIO more than {} (days)'.format(dio_level_high),
        ),
    ]
    return chartdata

##############################################
# Updated version
##############################################

def get_show_by_attribute(show_by):
    if show_by == 'quantity':
        return
    elif show_by == 'package':
        return 'stock__product__package'
    elif show_by == 'pallet':
        return 'stock__product__pallet'
    elif show_by == 'cost':
        return 'stock__product__cost'
    elif show_by == 'weight':
        return 'stock__product__weight'
    elif show_by == 'volume':
        return 'stock__product__volume'


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import django
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "DTS.settings")
django.setup()

from interface.models import Module, ModuleField

"""
多次申请核查V2导字段
"""

# ApplyLoanStr
_module = Module.objects.get(en_name='ApplyLoanStr')

ModuleField.objects.filter(module=_module).delete()

fd = open('ApplyLoanStr.csv')

for line in fd:
    line = line.strip().split(',')
    ModuleField.objects.create(module=_module, en=line[1], cn=line[2])

fd.close()


#_*_ coding: utf-8 _*_
#Continue - располагается внутри блока инструкций цикла
#           служит для пропуска одной из итераций(одно полное
#           исполнение инструкции внутри цикла)
#range()  - функция генерирующая последовательность
#           range(4) = 0,1,2,3.

for i in range( 1, 4 ) :
    for j in range( 1, 4 ) :
        if i == 2 and j == 1 :
            print( 'Continues inner loop at i = 1 j = 1' )
            continue
        print('Running i =', i, 'j = ', j)


import sys
import re

def cleanInput(corpus):
    # Replace all none alphanumeric characters with spaces
    corpus = re.sub(r'[^a-zA-Z0-9\s]', ' ', corpus)
    corpus = re.sub('\n+', " ", corpus)
    return corpus

def getNgrams(N):
    corpus = '''
    Mary had a little lamb its fleece was white as snow;
    And everywhere that Mary went, the lamb was sure to go.
    It followed her to school one day, which was against the rule;
    It made the children laugh and play, to see a lamb at school.
    And so the teacher turned it out, but still it lingered near,
    And waited patiently about till Mary did appear.
    "Why does the lamb love Mary so?" the eager children cry;"Why, Mary loves the lamb, you know" the teacher did reply."
    '''

    #corpus = 'ONE TWO ONE TWO THREE TWO THREE'

    corpus = cleanInput(corpus)
    tokens = [token for token in corpus.split(" ") if token != ""]
    grams = [tokens[i:i + N] for i in range(len(tokens) - N + 1)]
    return grams

def calc_probability(ngrams, wp):
    total_occurence = 0
    predicts = {}

    for gram in ngrams:
        prefix = ' '.join(gram[:-1])

        if wp == prefix:
            choice = gram[-1]
            total_occurence += 1
            predicts[choice] = predicts.get(choice, 0) + 1

    predicts = {k: v / total_occurence for k, v in predicts.items()}
    sorted_order = sorted(predicts.items(), key=lambda t: (-t[1], t[0]))
    return sorted_order

for line in sys.stdin:
    #print(line, end="")
    line = line.split(',')
    N, word = int(line[0]), line[1].rstrip().lstrip()
    ngrams = getNgrams(N)
    predictions = calc_probability(ngrams, word)

    sorted_order = calc_probability(ngrams, word)

    for key in sorted_order:
        print("{},{:.3f}".format(key[0], key[1]), end=";", flush=True)


import matplotlib.pyplot as plt
import matplotlib.cm as cm
import matplotlib as mpl
import os.path
import seaborn as sns
import pandas as pd
import numpy as np
import pynbody

@pynbody.derived_array
#def rhoOVI(f):
#    ovi = pynbody.analysis.ionfrac.calculate(f,ion='ovi',mode='new')
#    return f.gas['rho']*ovi*f.gas['OxMassFrac']  # Original from Pontzen

def N_OVI(f):
    ovi = pynbody.analysis.ionfrac.calculate(f,ion='ovi',mode='new')
    m_p = 1.6726 * 10**-24 #g
    #print(ovi)
    #print(f.gas['OxMassFrac'])
    #print(f.gas['rho'].in_units('g cm**-3')*ovi*f.gas['OxMassFrac']/(16*m_p))
    return f.gas['rho'].in_units('g cm**-3')*ovi*f.gas['OxMassFrac']/(16*m_p)

# Just using k = 1 and k = 2, for GM1 & GM4 for now
k = 2
## MOVED FILES FROM FABIO TO ALYSON BROOKS: /nobackupp8/ambrook2/fgoverna_pleiades_p8_files
sim = ['/nobackupp8/ambrook2/fgoverna_pleiades_p8_files/pioneer50h243.1536g1bwK1BH/pioneer50h243.1536gst1bwK1BH.00','/nobackupp8/ambrook2/fgoverna_pleiades_p8_files/pioneer50h243GM1.1536gs1bwK1BH/pioneer50h243GM1.1536gst1bwK1BH.00','/nobackupp8/ambrook2/fgoverna_pleiades_p8_files/pioneer50h243GM4.1536gst1bwK1BH/pioneer50h243GM4.1536gst1bwK1BH.00','/nobackup/nnsanche/pioneer50h243GM5.1536gst1bwK1BH/pioneer50h243GM5.1536gst1bwK1BH.00','/nobackupp8/fgoverna/pioneer50h243GM6.1536gst1bwK1BH/pioneer50h243GM6.1536gst1bwK1BH.00','/nobackup/nnsanche/pioneer50h243GM7.1536gst1bwK1BH/pioneer50h243GM7.1536gst1bwK1BH.004096']
labels = ['P0','GM1','GM4','GM5','GM6','GM7']
colors = sns.cubehelix_palette(8)
Z_sun = 0.0142 # (Asplund 2009; https://arxiv.org/pdf/0909.0948.pdf)
print('LOADING SIM:',labels[k])

ts = np.loadtxt('../'+labels[k]+'/timesteps.txt',dtype=str)
for t in range(len(ts)):
#    t = len(ts)-1
    print('Loading sim:',sim[k],' at timestep:',ts[t])
    if os.path.isfile(labels[k]+'_ionizb_'+ts[t]+'_xdata.np'):
        continue
        print('This plot has already been made. Skipping this snapshot.x')

    ######################
    # READ IN SIMULATION #
    ######################
    f = pynbody.load(sim[k]+ts[t])
    pynbody.analysis.halo.center(f.star)
    f.physical_units()
    h = f.halos()
    h1 = h[1]
    pynbody.analysis.angmom.faceon(h1)

    ###################
    # ISOLATE CGM GAS #
    ###################
    # Isolate and remove disk stars within radius 0-10 kpc & vertically 10 kpc
    r_max = 10  # kpc
    twenty_kpc_incm = 6.171*(10**22)
    #z_max = 10 #4 # kpc

    Rg_d = ((h1.g['x'].in_units('kpc'))**2. + (h1.g['y'].in_units('kpc'))**2. + (h1.g['z'].in_units('kpc'))**2.)**(0.5)
    disk_gas_xyzmax =  (Rg_d < r_max)
    #disk_gas_zmax  = (h1.g['z'].in_units('kpc') < z_max) & (h1.g['z'].in_units('kpc') > -z_max)
    disk_gas_mask = disk_gas_xyzmax #& disk_gas_zmax
    disk_gas = h1.g[disk_gas_mask] #& disk_gas_zmax]
    CGM_gas  = h1.g[~disk_gas_mask]

    CGM_temp = np.array(CGM_gas['temp'])
    transit_temp = 10**5.2
    CGM_gas = CGM_gas[CGM_temp < transit_temp]
    print('LOOKING AT CGM WITH TEMP < 10^5.2')

    #########################
    # CALCULATE OVI DENSITY #
    #########################
    # Ionization fraction of OVI compare to total Oxygen
    ovi = pynbody.analysis.ionfrac.calculate(CGM_gas,ion='ovi',mode='new')
    m_p = 1.6726 * 10**-24 #g

    # OVI density = total CGM gas density * fraction of oxygen * fraction of OVI / mass of oxygen
    OVI = CGM_gas['rho'].in_units('g cm**-3')*ovi*CGM_gas['OxMassFrac']/(16*m_p)
    print('Total mass in CGM:', np.sum(CGM_gas['mass']))
    if np.sum(CGM_gas['mass']) == 0.0:
        continue
    print('Total mass in metals:',np.sum(CGM_gas['mass']*CGM_gas['metals'])) # 'metals' *IS* metallicity
    print('Total mass Oxygen in CGM:', np.sum(CGM_gas['OxMassFrac']*CGM_gas['mass']),CGM_gas['mass'].units)
    print('Total mass in OVI in CGM:', np.sum(CGM_gas['OxMassFrac']*CGM_gas['mass']*ovi))
    #print('OVI Density: ',OVI,OVI.units)
    print('OVI fractions:',np.average(ovi))

    #############################################################
    # DIVIDE PARTICLES INTO SHELLS & CALCULATE COLUMN DENSITIES #
    #############################################################
    # In shells of 10 kpc, take an average density and line of sight L for each shell
    shell_bounds = np.arange(0,275,10)
    CGM_r = (CGM_gas['x']**2 + CGM_gas['y']**2)**(0.5)
    R_vir = int(np.max(CGM_r))
    print('R_vir',int(np.max(CGM_r)),CGM_gas['x'].units)

    #fig = plt.figure(figsize=(8, 8))
    #ax1 = fig.add_subplot(221)
    #ax2 = fig.add_subplot(222)
    CGM_Novi = []
    pathlength = []
    for i in range(len(shell_bounds)-1):
        shell = CGM_gas[(np.abs(CGM_r) > shell_bounds[i]) & (np.abs(CGM_r) < shell_bounds[i+1])]
        shell_ovi_frac = ovi[(np.abs(CGM_r) > shell_bounds[i]) & (np.abs(CGM_r) < shell_bounds[i+1])]
        shell_OVI_rho = shell['rho'].in_units('g cm**-3')*shell_ovi_frac*shell['OxMassFrac']/(16*m_p)
        #    pynbody.plot.sph.image(shell,qty="rho",units="g cm^-3",width=300,z_camera=)

        # Test some stuff to make sure I'm doing my path length measurements right
        shell_x = np.array(shell['x'])
        shell_y = np.array(shell['y'])
        shell_z = np.array(shell['z'])

        if len(shell_x) == 0:
            break

        #ax1.plot(shell_x,shell_y,'.')#color=sns.cubehelix_palette(8)[i])
        j = 1 - i/26.
        #print(j)
        #ax2.plot(shell_x,shell_z,'.',alpha=j)
        #print(np.min(shell_z),np.max(shell_z))

        avg_shell_OVI_rho = np.average(shell_OVI_rho)
        #print(avg_shell_OVI_rho,shell_OVI_rho.units)

        shell_z = np.max(shell['z'].in_units('cm')) - np.min(shell['z'].in_units('cm'))

        if i <= 1 :
            CGM_Novi.append(avg_shell_OVI_rho*(shell_z - twenty_kpc_incm))
            print('Remove 20 kpc in z because of empty center within 10 kpc radius')
        else:
            CGM_Novi.append(avg_shell_OVI_rho*shell_z) # Using shell_z to underestimate gas
            pathlength.append(shell_z/(3.086*10**21))

    #plt.show()
    #print(CGM_Novi)

    b_impact = shell_bounds + 5
    #print(b_impact)

    pathlength_kpc = pathlength #kpc

    #plt.plot(shell_bounds[:-3],pathlength_kpc)
    #plt.show()
    #quit()

    #print(float(ts[t]))
    #if (float(ts[t]) >= 2816.):
        #COS = pd.read_csv('COShalo_obs.txt',header=0,delim_whitespace=True,comment='#',index_col=False)
        #COS_ID = COS['ID']
        #COS_OVI = COS['LogN0VI']
        #COS_Rkpc = COS['Rho(kpc)'][COS_OVI != 0]
        #COS_OVI = COS_OVI[COS_OVI != 0]
        #print(COS)
        #plt.plot(COS_Rkpc[COS['RED?'] == 'yes'],COS_OVI[COS['RED?'] == 'yes'],marker='.',color='Red',linestyle=' ',label='COS Ellipticals')
        #plt.plot(COS_Rkpc[COS['RED?'] == 'no'],COS_OVI[COS['RED?'] == 'no'],marker='.',color='DodgerBlue',linestyle=' ',label='COS Spirals')

    np.savetxt(labels[k]+'_ionizb_'+ts[t]+'_xdata.np',shell_bounds[0:len(CGM_Novi)])
    np.savetxt(labels[k]+'_ionizNovi_'+ts[t]+'_ydata.np',CGM_Novi)
#    plt.plot(shell_bounds[0:len(CGM_Novi)],np.log10(CGM_Novi),marker='.',label=labels[k])
#    plt.ylabel(r'N$_{OVI}$ [cm$^{-2}$]')
#    plt.xlabel(r'$r$ [kpc]')
#    plt.ylim(13,16)
#    plt.xlim(-10,270)
#    plt.text(-5,15.75,'Collisional OVI (T > 10^${5.2}$) at '+str(labels[k]))
#    plt.legend()
#    plt.savefig(labels[k]+'_collNOVI_b_'+ts[t]+'.pdf')
#    plt.show()
#    quit()

quit()
##########################################
# PLOT COLUMN DENSITY AS FUNCTION OF X,Y #
##########################################
# From Pontzen's code
pynbody.plot.sph.image(CGM_gas,qty='N_OVI',width='500 kpc',cmap='magma',show_cbar=False,units="g cm^-2",vmin=1e13,vmax=1e15)

plt.text(200,200,str(labels[i]),color='White')
plt.colorbar(label=r'N$_{OVI}$ cm$^{-2}$')
#plt.savefig(str(labels[i])+'_OVI.pdf')
plt.show()


from core.advbase import *

def module():
    return Nadine

class Nadine(Adv):
    conf = {}
    conf['slots.a'] = [
    'Dragon_and_Tamer',
    'Flash_of_Genius',
    'Me_and_My_Bestie',
    'Chariot_Drift',
    'Dueling_Dancers'
    ]
    conf['acl'] = """
        `dragon, s=1
        `s3, not buff(s3)
        `s2
        `s1
        `s4
        `fs, x=5
        """
    conf['coabs'] = ['Blade', 'Wand', 'Serena']
    conf['share'] = ['Gala_Mym']

    def prerun(self):
        self.team_s1_hits = 1
        teammates = 2
        if self.condition(f'{teammates} teammates in s1'):
            self.team_s1_hits += teammates

    @staticmethod
    def prerun_skillshare(adv, dst):
        adv.team_s1_hits = 1
        teammates = 2
        if adv.condition(f'{teammates} teammates in s1'):
            adv.team_s1_hits += teammates

    def s1_proc(self, e):
        for _ in range(self.team_s1_hits):
            self.add_combo(e.name)
        aseq = 1 if e.group == 'default' else 3 + 1
        s1_hits = 1 if e.group == 'default' else 3 + self.team_s1_hits
        log('debug', 's1_hits', s1_hits, self.team_s1_hits)
        if s1_hits <= 3:
            self.hitattr_make(e.name, e.base, e.group, aseq, self.conf[e.name].extra_3)
            self.energy.add(1)
            return
        if s1_hits <= 5:
            self.hitattr_make(e.name, e.base, e.group, aseq, self.conf[e.name].extra_5)
            self.energy.add(3)
            return
        if s1_hits >= 6:
            self.hitattr_make(e.name, e.base, e.group, aseq, self.conf[e.name].extra_6)

            self.energy.add(5)
            return

if __name__ == '__main__':
    from core.simulate import test_with_argv
    test_with_argv(None, *sys.argv)


#link : = https://practice.geeksforgeeks.org/problems/overlapping-intervals/0
t=int(input())
for q in range(t):
    n=int(input())
    ar=[]
    ti=[int(b) for b in input().split()]
    for i in range(n):
        jk=[]
        jk.append(ti[2*i])
        jk.append(ti[2*i+1])
        ar.append(jk)
    ar.sort()
    m=[]
    s=-10000
    max= -10000
    for i in range(n):
        a=ar[i]
        if a[0]>max:
            if i!=0 :
                m.append([s,max])
            max=a[1]
            s=a[0]
        else:
            if a[1]>=max:
                max=a[1]
    if max!= -10000 and [s,max] not in m:
        m.append([s,max])
    for i in m:
        print(i[0],i[1],end=" ")
    print()


"""Library for running Bruker image ripping utility."""

import atexit
import glob
import logging
import os
import pathlib
import subprocess
import time

logger = logging.getLogger(__name__)

# Ripping process does not end cleanly, so the filesystem is polled to detect the
# processing finishing.  The following variables relate to the timing of that polling
# process.
RIP_TOTAL_WAIT_SECS = 3600  # Total time to wait for ripping before killing it.
RIP_EXTRA_WAIT_SECS = 10  # Extra time to wait after ripping is detected to be done.
RIP_POLL_SECS = 10  # Time to wait between polling the filesystem.

class RippingError(Exception):
    """Error raised if problems encountered during data conversion."""

def raw_to_tiff(dirname, ripper):
    """Convert Bruker RAW files to TIFF files using ripping utility specified with `ripper`."""
    def get_filelists():
        return set(dirname.glob('*Filelist.txt'))

    def get_rawdata():
        return set(dirname.glob('*RAWDATA*'))

    def get_tiffs():
        return set(dirname.glob('*.ome.tif'))

    filelists = get_filelists()
    if not filelists:
        raise RippingError('No *Filelist.txt files present in %s' % dirname)

    rawdata = get_rawdata()
    if not rawdata:
        raise RippingError('No RAWDATA files present in %s' % dirname)

    tiffs = get_tiffs()
    if tiffs:
        raise RippingError('Cannot rip because tiffs already exist in %s (%d found)' % (dirname, len(tiffs)))

    logger.info('Ripping from:\n %s\n %s', '\n '.join([str(f) for f in filelists]),
                '\n '.join([str(f) for f in rawdata]))

    # Normally, the fname is passed to -AddRawFile.  But there is a bug in the software, so
    # we have to pop up one level and use -AddRawFileWithSubFolders.
    cmd = [
        ripper,
        '-IncludeSubFolders',
        '-AddRawFileWithSubFolders',
        str(dirname),
        '-SetOutputDirectory',
        str(dirname),
        '-Convert',
    ]

    # Run a subprocess to execute the ripping.  Note this is non-blocking because the
    # ripper never exists.  (If we blocked waiting for it, we'd wait forever.)  Instead,
    # we wait for the input files to be consumed and/or output files to be finished.
    process = subprocess.Popen(cmd)

    # Register a cleanup function that will kill the ripping subprocess.  This handles the cases
    # where someone hits Cntrl-C, or the main program exits for some other reason.  Without
    # this cleanup function, the subprocess will just continue running in the background.
    def cleanup():
        timeout_sec = 5
        p_sec = 0
        for _ in range(timeout_sec):
            if process.poll() == None:
                time.sleep(1)
                p_sec += 1
        if p_sec >= timeout_sec:
            process.kill()
        logger.info('cleaned up!')

    atexit.register(cleanup)

    # Wait for the file list and raw data to disappear
    remaining_sec = RIP_TOTAL_WAIT_SECS
    last_tiffs = {}
    while remaining_sec >= 0:
        logging.info('Waiting for ripper to finish: %d seconds remaining', remaining_sec)
        remaining_sec -= RIP_POLL_SECS
        time.sleep(RIP_POLL_SECS)

        filelists = get_filelists()
        rawdata = get_rawdata()

        tiffs = get_tiffs()
        tiffs_changed = (last_tiffs != tiffs)
        last_tiffs = tiffs

        logging.info('  Found filelist files: %s', filelists)
        logging.info('  Found rawdata files: %s', rawdata)
        logging.info('  Found this many tiff files: %s', len(tiffs))

        if not filelists and not rawdata and not tiffs_changed:
            logging.info('Detected ripping is complete')
            time.sleep(RIP_EXTRA_WAIT_SECS)  # Wait before terminating ripper, just to be safe.
            logging.info('Killing ripper')
            process.kill()
            logging.info('Ripper has been killed')
            return

    raise RippingError('Killed ripper because it did not finish within %s seconds' % RIP_TOTAL_WAIT_SECS)


from path import path
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler

from onitu.api import Plug

plug = Plug()

# Ignore the next Watchdog event concerning those files
events_to_ignore = set()
# Store the mtime of the last write of each transfered file
last_mtime = {}

root = None

@plug.handler()
def get_chunk(filename, offset, size):
    filename = root.joinpath(filename)

    with open(filename, 'rb') as f:
        f.seek(offset)
        return f.read(size)

@plug.handler()
def start_upload(metadata):
    filename = root.joinpath(metadata.filename)

    # We ignore the next Watchdog events concerning this file
    events_to_ignore.add(metadata.filename)

    if not filename.exists():
        filename.dirname().makedirs_p()

    filename.open('w+b').close()

@plug.handler()
def end_upload(metadata):
    filename = root.joinpath(metadata.filename)

    # If this is the last chunk we stop ignoring event
    events_to_ignore.remove(metadata.filename)
    # this is to make sure that no further event concerning
    # this set of writes will be propagated to the Referee
    last_mtime[metadata.filename] = filename.mtime

    metadata.revision = filename.mtime
    metadata.write_revision()

@plug.handler()
def upload_chunk(filename, offset, chunk):
    abs_path = root.joinpath(filename)

    # We make sure events are ignored for this file
    events_to_ignore.add(filename)

    with open(abs_path, 'r+b') as f:
        f.seek(offset)
        f.write(chunk)

def check_changes():
    for abs_path in root.walkfiles():
        filename = abs_path.relpath(root).normpath()

        metadata = plug.get_metadata(filename)
        revision = metadata.revision
        revision = float(revision) if revision else .0

        if abs_path.mtime > revision:
            update_file(metadata, abs_path)

def update_file(metadata, path):
    metadata.size = path.size
    metadata.revision = path.mtime
    plug.update_file(metadata)

class EventHandler(FileSystemEventHandler):
    def on_moved(self, event):
        def handle_move(event):
            if event.is_directory:
                return

            #if event.src_path:
                #self._handle_deletion(event.src_path)
            self._handle_update(event.dest_path)

        handle_move(event)
        if event.is_directory:
            for subevent in event.sub_moved_events():
                handle_move(subevent)

    def on_modified(self, event):
        if event.is_directory:
            return

        self._handle_update(event.src_path)

    def _handle_update(self, abs_path):
        abs_path = path(abs_path)
        filename = root.relpathto(abs_path)

        if filename in events_to_ignore:
            return

        if filename in last_mtime:
            if last_mtime[filename] >= abs_path.mtime:
                # This event concerns a file that hasn't been changed
                # since the last write_chunk, we must ignore the event
                return
            else:
                del last_mtime[filename]

        metadata = plug.get_metadata(filename)
        update_file(metadata, abs_path)

def start(*args, **kwargs):
    plug.start(*args, **kwargs)

    global root
    root = path(plug.options['root'])

    check_changes()

    observer = Observer()
    observer.schedule(EventHandler(), path=root, recursive=True)
    observer.start()

    plug.wait()