import time
import os
from enum import Enum
from time import gmtime, strftime
from colorama import Fore, Back, init
import random
import sys
import math
import json
import sqlite3
import platform
import hmac
import hashlib
import base64
import string
import binascii
import re
import traceback
import threading
import elvex_module
import inspect
from socket import gethostname
import psutil
import prompt_toolkit
from time import gmtime, mktime
try:
    import readline
except ImportError:
    import pyreadline as readline
import npyscreen
try:
	import subprocess
except:
	subprocessUndefined = True
	# that's fine.
db_AdditionalsVer = 2
db_usersVer = 1
version = 4

oprint = print
oopen = open
PromoteFromWeb = True

def TempErrHook(exctype, val, tb):
	print('[{}] {} ({})'.format(exctype, val, tb))

sys.excepthook = TempErrHook # TEMP UNTIL MODULE FINISHED LOADING

def open(fn, t = "r"):
	global oopen
	Openned = False
	while not(Openned):
		try:
			fl = oopen(fn, t)
			Openned = True
		except IOError:
			Logger("Open of "+fn+" was failed. Trying again.", CT.ERROR)
			continue
	return fl

def Void():
	return
try:
	ohelp = help
except Exception:
	Void()

def clear():
	"""Clear the output."""
	if(platform.system() == "Windows"): os.system("cls")
	else: os.system("clear")

NotList = ["is_valid_command", "CompleterLoad","Void","Enum", "gmtime", "strftime", "init", "ohelp", "CT", "oprint", "round50", "create_self_signed_cert","completer","v", "dbgSwitchErrorDisplay", "dbgNoDebug"]
def help():
	"""List of all functions."""
	global NotList
	if not(isDebugger):
		return
	cmds = 0
	for name, val in elvex_module.__dict__.items():
		if callable(val) and name not in NotList:
			try:
				args = inspect.getfullargspec(val)[0]
			except Exception:
				continue
			args = ', '.join(args)
			if(name.startswith("nolist_")):
				continue
			if(name.startswith("dbg")):
				name = Fore.CYAN + name + Fore.RESET
			cmds += 1
			if(str(val.__doc__) == "None"):
				print(name + "(" + args + ") - No description.", CT.INFO)
			else:
				print(name + "(" + args + ") - " + val.__doc__, CT.INFO)
	if(cmds > 0):
		print("There is "+Fore.GREEN+str(cmds)+Fore.RESET + " functions in total.")
	else:
		print("There is "+Fore.RED+str(cmds)+Fore.RESET + " functions in total.")

StartTime = time.time()

isDebugger = False

true = True
false = False

def synchronized(func):

    func.__lock__ = threading.Lock()

    def synced_func(*args, **kws):
        with func.__lock__:
            return func(*args, **kws)

    return synced_func

def round50(n):
    return round(n * 2, -2)

init(autoreset=true)

if not(os.path.isfile("current.log")):
	with open("current.log", "w") as w:
		w.write("")

with open("current.log", "w") as w:
	w.write("")

try:
	os.mkdir("log")
except FileExistsError:
	idc = "i dont care"
	del idc

if not(os.path.isfile("log/unix"+str(StartTime)+".log")):
	with open("log/unix"+str(StartTime)+".log", "w") as w:
		w.write("")

with open("log/unix"+str(StartTime)+".log", "w") as w:
	w.write("")

class CT(Enum):
	WARN = 1
	INFO = 2
	ERROR = 3
	NONE = 4

def EncodedString(stri):
	"""Encodes string."""
	return str.encode(stri)

def eprint(t):
	"""Prints with newline."""
	sys.stdout.write(t+'\n')

def Logger(stri,type = CT.NONE):
	"""Add message to log."""
	global StartTime
	if(type == CT.NONE):
		with open("current.log", "a") as w:
			w.write("[NONE]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		with open("log/unix"+str(StartTime)+".log", "a") as w:
			w.write("[NONE]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		return
	if(type == CT.WARN):
		with open("current.log", "a") as w:
			w.write("[WARN]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		with open("log/unix"+str(StartTime)+".log", "a") as w:
			w.write("[WARN]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		return
	if(type == CT.INFO):
		with open("current.log", "a") as w:
			w.write("[INFO]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		with open("log/unix"+str(StartTime)+".log", "a") as w:
			w.write("[INFO]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		return
	if(type == CT.ERROR):
		with open("current.log", "a") as w:
			w.write("[ERROR]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		with open("log/unix"+str(StartTime)+".log", "a") as w:
			w.write("[ERROR]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		return

# "unix"+str(StartTime)+".log"

def is_json(myjson):
  """Check if string is json."""
  try:
    json_object = json.loads(myjson)
  except ValueError as e:
    return False
  return True

def LogPrint(stri,type = CT.NONE):
	"""Print and add to log."""
	global StartTime
	stri = str(stri)
	if(type == CT.NONE):
		eprint("[NONE]" + Back.WHITE+Fore.BLACK + "["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"]"+Back.RESET+Fore.RESET+" "+stri)
		with open("current.log", "a") as w:
			w.write("[NONE]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		with open("log/unix"+str(StartTime)+".log", "a") as w:
			w.write("[NONE]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		return
	if(type == CT.WARN):
		eprint(Fore.YELLOW+"[WARN]"+Back.WHITE+Fore.BLACK+"["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"]"+Back.RESET+Fore.RESET+" "+stri)
		with open("current.log", "a") as w:
			w.write("[WARN]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		with open("log/unix"+str(StartTime)+".log", "a") as w:
			w.write("[WARN]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		return
	if(type == CT.INFO):
		eprint(Fore.CYAN+"[INFO]"+Back.WHITE+Fore.BLACK+"["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"]"+Back.RESET+Fore.RESET+" "+stri)
		with open("current.log", "a") as w:
			w.write("[INFO]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		with open("log/unix"+str(StartTime)+".log", "a") as w:
			w.write("[INFO]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		return
	if(type == CT.ERROR):
		eprint(Fore.RED+"[ERROR]"+Back.WHITE+Fore.BLACK+"["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"]"+Back.RESET+Fore.RESET+" "+stri)
		with open("current.log", "a") as w:
			w.write("[ERROR]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		with open("log/unix"+str(StartTime)+".log", "a") as w:
			w.write("[ERROR]["+strftime("%Y-%m-%d %H:%M:%S", gmtime())+"] "+stri+"\n")
		return

class print():
	"""Print a message."""
	def __init__(self,msg,typeo = CT.NONE):
		LogPrint(msg,typeo)

def IsUserExists(login):
	"""Is user exists in Elvex DB?"""
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	r = c.execute("SELECT * FROM users WHERE lower(username) = '{}'".format(login.lower()))
	if(r.fetchone()):
		return True
	else:
		return False

def EStr(stre):
	"""Encode string to password."""
	a = hmac.new(b'_EaLEoELXoELWoXLOWQlWA_1+-2#)LC<E!!!(!0CC@@@@A', stre.encode(), hashlib.sha256)
	return str(a.hexdigest())

def AddUser(login, pswd, avatar = 0, electricity = 0, ppcount = 0.0, inventory = "[]", customization = '{"droidColor": "blue", "lampColor": "blue","hat": "none", "body": "none", "hands": "none", "legs": "none"}', bio = "Not specified.", stats = "{}", banned = False, regip = "0.0.0.0", accessible = True, ban_reason = -1, badges = "[]"):
	"""Create user in Elvex DB."""
	if(IsUserExists(login)):
		print("Creation user with username "+login+" failed. User already exists.", CT.ERROR)
		return "USER_EXISTS"
	if(len(login) < 4):
		return "USER_LEN"
	if(len(pswd) < 6):
		return "PSWD_LEN"
	if(login.isspace() or login == ""):
		return "USER_SPACE"
	pswd = EStr(pswd)
	if(re.search('[^a-zA-Z0-9_]', login) is not None):
		return "USER_CHARACTERS"
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	r = c.execute("SELECT * FROM bannednames WHERE lower(name) = '{}'".format(login.lower()))
	if(r.fetchone()):
		return "NAME_BANNED"
	#        TEXT, TEXT, INT, FLOAT, TEXT, TEXT, TEXT, TEXT, BOOL, TEXT
	c.execute("INSERT INTO users VALUES ('{}', '{}', {}, {}, {}, '{}', '{}', '{}', '{}', {}, '{}', {}, {}, '{}')".format(login, pswd, str(avatar), str(electricity), str(ppcount),inventory, customization,bio, stats, int(banned), regip, int(accessible), str(ban_reason), badges))
	conn.commit()
	conn.close()
	return "OK"

def RemoveUser(login):
	"""Remove user from Elvex DB."""
	if not (IsUserExists(login)):
		print("Removal failed. User doesn't exists. ("+login+")", CT.ERROR)
		return "USER_GONE"
	if(login.isspace()):
		return "USER_SPACE"
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	c.execute("UPDATE users SET accessible = false WHERE lower(username) = '{}'".format(login.lower()))
	conn.commit()
	conn.close()
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	c.execute("INSERT INTO bannednames VALUES ('{}')".format(login.lower()))
	conn.commit()
	conn.close()
	return "OK"

def RestoreUser(login):
	"""Restores user profile."""
	if not (IsUserExists(login)):
		print("Restoring failed. User doesn't exists. ("+login+")", CT.ERROR)
		return "USER_GONE"
	if(login.isspace()):
		return "USER_SPACE"
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	c.execute("UPDATE users SET accessible = true WHERE lower(username) = '{}'".format(login.lower()))
	conn.commit()
	conn.close()
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	c.execute("DELETE FROM bannednames WHERE name = '{}'".format(login.lower()))
	conn.commit()
	conn.close()
	return "OK"
def IsAccessibleUser(username):
	"""Is user profile accessible?"""
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	r = c.execute("SELECT accessible FROM users WHERE lower(username) = '{}'".format(username.lower()))
	r = r.fetchone()
	return bool(r[0])

def ListUsers():
	"""List all users."""
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	a = c.execute("SELECT * FROM users")
	for b in a:
		print("- "+b[0])
	conn.close()

def BanUser(login,reason = -1):
	"""Ban player from Elvex."""
	if not (IsUserExists(login)):
		print("Ban failed. User doesn't exists. ("+login+")", CT.ERROR)
		return "USER_GONE"
	if(login.isspace()):
		return "USER_SPACE"
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	c.execute("UPDATE users SET banned = true WHERE lower(username) = '{}'".format(login.lower()))
	c.execute("UPDATE users SET ban_reason = {} WHERE lower(username) = '{}'".format(reason,login.lower()))
	conn.commit()
	conn.close()
	return "OK"

def UnbanUser(login):
	"""Pardon a player."""
	if not (IsUserExists(login)):
		print("Ban failed. User doesn't exists. ("+login+")", CT.ERROR)
		return "USER_GONE"
	if(login.isspace()):
		return "USER_SPACE"
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	c.execute("UPDATE users SET banned = false WHERE lower(username) = '{}'".format(login.lower()))
	conn.commit()
	conn.close()
	return "OK"

flushing = False
flushingTried = 0

def FlushUsers():
	"""(WARNING) Delete all users."""
	global flushing
	global flushingTried
	if(flushing == False):
		print("To flush all users use FlushUsers() again.",CT.WARN)
		flushing = True
		flushingTried = time.time()
		return
	if(flushing == True and flushingTried+30 < time.time()):
		print("To flush all users use FlushUsers() again.",CT.WARN)
		flushing = True
		flushingTried = time.time()
		return
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	c.execute("DELETE FROM users")
	c.execute("DELETE FROM bannednames")
	c.execute('''INSERT INTO users VALUES ('act8', '', 0, 1000000, 0, '[]', '\{\}', '', '\{\}', false, '', true, -1, '["badge_bot"]')''')
	conn.commit()
	conn.close()
	print("Users were flushed.", CT.INFO)
	flushing = False

def IsUserBanned(username):
	"""Check if user is banned or no."""
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	a = c.execute("SELECT banned FROM users WHERE lower(username) = '{}'".format(username.lower()))
	a = bool(a.fetchone()[0])
	conn.close()
	return a

def GetBanReason(username):
	"""Get reason of ban for user."""
	if not IsUserExists(username):
		return "USER_GONE"
	if not IsUserBanned(username):
		return "USER_LAW"
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	a = c.execute("SELECT ban_reason FROM users WHERE lower(username) = '{}'".format(username.lower()))
	a = a.fetchone()[0]
	conn.close()
	return a

def GetUser(username, safe = True):
	"""Get user by name."""
	global isDebugger
	username = username.lower()
	if not (IsUserExists(username)):
		print("Tried to get user, but user with that name doesn't exists. ("+username+")", CT.ERROR)
		return "USER_GONE"
	elif IsUserBanned(username) and not isDebugger:
		return "USER_BANNED"
	elif not IsAccessibleUser(username) and not isDebugger:
		return "USER_GONE"
	if(username.isspace()):
		return "USER_SPACE"
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	if(safe):
		a = c.execute("SELECT username, electricity,avatar, ppcount, inventory, customization, bio, stats, banned, badges FROM users WHERE lower(username) = '{}'".format(username.lower()))
		a = a.fetchone()
	else:
		a = c.execute("SELECT username, passhash, electricity,avatar,ppcount, inventory, customization, bio, stats, banned, regip, badges FROM users WHERE lower(username) = '{}'".format(username.lower()))
		a = a.fetchone()
	return a

def AddRedeemKey(key, type_reedem, code):
	conn = sqlite3.connect("additional.db")
	c = conn.cursor()
	c.execute('INSERT INTO item_keys VALUES ("'+str(key)+'", "'+str(type_reedem)+'", "'+str(code)+'")')
	conn.commit()
	conn.close()
	return True

def isRedeemKey(key):
	conn = sqlite3.connect("additional.db")
	c = conn.cursor()
	r = c.execute('SELECT * FROM item_keys WHERE key = "'+key+'"')
	if(r.fetchone()):
		conn.close()
		return True
	else:
		conn.close()
		return False

def RemoveReedemKey(key):
	if not isRedeemKey(key): return False
	conn = sqlite3.connect("additional.db")
	c = conn.cursor()
	r = c.execute('DELETE FROM item_keys WHERE key = "'+key+'"')
	conn.commit()
	conn.close()
	return True

def UseRedeemKey(user,key):
	if not isRedeemKey(key): return False
	conn = sqlite3.connect("additional.db")
	c = conn.cursor()
	r = c.execute('SELECT * FROM item_keys WHERE key = "'+key+'"')
	r = r.fetchone()
	if(r[1] == "item"):
		AddInvUser(user, r[2])
	elif(r[1] == "badge"):
		triedOK = GiveUserBadge(user, r[2])
		if(type(triedOK) == str):
			return False, triedOK
	else:
		RemoveReedemKey(key)
		return False, "WRONG_KEYTYPE"
	RemoveReedemKey(key)
	return True, r[1], r[2]

def rndstr(stringLength=4):
	letters = string.ascii_uppercase + string.digits
	return ''.join(random.choice(letters) for i in range(stringLength))
def rndostr(stringLength=4):
	letters = string.ascii_uppercase
	return ''.join(random.choice(letters) for i in range(stringLength))

def GenerateRedeemKey(typer, code):
	key = rndstr()+"-"+rndstr(3)+"-"+rndstr()+"-"+rndostr()
	AddRedeemKey(key, typer, code)
	return key

# print(GetUserBalance("test"))

def GetUserBalance(username):
	"""Get user's balance."""
	s = GetUser(username)
	if(type(s) == str):
		return s
	return int(s[1])

def AddInvUser(username,item_code):
	"""Add item to player's inventory."""
	if not (IsUserExists(username)):
		print("Tried to add item to user's inventory, but user with that name doesn't exists. ("+username+")", CT.ERROR)
		return "USER_GONE"
	if(username.isspace()):
		return "USER_SPACE"
	conn = sqlite3.connect("users.db")
	c = conn.cursor()
	a = c.execute("SELECT inventory FROM users WHERE lower(username) = '{}'".format(username.lower()))
	a = json.loads(a.fetchone()[0])
	a.append(item_code)
	a = json.dumps(a)
	c.execute("UPDATE users SET inventory = '{}' WHERE lower(username) = '{}'".format(a,username))
	conn.commit()
	conn.close()
	return "OK"

def RemInvUser(username,index):
	"""Take item from player's inventory."""
	if not (IsUserExists(username)):
		print("Tried to take item from user's inventory, but user with that name doesn't exists. ("+username+")", CT.ERROR)
		return "USER_GONE"
	if(username.isspace()):
		return "USER_SPACE"
	conn = sqlite3.connect("users.db")
	c = conn.cursor()
	a = c.execute("SELECT inventory FROM users WHERE lower(username) = '{}'".format(username.lower()))
	a = json.loads(a.fetchone()[0])
	del a[index]
	a = json.dumps(a)
	c.execute("UPDATE users SET inventory = '{}' WHERE lower(username) = '{}'".format(a,username))
	conn.commit()
	conn.close()
	return "OK"

def SetCustomizationUser(username, n, h):
	"""Set item to customization slot of player."""
	if not (IsUserExists(username)):
		print("Tried to change user's customization, but user with that name doesn't exists. ("+username+")", CT.ERROR)
		return "USER_GONE"
	if(username.isspace()):
		return "USER_SPACE"
	conn = sqlite3.connect("users.db")
	c = conn.cursor()
	a = c.execute("SELECT customization FROM users WHERE lower(username) = '{}'".format(username.lower()))
	a = json.loads(a.fetchone()[0])
	a[n] = h
	a = json.dumps(a)
	c.execute("UPDATE users SET customization = '{}' WHERE lower(username) = '{}'".format(a,username))
	conn.commit()
	conn.close()
	return "OK"

def EditUserPassword(username, currentPass, newPass):
	"""Edit password of user (needs old)."""
	global isDebugger
	if not (IsUserExists(username)):
		print("Tried to change user's password, but user with that name doesn't exists. ("+username+")", CT.ERROR)
		return "USER_GONE"
	elif IsUserBanned(username) and not isDebugger:
		return "USER_BANNED"
	if(username.isspace()):
		return "USER_SPACE"
	conn = sqlite3.connect("users.db")
	c = conn.cursor()
	a = c.execute("SELECT passhash FROM users WHERE lower(username) = '{}'".format(username.lower()))
	a = str(a.fetchone()[0])
	if not(EStr(currentPass) == a):
		return "USER_WRONG_HASH"
	newPass = EStr(newPass)
	c.execute("UPDATE users SET passhash = '{}' WHERE lower(username) = '{}'".format(newPass, username))
	conn.commit()
	conn.close()
	return "OK"

def UnsafeEditUserPassword(username, newPass):
	"""Edit password of user."""
	if not (IsUserExists(username)):
		print("Tried to change user's password, but user with that name doesn't exists. ("+username+")", CT.ERROR)
		return "USER_GONE"
	if(username.isspace()):
		return "USER_SPACE"
	conn = sqlite3.connect("users.db")
	c = conn.cursor()
	a = c.execute("SELECT passhash FROM users WHERE lower(username) = '{}'".format(username.lower()))
	a = str(a.fetchone()[0])
	newPass = EStr(newPass)
	c.execute("UPDATE users SET passhash = '{}' WHERE lower(username) = '{}'".format(newPass, username))
	conn.commit()
	conn.close()
	return "OK"

def ChangeUserStat(username,stat,to):
	"""Change user's statistic."""
	if not (IsUserExists(username)):
		print("Tried to change user's stat, but user with that name doesn't exists. ("+username+")", CT.ERROR)
		return "USER_GONE"
	if(username.isspace()):
		return "USER_SPACE"
	conn = sqlite3.connect("users.db")
	c = conn.cursor()
	a = c.execute("SELECT stats FROM users WHERE lower(username) = '{}'".format(username.lower()))
	a = json.loads(a.fetchone()[0])
	a[stat] = to
	a = json.dumps(a)
	c.execute("UPDATE users SET stats = '{}' WHERE lower(username) = '{}'".format(a,username))
	conn.commit()
	conn.close()
	return "OK"

def EditUser(username, what, how):
	"""Edit user account."""
	global isDebugger
	if not (IsUserExists(username)):
		print("Tried to edit user, but user with that name doesn't exists. ("+username+")", CT.ERROR)
		return "USER_GONE"
	elif IsUserBanned(username) and not isDebugger:
		return "USER_BANNED"
	if(username.isspace()):
		return "USER_SPACE"
	stringlets = ["username", "bio"]
	warners = ["username"]
	if(what == "inventory"):
		print("To change inventory use: AddInvUser() or RemInvUser()")
		return "BAD_REQUEST"
	if(what == "customization"):
		print("To change customization use: SetCustomizationUser()")
		return "BAD_REQUEST"
	if(what == "passhash"):
		print("To change user's password use: EditUserPassword()")
		return "BAD_REQUEST"
	if(what == "stats"):
		print("To change stats use: ChangeUserStat()")
		return "BAD_REQUEST"
	if(what == "banned"):
		print("To change banned state use: BanUser() or UnbanUser()")
		return "BAD_REQUEST"
	if(what == "regip"):
		print("Regip cannot be changed.")
		return "BAD_REQUEST"

	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	if(what in stringlets):
		c.execute("UPDATE users SET {} = '{}' WHERE lower(username) = '{}'".format(what,how,username))
	else:
		c.execute("UPDATE users SET {} = {} WHERE lower(username) = '{}'".format(what,how,username))
	if(what in warners):
		print(username+" just changed "+what+" to "+how,CT.WARN)
	conn.commit()
	conn.close()
	return "OK"

def CreateCrate(item_code, loot_table):
	"""Creates a new crate."""
	conn = sqlite3.connect('additional.db')
	c = conn.cursor()
	if not (is_json(loot_table)):
		return "LOOTTABLE_NOT_JSON"
	c.execute("INSERT INTO crates VALUES ('{}', '{}')".format(item_code,loot_table))
	conn.commit()
	conn.close()
	return "OK"

def isCrate(item_code):
	"""Checks if item is a crate or no."""
	conn = sqlite3.connect('additional.db')
	c = conn.cursor()
	a = c.execute("SELECT * FROM crates WHERE item_code = '{}'".format(item_code))

	if(a.fetchone()):
		conn.commit()
		conn.close()
		return True
	else:
		conn.commit()
		conn.close()
		return False

def About():
	"""Get information about ELVEX SOCIAL"""
	global isDebugger
	global version
	if not isDebugger:
		return "FAILED"
	print('Elvex SOCIAL v'+str(version))

def GetRandomPoolItem():
	"""Select random item out of shop pool."""
	conn = sqlite3.connect("additional.db")
	c = conn.cursor()
	a = c.execute("SELECT * FROM shop_pool").fetchall()
	a = a
	conn.close()
	return random.choice(a)

def ForceUpdateStore():
	"""Force shop to update its containment."""
	conn = sqlite3.connect("additional.db")
	c = conn.cursor()
	c.execute("DELETE FROM shop_current")
	item = GetRandomPoolItem()
	price = round50(random.randint(item[1], item[2]))
	c.execute("INSERT INTO shop_current VALUES ('{}', {})".format(item[0],str(price)))
	item = GetRandomPoolItem()
	price = round50(random.randint(item[1], item[2]))
	c.execute("INSERT INTO shop_current VALUES ('{}', {})".format(item[0],str(price)))
	item = GetRandomPoolItem()
	price = round50(random.randint(item[1], item[2]))
	c.execute("INSERT INTO shop_current VALUES ('{}', {})".format(item[0],str(price)))
	a = c.execute("SELECT * FROM time_storage WHERE sett = 'store_update';").fetchone()[1]
	c.execute("UPDATE time_storage SET unix = {} WHERE sett = 'store_update'".format(str(int(time.time()))))
	conn.commit()
	conn.close()
	return "SHOP_UPDATED"

def LogStoreItems():
	"""Log current store items."""
	conn = sqlite3.connect("additional.db")
	c = conn.cursor()
	a = c.execute("SELECT * FROM shop_current").fetchall()
	for t in a:
		print(t[0] + " -- " + str(t[1]) +" electricity")
def GetStoreItems():
	"""Get current store items."""
	UpdateStore()
	conn = sqlite3.connect("additional.db")
	c = conn.cursor()
	a = c.execute("SELECT * FROM shop_current").fetchall()
	return a
def UpdateStore():
	"""Update store containment."""
	conn = sqlite3.connect("additional.db")
	c = conn.cursor()
	a = c.execute("SELECT * FROM time_storage WHERE sett = 'store_update';").fetchone()[1]
	if(a+3600 > int(time.time())):
		conn.close()
		return "TIMER_CONTINUES"
	else:
		c.execute("DELETE FROM shop_current")
		item = GetRandomPoolItem()
		price = round50(random.randint(item[1], item[2]))
		c.execute("INSERT INTO shop_current VALUES ('{}', {})".format(item[0],str(price)))
		item = GetRandomPoolItem()
		price = round50(random.randint(item[1], item[2]))
		c.execute("INSERT INTO shop_current VALUES ('{}', {})".format(item[0],str(price)))
		item = GetRandomPoolItem()
		price = round50(random.randint(item[1], item[2]))
		c.execute("INSERT INTO shop_current VALUES ('{}', {})".format(item[0],str(price)))
		c.execute("UPDATE time_storage SET unix = {} WHERE sett = 'store_update'".format(str(int(time.time()))))
		conn.commit()
		conn.close()
		return "SHOP_UPDATED"

def GetStoreTimer():
	"""Get amount of seconds before store updates."""
	UpdateStore()
	conn = sqlite3.connect("additional.db")
	c = conn.cursor()
	a = c.execute("SELECT * FROM time_storage WHERE sett = 'store_update';").fetchone()[1]
	conn.close()
	return 3600-(int(time.time())-a)

def AddItemPool(item_code, min_price, max_price):
	"""Adds item to store pool"""
	conn = sqlite3.connect("additional.db")
	c = conn.cursor()
	c.execute("INSERT INTO shop_pool VALUES ('{}', {}, {})".format(item_code, str(min_price), str(max_price)))
	conn.commit()
	conn.close()
	return "OK"

def GetStoreItem(index):
	"""Get store item by index."""
	items = GetStoreItems()
	if(index >= len(items)):
		return "NO_SUCH_INDEX"
	return items[index]

def ChanceTry(chance):
	"""Try the percentage chance."""
	return random.randrange(0,100) < chance

def GetCrateItem(crate_code):
	"""Get random item of crate."""
	conn = sqlite3.connect("additional.db")
	c = conn.cursor()
	cr = c.execute("SELECT items_in FROM crates WHERE item_code = '{}'".format(crate_code))
	cr = cr.fetchone()[0]
	cr = json.loads(cr)
	cr = sorted(cr.items(), key=lambda x: x[1])
	ItemCode_Result = ""
	while ItemCode_Result == "":
		for i in cr:
			a = ChanceTry(i[1])
			if(a):
				ItemCode_Result = i[0]
	conn.close()
	return ItemCode_Result

def haveUserBadge(user, badge):
	if not IsUserExists(user): return "USER_GONE"
	try:
		uinv = json.loads(GetUser(user)[9])
	except Exception:
		return False
	if(badge in uinv): return True
	else: return False

def GiveUserBadge(user, badge):
	if(haveUserBadge(user, badge)): return "BADGE_ALREADY"
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	cr = c.execute("SELECT badges FROM users WHERE lower(username) = '{}'".format(user)).fetchone()[0]
	cr = json.loads(cr)
	cr.append(badge)
	cr = json.dumps(cr)
	cr = c.execute("UPDATE users SET badges=\'{}\' WHERE lower(username) = '{}'".format(cr,user))
	conn.commit()
	conn.close()
	return True
BannedMethods = []
class VariableHandler(object):
	def __init__(self):
		self.Variables = {}
		self.DefaultsSet()
	def _set(self, varname, varcontent):
		self.Variables[varname] = varcontent
	def get(self, varname):
		if(varname in self.Variables):
			return self.Variables[varname]
		return False
	def DefaultsSet(self):
		self._set("TechWorkID", -1)
		self._set("MessageDelay", 0)

if(__name__ == "elvex_module"):
	vh = VariableHandler()

class CommandHandler(object):
	def __init__(self):
		self.currentCmd = ""
		# "": {"run": lambda: self.callFunc(""), "desc": "", "singleArg": False}
		self.Commands = {
			"help": {"run": lambda: self.callFunc("help"), "desc": "List of all commands.", "singleArg": False},
			"stop": {"run": lambda: self.callFunc("stop"), "desc": "Stop the server.", "singleArg": False},
			"togglemethod": {"run": lambda m: self.callFunc("togglemethod", m), "desc": "Toggles API method.", "singleArg": True},
			"yee": {"run": lambda: self.callFunc("yee"), "desc": "yee haw", "singleArg": False, "unlisted": True},
			"vhlist": {"run": lambda: self.callFunc("vhlist"), "desc": "list all variables of variablehandler", "singleArg": False, "unlisted": True},
			"fakedelay": {"run": lambda i: self.callFunc("fakedelay", i), "desc": "Adds fake delay before answering.", "singleArg": False},
			"maintenance": {"run": lambda i: self.callFunc("maintenance", i), "desc": "Set maintenance mode to specific.", "singleArg": False}
		}
		return
	def callFunc(self, cmd, *args):
		global vh
		if(cmd == "help"):
			print("Currently there is "+Fore.CYAN+str(len(self.Commands))+Fore.RESET+" total commands.")
			for cmd in self.Commands:
				if not ("unlisted" in self.Commands[cmd] and self.Commands[cmd]['unlisted']):
					print(Fore.CYAN+cmd+Fore.RESET+" - "+self.Commands[cmd]['desc'])
		elif(cmd == "stop"):
			print("Server is shutting down...", CT.INFO)
			os._exit(1)
		elif(cmd == "maintenance"):
			mid = int(args[0])
			if(mid < -1 or mid > 2):
				print("Wrong maintenance id.", CT.WARN)
				return
			vh._set("TechWorkID", mid)
			ReasonNames = ["Disabled", "Maintenance", "Update", "Not Listed"]
			CurrentReason = ReasonNames[mid+1]
			print("Switched current maintenance state to "+Fore.CYAN+CurrentReason+Fore.RESET+".", CT.INFO)
		elif(cmd == "fakedelay"):
			try:
				int(args[0])
			except Exception as e:
				print("Argument must be integer-type.", CT.ERROR)
				return

			vh._set("MessageDelay", int(args[0]))
			print("Set fake delay to "+Fore.CYAN+str(vh.get("MessageDelay"))+Fore.RESET+" seconds.")
		elif(cmd == "togglemethod"):
			global BannedMethods
			if(args[0] in BannedMethods):
				BannedMethods.remove(args[0])
				print("You enabled "+Fore.CYAN+args[0]+Fore.RESET+" method!", CT.INFO)
			else:
				BannedMethods.append(args[0])
				print("You disabled "+Fore.CYAN+args[0]+Fore.RESET+" method!", CT.INFO)
		elif(cmd == "yee"):
			print("haw")
		elif(cmd == "vhlist"):
			for v in vh.Variables:
				print(str(v)+" = "+str(vh.Variables[v]))
		return
	def Run(self, fullcmd):
		args = fullcmd.split(' ')
		args.pop(0)
		singlearg = ' '.join(args)
		cmd = fullcmd.split(' ', 1)[0]
		args = filter(None, args)
		self.currentCmd = cmd
		if(cmd in self.Commands):
			try:
				if(self.Commands[cmd]['singleArg']):
					self.Commands[cmd]['run'](singlearg)
				else:
					self.Commands[cmd]['run'](*args)
			except Exception as e:
				print(str(e), CT.ERROR)
		else:
			print("No command named "+Fore.RED+cmd+Fore.RESET+" found. Use `help` to list all commands.")

if(__name__ == "elvex_module"):
	cmdhand = CommandHandler()

# -- Debugger-only tools

def fprint(msg):
	print(fstr(msg))

def fstr(msg):
	Regex = r'[{]\w+[}]'
	retMsg = msg
	vars = re.findall(Regex, msg)
	for var in vars:
		varR = var[1:]
		varR = varR[:-1]
		if(varR in locals()):
			varR = str(locals()[varR])
		elif(varR in globals()):
			varR = str(globals()[varR])
		else:
			varR = "(Null)"
		retMsg = retMsg.replace(str(var), varR)
	return (retMsg)

class ESP(object):
	def __init__(self):
		self.inited = True
	def print(self, msg, mtype = CT.NONE):
		print(Fore.YELLOW+"[ESP] "+Fore.RESET+msg, mtype)
	class ErrorClassification(Enum):
		Unknown = -1,
		Unassigned = 0,
		Safe = 1,
		Unsafe = 2,
		Euclid = 3,
		Keter = 4,
		Thaumiel = 5,
		Neutralized = 6,
		Apocalypse = 7
	def GetClassClassification(self, classname):
		ClassifiedErrors = {
			"BaseException": [self.ErrorClassification.Safe, Fore.GREEN],
			"Exception": [self.ErrorClassification.Unknown, Fore.RED],
			"ArithmeticError": [self.ErrorClassification.Unsafe, Fore.YELLOW],
			"BufferError": [self.ErrorClassification.Euclid, Fore.RED],
			"LookupError": [self.ErrorClassification.Euclid, Fore.RED],
			"AssertionError": [self.ErrorClassification.Unsafe, Fore.YELLOW],
			"AttributeError": [self.ErrorClassification.Unsafe, Fore.YELLOW],
			"EOFError": [self.ErrorClassification.Thaumiel, Fore.CYAN],
			"FloatingPointError": [self.ErrorClassification.Neutralized,Fore.RESET],
			"ImportError": [self.ErrorClassification.Apocalypse,Fore.MAGENTA],
			"ModuleNotFound": [self.ErrorClassification.Apocalypse,Fore.MAGENTA],
			"IndexError": [self.ErrorClassification.Euclid, Fore.RED],
			"KeyError": [self.ErrorClassification.Euclid, Fore.RED],
			"MemoryError": [self.ErrorClassification.Apocalypse,Fore.MAGENTA],
			"NameError": [self.ErrorClassification.Euclid, Fore.RED],
			"NotImplementedError": [self.ErrorClassification.Euclid, Fore.RED],
			"OSError": [self.ErrorClassification.Keter,Fore.RED],
			"OverflowError": [self.ErrorClassification.Thaumiel,Fore.CYAN],
			"RecursionError": [self.ErrorClassification.Keter,Fore.RED],
			"ReferenceError": [self.ErrorClassification.Keter,Fore.RED],
			"RuntimeError": [self.ErrorClassification.Keter,Fore.RED],
			"StopInteration": [self.ErrorClassification.Unsafe, Fore.YELLOW],
			"StopAsyncIteration": [self.ErrorClassification.Unsafe, Fore.YELLOW],
			"SyntaxError": [self.ErrorClassification.Keter,Fore.RED],
			"IndentationError": [self.ErrorClassification.Unknown,Fore.RESET],
			"TabError": [self.ErrorClassification.Apocalypse,Fore.MAGENTA],
			"SystemError": [self.ErrorClassification.Keter,Fore.RED],
			"SystemExit": [self.ErrorClassification.Unknown,Fore.RESET],
			"TypeError": [self.ErrorClassification.Euclid, Fore.RED],
			"UnbondLocalError": [self.ErrorClassification.Keter,Fore.RED],
			"UnicodeError": [self.ErrorClassification.Keter,Fore.RED],
			"ValueError": [self.ErrorClassification.Thaumiel, Fore.CYAN],
			"ZeroDivisionError": [self.ErrorClassification.Apocalypse,Fore.MAGENTA],
			"EnvironmentError": [self.ErrorClassification.Thaumiel, Fore.CYAN],
			"IOError": [self.ErrorClassification.Thaumiel, Fore.CYAN],
			"WindowsError": [self.ErrorClassification.Thaumiel, Fore.CYAN]
		}
		if not classname in ClassifiedErrors:
			return [self.ErrorClassification.Unassigned, Fore.CYAN]
		else:
			return ClassifiedErrors[classname]
	def TracebackHook(self, e, v, t):
		self.print('Error just raised with errortype: '+str(e.__name__)+'.', CT.ERROR)
		esp_error = self.GetClassClassification(str(e.__name__))
		self.print('We classified this error as '+esp_error[1]+str(esp_error[0])+Fore.RESET+'.', CT.ERROR)
		self.print("ErrorValue: "+str(v), CT.ERROR)
		self.print("Traceback: "+str(traceback.extract_tb(t)), CT.ERROR)
		self.print("Server will be closed, because this error may broke something.", CT.INFO)
		self.print("// To-Do: In future wanted to keep working after error handled (sys module doesn't give to do that.)", CT.INFO)

esp = ESP()

sys.excepthook = esp.TracebackHook

class nolist_createUser(npyscreen.Form):
	def create(self):
		self.username = self.add(npyscreen.TitleText, name='Username')
		self.pswd = self.add(npyscreen.TitlePassword, name='Password')
		self.avatar = self.add(npyscreen.TitleSlider, name='Avatar', out_of=20,step=1,lowest=0)
		self.elec = self.add(npyscreen.TitleSlider, name='Electricity',step=10000,out_of=1000000000,lowest=0)
		self.pps = self.add(npyscreen.TitleSlider, name='PP', out_of=7450,step=50,lowest=0)
		self.banned = self.add(npyscreen.CheckBox, name='Is banned?')
		self.access = self.add(npyscreen.CheckBox, name='Is removed?')
	def realCreate(*args):
		f = nolist_createUser(name="New User")
		f.edit()
		return f

def dbgCreateUser():
	"""Shows form to create user from debugger."""
	if not(isDebugger): return
	curses.initscr()
	storeData = npyscreen.wrapper_basic(nolist_createUser.realCreate)
	a = AddUser(storeData.username.value, EStr(storeData.pswd.value), int(storeData.avatar.value), int(storeData.elec.value), float(storeData.pps.value), "[]", "\{\}", "", "\{\}", bool(storeData.banned.value), "0.0.0.0", bool(storeData.access.value), -1, '[]')
	if(a != "OK"):
		print("There was an error while creating user account ("+a+")", CT.ERROR)
	else:
		print("User "+Fore.CYAN+storeData.username.value+Fore.RESET+" was created successfully!", CT.INFO)

def CalculatePPv2(dmgDealt, dmgTaken, supportPoints):
	"""Perfomance Points calculator"""
	if(supportPoints == 0): bonus = -5
	elif(supportPoints > 0):
		bonus = math.fabs(round(math.atan2(supportPoints, -supportPoints)))+math.tan(supportPoints*11.3777)+(supportPoints/0.5*0.25)
	else:
		supportPoints = math.fabs(supportPoints)
		bonus = math.fabs(round(math.atan2(supportPoints, -supportPoints)))+math.tan(supportPoints*11.3777)+(supportPoints/0.5*0.25)+supportPoints * 2.5
		bonus = -(bonus)
	return -3 + (dmgDealt ** 1.25 - dmgTaken ** 1.2589)/10.5+bonus

# Checking dbs

if not (os.path.isfile("users.db")):
	conn = sqlite3.connect('users.db')
	c = conn.cursor()
	c.execute('''CREATE TABLE users
             (username text, passhash text, avatar int, electricity int, ppcount float, inventory text, customization text, bio text, stats text, banned boolean, regip text, accessible boolean,ban_reason int, badges text)''')
	c.execute('''INSERT INTO users VALUES ('act8', '', 0, 1000000, 0, '[]', '\{\}', '', '\{\}', false, '', true, -1, '["badge_bot"]')''')
	c.execute('''CREATE TABLE bannednames
		(name text)''')
	c.execute('''CREATE TABLE db_info
             (st text, data int)''')
	c.execute('''INSERT INTO db_info VALUES ('ver', {})'''.format(str(db_usersVer)))
	conn.commit()
	conn.close()
	Logger("Created new user database, because users.db was missing.", CT.INFO)

if not (os.path.isfile("additional.db")):
	conn = sqlite3.connect('additional.db')
	c = conn.cursor()
	c.execute('''CREATE TABLE crates
             (item_code text, items_in text)''')
	c.execute('''CREATE TABLE shop_pool
             (item_code text, min_price int, max_price int)''')
	c.execute('''CREATE TABLE shop_current
             (item_code text, price int)''')
	c.execute('''CREATE TABLE item_keys (key text, type text, code text)''')
	c.execute('''CREATE TABLE time_storage
             (sett text, unix int)''')
	c.execute('''CREATE TABLE db_info
             (st text, data int)''')
	c.execute('''INSERT INTO time_storage VALUES ('store_update', 0)''')
	c.execute('''INSERT INTO db_info VALUES ('ver', {})'''.format(str(db_AdditionalsVer)))
	conn.commit()
	conn.close()
	Logger("Created new additionals database, because additional.db was missing.", CT.INFO)

if(platform.system() == "Windows"):
	subprocess.check_call(["attrib","+H","users.db"])
	subprocess.check_call(["attrib","+H","additional.db"])

# Logging information about run

Logger("-----------------------")
Logger("Python: "+sys.version)
try:
	import pip
	Logger("Pip: "+pip.__version__)
except ImportError:
	Logger("Pip: Not found.")
foundProc = False
if(platform.system() == "Windows"):
	foundProc = True
	Logger("Processor: "+platform.processor())
elif platform.system() == "Darwin":
	foundProc = True
	os.environ['PATH'] = os.environ['PATH'] + os.pathsep + '/usr/sbin'
	command ="sysctl -n machdep.cpu.brand_string"
	Logger("Processor: "+str(subprocess.check_output(command).strip()))
elif platform.system() == "Linux":
	foundProc = True
	command = "cat /proc/cpuinfo"
	try:
		all_info = subprocess.check_output(command, shell=True).strip()
		for line in all_info.split("\n"):
			if "model name" in line:
				Logger("Processor: "+str(re.sub( ".*model name.*:", "", line,1)))
	except Exception:
		foundProc = False

if not foundProc:
	Logger("Processor: Unknown")

Logger("RAM: "+str(psutil.virtual_memory().total/1024/1024)+" MB")
partitions = psutil.disk_partitions()
Logger('Disks:')
diskCount = 0
for p in partitions:
	diskCount += 1
	Logger('[Disk '+str(diskCount)+'] '+p.device+"(mnt: {} / fstype: {})".format(p.mountpoint, p.fstype))

Logger("Users: ")
uCount = 0
for u in psutil.users():
	uCount += 1
	Logger("[User {}] {}".format(str(uCount), u.name))
try:
	pidarray = psutil.pids()
	npid = {}
	for p in pidarray:
		npid[p] = psutil.Process(p).name
	Logger("Current processes: ")
	for k,v in npid.items():
		Logger("[PID {}]: {}".format(str(k), v))
except Exception as ex:
	print("Error while storing debug information was raised. Check log file for more information.", CT.WARN)
	Logger("Error code: "+str(ex), CT.ERROR)
Logger("-----------------------")

def dbgSwitchErrorDisplay():
	global isFormattedError
	global isDebugger
	if not(isDebugger): return
	isFormattedError = not isFormattedError
	if(isFormattedError): print("Set formatting to "+Fore.CYAN+"PRETTY"+Fore.RESET+".")
	else: print("Set formatting to "+Fore.CYAN+"INFORMATIVE"+Fore.RESET+".")

def dbgNoDebug():
	global isDebugger
	if not isDebugger:
		print("You can only LEAVE debugger, not ENTER.")
		return
	isDebugger = False
	print("You are now moron.")
	return

conn = sqlite3.connect('users.db')
c = conn.cursor()
try:
	dInfo = c.execute("SELECT data FROM db_info WHERE st = 'ver'")
	dInfo = dInfo.fetchone()[0]
except Exception:
	print("Your version of users.db is very out of date. Please"+Fore.RED+" recreate "+Fore.RESET+"it ASAP.", CT.ERROR)
	dInfo = db_usersVer
if(dInfo < db_usersVer):
	print("Your version of users.db is out of date. Please be sure to recreate it or update.", CT.WARN)
elif(dInfo > db_usersVer):
	print("Your version of users.db is newer than this ELVEX SOCIAL version requires. Something may be broken.", CT.WARN)
conn.close()
conn = sqlite3.connect('additional.db')
c = conn.cursor()
try:
	dInfo = c.execute("SELECT data FROM db_info WHERE st = 'ver'")
	dInfo = dInfo.fetchone()[0]
except Exception:
	print("Your version of additional.db is very out of date. Please"+Fore.RED+" recreate "+Fore.RESET+"it ASAP.", CT.ERROR)
	dInfo = db_AdditionalsVer

if(dInfo < db_AdditionalsVer):
	print("Your version of additional.db is out of date. Please be sure to recreate it or update.", CT.WARN)
elif(dInfo > db_AdditionalsVer):
	print("Your version of additional.db is newer than this ELVEX SOCIAL version requires. Something may be broken.", CT.WARN)
conn.close()

def CompleterLoad():
	global history
	for name, val in elvex_module.__dict__.items():
		if callable(val) and name not in NotList:
			try:
				args = inspect.getfullargspec(val)[0]
			except Exception:
				continue
			if(name.startswith("nolist_")):
				continue
			history.append_string(name+"()")

def is_valid_command(v):
	return re.match(r"\b[^()]+\((.*)\)$", v) is not None

NotAScriptMessage = Fore.RESET + '''
⣿⣿⣿⣿⣿⣿⣿⣿⠿⠛⠛⢛⣉⣩⣤⣬⣉⣉⣉⠛⠛⠿⣿⣿⣿⣿⣿⣿⣿⣿
⣿⣿⣿⣿⣿⠿⠋⣀⣴⣾⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣦⣈⠻⢿⣿⣿⣿⣿⣿
⣿⣿⣿⠟⢁⣴⣾⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣦⡙⠿⣿⣿⣿
⣿⣿⠏⢠⣾⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⠋⠙⢻⣷⡆⠹⣿⣿
⣿⡇⢠⣿⣿⣿⣿⣿⣿⡟⠋⠛⢻⣿⣿⣿⣿⣿⣿⣿⣿⣷⣄⣀⣴⣿⣿⡄⢹⣿
⡟⢀⣿⣿⣿⣿⣿⣿⣿⣧⣀⣤⣼⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡄⢻
⠁⢸⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⠿⠟⢛⣋⣉⣉⣉⠙⢿⣿⣿⣿⣿⣿⡇⢸
⡄⢸⣿⣿⣿⣿⣿⣿⣿⣿⡿⠋⣡⣴⣶⣿⣿⣿⣿⣿⣿⣧⠄⢿⣿⣿⣿⣿⡇⢸
⣇⠈⣿⣿⣿⣿⣿⣿⣿⡟⠁⣼⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡆⢸⣿⣿⣿⣿⠃⣼
⣿⣆⠘⣿⣿⣿⣿⣿⣿⡇⣴⣤⣤⣬⣉⡛⠻⣿⣿⣿⣿⣿⡇⢸⣿⣿⣿⠃⢸⣿
⣿⣿⣆⠘⢿⣿⣿⣿⣿⢀⣿⣿⣿⣿⣿⠿⠷⠌⠛⢛⣋⣉⣁⣸⣿⡿⠋⣠⣿⣿
⣿⣿⣿⣶⡈⠙⢿⣿⣟⣈⣉⣩⣥⣤⣶⣶⣶⣾⣿⣿⣿⣿⣿⡿⠟⢁⣾⣿⣿⣿
⣿⣿⣿⣿⣿⣶⣄⠉⠛⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⠟⠋⣉⣤⣶⣿⣿⣿⣿⣿
⣿⣿⣿⣿⣿⣿⣿⣿⣷⣦⣤⣈⡉⠉⠛⣋⣉⣉⣤⣤⣶⣿⣿⣿⣿⣿⣿⣿⣿⣿'''+Fore.RESET+'''
This is not a script. This is library.
You cannot run it s/c.
'''

if(len(sys.argv) > 1 and sys.argv[1] == "debugger"):
	Logger("Debugger initialized.", CT.WARN)
	oprint("Debugger for Elvex SOCIAL v"+str(version)+", act8team.\nVisit act8team.com for more information.")
	import curses
	isFormattedError = True
	isDebugger = True
	history = prompt_toolkit.history.InMemoryHistory()
	CompleterLoad()
	session = prompt_toolkit.PromptSession(
        history=history,
        auto_suggest=prompt_toolkit.auto_suggest.AutoSuggestFromHistory(),
        enable_history_search=True)
	validator = prompt_toolkit.validation.Validator.from_callable(is_valid_command, error_message='Not a valid function. (Cannot be executed)', move_cursor_to_end=True)
	while(True):
		oprint(Fore.CYAN+ '> '+Fore.RESET,end='')
		if(isFormattedError):
			a = session.prompt('  ', validator=validator, validate_while_typing=True)
			try:
				eval(a)
			except Exception as e:
				print(str(e), CT.ERROR)
		else:
			a = input()
			eval(a)
elif(__name__ == "__main__"):
	try:
		print(NotAScriptMessage)
	except Exception:
		os._exit(-1)
	time.sleep(10)
	os._exit(-1)


import glob
import requests
import datetime
from tqdm import tqdm

tsv_paths = glob.glob('./datasets/*.tsv')
for path in tsv_paths:
    with open(path, 'r') as f:
        f.readline()
        for url in tqdm(f.readlines()):
            url = url.split('\t')[0]
            print(url)
            ext = url.split('.')[-1]
            save_path = './datasets/' + datetime.datetime.now().strftime('%Y%m%d_%H%M-%S_%f' + '.' + ext)
            response = requests.get(url)

            with open(save_path, 'wb') as wf:
                wf.write(response.content)


from pygame.locals import *
import pygame
import time
import paho.mqtt.client as mqtt
import uuid
import signal
import random
import math

class Player:
    x = 400
    y = 300
    speed = 1

    def moveRight(self, amt = None):
        if amt is None:
            self.x = self.x + self.speed
        else:
            self.x = self.x + amt

    def moveLeft(self, amt = None):
        if amt is None:
            self.x = self.x - self.speed
        else:
            self.x = self.x - amt

    def moveUp(self, amt = None):
        if amt is None:
            self.y = self.y - self.speed
        else:
            self.y = self.y - amt

    def moveDown(self, amt = None):
        if amt is None:
            self.y = self.y + self.speed
        else:
            self.y = self.y + amt

class Hole:
    def __init__(self):
        self.x = random.randrange(100, 700)
        self.y = random.randrange(100, 500)

    def regenHole(self):
        self.x = random.randrange(100, 700)
        self.y = random.randrange(100, 500)

topic = 'IDD/juicey/labyrinth/#'

topicq = 'IDD/juicey/labyrinth'
# topicX = 'IDD/juicey/labyrinth/X'
# topicY = 'IDD/juicey/labyrinth/Y'

# x_msg = 0.0
# y_msg = 0.0

class App:

    windowWidth = 800
    windowHeight = 600
    player = 0

    def __init__(self):
        self._running = True
        self._display_surf = None
        self._image_surf = None
        self._block_surf = None
        self.player = Player()
        self.hole = Hole()
        self.controls = [0.0, 0.0]

    def on_init(self):
        pygame.init()
        self._display_surf = pygame.display.set_mode((self.windowWidth,self.windowHeight), pygame.HWSURFACE)

        pygame.display.set_caption('Pygame pythonspot.com example')
        self._running = True
        self._image_surf = pygame.image.load("ball.png").convert()
        self._block_surf = pygame.image.load("hole.png").convert()

    def on_event(self, event):
        if event.type == QUIT:
            self._running = False

    def on_loop(self):
        if math.dist([self.player.x, self.player.y], [self.hole.x, self.hole.y]) < 40:
            self.hole.regenHole()

    def on_render(self):
        self._display_surf.fill((0,0,0))
        self._display_surf.blit(self._image_surf,(self.player.x,self.player.y))
        self._display_surf.blit(self._block_surf,(self.hole.x,self.hole.y))
        pygame.display.flip()

    def on_cleanup(self):
        pygame.quit()

    def on_execute(self):
        if self.on_init() == False:
            self._running = False

        while( self._running ):
            pygame.event.pump()
            keys = pygame.key.get_pressed()
            moves = list(map(abs, self.controls))
            if self.controls[0] > 0.5:
                self.player.moveRight(moves[0] - 0.5)

            if self.controls[0] < -0.5:
                self.player.moveLeft(moves[0] - 0.5)

            if self.controls[1] > 0.5:
                self.player.moveUp(moves[1] - 0.5)

            if self.controls[1] < -0.5:
                self.player.moveDown(moves[1] - 0.5)

            if (keys[K_ESCAPE]):
                self._running = False

            self.on_loop()
            self.on_render()
        self.on_cleanup()

theApp = App()

def on_connect(client, userdata, flags, rc):
    print(f"connected with result code {rc}")
    client.subscribe(topic)

no_players = 0
print("Number of players? [1 or 2]")

while no_players not in [1, 2]:
    no_players = int(input())
    print("1 or 2 players only!")

def on_message(cleint, userdata, msg):
    # if a message is recieved on topic, print message
    if no_players == 2:
        if topicq in msg.topic:
            player_axis = 0 if msg.topic[-1] == 'X' else 1
            m = float(msg.payload.decode('UTF-8'))
            theApp.controls[player_axis] = m
    else:
        if topicq in msg.topic:
            m = msg.payload.decode('UTF-8').split(',')
            theApp.controls = list(map(float, m))

# this lets us exit gracefully (close the connection to the broker)
def handler(signum, frame):
    print('exit gracefully')
    client.loop_stop()
    exit (0)

# hen sigint happens, do the handler callback function
signal.signal(signal.SIGINT, handler)

if __name__ == "__main__" :
    client = mqtt.Client(str(uuid.uuid1()))
    client.tls_set()
    client.username_pw_set('idd', 'device@theFarm')
    client.on_connect = on_connect
    client.on_message = on_message

    client.connect(
        'farlab.infosci.cornell.edu',
        port=8883)

    client.loop_start()

    theApp.on_execute()


import os
import re
import math
import time
import numpy as np
import snowballstemmer as stem
from scipy.sparse import csr_matrix, linalg
import sklearn.preprocessing

def gen_words(text, stemming=stem.stemmer('english')):
    """Create generator.

    :param text: some string
    :param stemming: variant of stemming algorithm
    :return: generator giving stemmed words from text
    """
    for word in stemming.stemWords(re.findall(r"[\w']+", text.lower())):
        yield word

def parse_file(file_path, bow, stemming):
    """Create bow set by parsing file.

    :param file_path: path to file
    :param bow: bag_of_words set
    :param stemming: variant of stemming algorithm
    :return: bag_of_words set for parsed file
    """
    terms = {}
    with open(file_path, 'r', encoding='utf-8') as file:
        for line in file.readlines():
            for word in gen_words(line.lower(), stemming):
                if word not in bow:
                    bow[word] = len(bow)

                if bow[word] in terms:
                    terms[bow[word]] += 1
                else:
                    terms[bow[word]] = 1

    return terms

def idf_matrix(matrix, bow):
    """Use inverse document frequency to reduce meaning of frequent words.

    :param bow: bag_of_words set
    :param matrix: term_by_document matrix
    :return: none
    """
    for word in bow:
        docs = sum(1 for doc in matrix if bow[word] in matrix[doc])
        idf = math.log10(len(matrix) / docs)
        for doc in matrix:
            if bow[word] in matrix[doc]:
                matrix[doc][bow[word]] *= idf
                if not matrix[doc][bow[word]]:
                    del matrix[doc][bow[word]]

def create_matrix(dir_path, svd=False, k=30, idf=True):
    """Create bag_of_words and term_by_document matrix.

    :param idf: whether to apply idf for each word
    :param k: parameter for svd
    :param svd: whether to apply low rank approximation
    :param dir_path: path to directory with documents
    :returns: bag_of_words, documents names, term_by_document matrix
    """
    bow = {}
    stemming = stem.stemmer('english')
    docs_terms = {name: parse_file(dir_path + "/" + name, bow, stemming)
                  for name in os.listdir(dir_path)}

    if idf:
        idf_matrix(docs_terms, bow)

    rows = []
    cols = []
    data = []
    docs = list(docs_terms.keys())
    for i, doc in enumerate(docs):
        col = docs_terms[doc].keys()
        data.extend([docs_terms[doc][x] for x in col])
        cols.extend(col)
        rows.extend([i] * len(col))

    matrix = csr_matrix((data, (rows, cols)),
                        shape=(len(docs_terms), len(bow)), dtype=float)

    if svd:
        u, s, v = linalg.svds(matrix, k)
        matrix = csr_matrix(u.dot(np.diag(s)).dot(v))

    sklearn.preprocessing.normalize(matrix, norm='l1', axis=1, copy=False)

    return bow, docs, matrix

def create_vec(text, bow):
    """Change user query into bag_of_words vector.

    :param text: user query
    :param bow: bag_of_words set
    :return: normalised bag_of_words vector
    """
    vec = np.zeros(shape=len(bow))
    for word in gen_words(text.lower()):
        if word in bow:
            vec[bow[word]] += 1
    return vec / np.linalg.norm(vec)

def fix_arr(max_values, indexes, n):
    """Sort in reverse order max_values.

    :param max_values: arr of max_values
    :param indexes: arr of indexes
    :param n: len of indexes/max_values
    :return: none
    """
    i = n - 1
    while i > 0 and max_values[i] > max_values[i - 1]:
        max_values[i], max_values[i - 1] = max_values[i - 1], max_values[i]
        indexes[i], indexes[i - 1] = indexes[i - 1], indexes[i]
        i -= 1

def query(matrix, vec, n=1):
    """Find n documents with highest correlation to query.

    :param matrix: term_by_document matrix
    :param vec: query from user
    :param n: number of documents to find
    :return: n max files indexes with highest correlation
    """
    arr = matrix.dot(vec)
    max_values = np.zeros(shape=n)
    indexes = np.zeros(shape=n, dtype=int)

    for i, elem in enumerate(arr):
        if elem > max_values[-1]:
            max_values[-1] = elem
            indexes[-1] = i
            fix_arr(max_values, indexes, n)
    return indexes

def display_results(docs, indexes):
    """Display results of query.

    :param docs: arr of names of files
    :param indexes: indexes of docs with highest correlation
    :return: none
    """
    print("\nTop %d results are:" % len(indexes))
    for index in indexes:
        print(docs[index])

def main(n=10, k=800, svd=True, idf=True, path='wiki'):

    bow, docs, matrix = create_matrix(path, svd, k, idf)

    text = input('\nEnter sentence: ')
    while text != 'quit':
        start_time = time.time()
        vec = create_vec(text, bow)
        display_results(docs, query(matrix, vec, n))
        end_time = time.time()
        print("time: " + str(end_time - start_time))
        text = input('\nEnter sentence: ')

if __name__ == "__main__":
    main()


	#-*- coding: utf-8 -*-，
#coding = utf-8
import openpyxl
from openpyxl.compat import range
from openpyxl.cell import get_column_letter
from django.http import HttpResponse
import MySQLdb as mdb
from openpyxl.writer.excel import save_virtual_workbook
import datetime

# {"app_download":[], "traffic":[]}
def queryset_to_xlsx(type, queryset):
    wb = openpyxl.Workbook()

    ws = wb.active

    today = datetime.datetime.now().date().strftime("%Y%m%d")

    if type == "APP":
        wb_name = "App_Download_" + today + ".xlsx"
        ws.title = u"App Download Data"
        generate_app_wb(wb, queryset)
    elif type == "PC":
        wb_name = "PC_Traffic_" + today + ".xlsx"
        ws.title = u"PC Traffic"
        generate_pc_wb(wb, queryset)
    elif type == "MOBILE":
        wb_name = "Mobile_Traffic_" + today + ".xlsx"
        ws. title = u"MOBILE Traffic"
        generate_mobile_wb(wb, queryset)

    response = HttpResponse(save_virtual_workbook(wb), content_type='application/vnd.ms-excel')
    response['Content-Disposition'] = 'attachment; filename=' + wb_name
    return response

def generate_app_wb(wb, app_queryset):
    print(u"开始写入APP数据。。。")
    ws = wb.get_sheet_by_name(u"App Download Data")
    # Append Table headers
    # headers = ["id", "app_id", "app_name", "app_url", "region", "os", "device", \
    #     "store", "source", "sector", "sub_sector", "type", "rank", "starting_date", "date_type", \
    #     "accumulative_downloads_estimate", "download_or_revenue", "unit", "app_average_price_usd", \
    #     "avg_rating_all", "app_version", "app_release_date", "publisher_id", "publisher_name", \
    #     "company_id", "company_name", "parent_company_id", "parent_company_name"]
    headers = ["id", "app_id", "app_name", "app_url", "region", "os", "device", \
        "store", "source", "sector", "sub_sector", "type", "rank", "starting_date", "date_type", \
        "accumulative_downloads_estimate", "download_or_revenue", "unit($)", "app_average_price_usd($)", \
        "avg_rating_all", "app_version", "app_release_date", "publisher_id", "publisher_name", \
        "company_id", "company_name", "parent_company_id", "parent_company_name"]
    ws.append(headers)
    for obj in app_queryset:
        row = [obj.id, obj.app_id, obj.app_name, obj.app_url, obj.region, obj.os, \
            obj.device, obj.store, obj.source, obj.sector, obj.sub_sector, obj.type, obj.rank, \
            obj.starting_date, obj.date_type, obj.accumulative_downloads_estimate, \
            obj.download_or_revenue, obj.unit, obj.app_average_price_usd, obj.avg_rating_all, \
            obj.app_version, obj.app_release_date, obj.publisher_id, obj.publisher_name, \
            obj.company_id, obj.company_name, obj.parent_company_id, obj.parent_company_name]
        ws.append(row)

    return wb

def generate_pc_wb(wb, pc_queryset):
    print(u"开始写入PC Traffic数据。。。")
    ws = wb.get_sheet_by_name(u"PC Traffic")
    # headers = ["id", "company_name", "source", "url", "domain_name", "url_type", "sector", \
    #     "sub_sector", "starting_date", "date_type", "total_visitors(K)", "visitor_penetration", \
    #     "total_visits(K)", "total_visits_share", "total_page_views(K)", "total_page_views_share", \
    #     "total_length_of_visits(Hours)", "total_length_of_visits_share", "daily_unique_visitors(K)", \
    #     "daily_unique_visitors_penetration", "daily_visits(K)", "daily_visits_share", \
    #     "daily_page_views(K)", "daily_page_views_share", "daily_length_of_visits(Hours)", \
    #     "daily_length_of_visits_share", "per_capita_daily_visits", "per_capita_length_of_daily_visits(MM:SS)",
    #     "per_capita_daily_page_views", "per_visit_page_views", "per_page_view_length", \
    #     "per_capita_days_of_visits(#)", "per_capita_visits(#)", "per_capita_page_views", \
    #     "per_capita_length_of_visits(MM:SS)", "createddate"]
    headers = ["id", "公司名", "source", "url", "domain_name", "url_type", "sector", \
        "sub_sector", "starting_date", "date_type", "总覆盖人数(千)", "总覆盖人数比例(%)", \
        "总访问次数(千次)", "总访问次数比例(%)", "总页面浏览量(千页)", "总页面浏览量比例(%)", \
        "总访问时长(小时)", "总访问时长比例(%)", "日均覆盖人数(千人)", \
        "日均覆盖人数比例(%)", "日均访问次数(千)", "日均访问次数比例", \
        "日均页面浏览量(千)", "日均页面浏览量比例", "日均访问时长(小时)", \
        "日均访问时长比例(%)", "人均单日访问次数(次)", "人均单日访问时长(分)",
        "人均单日页面浏览量", "人均单次页面浏览量", "人均单页平均浏览时间(分)", \
        "人均月度/周访问天数(天)", "人均月度/周访问次数(次)", "人均月度/周访问页面量", \
        "人均月度有效浏览时间(分)"]
    ws.append(headers)

    for obj in pc_queryset:
        row = [obj.id, obj.company_name, obj.source, obj.url, obj.domain_name, obj.url_type, obj.sector, obj.sub_sector,
            obj.starting_date, obj.date_type, obj.total_visitors, obj.visitor_penetration, obj.total_visits,
            obj.total_visits_share, obj.total_page_views, obj.total_page_views_share, obj.total_length_of_visits,
            obj.total_length_of_visits_share, obj.daily_unique_visitors, obj.daily_unique_visitors_penetration,
            obj.daily_visits, obj.daily_visits_share, obj.daily_page_views, obj.daily_page_views_share, obj.daily_length_of_visits,
            obj.daily_length_of_visits_share, obj.per_capita_daily_visits, obj.per_capita_length_of_daily_visits,
            obj.per_capita_daily_page_views, obj.per_visit_page_views, obj.per_page_view_length, obj.per_capita_days_of_visits,
            obj.per_capita_visits, obj.per_capita_page_views, obj.per_capita_length_of_visits]
        ws.append(row)

    return wb

def generate_mobile_wb(wb, pc_queryset):
    print(u"开始写入Mobile Traffic数据。。。")
    ws = wb.get_sheet_by_name(u"MOBILE Traffic")
    # headers = ["id", "source", "app_name", "sector", "sub_sector", "starting_date",
    #         "active_users(K)", "active_users_penetration", "total_startup_counts(MN)",
    #         "total_startup_counts_share", "total_use_time(Hours/K)", "total_use_time_share", "daily_active_user(K)",
    #         "daily_active_user_penetration", "daily_startup_counts(MN)", "daily_startup_counts_share",
    #         "daily_use_time(Hours/K)", "daily_use_time_share", "per_capita_days_in_use(#)", "per_capita_startup_counts(#)",
    #         "per_capita_run_time", "per_capita_daily_startup_counts(#)", "per_capita_daily_use_time(MMs)", "per_startup_use_time(MMs)",
    #         "app_installations(K)", "active_smart_devices", "active_android_devices", "Installation_penetration",
    #         "app_uninstall_rate", "createddate"]
    headers = ["id", "source", "app_name", "sector", "sub_sector", "starting_date",
            "总覆盖人数(千)", "总覆盖人数比例(%)", "总启动次数(千)",
            "总启动次数比例(%)", "总使用时长(千小时)", "总使用时长比例(%)", "DAU/日均覆盖人数（千）",
            "日均覆盖人数比例(%)/大盘覆盖率", "日均启动次数(千)", "日均启动次数比例",
            "日均使用时长(千小时)", "日均使用时长比例(%)", "人均使用天数（天）", "人均使用次数（次）",
            "人均有效使用时间（分钟）", "人均单日启动次数(次)", "人均每日有效使用时间（分钟）", "人均单次有效使用时间（分钟）",
            "装机量(千)", "active_smart_devices(个)", "Android大盘(覆盖机数)", "App安装覆盖率(%)/渗透率",
            "App流失率"]
    ws.append(headers)

    for obj in pc_queryset:
        row = [obj.id, obj.source, obj.app_name, obj.sector, obj.sub_sector, obj.starting_date,
            obj.active_users, obj.active_users_penetration, obj.total_startup_counts,
            obj.total_startup_counts_share, obj.total_use_time, obj.total_use_time_share, obj.daily_active_user,
            obj.daily_active_user_penetration, obj.daily_startup_counts, obj.daily_startup_counts_share,
            obj.daily_use_time, obj.daily_use_time_share, obj.per_capita_days_in_use, obj.per_capita_startup_counts,
            obj.per_capita_use_time, obj.per_capita_daily_startup_counts, obj.per_capita_daily_use_time, obj.per_startup_use_time,
            obj.app_installations, obj.active_smart_devices, obj.active_android_devices, obj.installation_penetration,
            obj.app_uninstall_rate]
        ws.append(row)

    return wb

# def queryset_to_xlsx(queryset):

#     wb = openpyxl.Workbook()
#     ws = wb.active
#     ws.title = u'App Data'

#     # Append Table headers
#     headers = ["id", "app_id", "app_name", "app_url", "app_iap", "region", "os", "device", \
#         "store", "source", "sector", "sub_sector", "type", "rank", "starting_date", "date_type", \
#         "accumulative_downloads_estimate", "download_or_revenue", "unit", "app_average_price_usd", \
#         "avg_rating_all", "app_version", "app_release_date", "publisher_id", "publisher_name", \
#         "company_id", "company_name", "parent_company_id", "parent_company_name"]
#     ws.append(headers)

#     print(u"queryset Size: " + str(len(queryset)))
#     for obj in queryset:
#         row = [obj.id, obj.app_id, obj.app_name, obj.app_url, obj.app_iap, obj.region, obj.os, \
#             obj.device, obj.store, obj.source, obj.sector, obj.sub_sector, obj.type, obj.rank, \
#             obj.starting_date, obj.date_type, obj.accumulative_downloads_estimate, \
#             obj.download_or_revenue, obj.unit, obj.app_average_price_usd, obj.avg_rating_all, \
#             obj.app_version, obj.app_release_date, obj.publisher_id, obj.publisher_name, \
#             obj.company_id, obj.company_name, obj.parent_company_id, obj.parent_company_name]
#         ws.append(row)

#     response = HttpResponse(save_virtual_workbook(wb), content_type='application/vnd.ms-excel')
#     response['Content-Disposition'] = 'attachment; filename=App_Download.xlsx'
#     return response

# def official_demo():
#     wb = openpyxl.Workbook()
#     # ws = wb.get_active_sheet()
#     # ws.title = 'Test sheet'

#     # row_num = 0
#     dest_filename = 'empty_book.xlsx'
#     ws1 = wb.active
#     ws1.title = "range names"

#     for row in range(1, 40):
#         ws1.append(range(600))

#     ws2 = wb.create_sheet(title="Pi")
#     ws2['F5'] = 3.14

#     ws3 = wb.create_sheet(title="Data")
#     for row in range(10, 20):
#         for col in range(27, 54):
#             _ = ws3.cell(column=col, row=row, value="%s" % get_column_letter(col))
#     wb.save(filename = dest_filename)


import geopandas
from shapely.geometry import Point
import pandas as pd
from glob import glob
from os.path import join as j
from src.utils import get_logger
import json

logger = get_logger(__file__, snakemake.log[0])

logger.info(f"loading {snakemake.input.cities}")
maf_cities = pd.read_csv(snakemake.input.cities)
maf_geometry = [Point(xy) for xy in zip(maf_cities.lon, maf_cities.lat)]
geo_df_maf = geopandas.GeoDataFrame(maf_cities, geometry=maf_geometry)

shape_file_dir = snakemake.input.shapefiles
logger.info(f"loading shapefiles in {shape_file_dir}")
glob_expression = j(shape_file_dir, 'OECD_MA_*/*.shp')
geo_df_list = []
for shape_file in glob(glob_expression):
    cur_df = geopandas.read_file(shape_file)
    geo_df_list.append(cur_df)

all_countries = geopandas.GeoDataFrame(pd.concat(geo_df_list, ignore_index=True))
del geo_df_list

logger.info(f"executing spatial join on")
cities_with_country = geopandas.sjoin(all_countries, geo_df_maf)

logger.info(f"writing mapping to file")
# create a unique name of the form "NAME, country" (NAME == str in shapefile)
cities_with_country['city_country'] = cities_with_country.apply(
    lambda x: x['NAME']+', '+x['country'], axis=1
)
city_to_FUA = dict(cities_with_country.set_index('city')['city_country'])

output_path = snakemake.output[0]
with open(output_path, 'w') as f:
    json.dump(city_to_FUA, f)
logger.info(f"successfully wrote city_country to {output_path}")

city_to_country = dict(cities_with_country.set_index('city')['country'])

output_path = snakemake.output[1]
with open(output_path, 'w') as f:
    json.dump(city_to_country, f)
logger.info(f"successfully wrote country to {output_path}")


from copy_paste import CopyPaste
from coco import CocoDetectionCP
import albumentations as A

transformScene = A.Compose([
        #A.RandomScale(scale_limit=(-0.9, 1), p=1), #LargeScaleJitter from scale of 0.1 to 2
        A.PadIfNeeded(256, 256, border_mode=0), #pads with image in the center, not the top left like the paper
        # A.Resize(534, 889, always_apply=True, p=1),
        A.Resize(800, 1333, always_apply=True, p=1),
    ]
)

transform = A.Compose([
        A.RandomScale(scale_limit=(-0.9, 1), p=1), #LargeScaleJitter from scale of 0.1 to 2
        A.PadIfNeeded(256, 256, border_mode=0), #pads with image in the center, not the top left like the paper
        A.Flip(p=0.5),
        A.Resize(800, 1333, always_apply=True, p=1),
        #A.Solarize(always_apply=True, p=1.0, threshold=(128, 128)),
        # A.RandomCrop(534, 889),
        #A.Resize(800, 1333, always_apply=True, p=1),
        # pct_objects is the percentage of objects to paste over
        CopyPaste(blend=True, sigma=1, pct_objects_paste=1, p=1., gray=False)
    ], bbox_params=A.BboxParams(format="coco")
)

#data = CocoDetectionCP(
#    '../agilent-repos/mmdetection/data/bead_cropped_detection/images',
#    '../agilent-repos/mmdetection/data/custom/object-classes.json',
#    transform
#)
data = CocoDetectionCP(
    '../Swin-Transformer-Object-Detection/data/flooding_high_cropped',
    '../Swin-Transformer-Object-Detection/data/w_nebulizer_save_4',
    #'../Swin-Transformer-Object-Detection/data/beading_basler',
    #'../Swin-Transformer-Object-Detection/data/basler_bead_non_cropped.json',
    '../Swin-Transformer-Object-Detection/data/w_nebulizer_train_iter1.json',
    transform,
    transformScene,
    gray=False
)

# Synthesize a new dataset using all of the scene images, and the available
# annotations from the objects images (randomly).
data.synthesize(pct=0.03,
                out_dir="data/out_02_pct_w_nebulizer/",
                json_name="data/out_02_pct_w_nebulizer.json")


import boto
import boto.s3.connection
import boto.s3
from boto.s3.key import Key

import swiftclient

import random
from nose.tools import eq_ as eq

import yaml
import sys
import os

## FROM https://github.com/ceph/swift/blob/master/test/__init__.py
def get_config():
    """
    Attempt to get a functional config dictionary.
    """
    # config_file = os.environ.get('CROSS_PROTOCOL_TEST_CONFIG_FILE')
    try:
        # Get config.yaml in the same directory
        __location__ = os.path.realpath(
        os.path.join(os.getcwd(), os.path.dirname(sys.argv[0])))
        f = open(os.path.join(__location__, 'config.yaml'))
        # use safe_load instead load
        conf = yaml.safe_load(f)
        f.close()
    except IOError:
        print >>sys.stderr, 'UNABLE TO READ FUNCTIONAL TESTS CONFIG FILE'
    return conf

## IMPORT FROM CONFIG.YAML

conf = get_config()
s3keys = conf['s3']
swiftkeys = conf['swift']

# For more parameters:
# https://github.com/boto/boto/blob/develop/boto/s3/connection.py
s3conn = boto.connect_s3(
    aws_access_key_id = s3keys['aws_access_key_id'],
    aws_secret_access_key = s3keys['aws_secret_access_key'],
    host = s3keys['host'],
    is_secure = True,
    port = None,
    proxy = None,
    proxy_port = None,
    https_connection_factory = None,
    calling_format = boto.s3.connection.OrdinaryCallingFormat()
    )

# For more parameters:
# https://github.com/openstack/python-swiftclient/blob/master/swiftclient/client.py
swiftconn = swiftclient.Connection(
    authurl = swiftkeys['authurl'],
    user = swiftkeys['user'],
    key = swiftkeys['key'],
    preauthurl = None
    # NOTE TO SELF: Port, HTTPS/HTTP, etc. all contained in authurl/preauthurl
    )

## HELPER FUNCTIONS

# FROM https://github.com/ceph/s3-tests/blob/master/s3tests/functional/utils.py
def assert_raises(excClass, callableObj, *args, **kwargs):
    """
    Like unittest.TestCase.assertRaises, but returns the exception.
    """
    try:
        callableObj(*args, **kwargs)
    except excClass as e:
        return e
    else:
        if hasattr(excClass, '__name__'):
            excName = excClass.__name__
        else:
            excName = str(excClass)
        raise AssertionError("%s not raised" % excName)

def swift_list_containers(swiftconn):
    # Returns list of containers
    buckets = swiftconn.get_account()[1]
    list_of_buckets = [bucket_dictionary[u'name'] \
    for bucket_dictionary in buckets]
    return list_of_buckets

def swift_list_objects(swiftconn, name):
    # Returns list of objects in a container
    objects = swiftconn.get_container(name)[1]
    list_of_objects = [object_dictionary[u'name'] \
    for object_dictionary in objects]
    return list_of_objects

def swift_delete_all_containers(swiftconn):
    # Deletes all containers, both empty and nonempty
    for container in swift_list_containers(swiftconn):
        objects = swift_list_objects(swiftconn, container)
        for name in objects:
            swiftconn.delete_object(container, name)
        swiftconn.delete_container(container)

def swift_md5(swiftconn, container, object_name):
    # Returns object md5
    return swiftconn.head_object(container, object_name)['etag']

def swift_size(swiftconn, container, object_name):
    # Returns object content-length
    return int(swiftconn.head_object(container, object_name)['content-length'])

def s3_list_buckets(s3conn):
    # Returns list of buckets
    buckets = s3conn.get_all_buckets()
    list_of_buckets = [bucket.name for bucket in buckets]
    return list_of_buckets

def s3_list_objects(s3conn, name):
    # Returns list of objects in a bucket
    bucket = s3conn.get_bucket(name)
    list_of_objects = [obj.key for obj in bucket.list()]
    return list_of_objects

def s3_delete_all_buckets(s3conn):
    # Deletes all buckets, both empty and nonempty
    for name in s3_list_buckets(s3conn):
        bucket = s3conn.get_bucket(name)
        keys = bucket.list()
        for key in keys:
            key.delete()
        s3conn.delete_bucket(bucket)

def s3_md5(s3conn, bucket, object_name):
    # Returns object md5
    resp = s3conn.make_request('HEAD', bucket, object_name)
    for x in resp.getheaders():
        if x[0]=='etag':
            return x[1][1:-1]

def s3_size(s3conn, bucket, object_name):
    # Returns object size
    resp = s3conn.make_request('HEAD', bucket, object_name)
    for x in resp.getheaders():
        if x[0]=='content-length':
            return x[1]

# 3-63 chars, must start/end with lowercase letter or number
# can contain lowercase letters, numbers, and dashes (no periods)
# cannot start/end with dash or period
# no consecutive period/dashes
def create_valid_name():
    name_length = random.randint(3,63)
    name = ''
    for x in range(name_length):
        if x == 0 or x == name_length-1:
            name+=chr(random.randint(97,122))
        elif x!=0 and name[x-1] in '-.':
            name+=chr(random.randint(97,122))
        else:
            name+=random.choice(chr(random.randint(97,122))+'-')
    return name

## TODO: to reestablish connection, set up tests as classes
## and use instance variables

## BUCKET TESTS

def test_list_buckets():
    # Delete all buckets
    s3_delete_all_buckets(s3conn)
    # Create bucket
    name = create_valid_name()
    bucket = s3conn.create_bucket(name)
    # Check buckets list
    # name = name.encode('utf-8')
    eq(swift_list_containers(swiftconn), [name])
    eq(s3_list_buckets(s3conn), [name])
    # FIXME: remove bucket or teardown?

def test_list_container():
    # Delete all buckets
    s3_delete_all_buckets(s3conn)
    # Create container
    name = create_valid_name()
    swiftconn.put_container(name)
    # Check containers list
    eq(s3_list_buckets(s3conn), [name])
    eq(swift_list_containers(swiftconn), [name])
    # FIXME: remove bucket or teardown?

def test_list_empty_bucket():
    # Create bucket
    name = create_valid_name()
    bucket = s3conn.create_bucket(name)
    # Get list of objects
    eq(swift_list_objects(swiftconn, name), [])
    eq(s3_list_objects(s3conn, name), [])

def test_list_empty_container():
    # Create container
    name = create_valid_name()
    swiftconn.put_container(name)
    # Get list of objects
    eq(s3_list_objects(s3conn, name), [])
    eq(swift_list_objects(swiftconn, name), [])

def test_list_nonempty_bucket():
    # Create bucket
    name = create_valid_name()
    bucket = s3conn.create_bucket(name)
    # Add test object using S3
    k = Key(bucket)
    k.key = 'test'
    k.set_contents_from_string('this is a test file')
    # Get list of objects using Swift
    eq(swift_list_objects(swiftconn, name), ['test'])

def test_list_nonempty_container():
    # Create container
    name = create_valid_name()
    swiftconn.put_container(name)
    # Add test object using Swift
    swiftconn.put_object(name, 'test', 'this is a test file')
    # Get list of objects using S3
    eq(s3_list_objects(s3conn, name), ['test'])

#test_head_bucket FIXME

#test_head_container FIXME

def test_delete_bucket():
    # Delete all buckets
    s3_delete_all_buckets(s3conn)
    # Create bucket
    name = create_valid_name()
    bucket = s3conn.create_bucket(name)
    # Delete using Swift
    swiftconn.delete_container(name)
    # List buckets
    eq(swift_list_containers(swiftconn), [])
    eq(s3_list_buckets(s3conn), [])
    # Try deleting again with S3/Swift
    s3_err = assert_raises(boto.exception.S3ResponseError, s3conn.delete_bucket, name)
    swift_err = assert_raises(swiftclient.ClientException, swiftconn.delete_container, name)
    eq(s3_err.status,404)
    eq(swift_err.http_status, 404)
    eq(swift_err.http_reason, 'Not Found')
    eq(s3_err.reason, 'Not Found')
    eq(swift_err.http_response_content, 'NoSuchBucket')
    eq(s3_err.error_code, 'NoSuchBucket')

def test_delete_container():
    # Delete all buckets
    s3_delete_all_buckets(s3conn)
    # Create bucket using Swift
    name = create_valid_name()
    swiftconn.put_container(name)
    # Delete using S3
    s3conn.delete_bucket(name)
    # List buckets
    eq(swift_list_containers(swiftconn), [])
    eq(s3_list_buckets(s3conn), [])
    # Try deleting again with S3/Swift
    s3_err = assert_raises(boto.exception.S3ResponseError, s3conn.delete_bucket, name)
    swift_err = assert_raises(swiftclient.ClientException, swiftconn.delete_container, name)
    eq(s3_err.status,404)
    eq(swift_err.http_status, 404)
    eq(swift_err.http_reason, 'Not Found')
    eq(s3_err.reason, 'Not Found')
    eq(swift_err.http_response_content, 'NoSuchBucket')
    eq(s3_err.error_code, 'NoSuchBucket')

def test_delete_non_empty_bucket():
    # Create bucket
    name = create_valid_name()
    bucket = s3conn.create_bucket(name)
    # Add test object using S3
    k = Key(bucket)
    k.key = 'test'
    k.set_contents_from_string('this is a test file')
    # Try to delete bucket using Swift
    swift_err = assert_raises(swiftclient.ClientException, swiftconn.delete_container, name)
    eq(swift_err.http_status, 409)
    eq(swift_err.http_reason, 'Conflict')
    eq(swift_err.http_response_content, 'BucketNotEmpty')

def test_delete_non_empty_container():
    # Create container
    name = create_valid_name()
    swiftconn.put_container(name)
    # Add test object using Swift
    swiftconn.put_object(name, 'test', 'this is a test file')
    # Try to delete container using S3
    s3_err = assert_raises(boto.exception.S3ResponseError, s3conn.delete_bucket, name)
    eq(s3_err.status, 409)
    eq(s3_err.reason, 'Conflict')
    eq(s3_err.error_code, 'BucketNotEmpty')

## OBJECT TESTS

def test_list_object_in_bucket():
    # Create container using Swift
    name = create_valid_name()
    swiftconn.put_container(name)
    # Create object using S3
    bucket = s3conn.get_bucket(name)
    k = Key(bucket)
    k.key = 'test'
    k.set_contents_from_string('Create object using S3')
    # Check list of objects
    eq(swift_list_objects(swiftconn, name), ['test'])
    eq(s3_list_objects(s3conn, name), ['test'])

def test_list_object_in_container():
    # Create bucket using S3
    name = create_valid_name()
    bucket = s3conn.create_bucket(name)
    # Create object using Swift
    swiftconn.put_object(name, 'test', 'Create object using Swift')
    # Check list of objects
    eq(s3_list_objects(s3conn, name), ['test'])
    eq(swift_list_objects(swiftconn, name), ['test'])

def test_size_object_in_container():
    # Create container
    name = create_valid_name()
    swiftconn.put_container(name)
    # Create object using S3
    bucket = s3conn.get_bucket(name)
    k = Key(bucket)
    k.key = 'test'
    f = open('/dev/urandom', 'r')
    data = f.read(random.randint(1,30000))
    f.close()
    k.set_contents_from_string(data)
    # Check size
    eq(k.size, swift_size(swiftconn, name, 'test'))

def test_size_object_in_bucket():
    # Create bucket
    name = create_valid_name()
    bucket = s3conn.create_bucket(name)
    # Create object using Swift
    f = open('/dev/urandom', 'r')
    data = f.read(random.randint(1,30000))
    f.close()
    swiftconn.put_object(name, 'test', data)
    k = Key(bucket, 'test')
    k.open_read()
    # Check size
    eq(k.size, swift_size(swiftconn, name, 'test'))

def test_checksum_object_in_container():
    # Create container
    name = create_valid_name()
    swiftconn.put_container(name)
    # Create object using S3
    bucket = s3conn.get_bucket(name)
    k = Key(bucket)
    k.key = 'test'
    f = open('/dev/urandom', 'r')
    data = f.read(random.randint(1,30000))
    f.close()
    k.set_contents_from_string(data)
    # Check checksum
    eq(s3_md5(s3conn, name, 'test'), swift_md5(swiftconn, name, 'test'))

def test_checksum_object_in_bucket():
    # Create bucket
    name = create_valid_name()
    bucket = s3conn.create_bucket(name)
    # Create object using Swift
    f = open('/dev/urandom', 'r')
    data = f.read(random.randint(1,30000))
    f.close()
    swiftconn.put_object(name, 'test', data)
    k = Key(bucket, 'test')
    # Check checksum
    eq(s3_md5(s3conn, name, 'test'), swift_md5(swiftconn, name, 'test'))

def test_delete_object_swift():
    # Create bucket
    name = create_valid_name()
    bucket = s3conn.create_bucket(name)
    # Create object using S3
    bucket = s3conn.get_bucket(name)
    k = Key(bucket)
    k.key = 'test'
    f = open('/dev/urandom', 'r')
    data = f.read(random.randint(1,30000))
    f.close()
    k.set_contents_from_string(data)
    # Delete object with Swift
    swiftconn.delete_object(name, 'test')
    # Check list of objects
    eq(s3_list_objects(s3conn, name), [])
    eq(swift_list_objects(swiftconn, name), [])

def test_delete_object_s3():
    # Create container
    name = create_valid_name()
    swiftconn.put_container(name)
    # Create object using Swift
    f = open('/dev/urandom', 'r')
    data = f.read(random.randint(1,30000))
    f.close()
    swiftconn.put_object(name, 'test', data)
    # Delete object with S3
    bucket = s3conn.get_bucket(name)
    k = Key(bucket, 'test')
    k.delete()
    # Check list of objects
    eq(s3_list_objects(s3conn, name), [])
    eq(swift_list_objects(swiftconn, name), [])

def test_copy_swift_object():
    # Create buckets
    name = create_valid_name()
    bucket = s3conn.create_bucket(name)
    name2 = create_valid_name()
    bucket2 = s3conn.create_bucket(name2)
    # Create object using S3
    bucket = s3conn.get_bucket(name)
    k = Key(bucket)
    k.key = 'test'
    f = open('/dev/urandom', 'r')
    data = f.read(random.randint(1,30000))
    f.close()
    k.set_contents_from_string(data)
    # Copy object using Swift
    swiftconn.put_object(container=name2,obj='test',contents=None,headers={'x-copy-from':name+'/'+'test'})
    # Check list of objects
    eq(s3_list_objects(s3conn, name2), ['test'])
    eq(swift_list_objects(swiftconn, name2), ['test'])
    # Check checksum
    eq(s3_md5(s3conn, name, 'test'), s3_md5(s3conn, name2, 'test'))
    # Check size
    eq(s3_size(s3conn, name, 'test'), s3_size(s3conn, name2, 'test'))

def test_copy_s3_object():
    # Create containers
    name = create_valid_name()
    swiftconn.put_container(name)
    name2 = create_valid_name()
    swiftconn.put_container(name2)
    # Create object using Swift
    f = open('/dev/urandom', 'r')
    data = f.read(random.randint(1,30000))
    f.close()
    swiftconn.put_object(name, 'test', data)
    # Copy object using S3
    bucket = s3conn.get_bucket(name2)
    bucket.copy_key('test', name, 'test')
    # Check list of objects
    eq(s3_list_objects(s3conn, name2), ['test'])
    eq(swift_list_objects(swiftconn, name2), ['test'])
    # Check checksum
    eq(s3_md5(s3conn, name, 'test'), s3_md5(s3conn, name2, 'test'))
    # Check size
    eq(s3_size(s3conn, name, 'test'), s3_size(s3conn, name2, 'test'))