#!/usr/bin/env python3
import sys
import os
from lxml import etree
import configparser
import subprocess
import time

#Variables globales que se utilizaran durante toda la practica
ordenes = ["crear", "arrancar", "parar", "destruir", "help", "prueba","check", "monitor", "haproxy"]
maquinas = ["lb","c1","s1","s2", "s3", "s4", "s5"]
multiples = ["crear", "arrancar", "parar","monitor"]
ips = ["","10.0.1.2","10.0.2.11","10.0.2.12","10.0.2.13","10.0.2.14","10.0.2.15"]
config = configparser.ConfigParser()

def help():
	print("AYUDA:")
	print("Para utilizar el script, escriba \"sudo ./pf1.py <comando> <opción>\"")
	print("Los comandos disponibles son: crear, arrancar, parar, destruir, check, monitor, haproxy y help\n")
	print("Escriba \"sudo ./pf1.py crear <Numero>\" para crear las MV referentes a los servidores.")
	print("	-Siendo <Numero> el número de servidores a crear")
	print("	-El <Número> debe estar comprendido entre 1 y 5.")
	print("	-Si no se introduce valor, se crearán 2 servidores.\n")
	print("Escriba \"sudo ./pf1.py arrancar <opción>\" para arrancar las MV.")
	print("	-La opción sirve para elegir qué máquina arrancar en caso de que sólo quiera arrancarse una.")
	print("	-No introducir opción (escribiendo solo \"sudo ./pf1.py arrancar\") arrancará todas las máquinas.")
	print("	-Las opciones posibles son lb, c1, s1, s2, s3, s4, s5 (en función de cuantos servidores se hayan creado anteriormente.)")
	print("		Por ejemplo, escribir \"sudo ./pf1.py arrancar s1\" arrancará solo el servidor s1.\n")
	print("Escriba \"sudo ./pf1.py parar <opción>\" para parar las MV.\n")
	print("	-La opción sirve para elegir qué máquina parar en caso de que sólo quiera pararse una.")
	print("	-No introducir opción (escribiendo solo \"sudo ./pf1.py parar\") parará todas las máquinas.")
	print("	-Las opciones posibles son lb, c1, s1, s2, s3, s4, s5 (en función de cuantos servidores se hayan arrancado anteriormente.)")
	print("		Por ejemplo, escribir \"sudo ./pf1.py arrancar s1\" arrancará solo el servidor s1.\n")
	print("Escriba \"sudo ./pf1.py destruir\" para eliminar todos los archivos que hayan sido creados.\n")
	print("Escriba \"sudo ./pf1.py check\" para comprobar las conexiones del host con el resto de máquinas.\n")
	print("Escriba \"sudo ./pf1.py monitor <opción> <opción2>\" para monitorizar aspectos del entorno.")
	print("	-Las opciones principales de monitor son: -state -cpus -info")
	print("	-Escribir sólo la primera opción monitorizará todas las máquinas.")
	print("		Por ejemplo, escribir \"sudo ./pf1.py monitor -info\" dará la información de todas las máquinas")
	print("	-Escribir el nombre de una máquina en la segunda opción permite monitorizarla.")
	print("		Por ejemplo, escribir \"sudo ./pf1.py monitor -cpus s1\" dará la información de los cpus sólo de s1\n")
	print("Escriba \"sudo ./pf1.py haproxy\" para comprobar que el haproxy funciona correctamente.")

def check():
	for maquina in maquinas:
		if os.path.isfile('./'+maquina+'.qcow2'):
			print("El fichero qcow2 de "+maquina+" ha sido creado correctamente.")
		if os.path.isfile('./'+maquina+'.xml'):
			print("El fichero xml de "+maquina+" ha sido creado correctamente.")

def crear(num):
#Esto simplemente crea el fichero de configuracion pf1.cfg que lleva dentro el numero de servidores arrancados
#Usamos el modulo configparser por lo tanto se guarda el numero pasado por el comando crear en num_serv de pf1.cfg
	if not os.path.exists('./pf1.cfg'):
		os.mknod('./pf1.cfg')
		config.read('pf1.cfg')
		config.add_section('main')
		for index in range (0, num + 2):
			config.set('main',maquinas[index],maquinas[index])
		config.set('main','num_serv', str(num))
		with open('pf1.cfg', 'w') as f:
			config.write(f)
	else:
		config.read('pf1.cfg')
		for index in range (0, num + 2):
			config.set('main',maquinas[index],maquinas[index])
		config.set('main','num_serv', str(num))
		with open('pf1.cfg', 'w') as f:
			config.write(f)

#Creacion de MVs de los Servidores. Como numero es el introducido con el comando, se crearán ese número de Servidores
#siendo de forma predeterminada 2
	for index in range(num+2):
		server=maquinas[index]
		os.system('qemu-img create -f qcow2 -b cdps-vm-base-pf1.qcow2 '+server+'.qcow2')
		os.system('cp plantilla-vm-pf1.xml '+server+'.xml')

		# Cargamos el fichero xml
		tree = etree.parse(server+'.xml')
		# Obtenemos el nodo raiz e imprimimos su nombre y el valor del atributo 'tipo'
		root = tree.getroot()
		# Buscamos la etiqueta 'nombre' imprimimos su valor y luego lo cambiamos
		name = root.find("name")
		name.text = server

		# Buscamos la etiqueta 'nombre' bajo el nodo 'parte3' e imprimimos su valor y luego lo cambiamos
		sourcefile=root.find("./devices/disk/source[@file='/mnt/tmp/XXX/XXX.qcow2']")
		sourcefile.set("file", "/home/jepeme/Documentos/CDPS/pf1/" +server+".qcow2")

		#Esto solo será necesario en lb (añadir el segundo bridge)
		if server == "lb":
			sourceInterface2 = root.find("./devices")
			interface2 = etree.SubElement(sourceInterface2, "interface", type='bridge')
			source2 = etree.SubElement(interface2, "source", bridge='LAN2')
			model2 = etree.SubElement(interface2, "model", type='virtio')

		#Ahora en función de qué sea tendrá de configuración LAN1 (c1 y lb) o LAN2 (s1-s5)
		sourcebridge=root.find("./devices/interface/source[@bridge='XXX']")
		if server == "lb" or server == "c1":
			sourcebridge.set("bridge", "LAN1")
		else:
			sourcebridge.set("bridge", "LAN2")

		#Se modifican los ficheros xml de cada una de las máquinas
		out=open(server+".xml", "w")
		out.write(etree.tostring(tree, encoding="unicode"))
		out.close()

	#Creacion de los bridges correspondientes a las dos redes virtuales
	os.system("sudo brctl addbr LAN1")
	os.system("sudo brctl addbr LAN2")
	os.system("sudo ifconfig LAN1 up")
	os.system("sudo ifconfig LAN2 up")

	os.system("sudo ifconfig LAN1 10.0.1.3/24")
	os.system("sudo ip route add 10.0.0.0/16 via 10.0.1.1")
	config.read('pf1.cfg')
	numservers= int(config.get('main', 'num_serv'))
	print ("El número de servidores a arrancar es: " +str(numservers))
	print("Configurando interfaces, por favor, espere...")
	os.system("sudo virt-copy-out -a s1.qcow2 /etc/network/interfaces . ; mv interfaces plantillainterfaces")

	for index in range(0,numservers+2):
		maquina = maquinas[index]
		print("Cambiando valores de "+maquina+", por favor, espere...")
		file = open('hostname', 'w+')
		file.write(maquina)
		file.close()
		os.system("sudo virt-copy-in -a "+maquina+".qcow2 hostname /etc")
		file = open('hosts', 'w+')
		file.writelines(["127.0.1.1 "+maquina+" "+maquina,
		"\n127.0.0.1 localhost",
		"\n::1 ip6-localhost ip6-loopback" ,
		"\nfe00::0 ip6-localnet" ,
		"\nff00::0 ip6-mcastprefix",
		"\nff02::1 ip6-allnodes",
		"\nff02::2 ip6-allrouters",
		"\nff02::3 ip6-allhosts"])
		file.close()
		print ("Definiendo "+maquina)
		# Lanzamos s1
		os.system('sudo virsh define ' +maquina+'.xml')
		os.system("sudo virt-copy-in -a "+maquina+".qcow2 hosts /etc")

		if(maquina) == "lb":
			aux = open("plantillainterfaces", "r")
			interfaces = open("interfaces", "w+")
			for line in aux:
				if "iface lo inet loopback" in line:
					interfaces.writelines([line,
					 "\nauto eth1" ,
					 "\niface eth1 inet static",
					 "\naddress 10.0.2.1",
					 "\nnetmask 255.255.255.0",
					 "\nnetwork 10.0.2.0",
					 "\nbroadcast 10.0.2.255",
					 "\nauto eth0",
					 "\niface eth0 inet static",
					 "\naddress 10.0.1.1",
					 "\nnetmask 255.255.255.0",
					 "\nnetwork 10.0.1.0",
					 "\nbroadcast 10.0.1.255\n"])
				else:
					interfaces.write(line)
			interfaces.close()
			aux.close()
			os.system("sudo virt-copy-in -a "+maquina+".qcow2 interfaces /etc/network")

			#Para que el balanceador de tráfico funcione como router al arrancar, modificamos el fichero
			#/etc/sysctl.conf
			print("Configurando correctamente el balanceador de tráfico...")
			os.system("sudo virt-copy-out -a lb.qcow2 /etc/sysctl.conf . ; mv sysctl.conf plantillasysctl.conf")
			aux = open("plantillasysctl.conf", "r")
			sysctl = open("sysctl.conf", "w+")
			for line in aux:
				if "#net.ipv4.ip_forward=1" in line:
					sysctl.write("net.ipv4.ip_forward=1\n")
				else:
					sysctl.write(line)
			sysctl.close()
			aux.close()
			os.system("sudo virt-copy-in -a lb.qcow2 sysctl.conf /etc")
			os.system('rm -f plantillasysctl.conf')
			os.system('rm -f sysctl.conf')

			print(maquina + " :Configurando correctamente el balanceador de tráfico...")
			os.system("sudo virt-copy-out -a lb.qcow2 /etc/haproxy/haproxy.cfg .")
			with open("haproxy.cfg", "a") as file:
			    file.writelines(["frontend lb\n",
							"bind *:80\n",
							"mode http\n",
							"default_backend webservers\n",
							"backend webservers\n",
							"mode http\n",
							"balance roundrobin\n",
							"server s1 10.0.2.11:80 check\n",
							"server s2 10.0.2.12:80 check\n",
							"server s3 10.0.2.13:80 check\n",
							"server s2 10.0.2.14:80 check\n",
							"server s2 10.0.2.15:80 check\n"])
			os.system("sudo virt-copy-in -a lb.qcow2 haproxy.cfg /etc/haproxy")
			os.system('rm -f haproxy.cfg')

		elif(maquina) == "c1":
			aux = open("plantillainterfaces", "r")
			interfaces = open("interfaces", "w+")
			for line in aux:
				if "iface lo inet loopback" in line:
					interfaces.writelines([line,
					 "\nauto eth0" ,
					 "\niface eth0 inet static",
					 "\naddress 10.0.1.2",
					 "\nnetmask 255.255.255.0",
					 "\nnetwork 10.0.1.0",
					 "\nbroadcast 10.0.1.255",
					 "\ngateway 10.0.1.1\n"])
				else:
					interfaces.write(line)
			interfaces.close()
			aux.close()
			os.system("sudo virt-copy-in -a "+maquina+".qcow2 interfaces /etc/network")

		else:
			aux = open("plantillainterfaces", "r")
			interfaces = open("interfaces", "w+")
			for line in aux:
				if "iface lo inet loopback" in line:
					interfaces.writelines([line,
					 "\nauto eth0" ,
					 "\niface eth0 inet static",
					 "\naddress 10.0.2.1"+ str(index-1),
					 "\nnetmask 255.255.255.0",
					 "\nnetwork 10.0.2.0",
					 "\nbroadcast 10.0.1.255",
					 "\ngateway 10.0.2.1\n"])
				else:
					interfaces.write(line)
			interfaces.close()
			aux.close()

			htmlfile = open("index.html", "w+")
			htmlfile.write(maquina)
			htmlfile.close()
			os.system("sudo virt-copy-in -a "+maquina+".qcow2 interfaces /etc/network")
			os.system("sudo virt-copy-in -a "+maquina+".qcow2 index.html /var/www/html")

	check()

def parar():
	if not os.path.exists('./pf1.cfg'):
		print("No existe el archivo pf1.cfg, ¿Has creado ya las Máquinas Virtuales necesarias?")
		print("A continuación verá el comando de ayuda:")
		help()
	else:
		config.read('pf1.cfg')
		numservers= int(config.get('main', 'num_serv'))

		os.system("HOME=/Documentos/CDPS/pf1 sudo virt-manager")
		if data == "all":
			print("Paramos todas")
			for server in range(0,numservers+2):
				os.system("sudo virsh shutdown "+ maquinas[server])
		else:
			# Paramos la maquina pedida
			print ("Deteniendo "+ data)
			os.system('sudo virsh shutdown ' +data)

def destruir():
	if not os.path.exists('./pf1.cfg'):
		print("No existe el archivo pf1.cfg, ¿Has creado ya las Máquinas Virtuales necesarias?")
		print("A continuación verá el comando de ayuda:")
		help()
	else:
		config.read('pf1.cfg')
		numservers= int(config.get('main', 'num_serv'))
		print ("El número de servidores a destruir es: " +str(numservers))
		os.system("sudo ifconfig LAN1 down")
		os.system("sudo ifconfig LAN2 down")
		os.system("sudo brctl delbr LAN1")
		os.system("sudo brctl delbr LAN2")

		for server in maquinas:
			if os.path.isfile('./'+server+'.qcow2'):
				os.system('rm -f '+server+'.qcow2')
			if os.path.isfile('./'+server+'.xml'):
				os.system('rm -f '+server+'.xml')

		for server in range(0,numservers+2):
			os.system("sudo virsh shutdown "+ maquinas[server])
			os.system("sudo virsh destroy "+maquinas[server])
			os.system("sudo virsh undefine "+maquinas[server])

		if os.path.exists('hosts'):
			os.system("rm -f hosts")
		if os.path.exists('hostname'):
			os.system("rm -f hostname")
		if os.path.exists('index.html'):
			os.system('rm -f index.html')
		if os.path.exists("pf1.cfg"):
			os.system("rm -f pf1.cfg")
		if os.path.exists("plantillainterfaces"):
			os.system("rm -f plantillainterfaces")
		if os.path.exists("interfaces"):
			os.system("rm -f interfaces")

	print("El proyecto ha sido destruido")

def arrancar():
	if not os.path.exists('./pf1.cfg'):
		print("No existe el archivo pf1.cfg, ¿Has creado ya las Máquinas Virtuales necesarias?")
		print("A continuación verá el comando de ayuda:")
		help()
	else:
		config.read('pf1.cfg')
		numservers= int(config.get('main', 'num_serv'))

		os.system("HOME=/Documentos/CDPS/pf1 sudo virt-manager")
		if data == "all":
			print("Arrancamos todas")
			for index in range(0, numservers+2):
				maquina = maquinas[index]
				print ("Arrancando "+maquina+ "...")
				os.system('sudo virsh start ' +maquina)
				os.system("xterm -rv -sb -rightbar -fa  monospace -fs  10  -title '"+maquina+"' -e  'sudo virsh console "+maquina+"' &")
		else:
			# Lanzamos la elegida
			print ("Arrancando "+data+ "...")
			os.system('sudo virsh start ' +data)
			os.system("xterm -rv -sb -rightbar -fa  monospace -fs  10  -title '"+data+"' -e  'sudo virsh console "+data+"' &")

def haproxy():
	print("Probando que el haproxy funciona correctamente:")
	t_end = time.time() + 8
	while time.time() < t_end:
		os.system("curl 10.0.1.1")
		time.sleep(0.4)
		print("")

#Comprueba si hay conexión al resto de máquinas desde el host
def checkConnections():
	if not os.path.exists('./pf1.cfg'):
		print("No existe el archivo pf1.cfg, ¿Has creado ya las Máquinas Virtuales necesarias?")
		print("A continuación verá el comando de ayuda:")
		help()
	else:
		config.read('pf1.cfg')
		numservers= int(config.get('main', 'num_serv'))
	for index in range (1,numservers+2):
		print("Comprobando conexión con " + maquinas[index])
		os.system("ping " + ips[index] + " -c 4")
		print("\n")
	for index in range (1,numservers+1):
		print("Comprobando conexión con el html de " + maquinas[index+1])
		os.system("curl 10.0.2.1" + str(index))
		print("\n")

def monitor():
	config.read('pf1.cfg')
	numservers= int(config.get('main', 'num_serv'))
	if data == "-state":
		if len(sys.argv) == 3:
			print("Mostrando la info de las cpus de todas las máquinas.")
			for maquina in upList:
				print("Información sobre la máquina " + maquina)
				os.system("sudo virsh domstate " + maquina)
		elif len(sys.argv) == 4:
			if sys.argv[3] in upList:
				print("Información sobre la máquina " + sys.argv[3])
				os.system("sudo virsh domstate " + sys.argv[3])
			else:
				print("Dicha máquina no existe")
		else:
			print("Dicho comando no existe")
			time.sleep(1)
			help()
	elif data == "-cpus":
		if len(sys.argv) == 3:
			for maquina in upList:
				os.system("clear")
				print("Mostrando la info de las cpus de todas las máquinas.")
				print("Información sobre la máquina " + maquina)
				os.system("sudo virsh cpu-stats " + maquina)
				time.sleep(2)
		elif len(sys.argv) == 4:
			if sys.argv[3] in upList:
				print("Mostrando la info de " + sys.argv[3])
				print("Información sobre la máquina " + sys.argv[3])
				os.system("sudo virsh cpu-stats " + sys.argv[3])
			else:
				print("Dicha máquina no existe")
		else:
			print("Dicho comando no existe")
			time.sleep(1)
			help()
	elif data == "-info":
		if len(sys.argv) == 3:
			for maquina in upList:
				os.system("clear")
				print("Mostrando la info de las cpus de todas las máquinas.")
				print("Información sobre la máquina " + maquina)
				os.system("sudo virsh dominfo " + maquina)
				time.sleep(2)
		elif len(sys.argv) == 4:
			if sys.argv[3] in upList:
				print("Mostrando la info de " + sys.argv[3])
				print("Información sobre la máquina " + sys.argv[3])
				os.system("sudo virsh dominfo " + sys.argv[3])
			else:
				print("Dicha máquina no existe")
		else:
			print("Dicho comando no existe")
			time.sleep(1)
			help()
	else:
		print("Comando no disponible")
		time.sleep(2)
		help()

def ejecutarOrden(orden):
    if orden in ordenes:
        if orden == "crear":
            crear(numero)
        if orden == "arrancar":
            arrancar()
        if orden == "parar":
            parar()
        if orden == "destruir":
            destruir()
        if orden == "haproxy":
            haproxy()
        if orden == "check":
            checkConnections()
        if orden == "help":
            help()
        if orden == "monitor":
            monitor()
    else:
        print("La orden " + orden + " NO existe")

try:
	orden = sys.argv[1]
	if sys.argv[1] in multiples:
		if sys.argv[1] == "crear":
			if len(sys.argv) == 3:
				if int(sys.argv[2]) < 6 and int(sys.argv[2]) > 0:
					numero = int(sys.argv[2])
					ejecutarOrden(orden)
				else:
					print("El número elegido es incorrecto, escoja otro distinto")
			else:
				numero = 2
				ejecutarOrden(orden)

		if sys.argv[1] == "monitor":
			if not os.path.exists('./pf1.cfg'):
				print("No existe el archivo pf1.cfg, ¿Has creado y arrancado ya las Máquinas Virtuales necesarias?")
				print("A continuación verá el comando de ayuda:")
				help()
			else:
				try:
					config.read('pf1.cfg')
					maquinascreadas= int(config.get('main',"num_serv"))
					upList = []
					for index in range (0,maquinascreadas+2):
						upList.append(maquinas[index])
					if len(sys.argv) > 2:
						if sys.argv[2] == "-state":
							data = "-state"
							ejecutarOrden(orden)
						elif sys.argv[2] == "-cpus":
							data = "-cpus"
							ejecutarOrden(orden)
						elif sys.argv[2] == "-info":
							data = "-info"
							ejecutarOrden(orden)
						else:
							print("Dicha opción no existe, pruebe con otra.")
							print("A continuación verá el comando de ayuda.")
							time.sleep(2)
							help()
					else:
						print("Debes seleccionar una opción para monitorizar")
						time.sleep(2)
						help()
				except:
					print("Error, posiblemente no haya arrancado aún el sistema.")
					print("A continuación verá el comando de ayuda.")
					time.sleep(2)
					help()

		if sys.argv[1] == "arrancar":
			if not os.path.exists('./pf1.cfg'):
				print("No existe el archivo pf1.cfg, ¿Has creado ya las Máquinas Virtuales necesarias?")
				print("A continuación verá el comando de ayuda:")
				help()
			else:
				try:
					config.read('pf1.cfg')
					maquinascreadas= int(config.get('main',"num_serv"))
					upList = []
					for index in range (0,maquinascreadas+2):
						upList.append(maquinas[index])
					if len(sys.argv) > 2:
						if sys.argv[2] in upList:
							print("Arrancando máquina " + sys.argv[2])
							data = sys.argv[2]
							ejecutarOrden(orden)
						elif sys.argv[2] in maquinas:
							print("La máquina " + sys.argv[2] + " no ha sido creada, no puede arrancarse.")
							print("Solo han sido creadas las máquinas: ")
							print(upList)
						else:
							print("Dicha opción no existe, pruebe con otra.")
							print("A continuación verá el comando de ayuda.")
							time.sleep(2)
							help()
					else:
						data = "all"
						ejecutarOrden(orden)
				except:
					print("Error, posiblemente no haya creado aún el sistema.")
					print("A continuación verá el comando de ayuda.")
					time.sleep(2)
					help()

		if sys.argv[1] == "parar":
			if not os.path.exists('./pf1.cfg'):
				print("No existe el archivo pf1.cfg, ¿Has creado ya las Máquinas Virtuales necesarias?")
				print("A continuación verá el comando de ayuda:")
				help()
			else:
				try:
					config.read('pf1.cfg')
					maquinascreadas= int(config.get('main',"num_serv"))
					upList = []
					for index in range (0,maquinascreadas+2):
						upList.append(maquinas[index])
					if len(sys.argv) > 2:
						if sys.argv[2] in upList:
							print("Parando máquina " + sys.argv[2])
							data = sys.argv[2]
							try:
								ejecutarOrden(orden)
							except:
								print("Esa máquina no está arrancada")
						elif sys.argv[2] in maquinas:
							print("La máquina " + sys.argv[2] + " no ha sido arrancada, no puede arrancarse.")
							print("Solo han sido creadas las máquinas: ")
							print(upList)
						else:
							print("Dicha opción no existe, pruebe con otra.")
							print("A continuación verá el comando de ayuda.")
							time.sleep(2)
							help()
					else:
						data = "all"
						ejecutarOrden(orden)
				except:
					print("Error, posiblemente no haya creado aún el sistema.")
					print("A continuación verá el comando de ayuda.")
					time.sleep(2)
					help()
	else:
		if  len(sys.argv) == 3 and sys.argv[2] not in multiples:
			print("Solo se permite un tercer argumento para crear las MV o para monitorizar el entorno con la orden monitor.")
		else:
			ejecutarOrden(orden)
except:
	help()


'''
Author: Aslan
Type: Practice, also a useless implementation. This problem is not categorical but my attempt is for categorical...
What: Deep-learning, policy gradient, gradient descent
Entropy: https://towardsdatascience.com/demystifying-entropy-f2c3221e2550
Cross entropy: https://towardsdatascience.com/demystifying-cross-entropy-e80e3ad54a8, https://ml-cheatsheet.readthedocs.io/en/latest/loss_functions.html
'''

import tensorflow as tf
import numpy as np
import gym

# Openai
env = gym.make('Pendulum-v0')
env = env.unwrapped
env.seed(1)

# Parameters
state_size = 3
action_size = 2
max_epochs = 200
learnRate = 0.01
gamma = 0.95

# Reward handling
def rewardHandle(reward_to_handle):
    adjusted_rewards = np.zeros_like(reward_to_handle)
    cumulative = 0.0
    for i in reversed(range(len(reward_to_handle))):
        cumulative = cumulative * gamma + reward_to_handle[i]
        adjusted_rewards[i] = cumulative
    mean = np.mean(adjusted_rewards)
    std = np.std(adjusted_rewards)
    adjusted_rewards = (adjusted_rewards - mean) / std

    return adjusted_rewards

with tf.name_scope("inputs"):
    input_ = tf.placeholder(tf.float32, [None, state_size], name="input_")
    actions = tf.placeholder(tf.float32, [None, action_size], name="actions")
    adjusted_rewards = tf.placeholder(tf.float32, [None,], name="adjusted_rewards")

    # Build layers
    with tf.name_scope("f1"):
        fc1 = tf.contrib.layers.fully_connected(inputs=input_,
                                                num_outputs=16,
                                                activation_fn=tf.nn.relu,
                                                weights_initializer=tf.contrib.layers.xavier_initializer())

    with tf.name_scope("f2"):
        fc2 = tf.contrib.layers.fully_connected(inputs=fc1,
                                                num_outputs=16,
                                                activation_fn=tf.nn.relu,
                                                weights_initializer=tf.contrib.layers.xavier_initializer())

    with tf.name_scope("f3"):
        fc3 = tf.contrib.layers.fully_connected(inputs=fc2,
                                                num_outputs=action_size,
                                                activation_fn=None,
                                                weights_initializer=tf.contrib.layers.xavier_initializer())

    with tf.name_scope("softmax"):
        action_distribution = tf.nn.softmax(fc3)

    with tf.name_scope("loss"):
        neg_log_prob = tf.nn.softmax_cross_entropy_with_logits_v2(logits=fc3, labels=actions)
        loss = tf.reduce_mean(neg_log_prob * adjusted_rewards)

    with tf.name_scope("train"):
        train_opt = tf.train.AdamOptimizer(learning_rate=learnRate).minimize(loss)

# Training
ep_states, ep_actions, ep_rewards = [], [], []
max_reward = 0

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    for epochs in range(max_epochs):
        state = env.reset()
        env.render()

        for _ in range(200):

            # non-deterministic env
            actions_prob_dist = sess.run(action_distribution, feed_dict={input_: state.reshape([1, 3])})
            action = np.random.choice(range(actions_prob_dist.shape[1]), p=actions_prob_dist.ravel())

            if action == 0:
                oAction = [1,-2,2]
            else:
                oAction = [-1,-2,2]

            new_state, reward, done, extra = env.step(oAction)
            ep_states.append(state)
            action_ = np.zeros(action_size)
            action_[action] = 1
            ep_actions.append(action_)
            ep_rewards.append(reward)

        input_reward = rewardHandle(ep_rewards)
        sess.run(train_opt, feed_dict={input_: np.vstack(np.array(ep_states)),
                                        actions: np.vstack(np.array(ep_actions)),
                                        adjusted_rewards: input_reward})

        max_reward = max(max_reward, np.sum(ep_rewards))
        print("Max reward so far: ", max_reward)
        ep_states, ep_actions, ep_rewards = [], [], []
        state = new_state


from django.db import models
from django.contrib.comments.models import Comment
from django.db.models.signals import pre_save, post_save, post_delete
from customsetting.utility import remove_bad_char
from django.db import close_connection
from django.core.cache import cache

def count_comments( sender, instance, **kwargs ):
    """
    Signal Listener to count all comments of a particular question whenever a comment is posted.
    """

    if instance.content_type.name == 'question':
        from content.models import Question
#        close_connection()
        count = Comment.objects.using( 'htcMaster' ).filter( object_pk = instance.object_pk, content_type = instance.content_type ).count()
        question = Question.objects.get( pk = instance.object_pk )
        question.anscount = count
        question.save()
    if instance.content_type.name in ['article','question'] and cache.has_key( 'question_article_frontend' ):
        cache.delete( 'question_article_frontend' )

def modify_comment( sender, instance, **kwargs ):
    #Truncate the URLS and Phone Numbers when Question is posted.
    #Get the question object
    from content.models import Question
    if isinstance( instance, Question ):
        if not instance.author.is_staff:
            instance.content = remove_bad_char( instance.content )
    else:
        if not instance.user.is_staff:
            instance.comment = remove_bad_char( instance.comment )

class NewComment( Comment ):
    abuse_count = models.IntegerField( default = 0, blank = True, null = True )
    class Meta:
        verbose_name_plural = "Comments"
        ordering = ['-submit_date']

    def save(instance, *args, **kwargs):
        super(NewComment,instance).save()
        if instance.content_type.name == 'question':
            from content.models import Question
            close_connection()
            count = Comment.objects.using('htcMaster').filter(object_pk=instance.object_pk,content_type=instance.content_type).count()
            question = Question.objects.get(pk=instance.object_pk)
            question.anscount = count
            question.save()
        if instance.content_type.name in ['article','question'] and cache.has_key('question_article_frontend'):
            cache.delete('question_article_frontend')

pre_save.connect(modify_comment, sender=NewComment)

#post_save.connect(count_comments, sender=NewComment)

post_delete.connect( count_comments, sender = Comment )


# -*- coding: utf-8 -*-

from odoo.tests import TransactionCase, tagged, Form
from odoo.exceptions import ValidationError
from odoo import fields
from .test_sale_common import TestCommonSaleNoChart
import logging

_logger = logging.getLogger(__name__)

@tagged('ipx_product_fob_compute', 'post_install', '-at_install')
class TestSaleOrderLine(TestCommonSaleNoChart):
    def setUp(self):
        super(TestSaleOrderLine, self).setUp()

        self.uom_unit = self.env.ref('uom.product_uom_unit')

        self.product = self.env['product'].create({
            'name':'Producto',
            'type':'consu',
            'uom_id':self.uom_unit.id,
            'uom_po_id':self.uom_unit.id,
        })

        self.sale_order = self.env['sale.order'].create({

        })


import os
import glob
import time
import random
import json
import traceback
from multiprocessing import Process, Lock
from multiprocessing.managers import BaseManager
from tqdm import tqdm
from kafka import KafkaConsumer, KafkaProducer
import sklearn.metrics as metrics
from sklearn.preprocessing import label_binarize
import numpy as np
import pandas as pd
import scipy.io.arff as arff
import tensorflow as tf
from datastream_minerva.models import create_model

# Object shared among processes
class Consumer:

    def __init__(self, batch_size=10, num_batches_fed=40, topic='', results_path='./', bootstrap_servers=None,
                 from_beginning=False, debug=True, two_gpu=False, create_model_func='cnn', clf_name=None, time_out_ms=10000):
        self.batch_size = batch_size
        self.num_batches_fed = num_batches_fed
        self.topic = topic
        self.data_set_name = self.topic.replace('streams_', '')
        self.results_path = os.path.join(results_path, self.data_set_name)
        self.bootstrap_servers = bootstrap_servers
        self.from_beginning = from_beginning
        self.debug = debug
        self.two_gpu = two_gpu
        self.create_model_func = create_model_func
        self.time_out_ms = time_out_ms
        self.new_model_available = False
        self.buffer = []
        self.time_out = False
        self.finished = False
        self.history = {self.data_set_name: {
            'data': pd.DataFrame(),
            'metrics': pd.DataFrame()
        }}
        self.train_time = 0.
        self.x_training = None
        self.y_training = None
        self.batch_counter = 0
        self.count = 0
        self.initial_training_set = []
        self.num_features = None
        self.num_classes = None
        self.weights = None
        self.available = True
        self.clf_name = clf_name if clf_name is not None else 'ADLStream_{0}_{1}x{2}'.format(str(self.create_model_func),str(num_batches_fed),str(batch_size))
        self.average = 'weighted'
        self.loss_func = 'categorical_crossentropy'
        self.columns = None
        self.classes = None
        self.prequential_kappa = None
        self.first_history = 0

    def add(self, ls):
        if not self.initial_training_set:
            self.initial_training_set = [] + ls
        else:
            self.buffer = self.buffer + ls
            print("#1", len(self.buffer))

    def next(self):
        if len(self.buffer) == 0:
            return None
        res = self.buffer[0]
        self.buffer.pop(0)
        self.count = self.count + 1
        return res

    def get_timeout_ms(self):
        return self.time_out_ms

    def is_time_out(self):
        return self.time_out

    def set_time_out(self, b=True):
        self.time_out = b

    def get_count(self):
        return self.count

    def get_batch_size(self):
        return self.batch_size

    def get_num_batches_fed(self):
        return self.num_batches_fed

    def get_bootstrap_servers(self):
        return self.bootstrap_servers

    def get_from_begining(self):
        return self.from_beginning

    def get_debug(self):
        return self.debug

    def is_two_gpu(self):
        return self.two_gpu

    def get_topic(self):
        return self.topic

    def get_data_set_name(self):
        return self.data_set_name

    def get_history(self):
        print("#2", len(self.history),
              len(self.history[list(self.history.keys())[0]]),
              self.history[list(self.history.keys())[0]]['data'].values.shape,
              self.history[list(self.history.keys())[0]]['metrics'].values.shape,)
        return self.history[self.data_set_name]

    def is_first_history(self):
        if self.first_history<2:
            self.first_history += 1
            return True
        else:
            return False
    def append_history(self, k, v):
        dir_name = os.path.dirname(__file__)
        file_path = os.path.join(dir_name, self.get_results_path(), 'ADLStream_' + self.get_clf_name())
        if not os.path.exists(file_path):
            os.makedirs(file_path)
        file_path = os.path.join(file_path, k + '.csv')
        if self.is_first_history():
            with open(file_path, 'w') as fd:
                fd.write(','.join([str(c) for c in v.columns])+'\n')
        if k == 'metrics':
            values = v.values if len(v.values.shape)>1 else v.values.reshape((1,v.values.shape[0]))
            values_csv = '\n'.join([ ','.join([str(c) for c in r]) for r in values])+'\n'
            with open(file_path, 'a') as fd:
                fd.write(values_csv)
        elif k == 'data':
            values = v.values if len(v.values.shape) > 1 else v.values.reshape((1, v.values.shape[0]))
            values_csv = '\n'.join([','.join([str(c) for c in r]) for r in values]) + '\n'
            with open(file_path, 'a') as fd:
                fd.write(values_csv)
        else:
            raise Exception("history key must be 'data' or 'metrics'")

    def get_results_path(self):
        return self.results_path

    def get_clf_name(self):
        return self.clf_name

    def get_average(self):
        return self.average

    def set_average(self, av):
        self.average = av

    def get_loss_function(self):
        return self.loss_func

    def set_loss_function(self, lf):
        self.loss_func = lf

    def get_columns(self):
        return self.columns

    def set_columns(self, c):
        self.columns = c

    def get_classes(self):
        return self.classes

    def set_classes(self, c):
        self.classes = c

    def get_buffer_len(self):
        return len(self.buffer)

    def get_initial_training_set(self):
        return self.initial_training_set

    def set_num_features(self, nf):
        self.num_features = nf

    def get_num_features(self):
        return self.num_features

    def get_num_classes(self):
        return len(self.get_classes())

    def set_weights(self, w):
        self.weights = w

    def get_weights(self):
        return self.weights

    def set_new_model_available(self, b=True):
        self.new_model_available = b

    def is_new_model_available(self):
        return self.new_model_available

    def is_finished(self):
        return self.finished

    def set_finished(self):
        self.finished = True

    def is_available(self):
        return self.available

    def set_available(self, b):
        self.available = b

    def add_train_time(self, new_train_time):
        self.train_time += new_train_time

    def get_train_time(self):
        res = self.train_time
        self.train_time = 0
        return res

    def add_training_data(self, x, y):
        if self.x_training is None or self.y_training is None:
            self.x_training = x
            self.y_training = y
        else:
            self.x_training = np.vstack((self.x_training, x))
            self.y_training = np.vstack((self.y_training, y))
            self.batch_counter += 1
        print("#3", self.x_training.shape, self.y_training.shape)

    def get_training_data(self):
        x, y = np.copy(self.x_training), np.copy(self.y_training)

        num_instances_to_train = self.batch_size * self.num_batches_fed
        if len(x) > num_instances_to_train:
            self.x_training = self.x_training[-num_instances_to_train:]
            self.y_training = self.y_training[-num_instances_to_train:]
            indices = list(range(len(x)))
            k, m = divmod(len(indices), 4)
            w = [1/8]*(k+1 if m > 0 else k) + [1/4]*(k+1 if m > 1 else k) + [1/2]*(k+1 if m > 2 else k) + [10]*k
            w = [e/sum(w) for e in w]
            indices = sorted(np.random.choice(indices, num_instances_to_train, False, w))
            x = np.take(x, indices, axis=0)
            y = np.take(y, indices, axis=0)

        return x, y

    def create_model(self, num_features, num_classes, loss_func):
        return create_model(self.create_model_func, num_features, num_classes, loss_func)
        # try:
        #     model = self.create_model_func[0](num_features, num_classes, loss_func)
        #     return model
        # except Exception:
        #     raise ValueError('Create-model function is not well defined')

    def get_create_model_func(self):
        return self.create_model_func

    def update_prequential_kappa(self, new_kappa):
        if self.prequential_kappa is None:
            self.prequential_kappa = new_kappa
        else:
            fading_factor = 0.98
            self.prequential_kappa = (new_kappa + fading_factor * self.prequential_kappa) / (1 + fading_factor)
        print("#4", self.prequential_kappa)
        return self.prequential_kappa

# Get next message from Kafka
def read_message(consumer):
    message = consumer.next()
    return message

# Save Metrics
def save_history(consumer, x, y, y_pred, test_time, train_time):
    # print(y_pred)
    # print(y)
    y_pred = np.argmax(y_pred, axis=1)
    y = np.argmax(y, axis=1)
    # print(y_pred)
    # print(y)

    x = np.array(x)
    if len(x.shape) == 3:
        x = x.reshape(x.shape[0], -1)

    records = pd.DataFrame(np.hstack((x, y.reshape(-1, 1), y_pred.reshape(-1, 1))))
    records.columns = consumer.get_columns()
    consumer.append_history('data', records)

    prec, recall, fbeta, support = metrics.precision_recall_fscore_support(y, y_pred, average=consumer.get_average())
    accuracy = metrics.accuracy_score(y, y_pred)
    f1_score = metrics.f1_score(y, y_pred, average='weighted')
    conf_matrix = metrics.confusion_matrix(y, y_pred)

    try:
        tn, fp, fn, tp = conf_matrix.ravel()
    except ValueError as ve:
        tn, fp, fn, tp = conf_matrix[0][0], 0, 0, 0

    metrics_record = pd.DataFrame({
        'total': [len(consumer.get_history()['data'])],
        'tn': [tn],
        'fp': [fp],
        'fn': [fn],
        'tp': [tp],
        'precision': [prec],
        'recall': [recall],
        'f1': [f1_score],
        'fbeta': [fbeta],
        'accuracy': [accuracy],
        'train_time': [train_time],
        'test_time': [test_time]
    })
    consumer.append_history('metrics', metrics_record)

# Create results files
def write_results_file(consumer):
    dir_name = os.path.dirname(__file__)
    #file_path = os.path.join(dir_name, consumer.get_results_path(), 'keras_parallel_' + consumer.get_clf_name())
    file_path = os.path.join(dir_name, consumer.get_results_path(), 'ADLStream_' + consumer.get_clf_name())
    if not os.path.exists(file_path):
        os.makedirs(file_path)

    history_data = consumer.get_history()['data']
    history_data.columns = consumer.get_columns()

    # history_data.to_csv(os.path.join(file_path, 'data.csv'), index=False)
    # consumer.get_history()['metrics'].to_csv(os.path.join(file_path, 'metrics.csv'),
    #                                          columns=('total', 'tp', 'tn', 'fp', 'fn', 'precision',
    #                                                   'recall', 'f1', 'fbeta',
    #                                                   'accuracy', 'train_time', 'test_time'),
    #                                          index=False)

def train_model(x_train, y_train, consumer, model, index=-1, device=None):
    # Train
    # if consumer.get_debug():
    print('P' + str(index) + ' (' + device + '):  Training with the last {} last instances'.format(len(x_train)))
    train_time_start = time.time()
    model.fit(x_train, y_train, consumer.get_batch_size(), epochs=1, verbose=0)
    train_time_end = time.time()
    train_time = train_time_end - train_time_start

    # Save model weights
    consumer.set_weights(model.get_weights())

    # Notify new model available
    consumer.set_new_model_available()

    return train_time

def classify(model, input_data):
    test_time_start = time.time()
    y_pred = model.predict(input_data, verbose=0)
    test_time_end = time.time()
    test_time = test_time_end - test_time_start

    return y_pred, test_time

# DNN training process
def dnn_train(index, consumer, lock_training_data):
    # # Specify GPU to use
    gpus = tf.config.experimental.list_physical_devices('GPU')
    if len(gpus) >= 2:
        device = gpus[0]
        tf.config.experimental.set_memory_growth(device, True)
        tf.config.experimental.set_visible_devices(device, 'GPU')
        device_name = device[0]
    else:
        device_name = '/GPU:0'
        print("ERROR - TRAINING PROCESS: ADLStream needs at least 2 GPUs.")
        print("ERROR - TRAINING PROCESS: {0} GPUs found: {1}".format(len(gpus), gpus))

    # Wait until first window available.
    while not consumer.get_initial_training_set():
        if consumer.is_time_out():
            return
        pass

    # Create and train model for the first time.
    initial_training_set = consumer.get_initial_training_set()
    window_y, window_x = [], []

    for record in initial_training_set:
        consumer.set_classes(record.pop('classes', None))
        window_y.append(record.pop('class', None))
        window_x.append(list(record.values()))

        if not consumer.get_columns():
            aux_columns = list(record.keys())
            aux_columns.extend(['class', 'prediction'])
            consumer.set_columns(aux_columns)

    if consumer.get_num_classes() == 2:
        consumer.set_loss_function('binary_crossentropy')
        consumer.set_average('binary')

    x = np.expand_dims(np.array(window_x), axis=-1)
    y = label_binarize(window_y, consumer.get_classes())
    if consumer.get_num_classes()==2:
        y = np.eye(2)[y.flatten()]
    with lock_training_data:
        consumer.add_training_data(x, y)
    consumer.set_num_features(x.shape[1])

    # create model, train it and save the weights.

    # model = consumer.create_model(consumer.get_num_features(), consumer.get_num_classes(), consumer.get_loss_function())
    model = create_model(consumer.get_create_model_func(), consumer.get_num_features(), consumer.get_num_classes(), consumer.get_loss_function())

    with lock_training_data:
        x_train, y_train = consumer.get_training_data()
    train_model(x_train, y_train, consumer, model, index, device_name)

    # Main loop
    while not consumer.is_finished():
        with lock_training_data:
            x_train, y_train = consumer.get_training_data()
        train_time = train_model(x_train, y_train, consumer, model, index, device_name)
        consumer.add_train_time(train_time)

# DNN classifying process
def dnn_classify(index, consumer, lock_messages, lock_training_data):
    #
    # Specify GPU to use
    gpus = tf.config.experimental.list_physical_devices('GPU')
    if len(gpus) >= 2:
        device = gpus[1]
        tf.config.experimental.set_memory_growth(device, True)
        tf.config.experimental.set_visible_devices(device, 'GPU')
        device_name = device[0]
    else:
        device_name = "/CPU:0"
        print("ERROR - CLASSIFYING PROCESS: ADLStream needs at least 2 GPUs.")
        print("ERROR - CLASSIFYING PROCESS: {0} GPUs found: {1}".format(len(gpus), gpus))
        # consumer.set_finished()

    window_y, window_x = [], []
    batch_size = consumer.get_batch_size()
    count_messages = 0
    y_pred_history = None
    x_pred_history = None
    y_history = None
    test_time = 0.

    # Wait until model is created and trained for the first time.
    while not consumer.is_new_model_available():
        if consumer.is_time_out():
            return
        pass

    # create model and load weights
    model = create_model(consumer.get_create_model_func(), consumer.get_num_features(), consumer.get_num_classes(),
                         consumer.get_loss_function())

    model.set_weights(consumer.get_weights())

    # Main loop
    while True:
        # Read messages
        with lock_messages:
            record = read_message(consumer)
        # if no message received...
        if record is None:
            # if timeout = true --> break
            if consumer.is_time_out():
                consumer.set_finished()
                if consumer.get_debug():
                    print("P" + str(index) + ": Finish")
                if y_pred_history is not None:
                    save_history(consumer, x_pred_history, y_history, y_pred_history, test_time, 0.)
                # TODO: WHY????? write_results_file(consumer)
                break
            # ToDo: what if time_out is not True (wait? continue asking?)
            if not consumer.is_time_out():
                if consumer.get_debug():
                    print("P" + str(index) + ": ... waiting feeder")
                time.sleep(0.2)

        else:
            _ = record.pop('classes', None)
            record_class = record.pop('class', None)
            record_values = list(record.values())

            window_y.append(record_class)
            window_x.append(record_values)

            y = label_binarize([window_y[-1]], consumer.get_classes())
            if consumer.get_num_classes() == 2:
                y = np.eye(2)[y.flatten()]

            x_pred = np.expand_dims(np.array([list(record_values)]), axis=-1)
            # Update model if new model available
            if consumer.is_new_model_available():
                model.set_weights(consumer.get_weights())
                consumer.set_new_model_available(False)
            # Classify
            y_pred, test_time_aux = classify(model, x_pred)
            test_time += test_time_aux

            if y_pred_history is None:
                y_pred_history = np.array(y_pred)
            else:
                y_pred_history = np.append(y_pred_history, y_pred, axis=0)

            if x_pred_history is None:
                x_pred_history = np.array(x_pred)
            else:
                x_pred_history = np.append(x_pred_history, x_pred, axis=0)

            if y_history is None:
                y_history = np.array(y)
            else:
                y_history = np.append(y_history, y, axis=0)

            # if window completed
            if len(window_x) % batch_size == 0:
                count_messages += batch_size
                if consumer.get_debug():
                    print('P' + str(index) + ' (' + device_name + '):  {} instances classified'.format(
                        count_messages))
                # Format window data
                x = np.expand_dims(np.array(window_x), axis=-1)
                window_y = list(np.array(window_y))
                y = label_binarize(window_y, consumer.get_classes())
                if consumer.get_num_classes() == 2:
                    y = np.eye(2)[y.flatten()]

                with lock_training_data:
                    consumer.add_training_data(x, y)
                window_y, window_x = [], []

                # Save metrics
                save_history(consumer, x_pred_history, y_history, y_pred_history, test_time, consumer.get_train_time())
                y_pred_history = None
                x_pred_history = None
                y_history = None
                test_time = 0.

# Buffer feeder
def buffer_feeder(consumer, lock):
    batch_size = consumer.get_batch_size()
    num_batches_fed = consumer.get_num_batches_fed()
    kafka_consumer = KafkaConsumer(consumer.get_topic(),
                                   group_id='DSClassification_Consumer_par_{}'.format(random.randrange(999999)),
                                   bootstrap_servers=consumer.get_bootstrap_servers(),
                                   auto_offset_reset='earliest' if consumer.get_from_begining() else 'latest',
                                   consumer_timeout_ms=consumer.get_timeout_ms(),
                                   value_deserializer=lambda m: json.loads(m.decode('ascii')))
    new_batch = []
    i = 0
    while True:
        try:
            message = next(kafka_consumer).value
            i += 1
            new_batch.append(message)
        except StopIteration:
            if consumer.get_debug():
                print("BF: finish" + " - " + str(i) + " messages")
            if new_batch:
                with lock:
                    consumer.add(new_batch)
            consumer.set_time_out()
            break
        except Exception:
            print('Exception getting new messages from kafka')
            print(traceback.format_exc())

        if len(new_batch) == batch_size:
            with lock:
                consumer.add(new_batch)
                len_buffer = consumer.get_buffer_len()
            new_batch = []
            # Control buffer size
            if len_buffer > 100 * batch_size:
                print("PB  BUFFER FULL ({}). Waiting {}s".format(len_buffer, (len_buffer/(100*batch_size))))
                time.sleep(len_buffer / (100 * batch_size))

def runADLStream(topic, create_model_func='cnn', two_gpu=True, batch_size=10, num_batches_fed=40, debug=True, output_path='./ADLStreamResults/', from_beginning=True, time_out_ms=10000, bootstrap_servers='localhost:9092', clf_name=None):
    bootstrap_servers = bootstrap_servers.split(' ')
    print("Consuming from bootstrap_servers: " + str(bootstrap_servers))
    print('Topic:', topic)
    print('Two GPU:', two_gpu)
    print('Batch size:', batch_size)
    print('Number of batches fed:', num_batches_fed)
    print('Output path:', output_path)
    print('time out (ms)', time_out_ms)

    BaseManager.register('Consumer', Consumer)
    manager = BaseManager()
    manager.start()
    consumer = manager.Consumer(batch_size=batch_size, num_batches_fed=num_batches_fed, results_path=output_path,
                                topic=topic, clf_name=clf_name,
                                bootstrap_servers=bootstrap_servers, from_beginning=from_beginning, debug=debug,
                                two_gpu=two_gpu, create_model_func=create_model_func, time_out_ms=time_out_ms)

    lock_messages = Lock()
    lock_training_data = Lock()

    pb = Process(target=buffer_feeder, args=[consumer, lock_messages])
    p0 = Process(target=dnn_train, args=[0, consumer, lock_training_data])
    p1 = Process(target=dnn_classify, args=[1, consumer, lock_messages, lock_training_data])
    pb.start()
    p0.start()
    p1.start()
    pb.join()
    p0.join()
    p1.join()

def decode_class(c):
    is_dataset = type(c) == type(pd.Series())
    if is_dataset:
        decoded = c.str.decode("utf-8") if type(c[0]) == type(b'') else c.astype('int').astype('str')
    else:
        decoded = c.decode("utf-8") if isinstance(c, (bytes, np.bytes_)) else str(int(c))

    if is_dataset:
        if 'class' in decoded[0]:
            decoded = decoded.str.split('class', expand=True)[1]
        elif 'level' in decoded[0]:
            decoded = decoded.str.split('class', expand=True)[1]
        elif 'group' in decoded[0]:
            decoded = decoded.str.split('group', expand=True)[1]
            decoded[decoded == 'A'] = 0
            decoded[decoded == 'B'] = 1
    else:
        if 'class' in decoded:
            decoded = decoded.split('class')[1]
        elif 'group' in decoded:
            decoded = decoded.split('group')[1]
            decoded = 0 if decoded == 'A' else decoded
            decoded = 1 if decoded == 'B' else decoded

    return decoded

def runMultiARFFProducer(dir_path, bootstrap_servers='localhost:9092'):
    bootstrap_servers = bootstrap_servers.split(' ')

    topics = []

    arff_files = glob.glob(os.path.join( dir_path, '**/*.arff'), recursive=True)
    for file_path in arff_files:
        producer = KafkaProducer(bootstrap_servers=bootstrap_servers,
                                 client_id='ARFF_Producer_{}'.format(random.randrange(999999)),
                                 value_serializer=lambda m: json.dumps(m).encode('ascii'))
        topic = 'streams_' + os.path.basename(file_path).replace('.arff', '')
        print('Creating topic for {}'.format(topic))
        data, meta = arff.loadarff(file_path)
        attributes = [x for x in meta]
        df = pd.DataFrame(data)
        class_name = 'class' if 'class' in df.columns else 'target'

        classes = np.unique(decode_class(df[class_name])).astype(np.int)
        print(classes)
        min_classes = min(classes)
        classes = classes - min_classes
        classes = [int(c) for c in classes]

        for _, entry in tqdm(enumerate(data)):
            record = {'classes': list(classes)}
            for _, attr in enumerate(attributes):
                value = entry[attr]
                if attr == class_name:
                    attr = 'class'
                    value = int(decode_class(value)) - int(min_classes)

                elif type(value) == np.bytes_:
                    value = value.decode('UTF-8')

                    if meta[attr][0] == 'nominal':
                        # print(meta[attr], value)
                        value = meta[attr][1].index(value)

                record[attr] = value
            producer.send(topic, record)
        producer.close()

    return topics

def runARFFProducer(file_path, bootstrap_servers='localhost:9092'):
    bootstrap_servers = bootstrap_servers.split(' ')

    producer = KafkaProducer(bootstrap_servers=bootstrap_servers,
                             client_id='ARFF_Producer_{}'.format(random.randrange(999999)),
                             value_serializer=lambda m: json.dumps(m).encode('ascii'))
    topic = 'streams_' + os.path.basename(file_path).replace('.arff', '')
    print('Creating topic for {}'.format(topic))
    data, meta = arff.loadarff(file_path)
    attributes = [x for x in meta]
    df = pd.DataFrame(data)
    class_name = 'class' if 'class' in df.columns else 'target'
    classes = np.unique(decode_class(df[class_name])).astype(np.int)
    min_classes = min(classes)
    classes = classes - min_classes
    classes = [int(c) for c in classes]

    for _, entry in tqdm(enumerate(data)):
        record = {'classes': classes}
        for _, attr in enumerate(attributes):
            value = entry[attr]
            if attr == class_name:
                attr = 'class'
                value = int(decode_class(value)) - int(min_classes)

            elif type(value) == np.bytes_:
                value = value.decode('UTF-8')

                if meta[attr][0] == 'nominal':
                    # print(meta[attr], value)
                    value = meta[attr][1].index(value)

            record[attr] = value

        producer.send(topic, record)
    producer.close()

    return topic


from pathlib import Path

import cv2
import numpy as np

import depthai as dai

if __name__ == "__main__":
    model_path = Path(__file__).parent / 'custom_ops/out'
    pipeline = dai.Pipeline()
    # Source
    camera = pipeline.createColorCamera()
    camera.setPreviewSize(300, 300)
    camera.setCamId(0)
    camera.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
    camera.setInterleaved(True)
    # Ops
    detection = pipeline.createNeuralNetwork()
    blob_path = model_path / 'model.blob'
    detection.setBlobPath(f'{blob_path.as_posix()}')
    camera.preview.link(detection.input)
    # Link Outputs for Detection
    x_out = pipeline.createXLinkOut()
    x_out.setStreamName('custom')
    detection.out.link(x_out.input)

    device = dai.Device(pipeline)
    device.startPipeline()

    frame_buffer = device.getOutputQueue(name='custom', maxSize=4)

    while True:
        frame = frame_buffer.get()
        # Returns a list
        layer = frame.getFirstLayerFp16()
        layer = np.array(layer, dtype=np.uint8)
        shape = (300, 300, 3)
        frame_data = layer.reshape(shape)
        cv2.imshow('Image', frame_data)
        if cv2.waitKey(1) == ord('q'):
            break


# -*- coding: utf-8 -*-
"""

    mslib.msui._tests.test_mscolab_version_history
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    This module is used to test mscolab-project related gui.

    This file is part of mss.

    :copyright: Copyright 2020 Tanish Grover
    :copyright: Copyright 2020-2021 by the mss team, see AUTHORS.
    :license: APACHE-2.0, see LICENSE for details.

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
"""
import os
import sys
import pytest
import mock

from mslib._tests.utils import mscolab_start_server
from mslib.msui.mscolab import MSSMscolabWindow
from mslib.mscolab.conf import mscolab_settings
from PyQt5 import QtCore, QtTest, QtWidgets

PORTS = list(range(9591, 9620))

@pytest.mark.skipif(os.name == "nt",
                    reason="multiprocessing needs currently start_method fork")
class Test_MscolabVersionHistory(object):
    def setup(self):
        self.process, self.url, self.app, _, self.cm, self.fm = mscolab_start_server(PORTS)
        QtTest.QTest.qWait(500)
        self.application = QtWidgets.QApplication(sys.argv)
        self.window = MSSMscolabWindow(data_dir=mscolab_settings.MSCOLAB_DATA_DIR,
                                       mscolab_server_url=self.url)
        self._connect_to_mscolab()
        self._login()
        self._activate_project_at_index(0)
        # activate project window here by clicking button
        QtTest.QTest.mouseClick(self.window.versionHistoryBtn, QtCore.Qt.LeftButton)
        QtWidgets.QApplication.processEvents()
        QtTest.QTest.qWait(100)
        QtWidgets.QApplication.processEvents()
        QtWidgets.QApplication.processEvents()
        self.version_window = self.window.version_window
        assert self.version_window is not None
        QtTest.QTest.qWaitForWindowExposed(self.window)
        QtWidgets.QApplication.processEvents()

    def teardown(self):
        if self.window.version_window:
            self.window.version_window.close()
        if self.window.conn:
            self.window.conn.disconnect()
        self.application.quit()
        QtWidgets.QApplication.processEvents()
        self.process.terminate()

    def test_changes(self):
        self._change_version_filter(1)
        len_prev = self.version_window.changes.count()
        # make a changes
        self.window.waypoints_model.invert_direction()
        QtWidgets.QApplication.processEvents()
        QtTest.QTest.qWait(100)
        self.window.waypoints_model.invert_direction()
        QtWidgets.QApplication.processEvents()
        QtTest.QTest.qWait(100)
        self.version_window.load_all_changes()
        QtWidgets.QApplication.processEvents()
        len_after = self.version_window.changes.count()
        assert len_prev == (len_after - 2)

    @mock.patch("PyQt5.QtWidgets.QInputDialog.getText", return_value=["MyVersionName", True])
    def test_set_version_name(self, mockbox):
        self._change_version_filter(1)
        # make a changes
        self.window.waypoints_model.invert_direction()
        QtWidgets.QApplication.processEvents()
        QtTest.QTest.qWait(100)
        self.version_window.load_all_changes()
        QtWidgets.QApplication.processEvents()
        self._activate_change_at_index(0)
        QtWidgets.QApplication.processEvents()
        QtTest.QTest.mouseClick(self.version_window.nameVersionBtn, QtCore.Qt.LeftButton)
        QtWidgets.QApplication.processEvents()
        QtTest.QTest.qWait(100)
        assert self.version_window.changes.currentItem().version_name == "MyVersionName"
        assert self.version_window.changes.count() == 1

    def test_version_name_delete(self):
        pytest.skip("skipped because the next line triggers an assert")
        self._activate_change_at_index(0)
        QtTest.QTest.mouseClick(self.version_window.deleteVersionNameBtn, QtCore.Qt.LeftButton)
        QtWidgets.QApplication.processEvents()
        QtTest.QTest.qWait(100)
        assert self.version_window.changes.count() == 0

    @mock.patch("PyQt5.QtWidgets.QMessageBox.question", return_value=QtWidgets.QMessageBox.Yes)
    def test_undo(self, mockbox):
        self._change_version_filter(1)
        # make changes
        for i in range(2):
            self.window.waypoints_model.invert_direction()
            QtWidgets.QApplication.processEvents()
            QtTest.QTest.qWait(100)
        self.version_window.load_all_changes()
        QtWidgets.QApplication.processEvents()
        changes_count = self.version_window.changes.count()
        self._activate_change_at_index(1)
        QtTest.QTest.mouseClick(self.version_window.checkoutBtn, QtCore.Qt.LeftButton)
        QtWidgets.QApplication.processEvents()
        QtTest.QTest.qWait(200)
        new_changes_count = self.version_window.changes.count()
        assert changes_count + 1 == new_changes_count

    def test_refresh(self):
        self._change_version_filter(1)
        changes_count = self.version_window.changes.count()
        self.window.waypoints_model.invert_direction()
        QtWidgets.QApplication.processEvents()
        QtTest.QTest.qWait(100)
        self.window.waypoints_model.invert_direction()
        QtWidgets.QApplication.processEvents()
        QtTest.QTest.qWait(100)
        QtTest.QTest.mouseClick(self.version_window.refreshBtn, QtCore.Qt.LeftButton)
        QtWidgets.QApplication.processEvents()
        QtTest.QTest.qWait(100)
        new_changes_count = self.version_window.changes.count()
        assert new_changes_count == changes_count + 2

    def _connect_to_mscolab(self):
        assert self.window is not None
        self.window.url.setEditText(self.url)
        QtTest.QTest.mouseClick(self.window.toggleConnectionBtn, QtCore.Qt.LeftButton)
        QtTest.QTest.qWait(100)

    def _login(self):
        assert self.window is not None
        self.window.emailid.setText('a')
        self.window.password.setText('a')
        QtTest.QTest.mouseClick(self.window.loginButton, QtCore.Qt.LeftButton)
        QtWidgets.QApplication.processEvents()

    def _activate_project_at_index(self, index):
        assert self.window is not None
        assert index < self.window.listProjects.count()
        item = self.window.listProjects.item(index)
        point = self.window.listProjects.visualItemRect(item).center()
        QtTest.QTest.mouseClick(self.window.listProjects.viewport(), QtCore.Qt.LeftButton, pos=point)
        QtWidgets.QApplication.processEvents()
        QtTest.QTest.mouseDClick(self.window.listProjects.viewport(), QtCore.Qt.LeftButton, pos=point)
        QtWidgets.QApplication.processEvents()

    def _activate_change_at_index(self, index):
        assert self.version_window is not None
        assert index < self.version_window.changes.count()
        item = self.version_window.changes.item(index)
        point = self.version_window.changes.visualItemRect(item).center()
        QtTest.QTest.mouseClick(self.version_window.changes.viewport(), QtCore.Qt.LeftButton, pos=point)
        QtWidgets.QApplication.processEvents()
        QtTest.QTest.keyClick(self.version_window.changes.viewport(), QtCore.Qt.Key_Return)
        QtWidgets.QApplication.processEvents()
        QtTest.QTest.qWait(100)

    def _change_version_filter(self, index):
        assert self.version_window is not None
        assert index < self.version_window.versionFilterCB.count()
        self.version_window.versionFilterCB.setCurrentIndex(index)
        self.version_window.versionFilterCB.currentIndexChanged.emit(index)
        QtWidgets.QApplication.processEvents()
        QtTest.QTest.qWait(100)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Oct 10 22:38:25 2019

@author: xinlu
"""
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Oct 11 16:09:00 2019

@author: xinlu
"""

import string
from collections import Counter
import csv
import nltk
from nltk.tokenize import casual_tokenize
from nltk.util import ngrams
from nltk.util import bigrams
from nltk.lm import NgramCounter
from nltk.corpus import stopwords
import matplotlib.pyplot as plt
import re
import collections
import pandas as pd
import numpy as np
from nltk.corpus import stopwords
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk.classify import NaiveBayesClassifier
from sklearn import metrics
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report

DATA = 'INPUT.txt'

punctuation = list(string.punctuation) + ['…', '...','’']
stop_words = stopwords.words('english') + ['rt', 'via']

# The total number of distinct words (vocabulary)
# The total number of tokens corresponding to the top 10 most frequent words (types) in the vocabulary
def totals(text):
    token_count = 0
    counts = Counter(text)
    for key, val in counts.most_common(10):
        token_count += val
    return len(counts), counts.most_common(10), token_count

# The number of words, characters and av char count for each tweet
def tweet_counts(tweet):
    char_counts = [len(word) for word in tweet]
    total_chars = sum(char_counts)
    av_chars = total_chars / len(char_counts)
    return len(tweet), total_chars, av_chars

# calculate standard deviation of characters per token in a tweet
def standard_deviation(text):
    char_count = []
    words = text.split(' ')
    for word in words:
        char_count.append(len(word))

    diffs = 0
    average = sum(char_count)/len(char_count)
    for n in char_count:
        diffs += (n - average)**(2)
    return (diffs/(len(char_count)-1))**(0.5)

# The average number and standard deviation of characters per token
# The total number of characters
def token_counts(tweet_list):
    # sum the values with same keys for char_count and av_char_count
    char_counts = []
    word_counts = []

    for tweet in tweet_list:
        char_counts.append(tweet['char_count'])
        word_counts.append(tweet['word_count'])

    sd = standard_deviation(tweet['text'])
    total_chars = sum(char_counts)
    total_words = sum(word_counts)
    av_char_count = total_chars / total_words
    return total_chars, av_char_count, sd

# debugging
def write_counter_to_file(counters):
    filenames = ['bigrams','trigrams','fourgrams','fivegrams']
    num = 0
    for counter in counters:
        filename = filenames[num]
        with open(filename + '.csv', 'w') as csvfile:
            writer = csv.writer(csvfile)
            for key, value in counter.items():
                writer.writerow(list(key) + [value])
            num = num+1

# The total number of distinct n-grams (of words) that appear in the dataset for n=2,3
# You can use sets or Counters pretty easily
# https://dbader.org/blog/sets-and-multiset-in-python
# Check http://www.nltk.org/api/nltk.html#nltk.util.bigrams
# This also looks promising
# http://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html
def ngram_counts(tweet_list):
    bigram_counter = Counter()
    trigram_counter = Counter()
    fourgram_counter = Counter()
    fivegram_counter = Counter()
    list_counters = []

    for tweet in tweet_list:
        tokens = tweet['text'].split()
        bigrams = list(nltk.ngrams(tokens, 2))
        trigrams = list(nltk.ngrams(tokens, 3))
        fourgrams = list(nltk.ngrams(tokens, 4))
        fivegrams = list(nltk.ngrams(tokens, 5))
        bigram_counter.update(bigrams)
        trigram_counter.update(trigrams)
        fourgram_counter.update(fourgrams)
        fivegram_counter.update(fivegrams)
    distinct_bigram_counter = len(bigram_counter) # distinct counts
    distinct_trigram_counter = len(trigram_counter) # distinct counts
    distinct_fourgram_counter = len(fourgram_counter) # distinct counts
    distinct_fivegram_counter = len(fivegram_counter) # distinct counts

    # debug... trigrams count slightly higher??
    list_counters = [bigram_counter, trigram_counter, fourgram_counter, fivegram_counter]
    write_counter_to_file(list_counters)

    #total_counter = sum(ngram_counter.values()) # total counts
    return distinct_bigram_counter, distinct_trigram_counter, distinct_fourgram_counter, distinct_fivegram_counter

# Plot a token log frequency. Describe what this plot means and how to interpret it.
# A useful way to do this is by plotting log-frequency against the log-rank
# You could also plot Heap's law - types versus tokens
# You could also do a log frequency graph of the top n tokens
# Describe out it might help you understand coverage when training a model?

def token_log_freq(corpus):
    corpus_counts = Counter(corpus)
    plt.loglog([val for word, val in corpus_counts.most_common(1000)])
    plt.xlabel('rank')
    plt.ylabel('frequency')
    #plt.show()
    return

with open(DATA, 'r') as data:
    tweet_list = []
    corpus = []
    for line in data:
        #print(line.split('\t'))

        info = line.strip().split('\t')
        id, label, text = info[0], info[1], ' '.join(info[2:])

        #tokens = tokenize(text)
        tokens = casual_tokenize(text)
        tokens = [token.lower() for token in tokens]
        tokens = [token for token in tokens if not token in stop_words]
        tokens = [token for token in tokens if not token in punctuation]
        #tokens = [tokens for token in tokens if not token.isdigit()]

        corpus.extend(tokens) # one giant list
        word_count, char_count, av_char_count = tweet_counts(tokens)

        row = {
            'id': id,
            'label': label,
            'text': ' '.join(tokens),
            'word_count': word_count,
            'char_count': char_count,
        }
        tweet_list.append(row)

        total_tweets = len(tweet_list)

total_chars, av_char_count, sd = token_counts(tweet_list)

####################################Answers to questions#############################
print("total tweets = ", total_tweets)
print("total tokens = ", len(corpus))
print("total number of characters = ", total_chars)
print("av chars per token = ", av_char_count)
print("sd for chars per token = ", sd)

vocab, most_common, tokens_common = totals(corpus)

print("vocabulary size = ", vocab)
print("10 most common words = ", most_common)
print("token count of 10 most common words = ", tokens_common)

# The token/type ratio in the dataset
# Type-Token Ratio can be obtained by dividing the total type count by the total token count.
# The basic idea is that the higher the number, the more lexically diverse
print("token/type ratio = ", vocab / len(corpus))

bigram_counts, trigram_counts, fourgram_counts, fivegram_counts = ngram_counts(tweet_list)

print("unigram count = ", len(corpus))
print("bigram count = ", bigram_counts)
print("trigram count = ", trigram_counts)
print("four-gram count = ", fourgram_counts)
print("five-gram count = ", fivegram_counts)

#- The total number of distinct n-grams (of words) that appear in the dataset for n=2,3,4,5.
with open('INPUT.txt') as f:
    df = f.read()

##remove the user id 80xxx (first 18 digits) from each line
df = re.sub(r"(?<!\d)\d{18}(?!\d)","",df)
#remove url
def remove_url(txt):
    """Replace URLs found in a text string with nothing
    (i.e. it will remove the URL from the string).

    Parameters
    ----------
    txt : string
        A text string that you want to parse and remove urls.

    Returns
    -------
    The same txt string with url's removed.
    """
    return " ".join(re.sub("([^0-9A-Za-z \t])|(\w+:\/\/\S+)", " ", txt).split())

df1 = [remove_url(df)]
df2 = str(df1).split()
df3 = [word.lower() for word in df2]

stop_words = set(stopwords.words('english'))
filtered_sentence = [w for w in df3 if not w in stop_words]
list(stop_words)[0:20]
filtered_sentence = []

for w in df3:
    if w not in stop_words:
        filtered_sentence.append(w)
df9 = filtered_sentence

#n=5
s1 = re.sub(r'[^a-zA-Z0-9\s]', ' ', str(df9))
tokens1 = [token for token in s1.split(" ") if token != ""]
output1 = list(ngrams(tokens, 5))
ss1 = collections.Counter(output1)
ss1 = pd.DataFrame(ss1.most_common(),columns = ['ngrams','count'])
ss1

#n=4
s2 = re.sub(r'[^a-zA-Z0-9\s]', ' ', str(df9))
tokens2 = [token for token in s2.split(" ") if token != ""]
output2 = list(ngrams(tokens2, 4))
ss2 = collections.Counter(output2)
ss2 = pd.DataFrame(ss2.most_common(),columns = ['ngrams','count'])
ss2

#n=3
s3 = re.sub(r'[^a-zA-Z0-9\s]', ' ', str(df9))
tokens3 = [token for token in s3.split(" ") if token != ""]
output3 = list(ngrams(tokens3, 3))
ss3 = collections.Counter(output3)
ss3 = pd.DataFrame(ss3.most_common(),columns = ['ngrams','count'])
ss3

#n=2
s4 = re.sub(r'[^a-zA-Z0-9\s]', ' ', str(df9))
tokens4 = [token for token in s4.split(" ") if token != ""]
output4 = list(ngrams(tokens4, 2))
ss4 = collections.Counter(output4)
ss4 = pd.DataFrame(ss4.most_common(),columns = ['ngrams','count'])
ss4

# If the graph approximately accurate, it should look linear
# The idea here is that word count distributions follow a power law
# According to Zipf's law, the frequency of any word is inversely proportional
# to its rank in the frequency table.
token_log_freq(corpus)

######################################################################################
#############################Gold (Gold directory)####################################
######################################################################################

#What is the number of types that appear in the dev data but not the training data (OOV).?
#[Hint: compare vocabulary of all the words across your GOLD files with your INPUT.txt.
#Out-of-vocabulary words are those that appear in INPUT.txt that are not in your training data.]
with open('Gold/dev.txt') as dev:
    devdata = dev.read()
with open('Gold/train.txt') as train:
    traindata = train.read()
with open('Gold/devtest.txt') as devtest:
    devtest = devtest.read()
with open('Gold/test.txt') as test:
    test = test.read()

##remove the user id 80xxx (first 18 digits) from each line
devdata = re.sub(r"(?<!\d)\d{18}(?!\d)","",devdata)
traindata = re.sub(r"(?<!\d)\d{18}(?!\d)","",traindata)
devtest = re.sub(r"(?<!\d)\d{18}(?!\d)","",devtest)
test = re.sub(r"(?<!\d)\d{18}(?!\d)","",test)

#remove url
devdata = str([remove_url(devdata)]).split()
traindata = str([remove_url(traindata)]).split()
devtest = str([remove_url(devtest)]).split()
test = str([remove_url(test)]).split()

#combine 4 gold data sets

filenames = ['dev.txt', 'train.txt', 'devtest.txt','test.txt']
with open('Gold/golddata.txt', 'w') as outfile:
    for fname in filenames:
        with open(fname) as infile:
            outfile.write(infile.read())

with open('Gold/golddata.txt') as golddata:
    golddata = golddata.read()

#find types appear in the dev data but not the training data (OOV)
def notMatches(a,b):
    return [x for x in a if x not in b]

devNotTraining = pd.DataFrame(set(notMatches(devdata,traindata)), columns=['Types'])
print("The number of types that appear in the dev data but not the training data (OOV)")
print(len(devNotTraining))

#Look at the vocabulary growth (types) combining your four gold data sets against your input data.
#Plot vocabulary growth at difference sample sizes N. [Hint:The idea here is to take a value
#N around some percentage of your data such as 25%, 50%, 75%, 100% and look at your vocabulary size.]
##Combine all the input.

#remove nonsense words
golddata1 = str(golddata).split()
golddata2 = [word.lower() for word in golddata1]

N_tweet = len(golddata)
print('The total number of distinct words in gold dataset:', N_tweet)

#different sample sizes
N_tweet25 = round(.25 * N_tweet)
N_tweet50 = round(.5 * N_tweet)
N_tweet75 = round(.75 * N_tweet)

sample25 = golddata.splitlines()[0:N_tweet25]
sample50 = golddata.splitlines()[0:N_tweet50]
sample75 = golddata.splitlines()[0:N_tweet75]

#get list of words in different sample sizes
line25 = str(sample25).split()
word25 = [word.lower() for word in line25]

line50 = str(sample50).split()
word50 = [word.lower() for word in line50]

line75 = str(sample75).split()
word75 = [word.lower() for word in line75]

samplesizes = ('size = 25%', 'size = 50%', 'size = 75%', 'size = 100%')
y_pos = np.arange(len(samplesizes))
performance = [len(word25), len(word50), len(word75), len(golddata2)]

##please run the following 5 rows of code separately with parts of code of other questions
#otherwise graph will be crunched and looks weird
plt.bar(y_pos, performance, align='center', alpha=0.5)
plt.xticks(y_pos, samplesizes)
plt.ylabel('Vocab Size')
plt.title('vocabulary growth at difference sample sizes')

plt.show()

#What is the class distribution of the training data set - how many negative, neutral, positive tweets?
df_train = pd.read_csv('train.txt', sep = '\t', header = None)
df_train.columns = ['id', 'label', 'text']
df_train

df_pos_train = df_train[df_train['label'] == 'positive']
pos_tweets = df_pos_train['text'].tolist()

df_neg_train = df_train[df_train['label'] == 'negative']
neg_tweets = df_neg_train['text'].tolist()

df_neutral_train = df_train[df_train['label'] == 'neutral']
neutral_tweets = df_neutral_train['text'].tolist()

print('Class distribution of the training data set:')
print('Positive: ', len(pos_tweets))
print('Negative: ', len(neg_tweets))
print('Neutral: ', len(neutral_tweets))

#Look at the difference between the top word types across these three classes. [Recall these are your "unique words"
#- not tokens]
poswords1 = str(pos_tweets).split()
poswords2 = [word.lower() for word in poswords1]

stop_words = set(stopwords.words('english'))
filtered_sentence = [w for w in poswords2 if not w in stop_words]
list(stop_words)[0:20]

filtered_sentence = []

for w in poswords2:
    if w not in stop_words:
        filtered_sentence.append(w)
poswords3 = filtered_sentence
poswords4 = collections.Counter(poswords3)
poswords4.most_common(10)
poswords = pd.DataFrame(poswords4.most_common(10), columns=['words', 'count'])

negwords1 = str(neg_tweets).split()
negwords2 = [word.lower() for word in negwords1]

filtered_sentence2 = [w for w in negwords2 if not w in stop_words]

filtered_sentence2 = []

for w in negwords2:
    if w not in stop_words:
        filtered_sentence.append(w)
negwords3 = filtered_sentence
negwords4 = collections.Counter(negwords3)
negwords4.most_common(10)
negwords = pd.DataFrame(negwords4.most_common(10), columns=['words', 'count'])

neuwords1 = str(neutral_tweets).split()
neuwords2 = [word.lower() for word in neuwords1]

filtered_sentence3 = [w for w in neuwords2 if not w in stop_words]

filtered_sentence3 = []

for w in neuwords2:
    if w not in stop_words:
        filtered_sentence.append(w)
neuwords3 = filtered_sentence
neuwords4 = collections.Counter(neuwords3)
neuwords4.most_common(10)
neuwords = pd.DataFrame(neuwords4.most_common(10), columns=['words', 'count'])
neuwords

print('The top word types in positive tweets:')
print(poswords)

print('The top word types in negative tweets:')
print(negwords)

print('The top word types in neutral tweets:')
print(neuwords)
##Answer: there is not much difference between the top word types across these three classes

#What words are particularly characteristic of your training set and dev set? Are they the same?
#[Hint: we have not actually learned this concept yet -- just stick with most common words unless
#you've already read ahead and used something like PMI.]

df2 = str(traindata).split()
df3 = [word.lower() for word in df2]
df4 = collections.Counter(df3)
trainCommon = pd.DataFrame(df4.most_common(10), columns=['words', 'count'])

print('Most common words in the training data set:')
print(trainCommon)

df22 = str(devdata).split()
df33 = [word.lower() for word in df22]
df44 = collections.Counter(df33)
devCommon = pd.DataFrame(df44.most_common(10), columns=['words', 'count'])

print('Most common words in the dev data set:')
print(devCommon)

#The top 10 most common words of training set and dev set are exactly the same.

#######Part2
df_train = open('train.txt').readlines()
df_input = open('INPUT.txt').readlines()

# lowercase all the words and remove url
stop_words = set(stopwords.words('english'))
def preprocess_tweet(tweet):
    words = tweet.split(' ')
    temp = []
    for word in words:
        s = word.lower()
        if "http" in s or s in stop_words:
            continue
        else:
            temp.append(s)
    return " ".join(temp)

def split_files(file):
    split = []
    for line in file:
        s = line.split('\t',2)
        split.append((preprocess_tweet(s[2]),s[1]))
    return split

def features(sentence):
    words = sentence.lower().split()
    return dict(('contains(%s)' % w, True) for w in words)

df_train = split_files(df_train)
df_train

input_paper = []
input_tweets = []
num_of_tweets = 0
for line in df_input:
    num_of_tweets = num_of_tweets+1
    input_id = line.split('\t',2)[0]
    s = line.split('\t',2)[2]
    input_paper.append((input_id,preprocess_tweet(s)))
    input_tweets.append(preprocess_tweet(s))
#using vader
input_vader_ans = []

analyzer = SentimentIntensityAnalyzer()
#sentences = input_tweets
for sentence in input_tweets:
    vs = analyzer.polarity_scores(sentence)
    if vs['compound'] >= 0.10:
        input_vader_ans.append("positive")
    elif vs['compound'] <= - 0.05:
        input_vader_ans.append("negative")
    else :
        input_vader_ans.append("neutral")

#using naive bayes
classify=nltk.classify.apply_features(features,df_train)
trainer=NaiveBayesClassifier.train(classify)

input_list = [features(t) for t in input_tweets]
input_nb_ans = [trainer.classify(t) for t in input_list]

ids = [tw[0] for tw in input_paper]

df = pd.DataFrame(list(zip(ids, input_nb_ans, input_vader_ans)),
               columns =['ID', 'NB_ans', 'VD_ans'])
df.head(20)

df.to_csv('answer.csv')

######Part3
df_t = open('test.txt').readlines()
df_t = split_files(df_t)
input_tweets = [tw[0] for tw in df_t]
input_vader_ans1 = []
for sentence in input_tweets:
    vs = analyzer.polarity_scores(sentence)
    if vs['compound'] >= 0.10:
        input_vader_ans.append("positive")
    elif vs['compound'] <= - 0.05:
        input_vader_ans.append("negative")
    else :
        input_vader_ans.append("neutral")

classify=nltk.classify.apply_features(features,df_t)
trainer=NaiveBayesClassifier.train(classify)

input_list = [features(t) for t in input_tweets]
input_nb_ans = [trainer.classify(t) for t in input_list]

df = pd.DataFrame(list(zip(input_nb_ans, input_vader_ans)),
               columns =['NB_ans', 'VD_ans'])

df.head(20)

df_test = pd.read_csv('test.txt', sep = '\t', header = None)
df_test.columns = ['id', 'label', 'text','1']
NBlabels = df.iloc[:, 0].values
VDlabels = df.iloc[:, 1].values
TRlabels = df_test['label']
Text = df_test['text']

NBlabels = list(NBlabels)
VDlabels = list(VDlabels)
TRlabels = list(TRlabels)
Text = list(Text)
df1 = pd.DataFrame(zip(NBlabels,TRlabels, Text),
                   columns = ['NBlabels','TRlabels','Text'])
print(df1.loc[[11]].values)

### F1 for Vader model
confusion_matrix(y_true=TRlabels, y_pred=VDlabels)
target_names = ['positive', 'neutral', 'negative']
score = metrics.precision_recall_fscore_support(y_true=TRlabels, y_pred=VDlabels)
print(classification_report(y_true=TRlabels, y_pred=VDlabels, target_names=target_names))

### F1 for Bayes model
confusion_matrix(y_true=TRlabels, y_pred=NBlabels)
target_names = ['positive', 'neutral', 'negative']
score = metrics.precision_recall_fscore_support(y_true=TRlabels, y_pred=NBlabels)
print(classification_report(y_true=TRlabels, y_pred=NBlabels, target_names=target_names))