"""
.. py:module:: environment

This module holds ``Enviroment``-class, an universe where the agents live.
Environment holds methods for inter-agent communication and some utilities that
are usually needed when implementing creative multi-agent systems.

All implementations should subclass ``Environment`` in order to provide basic
functionality for the system to operate.

Environments are used by defining their address at the instantation time, and
adding agents to their container.
"""
import asyncio

import logging
from random import choice, shuffle

from aiomas import Container

from creamas.logging import ObjectLogger
from creamas.util import run_or_coro
from creamas.serializers import get_serializers

__all__ = ['Environment']

class Environment(Container):
    """Base environment class inherited from :py:class:`aiomas.Container`.
    """
    @classmethod
    def create(cls, *args, **kwargs):
        """Create a new environment, see :py:func:`~aiomas.Container.create` for details.

        If ``extra_serializers`` keyword argument is None, serializers returned by
        :func:`~creamas.serializers.get_serializers` are added to it.
        """
        extra_ser = kwargs.pop('extra_serializers', None)
        if extra_ser is None:
            extra_ser = get_serializers()
        kwargs.update({'extra_serializers': extra_ser})
        return super().create(*args, **kwargs)

    def __init__(self, base_url, loop, clock, connect_kwargs):
        super().__init__(base_url, loop, clock, connect_kwargs)
        self._age = 0
        self._logger = None
        self._log_folder = None
        self._artifacts = []
        self._candidates = []
        self._name = base_url

        # Try setting the process name to easily recognize the spawned
        # environments with 'ps -x' or 'top'
        try:
            import setproctitle as spt
            spt.setproctitle('Creamas: {}'.format(str(self)))
        except:
            pass

    @property
    def name(self):
        """Name of the environment."""
        return self._name

    @property
    def artifacts(self):
        """Published artifacts for all agents."""
        return self._artifacts

    @property
    def age(self):
        """Age of the environment.
        """
        return self._age

    @age.setter
    def age(self, a):
        self._age = a

    @property
    def logger(self):
        """Logger for the environment.
        """
        return self._logger

    @property
    def log_folder(self):
        """Logging folder for the environment. If set, will create py:class:`creamas.logging.ObjectLogger` for that
        folder.
        """
        return self._log_folder

    @log_folder.setter
    def log_folder(self, _log_folder):
        assert(type(_log_folder) is str)
        self._log_folder = _log_folder
        self._logger = ObjectLogger(self, _log_folder, add_name=True,
                                    init=True)

    def get_agents(self, addr=True, agent_cls=None, include_manager=False):
        """Get agents in the environment.

        :param bool addr: If ``True``, returns only addresses of the agents.
        :param agent_cls: Optional, if specified returns only agents belonging to that particular class.

        :param bool include_manager:
            If ``True`` includes the environment's manager, i.e. the agent in the address
            ``tcp://environment-host:port/0``, to the returned list if the environment has attribute :attr:`manager`.
            If environment does not have :attr:`manager`, then the parameter does nothing.

        :returns: A list of agents in the environment.
        :rtype: list

        .. note::
            Manager agents are excluded from the returned lists of agents by default.
        """
        agents = list(self.agents.dict.values())
        if hasattr(self, 'manager') and self.manager is not None:
            if not include_manager:
                agents = [a for a in agents if a.addr.rsplit('/', 1)[1] != '0']
        if agent_cls is not None:
            agents = [a for a in agents if type(a) is agent_cls]
        if addr:
            agents = [agent.addr for agent in agents]
        return agents

    def get_agent(self, addr):
        """Get agent in given address.

        :raises ValueError: If given address is not part of this environment
        :raises KeyError: If no such agent in the environment
        """
        base_url, agent_number = addr.rsplit('/', 1)
        base_url += "/"
        if base_url != self._base_url:
            raise ValueError("Given address' base URL ({}) does not match with the environment ({})."
                             .format(base_url, self._base_url))
        agent = self.agents.dict[agent_number]
        return agent

    async def trigger_act(self, *args, addr=None, agent=None, **kwargs):
        """Trigger agent to act.

        If *agent* is None, then looks the agent by the address.

        :raises ValueError: if both *agent* and *addr* are None.
        """
        if agent is None and addr is None:
            raise TypeError("Either addr or agent has to be defined.")
        if agent is None:
            agent = self.get_agent(addr)
        self._log(logging.DEBUG, "Triggering agent in {}".format(agent.addr))
        ret = await agent.act(*args, **kwargs)
        return ret

    async def trigger_all(self, *args, **kwargs):
        """Trigger all agents in the environment to act asynchronously.

        :returns: A list of agents' :meth:`act` return values.

        Given arguments and keyword arguments are passed down to each agent's
        :meth:`creamas.core.agent.CreativeAgent.act`.

        .. note::
            The environment's manager agent, i.e. if the environment has :attr:`manager`, is excluded from acting.
        """
        tasks = []
        for a in self.get_agents(addr=False, include_manager=False):
            task = asyncio.ensure_future(self.trigger_act
                                         (*args, agent=a, **kwargs))
            tasks.append(task)
        rets = await asyncio.gather(*tasks)
        return rets

    def is_ready(self):
        """Check if the environment is fully initialized.

        The function is mainly used by the multiprocessing environment managers
        and distributed environments to ensure that the environment has been
        correctly initialized before any other preparations are done for the
        environments or the simulation is started.

        Override the function in the subclasses which need more time consuming
        initialization routines. The function should return True when the
        environment is ready be used in a simulation, or when any
        cross-environment initialization routines can be run. That is, the
        environment is inherently in a coherent state, and can execute orders
        from managers or simulations.

        :rtype: bool
        :returns: This basic implementation returns always True.
        """
        return True

    def create_random_connections(self, n=5):
        """Create random connections for all agents in the environment.

        :param int n: the number of connections for each agent

        Existing agent connections that would be created by chance are not
        doubled in the agent's :attr:`connections`, but count towards
        connections created.
        """
        if type(n) != int:
            raise TypeError("Argument 'n' must be of type int.")
        if n <= 0:
            raise ValueError("Argument 'n' must be greater than zero.")
        for a in self.get_agents(addr=False):
            others = self.get_agents(addr=False)[:]
            others.remove(a)
            shuffle(others)
            for r_agent in others[:n]:
                a.add_connection(r_agent)

    def create_connections(self, connection_map):
        """Create agent connections from a given connection map.

        :param dict connection_map:
            A map of connections to be created. Dictionary where keys are agent addresses and values are lists of
            (addr, attitude)-tuples suitable for :meth:`~creamas.core.agent.CreativeAgent.add_connections`.

        Only connections for agents in this environment are made.
        """
        agents = self.get_agents(addr=False)
        rets = []
        for a in agents:
            if a.addr in connection_map:
                r = a.add_connections(connection_map[a.addr])
                rets.append(r)
        return rets

    def get_connections(self, data=True):
        """Return connections from all the agents in the environment.

        :param bool data: If ``True`` return also the dictionary associated with each connection

        :returns:
            A list of ``(addr, connections)``-tuples, where ``connections`` is a list of addresses agent in ``addr`` is
            connected to. If ``data`` parameter is ``True``, then the ``connections`` list contains tuples of
            ``(nb_addr, data)``-pairs , where ``data`` is a dictionary.

        :rtype: dict

        .. note::
            Environment's manager agent is excluded from the returned list.
        """
        connections = []
        for a in self.get_agents(addr=False):
            c = (a.addr, a.get_connections(data=data))
            connections.append(c)
        return connections

    def clear_connections(self):
        """Clear all connections from the agents in the environment.
        """
        for a in self.get_agents(addr=False):
            a.clear_connections()

    def get_random_agent(self, agent):
        """Return random agent that is not the same as the agent given as a parameter.

        :param agent: Agent that is not wanted to return
        :type agent: :py:class:`~creamas.core.agent.CreativeAgent`
        :returns: random, non-connected, agent from the environment
        :rtype: :py:class:`~creamas.core.agent.CreativeAgent`
        """
        r_agent = choice(self.get_agents(addr=False))
        while r_agent.addr == agent.addr:
            r_agent = choice(self.get_agents(addr=False))
        return r_agent

    def add_artifact(self, artifact):
        """Add artifact with given framing to the environment.

        :param object artifact: Artifact to be added.
        """
        artifact.env_time = self.age
        self.artifacts.append(artifact)
        self._log(logging.DEBUG, "ARTIFACTS appended: '{}', length={}"
                  .format(artifact, len(self.artifacts)))

    def add_artifacts(self, artifacts):
        """Add artifacts to :attr:`artifacts`.

        :param artifacts: list of :py:class:`~creamas.core.artifact.Artifact` objects
        """
        for artifact in artifacts:
            self.add_artifact(artifact)

    async def get_artifacts(self, agent=None):
        """Return artifacts published to the environment.

        :param agent: If not ``None``, then returns only artifacts created by the agent.

        :returns: All artifacts published (by the agent).
        :rtype: list

        If environment has a :attr:`manager` agent, e.g. it is a slave environment in
        :class:`~creamas.mp.MultiEnvironment`, then the manager's :meth:`~creamas.mp.EnvManager.get_artifacts` is called.
        """
        # TODO: Figure better way for this
        if hasattr(self, 'manager') and self.manager is not None:
            artifacts = await self.manager.get_artifacts()
        else:
            artifacts = self.artifacts
        if agent is not None:
            artifacts = [a for a in artifacts if agent.name == a.creator]
        return artifacts

    def _log(self, level, msg):
        if self.logger is not None:
            self.logger.log(level, msg)

    def save_info(self, folder, *args, **kwargs):
        """Save information accumulated during the environments lifetime.

        Called from :py:meth:`~creamas.core.Environment.destroy`. Override in subclass.

        :param str folder: root folder to save information
        """
        pass

    def destroy(self, folder=None, as_coro=False):
        """Close the environment.

        .. deprecated:: 0.4.0
            Use :func:`close` instead

        """
        DeprecationWarning("{0}.destroy is deprecated, use {0}.close instead.".format(str(self.__class__.__name__)))
        return self.close(folder=folder, as_coro=as_coro)

    def close(self, folder=None, as_coro=False):
        """Close the environment.

        Does the following:

        1. calls :py:meth:`~creamas.core.Environment.save_info`
        2. for each agent: calls :py:meth:`close`
        3. Shuts down its RPC-service.
        """
        async def _close(folder):
            ret = self.save_info(folder)
            for a in self.get_agents(addr=False):
                a.close(folder=folder)
            await self.shutdown(as_coro=True)
            return ret

        return run_or_coro(_close(folder), as_coro)

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return "{}({})".format(self.__class__.__name__, self.name)


import time
from typing import Tuple

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class base_parser:
    """Parent class for parsers of all marketplaces"""
    def __init__(self):
        """Default constructor which created a web driver for selenium parser"""
        options = Options()
        self.driver = webdriver.Chrome(options=options)

    def parse(self, url):
        """Function parses a site via provided url. Should be reimplemented in each dependent class"""
        pass

class ozon_parser(base_parser):
    """Ozon market parser. Extends from base parser class"""
    def parse(self, url: str) -> Tuple[str, int]:
        """
        Reimplementation of parse function for ozon market. Parses price and name of product using provided url.
        :param url: url for the product
        :return: tuple (name, price)
        """
        try:
            self.driver.get(url)

            price = self._parse_price(self.driver.find_element(By.CLASS_NAME, 'k1y').text)
            name = self.driver.find_element(By.CLASS_NAME, 'lm').text

            return name, price

        except Exception as ex:
            print(ex)
            return "-1", -1

        finally:
            self.driver.close()
            self.driver.quit()

    def _parse_price(self, price: str) -> int:
        """
        Parses price from sting to integer.
        :param price: price in string format.
        :return: price in integer format.
        """
        price = price.rsplit(u'\u2009', 1)[0].replace(u'\u2009', '')

        return int(price)

class wildberries_parser(base_parser):
    """Wildberries market parser. Extends from base parser class"""
    def parse(self, url: str) -> Tuple[str, int]:
        """
        Reimplementation of parse function for wildberries market. Parses price and name of product using provided url.
        :param url: url for the product
        :return: tuple (name, price)
        """
        try:
            self.driver.get(url)
            WebDriverWait(self.driver, 60).until(
                EC.element_to_be_clickable((By.CLASS_NAME, 'price-block__final-price')))

            price = self._parse_price(self.driver.find_element(By.CLASS_NAME, 'price-block__final-price').text)
            name = self.driver.find_element(By.CLASS_NAME, 'product-page__header').text

            return name, price

        except Exception as ex:
            print(ex)
            return "-1", -1

        finally:
            self.driver.close()
            self.driver.quit()

    def _parse_price(self, price: str) -> int:
        """
        Parses price from sting to integer.
        :param price: price in string format.
        :return: price in integer format.
        """
        price_parts = price.rsplit(' ', 1)[0]
        merged_price = price_parts.replace(' ', '')

        return int(merged_price)

class sbermegamarket_parser(base_parser):
    """Sbermegamarket parser. Extends from base parser class"""
    def parse(self, url: str) -> Tuple[str, int]:
        """
        Reimplementation of parse function for sbermegamarket. Parses price and name of product using provided url.
        :param url: url for the product
        :return: tuple (name, price)
        """
        try:
            self.driver.get(url=url)
            time.sleep(3)
            try:
                self.driver.find_element(By.CLASS_NAME, 'header-region-selector-view__footer-ok').click()
            except Exception as ex:
                pass
            price_div = self.driver.find_element(By.CLASS_NAME, "pdp-sales-block__price-final")
            price_div_text = price_div.text
            item_price = ''
            for i in price_div_text:
                if i == '.' or i == ',' or i.isdigit():
                    item_price += i
            item_price = item_price.replace(',', '.')
            item_price = float(item_price)
            return ' ', item_price

        except Exception as ex:
            print(ex)
            return "-1", -1

        finally:
            self.driver.close()
            self.driver.quit()

class kazanexpress_parser(base_parser):
    """KazanExpress parser. Extends from base parser class"""
    def parse(self, url: str) -> Tuple[str, float]:
        """
        Reimplementation of parse function for kazanexpress market. Parses price and name of product using provided url.
        :param url: url for the product
        :return: tuple (name, price)
        """
        try:
            self.driver.get(url=url)
            time.sleep(5)
            prod_name = self.driver.find_element(By.XPATH, '//*[@id="product-info"]/div[1]/h1').text
            price_div = self.driver.find_element(By.XPATH, '//*[@id="product-info"]/div[2]/div[4]/div[2]/div[1]/span')
            price_div_text = price_div.text
            item_price = ''
            for i in price_div_text:
                if i == '.' or i == ',' or i.isdigit():
                    item_price += i
            item_price = item_price.replace(',', '.')
            item_price = float(item_price)
            return prod_name, item_price

        except Exception as ex:
            return "-1", -1

        finally:
            self.driver.close()
            self.driver.quit()

class yandex_market_parser(base_parser):
    """Yandex market parser. Extends from base parser class"""
    def parse(self, url: str) -> Tuple[str, float]:
        """
        Reimplementation of parse function for yandex market. Parses price and name of product using provided url.
        :param url: url for the product
        :return: tuple (name, price)
        """
        try:
            self.driver.get(url)
            time.sleep(15)
            xpath_for_name_list = ['/html/body/div[2]/div[2]/main/div[4]/div/div/div[2]/div/div/div[1]/div[1]/h1',
                                   '/html/body/div[2]/div[2]/main/div[3]/div/div/div[2]/div/div/div[1]/div[1]/h1',
                                   '/html/body/div[2]/div[2]/div/div/div[1]/div/div/div/div/div[1]/div[1]/h1']
            prod_name = ''
            for i in range(len(xpath_for_name_list)):
                xpath = xpath_for_name_list[i]
                flag = 0
                try:
                    prod_name = self.driver.find_element(By.XPATH, xpath).text
                except Exception as ex:
                    flag = 1
                if not flag:
                    break

            try:
                price_div = self.driver.find_element(By.XPATH, '//*[@id="cardAddButton"]/div[1]/div/div/div[1]/div/div[4]/div[1]/div/div[1]/div/h3/span[2]')
            except Exception as ex:
                price_div = self.driver.find_element(By.XPATH, '/html/body/div[2]/div[2]/div/div/div[3]/div/div[3]/div[1]/div[1]/div/div/div[3]/div[2]/div/div/div[1]/span/span[1]')
            price_div_text = price_div.text
            item_price = ''
            for i in price_div_text:
                if i == '.' or i == ',' or i.isdigit():
                    item_price += i
            item_price = item_price.replace(',', '.')
            item_price = float(item_price)
            return prod_name, item_price

        except Exception as ex:
            return "-1", -1

        finally:
            self.driver.close()
            self.driver.quit()


# !/usr/bin/env python
# -*- coding: utf-8 -*-
#--------------------------------------------------------------------------------------------------
# Name          : TieResolver.py
# Author  		: Mark Pooley (mark-pooley@uiowa.edu)
# Link    		: http://www.ppc.uiowa.edu
# Date    		: 2015-01-20 12:39:50
# Version		: $1.0$
# Description	:Takes as input a table of ties and a table of near pologyons.
# Ties are resolved by choosing the provider ZCTA that shares the longest
# common boundary with the recipient ZCTA. If none of the candidate provider
# ZCTAs are adjacent, then the nearest is selected calculating the distance
# between the rec_ZCTA and the prov_ZCTA
# ---------------------------------------------------------------------------

###################################################################################################
#Import python modules
###################################################################################################

import arcpy
from arcpy import env
import math
import csv
import os
from operator import itemgetter
from collections import defaultdict

###################################################################################################
#Input Variable loading and environment declaration
###################################################################################################
tieTable = arcpy.GetParameterAsText(0) #table of recipient provider ties
nbrTable = arcpy.GetParameterAsText(1) #table of polygon neighbors generated from ZCTAs
ZCTAs = arcpy.GetParameterAsText(2) #ZCTAs input
outputLocation = arcpy.GetParameterAsText(3) #location of output file
outFile = arcpy.GetParameterAsText(4) #name of ouput file
#check if '.csv' at end of outFile name, if not add it.
if outFile[:-4] != '.csv':
	outFile = outFile + '.csv'

outFile = os.path.join(outputLocation,outFile) #concantenate path and file name so it gets stored in correct location

env.workspace = arcpy.GetParameterAsText(5)
env.overwriteOutput = True

###################################################################################################
# Defining global functions
###################################################################################################

#define function that will be used to calculate the distance between paired points. Cursor data will be fed into this
def distanceXY(lat1, long1, lat2, long2):

    # Convert latitude and longitude to
    # spherical coordinates in radians.
    degrees_to_radians = math.pi/180.0

    # phi = 90 - latitude
    phi1 = (90.0 - lat1)*degrees_to_radians
    phi2 = (90.0 - lat2)*degrees_to_radians

    # theta = longitude
    theta1 = long1*degrees_to_radians
    theta2 = long2*degrees_to_radians

    cos = (math.sin(phi1)*math.sin(phi2)*math.cos(theta1 - theta2) + math.cos(phi1)*math.cos(phi2))
    arc = math.acos(cos)

    #multiple by 3960 to get miles
    arc = arc *3960
    return arc

###################################################################################################
#Global variables to be used in process
###################################################################################################
tempDict = defaultdict(list) #establish default dictionary to be populated with all csv entries
tieDict = {} #dictionary that will keep just ties
resolvedDict = {} #dictionary to contain resolved/matched ties
nbrTable_FieldList = [f.name for f in arcpy.ListFields(nbrTable)]
ZCTAs_FieldList = [f.name for f in arcpy.ListFields(ZCTAs)]
outList = [] # output list of entries that will be written to output CSV

###################################################################################################
#Pull field variables from field lists
###################################################################################################
src_ZCTA_field = [f for f in nbrTable_FieldList if 'src_' in f][0] #find src field within field list
nbr_ZCTA_field = [f for f in nbrTable_FieldList if 'nbr_' in f][0] #find nbr field within field list
ZCTA_field = [f for f in ZCTAs_FieldList if 'ZCTA' in f or 'ZIP' in f][0] #find ZCTA field within field list
length_Field = [f for f in nbrTable_FieldList if 'LENGTH' in f][0] #find LENGTH field within field list

###################################################################################################
#read in input CSV and create dictionaries of tied Rec ZCTAs
###################################################################################################

#open tieTable csv as reader csv and create a temporary dictionary of rec and prov ZCTAs from rows
with open(tieTable, "rb") as inFile:
	rowCount = 0
	#sniff into 10kb of csv to check dialect create a reader using the identified dialect and
	dialect = csv.Sniffer().sniff(inFile.read(10*1024))
	inFile.seek(0)
	reader = csv.reader(inFile, dialect) #reader object
	header = reader.next() #create header

	for row in reader:
		tempDict[row[0]].append(row[1])
		outList.append(row)
		rowCount +=1

#create new dictionary by iterating through current one and checking for value lists greater than 1
#lists greater than 1 indicate a tie
tieDict = {}
for key,value in tempDict.iteritems():
	if len(value) > 1:
		tieDict[key] = value
arcpy.AddMessage(str(len(tieDict)) + " ties found in data... ")

###################################################################################################
#Find best provider match where providers are
#adjacent/touching recipient ZCTA
###################################################################################################

#main loop to iterate through the ties dictionary and
arcpy.SetProgressor("step","Checking for matching provider ZCTAs adjacent to recipients...",0,len(tieDict),1)
for key,values in tieDict.iteritems():
	srcQuery =  src_ZCTA_field + " = '" + str(key) + "'"
	dyadBorder = 0

	#first check that the rec and prov don't match. If they do, consider the tie resolved by making the provider
	#zcta that of the recipient
	if key in values:
		resolvedDict[key] = values[values.index(key)]

	else:
		#create search cursor that will look through the neighbor table first.
		#in the future look to see if neighbors and shared boreder length can be identified
		#using geomtries instead of a bulky neighbor table
		with arcpy.da.SearchCursor(nbrTable,nbrTable_FieldList,srcQuery) as cursor:
			for row in cursor:
				if row[nbrTable_FieldList.index(nbr_ZCTA_field)] in values:
					if row[nbrTable_FieldList.index(length_Field)] > dyadBorder:
						dyadBorder = row[nbrTable_FieldList.index(length_Field)]
						provZCTA = row[nbrTable_FieldList.index(nbr_ZCTA_field)]
						arcpy.SetProgressorLabel("Match Found!")
						#resolvedDict[key] = provZCTA
		if dyadBorder > 0:
			#create match from the rec and provider sharing the most boundary length
			resolvedDict[key] = provZCTA
	arcpy.SetProgressorPosition() #update progressor positon

arcpy.AddMessage(str(len(resolvedDict)) + " ties resolved by finding martching or adjacent provider ZCTAs..." )

#create set of those where neighbor search is needed because provider ZCTAS
#aren't adjacent.  Use iteritems to iterate through a dictionary correctly.
for key,value in resolvedDict.iteritems():
	if tieDict.has_key(key):
		#if key is in resolved dictionary, pop identified key from tie dictionary and keep none of the values
		tieDict.pop(key,None)

arcpy.AddMessage(str(len(tieDict)) + " remaining ties to be resolved by finding nearest provider ZCTA...")

###################################################################################################
#Find best provider match - being the closest provider to the recipient ZCTA based on the
#distance between recipient and candidate provider centroid
###################################################################################################

#loop through remaining ties to find the nearest provider ZCTA
arcpy.SetProgressor('step','finding nearest provider ZCTAs for remaining ties...',0,len(tieDict),1)
for key,values in tieDict.iteritems():
	tempDict = {} #temporary dictionary to track the best recipient provider match
	recQuery = ZCTA_field + " = '" + str(key) + "'" #query to be used for search cursor
	provDistance = 1e309 #check variable for distance checking. set to infinity for each key value

	#get recipient coordinates using Shape@TRUECENTROID
	with arcpy.da.SearchCursor(ZCTAs,["ZCTA5CE10","SHAPE@TRUECENTROID"],recQuery) as cursor:
		for row in cursor:
			recCoord = row[1]
			recX = recCoord[0] #rec lat/X var
			recY = recCoord[1] #rec lon/Y var

	#-------------------------------------------------------------------------------------------
	#iterate through provider zctas to find coordinates and pass them through the
	#distance function.
	#-------------------------------------------------------------------------------------------
	for value in values:
		provQuery = ZCTA_field + " = '" + str(value) + "'" #queary to be used for search cursor
		with arcpy.da.SearchCursor(ZCTAs,[ZCTA_field,"SHAPE@TRUECENTROID"],provQuery) as cursor:
			for row in cursor:
				provCoord = row[1] #pull coordinate pairs
				provX = provCoord[0] #prov lat/X var
				provY = provCoord[1] #prov long/Y var

				#feed coordinates into distanceXY funcation to calculate distance between the two pairs
				distance = distanceXY(recX,recY,provX,provY)
				if distance < provDistance:
					provDistance = distance # declare newly caluclate distance as the minimum distance
					tempDict[key] = value #reassign recipient and provider paring based on caluclation

	resolvedDict.update(tempDict)#update the resovled dictionary with the best fitting pair
	arcpy.SetProgressorPosition() #update progressor position through each iteration

###################################################################################################
#write final output file to CSV
###################################################################################################

arcpy.SetProgressor("step", "writing new CSV with all ties resolved",0,rowCount,1)

#open tieTable csv as reader csv and create a temporary dictionary of rec and prov ZCTAs from rows
readerFile =  open(tieTable, "rb")
dialect = csv.Sniffer().sniff(readerFile.read(10*1024)) #sniff into 10kb of csv to check dialect create a reader using the identified dialect and
readerFile.seek(0) #not sure what this does, but it's important
reader = csv.reader(readerFile, dialect) #reader object using snieefed dialect
header = reader.next() #create header

outCSV = open(outFile, 'wb') #open writer object
writer = csv.writer(outCSV,dialect) #writer object using the reader dialect
writer.writerow(header) #write header

for row in reader:
	if row[0] not in resolvedDict: #if rec ZCTA is not in the resolved Dictionary, just write the row
		writer.writerow(row)

	elif resolvedDict[row[0]] == row[1]: #if resolved dictionary value for key is equal to prov_ZCTA, write row
		writer.writerow(row)

	else:
		pass #pass, nothing to see hear, not even a slight weapons malfunction

	arcpy.SetProgressorPosition() #udpate progressor position

#close csv files.
outCSV.close()
readerFile.close()

arcpy.AddMessage("Process complete!\n" + "Output csv location: " + str(os.path.realpath(outFile)))


from __future__ import unicode_literals

import codecs
import io
import json
import logging
import math
import os
import time
import uuid

import numpy
import pandas

from timesketch_api_client import timeline
from timesketch_api_client import definitions
from timesketch_import_client import utils

logger = logging.getLogger("timesketch_importer.importer")

class ImportStreamer(object):
    """Upload object used to stream results to Timesketch."""

    # The number of entries before automatically flushing
    # the streamer.
    DEFAULT_ENTRY_THRESHOLD = 50000

    # Number of bytes in a binary file before automatically
    # chunking it up into smaller pieces.
    DEFAULT_FILESIZE_THRESHOLD = 104857600  # 100 Mb.

    # Define default values.
    DEFAULT_TEXT_ENCODING = "utf-8"
    DEFAULT_TIMESTAMP_DESC = "Time Logged"

    # Define the maximum amount of retries for a file/chunk upload.
    DEFAULT_RETRY_LIMIT = 3

    def __init__(self):
        """Initialize the upload streamer."""
        self._celery_task_id = ""
        self._count = 0
        self._config_helper = None
        self._data_label = ""
        self._dict_config_loaded = False
        self._csv_delimiter = None
        self._data_lines = []
        self._data_type = None
        self._datetime_field = None
        self._format_string = None
        self._index = ""
        self._last_response = None
        self._provider = "Importer library"
        self._resource_url = ""
        self._sketch = None
        self._timeline_id = None
        self._timeline_name = None
        self._upload_context = ""

        self._chunk = 1

        self._text_encoding = self.DEFAULT_TEXT_ENCODING
        self._timestamp_desc = self.DEFAULT_TIMESTAMP_DESC
        self._threshold_entry = self.DEFAULT_ENTRY_THRESHOLD
        self._threshold_filesize = self.DEFAULT_FILESIZE_THRESHOLD

    def _fix_dict(self, my_dict):
        """Adjusts a dict with so that it can be uploaded to Timesketch.

        This function will take a dictionary and modify it. Summary of the
        changes are:
          * If "message" is not a key and a format message string has been
              defined, a message field is constructed.
          * If "datetime" is not a key, an attempt to generate it is made.
          * If "timestamp_desc" is not set but defined by the importer it
              is added.
          * All keys that start with an underscore ("_") are removed.

        Args:
            my_dict: a dictionary that may be missing few fields needed
                    for Timesketch.
        """
        if "message" not in my_dict:
            format_string = self._format_string or utils.get_combined_message_string(
                mydict=my_dict
            )
            my_dict["message"] = format_string.format(**my_dict)

        _ = my_dict.setdefault("timestamp_desc", self._timestamp_desc)
        if self._data_type:
            _ = my_dict.setdefault("data_type", self._data_type)

        if "datetime" not in my_dict:
            date = ""
            if self._datetime_field:
                value = my_dict.get(self._datetime_field)
                if value:
                    date = utils.get_datestring_from_value(value)
            if not date:
                for key in my_dict:
                    key_string = key.lower()
                    if "time" not in key_string:
                        continue

                    if key_string == "timestamp_desc":
                        continue

                    value = my_dict[key]
                    date = utils.get_datestring_from_value(value)
                    if date:
                        break

            if date:
                my_dict["datetime"] = date
        else:
            my_dict["datetime"] = utils.get_datestring_from_value(my_dict["datetime"])

        # We don't want to include any columns that start with an underscore.
        underscore_columns = [x for x in my_dict if x.startswith("_")]
        if underscore_columns:
            for column in underscore_columns:
                del my_dict[column]

    def _fix_data_frame(self, data_frame):
        """Returns a data frame with added columns for Timesketch upload.

        Args:
            data_frame: a pandas data frame.

        Returns:
            A pandas data frame with added columns needed for Timesketch.
        """
        if "message" not in data_frame:
            format_string = self._format_string or utils.get_combined_message_string(
                dataframe=data_frame
            )
            utils.format_data_frame(data_frame, format_string)

        if "timestamp_desc" not in data_frame:
            data_frame["timestamp_desc"] = self._timestamp_desc

        if self._data_type and "data_type" not in data_frame:
            data_frame["data_type"] = self._data_type

        if "datetime" not in data_frame:
            if self._datetime_field and self._datetime_field in data_frame:
                try:
                    data_frame["timestamp"] = pandas.to_datetime(
                        data_frame[self._datetime_field], utc=True
                    )
                except ValueError as e:
                    logger.info(
                        "Unable to convert timestamp in column: %s, error %s",
                        self._datetime_field,
                        e,
                    )
            else:
                for column in data_frame.columns[
                    data_frame.columns.str.contains("time", case=False)
                ]:
                    if column.lower() == "timestamp_desc":
                        continue
                    try:
                        data_frame["timestamp"] = pandas.to_datetime(
                            data_frame[column], utc=True
                        )
                        # We want the first successful timestamp value.
                        break
                    except ValueError as e:
                        logger.info(
                            "Unable to convert timestamp in column: " "%s, error %s",
                            column,
                            e,
                        )

            if "timestamp" in data_frame:
                data_frame["datetime"] = data_frame["timestamp"].dt.strftime(
                    "%Y-%m-%dT%H:%M:%S%z"
                )
                data_frame["timestamp"] = (
                    data_frame["timestamp"].astype(numpy.int64) / 1e9
                )
        else:
            try:
                date = pandas.to_datetime(data_frame["datetime"], utc=True)
                data_frame["datetime"] = date.dt.strftime("%Y-%m-%dT%H:%M:%S%z")
            except Exception:  # pylint: disable=broad-except
                logger.error(
                    "Unable to change datetime, is it badly formed?", exc_info=True
                )

        # TODO: Support labels in uploads/imports.
        if "label" in data_frame:
            del data_frame["label"]
            logger.warning(
                "Labels cannot be imported at this time. Therefore the "
                "label column was dropped from the dataset."
            )

        # We don't want to include any columns that start with an underscore.
        columns = list(data_frame.columns[~data_frame.columns.str.contains("^_")])
        return data_frame[columns]

    def _ready(self):
        """Check whether all variables have been set.

        Raises:
            ValueError: if the streamer has not yet been fully configured.
        """
        if self._sketch is None:
            raise ValueError("Sketch has not yet been set.")

    def _reset(self):
        """Reset the buffer."""
        self._count = 0
        self._data_lines = []

    def _upload_data_buffer(self, end_stream, retry_count=0):
        """Upload data to Timesketch.

        Args:
            end_stream: boolean indicating whether this is the last chunk of
                the stream.
            retry_count: optional int that is only set if this is a retry
                of the upload.

        Raises:
            RuntimeError: If the data buffer is not successfully uploaded.
        """
        if not self._data_lines:
            return None

        start_time = time.time()
        data = {
            "name": self._timeline_name,
            "sketch_id": self._sketch.id,
            "enable_stream": not end_stream,
            "data_label": self._data_label,
            "provider": self._provider,
            "events": "\n".join([json.dumps(x) for x in self._data_lines]),
        }
        if self._index:
            data["index_name"] = self._index

        if self._upload_context:
            data["context"] = self._upload_context

        logger.debug(
            "Data buffer ready for upload, took {0:.2f} seconds to "
            "prepare.".format(time.time() - start_time)
        )

        response = self._sketch.api.session.post(self._resource_url, data=data)

        # TODO: Investigate why the sleep is needed, fix the underlying issue
        # and get rid of it here.
        # To prevent unexpected errors with connection refusal adding a quick
        # sleep.
        time.sleep(2)

        if response.status_code not in definitions.HTTP_STATUS_CODE_20X:
            if retry_count >= self.DEFAULT_RETRY_LIMIT:
                raise RuntimeError(
                    "Error uploading data: [{0:d}] {1!s} {2!s}, "
                    "index {3:s}".format(
                        response.status_code,
                        response.reason,
                        response.text,
                        self._index,
                    )
                )

            logger.warning(
                "Unable to upload data buffer: {0:d}, retrying (attempt "
                "{1:d}/{2:d})".format(
                    self._chunk, retry_count + 1, self.DEFAULT_RETRY_LIMIT
                )
            )

            return self._upload_data_buffer(
                end_stream=end_stream, retry_count=retry_count + 1
            )

        logger.debug(
            "Data buffer nr. {0:d} uploaded, total time: {1:.2f}s".format(
                self._chunk, time.time() - start_time
            )
        )
        self._chunk += 1
        response_dict = response.json()
        object_dict = response_dict.get("objects", [{}])[0]
        meta_dict = response_dict.get("meta", {})
        self._celery_task_id = meta_dict.get("task_id", "")

        self._timeline_id = object_dict.get("id")
        self._index = object_dict.get("searchindex", {}).get("index_name")
        self._last_response = response_dict

        return None

    def _upload_data_frame(self, data_frame, end_stream, retry_count=0):
        """Upload data to Timesketch.

        Args:
            data_frame: a pandas DataFrame with the content to upload.
            end_stream: boolean indicating whether this is the last chunk of
                the stream.
            retry_count: optional int that is only set if this is a retry
                of the upload.

        Raises:
            RuntimeError: If the dataframe is not successfully uploaded.
        """
        data = {
            "name": self._timeline_name,
            "sketch_id": self._sketch.id,
            "enable_stream": not end_stream,
            "data_label": self._data_label,
            "provider": self._provider,
            "events": data_frame.to_json(orient="records", lines=True),
        }
        if self._index:
            data["index_name"] = self._index

        if self._upload_context:
            data["context"] = self._upload_context

        response = self._sketch.api.session.post(self._resource_url, data=data)
        if response.status_code not in definitions.HTTP_STATUS_CODE_20X:
            if retry_count >= self.DEFAULT_RETRY_LIMIT:
                raise RuntimeError(
                    "Error uploading data: [{0:d}] {1!s} {2!s}, "
                    "index {3:s}".format(
                        response.status_code,
                        response.reason,
                        response.text,
                        self._index,
                    )
                )

            logger.warning(
                "Unable to upload dataframe, retrying (attempt "
                "{0:d}/{1:d})".format(retry_count + 1, self.DEFAULT_RETRY_LIMIT)
            )

            return self._upload_data_frame(
                data_frame=data_frame,
                end_stream=end_stream,
                retry_count=retry_count + 1,
            )

        self._chunk += 1
        response_dict = response.json()
        object_dict = response_dict.get("objects", [{}])[0]
        meta_dict = response_dict.get("meta", {})
        self._celery_task_id = meta_dict.get("task_id", "")

        self._timeline_id = object_dict.get("id")
        self._index = object_dict.get("searchindex", {}).get("index_name")
        self._last_response = response_dict
        return None

    def _upload_binary_file(self, file_path):
        """Upload binary data to Timesketch, potentially chunking it up.

        Args:
            file_path: a full path to the file that is about to be uploaded.
        """
        file_size = os.path.getsize(file_path)

        if self._timeline_name:
            timeline_name = self._timeline_name
        else:
            file_name = os.path.basename(file_path)
            file_name_no_ext, _, _ = file_name.rpartition(".")
            timeline_name = file_name_no_ext

        data = {
            "name": timeline_name,
            "sketch_id": self._sketch.id,
            "total_file_size": file_size,
            "provider": self._provider,
            "data_label": self._data_label,
        }
        if self._index:
            data["index_name"] = self._index

        if self._upload_context:
            data["context"] = self._upload_context

        if file_size <= self._threshold_filesize:
            file_dict = {"file": open(file_path, "rb")}
            response = self._sketch.api.session.post(
                self._resource_url, files=file_dict, data=data
            )
        else:
            chunks = int(math.ceil(float(file_size) / self._threshold_filesize))
            data["chunk_total_chunks"] = chunks
            data["chunk_index_name"] = uuid.uuid4().hex

            for index in range(0, chunks):
                data["chunk_index"] = index
                start = self._threshold_filesize * index
                data["chunk_byte_offset"] = start
                fh = open(file_path, "rb")
                fh.seek(start)
                binary_data = fh.read(self._threshold_filesize)
                file_stream = io.BytesIO(binary_data)
                file_stream.name = file_path
                file_dict = {"file": file_stream}

                retry_count = 0
                while True:
                    if retry_count >= self.DEFAULT_RETRY_LIMIT:
                        raise RuntimeError(
                            "Error uploading data chunk: {0:d}/{1:d}. Status "
                            "code: {2:d} - {3!s} {4!s}".format(
                                index,
                                chunks,
                                response.status_code,
                                response.reason,
                                response.text,
                            )
                        )

                    response = self._sketch.api.session.post(
                        self._resource_url, files=file_dict, data=data
                    )

                    if response.status_code in definitions.HTTP_STATUS_CODE_20X:
                        break

                    retry_count += 1
                    logger.warning(
                        "Error uploading data chunk {0:d}/{1:d}, retry "
                        "attempt {2:d}/{3:d}".format(
                            index, chunks, retry_count, self.DEFAULT_RETRY_LIMIT
                        )
                    )

        if response.status_code not in definitions.HTTP_STATUS_CODE_20X:
            raise RuntimeError(
                "Error uploading data: [{0:d}] {1!s} {2!s}, file: {3:s}, "
                "index {4:s}".format(
                    response.status_code,
                    response.reason,
                    response.text,
                    file_path,
                    self._index,
                )
            )

        response_dict = response.json()
        object_dict = response_dict.get("objects", [{}])[0]
        meta_dict = response_dict.get("meta", {})
        self._celery_task_id = meta_dict.get("task_id", "")

        self._timeline_id = object_dict.get("id")
        self._index = object_dict.get("searchindex", {}).get("index_name")
        self._last_response = response_dict

    def add_data_frame(self, data_frame, part_of_iter=False):
        """Add a data frame into the buffer.

        Args:
              data_frame: a pandas data frame object to add to the buffer.
              part_of_iter: if it is expected that this function is called
                  as part of an iterator, or that many data frames may be
                  added to the importer set to True, defaults to False.

        Raises:
              ValueError: if the data frame does not contain the correct
                  columns for Timesketch upload.
        """
        self._ready()

        if not isinstance(data_frame, pandas.DataFrame):
            raise TypeError("Entry object needs to be a DataFrame")

        size = data_frame.shape[0]

        if self._config_helper:
            data_type = ""
            if "data_type" in data_frame:
                data_types = data_frame.data_type.unique()
                if len(data_types) == 1:
                    data_type = data_types[0]

            df_columns = list(data_frame.columns)
            self._config_helper.configure_streamer(
                self, data_type=data_type, columns=df_columns
            )

        data_frame_use = self._fix_data_frame(data_frame)

        if "datetime" not in data_frame_use:
            raise ValueError(
                "Need a field called datetime in the data frame that is "
                "formatted according using this format string: "
                "%Y-%m-%dT%H:%M:%S%z. If that is not provided the data frame "
                'needs to have a column that has the word "time" in it, '
                "that can be used to convert to a datetime field."
            )

        if "message" not in data_frame_use:
            raise ValueError(
                "Need a field called message in the data frame, use the "
                "formatting string to generate one automatically."
            )

        if "timestamp_desc" not in data_frame_use:
            raise ValueError("Need a field called timestamp_desc in the data frame.")

        if size <= self._threshold_entry:
            end_stream = not part_of_iter
            self._upload_data_frame(data_frame_use, end_stream=end_stream)
            return

        chunks = int(math.ceil(float(size) / self._threshold_entry))
        for index in range(0, chunks):
            chunk_start = index * self._threshold_entry
            data_chunk = data_frame_use[
                chunk_start : chunk_start + self._threshold_entry
            ]
            end_stream = bool(index == chunks - 1)
            self._upload_data_frame(data_chunk, end_stream=end_stream)

    def add_dict(self, entry):
        """Add an entry into the buffer.

        Args:
            entry: a dict object to add to the buffer.

        Raises:
            TypeError: if the entry is not a dict.
        """
        self._ready()
        if not isinstance(entry, dict):
            raise TypeError("Entry object needs to be a dict.")

        if self._count >= self._threshold_entry:
            self.flush(end_stream=False)
            self._reset()

        if self._config_helper and not self._dict_config_loaded:
            data_type = entry.get("data_type", "")
            columns = entry.keys()
            self._config_helper.configure_streamer(
                self, data_type=data_type, columns=columns
            )
            self._dict_config_loaded = True

        # Changing the dictionary to add fields, such as timestamp description,
        # message field, etc. See function docstring for further details.
        self._fix_dict(entry)
        self._data_lines.append(entry)
        self._count += 1

    def add_excel_file(self, filepath, **kwargs):
        """Add a Microsoft Excel sheet to importer.

        Args:
            filepath: full file path to a XLS or XLSX file to add to the
                importer.
            kwargs:
                Other parameters can be passed in that match the
                pandas.read_excel parameters. Including:

                sheet_name: str, int, list, or None, default 0. Strings are
                    used for sheet names. Integers are used in zero-indexed
                    sheet positions. Lists of strings/integers are used to
                    request multiple sheets. Specify None to get all sheets.
                header : int, list of int, default 0
                    Row (0-indexed) to use for the column labels of the
                    parsed DataFrame. If a list of integers is passed those
                    row positions will be combined into a ``MultiIndex``. Use
                    None if there is no header.
                names : array-like, default None
                    List of column names to use. If file contains no header
                    row then you should explicitly pass header=None.
                index_col : int, list of int, default None
                    Column (0-indexed) to use as the row labels of the
                    DataFrame. Pass None if there is no such column. If a list
                    is passed, those columns will be combined into a
                    ``MultiIndex``. If a subset of data is selected with
                    ``usecols``, index_col is based on the subset.

        Raises:
            TypeError: if the entry is not an Excel sheet.
        """
        self._ready()
        if not os.path.isfile(filepath):
            raise TypeError("File path is not a real file.")

        file_ending = filepath.lower().split(".")[-1]
        if file_ending not in ["xls", "xlsx"]:
            raise TypeError("File name needs to end with xls or xlsx")

        data_frame = pandas.read_excel(filepath, **kwargs)
        if data_frame.empty:
            raise TypeError("Not able to read any rows from sheet.")

        self.add_data_frame(data_frame)

    def add_file(self, filepath, delimiter=","):
        """Add a CSV, JSONL or a PLASO file to the buffer.

        Args:
            filepath: the path to the file to add.
            delimiter: if this is a CSV file then a delimiter can be defined.

        Raises:
            TypeError: if the entry does not fulfill requirements.
        """
        self._ready()

        if not os.path.isfile(filepath):
            raise TypeError("Entry object needs to be a file that exists.")

        if not self._timeline_name:
            base_path = os.path.basename(filepath)
            default_timeline_name, _, _ = base_path.rpartition(".")
            self.set_timeline_name(default_timeline_name)

        file_ending = filepath.lower().split(".")[-1]

        if not self._data_label:
            self._data_label = file_ending

        if file_ending == "csv":
            if self._csv_delimiter:
                delimiter = self._csv_delimiter

            with codecs.open(
                filepath, "r", encoding=self._text_encoding, errors="replace"
            ) as fh:
                for chunk_frame in pandas.read_csv(
                    fh, delimiter=delimiter, chunksize=self._threshold_entry
                ):
                    self.add_data_frame(chunk_frame, part_of_iter=True)
        elif file_ending == "plaso":
            self._upload_binary_file(filepath)

        elif file_ending == "jsonl":
            with codecs.open(
                filepath, "r", encoding=self._text_encoding, errors="replace"
            ) as fh:
                for line in fh:
                    try:
                        self.add_json(line.strip())
                    except TypeError as e:
                        logger.error("Unable to decode line: {0!s}".format(e))

        else:
            raise TypeError(
                "File needs to have a file extension of: .csv, .jsonl or " ".plaso"
            )

    def add_json(self, json_entry, column_names=None):
        """Add an entry that is in a JSON format.

        Args:
            json_entry: a single entry encoded in JSON.
            column_names: a list of column names if the JSON object
                is a list as an opposed to a dict.

        Raises:
            TypeError: if the entry is not JSON or in the wrong JSON format.
        """
        try:
            json_obj = json.loads(json_entry)
        except json.JSONDecodeError as e:
            raise TypeError("Data not as JSON, error: {0!s}".format(e)) from e

        json_dict = {}
        if isinstance(json_obj, (list, tuple)):
            if not column_names:
                raise TypeError(
                    "Data is a list, but there are no defined column names."
                )
            if not len(json_obj) != len(column_names):
                raise TypeError(
                    "The number of columns ({0:d}) does not match the number "
                    "of columns in the JSON list ({1:d})".format(
                        len(column_names), len(json_obj)
                    )
                )
            json_dict = dict(zip(column_names, json_obj))
        elif isinstance(json_obj, dict):
            json_dict = json_obj
        else:
            raise TypeError(
                "The JSON object needs to be either a dict or a list with "
                "defined column names."
            )

        self.add_dict(json_dict)

    @property
    def celery_task_id(self):
        """Return the celery task identification for the upload."""
        return self._celery_task_id

    def close(self):
        """Close the streamer."""
        try:
            self._ready()
        except ValueError:
            return

        if self._data_lines:
            self.flush(end_stream=True)

        # Trigger auto analyzer pipeline to kick in.
        pipe_resource = "{0:s}/sketches/{1:d}/analyzer/".format(
            self._sketch.api.api_root, self._sketch.id
        )
        data = {"index_name": self._index}
        _ = self._sketch.api.session.post(pipe_resource, json=data)

    def flush(self, end_stream=True):
        """Flushes the buffer and uploads to timesketch.

        Args:
            end_stream: boolean that determines whether this is the final
                data to be flushed or whether there is more to come.

        Raises:
            ValueError: if the stream object is not fully configured.
            RuntimeError: if the stream was not uploaded.
        """
        if not self._data_lines:
            return

        self._ready()
        self._upload_data_buffer(end_stream=end_stream)

    @property
    def response(self):
        """Returns the last response from an upload."""
        return self._last_response

    def set_csv_delimiter(self, delimiter):
        """Set the CSV delimiter for CSV file parsing."""
        self._csv_delimiter = delimiter

    def set_data_label(self, data_label):
        """Set the data label of the imported data."""
        self._data_label = data_label

    def set_data_type(self, data_type):
        """Sets the column where the data_type is defined in."""
        self._data_type = data_type

    def set_datetime_column(self, column):
        """Sets the column where the timestamp is defined in."""
        self._datetime_field = column

    def set_entry_threshold(self, threshold):
        """Set the threshold for number of entries per chunk."""
        self._threshold_entry = threshold

    def set_filesize_threshold(self, threshold):
        """Set the threshold for file size per chunk."""
        self._threshold_filesize = threshold

    def set_config_helper(self, helper):
        """Set the config helper object."""
        self._config_helper = helper

    def set_index_name(self, index):
        """Set the index name."""
        self._index = index

    def set_provider(self, provider):
        """Set the data provider."""
        self._provider = provider

    def set_upload_context(self, upload_context):
        """Set the upload context for the data import."""
        self._upload_context = upload_context

    def generate_index_name(self):
        """Generates a new index name."""
        self._index = uuid.uuid4().hex

    def set_message_format_string(self, format_string):
        """Set the message format string."""
        self._format_string = format_string

    def set_sketch(self, sketch):
        """Set a client for the streamer.

        Args:
            sketch: an instance of Sketch that is used to communicate
                with the API to upload data.
        """
        self._sketch = sketch
        self._resource_url = "{0:s}/upload/".format(sketch.api.api_root)

    def set_text_encoding(self, encoding):
        """Set the default encoding for reading text files."""
        self._text_encoding = encoding

    def set_timeline_name(self, name):
        """Set the timeline name."""
        self._timeline_name = name

    def set_timestamp_description(self, description):
        """Set the timestamp description field."""
        self._timestamp_desc = description

    @property
    def state(self):
        """Returns a state string for the indexing process."""
        if not self._celery_task_id:
            return "Unknown"

        tasks = self._sketch.api.check_celery_status(job_id=self._celery_task_id)

        if len(tasks) > 1:
            for task in tasks:
                if task.get("task_id", "") == self._celery_task_id:
                    return task.get("state", "Unknown")

        task = tasks[0]
        return task.get("state", "Unknown")

    @property
    def timeline(self):
        """Returns a timeline object."""
        if not self._timeline_id:
            logger.warning("No timeline ID has been stored as of yet.")
            return None

        timeline_obj = timeline.Timeline(
            timeline_id=self._timeline_id,
            sketch_id=self._sketch.id,
            api=self._sketch.api,
            name=self._timeline_name,
            searchindex=self._index,
        )
        return timeline_obj

    def __enter__(self):
        """Make it possible to use "with" statement."""
        self._reset()
        return self

    # pylint: disable=unused-argument
    def __exit__(self, exception_type, exception_value, traceback):
        """Make it possible to use "with" statement."""
        self.close()


"""Add vote table and poll numbers

Revision ID: 8c47e9ed82c0
Revises: 8901d961d1d9
Create Date: 2021-06-25 20:23:16.229606

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '8c47e9ed82c0'
down_revision = '8901d961d1d9'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('vote',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('guid', sa.Integer(), nullable=False),
    sa.Column('pollNumber', sa.Integer(), nullable=False),
    sa.Column('reason', sa.String(length=256), nullable=True),
    sa.Column('electionID', sa.Integer(), nullable=False),
    sa.ForeignKeyConstraint(['electionID'], ['election.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.add_column('participant', sa.Column('pollNumber', sa.Integer(), nullable=True))
    op.create_foreign_key(None, 'participantelection', 'participant', ['participantId'], ['id'])
    op.create_foreign_key(None, 'participantelection', 'election', ['electionId'], ['id'])
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_constraint(None, 'participantelection', type_='foreignkey')
    op.drop_constraint(None, 'participantelection', type_='foreignkey')
    op.drop_column('participant', 'pollNumber')
    op.drop_table('vote')
    # ### end Alembic commands ###


import time
import datetime

class CTime1000:
    def __init__(self, txt_msg="time", end_time = 0 ):
        self.name = txt_msg
        self.start_time = self.get_mmsec()
        self.end_time = self.start_time + end_time
        self.old_txt = ""
        self.debug_Timer = 0
        self.pre_second = 0
        self.sample_cnt = 0
        self.sample_val = 0

    def __del__(self):
        print( "{} - class delete".format( self.name ))

    def get_mmsec( self ):
        cur_time = time.time() * 0.001
        return cur_time

    def get_time(self):
        cur_time = self.get_mmsec()
        delta_time = self.end_time - cur_time

        return delta_time

    def startTime(self, end_time ):
        self.start_time = self.get_mmsec()
        self.end_time = self.start_time + end_time

    def endTime(self, end_time = -1):
        ret_code = False
        if  end_time > 0:
            self.end_time = self.start_time + end_time

        cur_time = self.get_mmsec()
        delta_time = self.end_time - cur_time
        if delta_time < 0.:
              ret_code = True

        return  ret_code

    def sampleTime(self):
        self.sample_cnt += 1
        now = datetime.datetime.now()
        if now.second != self.pre_second:
            self.pre_second = now.second
            self.sample_val = self.sample_cnt
            self.sample_cnt = 0

        return  self.sample_val


import argparse
import tomli

def get_args():
    parser = argparse.ArgumentParser(description="Process .txt and .md files to .html")
    parser.add_argument(
        "path", nargs="?",
        help="path to the file or folder to be processed"
    )
    parser.add_argument(
        "--version", "-v", action="store_true",
        help="print the tool's name and version"
    )
    parser.add_argument(
        "--output",
        "-o",
        default="./html/examples",
        help="Specify a different output directory",
    )
    parser.add_argument(
        "--lang",
        "-l",
        default="en-CA",
        help="Specify the language for the lang attribute in the HTML document",
    )
    parser.add_argument(
        "--config", "-c",
        help="Specify a different config file"
    )

    args = parser.parse_args()

    # Handle TOML config logic
    if args.config:
        try:
            with open(args.config, "rb") as f:
                default_values = {
                    "path": None,
                    "output": "./html/examples",
                    "lang": "en-CA",
                    "version": False,
                }

                toml_dict = tomli.load(f)

                # Convert the dict to a Namespace object
                args = argparse.Namespace(**toml_dict)

                # Set any missing args to their default values
                for key, value in default_values.items():
                    setattr(args, key, getattr(args, key, value))
        except tomli.TOMLDecodeError:
            print(f"Error: {args.config} is not a valid config TOML file.")
            return None
        except FileNotFoundError:
            print(f"Error: {args.config} does not exist.")
            return None
        except IsADirectoryError:
            print(
                f"Error: {args.config} is a directory. Must provide path to config file."
            )
            return None

    return args


# What the Gang of Four’s original Singleton Pattern
# might look like in Python.
import matplotlib.pyplot as plt
import pandas as pd
from Action import Action
from datetime import datetime
from pathlib import Path

PLOT_DIR = "plots"

class Listener(object):
    _instance = None

    def __init__(self):
        raise RuntimeError('Call instance() instead')

    @classmethod
    def instance(cls):
        if cls._instance is None:
            cls._instance = cls.__new__(cls)
            cls._instance.tick = 0
            cls._instance.stockPrice = {}
            cls._instance.agentWealths = {}
            cls._instance.agentWorths = {}
            cls._instance.cooperations = {}
            cls._instance.caActions = {}

            # create directory to store plots
            path = (Path(PLOT_DIR) / datetime.now().strftime("%Y%m%d%H%M%S")).resolve()
            path.mkdir(parents=True)
            cls._instance.plotDir = path
        return cls._instance

    def newTick(self):
        self.tick += 1

    def newStockPrice(self, price: float):
        if len(self.stockPrice) == 0:
            self.initialPrice = price

        self.stockPrice[self.tick] = price

    def addAgentWealth(self, agentName: str, agentBalance: float):
        if self.tick not in self.agentWealths:
            self.agentWealths[self.tick] = {}
        self.agentWealths[self.tick][agentName] = agentBalance

    def addAgentWorth(self, agentName: str, agentBalance: float):
        if self.tick not in self.agentWorths:
            self.agentWorths[self.tick] = {}
        self.agentWorths[self.tick][agentName] = agentBalance

    def addCooperation(self, agentName: str):
        if self.tick not in self.cooperations:
            self.cooperations[self.tick] = []

        self.cooperations[self.tick].append(agentName)

    def addCaAction(self, action):
        if self.tick not in self.caActions:
            self.caActions[self.tick] = {}

        if action not in self.caActions[self.tick]:
            self.caActions[self.tick][action] = 0

        self.caActions[self.tick][action] += 1

    def __plotStockTickVariation(self, axis):
        xx = []
        yy = []
        for (t, v) in self.stockPrice.items():
            if t-1 in self.stockPrice:
                yy.append(v - self.stockPrice[t-1])
                xx.append(t)

        axis.plot(xx, yy)

    def __plotStockBeginDiff(self, axis):
        xx = []
        yy = []
        for (t, v) in self.stockPrice.items():
            yy.append(v - self.initialPrice)
            xx.append(t)

        axis.plot(xx, yy)

    def __plotMeanAgentWealth(self, axis):
        xx = []
        yy = []
        for (t, d) in self.agentWealths.items():
            xx.append(t)
            yy.append(sum(d.values())/len(d))

        axis.plot(xx, yy)

    def __plotMeanAgentWorth(self, axis):
        xx = []
        yy = []
        for (t, d) in self.agentWorths.items():
            xx.append(t)
            yy.append(sum(d.values())/len(d))

        axis.plot(xx, yy)

    def __plotCoopPercentPerTick(self):
        plt.figure(figsize=(10,5))

        # number of cooperative agents
        n = len([a for a in self.agentWorths[0].keys() if "ca_" in a])

        xx = self.cooperations.keys()
        yy = [ len(l)/n for l in self.cooperations.values() ]

        plt.ylim(0, 1)
        plt.ylabel("Coop. percent.")
        plt.xlabel("Ticks")
        plt.plot(xx, yy)
        plt.title("Cooperation percentage per tick")
        plt.savefig(self.plotDir / '1_coopPercentPerTick.png')

    def __plotStockPrices(self):
        plt.figure(figsize=(10,5))

        xx = []
        yy = []
        for (t, v) in self.stockPrice.items():
            xx.append(t)
            yy.append(v)

        plt.plot(xx, yy)
        plt.ylabel("Stock Price")
        plt.xlabel("Ticks")
        plt.title('Stock price per tick')
        plt.savefig(self.plotDir / '2_stockPricePerTick.png')

    def __plotFinalAgentsWorth(self):
        plt.figure(figsize=(10,5))

        lastTick = max(self.agentWorths.keys())
        xx = []
        heights = []
        wia_sum = 0
        wia_count = 0
        ca_sum = 0
        ca_count = 0
        for agent, balance in self.agentWorths[lastTick].items():
            agentClass = agent[:agent.find("_")]
            if agent == "sia_0":
                xx.append("Strong Isolated\nAgents")
                heights.append(balance - self.agentWorths[0][agent])
            elif agentClass == "wia":
                wia_sum += balance - self.agentWorths[0][agent]
                wia_count += 1
            elif agentClass == "ca":
                ca_sum += balance - self.agentWorths[0][agent]
                ca_count += 1

        wia_avg = 0
        ca_avg = 0
        if wia_count != 0:
            wia_avg = wia_sum / wia_count
        if ca_count != 0:
            ca_avg = ca_sum / ca_count

        xx.append("Weak Isolated\nAgents")
        heights.append(wia_avg)
        xx.append("Cooperative\nAgents")
        heights.append(ca_avg)

        plt.bar(x=xx, height=heights)
        plt.ylabel("Final worth balance")
        plt.title("Average final worth balance per agent type")
        plt.savefig(self.plotDir / '3_avgFianlWorthBalancePerAgent.png')

    def __plotFinalAgentsWealth(self):
        plt.figure(figsize=(10,5))

        lastTick = max(self.agentWealths.keys())
        xx = []
        heights = []
        wia_sum = 0
        wia_count = 0
        ca_sum = 0
        ca_count = 0
        for agent, balance in self.agentWealths[lastTick].items():
            agentClass = agent[:agent.find("_")]
            if agent == "sia_0":
                xx.append("Strong Isolated\nAgents")
                heights.append(balance - self.agentWealths[0][agent])
            elif agentClass == "wia":
                wia_sum += balance - self.agentWealths[0][agent]
                wia_count += 1
            elif agentClass == "ca":
                ca_sum += balance - self.agentWealths[0][agent]
                ca_count += 1

        wia_avg = 0
        ca_avg = 0
        if wia_count != 0:
            wia_avg = wia_sum / wia_count
        if ca_count != 0:
            ca_avg = ca_sum / ca_count

        xx.append("Weak Isolated\nAgents")
        heights.append(wia_avg)
        xx.append("Cooperative\nAgents")
        heights.append(ca_avg)

        plt.bar(x=xx, height=heights)
        plt.ylabel("Final wealth balance")
        plt.title("Average final wealth balance per agent type")
        plt.savefig(self.plotDir / '4_avgFinalWealthBalancePerAgent.png')

    def __plotAvgWorthPerAgentType(self):
        plt.figure(figsize=(10,5))

        xx = []
        yy_wia = []
        yy_sia = []
        yy_ca  = []

        lastTick = max(self.agentWorths.keys())
        for t in range(1, lastTick):
            xx.append(t)
            wia_sum = 0
            sia_sum = 0
            ca_sum  = 0

            wia_count = 0
            ca_count  = 0
            for agent, balance in self.agentWorths[t].items():
                agentClass = agent[:agent.find("_")]
                y = balance - self.agentWorths[0][agent]
                if agent == "sia_0":
                    yy_sia.append(y)
                elif agentClass == "wia":
                    wia_sum += y
                    wia_count += 1
                elif agentClass == "ca":
                    ca_sum += y
                    ca_count += 1

            if wia_count != 0:
                yy_wia.append(wia_sum / wia_count)
            if ca_count != 0:
                yy_ca.append(ca_sum / ca_count)

        plt.title("Average Worth Balance per tick")
        plt.ylabel("Mean Agent Worth Balance")
        plt.xlabel("Ticks")
        plt.plot(xx, yy_wia, label="WIA")
        plt.plot(xx, yy_sia, label="SIA")
        plt.plot(xx, yy_ca,  label="CA")
        # plt.legend(bbox_to_anchor = (0.75, 0.21), ncol = 3)
        plt.legend(["WIA", "SIA", "CA"])

        plt.savefig(self.plotDir / '5_avgWorthPerTick.png')

    def __plotCooperationImmediateEffect(self):
        plt.figure(figsize=(10,5))

        coop = []
        ncoop = []

        lastTick = max(self.agentWorths.keys())
        for t in range(0, lastTick, 1):
            for agent, worth in self.agentWorths[t].items():
                if "ca" not in agent:
                    continue

                if agent in self.cooperations[t]:
                    coop.append(self.agentWorths[t+1][agent] - worth)
                else:
                    ncoop.append(self.agentWorths[t+1][agent] - worth)

        # heights = [coop_sum, ncoop_sum]
        xx = [coop, ncoop]
        plt.title("Cooperation immediate effect in agent worth")

        red_square = dict(markerfacecolor='g', marker='o', alpha=0.2)

        plt.boxplot(x=xx, labels=["Cooperate", "Not cooperate"], flierprops=red_square)
        plt.savefig(self.plotDir / '6_cooperationImmediateEffect.png')

    def __plotCaActions(self):
        plt.figure(figsize=(10,5))

        nca = sum([1 for agent in self.agentWorths[0].keys() if "ca" in agent])
        if nca == 0:
            return

        actions = [Action.SKIP, Action.HOLD, Action.SELL, Action.BUY]
        lastTick = max(self.agentWorths.keys())

        xx = []
        yy = {}
        for a in actions:
            yy[a] = []

        variationThreshold = 3
        for t in range(0, lastTick, 1):
            if abs(self.stockPrice[t] - self.stockPrice[t+1]) < variationThreshold:
                # ignore ticks with small variation
                continue

            xx.append(t)
            s = 0
            for a in actions:
                if a not in self.caActions[t]:
                    self.caActions[t][a] = 0

                percent = self.caActions[t][a] / nca
                s += percent
                yy[a].append(s)

        for a in actions:
            df = pd.DataFrame(yy[a])
            df.rolling(100, min_periods=1)
            plt.plot(xx, df.values.tolist())

        plt.fill_between(xx, 0, yy[actions[0]])
        plt.fill_between(xx, yy[actions[0]], yy[actions[1]])
        plt.fill_between(xx, yy[actions[1]], yy[actions[2]])
        plt.fill_between(xx, yy[actions[2]], yy[actions[3]])

        plt.savefig(self.plotDir / '7_caActions.png')

    def spitData(self):
        # self.__plotCoopPercentPerTick()
        # self.__plotStockPrices()
        # self.__plotFinalAgentsWorth()
        # self.__plotFinalAgentsWealth()
        # self.__plotAvgWorthPerAgentType()
        # self.__plotCooperationImmediateEffect()
        # self.__plotCaActions()
        print(sum(self.stockPrice.values())/len(self.stockPrice.keys()))

listener = Listener.instance()


from django.db import models
from django.utils.translation import gettext_lazy as _

from cms.models import CMSPlugin

from djangocms_bootstrap4.constants import COLOR_STYLE_CHOICES
from djangocms_bootstrap4.fields import AttributesField, TagTypeField

from .constants import LISTGROUP_STATE_CHOICES

class Bootstrap4ListGroup(CMSPlugin):
    """
    Components > "List Group" Plugin
    https://getbootstrap.com/docs/4.0/components/list-group/
    """
    list_group_flush = models.BooleanField(
        verbose_name=_('List group flush'),
        default=False,
        help_text=_('Create lists of content in a card with a flush list group.')
    )
    tag_type = TagTypeField()
    attributes = AttributesField()

    def __str__(self):
        return str(self.pk)

    def get_short_description(self):
        text = ''
        if self.list_group_flush:
            text += '.list-group-flush'
        return text

class Bootstrap4ListGroupItem(CMSPlugin):
    """
    Components > "List Group Ite" Plugin
    https://getbootstrap.com/docs/4.0/components/list-group/
    """
    list_context = models.CharField(
        verbose_name=_('Context'),
        choices=COLOR_STYLE_CHOICES,
        blank=True,
        max_length=255,
    )
    list_state = models.CharField(
        verbose_name=_('State'),
        choices=LISTGROUP_STATE_CHOICES,
        blank=True,
        max_length=255,
    )
    tag_type = TagTypeField()
    attributes = AttributesField()

    def __str__(self):
        return str(self.pk)

    def get_short_description(self):
        text = []
        if self.list_context:
            text.append(f'.list-group-item-{self.list_context}')
        if self.list_state:
            text.append(f'.{self.list_state}')
        return ' '.join(text)


# grid search hypers

params = [
    ('epochs', [50, 75]),
    ('batch_size', [64]),
    ('validation_split', [0.]),
    ('filters', [320]),
    ('kernel_size', [5]),
    ('conv_activation', ['relu']),
    ('conv_l2_regularizer', [0.1, 0.01, 0.001, 0.]),
    ('dropout_rate', [0., 0.2, 0.6]),
    ('dense_activation', ['tanh']),
    ('dense_l2_regularizer', [0.1, 0.01, 0.001, 0.]),
    ('activation', ['sigmoid']),
    ('optimizer', ['nadam']),
    ('loss_function', ['binary_crossentropy']),
    ('units', [128]),
    ('trainable', [False]),
    ('dense_layers', [1, 2]),
]

param_grid = dict(params)

import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

import sys
try:
    sys.path.insert(0, '.')
    from constants import Const
    sys.path.insert(0, Const.ROOT)
except:
    sys.path.insert(0, '..')
    from constants import Const

import utils

from MyClassifier import MyClassifier, Model
from sklearn.base import BaseEstimator, ClassifierMixin

from tensorflow.keras import backend as K
from tensorflow.keras import optimizers, regularizers
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.layers import (GRU, LSTM, RNN, Bidirectional,
                          Dense, Dropout, Lambda, RepeatVector,
                          TimeDistributed, Concatenate)
from tensorflow.keras.layers import Conv1D
from tensorflow.keras.layers import Embedding
from tensorflow.keras.layers import (AveragePooling1D, GlobalMaxPooling1D,
                                  MaxPooling1D)
from tensorflow.keras import Input, Sequential
from tensorflow.keras.models import load_model
from tensorflow.keras.preprocessing import sequence, text
import dill
import numpy as np

import matplotlib.pyplot as plt
import itertools
from sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score, confusion_matrix
from sklearn.model_selection import train_test_split

class CNNSentimentPolarityClassifier (MyClassifier):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        self.WEIGHTS_PATH = 'model/cnn/weights/best_{}.hdf5'
        self.MODEL_PATH = 'model/cnn/best_{}.model'
        self.WE_PATH = '../we/embedding_matrix.pkl'
        self.THRESHOLD = 0.5
        self.target_names = ['polarity']

        self.cnn_model = None
        for key, value in kwargs.items():
            setattr(self, key, value)

    def fit(self, X, y,

        filters = 320,
        kernel_size = 5,
        conv_activation = 'tanh',
        conv_l2_regularizer = 0.01,
        dropout_rate = 0.6,
        dense_activation = 'relu',
        dense_l2_regularizer = 0.01,
        activation = 'sigmoid',
        optimizer = "nadam",
        loss_function = 'binary_crossentropy',
        units = 256,
        trainable = False,
        dense_layers = 1,
        class_weight = None,

        is_save = False,
        category = 'NOTSPECIFIED',

        **kwargs):
        self.cnn_model = self._create_model(
        	filters,
            kernel_size,
            conv_activation,
            conv_l2_regularizer,
            dropout_rate,
            dense_activation,
            dense_l2_regularizer,
            activation,
            optimizer,
            loss_function,
            units,
            trainable,
            dense_layers,
        )
        mode = kwargs.get('mode', 'train_validate_split')
        if mode == "train_validate_split":
            # self.cnn_model.summary()
            self.cnn_model.fit(
                X, y,
                class_weight=class_weight,
                **kwargs
            )
        if is_save:
            self.cnn_model.save(self.MODEL_PATH.format(category))

    def predict(self, X, **kwargs):
        y_pred = self.cnn_model.predict(X)
        if y_pred.shape[1] > 1:
            return y_pred.argmax(axis=-1)
        else:
            THRESHOLD = self.THRESHOLD
            y_pred[y_pred > THRESHOLD] = 1.
            y_pred[y_pred <= THRESHOLD] = 0.
            return y_pred

    def _fit_train_validate_split(self, X, y):
        pass

    def _fit_gridsearch_cv(self, X, y, param_grid, category="NOTSPECIFIED", **kwargs):
        from sklearn.model_selection import GridSearchCV
        from tensorflow.keras.wrappers.scikit_learn import KerasClassifier

        np.random.seed(7)

        # Wrap in sklearn wrapper
        model = KerasClassifier(build_fn = self._create_model, verbose=0)

        # train
        IS_REFIT = kwargs.get('is_refit','f1_macro')
        grid = GridSearchCV(estimator=model, param_grid=param_grid, cv=5, refit=IS_REFIT, scoring=['f1_macro', 'precision_macro', 'recall_macro'], verbose=1)
        grid_result = grid.fit(X, y)
        # print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
        print(grid_result.cv_results_.keys())
        means = [grid_result.cv_results_['mean_test_f1_macro'], grid_result.cv_results_['mean_test_precision_macro'], grid_result.cv_results_['mean_test_recall_macro']]
        stds = [grid_result.cv_results_['std_test_f1_macro'], grid_result.cv_results_['std_test_precision_macro'], grid_result.cv_results_['std_test_recall_macro']]
        for mean, stdev in zip(means, stds):
            print("\n{} ({})".format(mean, stdev))
        params = grid_result.best_params_
        print("with:", params)
        with open('output/gridsearch_cnn_{}.pkl'.format(category), 'wb') as fo:
            dill.dump(grid_result.cv_results_, fo)
        if IS_REFIT:
            grid.best_estimator_.model.save('model/cnn/best_{}.model'.format(category))

    def _create_model(
        self,

        filters = 320,
        kernel_size = 5,
        conv_activation = 'tanh',
        conv_l2_regularizer = 0.01,
        dropout_rate = 0.6,
        dense_activation = 'relu',
        dense_l2_regularizer = 0.01,
        activation = 'sigmoid',
        optimizer = "nadam",
        loss_function = 'binary_crossentropy',
        units = 256,
        trainable = False,
        dense_layers = 1,

        **kwargs
    ):
        K.clear_session()
        MAX_SEQUENCE_LENGTH = kwargs.get("max_sequence_length", 150)

        # Define Architecture
        layer_input = Input(shape=(MAX_SEQUENCE_LENGTH,))
        layer_embedding = self._load_embedding(self.WE_PATH, trainable=trainable, vocabulary_size=15000, embedding_vector_length=500)(layer_input)
        layer_conv = Conv1D(filters=filters, kernel_size=kernel_size, padding='same', activation=conv_activation,
        kernel_regularizer=regularizers.l2(conv_l2_regularizer))(layer_embedding)
        layer_pooling = GlobalMaxPooling1D()(layer_conv)
        layer_dropout = Dropout(dropout_rate, seed=7)(layer_pooling)
        for i in range(dense_layers):
            layer_dense = Dense(units, activation=dense_activation, kernel_regularizer=regularizers.l2(dense_l2_regularizer))(layer_dropout)
            layer_dropout = Dropout(dropout_rate, seed=7)(layer_dense)
        layer_softmax = Dense(1, activation=activation)(layer_dropout)

        # Create Model
        cnn_model = Model(inputs=layer_input, outputs=layer_softmax)

        # Create Optimizer
        # optimizer = optimizers.Nadam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay=0.004)
        # optimizer = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
        cnn_model.compile(loss=loss_function, optimizer=optimizer, metrics=['accuracy'])
        return cnn_model

    def load_weights(self, path):
        self._create_model()
        self.cnn_model.load_weights(path)

    def get_threshold(self):
        return self.THRESHOLD

    def load_best_model(self, category):
        best_model = load_model(Const.SPC_ROOT + 'model/cnn/best_{}.model'.format(category))
        del self.cnn_model
        self.cnn_model = best_model

def get_best_params():
    best_params = {
        'food':
        {
            'epochs': 75,
            'batch_size': 64,
            'filters': 320,
            'kernel_size': 5,
            'conv_activation': 'relu',
            'conv_l2_regularizer': 0.01,
            'dropout_rate': 0.2,
            'dense_activation': 'tanh',
            'dense_l2_regularizer': 0.001,
            'activation': 'sigmoid',
            'loss_function': 'binary_crossentropy',
            'units': 128,
            'trainable': False,
            'dense_layers': 1,
        },

        'service':
        {
            'epochs': 75,
            'batch_size': 64,
            'filters': 320,
            'kernel_size': 5,
            'conv_activation': 'relu',
            'conv_l2_regularizer': 0.001,
            'dropout_rate': 0.6,
            'dense_activation': 'tanh',
            'dense_l2_regularizer': 0.001,
            'activation': 'sigmoid',
            'loss_function': 'binary_crossentropy',
            'units': 128,
            'trainable': False,
            'dense_layers': 1,
        },

        'price':
        {
            'epochs': 50,
            'batch_size': 64,
            'filters': 320,
            'kernel_size': 5,
            'conv_activation': 'relu',
            'conv_l2_regularizer': 0.01,
            'dropout_rate': 0.,
            'dense_activation': 'tanh',
            'dense_l2_regularizer': 0.,
            'activation': 'sigmoid',
            'loss_function': 'binary_crossentropy',
            'units': 128,
            'trainable': False,
            'dense_layers': 1,
        },

        'place':
        {
            'epochs': 50,
            'batch_size': 64,
            'filters': 320,
            'kernel_size': 5,
            'conv_activation': 'relu',
            'conv_l2_regularizer': 0.01,
            'dropout_rate': 0.2,
            'dense_activation': 'tanh',
            'dense_l2_regularizer': 0.,
            'activation': 'sigmoid',
            'loss_function': 'binary_crossentropy',
            'units': 128,
            'trainable': False,
            'dense_layers': 1,
        },
    }

    return best_params

def main():
    categories = ['food', 'service', 'price', 'place']
    f1_scores = []
    p_scores = []
    r_scores = []
    for category in categories:
        print("\n\n========= CHECKING CATEGORY:", category, "==========")
        """
            Initialize data
        """
        X, y, X_test, y_test = utils.get_spc_dataset(category)
        # X_train, X_validate, y_train, y_validate = train_test_split(X, y, test_size=0.20, random_state=7)

        """
            Make the model
        """
        # np.random.seed(7)

        # checkpointer = ModelCheckpoint(filepath='model/cnn/weights/CNN.hdf5', verbose=0, save_best_only=True)
        spc = CNNSentimentPolarityClassifier()

        """
            Fit the model
        """

        from tensorflow.keras.utils import to_categorical
        # y = to_categorical(y)
        # spc._fit_new_gridsearch_cv(X, y, params, result_path="output/gridsearch_cv_result_{}.csv".format(category), score_verbose=True)
        best_params = get_best_params()

        is_save = {
            'food': False,
            'service': False,
            'price': False,
            'place': False,
        }
        if is_save[category]:
            spc.fit(X, y,
                **best_params[category],
                verbose=1,
                category=category,
                is_save = is_save[category],
                validation_split=0.,
                optimizer='nadam',
            )

        """
            Load best estimator and score it
        """

        spc.load_best_model(category)

        # preds = spc.predict(X_test)
        # for pred in preds:
        #     if pred[0] == pred[1]:
        #         print("SAMA", pred[0])

        score = spc.score(X_test, y_test, verbose=1)
        f1_scores.append(score['f1_score_macro'])
        p_scores.append(score['precision_score_macro'])
        r_scores.append(score['recall_score_macro'])
    print("F1-MEAN-MACRO:", np.array(f1_scores).mean())
    print("P -MEAN-MACRO:", np.array(p_scores).mean())
    print("R -MEAN-MACRO:", np.array(r_scores).mean())

def get_wrong_preds(data='train'):
    categories = ['food', 'service', 'price', 'place']

    for category in categories:
        print()
        print()
        print("\t######################################################")
        print("\t############## CHECKING CATEGORY: {} ##############".format(category.upper()))
        print("\t######################################################")
        print()
        spc = CNNSentimentPolarityClassifier()
        spc.load_best_model(category=category)
        X, y, X_test, y_test, df, df_test = utils.get_spc_dataset(category, return_df = True)

        data = 'train'
        if data == 'test':
            df = df_test
            X = X_test
            y = y_test

        print(len(df))
        y_pred = spc.predict(X)

        spc.score(X, y)

        cnt = 0
        for i, (review, y_pred_single, y_single) in enumerate(list(zip(df['review'], y_pred, y.values.tolist()))):
            y_pred_single = int(y_pred_single[0])
            if y_pred_single != y_single:
                cnt += 1
                print("================= {} =================".format(i))
                print(review)
                print('PRED:', y_pred_single)
                print('ACTL:', y_single)
                print()
        print(cnt, "sentences missclasified")

if __name__ == "__main__":
    # utils.time_log(main)
    utils.time_log(get_wrong_preds)


import json
import boto3
import re
import os
import datetime
import urllib.request
import traceback

USER_ID = os.environ['USERID']
ACCESS_TOKEN = {}

def getAccessToken(shopId="dev", manage=False):
    global ACCESS_TOKEN
    if not shopId in ACCESS_TOKEN:
        dynamoDB = boto3.resource("dynamodb")
        table = dynamoDB.Table("userMaster")  # DynamoDBのテーブル名
        item = table.get_item(Key={"userId": shopId, "key": "ユーザー設定"})

        ACCESS_TOKEN[shopId] = item["Item"]["data"]

    if manage:
        return ACCESS_TOKEN[shopId]["AccessTokenManage"]
    else:
        return ACCESS_TOKEN[shopId]["AccessToken"]

def manageMessage(mes, shopId="dev", to=None):
    message(mes, to, True, shopId)

def message(mes, to=None, isManage=False, shopId="dev"):
    url = "https://api.line.me/v2/bot/message/push"

    messages = []

    if type(mes) is str:
        messages += [{
            'type': 'text',
            'text': mes
        }]
    else:
        messages += [{"type": "template",
                      "altText": mes["text"],
                      "template": mes}]

    print(messages)

    if to is None:
        to = USER_ID

    postData = {
        "to": to,
        "messages": messages
    }
    accessToken = getAccessToken(shopId, isManage)

    print(f"accessToken:{accessToken}")

    headers = {
        "Content-Type": "application/json",
        'Authorization': 'Bearer ' + accessToken,
    }

    req = urllib.request.Request(url, json.dumps(postData).encode(), headers)

    with urllib.request.urlopen(req) as res:
        body = json.loads(res.read())