from socket import *
import os
import threading
import sys
from termcolor import colored   #for highlighting
import logging

#now we will Create and configure logger
logging.basicConfig(filename="proxy.log",
					format='%(asctime)s %(message)s',
					filemode='w')

#Let us Create an object
logger=logging.getLogger()

#Now we are going to Set the threshold of logger to DEBUG
logger.setLevel(logging.DEBUG)

threads = []
clients = []
tcpSerSock = socket(AF_INET, SOCK_STREAM)
tcpSerSock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
PROXY_SERV = '127.0.0.1'
PROXY_PORT = 8888
tcpSerSock.bind((PROXY_SERV,PROXY_PORT))
tcpSerSock.listen(5)
print(colored("[INFO] ",'green'),end="")
print(colored('Serving on Port '+str(PROXY_PORT),'white'))
logger.debug("Server running on port "+str(PROXY_PORT))

def proxy(tcpCliSock, addr):
		while True:
			try:
				message =  tcpCliSock.recv(2048)
				if message.decode() == "quit":
					logger.debug("Closing Proxy Server socket")
					tcpSerSock.close()
					for client in clients:
						client.send("exit".encode())
						client.close()
					break

				elif message.decode() != "":
					print(colored("[INFO] ",'green'),end="")
					print(colored("Message from ",'white'),end="")
					print(colored(str(addr[0])+":"+str(addr[1]),'cyan'),end="")
					print(colored(' :- '+message.decode(),'white'))
					filename = message.decode()
					fileExist = "false"
					filepath = "p_www/" + filename
					logger.debug(str(addr[0])+":"+str(addr[1])+" Requesting "+filepath)
					arr = os.listdir("p_www")
					if filename in arr:
						print(colored("[INFO] ",'green'),end="")
						print(colored(filename+" found in Proxy Server. Sending to "+str(addr[0])+':'+str(addr[1]),'white'))
						logger.debug(filename+" found in Proxy Server. Sending to "+str(addr[0])+':'+str(addr[1]))
						f = open(filepath,'r')
						data = f.read()
						tcpCliSock.send(data.encode())
					else:
						print(colored("[INFO] ",'green'),end="")
						print(colored("File not found in Proxy Server. Requesting from Server",'yellow'))
						logger.info(filepath+" not found in Proxy Server. Requesting from Server")
						server = socket(AF_INET, SOCK_STREAM)
						server.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
						try:
							server.connect(("127.0.0.1",8081))
							print(colored("[INFO] ",'green'),end="")
							print(colored("Connected to Server. Fetching from server",'white'))
							logger.info("Connected to Server. Fetching "+filename+" from server")
							server.send(filename.encode())
							data = server.recv(2048)
							if data.decode() != 'Nil':
								print(colored("[INFO] ",'green'),end="")
								print(colored(filename+" found in Server. Sending to "+str(addr[0])+':'+str(addr[1]),'white'))
								tcpCliSock.send(data)
							else:
								print(colored("[INFO] ",'green'),end="")
								print(colored(filename+" not found in Server.",'yellow'))
								logger.info(filename+" not found in Server.")
								tcpCliSock.send("File not found".encode())

							server.send("exit".encode())	#disconnect from server
							server.close()
							if "Nil" not in data.decode():
								print(colored("[INFO] ",'green'),end="")
								print(colored("Writing "+filename+"to Proxy Server",'yellow'))
								logger.debug("Writing "+filepath+" to Proxy Server")
								w = open(filepath,'w')
								w.write(data.decode())
								w.close()

						except Exception as e:
							tcpCliSock.send("Unable to fetch file / file not found".encode())
							print(colored("[INFO] ",'green'),end="")
							print(colored("Unable to connect to Server",'yellow'))
							logger.info("Unable to fetch file / file not found")
							logger.error(e)

				else:
					continue

			except Exception as e:
				logger.warning("something went wrong")
				logger.error(e)
				break

while True:
	try:
		tcpCliSock, addr = tcpSerSock.accept()
		clients.append(tcpCliSock)
		print(colored("[INFO] ",'green'),end="")
		print(colored(str(addr[0])+':'+str(addr[1]),'cyan'),end="")
		print(colored(' connected','white'))
		logger.info(str(addr[0])+':'+str(addr[1])+" connected")
		t = threading.Thread(target=proxy, args=(tcpCliSock,addr))
		threads.append(t)
		t.start()
	except:
		logger.warning("Shutting down")
		break

for t in threads:
    t.join()

print(colored("[SHUTDOWN] ",'green'),end="")
print(colored("Proxy Server has shutdown",'white'))
logger.info("Server has shutdown")


from django.shortcuts import render,redirect,get_object_or_404
from django.http import HttpResponse
from .models import Userstore
from django.views.decorators.csrf import csrf_exempt
import json
from django.utils import timezone
from pharmacist.views import getMedicineList
from diagnosticexecutive.views import getDiagnosisList

def deskHome(request):  #Deskexecutive HomePage after login
	return render(request,'deskexecutive/deskHome.html')

def addPatient(request):
	if request.method == "POST":  # checking whether data coming is from post request or not
		# now store all the details from the user in the database
		ssnid = request.POST['ssnid']
		age = request.POST['age']
		name = request.POST['name']
		roomtype = request.POST['roomtype']
		adrs = request.POST['adrs']
		city = request.POST['city']
		state = request.POST['state']
		patientobj = Userstore() #Creating object of the userstore database
		patientobj.ws_ssn = ssnid
		patientobj.ws_pat_name = name
		patientobj.ws_pat_age = age
		patientobj.ws_city = city
		patientobj.ws_state = state
		patientobj.ws_adrs = adrs
		patientobj.ws_status = "Active"
		patientobj.ws_rtype = roomtype
		patientobj.save()
		return redirect('deskexecutive:deskHome') #After adding patient desk executive will be redirected to the Desk executive's Home page
	else:
		return render(request,'deskexecutive/addPatient.html')

def showAllPatient(request):
	user = Userstore.objects  # Getting objects of all the patient in our database
	return render(request,'deskexecutive/showAllPatient.html',{'user':user})

def updatePatient(request):
	if request.method == "POST":
		ws_pat_id=request.POST['uid']
		patientobj = Userstore.objects.get(ws_pat_id=ws_pat_id)

		if request.POST['age']!= patientobj.ws_pat_age:
			patientobj.ws_pat_age = request.POST['age']
		if request.POST['name']!=patientobj.ws_pat_name:
			patientobj.ws_pat_name = request.POST['name']
		if request.POST['roomtype']!=patientobj.ws_rtype:
			patientobj.ws_rtype = request.POST['roomtype']
		if request.POST['adrs']!=patientobj.ws_adrs:
			patientobj.ws_adrs = request.POST['adrs']
		if request.POST['city']!=patientobj.ws_city:
			patientobj.ws_city = request.POST['city']
		if request.POST['state']!=patientobj.ws_state:
			patientobj.ws_state = request.POST['state']
		patientobj.save()
		return redirect('deskexecutive:deskHome')
	else:
		return render(request,'deskexecutive/updatePatient.html')

def deletePatient(request):
	if request.method == "POST":
		ws_pat_id = request.POST['uid']
		patientobj = Userstore.objects.get(ws_pat_id=ws_pat_id)
		# Userstore.objects.filter(ws_pat_id=ws_pat_id).delete()
		patientobj.delete()
		return redirect('deskexecutive:deskHome')
	else:
		return render(request,"deskexecutive/deletePatient.html")

@csrf_exempt
def fetchData(request):
	if request.method == "POST":
		ws_pat_id = request.POST['uid']
		patientobj= get_object_or_404(Userstore,pk=ws_pat_id)

		patientDetails={
		'patientId' : ws_pat_id,
		'patientName': patientobj.ws_pat_name,
		'patientSSN' : patientobj.ws_ssn,
		'patientAge' : patientobj.ws_pat_age,
		'patientCity': patientobj.ws_city,
		'patientState': patientobj.ws_state,
		'patientAdrs' : patientobj.ws_adrs,
		'patientStatus': patientobj.ws_status,
		'patientRtype' : patientobj.ws_rtype,
		'patientDOJ' : str(patientobj.prettyTime)
		}
	return HttpResponse(json.dumps(patientDetails))

@csrf_exempt
def fetchActivePatient(request):
	if request.method == "GET":
		ws_pat_id = request.GET['uid']
		if not (Userstore.objects.filter(ws_pat_id=ws_pat_id).exists()):
			return HttpResponse(json.dumps({'exists':'no'}))
		patientobj= get_object_or_404(Userstore,pk=ws_pat_id)
		patientDetails={
		'exists':'yes',
		'patientId' : ws_pat_id,
		'patientName': patientobj.ws_pat_name,
		'patientSSN' : patientobj.ws_ssn,
		'patientAge' : patientobj.ws_pat_age,
		'patientCity': patientobj.ws_city,
		'patientState': patientobj.ws_state,
		'patientAdrs' : patientobj.ws_adrs,
		'patientStatus': patientobj.ws_status,
		'patientRtype' : patientobj.ws_rtype,
		'patientDOJ' : str(patientobj.prettyTime)
		}
	return HttpResponse(json.dumps(patientDetails))

def billing(request):
	if request.method == "POST":
		pass
	else:
		return render(request,"deskexecutive/billing.html")

@csrf_exempt
def checkout(request):
	if request.method=="GET":
		pid = request.GET['uid']
		patientobj = get_object_or_404(Userstore,ws_pat_id=pid)
		days = str(timezone.now() - patientobj.ws_doj)
		days = int(days[0:days.index(" ")])+1
		if (patientobj.ws_rtype == "General Ward"):
			bedPrice=2000*days
		elif (patientobj.ws_rtype=="Semi-Sharing"):
			bedPrice= days*4000
		else:
			bedPrice=days*8000
		patientDetails={
		'patientId' : patientobj.ws_pat_id,
		'patientName': patientobj.ws_pat_name,
		'patientSSN' : patientobj.ws_ssn,
		'patientRtype' : patientobj.ws_rtype,
		'patientAge' : patientobj.ws_pat_age,
		'patientAdrs' : patientobj.ws_adrs,
		'patientCity': patientobj.ws_city,
		'patientDOJ' : str(patientobj.prettyTime())
		}
		DiagnosTest, testcost= getDiagnosisList(pid)
		medicine, medcost = getMedicineList(pid)
		print(medicine)
		print(DiagnosTest)
		bedDetails={
		'days':days,
		'bedPrice':bedPrice
		}
		data = {"patientDetails":patientDetails,'beds':bedDetails,'medicine':medicine,'DiagnosTest':DiagnosTest}
		return render(request,'deskexecutive/checkout.html',{"total":medcost+testcost+bedPrice,"medcost":medcost,"testcost":testcost,"patientDetails":patientDetails,'days':days,'bedPrice':bedPrice,'medicine':medicine,'DiagnosTest':DiagnosTest})

@csrf_exempt
def searchPatient(request):
	if request.method=="POST":
		ws_pat_id = request.POST['uid']
		if not (Userstore.objects.filter(ws_pat_id=ws_pat_id).exists()):
			return HttpResponse(json.dumps({'exists':'no'}))
		patientobj= get_object_or_404(Userstore,pk=ws_pat_id)
		patientDetails={
		'exists':'yes',
		'patientId' : ws_pat_id,
		'patientName': patientobj.ws_pat_name,
		'patientSSN' : patientobj.ws_ssn,
		'patientAge' : patientobj.ws_pat_age,
		'patientCity': patientobj.ws_city,
		'patientState': patientobj.ws_state,
		'patientAdrs' : patientobj.ws_adrs,
		'patientStatus': patientobj.ws_status,
		'patientRtype' : patientobj.ws_rtype,
		'patientDOJ' : str(patientobj.prettyTime)
		}
		return HttpResponse(json.dumps(patientDetails))
	else:
		return render(request,'deskexecutive/searchPatient.html')


# -*- coding: utf-8 -*-
#
#       ramstk.modules.function.Controller.py is part of The RAMSTK
#       Project
#
# All rights reserved.
# Copyright 2007 - 2017 Doyle Rowland doyle.rowland <AT> reliaqual <DOT> com
"""Function Package Data Controller."""

# Import third party modules.
from pubsub import pub

# Import other RAMSTK modules.
from ramstk.modules import RAMSTKDataController
from ramstk.modules import RAMSTKDataMatrix
from ramstk.dao import RAMSTKFunction, RAMSTKHardware, RAMSTKSoftware
from . import dtmFunction

class FunctionDataController(RAMSTKDataController):
    """
    Provide interface between Function data model and RAMSTK view model.

    A single Function controller can manage one or more Function data
    models.  The attributes of a Function data controller are:
    """

    def __init__(self, dao, configuration, **kwargs):
        """
        Initialize a Function data controller instance.

        :param dao: the RAMSTK Program DAO instance to pass to the
                    Function Data Model.
        :type dao: :class:`ramstk.dao.DAO`
        :param configuration: the Configuration instance associated with
                              the current instance of the RAMSTK
                              application.
        :type configuration: :class:`ramstk.Configuration.Configuration`
        """
        RAMSTKDataController.__init__(
            self,
            configuration,
            model=dtmFunction(dao, **kwargs),
            ramstk_module='function',
            **kwargs)

        # Initialize private dictionary attributes.

        # Initialize private list attributes.

        # Initialize private scalar attributes.
        self._dmx_fctn_hw_matrix = RAMSTKDataMatrix(dao, RAMSTKFunction,
                                                    RAMSTKHardware)
        self._dmx_fctn_sw_matrix = RAMSTKDataMatrix(dao, RAMSTKFunction,
                                                    RAMSTKSoftware)

        # Initialize public dictionary attributes.

        # Initialize public list attributes.

        # Initialize public scalar attributes.

        # Subscribe to PyPubSub messages.
        pub.subscribe(self.request_do_select_all, 'selected_revision')
        pub.subscribe(self.request_do_delete, 'request_delete_function')
        pub.subscribe(self.request_do_insert, 'request_insert_function')
        pub.subscribe(self.request_do_update, 'request_update_function')
        pub.subscribe(self.request_do_update_all,
                      'request_update_all_functions')
        pub.subscribe(self.request_set_attributes, 'editing_function')

    def request_do_create(self, revision_id, matrix_type):
        """
        Request to create or refresh a Function matrix.

        :param int revision_id: the ID of the Revision the desired Matrix is
                                associated with.
        :param str matrix_type: the type of the Matrix to select all rows and
        """
        if matrix_type == 'fnctn_hrdwr':
            self._dmx_fctn_hw_matrix.do_create(
                revision_id,
                matrix_type,
                rkey='function_id',
                ckey='hardware_id')

        return

    def request_do_select_all(self, revision_id):
        """
        Retrieve the treelib Tree() from the Function Data Model.

        :return: tree; the treelib Tree() of RAMSTKFunction models in the
                 Function tree.
        :rtype: :class:`treelib.Tree`
        """
        return self._dtm_data_model.do_select_all(revision_id=revision_id)

    def request_do_select_all_matrix(self, revision_id, matrix_type):
        """
        Retrieve all the Matrices associated with the Function.

        :param int revision_id: the Revision ID to select the matrices for.
        :param int matrix_type: the type of the Matrix to retrieve.  Current
                                Function matrix types are:

                                fnctn_hrdwr = Function:Hardware
                                fnctn_sftwr = Function:Software
                                fnctn_vldtn = Function:Validation

        :return: (_matrix, _column_hdrs, _row_hdrs); the Pandas Dataframe,
                 noun names to use for column headings, noun names to use for
                 row headings.
        :rtype: (:class:`pandas.DataFrame`, dict, dict)
        """
        _matrix = None
        _column_hdrs = []
        _row_hdrs = []

        if matrix_type == 'fnctn_hrdwr':
            self._dmx_fctn_hw_matrix.do_select_all(
                revision_id,
                matrix_type,
                rkey='function_id',
                ckey='hardware_id',
                rheader='function_code',
                cheader='comp_ref_des')
            _matrix = self._dmx_fctn_hw_matrix.dtf_matrix
            _column_hdrs = self._dmx_fctn_hw_matrix.dic_column_hdrs
            _row_hdrs = self._dmx_fctn_hw_matrix.dic_row_hdrs

        return (_matrix, _column_hdrs, _row_hdrs)

    def request_do_insert(self, revision_id, parent_id, **kwargs):  # pylint: disable=unused-argument
        """
        Request to add an RAMSTKFunction table record.

        :return: False if successful or True if an error is encountered.
        :rtype: bool
        """
        _error_code, _msg = self._dtm_data_model.do_insert(
            revision_id=revision_id, parent_id=parent_id)

        return RAMSTKDataController.do_handle_results(self, _error_code, _msg,
                                                      None)

    def request_do_insert_matrix(self, matrix_type, item_id, heading,
                                 row=True):
        """
        Request the to add a new row or column to the Data Matrix.

        :param str matrix_type: the type of the Matrix to retrieve.  Current
                                Function matrix types are:

                                fnctn_hrdwr = Function:Hardware
                                fnctn_sftwr = Function:Software
                                fnctn_vldtn = Function:Validation

        :param int item_id: the ID of the row or column item to insert into the
                            Matrix.
        :param str heading: the heading for the new row or column.
        :keyword bool row: indicates whether to insert a row (default) or a
                           column.
        :return: False if successful or True if an error is encountered.
        :rtype: bool
        """
        if matrix_type == 'fnctn_hrdwr':
            _error_code, _msg = self._dmx_fctn_hw_matrix.do_insert(
                item_id, heading, row=row)

        if _error_code == 0 and not self._test:
            pub.sendMessage(
                'insertedMatrix',
                matrix_type=matrix_type,
                item_id=item_id,
                row=row)

        return RAMSTKDataController.do_handle_results(self, _error_code, _msg,
                                                      None)

    def request_do_delete(self, node_id):
        """
        Request to delete an RAMSTKFunction table record.

        :param int node_id: the PyPubSub Tree() ID of the Function to delete.
        :return: False if successful or True if an error is encountered.
        :rtype: bool
        """
        _error_code, _msg = self._dtm_data_model.do_delete(node_id)

        return RAMSTKDataController.do_handle_results(self, _error_code, _msg,
                                                      None)

    def request_do_delete_matrix(self, matrix_type, item_id, row=True):
        """
        Request to remove a row or column from the selected Data Matrix.

        :param str matrix_type: the type of the Matrix to retrieve.  Current
                                Function matrix types are:

                                fnctn_hrdwr = Function:Hardware
                                fnctn_sftwr = Function:Software
                                fnctn_vldtn = Function:Validation

        :param int item_id: the ID of the row or column item to remove from the
                            Matrix.
        :keyword bool row: indicates whether to insert a row (default) or a
                           column.
        :return: False if successful or True if an error is encountered.
        :rtype: bool
        """
        if matrix_type == 'fnctn_hrdwr':
            _error_code, _msg = self._dmx_fctn_hw_matrix.do_delete(
                item_id, row=row)

        return RAMSTKDataController.do_handle_results(self, _error_code, _msg,
                                                      'deletedMatrix')

    def request_do_update(self, node_id):
        """
        Request to update an RAMSTKFunction table record.

        :param int node_id: the PyPubSub Tree() ID of the Function to save.
        :return: False if successful or True if an error is encountered.
        :rtype: bool
        """
        _error_code, _msg = self._dtm_data_model.do_update(node_id)

        return RAMSTKDataController.do_handle_results(self, _error_code, _msg,
                                                      None)

    def request_do_update_matrix(self, revision_id, matrix_type):
        """
        Request to update the selected Data Matrix.

        :param int revision_id: the ID of the Revision is the matrix to update
                                is associated with.
        :param int matrix_type: the type of the Matrix to save.
        :return: False if successful or True if an error is encountered.
        :rtype: bool
        """
        if matrix_type == 'fnctn_hrdwr':
            _error_code, _msg = self._dmx_fctn_hw_matrix.do_update(
                revision_id, matrix_type)
        else:
            _error_code = 6
            _msg = 'RAMSTK ERROR: Attempted to update non-existent matrix ' \
                   '{0:s}.'.format(matrix_type)

        return RAMSTKDataController.do_handle_results(self, _error_code, _msg,
                                                      'savedMatrix')

    def request_do_update_all(self, **kwargs):  # pylint: disable=unused-argument
        """
        Request to update all records in the RAMSTKFunction table.

        :return: (_error_code, _msg); the error code and associated message.
        :rtype: (int, str)
        """
        _error_code, _msg = self._dtm_data_model.do_update_all()

        return RAMSTKDataController.do_handle_results(self, _error_code, _msg,
                                                      None)

    def request_set_attributes(self, module_id, key, value):
        """
        Request to set a Function attribute.

        :param int module_id: the ID of the entity who's attribute is to
                              be set.
        :param str key: the key of the attributes to set.
        :param value: the value to set the attribute identified by the
                      key.
        :return:
        :rtype:
        """
        _entity = self.request_do_select(module_id)
        _attributes = _entity.get_attributes()

        _attributes[key] = value

        return _entity.set_attributes(_attributes)


from django.urls import path

from . import views

urlpatterns = [
    path('', views.index, name='gallery-index'),
    path('videos/', views.videos, name='videos'),
    path('backstage/', views.backstage, name='backstage'),
    path('stories/', views.stories, name='stories'),
    path('large_format/', views.large_format, name='large_format'),
    path('multimedia/', views.multimedia, name='multimedia'),
]


# ----------------------------------------------------------------------------------------------------------- #
#
# Program developped for simple S(E)IR model of corona virus infections in Denmark.
#
# Data from:
#
#   Developed by: Troels C. Petersen (NBI, petersen@nbi.dk), Mathias Heltberg, and Christian Michelsen
#   First version:  24. Mar. 2020 by Troels Petersen
#   Latest version: 15. Sep. 2020
#
# ----------------------------------------------------------------------------------------------------------- #

from __future__ import division, print_function
from numba import jit
import importlib
import Config
from ExternalFunctions import nice_string_output, add_text_to_ax
from ExternalFunctions import UnbinnedLH, Chi2Regression

import sys
import numpy as np
import matplotlib.pyplot as plt
# import seaborn as sns
from iminuit import Minuit
from scipy import stats
import pandas as pd
from scipy import interpolate

sys.path.append('External_Functions')

# ----------------------------------------------------------------------------------------------------------- #
plt.close('all')

Config = importlib.reload(Config)
case = Config.load_case

# 61000 tested with 2 days between
# 21000 Tested positive
# 10000 from week 19 and forward

# Antal tested 22576, med 10053 efter uge 19 (hvor tracing startede) hvoraf 650 positive (6.5%).

# Ialt er der 10000 testede positive efter uge 19.

# Det lader til, at dem, som tester negativ paa dag 4, men positiv paa dag 6 (69 ialt) er "meget normalt",
# idet de udgoer 0.3% af alle testede, hvilket ogsaa er hvad man finder generelt.

# Saa skal man stadig teste paa dag 6?

# Desuden:
# Lokale udbrud bliver maaske "forstoerret" i data, fordi der bliver testet mere (ved mobilt telt).

# ----------------------------------------------------------------------------------------------------------- #
# Load specific case from config file

# # Municipalities considered Copenhagen area (from a Covid-19 outbreak point of view):
# area_cph = ["Copenhagen", "Frederiksberg", "Gentofte", "Dragør", "Vallensbæk", "Ballerup", "Gladsaxe", "Herlev", "Hvidovre", "Lyngby-Taarbæk", "Rødovre", "Brøndby", "Glostrup", "Høje-Taastrup", "Tårnby", "Ishøj", "Albertslund"]

# # Municipalities considered Aarhus area (from a Covid-19 outbreak point of view):
# area_aarhus = ["Aarhus"]

# ----------------------------------------------------------------- #
# The data:
# ----------------------------------------------------------------- #

data_posi = pd.read_csv("Municipality_cases_time_series.csv",
                        sep=';', thousands='.', index_col=0)
data_test = pd.read_csv(
    "Municipality_tested_persons_time_series.csv", sep=';', thousands='.', index_col=0)

print(data_test.head(10))
print("...")
print(data_test.tail(10))

# Load Municipalities
Nposi = data_posi.loc["2020-03-01":"2020-09-24"][case['Municipalities']
                                                 ].values.sum(axis=1)
Ntest = data_test.loc["2020-03-01":"2020-09-24"][case['Municipalities']
                                                 ].values.sum(axis=1)
eNtest = np.sqrt(Ntest)
# # Two 1D arrays:
# Nposi_aarhus = data_posi.loc["2020-03-01":"2020-09-16"][area_aarhus].values.sum(axis=1)
# Ntest_aarhus = data_test.loc["2020-03-01":"2020-09-16"][area_aarhus].values.sum(axis=1)
# eNtest_aarhus = np.sqrt(Ntest_aarhus)

# Nposi_cph = data_posi.loc["2020-03-01":"2020-09-16"][area_cph].values.sum(axis=1)
# Ntest_cph = data_test.loc["2020-03-01":"2020-09-16"][area_cph].values.sum(axis=1)
# eNtest_cph = np.sqrt(Ntest_cph)

# # All of Denmark
# Nposi_all = data_posi.loc["2020-03-01":"2020-09-16"].sum(axis=1)
# Ntest_all = data_test.loc["2020-03-01":"2020-09-16"].sum(axis=1)
# eNtest_all = np.sqrt(Ntest_all)

# Make a list of days with the proper length:
print("Number of days for which there is data:", len(data_posi))

# Get the number of positive tests and uncertainty, assume a fixed number of daily tests (=3000):
dayLockdown_aarhus = case['dayLockdown']
# nAveDailyTests_aarhus = case['scale_test']
fPos = Nposi / Ntest
nPos_aarhus = case['scale_test'] * fPos
enPos_aarhus = case['scale_test'] * \
    np.sqrt(fPos*(1.0-fPos)/Ntest + case['SystError']**2)
day = np.arange(0, len(nPos_aarhus))
# dayLockdown_cph = 192   # Unknown!
# nAveDailyTests_cph = 12000.0
# fPos_cph = Nposi_cph / Ntest_cph
# nPos_cph  = nAveDailyTests_cph * fPos_cph
# enPos_cph = nAveDailyTests_cph * np.sqrt(fPos_cph*(1.0-fPos_cph)/Ntest_cph + SystError**2)

Plot_StartDay, Plot_EndDay = case['Plot_range']
Fit_StartDay, Fit_EndDay = case['Fit_range']
save_plots = case['save_plots']
if False:
    # Number of tests in Aarhus and Cph as a function of time:
    fig_Ntests, ax_Ntests = plt.subplots(figsize=(12, 7))
    ax_Ntests.set(xlabel="Day (1st of March is day 0)",
                  ylabel="Tests / day", title=f"Test per day for {case['Name']}")
    ax_Ntests.errorbar(day[Plot_StartDay:Plot_EndDay], Ntest[Plot_StartDay:Plot_EndDay],
                       yerr=eNtest[Plot_StartDay:Plot_EndDay], fmt='.', linewidth=2, label=f"N tests in {case['Name']}", color='red')
    # ax_Ntests.errorbar(day[Plot_StartDay:Plot_EndDay], Ntest_cph[Plot_StartDay:Plot_EndDay], yerr=eNtest_cph[Plot_StartDay:Plot_EndDay], fmt='.', linewidth=2, label='N tests in CphArea', color='blue')
    ax_Ntests.legend(loc=(0.05, 0.75), fontsize=16)
    d_N = {
        'Mean daily tests:': f"{int(np.mean(Ntest[Fit_StartDay:Fit_EndDay]))}"}
    add_text_to_ax(0.05, 0.65, nice_string_output(
        d_N, decimals=0), ax=ax_Ntests, fontsize=12)
    plt.tight_layout()
    plt.show(block=False)
    if (save_plots):
        plt.savefig(f"figures/{case['Name']}_NDailyTests.pdf")

# ----------------------------------------------------------------- #
# SIR model with time variation in beta:
# ----------------------------------------------------------------- #

# SIR Model, including modelling varying beta:
def func_SIRmodel(x, dayLockdown, nI0, beta0, beta1, gamma):

    # These numbers are the current status for each type of case, NOT cumulative.
    NdaysModelExtendsBeyondData = 5
    NdaysModel = len(nPos_aarhus) + NdaysModelExtendsBeyondData
    mS = np.zeros(NdaysModel)   # Susceptible
    mI = np.zeros(NdaysModel)   # Infected
    mR = np.zeros(NdaysModel)   # Recovered
    mTot = np.zeros(NdaysModel)   # Overall number (for check)

    # Initial numbers:
    dayN = Fit_StartDay
    mS[dayN] = 5800000-nI0
    mI[dayN] = nI0
    mR[dayN] = 0
    mTot[dayN] = mS[dayN]+mI[dayN]     # There are no Recovered, yet!

    # Model loop:
    # -----------------
    while (dayN < len(nPos_aarhus)-1 and mI[dayN] > 0):
        dayN += 1
        if (dayN < dayLockdown):     # Could potentially be a fitting parameter!
            beta = beta0
#        elif (dayN < 25) :
#            beta = beta1
        else:
            beta = beta1
            # beta = max(beta0 - dbeta*(dayN-dayLockdown), beta_inf)

        dI = beta*mI[dayN-1] * (mS[dayN-1] / mTot[dayN-1])
        dR = gamma*mI[dayN-1]
        mS[dayN] = mS[dayN-1] - dI
        mI[dayN] = mI[dayN-1] + dI - dR
        mR[dayN] = mR[dayN-1] + dR
        mTot[dayN] = mS[dayN] + mI[dayN] + \
            mR[dayN]       # Should remain constant!

    return mI[x]

func_SIRmodel_vec = np.vectorize(func_SIRmodel)

# ----------------------------------------------------------------- #
# SEIR model with time variation in beta:
# ----------------------------------------------------------------- #

# SEIR model, including modelling of time delays and varying beta:
@jit
def func_SEIRmodel(x, dayLockdown, nI0, beta0, beta1, lambdaE, lambdaI):

    # Initial numbers:
    N_tot = 5800000
    S = N_tot - nI0

    # The initial number of exposed and infected are scaled to match beta0. Factors in front are ad hoc!
    Norm = np.exp(0.8*lambdaE * beta0) + np.exp(0.7*lambdaE * beta0) + np.exp(0.6*lambdaE * beta0) + np.exp(0.5*lambdaE * beta0) +\
        np.exp(0.4*lambdaI * beta0) + np.exp(0.3*lambdaI * beta0) + \
        np.exp(0.2*lambdaI * beta0) + np.exp(0.1*lambdaI * beta0)
    E1 = nI0 * np.exp(0.8*lambdaE * beta0) / Norm
    E2 = nI0 * np.exp(0.7*lambdaE * beta0) / Norm
    E3 = nI0 * np.exp(0.6*lambdaE * beta0) / Norm
    E4 = nI0 * np.exp(0.5*lambdaE * beta0) / Norm
    I1 = nI0 * np.exp(0.4*lambdaI * beta0) / Norm
    I2 = nI0 * np.exp(0.3*lambdaI * beta0) / Norm
    I3 = nI0 * np.exp(0.2*lambdaI * beta0) / Norm
    I4 = nI0 * np.exp(0.1*lambdaI * beta0) / Norm
    """
    E1 = nI0/8
    E2 = nI0/8
    E3 = nI0/8
    E4 = nI0/8
    I1 = nI0/8
    I2 = nI0/8
    I3 = nI0/8
    I4 = nI0/8
    """
    R = 0
    # We define the first day given in the array of days (x) as day 0:
    dayN = Fit_StartDay
    NdaysModelExtendsBeyondData = 5
    NdaysModel = len(nPos_aarhus) + NdaysModelExtendsBeyondData

    # We store the results (time, S, E, I, and R) for each day here:
    SEIR_result = np.zeros((NdaysModel, 5))
    SEIR_result[dayN] = [dayN, S, E1+E2+E3+E4, I1+I2+I3+I4, R]

    # Numerical settings:
    nStepsPerDay = 24           # We simulated in time steps of 1 hour
    dt = 1.0 / nStepsPerDay     # Time step length in days

    # Model loop:
    # -----------------
    while (dayN < NdaysModel-1 and I1 >= 0):

        dayN += 1
        if (dayN < dayLockdown):
            beta = beta0
        else:
            beta = beta1
            # beta = max(beta0 - dbeta*(dayN-dayLockdown), beta_inf)   # Minimum value of beta (at time infinity)

        # Now divide the daily procedure into time steps:
        for _ in range(nStepsPerDay):
            dS = -beta*(I1+I2+I3+I4) * (S / N_tot)
            dE1 = -dS - lambdaE * E1
            dE2 = lambdaE * E1 - lambdaE * E2
            dE3 = lambdaE * E2 - lambdaE * E3
            dE4 = lambdaE * E3 - lambdaE * E4
            dI1 = lambdaE * E4 - lambdaI * I1
            dI2 = lambdaI * I1 - lambdaI * I2
            dI3 = lambdaI * I2 - lambdaI * I3
            dI4 = lambdaI * I3 - lambdaI * I4
            dR = lambdaI * I4

            S += dt * dS
            E1 += dt * dE1
            E2 += dt * dE2
            E3 += dt * dE3
            E4 += dt * dE4
            I1 += dt * dI1
            I2 += dt * dI2
            I3 += dt * dI3
            I4 += dt * dI4
            R += dt * dR
            Tot = S + E1+E2+E3+E4 + I1+I2+I3+I4 + R

        # Record status of model every day:
        SEIR_result[dayN] = [dayN, S, E1+E2+E3+E4, I1+I2+I3+I4, R]

        # print(dayN, E1+E2+E3+E4, I1+I2+I3+I4, R)

    # Return only the number of infected and only for the relevant days:
    return SEIR_result[x, 3]

func_SEIRmodel_vec = np.vectorize(func_SEIRmodel)

# ----------------------------------------------------------------- #
# Plot/print SIR and/or SEIR model with given parameters (for testing):
# ----------------------------------------------------------------- #

# Plot data (scaled to a fixed number of daily tests):
doSIRModelPlot = case['doSIRModelPlot']
doSEIRModelPlot = case['doSEIRModelPlot']
if (doSIRModelPlot or doSEIRModelPlot):
    fig, ax = plt.subplots(figsize=(12, 7))
    ax.set(xlabel="Day (1st of March is day 0)",
           ylabel="Newly infected / day", title="")
    ax.errorbar(day[Plot_StartDay:Plot_EndDay], nPos_aarhus[Plot_StartDay:Plot_EndDay],
                yerr=enPos_aarhus[Plot_StartDay:Plot_EndDay], fmt='.', linewidth=2, label='Data (scaled)', color='red')

# Plot SIR model on top of data (with fixed parameters):
if (doSIRModelPlot):

    # Model (fixed parameters):
    nI0_plot = 2.0
    beta0_plot = 0.34
    beta1_plot = 0.06
    gamma_plot = 1.0/7.0
    ax.plot(day[Fit_StartDay:Fit_EndDay], func_SIRmodel_vec(day[Fit_StartDay:Fit_EndDay], dayLockdown_aarhus,
                                                            nI0_plot, beta0_plot, beta1_plot, gamma_plot), 'blue', linewidth=1.0, label='SIR Model')

    # Calculate Chi2, Ndof, and Chi2-Probability for this model:
    nPos_est = func_SIRmodel(day[Fit_StartDay:Fit_EndDay],
                             dayLockdown_aarhus, nI0_plot, beta0_plot, beta1_plot, gamma_plot)
    chi2 = np.sum(((nPos_aarhus[Fit_StartDay:Fit_EndDay] -
                    nPos_est) / enPos_aarhus[Fit_StartDay:Fit_EndDay])**2)
    Ndof = len(day[Fit_StartDay:Fit_EndDay]) - 5
    Prob = stats.chi2.sf(chi2, Ndof)
    print(
        f"  SIR Model (fixed for plot):  Prob(Chi2={chi2:6.1f}, Ndof={Ndof:3d}) = {Prob:7.5f}")

# Plot SEIR model on top of data (with fixed parameters):
if (doSEIRModelPlot):

    # Model (fixed parameters):
    nI0_plot = case['nI0_plot']
    beta0_plot = case['beta0_plot']
    beta1_plot = case['beta1_plot']
    lambdaE_plot = case['lambdaE_plot']
    lambdaI_plot = case['lambdaE_plot']
    ax.plot(day[Fit_StartDay:Fit_EndDay], func_SEIRmodel_vec(day[Fit_StartDay:Fit_EndDay], dayLockdown_aarhus-3,
                                                             nI0_plot, beta0_plot, beta1_plot, lambdaE_plot, lambdaI_plot), 'green', linewidth=1.0, label='SEIR Model')

    # Calculate Chi2, Ndof, and Chi2-Probability for this model:
    nPos_est = func_SEIRmodel_vec(day[Fit_StartDay:Fit_EndDay], dayLockdown_aarhus,
                                  nI0_plot, beta0_plot, beta1_plot, lambdaE_plot, lambdaI_plot)
    chi2 = np.sum(((nPos_aarhus[Fit_StartDay:Fit_EndDay] -
                    nPos_est) / enPos_aarhus[Fit_StartDay:Fit_EndDay])**2)
    # There are 5 parameters in the fit!
    Ndof = len(day[Fit_StartDay:Fit_EndDay]) - 5
    Prob = stats.chi2.sf(chi2, Ndof)
    print(
        f"  SEIR Model (fixed for plot):  Prob(Chi2={chi2:6.1f}, Ndof={Ndof:3d}) = {Prob:7.5f}")

if (doSIRModelPlot or doSEIRModelPlot):
    ax.legend(loc=(0.05, 0.75), fontsize=16)
    plt.tight_layout()
    plt.show(block=False)
    # if (save_plots) :
    #     plt.savefig("figures/fig_CoronaDataDK_ModelPlot_SEIR2_Sep23.pdf")

# ----------------------------------------------------------------- #
# Fit to data with SIR model:
# ----------------------------------------------------------------- #

# Plot Data:
# ----------
if (case['doFitSIR'] or case['doFitSEIR']):
    print("\n  ------------------------------------  Fitting Stage  --------------------------------------- \n")

    fig_fit, ax_fit = plt.subplots(figsize=(12, 7))
    ax_fit.errorbar(day[Plot_StartDay:Plot_EndDay], nPos_aarhus[Plot_StartDay:Plot_EndDay], yerr=enPos_aarhus[Plot_StartDay:Plot_EndDay],
                    fmt='.', linewidth=2, color="red", label='Newly infected (scaled to 3000 tests)')
    ax_fit.set(xlabel="Day (1st of March is day 0)",
               ylabel="Newly infected / day (scaled)", title="")

# SIR model fit:
# --------------
if (case['doFitSIRinit']):

    endday_OK = Fit_StartDay
    for i_endday in range(Fit_StartDay, Fit_EndDay):
        # Defining Chi2 calculation:
        def ChiSquareCalcSIRinit(dayL, nI0, beta0, beta1, gamma):
            nPos_est = func_SIRmodel(
                day[Fit_StartDay:i_endday], dayL, nI0, beta0, beta1, gamma)
            chi2 = np.sum(
                ((nPos_aarhus[Fit_StartDay:i_endday] - nPos_est) / enPos_aarhus[Fit_StartDay:i_endday])**2)
            return chi2

        minuit_SIRinit = Minuit(ChiSquareCalcSIRinit, pedantic=False, print_level=0, dayL=1000,
                                fix_dayL=True, nI0=0.5, beta0=0.45, beta1=0.06, gamma=0.14, fix_gamma=True)
        minuit_SIRinit.migrad()
        # if (not minuit_SIRinit.fmin.is_valid) :
        # print("  WARNING: The ChiSquare fit DID NOT converge!!! ")

        chi2 = minuit_SIRinit.fmin.fval
        Ndof = len(day[Fit_StartDay:Fit_EndDay]) - len(minuit_SIRinit.args)
        Prob = stats.chi2.sf(chi2, Ndof)
        print(
            f"  SIRinit Model fit:  Day={i_endday:3d}   Prob(Chi2={chi2:6.1f}, Ndof={Ndof:3d}) = {Prob:7.5f}")

        # If model is acceptable (prob > 5%), then save the parameters:
        if (Prob > 0.05):
            dayL_fit, nI0_fit, beta0_fit, beta1_fit, gamma_fit = minuit_SIRinit.args
            endday_OK = i_endday
        else:
            continue

    # Plot the last acceptable fitted model on top of data extending in time:
    ax_fit.plot(day[Fit_StartDay:endday_OK], func_SIRmodel_vec(day[Fit_StartDay:endday_OK], dayL_fit, nI0_fit,
                                                               beta0_fit, beta1_fit, gamma_fit), 'blue', linewidth=1.0, label='Fit with SIR Model - no intervention')
    ax_fit.plot(day[endday_OK-1:endday_OK+2], func_SIRmodel_vec(day[endday_OK-1:endday_OK+2],
                                                                dayL_fit, nI0_fit, beta0_fit, beta1_fit, gamma_fit), 'blue', linewidth=1.0, linestyle="dotted")

Fit_EndDay_delay = case['Fit_EndDay_delay']

# SIR model fit:
# --------------
if (case['doFitSIR']):

    Fit_EndDay = endday_OK + Fit_EndDay_delay

    # Defining Chi2 calculation:
    def ChiSquareCalcSIR(dayL, nI0, beta0, beta1, gamma):
        nPos_est = func_SIRmodel(
            day[Fit_StartDay:Fit_EndDay], dayL, nI0, beta0, beta1, gamma)
        chi2 = np.sum(((nPos_aarhus[Fit_StartDay:Fit_EndDay] -
                        nPos_est) / enPos_aarhus[Fit_StartDay:Fit_EndDay])**2)
        return chi2

    minuit_SIR = Minuit(ChiSquareCalcSIR, pedantic=False, print_level=0,
                        dayL=dayLockdown_aarhus, limit_dayL=[150, 208], nI0=2.0, beta0=0.45, beta1=0.04, gamma=0.14, fix_gamma=True)
    minuit_SIR.migrad()
    if (not minuit_SIR.fmin.is_valid):
        print("  WARNING: The ChiSquare fit DID NOT converge!!! ")

    # Same as minuit_SIR.args
    dayL_fit, nI0_fit, beta0_fit, beta1_fit, gamma_fit = minuit_SIR.values.values()
    edayL_fit, enI0_fit, ebeta0_fit, ebeta1_fit, egamma_fit = minuit_SIR.errors.values()
    if dayL_fit > max(day):
        dayL_fit = 208
    for name in minuit_SIR.parameters:
        print("Fit value (exp): {0} = {1:.5f} +/- {2:.5f}".format(name,
                                                                  minuit_SIR.values[name], minuit_SIR.errors[name]))
    chi2 = minuit_SIR.fmin.fval
    Ndof = len(day[Fit_StartDay:Fit_EndDay]) - len(minuit_SIR.args)
    Prob = stats.chi2.sf(chi2, Ndof)
    print(
        f"  SIR Model fit:  Prob(Chi2={chi2:6.1f}, Ndof={Ndof:3d}) = {Prob:7.5f}")

    # Plot fitted model on top of data extending in time:
    ax_fit.plot(day[Fit_StartDay:Fit_EndDay], func_SIRmodel_vec(day[Fit_StartDay:Fit_EndDay], dayL_fit,
                                                                nI0_fit, beta0_fit, beta1_fit, gamma_fit), 'blue', linewidth=2.0, label='Fit with SIR Model')
    ax_fit.plot(day[Fit_EndDay-1:Fit_EndDay+15], func_SIRmodel_vec(day[Fit_EndDay-1:Fit_EndDay+15],
                                                                   dayL_fit, nI0_fit, beta0_fit, beta1_fit, gamma_fit), 'blue', linewidth=1.0, linestyle="dotted")
    ax_fit.legend(loc=(0.05, 0.80), fontsize=12)
    d_fit = {'SIR-model': "",
             'Prob Chi2': f"{Prob:.5f}",
             r'$\beta_0$     ': f"{minuit_SIR.values['beta0']:.4f}+-{minuit_SIR.errors['beta0']:.4f}",
             r'$\beta_1$    ': f"{minuit_SIR.values['beta1']:.4f}+-{minuit_SIR.errors['beta1']:.4f}",
             "Lockdown": f"{data_posi.iloc[int(dayL_fit)].name}, ({int(dayL_fit)})"}
    add_text_to_ax(0.05, 0.55, nice_string_output(
        d_fit, decimals=0), ax=ax_fit, fontsize=9)

animate = False
# SEIR model fit:
# ---------------
if (case['doFitSEIR']):
    Fit_EndDay = endday_OK + Fit_EndDay_delay

    # Defining Chi2 calculation:
    def ChiSquareCalcSEIR(dayL, nI0, beta0, beta1, lambdaE, lambdaI):
        nPos_est = func_SEIRmodel(
            day[Fit_StartDay:Fit_EndDay], dayL, nI0, beta0, beta1, lambdaE, lambdaI)
        chi2 = np.sum(((nPos_aarhus[Fit_StartDay:Fit_EndDay] -
                        nPos_est) / enPos_aarhus[Fit_StartDay:Fit_EndDay])**2)
        return chi2

    minuit_SEIR = Minuit(ChiSquareCalcSEIR, pedantic=False, print_level=0, dayL=dayLockdown_aarhus-4, fix_dayL=False,
                         limit_dayL=[150, 208], nI0=5.0, beta0=0.40, beta1=0.15, lambdaE=1.0, lambdaI=1.0, fix_lambdaE=True, fix_lambdaI=True)
    minuit_SEIR.migrad()
    if (not minuit_SEIR.fmin.is_valid):
        print("  WARNING: The ChiSquare fit DID NOT converge!!! ")

    dayL_fit, nI0_fit, beta0_fit, beta1_fit, lambdaE_fit, lambdaI_fit = minuit_SEIR.args
    dayL_fiterr, nI0_fiterr, beta0_fiterr, beta1_fiterr = minuit_SEIR.errors[
        "dayL"], minuit_SEIR.errors["nI0"], minuit_SEIR.errors["beta0"], minuit_SEIR.errors["beta1"]
    if dayL_fit > 208:
        dayL_fit = 208
    for name in minuit_SEIR.parameters:
        print("Fit value (exp): {0} = {1:.5f} +/- {2:.5f}".format(name,
                                                                  minuit_SEIR.values[name], minuit_SEIR.errors[name]))
    chi2 = minuit_SEIR.fmin.fval
    Ndof = len(day[Fit_StartDay:Fit_EndDay]) - len(minuit_SEIR.args)
    Prob = stats.chi2.sf(chi2, Ndof)
    print(
        f"  SEIR Model fit:  Prob(Chi2={chi2:6.1f}, Ndof={Ndof:3d}) = {Prob:7.5f}")

    # Plot fitted model on top of data extending in time:
    ax_fit.plot(day[Fit_StartDay:Fit_EndDay], func_SEIRmodel_vec(day[Fit_StartDay:Fit_EndDay], dayL_fit, nI0_fit,
                                                                 beta0_fit, beta1_fit, lambdaE_fit, lambdaI_fit), 'green', linewidth=2.0, label='Fit with SEIR Model')
    ax_fit.plot(day[Fit_StartDay:Fit_EndDay+15], func_SEIRmodel_vec(day[Fit_StartDay:Fit_EndDay+15], dayL_fit,
                                                                    nI0_fit, beta0_fit, beta1_fit, lambdaE_fit, lambdaI_fit), 'green', linewidth=2.0, linestyle="dotted")
    ax_fit.legend(loc=(0.03, 0.75), fontsize=12)

    # Plot an "envelope" of possible predictions:
    ax_fit.plot(day[Fit_StartDay:Fit_EndDay+10], func_SEIRmodel_vec(day[Fit_StartDay:Fit_EndDay+10], dayL_fit, nI0_fit,
                                                                    beta0_fit, beta1_fit+1*ebeta1_fit, lambdaE_fit, lambdaI_fit), 'green', linewidth=1.0, linestyle="dotted")
    ax_fit.plot(day[Fit_StartDay:Fit_EndDay+10], func_SEIRmodel_vec(day[Fit_StartDay:Fit_EndDay+10], dayL_fit, nI0_fit,
                                                                    beta0_fit, beta1_fit-1*ebeta1_fit, lambdaE_fit, lambdaI_fit), 'green', linewidth=1.0, linestyle="dotted")
    ax_fit.fill_between(day[Fit_StartDay:Fit_EndDay+10], func_SEIRmodel_vec(day[Fit_StartDay:Fit_EndDay+10], dayL_fit, nI0_fit, beta0_fit, beta1_fit+1*ebeta1_fit, lambdaE_fit,
                                                                            lambdaI_fit), func_SEIRmodel_vec(day[Fit_StartDay:Fit_EndDay+10], dayL_fit, nI0_fit, beta0_fit, beta1_fit-1*ebeta1_fit, lambdaE_fit, lambdaI_fit), color='green', alpha=0.3)

    ax_fit.set_title(f"Fit of outbreak in {case['Name']}")
    d_fit = {'SEIR-model': "",
             'Prob Chi2': f"{Prob:.5f}",
             r'$\beta_0$      ': f"{minuit_SEIR.values['beta0']:.4f}+-{minuit_SEIR.errors['beta0']:.4f}",
             r'$\beta_1$      ': f"{minuit_SEIR.values['beta1']:.4f}+-{minuit_SEIR.errors['beta1']:.4f}",
             "Lockdown": f"{data_posi.iloc[int(dayL_fit)].name}, ({int(dayL_fit)})"}
    add_text_to_ax(0.05, 0.72, nice_string_output(
        d_fit, decimals=0), ax=ax_fit, fontsize=9)

    if (case['doFitSIR'] or case['doFitSEIR']):
        plt.tight_layout()
        plt.show(block=False)
        if (save_plots):
            plt.savefig(f'figures/{case["Name"]}_modelFit.pdf')

    fig_res, ax_res = plt.subplots(2, figsize=(9, 9))
    ax_res[0].errorbar(day[Fit_StartDay:Fit_EndDay+10], (nPos_aarhus[Fit_StartDay:Fit_EndDay+10]-func_SEIRmodel_vec(day[Fit_StartDay:Fit_EndDay+10], dayL_fit, nI0_fit,
                                                                                beta0_fit, beta1_fit+1*ebeta1_fit, lambdaE_fit, lambdaI_fit)), enPos_aarhus[Fit_StartDay:Fit_EndDay+10],
                       fmt='.', linewidth=2, color="red", label='Residuals)')
    ax_res[0].plot(day[Fit_StartDay:Fit_EndDay+10],
                   np.zeros(len(day[Fit_StartDay:Fit_EndDay+10])), 'k--')
    ax_res[0].set(xlabel='Day after day 0', ylabel="fit residual",
                  title=f"Residuals for {case['Name']}")
    ax_res[0].legend()
    simulation = True
    if simulation:
        days_pos = day[int(np.rint(dayL_fit)):]
        val = minuit_SEIR.args
        from tqdm import tqdm

        def simul(days, vals, cov, N=int(1e5)):
            ys = []
            mu = []
            sig_up = []
            sig_down = []
            sig2_up, sig2_down = [], []
            means = [vals[0], vals[1], vals[2], vals[3]]
            vals = np.random.multivariate_normal(means, cov, size=N).T
            day_L = vals[0]
            n_I0 = vals[1]
            beta_0 = vals[2]
            beta_1 = vals[3]
            print('Simulating MC')
            for i in tqdm(range(N)):
                ys.append(func_SEIRmodel(
                    days, day_L[i], n_I0[i], beta_0[i], beta_1[i], lambdaE_fit, lambdaI_fit))
            ys = np.array(ys)
            ys = ys.T
            for day in ys:
                mu.append(np.mean(day))
                per = np.quantile(day, [0.5+0.34, 0.5-0.34])
                per2 = np.quantile(day, [0.5+0.49, 0.5-0.49])
                sig_up.append(per[0])
                sig_down.append(per[1])
                sig2_up.append(per2[0])
                sig2_down.append(per2[1])
            return np.array(mu), np.array(sig_up), np.array(sig_down), np.array(sig2_up), np.array(sig2_down)
        # playtime tomorrow
        multi = False
        if not multi:
            mu, sigup, sigdown, sig2up, sig2down = simul(days=days_pos, vals=minuit_SEIR.args,
                                                         cov=minuit_SEIR.matrix())
        else:
            import multiprocessing as mp
            import os
            mus, sigs = [], []
            ps = []

            def p(x, txt='Hello'):
                print(txt+x)
                return(x)
            for _ in range(os.cpu_count()):
                # p = mp.Process(target=simul, args=[days_pos)
                p = mp.Process(target=p, args=['Ciara'])
                p.start()
                ps.append(p)
            for process in ps:
                process.join()
        ax_res[1].errorbar(day[Fit_StartDay:Fit_EndDay], nPos_aarhus[Fit_StartDay:Fit_EndDay], yerr=enPos_aarhus[Fit_StartDay:Fit_EndDay],
                           fmt='.', linewidth=2, color="red", alpha=0.8, label='Newly infected (scaled to 3000 tests)')
        ax_res[1].errorbar(day[Fit_EndDay:], nPos_aarhus[Fit_EndDay:], yerr=enPos_aarhus[Fit_EndDay:],
                           fmt='.', linewidth=2, color="purple", alpha=0.8, label='Points for evaluating prediction')
        diff = Fit_EndDay-int(np.rint(dayL_fit))
        ax_res[1].plot(days_pos[:diff+1], mu[:diff+1],
                       'g-', label='Central values')
        ax_res[1].plot(days_pos[diff:], mu[diff:],
                       color='green', linestyle='dotted')
        ax_res[1].plot(days_pos, sigup, 'g--',  label='One sigma deviation')
        ax_res[1].plot(days_pos, sigdown, "g--")
        ax_res[1].fill_between(days_pos, sigup, sigdown,
                               color='green', alpha=0.4)
        ax_res[1].plot(days_pos, sig2up, 'g--',  label='Two sigma deviation')
        ax_res[1].plot(days_pos, sig2down, "g--")
        ax_res[1].fill_between(days_pos, sig2up, sig2down,
                               color='darkgreen', alpha=0.2)
        ax_res[1].set(xlabel='Day after March 1st',
                      ylabel="Scaled cases", title="MC errors")
        ax_res[1].vlines(dayL_fit, 0, max(nPos_aarhus[150:]),
                         linestyles='dashed', alpha=0.6, label='Fitted lockdown')
        ax_res[1].vlines(Fit_EndDay-0.5, 0, max(nPos_aarhus[150:]),
                         linestyles='solid', label='Extrapolating from')
        ax_res[1].legend(loc='best', fontsize=9)
        delay = diff
        case_pos = nPos_aarhus[int(np.rint(dayL_fit))+delay:]
        pos_err = np.sqrt(enPos_aarhus[int(np.rint(
            dayL_fit))+delay:]**2-case['SystError']**2)
        mu, sigup, sigdown = mu[delay:], sigup[delay:], sigdown[delay:]
        chi2list = 0
        diflist = case_pos-mu
        k = 0
        for dif in diflist:
            if dif > 0:
                chi2list += (mu[k]-case_pos[k])**2 / \
                    ((pos_err[k]**2+sigup[k]**2))
            if dif < 0:
                chi2list += (mu[k]-case_pos[k])**2 / \
                    ((pos_err[k]**2+sigdown[k]**2))
            k += 1
        chi2 = chi2list
        Ndof = len(case_pos)-len(minuit_SEIR.np_errors())
        Prob = stats.chi2.sf(chi2, Ndof)
        d_res = {'Chi2': f"{chi2:3.1f}",
                 'Ndof': f"{Ndof}",
                 'Prob': f"{Prob:1.6f}"}
        add_text_to_ax(0.05, 0.95, nice_string_output(
            d_res, decimals=0), ax=ax_res[1], fontsize=9)
    fig_res.tight_layout()
    fig_res.show()
# Plot if any fits are performed:
# -------------------------------

# ----------------------------------------------------------------- #
# Fit to data with SIR and SIER model:
# ----------------------------------------------------------------- #

try:
    __IPYTHON__
except:
    input('Press Enter to Exit!')


import cv2
import numpy
import math
from PIL import Image
#from matplotlib import pyplot

with open("final_image.txt", "r") as text_file:

	iksevi = text_file.readlines() #iksevi = redovi
	kolone = iksevi[1].split(' ')
	print('iksevi : ',len(iksevi))
	print('kolone : ',len(kolone))
	imgArray = numpy.empty([len(iksevi),len(kolone)], dtype=numpy.int16)
	xerino = 0

	for x in iksevi:
		brojevi = x.split(' ')
		y = 0
		for broj in brojevi:
			if (broj != '\n' and broj != ''):
				imgArray[xerino][y] = int(broj)
				y+=1
		xerino+=1

	img = Image.fromarray(imgArray)
	img.show()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# File : core.classifiers.RCNLPTextClassifier.py
# Description : Echo State Network for text classification.
# Auteur : Nils Schaetti <nils.schaetti@unine.ch>
# Date : 01.02.2017 17:59:05
# Lieu : Nyon, Suisse
#
# This file is part of the Reservoir Computing NLP Project.
# The Reservoir Computing Memory Project is a set of free software:
# you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Foobar is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with Foobar.  If not, see <http://www.gnu.org/licenses/>.
#

import argparse
from core.tools.RCNLPLogging import RCNLPLogging
import core.clustering.functions as cf
import numpy as np
import os
from scipy.stats import ttest_1samp
import pickle
from core.converters.RCNLPPosConverter import RCNLPPosConverter
from core.converters.RCNLPTagConverter import RCNLPTagConverter
from core.converters.RCNLPFuncWordConverter import RCNLPFuncWordConverter
from core.converters.RCNLPWordVectorConverter import RCNLPWordVectorConverter

#########################################################################
#
# Experience settings
#
#########################################################################

# Exp. info
ex_name = "Author clustering Experience"
ex_instance = "Author clustering One Reservoir"

# Reservoir Properties
rc_leak_rate = 0.5  # Leak rate
rc_input_scaling = 0.5  # Input scaling
rc_size = 500  # Reservoir size
rc_spectral_radius = 0.99  # Spectral radius
rc_w_sparsity = 0.1
rc_input_sparsity = 0.2

####################################################
# Main function
####################################################

if __name__ == "__main__":

    # Argument parser
    parser = argparse.ArgumentParser(description="RCNLP - Author clustering with Part-Of-Speech to Echo State Network")

    # Argument
    parser.add_argument("--texts", type=str, help="Text directory.")
    parser.add_argument("--startup", type=int, help="Number of start-up states to remove.", default=20)
    parser.add_argument("--out-components", type=int, help="Number of principal component to reduce reservoir states.", default=-1)
    parser.add_argument("--in-components", type=int, help="Number of principal component to reduce inputs.", default=-1)
    parser.add_argument("--homogene", action='store_true', help="Keep the same number of states for each authors.",
                        default=False)
    parser.add_argument("--pca-images", action='store_true', help="Generate image of principal components.",
                        default=False)
    parser.add_argument("--converter", type=str, help="The text converter to use (fw, pos, tag, wv).")
    parser.add_argument("--lang", type=str, help="Language model", default='en')
    parser.add_argument("--show-states", type=int, help="Number of states to show", default=500)
    parser.add_argument("--samples", type=int, help="Samples to estimate performances", default=20)
    parser.add_argument("--nreservoir", type=int, help="Number of reservoir to generate.", default=20)
    parser.add_argument("--pca-model", type=str, help="PCA model to load", default='')
    parser.add_argument("--output", type=str, help="Where to save the reservoir.", default="reservoir.p")
    args = parser.parse_args()

    # Logging
    logging = RCNLPLogging(exp_name=ex_name, exp_inst=ex_instance,
                           exp_value=RCNLPLogging.generate_experience_name(locals()))
    logging.save_globals()
    logging.save_variables(locals())

    # PCA model
    pca_model = None
    if args.pca_model != "":
        pca_model = pickle.load(open(args.pca_model, 'r'))
    # end if

    # Results
    state_results = np.array([])
    doc_results = np.array([])

    # >> 1. Convert the text to symbolic or continuous representations
    if args.converter == "pos":
        converter = RCNLPPosConverter(resize=args.in_components, pca_model=pca_model)
    elif args.converter == "tag":
        converter = RCNLPTagConverter(resize=args.in_components, pca_model=pca_model)
    elif args.converter == "fw":
        converter = RCNLPFuncWordConverter(resize=args.in_components, pca_model=pca_model)
    else:
        converter = RCNLPWordVectorConverter(resize=args.in_components, pca_model=pca_model)
        # end if

    # Whathever
    max_score = 0.0

    # Iterate over reservoirs
    for r in range(args.nreservoir):
        # Create a reservoir
        flow = cf.create_reservoir(converter.get_n_inputs(), rc_size, rc_input_scaling, rc_leak_rate, rc_spectral_radius,
                                   rc_input_sparsity, rc_w_sparsity)

        # Iterate
        for i in np.arange(0, args.samples):
            authors_id = np.random.choice(49, 2, replace=False) + 1
            texts1 = os.path.join(args.texts, str(authors_id[0]))
            texts2 = os.path.join(args.texts, str(authors_id[1]))
            print("Round %d with author %s and %s" % (i, texts1, texts2))
            state_score, doc_score = cf.clustering_states(args=args, texts1=texts1, texts2=texts2, ex_name=ex_name,
                                                          ex_instance=ex_instance, input_scaling=rc_input_scaling,
                                                          input_sparsity=rc_input_sparsity, leak_rate=rc_leak_rate,
                                                          logging=logging, size=rc_size, spectral_radius=rc_spectral_radius,
                                                          w_sparsity=rc_w_sparsity, save_graph=True if i == 0 else False,
                                                          pca_model=pca_model, flow=flow)

            # Log results
            #logging.save_results("V state score " + str(i), state_score, display=True)
            #logging.save_results("V doc score " + str(i), doc_score, display=True)

            # Save results
            state_results = np.append(state_results, state_score)
            doc_results = np.append(doc_results, doc_score)
        # end for

        # Log overall state results
        print("#############################################################################")
        logging.save_results("Overall state score", np.average(state_results), display=True)
        logging.save_results("Overall state score std", np.std(state_results), display=True)
        logging.save_results("State score T-test", ttest_1samp(state_results, 0).pvalue * 100.0, display=True)

        # Log overall doc results
        logging.save_results("Overall doc score", np.average(doc_results), display=True)
        logging.save_results("Overall doc score std", np.std(doc_results), display=True)
        logging.save_results("Doc score T-test", ttest_1samp(doc_results, 0).pvalue * 100.0, display=True)

        # Save reservoir
        if np.average(doc_results) > max_score:
            pickle.dump(flow, open(args.output, 'w'))
        # end if
    # end for

    # Open logging dir
    logging.open_dir()

# end if


import ConfigParser
import logging
import os

_VARIABLE_NAMES = 'envvars.ini'
_CONFIG_FILE = 'config.ini'

def get_configuration(section_name, config_file=_CONFIG_FILE,
                      variables=_VARIABLE_NAMES):
    var_config = ConfigParser.SafeConfigParser()
    var_config.read(variables)
    try:
        items = var_config.items(section=section_name)
    except ConfigParser.NoSectionError:
        logging.debug('Var config file %s not found', variables)
        return None

    config = ConfigParser.SafeConfigParser()
    config.read(config_file)
    properties = {}
    for name, variable in items:
        if variable in os.environ:
            properties[name] = os.getenv(variable)
        else:
            try:
                properties[name] = config.get(section_name, name.lower())
            except:
                logging.error('Unable to initialize %s for %s', name,
                              section_name)
                return None
    return properties


#!/usr/bin/python3
from flask import Flask, render_template
from models import storage
from os import getenv
from models.state import State
from models.city import City
app = Flask(__name__)

def show_state():
    my_list = storage.all("State").values()
    return render_template('7-states_list.html', my_list=my_list)

@app.route('/cities_by_states')
def cities_by_states_html():
    my_list = storage.all("State").values()
    return render_template('8-cities_by_states.html', my_list=my_list)

@app.teardown_appcontext
def teardown(self):
    storage.close()

if __name__ == "__main__":
    app.run(host="0.0.0.0", port="5000")


'''
Created on Dec 15, 2016

@author: njand
'''
import socket, json
#ip = "169.254.26.174"
ip = "127.0.0.1"
port = 63819
message = json.JSONEncoder().encode({"1":{"wp":[[0, 0, 1],[0,1,1]]}})

sock = socket.socket(socket.AF_INET,
                      socket.SOCK_DGRAM)
sock.sendto(message, (ip, port))


#!/bin/env python
import requests
import re
from months import months
from datetime import datetime
import json

class CheckLatestCODUpdate:
    def __init__(self):
        self.url = None
        self.loc = "./resources"
        self.current_patch_date = None
        self.previous_date = None
        self.first_date = None
        self.second_date = None
        self.url = None

    def query_google(self):
        print("Querying Google")
        self.output_message_to_logfile(f"{datetime.today()}: Querying Google")
        res = requests.get("https://www.google.com/search?&q=call+of+duty+patch+notes")
        #self.output_message_to_logfile(f"{datetime.today()}: Google Response:\n{res.content.decode('ISO-8859-1')}")
        return res.content.decode("ISO-8859-1")

    def search_for_cod_patch_notes(self, res):
        print("Searching for COD Patch Notes")
        self.output_message_to_logfile(f"{datetime.today()}: Searching for COD Patch Notes")
        pattern = '(https:\/\/www\.infinityward\.com\/news\/[0-9][0-9][0-9][0-9]-[0-9][0-9]\/.*_PATCH_NOTES_..._...)'
        x = re.search(pattern, res)
        self.output_message_to_logfile(f"{datetime.today()}: Found: {x.group(0)}")
        self.url = x.group(0);
        return x.group(0)

    def get_first_part_of_date(self, response):
        print(f"Check date of patch notes in rgex line: {response}")
        self.output_message_to_logfile(f"{datetime.today()}: Check date of patch notes in rgex line: {response}")
        first_part_of_date = re.search("[0-9][0-9][0-9][0-9]-[0-9][0-9]", response).group(0)
        print(first_part_of_date)
        self.output_message_to_logfile(f"{datetime.today()}: Found: {first_part_of_date}")
        return first_part_of_date

    def get_second_part_of_date(self, response):
        print(f"Check date of patch notes in regex line: {response}")
        self.output_message_to_logfile(f"{datetime.today()}: Getting second part of date")
        for month in months:
            if month in response.lower():
                print(f"Month found: {month}")
                self.output_message_to_logfile(f"{datetime.today()}: Month found {month}")
                pattern = f'({month.upper()}_[0-9].)'
                second_part_of_date = re.search(pattern, response).group(0)

                if second_part_of_date[-1].isdigit():
                    second_part_of_date = second_part_of_date
                else:
                    second_part_of_date = second_part_of_date[:-1]
                print(second_part_of_date)
                self.output_message_to_logfile(f"{datetime.today()}: Second part of date: {second_part_of_date}")
                return second_part_of_date
        print("Month not found")
        self.output_message_to_logfile(f"{datetime.today()}: Month not found")
        return ""

    def format_date(self, first_date, second_date):
        print("Formatting Dates")
        self.output_message_to_logfile(f"{datetime.today()}: Formatting Dates")
        second_date = second_date.split("_")[1]
        formatted_date = f"{first_date}-{second_date}"
        return formatted_date

    def save_date_of_most_recent_patch_notes(self, date):
        print(f"Saving most recent patch date to file: {date}")
        self.output_message_to_logfile(f"{datetime.today()}: Saving most recent patch date to file: {date}")
        f = open(f"{self.loc}resources/patch-date.txt", "w")
        f.write(date)
        f.close()

    def read_date_of_last_patch(self):
        self.output_message_to_logfile(f"{datetime.today()}: Reading last date from file..")
        f = open(f"{self.loc}resources/patch-date.txt", "r")
        self.previous_patch_date = f.read()
        self.output_message_to_logfile(f"{datetime.today()}: Last date from file: {self.previous_patch_date}")
        return self.previous_patch_date

    def new_patch_is_released(self, current_date):
        print(f"Current Date: {current_date}\nPrevious Date: {self.previous_patch_date}")
        self.output_message_to_logfile(
            f"{datetime.today()}: Current Date: {current_date}\nPrevious Date: {self.previous_patch_date}"
        )
        current_date_converted = datetime.strptime(current_date, "%Y-%m-%d")
        previous_update_date_converted = datetime.strptime(self.previous_patch_date, "%Y-%m-%d")
        self.output_message_to_logfile(f"{datetime.today()}: Current Date Converted: {current_date_converted}\n{datetime.today()}: Previous Date Converted: {previous_update_date_converted}")
        print(f"Current Date Converted: {current_date_converted}\nPrevious Date Converted: {previous_update_date_converted}")
        if current_date_converted > previous_update_date_converted:
            print("Update now available!")
            self.output_message_to_logfile(f"{datetime.today()}: Update Available!")
            self.save_date_of_most_recent_patch_notes(current_date)
            return True
        else:
            print("No update available")
            self.output_message_to_logfile(f"{datetime.today()}: No update available")
            return False

    def generate_patch_notes_url(self, first_date, second_date):
        print("Generating URL...")
        self.output_message_to_logfile(f"{datetime.today()}: Generating URL")
        global url
        url = self.url
        print(f"Generated: {url}")
        self.output_message_to_logfile(f"{datetime.today()}: Generated: {url}")
        return url

    def verify_if_correct_url_is_generated(self, url):
        res = requests.get(url)
        res = res.content.decode("utf8")
        if "Sorry, an unexpected error occurred" in res:
            print("Incorrect URL Generated")
            self.output_message_to_logfile(f"{datetime.today()}: Incorrect URL generated")
            return None
        else:
            print("URL is valid")
            self.output_message_to_logfile(f"{datetime.today()}: URL is valid")
            return True

    def check_if_new_patch_is_released(self, current_patch_date):
        if self.new_patch_is_released(current_patch_date):
            self.patch_url = self.generate_patch_notes_url(self.first_part_of_date, self.second_part_of_date)
            verify_url = self.verify_if_correct_url_is_generated(url)
            return verify_url

    def send_slack_message(self, message, hook):
        print("Sending Slack message...")
        self.output_message_to_logfile(f"{datetime.today()}: Sending Slack message...")
        slack_hook = f"https://hooks.slack.com/services/{hook}"
        payload = {"text": message}
        json_payload = json.dumps(payload)
        res = requests.post(slack_hook, data=json_payload)
        print(res.status_code)
        self.output_message_to_logfile(f"{datetime.today()}: {res.status_code}")

    def output_messages_to_logfile(self, messages):
        f = open(f"{self.loc}resources/log.txt", "a")
        for message in messages:
            f.write(f"{message}\n")
        f.close()

    def output_message_to_logfile(self, message):
        self.loc = "./"
        f = open(f"{self.loc}resources/log.txt", "a")
        f.write(f"{message}\n")
        f.close()

    @property
    def location(self):
        return self.loc

    @location.setter
    def location(self, loc):
        self.loc = loc

    @property
    def patch_url(self):
        return self.url

    @patch_url.setter
    def patch_url(self, url):
        self.url = url

    @property
    def previous_patch_date(self):
        return self.previous_date

    @previous_patch_date.setter
    def previous_patch_date(self, previous_date):
        self.previous_date = previous_date

    @property
    def first_part_of_date(self):
        return self.first_date

    @first_part_of_date.setter
    def first_part_of_date(self, first_date):
        self.first_date = first_date

    @property
    def second_part_of_date(self):
        return self.second_date

    @second_part_of_date.setter
    def second_part_of_date(self, second_date):
        self.second_date = second_date


from typing import Optional, Tuple
from uuid import uuid4

from anytree import search

from tree.node import Node

class Tree:
    def __init__(self):
        self.tree = Node(1, 'root')

    def reset(self):
        self.__init__()

    def delete_node(self, node_id: int):
        node = self.get_node_by_id(node_id)
        node.mark_as_deleted()
        for child in node.children:
            self.delete_node(child.id)

    def create_node(
        self,
        parent_id: Optional[int],
        value: str,
        node_id: int = None,
        children: Tuple[Node] = tuple(),
        is_deleted: bool = False,
    ):
        node_id = node_id if node_id else uuid4().int
        parent_node = self.get_node_by_id(parent_id)
        parent_node = parent_node if parent_node else self.tree
        new_node = Node(
            node_id,
            value=value,
            parent_id=parent_id,
            parent=parent_node,
            children=children,
            is_deleted=parent_node.is_deleted,
        )
        return new_node

    def get_node_by_id(self, node_id: int) -> Node:
        return search.find(self.tree, lambda node: node.id == node_id)


import numpy as np
from random import shuffle
from dash.sn_processing import PreProcessing
from dash.combine_sn_and_host import CombineSnAndHost
import multiprocessing as mp
import random
import time

class AgeBinning(object):
    def __init__(self, minAge, maxAge, ageBinSize):
        self.minAge = minAge
        self.maxAge = maxAge
        self.ageBinSize = ageBinSize

    def age_bin(self, age):
        ageBin = int(round(age / self.ageBinSize)) - int(round(self.minAge / self.ageBinSize))

        return ageBin

    def age_labels(self):
        ageLabels = []

        ageBinPrev = 0
        ageLabelMin = self.minAge
        for age in np.arange(self.minAge, self.maxAge, 0.5):
            ageBin = self.age_bin(age)

            if ageBin != ageBinPrev:
                ageLabelMax = int(round(age))
                ageLabels.append(str(int(ageLabelMin)) + " to " + str(ageLabelMax))
                ageLabelMin = ageLabelMax

            ageBinPrev = ageBin

        ageLabels.append(str(int(ageLabelMin)) + " to " + str(int(self.maxAge)))

        return ageLabels

class CreateLabels(object):

    def __init__(self, nTypes, minAge, maxAge, ageBinSize, typeList, hostList, nHostTypes):
        self.nTypes = nTypes
        self.minAge = minAge
        self.maxAge = maxAge
        self.ageBinSize = ageBinSize
        self.typeList = typeList
        self.ageBinning = AgeBinning(self.minAge, self.maxAge, self.ageBinSize)
        self.numOfAgeBins = self.ageBinning.age_bin(self.maxAge-0.1) + 1
        self.nLabels = self.nTypes * self.numOfAgeBins
        self.ageLabels = self.ageBinning.age_labels()
        self.hostList = hostList
        self.nHostTypes = nHostTypes

    def label_array(self, ttype, age, host=None):
        ageBin = self.ageBinning.age_bin(age)

        try:
            typeIndex = self.typeList.index(ttype)
        except ValueError as err:
            print("INVALID TYPE: {0}".format(err))
            raise ValueError

        if host is None:
            labelArray = np.zeros((self.nTypes, self.numOfAgeBins))
            labelArray[typeIndex][ageBin] = 1
            labelArray = labelArray.flatten()
            typeName = ttype + ": " + self.ageLabels[ageBin]
        else:
            hostIndex = self.hostList.index(host)
            labelArray = np.zeros((self.nHostTypes, self.nTypes, self.numOfAgeBins))
            labelArray[hostIndex][typeIndex][ageBin] = 1
            labelArray = labelArray.flatten()
            typeName = "{}: {}: {}".format(host, ttype, self.ageLabels[ageBin])

        labelIndex = np.argmax(labelArray)

        return labelIndex, typeName

    def type_names_list(self):
        typeNamesList = []
        if self.hostList is None:
            for tType in self.typeList:
                for ageLabel in self.ageBinning.age_labels():
                    typeNamesList.append("{}: {}".format(tType, ageLabel))
        else:
            for host in self.hostList:
                for tType in self.typeList:
                    for ageLabel in self.ageBinning.age_labels():
                        typeNamesList.append("{}: {}: {}".format(host, tType, ageLabel))

        return typeNamesList

class TempList(object):
    def temp_list(self, tempFileList):
        f = open(tempFileList, 'rU')

        fileList = f.readlines()
        for i in range(0,len(fileList)):
            fileList[i] = fileList[i].strip('\n')

        f.close()

        return fileList

class ReadSpectra(object):

    def __init__(self, w0, w1, nw, snFilename, galFilename=None):
        self.w0 = w0
        self.w1 = w1
        self.nw = nw
        self.snFilename = snFilename
        if galFilename is None:
            self.data = PreProcessing(snFilename, w0, w1, nw)
        else:
            self.galFilename = galFilename
            self.dataCombined = CombineSnAndHost(snFilename, galFilename, w0, w1, nw)

    def sn_plus_gal_template(self, snAgeIdx, snCoeff, galCoeff, z):
        wave, flux, minIndex, maxIndex, nCols, ages, tType = self.dataCombined.training_template_data(snAgeIdx, snCoeff, galCoeff, z)

        return wave, flux, nCols, ages, tType, minIndex, maxIndex

    def snid_template_data(self, ageIdx, z):
        """ lnw template files """
        wave, flux, nCols, ages, tType, minIndex, maxIndex = self.data.snid_template_data(ageIdx, z)

        return wave, flux, nCols, ages, tType, minIndex, maxIndex

    def sf_age(self):
        snName, extension = self.snFilename.strip('.dat').split('.')
        ttype, snName = snName.split('/')

        if extension == 'max':
            age = 0
        elif extension[0] == 'p':
            age = float(extension[1:])
        elif extension[0] == 'm':
            age = -float(extension[1:])
        else:
            print("Invalid Superfit Filename: " + self.snFilename)

        return snName, ttype, age

    def superfit_template_data(self, z):
        """ Returns wavelength and flux after all preprocessing """
        wave, flux, minIndex, maxIndex = self.data.two_column_data(z, minWave=self.w0, maxWave=self.nw)
        snName, ttype, age = self.sf_age()

        print(snName, ttype, age)

        return wave, flux, minIndex, maxIndex, age, snName, ttype

    def input_spectrum(self, z, smooth, minWave, maxWave):
        wave, flux, minIndex, maxIndex = self.data.two_column_data(z, smooth, minWave, maxWave)

        return wave, flux, int(minIndex), int(maxIndex)

class ArrayTools(object):

    def __init__(self, nLabels, nw):
        self.nLabels = nLabels
        self.nw = nw

    def shuffle_arrays(self, images, labels, filenames, typeNames):
        arraySize = len(labels)
        imagesShuf = np.empty((arraySize, int(self.nw)), np.float16)
        labelsShuf = np.empty(arraySize, np.uint16)
        filenamesShuf = np.empty(arraySize, dtype=object)
        typeNamesShuf = np.empty(arraySize, dtype=object)
        idx = 0
        print("Shuffle2")
        # Randomise order
        indexShuf = list(range(arraySize))
        shuffle(indexShuf)
        for i in indexShuf:
            imagesShuf[idx] = images[i]
            labelsShuf[idx] = labels[i]
            filenamesShuf[idx] = filenames[i]
            typeNamesShuf[idx] = typeNames[i]
            idx += 1
        print("LenLabels")
        print(len(labels), idx)

        print(imagesShuf)
        print(typeNamesShuf)
        print("Shuffle3")
        return imagesShuf, labelsShuf, filenamesShuf, typeNamesShuf

    def count_labels(self, labels):
        counts = np.zeros(self.nLabels)

        for i in range(len(labels)):
            counts[labels[i]] += 1

        return counts

    def div0(self, a, b):
        """ ignore / 0, div0( [-1, 0, 1], 0 ) -> [0, 0, 0] """
        with np.errstate(divide='ignore', invalid='ignore'):
            c = np.true_divide(a, b)
            c[~ np.isfinite(c)] = 0  # -inf inf NaN
        return c

    def over_sample_arrays(self, images, labels, filenames, typeNames):
        counts = self.count_labels(labels)
        idx = 0
        print("Before OverSample")  #
        print(counts)  #

        overSampleAmount = self.div0(1 * max(counts), counts)  # ignore zeros in counts
        overSampleArraySize = int(sum(np.array(overSampleAmount, int) * counts))
        print(np.array(overSampleAmount, int) * counts)
        print(np.array(overSampleAmount, int))
        print(overSampleArraySize, len(labels))
        imagesOverSampled = np.zeros((overSampleArraySize, int(self.nw)), np.float16)
        labelsOverSampled = np.zeros(overSampleArraySize, np.uint16)
        filenamesOverSampled = np.empty(overSampleArraySize, dtype=object)
        typeNamesOverSampled = np.empty(overSampleArraySize, dtype=object)

        counts1 = np.zeros(self.nLabels)

        imagesShuf, labelsShuf, filenamesShuf, typeNamesShuf = self.shuffle_arrays(images, labels, filenames, typeNames)
        print(len(labelsShuf))
        for i in range(len(labelsShuf)):
            label = labelsShuf[i]
            image = imagesShuf[i]
            filename = filenamesShuf[i]
            typeName = typeNamesShuf[i]

            labelIndex = label # np.argmax(label)

            print(idx, i, int(overSampleAmount[labelIndex]))
            for r in range(int(overSampleAmount[labelIndex])):
                imagesOverSampled[idx] = image #np.concatenate(imagesOverSampled, np.array([image]), axis=0)
                labelsOverSampled[idx] = label #np.concatenate(labelsOverSampled, np.array([label]), axis=0)
                filenamesOverSampled[idx] = filename #filenamesOverSampled = np.append(filenamesOverSampled, filename)
                typeNamesOverSampled[idx] = typeName #typeNamesOverSampled = np.append(typeNamesOverSampled, typeName)
                counts1[labelIndex] += 1
                idx += 1

        print("After OverSample")  #
        print(counts1)  #

        print("Before Shuffling")
        imagesOverSampledShuf, labelsOverSampledShuf, filenamesOverSampledShuf, typeNamesOverSampledShuf = self.shuffle_arrays(imagesOverSampled, labelsOverSampled, filenamesOverSampled, typeNamesOverSampled)
        print("After Shuffling")
        return imagesOverSampledShuf, labelsOverSampledShuf, filenamesOverSampledShuf, typeNamesOverSampledShuf

class CreateArrays(object):
    def __init__(self, w0, w1, nw, nTypes, minAge, maxAge, ageBinSize, typeList, minZ, maxZ, redshiftPrecision, hostTypes=None, nHostTypes=None):
        self.w0 = w0
        self.w1 = w1
        self.nw = nw
        self.nTypes = nTypes
        self.minAge = minAge
        self.maxAge = maxAge
        self.ageBinSize = ageBinSize
        self.typeList = typeList
        self.minZ = minZ
        self.maxZ = maxZ
        self.numOfRedshifts = (maxZ - minZ) * 1./redshiftPrecision
        self.ageBinning = AgeBinning(minAge, maxAge, ageBinSize)
        self.numOfAgeBins = self.ageBinning.age_bin(maxAge-0.1) + 1
        self.nLabels = nTypes * self.numOfAgeBins * nHostTypes
        self.createLabels = CreateLabels(self.nTypes, self.minAge, self.maxAge, self.ageBinSize, self.typeList, hostTypes, nHostTypes)
        self.hostTypes = hostTypes

    def snid_templates_to_arrays(self, snidTemplateLocation, tempfilelist):
        """ This function is for the SNID processed files, which
            have been preprocessed to negatives, and so cannot be
            imaged yet """

        tempList = TempList().temp_list(tempfilelist) #Arbrirary redshift to read filelist
        typeList = []
        images = np.empty((0, int(self.nw)), np.float16)  # Number of pixels
        labelsIndexes = [] # labels = np.empty((0, self.nLabels), np.uint8)  # Number of labels (SN types)
        filenames = []#np.empty(0)
        typeNames = []#np.empty(0)
        agesList = []

        for i in range(0, len(tempList)):
            ncols = 15
            readSpectra = ReadSpectra(self.w0, self.w1, self.nw, snidTemplateLocation + tempList[i])

            for ageidx in range(0, 1000):
                if ageidx < ncols:
                    for z in np.linspace(self.minZ, self.maxZ, self.numOfRedshifts + 1):
                        tempwave, tempflux, ncols, ages, ttype, tminindex, tmaxindex = readSpectra.snid_template_data(ageidx, z)
                        agesList.append(ages[ageidx])
                        if not tempflux.any():
                            print("NO DATA for {} ageIdx:{} z>={}".format(tempList[i], ageidx, z))
                            break

                        if self.minAge < float(ages[ageidx]) < self.maxAge:
                            labelIndex, typeName = self.createLabels.label_array(ttype, ages[ageidx])
                            nonzeroflux = tempflux[tminindex:tmaxindex + 1]
                            newflux = (nonzeroflux - min(nonzeroflux)) / (max(nonzeroflux) - min(nonzeroflux))
                            newflux2 = np.concatenate((tempflux[0:tminindex], newflux, tempflux[tmaxindex + 1:]))
                            images = np.append(images, np.array([newflux2]), axis=0)  # images.append(newflux2)
                            labelsIndexes.append(labelIndex) # labels = np.append(labels, np.array([label]), axis=0)  # labels.append(ttype)
                            filenames.append(tempList[i] + '_' + ttype + '_' + str(ages[ageidx]) + '_z' + str(z))
                            typeNames.append(typeName)

                    print(tempList[i], ageidx, ncols)
                else:
                    break

            # Create List of all SN types
            if ttype not in typeList:
                typeList.append(ttype)
        print(len(images))

        try:
            print("SIZE OF ARRAYS:")
            print(images.nbytes)
            print(np.array(labelsIndexes).nbytes)
            print(np.array(filenames).nbytes)
            print(np.array(typeNames).nbytes)
        except:
            print("Exception Raised")

        return typeList, images, np.array(labelsIndexes), np.array(filenames), np.array(typeNames)

    def combined_sn_gal_templates_to_arrays(self, snTemplateLocation, snTempFileList, galTemplateLocation, galTempList):
        snTempList = TempList().temp_list(snTempFileList)
        typeList = []
        images = np.empty((0, int(self.nw)), np.float16)  # Number of pixels
        labelsIndexes = [] # labels = np.empty((0, self.nLabels), np.uint8)  # Number of labels (SN types)
        filenames = []  # np.empty(0)
        typeNames = []  # np.empty(0)
        agesList = []
        for j in range(len(galTempList)):
            for i in range(0, len(snTempList)):
                ncols = 15
                readSpectra = ReadSpectra(self.w0, self.w1, self.nw, snTemplateLocation + snTempList[i], galTemplateLocation + galTempList[j])
                for ageidx in range(0, 1000):
                    if ageidx < ncols:
                        for snCoeff in [0.01, 0.02, 0.05, 0.07, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]:
                            galCoeff = 1 - snCoeff
                            for z in np.linspace(self.minZ, self.maxZ, self.numOfRedshifts + 1):
                                tempwave, tempflux, ncols, ages, ttype, tminindex, tmaxindex = readSpectra.sn_plus_gal_template(ageidx, snCoeff, galCoeff, z)
                                agesList.append(ages[ageidx])
                                if not tempflux.any():
                                    print("NO DATA for {} {} ageIdx:{} z>={}".format(galTempList[j], snTempList[i], ageidx, z))
                                    break

                                if self.minAge < float(ages[ageidx]) < self.maxAge:
                                    if self.hostTypes is None: # Checks if we are classifying by host as well
                                        labelIndex, typeName = self.createLabels.label_array(ttype, ages[ageidx], host=None)
                                    else:
                                        labelIndex, typeName = self.createLabels.label_array(ttype, ages[ageidx], host=galTempList[j])
                                    nonzeroflux = tempflux[tminindex:tmaxindex + 1]
                                    newflux = (nonzeroflux - min(nonzeroflux)) / (max(nonzeroflux) - min(nonzeroflux))
                                    newflux2 = np.concatenate((tempflux[0:tminindex], newflux, tempflux[tmaxindex + 1:]))
                                    images = np.append(images, np.array([newflux2]), axis=0)
                                    labelsIndexes.append(labelIndex) # labels = np.append(labels, np.array([label]), axis=0)
                                    filenames.append("{0}_{1}_{2}_{3}_snCoeff{4}_z{5}".format(snTempList[i], ttype, str(ages[ageidx]), galTempList[j], snCoeff, (z)))
                                    typeNames.append(typeName)

                        print(snTempList[i], ageidx, ncols, galTempList[j], snCoeff)
                    else:
                        break

                # Create List of all SN types
                if ttype not in typeList:
                    typeList.append(ttype)
        print(len(images))

        try:
            print("SIZE OF ARRAYS:")
            print(images.nbytes)
            print(np.array(labelsIndexes).nbytes)
            print(np.array(filenames).nbytes)
            print(np.array(typeNames).nbytes)
        except:
            print("Exception Raised")

        return typeList, images, np.array(labelsIndexes), np.array(filenames), np.array(typeNames)

    def combined_sn_gal_arrays_multiprocessing(self, snTemplateLocation, snTempFileList, galTemplateLocation, galTempFileList):
        galTempList = TempList().temp_list(galTempFileList)

        images = np.empty((0, int(self.nw)), np.float16)
        labelsIndexes = np.empty(0, np.uint16)
        filenames = np.empty(0)
        typeNames = np.empty(0)

        t1 = time.time()
        pool = mp.Pool(processes=11)
        results = [pool.apply_async(self.combined_sn_gal_templates_to_arrays, args=(snTemplateLocation, snTempFileList, galTemplateLocation, [gList],)) for gList in galTempList]

        outputs = [p.get() for p in results]

        for out in outputs:
            typeList, imagesPart, labelsPart, filenamesPart, typeNamesPart = out
            images = np.append(images, imagesPart, axis=0)
            labelsIndexes = np.append(labelsIndexes, labelsPart, axis=0)
            filenames = np.append(filenames, filenamesPart)
            typeNames = np.append(typeNames, typeNamesPart)

        t2 = time.time()
        print("time spent: {0:.2f}".format(t2 - t1))

        try:
            print("SIZE OF ARRAYS:")
            print(images.nbytes)
            print(labelsIndexes.nbytes)
            print(filenames.nbytes)
            print(typeNames.nbytes)
        except:
            print("Exception Raised")

        return typeList, images, labelsIndexes, filenames, typeNames


from math import pow, sqrt
import Body
from wx.lib.pubsub import Publisher as pub
import random

SIZE = 250

class GravitySimulator():
    def __init__(self, numberOfBodies, maxVelocity, minMass, maxMass):
        self.maxVelocity = maxVelocity
        self.minMass = minMass
        self.maxMass = maxMass
        self.bodies = {}
        self.creatBodies(numberOfBodies, self.bodies)

    @classmethod
    def initFromDictionary(cls, bodies):
        newSimulator = GravitySimulator(0,0,0,0) #python is fucking stupid
        newSimulator.bodies = bodies
        return newSimulator

    def creatBodies(self, numberOfBodies, bodies):
        """generate random bodies and add them to bodies dictionary"""
        random.seed()
        for _ in range(numberOfBodies):
            tempPosition = Body.position(random.randint(-SIZE,SIZE), random.randint(-SIZE,SIZE), random.randint(-SIZE,SIZE))

            tempVelocity = Body.velocity(random.randint(-self.maxVelocity,self.maxVelocity),random.randint(-self.maxVelocity,self.maxVelocity),random.randint(-self.maxVelocity,self.maxVelocity))

            tempBody = Body.Body(random.uniform(self.minMass,self.maxMass), tempPosition, tempVelocity)
            self.bodies.update({tempPosition:tempBody})

    def moveBodies(self):

        #gravity
        for Body.position1, body1 in self.bodies.iteritems():
                for Body.position2, body2 in self.bodies.iteritems():
                    if(body1!=body2):
                        if (body1.position.x < body2.position.x):
                            body2.velocity.x + Newtonian.gravity(body1, body2)
                        else:
                            body2.velocity.x - Newtonian.gravity(body1, body2)

                        if (body1.position.y < body2.position.y):
                            body2.velocity.y + Newtonian.gravity(body1, body2)
                        else:
                            body2.velocity.y - Newtonian.gravity(body1, body2)

                        if (body1.position.z < body2.position.z):
                            body2.velocity.z + Newtonian.gravity(body1, body2)
                        else:
                            body2.velocity.z - Newtonian.gravity(body1, body2)

        #move the bodies
        newBodies = {}
        for Body.position,body in self.bodies.iteritems():
            body.move()
            newBodies.update({body.position:body})

        self.Bodies = newBodies

        pub.sendMessage("BODIES CHANGED", self.bodies)

class Newtonian:
    Gravity = .0000000000667259 #m^2/kg^2

    @staticmethod
    def gravity(mass1, mass2):
        distance = Newtonian.distance(mass1, mass2)
        return Newtonian.Gravity*((mass1.mass*mass2.mass)/pow(distance,2))

    @staticmethod
    def distance(mass1, mass2):
        p1 = mass1.position.x
        p2 = mass1.position.y
        p3 = mass1.position.z

        q1 = mass2.position.x
        q2 = mass2.position.y
        q3 = mass2.position.z
        return sqrt( pow(p1-q1,2) + pow(p2-q2,2) + pow(p3-q3,2) )


'''
mapclean class is responsible for reading the OSM map xml, auditing, cleaning, generating JSON and finally to insert the Modeled Data into DB.
Created on May 30, 2015

@author: Feby Thomas
'''
import xml.etree.cElementTree as ET
from collections import defaultdict
import re
import pprint
import codecs
import json
from OSM.dbutil import get_db

#The source XML file from OSM
OSMFILE = "milwaukee_wisconsin.osm"

#Regular expression to find the ending of street names
street_type_re = re.compile(r'\b\S+\.?$', re.IGNORECASE)

#Array with expected and correct values for street name ending
expected = ["Street", "Avenue", "Boulevard", "Drive", "Court", "Place", "Square", "Lane", "Road",
            "Trail", "Parkway", "Commons"]

#Regular expressions to check the various tag 'k' sub elements
lower = re.compile(r'^([a-z]|_)*$')
lower_colon = re.compile(r'^([a-z]|_)*:([a-z]|_)*$')
problemchars = re.compile(r'[=\+/&<>;\'"\?%#$@\,\. \t\r\n]')

#Array to check and group the child for Created tag
CREATED = [ "version", "changeset", "timestamp", "user", "uid"]

#Mapping for correcting the street names
mapping = { "St": "Street",
            "St.": "Street",
            "Ave": "Avenue",
            "Ave.": "Avenue",
            "Rd.": "Road",
            "Blvd": "Boulevard",
            "Cir": "Circle",
            "Ct": "Court",
            "Dr": "Drive",
            "Dr.": "Drive",
            "Ln": "Lane",
            "PL": "Place",
            "Pl.": "Place",
            "Pkwy": "Parkway",
            "Rd": "Road",
            "W": "West"
            }

def update_name(name, mapping):
    """
    Function to correct the street name using regular expression and the mapping array above.
    """
    m = street_type_re.search(name)
    if m:
        street_type = m.group()
        for key in mapping:
            if key == street_type:
                name = street_type_re.sub(mapping[key],name)

    return name

def isright_street_type(street_types, street_name):
    """
    Function to check the street type are correct and if not it adds to a set a set dictionary and returns True or False
    """
    m = street_type_re.search(street_name)
    if m:
        street_type = m.group()
        if street_type not in expected:
            street_types[street_type].add(street_name)
            return False
        else:
            return True

def isright_state(state_set,state_name):
    """
    Function to check if the state name is consistent and correct, returns boolean
    """
    if len(state_name) > 2 and state_name != 'WI':
        state_set.add(state_name)
        return False
    else:
        return True

def audit_city(city_set,city_name):
        city_set.add(city_name)

def audit_postcode(postcode):
    """
    Function to check if the post code is consistent and correct, was used to audit only
    """
    if len(postcode) != 5 and len(postcode) != 10 or not postcode.startswith('53'):
        print(postcode)

def is_street_name(elem):
    """
    Function to if XML element attribute is for street
    """
    return (elem.attrib['k'] == "addr:street")

def is_postcode(elem):
    """
    Function to if XML element attribute is for postcode
    """
    return (elem.attrib['k'] == "addr:postcode")

def is_state(elem):
    """
    Function to if XML element attribute is for state
    """
    return (elem.attrib['k'] == "addr:state")

def is_city(elem):
    """
    Function to if XML element attribute is for city
    """
    return (elem.attrib['k'] == "addr:city")

def shape_element(element):
    """
    Function to shape a XML element into data structure as expected for final JSON.
    """
    node = {}
    if element.tag == "node" or element.tag == "way" :

        pos = []
        created = {}
        node['type'] = element.tag
        #Looping for all the attributes to be child of 'created' node.
        for attr in element.attrib:
            if attr in CREATED:
                created[attr] = element.attrib[attr]
                node['created'] = created
            elif attr == 'lat':
                pos.insert(0,float(element.attrib[attr]))
            elif attr == 'lon':
                pos.insert(1,float(element.attrib[attr]))
            else:
                node[attr] = element.attrib[attr]
        node['pos'] = pos
        address = {}
        nd = []
        #Looping for sub elements and attributes of 'tag' elements
        for tag in element.iter("tag"):
            tagk = tag.attrib['k']
            if problemchars.search(tagk):
                pass
            m = lower_colon.match(tagk)
            if m:
                if tagk.startswith("addr:"):
                    address[tagk.split(':')[1]] = tag.attrib['v']
                elif ':' not in tagk:
                    #print(tagk)
                    node[tagk] = tag.attrib['v']
            else:
                node[tagk] = tag.attrib['v']
        if len(address) > 0:
            node['address'] = address
        #print node
        #handling for Node References in the XML
        for tag in element.iter("nd"):
            nd.append(tag.attrib['ref'])
        if len(nd) > 0:
            node['node_refs'] = nd
        return node
    else:
        return None

def auditncleannjson(osmfile,pretty = False):
    """
    Function to audit, clean and structure the XML to the JSON like data dictionary.
    """
    street_types = defaultdict(set)
    cntry = set()
    city = set()
    data = []
    #opening file for JSON output
    file_out = "{0}.json".format(osmfile)
    with codecs.open(file_out, "w") as fo:
        # Looping through the XML Element Tree
        for event, elem in ET.iterparse(osmfile, events=("start",)):
            #Initial audit related checks happen here
            if elem.tag == "node" or elem.tag == "way":
                for tag in elem.iter("tag"):
                    if is_street_name(tag):
                        if not isright_street_type(street_types, tag.attrib['v']):
                            tag.set('v',update_name(tag.attrib['v'], mapping))
                            #print(tag.attrib['v'])
                    if is_state(tag):
                        if not isright_state(cntry, tag.attrib['v']):
                            tag.set('v','WI')
                    #Optional postcode check was used only to audit
                    #if is_postcode(tag):
                    #    audit_postcode(tag.attrib['v'])
                    if is_city(tag):
                        audit_city(city,tag.attrib['v'])
            #Handling within a  single overall loop to shape the element into the  required dictionary data structure.
            el = shape_element(elem)
            if el:
                data.append(el)
                #Writing to the JSON file
                if pretty:
                    fo.write(json.dumps(el, indent=2)+"\n")
                else:
                    fo.write(json.dumps(el) + "\n")

    print('City Audit List: ')
    pprint.pprint(city)
    print('Street Audit List: ')
    pprint.pprint(dict(street_types))
    #print(data[0])
    #returning the full XML converted to Data Dictionary
    return data

def process_map(file_in, pretty = False):
    """
    Starts the audit and cleanup steps.
    """
    return auditncleannjson(file_in, pretty)

def insert_db(osmdata):
    """
    Inserts the data dictionary into the Mongo DB
    """
    db = get_db()
    db.osmdatamke.insert(osmdata)

def process(file_in):
    """
    Overall processing function, which first calls the cleanup and XML to JSON conversion and alter inserts teh data into the database.
    """
    osmdata = process_map(OSMFILE,True)
    print('Process Completed Before DB insert')
    insert_db(osmdata)
    print('DB Insert completed')

if __name__ == '__main__':
    process(OSMFILE)