'''
	Combines Real and Imaginary optical constants using Effective Medium Theory for SCIENCE
	Then calculates emissivites based on the optical constants derived- also for SCIENCE

	Author: Dr. Farisa Morales (original IDL code) transcribed and modified by Cody King

	TODO:
		-Continue to impliment logic flow for choice of materials and such
		-Remove superfluous code
		-Change to usage of numpy readers(?)
		-Generalize variable names to accomodate other substances?
		-Make stylistic changes as per suggestion
		-Add error handling where errors would be common (e.g. file openers)
'''

try:
	from numpy import interp, real, imag, pi
#interp(x-coord of desired interpolates, x-coord of input data, y-coord of input data)
#Returns a numpy array object; this can be quickly converted to a standard python list
#I don't know if it's necessary (probably not) but I do it anyway just in case
	from kmod import whiteSpaceParser, getStart, getCol, listifier, columnizer
	from cmath import sqrt
	from math import log10
except ImportError:
	print('One or more required modules could not be found; check the python distribution library.')
	raw_input('Press ENTER to exit')
	exit()

timey = True
try:
	from os import times
except ImportError:
	timey = False

if timey: TIME1 = times()[0]

print('Initializing...')

#Thinking of adding a default variable loader
DEFAULTDIR = r'C:\Users\Cody\Desktop\School Documents\Physics\Independent Study\DielectricEffect'

#-----------------------------------------------------------
#Tiny simplifying functions that need to be declared earlier
def logicFlow (Q):
	resp = raw_input(Q)
	if resp == '1':
		return True
	elif resp == '2':
		return False
	else:
		print(resp + ' is not valid input.')
		return None
#-----------------------------------------------------------

#--------------------
#- Global Variables -
#--------------------
#For whether or not we will be making our own polluted matrix prior to re-inclusion;
#if true we do not make our own
preDMat = None
while preDMat == None:
	preDMat = logicFlow(r'Use a predetermined matrix composition? 1)Yes 2)No: ')

#For whether or not we will be running MIE theory on the optical constants calculated
#here. The value of this constant should be self-explanitory to what it means.
RUNMIE = None
while RUNMIE == None:
	RUNMIE = logicFlow(r'Run Mie Theory? 1)Yes 2)No: ')

#For whether or not we will be writing new lists to memory. If true, checks if we want to
#write the results of interpolations, EMT, and if running MIE, the results of MIE theory
WRITE = None
while WRITE == None:
	WRITE = logicFlow(r'Write results to disk? 1)Yes 2)No: ')

if WRITE:
	WRITE_INT = None
	while WRITE_INT == None:
		WRITE_INT = logicFlow(r'Write results of interpolates to disk? 1)Yes 2)No: ')
	WRITE_EMT = None
	while WRITE_EMT == None:
		WRITE_EMT = logicFlow(r'Write results of EMT to disk? 1)Yes 2)No: ')
	if RUNMIE:
		WRITE_MIE = None
		while WRITE_MIE == None:
			WRITE_MIE = logicFlow(r'Write results of MIE Theory to disk? 1)Yes 2)No: ')

#For whether or not we will be generating graphs of the results of the intpolations,
#EMT, and if running MIE, the results of MIE theory.
GRAPH = None
while GRAPH == None:
	GRAPH = logicFlow(r'Graph results? 1)Yes 2)No: ')

if GRAPH:
	GRAPH_EMT = None
	while GRAPH_EMT == None:
		GRAPH_EMT = logicFlow(r'Graph results of EMT? 1)Yes 2)No: ')
	if RUNMIE:
		GRAPH_MIE = None
		while GRAPH_MIE == None:
			GRAPH_MIE = logicFlow(r'Graph results of MIE Theory? 1)Yes 2)No: ')
	import matplotlib.pyplot as plt

print('Orbital Variables:\n\tpreDMat-' + str(preDMat) + '\n\tRUNMIE-' + str(RUNMIE) + '\n\tWRITE-' + str(WRITE) + '\n\tGRAPH-' + str(GRAPH) + '\nSuborbital Variables:')
if WRITE:
	print('\tWRITE_INT- ' + str(WRITE_INT) + '\n\tWRITE_EMT- ' + str(WRITE_EMT))
	if RUNMIE:
		print('\tWRITE_MIE- ' + str(WRITE_MIE))
if GRAPH:
	print('\tGRAPH_EMT- ' + str(GRAPH_EMT))
	if RUNMIE:
		print('\tGRAPH_MIE- ' + str(GRAPH_MIE))

#---------------------------------------------
#- File directories for the necessary tables -
#---------------------------------------------

#File directory for written tables
directory = raw_input(r'Please specifiy the file directory (Press ENTER for default): ')
if directory == '':
	directory = DEFAULTDIR

print('File Directory:\n' + directory)

if not preDMat:
	#Amorphous Carbon Optical Constants
	dir_AC = directory + r'\AmorphCarbOptConst.dat'

	#Water Optical Constants
	dir_WA = directory + r'\waterOpticalConstants.dat'

else:
   #Dirty Ice Optical Constants
	dir_DI = directory + r'\DirtyIceOptConst.dat'

#AstroSil Optical Constants
dir_AS = directory+ r'\AstroSilOptConst.dat'

#Desired Wavelengths
dir_W = directory + r'\wav_set.dat'

#Desired Grain Sizes; current table is currently unreadable by my modules so it needs to manually
#be modified; sorry!
dir_G = directory + r'\grain_size_grid_suvsil.list'

#------------------------------------------------------
#- Short functions which simplify (maybe) some things -
#------------------------------------------------------
#----------------------------------------------------------------
#Short function to trim lists based on designated mins and maxes
def trimmer(LIST, mini, maxi):
	tempList = []
	for ele in LIST:
		if ele >= mini and ele <= maxi:
			tempList.append(ele)
	return tempList
#End trimming function
#-------------------------------------------------------------
#Short function to write to a file data from a dictionary.
#Writes to a csv file the keys as a header followed by the values
#associated with each.
#Python doesn't care if we want the string read-in as a raw string
#if it sees a \' at the end of a string it complains about end-of-line errors!
def writerD(DICT, FILENAME):
	tempFile = open(directory + '\\' + FILENAME, 'w')
	for ele in DICT.keys():
		if ele is not 'NAME':
			tempFile.write(str(ele) + ',')
	tempFile.write('\n')
	row = 0
	while row < len(DICT[DICT.keys()[0]]):
		for key in DICT.keys():
			if key is not 'NAME':
				tempFile.write(str((DICT[key][row])) + ',')
		tempFile.write('\n')
		row += 1
	tempFile.close()
#End short writing function
#--------------------------------------------------------
#Short writing function specifically for the emissivities.
#Writes to a csv file a header of wavelengths followed by
#rows of emissivities with the first value in each row being
#the associated grain radius.
#Done as dictionaries are (seemingly) randomly ordered when it
#comes to numerical keys
def writerE(HEADER, ARRAYARRAY, FILENAME):
	tempFile = open(directory + '\\' + FILENAME, 'w')
	tempFile.write(str(None) + ',')	#Include header explaining table and content and generating parameters?
	for ele in HEADER:
		tempFile.write(str(ele) + ',')
	tempFile.write('\n')
	for ARRAY in ARRAYARRAY:
		for ele in ARRAY:
			tempFile.write(str(ele) + ',')
		tempFile.write('\n')
	tempFile.close()
#End short writing function
#------------------------------------------------------------------------------------------
#Short function to calculate the average optical constants based on Effective Medium Theory
#NM: Refractive Index of Matrix, KM: Extinction Coefficient of Matrix
#NI: Refractive Index of Inclusion, KI: Extinction Coefficient of Inclusion
#V: Ratio of pollutant volume to total volume
def EMT(NM, KM, NI, KI, V=.5):
	M, I = (NM + KM*1j) ** 2, (NI + KI*1j) ** 2
	F = (I - M) / (I + 2.0 * M)
	AMOC = sqrt(M * (1 + ((3.0 * V * F) / (1 - V * F))))
	return float(real(AMOC)), float(imag(AMOC))
#End REALLY short EMT function
#-------------------------------------------------------------
#Short function to utilize the above EMT function in recursion
#Argument is a list of tuples, each tuple comprised of the dictionary of the matrix
#material, the dictionary of the inclusion material, and then the decimal volume
#fraction of the matrix to total volume. Keyword arguments are: 'recur' and 'vacuum'.
#'recur' can take the place of the matrix or inclusion except on the first iteration;
#'vacuum' can only take the place of the inclusion.
def EMTRecursion(*mivargs):
	lastIter, iteration = {'Refractive Index':[], 'Extinction Coefficient':[]}, 0
	for miv in mivargs:
		m, i, v = miv
		if m == 'vacuum' or (iteration == 0 and (m == 'recur' or i == 'recur')) or v >= 1 or v <= 0:
			print('An error has occured; vacuum has been designated as the matrix material or\na recur was encountered on the first iteration\nIteration: ' + str(iteration) + ' or an invalid ratio was encountered')
			return None
		if m == 'recur':
			m = {'Refractive Index':[], 'Extinction Coefficient':[]}
			m.update(lastIter)
		if i == 'recur':
			i = {'Refractive Index':[], 'Extinction Coefficient':[]}
			i.update(lastIter)
		lastIter.update({'Refractive Index':[], 'Extinction Coefficient':[]})
		for it, mRI in enumerate(m['Refractive Index']):
			if i == 'vacuum':
				RI, EC = EMT(mRI, m['Extinction Coefficient'][it], 1, 0, v)
			else:
				RI, EC = EMT(mRI, m['Extinction Coefficient'][it], i['Refractive Index'][it], i['Extinction Coefficient'][it], v)
			lastIter['Refractive Index'].append(RI)
			lastIter['Extinction Coefficient'].append(EC)
		iteration += 1
	return lastIter
#End short recursion function

#-----------------------------------------------
#- Writing data from tables in files to memory -
#-----------------------------------------------

print('Writing required files to memory...')

if not preDMat:
	#--------------------
	#- Amorphous Carbon -
	#--------------------
	#Open the Amorphous Carbon table in binary mode
	AC_Tab = open(dir_AC, 'rb')

	#Start the reader at the appropriate postion
	AC_Tab.seek(getStart(AC_Tab))

	#Do the thing! Seperate the values!
	#Assign to mList the list of lists returned from Columnizer which took a single
	#list returned by listifier (as well as the number of columns in the table
	#found using the getCol module) which itself took a tuple returned by
	#whiteSpaceParser which consisted of the modified contents of the read file
	#in addition to the index location of whitespaces in that file.
	#For more information, check kmod.py
	mList = columnizer(listifier(whiteSpaceParser(AC_Tab.read())), getCol(AC_Tab))
	AmorphCarb = {'Wavelength(Microns)':mList[0], 'Refractive Index':mList[1], 'Extinction Coefficient':mList[2], 'NAME':'Amorphous Carbon'}

	#Close the file and delete variables no longer needed
	AC_Tab.close()
	del(AC_Tab, mList, dir_AC)

	#-------------
	#- Water Ice -
	#-------------
	#Similar proceedure for Water
	WA_Tab = open(dir_WA, 'rb')

	WA_Tab.seek(getStart(WA_Tab))

	mList = columnizer(listifier(whiteSpaceParser(WA_Tab.read())), getCol(WA_Tab))

	Water = {'Wavelength(Microns)':mList[0], 'Refractive Index':mList[1], 'Extinction Coefficient':mList[2], 'NAME':'Water Ice'}

	WA_Tab.close()

	del(WA_Tab, mList, dir_WA)

else:
	#-------------
	#- Dirty Ice -
	#-------------
	#Similar proceedure for Dirty Ice
	DI_Tab = open(dir_DI, 'rb')

	DI_Tab.seek(getStart(DI_Tab))

	mList = columnizer(listifier(whiteSpaceParser(DI_Tab.read())), getCol(DI_Tab))

	DirtyIce = {'Wavelength(Microns)':mList[0], 'Refractive Index':mList[1], 'Extinction Coefficient':mList[2], 'NAME':'Dirty Ice'}

	DI_Tab.close()

	del(DI_Tab, mList, dir_DI)

#------------
#- AstroSil -
#------------
#Similar proceedure for AstroSil
AS_Tab = open(dir_AS, 'rb')

AS_Tab.seek(getStart(AS_Tab))

mList = columnizer(listifier(whiteSpaceParser(AS_Tab.read())), getCol(AS_Tab))

#Values must be put in reverse order for ease of interpolation later
#Reversing can't be done in a dictionary assignment because it returns
#a None type for the value for some reason
mList[0].reverse()
mList[3].reverse()
mList[4].reverse()
AstroSil = {'Wavelength(Microns)':mList[0], 'Refractive Index':mList[3], 'Extinction Coefficient':mList[4], 'NAME':'Astro Silicates'}

#Apparently the author of the AstroSil table created the table with the real parts decremented by 1?
#Some of the values in the list now incremented by 1 show floating point arithmetic relics
#I'm pretty confident that's just a visual problem
for i, ele in enumerate(AstroSil['Refractive Index']):
	AstroSil['Refractive Index'][i] = ele + 1.0

AS_Tab.close()

del(AS_Tab, mList, dir_AS, i, ele)

#-----------------
#- Desired Waves -
#-----------------
#Similar proceedure for the desired Wavelengths
W_Tab = open(dir_W, 'rb')

W_Tab.seek(getStart(W_Tab))

mList = columnizer(listifier(whiteSpaceParser(W_Tab.read())), getCol(W_Tab))

Waves = {'Wavelength(Microns)':mList[0]}

W_Tab.close()

del(W_Tab, mList, dir_W)

#-----------------------
#- Desired Grain Sizes -
#-----------------------
#A litte different proceedure for the desired Grain Sizes
G_Tab = open(dir_G, 'rb')

G_Tab.seek(getStart(G_Tab))

mList = columnizer(listifier(whiteSpaceParser(G_Tab.read())), getCol(G_Tab))

Waves.update({'SIZE':mList[0]})

G_Tab.close()

del(G_Tab, mList, dir_G)

#-----------------------------------------------------------
#- Trimming Wave lists for useful Wavelengths; the prequel -
#-----------------------------------------------------------
#New Wavelengths based on Primary Matrix Material; determines Wavelength for all others.
#As this is needed for interpolation it must be done before the interpolation
#but assignment to other dictionaries must be done after.

print('Trimming Wave list...')

if not preDMat:
	Waves.update({'Wavelength(Microns)':trimmer(Waves['Wavelength(Microns)'], min(Water['Wavelength(Microns)']), max(Water['Wavelength(Microns)']))})

else:
	Waves.update({'Wavelength(Microns)':trimmer(Waves['Wavelength(Microns)'], min(DirtyIce['Wavelength(Microns)']), max(DirtyIce['Wavelength(Microns)']))})

#---------------------------------------------------------------------------
#- Interpolate using SciPy's (NumPy's?) interp function for missing values -
#---------------------------------------------------------------------------
#Update the dictionaries with these interpolations, changing complex coefficients
#into complex type. For some reason I get erroneous optical constants from EMT if
#I change these into complex numbers piecewise during EMT, I still can't figure
#out why

print('Interpolating for missing values and updating the appropriate dictionaries...')

if not preDMat:
	#Update Amorphous Carbon Dict with interpolates for reals
	AmorphCarb.update({'Refractive Index':list(interp(Waves['Wavelength(Microns)'], AmorphCarb['Wavelength(Microns)'], AmorphCarb['Refractive Index']))})

	#Update Amorphous Carbon Dict with interpolates for complex
	AmorphCarb.update({'Extinction Coefficient':list(interp(Waves['Wavelength(Microns)'], AmorphCarb['Wavelength(Microns)'], AmorphCarb['Extinction Coefficient']))})

	#Update Water Dict with interpolates for reals
	Water.update({'Refractive Index':list(interp(Waves['Wavelength(Microns)'], Water['Wavelength(Microns)'], Water['Refractive Index']))})

	#Update Water Dict with interpolates for complex
	Water.update({'Extinction Coefficient':list(interp(Waves['Wavelength(Microns)'], Water['Wavelength(Microns)'], Water['Extinction Coefficient']))})

else:
	#Update DirtyIce Dict with interpolates for reals
	DirtyIce.update({'Refractive Index':list(interp(Waves['Wavelength(Microns)'], DirtyIce['Wavelength(Microns)'], DirtyIce['Refractive Index']))})

	#Update DirtyIce Dict with interpolates for complex
	DirtyIce.update({'Extinction Coefficient':list(interp(Waves['Wavelength(Microns)'], DirtyIce['Wavelength(Microns)'], DirtyIce['Extinction Coefficient']))})

#Update AstroSil Dict with interpolates for reals
AstroSil.update({'Refractive Index':list(interp(Waves['Wavelength(Microns)'], AstroSil['Wavelength(Microns)'], AstroSil['Refractive Index']))})

#Update AstroSil Dict with interpolates for complex
AstroSil.update({'Extinction Coefficient':list(interp(Waves['Wavelength(Microns)'], AstroSil['Wavelength(Microns)'], AstroSil['Extinction Coefficient']))})

#----------------------------------------------------------
#- Trimming Wave lists for useful Wavelengths; the sequel -
#----------------------------------------------------------

print('Doin\' some more trimming...')

if not preDMat:
	#New Wavelenghts for Amorphous Carbon
	AmorphCarb.update({'Wavelength(Microns)':Waves['Wavelength(Microns)']})

	#New Wavelenghts for Water
	Water.update({'Wavelength(Microns)':Waves['Wavelength(Microns)']})

else:
	#New Wavelenghts for DirtyIce
	DirtyIce.update({'Wavelength(Microns)':Waves['Wavelength(Microns)']})

#New Wavelenghts for AstroSil
AstroSil.update({'Wavelength(Microns)':Waves['Wavelength(Microns)']})

#--------------------------------------------------------------
#- Computing Optical Constants for Inclusion-Matrix Particles -
#--------------------------------------------------------------
#Finally doing part of what this program is named after! Yay!
#Initialize the Optical Constants dictionary

print('Implimenting EMT to compute Optical Constants...')

IMPOptConst = {'Wavelength(Microns)':Waves['Wavelength(Microns)'], 'Refractive Index':[], 'Extinction Coefficient':[], 'NAME':'Inclusion Matrix Particles'}

#Iterate through designated reals and utilize EMT to generate new
#optical constants

if not preDMat:
	IMPOptConst.update(EMTRecursion((Water, AmorphCarb, 0.5), ('recur', AstroSil, 0.5)))

else:
	IMPOptConst.update(EMTRecursion((DirtyIce, AstroSil, 0.5)))

if WRITE:
	if WRITE_INT:
		print('Writing Interpolate Data to disk...')
		if not preDMat:
			writerD(AmorphCarb, 'AmorphCarbInterp2.csv')
			writerD(Water, 'WaterInterp2.csv')
		else:
			writerD(DirtyIce, 'DirtyIceInterp2.csv')
			writerD(AstroSil, 'AstroSilInterp2.csv')
	if WRITE_EMT:
		print('Writing EMT Data to disk...')
		writerD(IMPOptConst, 'IMPOptConst2.csv')

if GRAPH and GRAPH_EMT:
#--------------------------------------------------
#- Plot appropriate values if designated to do so -
#--------------------------------------------------
	def flotEMT(*flargs):
		plt.xlabel('Wavelength(Microns)')
		plt.ylabel('Exitinction Coefficient')
		plt.title('Extinction Coefficient Vs. Wavelength')
		plt.xscale('log')
		plt.yscale('log')
		plt.grid(True)
		for Exti in flargs:
			plt.plot(Exti['Wavelength(Microns)'], Exti['Extinction Coefficient'], label = Exti['NAME'])
		plt.legend(loc = 0)
		plt.show()

		plt.xlabel('Wavelength(Microns)')
		plt.ylabel('Refractive Index')
		plt.title('Refractive Index Vs. Wavelength')
		plt.xscale('log')
		plt.grid(True)
		for Refracti in flargs:
			plt.plot(Refracti['Wavelength(Microns)'], Refracti['Refractive Index'], label = Refracti['NAME'])
		plt.legend(loc = 0)
		plt.show()

	if not preDMat:
		flotEMT(Water, AmorphCarb, AstroSil, IMPOptConst)
	else:
		flotEMT(DirtyIce, AstroSil, IMPOptConst)

if RUNMIE:
#--------------
#- MIE Theory -
#--------------
#Each refractive index value is associated with a particular Wavelength.
#Inside an iteration of the values of the refractive index, we iterate
#through the grain sizes. We call bhmie with the size parameter associated
#with that particular grain size with that particular Wavelength, along
#with the sum of the refractive index and extinction coefficient associated
#with that Wavelength. The resulting Emiss list is a list of lists; each
#sublist is the emissivity of the grains of the various sizes associated
#with the Wavelength of incident light.

	print('Utilizing MIE Theory to calculate Emissivity of the IMPs.\nDepending on the number of Wavelengths and Grain Sizes,\nthis can take qite a bit of time... [>1hr]')

	if not preDMat:
		del(AmorphCarb, Water, IntermediaryMatrix)
	else:
		del(DirtyIce)
	del(AstroSil)

	from bhmie_herbert_kaiser_july2012_GEOFF_EDITION import bhmie
	Emiss = []
	if timey: Time0 = times()[0]
	sizey = len(Waves['SIZE'])
	for iter, radius in enumerate(Waves['SIZE']):
		print('Emissivites for grains of radius: ' + str(radius) + ' microns. ' + str(iter + 1) + ' of ' + str(sizey))
		if timey:
			print('\nTime for last radius: ' + str(times()[0] - Time0) + ' seconds')
			Time0 = times()[0]
		Emiss.append([radius])
		for i, lamda in enumerate(Waves['Wavelength(Microns)']):
			RI = IMPOptConst['Refractive Index'][i]
			if type(IMPOptConst['Extinction Coefficient'][i]) is not type(1j):
				EC = IMPOptConst['Extinction Coefficient'][i]*1j
			else:
				EC = IMPOptConst['Extinction Coefficient'][i]
			qabs = bhmie((2*pi*radius)/lamda, RI + EC, 2)
			if (qabs <= 0):
				qabs = 1.0e-11
			Emiss[iter].append(qabs)

if WRITE and RUNMIE:
	if WRITE_MIE:
		print('Writing MIE Data to disk...')
		writerE(Waves['Wavelength(Microns)'], Emiss, 'Emissivities.csv')

if GRAPH and RUNMIE:
	if GRAPH_MIE:
		def flotMIE(EMISS):
			plt.xlabel('Wavelength(Microns)')
			plt.ylabel('Emissivity')
			plt.title('Emissivity Vs. Wavelength')
			plt.xscale('log')
			plt.yscale('log')
			plt.grid(True)
			for ARRAY in EMISS:
				if (log10(ARRAY[0]) == int(log10(ARRAY[0]))):
					plt.plot(Waves['Wavelength(Microns)'], ARRAY[1:], label = ARRAY[0])
			plt.legend(loc = 0)
			plt.show()

		flotMIE(Emiss)
if timey:
	print('Run Time: ' + str(times()[0] - TIME1) + ' seconds')
raw_input('Press ENTER to exit')

'''
if not preDMat:
	for i, ele in enumerate(Water['Refractive Index']):

		#Utilize EMT for Amorphous Carbon and Water
		RI, EC = EMT(ele, Water['Extinction Coefficient'][i], AmorphCarb['Refractive Index'][i], AmorphCarb['Extinction Coefficient'][i])

		#Append the appropriate values to the Intermediary Matrix dictionary
		IntermediaryMatrix['Refractive Index'].append(RI)
		if EC < 0:
			EC = 1.0e-11
		IntermediaryMatrix['Extinction Coefficient'].append(EC)

		#Utilize the results from the last utilization of EMT for this run of EMT
		#now with AstroSil
		RI, EC = EMT(IntermediaryMatrix['Refractive Index'][i], IntermediaryMatrix['Extinction Coefficient'][i], AstroSil['Refractive Index'][i], AstroSil['Extinction Coefficient'][i])

		#Append the appropriate values to the IMP dictionary
		IMPOptConst['Refractive Index'].append(RI)
		if EC < 0:
			EC = 1.0e-11
		IMPOptConst['Extinction Coefficient'].append(EC)

else:
	for i, ele in enumerate(DirtyIce['Refractive Index']):

		#Utilize EMT with the Predetermined Matrix and Inclusion
		RI, EC = EMT(ele, DirtyIce['Extinction Coefficient'][i], AstroSil['Refractive Index'][i], AstroSil['Extinction Coefficient'][i])

		#Append the appropriate values to the IMP dictionary
		IMPOptConst['Refractive Index'].append(RI)
		if EC < 0:
			EC = 1.0e-11
		IMPOptConst['Extinction Coefficient'].append(EC)

del(i, ele, RI, EC)
'''


import boto3

client = boto3.client('rds')
response = client.create_event_subscription(
    Enabled=True,
    EventCategories=[
        'availability',
    ],
    SnsTopicArn='arn:aws:sns:us-east-2:585584209241:mysql',
    SourceIds=[
        'database-instance-01',
    ],
    SourceType='db-instance',
    SubscriptionName='mysqleventsubscription',
)
print(response)


import re
import time
from xlwt import *
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

excelFile = Workbook(encoding='utf-8')
excelTable = excelFile.add_sheet('Heller')

optionChrome = Options()
optionChrome.add_argument('--headless')
optionChrome.add_argument('--disable-gpu')
optionChrome.add_argument('disable-plugins')
optionChrome.add_argument('disable-extensions')

row = 0
###################################################################################

u = ['https://www.heller.biz/en/machines-and-solutions/4-axis-machining-centres-h/','https://www.heller.biz/en/machines-and-solutions/5-axis-machining-centres-f/','https://www.heller.biz/en/machines-and-solutions/5-axis-milling-turning-machining-centres-c/']

for z in range(len(u)):
    driverChrome = webdriver.Chrome(options=optionChrome)
    driverChrome.get(u[z])
    time.sleep(2)
    htmlResult = driverChrome.page_source
    driverChrome.quit()

    soup = BeautifulSoup(htmlResult, 'lxml')

####################################################################################
    rowNum_Name = row
    colNum_Name = 0

    Name = soup.find('tr', {"class": "technicaldata-title"})
    machineLabel = str(re.findall(r'.*', str(Name))).split('	')

    Name_Iterator = iter(machineLabel)
    for i in Name_Iterator:
        End_Name = str(i).replace('', '').replace('', '').replace('', '').replace('', '').replace("['", '').replace("']", '')
        excelTable.write(rowNum_Name, colNum_Name, End_Name)
        rowNum_Name += 1

####################################################################################

    LabelList = []
    UnitList = []
    rowNum_Label = row
    colNum_Label = 0
    counter = 0
    Tech = soup.find('tr', attrs={"class": "technicaldata-content"})

    Unit = Tech.find('td', attrs={"class": "technicaldata-unit"})
    while Unit is not None:
        if '	' in str(Unit):
            UnitList.append('')
            break
        Spec_Unit = str(re.findall(r'.*	', str(Unit)))[34:][:-8]
        UnitList.append(Spec_Unit)
        Unit = Unit.find_next('td', attrs={"class": "technicaldata-unit"})
#fcsdfsd

    Spec = Tech.find('td', {"class": "technicaldata-th hyphenate"})

    while Spec is not None:
        SpecLabel_1 = str(re.findall(r'.*
', str(Spec)))[42:][:-8].replace(
            "\\xad", '')
        SpecLabel_2 = str(re.findall(r'.*', str(Spec)))[25:][:-10].replace("\\xad", '')
        LabelList.append(SpecLabel_1 + " " + SpecLabel_2 + " " + UnitList[counter])
        counter += 1
        Spec = Spec.find_next('td', {"class": "technicaldata-th hyphenate"})

    for k in range(len(LabelList)):
        for p in range(len(machineLabel)):
            excelTable.write(rowNum_Label, 2 * colNum_Label + 1, LabelList[k])
            rowNum_Label += 1
        rowNum_Label = row
        colNum_Label += 1

    ########################################################################################################################
    TechList = []
    rowNum_Data = row
    colNum_Data = 0
    Tech = soup.find('tr', attrs={"class": "technicaldata-content"})

    while Tech is not None:
        TechList.append(str(Tech))
        Tech = Tech.find_next('tr', attrs={"class": "technicaldata-content"})
        if 'Hyphenator' in str(Tech):
            break

    Tech_Iterator = iter(TechList)
    for i in Tech_Iterator:
        Data = str(re.findall(r'.*', str(i))).split('	')
        Data_Iterator = iter(Data)
        for j in Data_Iterator:
            End_Data = str(j).replace('', '').replace('', '').replace('', '').replace("['", '').replace("']", '')
            excelTable.write(rowNum_Data, 2 * colNum_Data + 2, End_Data)
            rowNum_Data += 1
        rowNum_Data = row
        colNum_Data += 1
    row += len(machineLabel)

########################################################################################################################
excelFile.save('Heller.xls')


"""
Code for linting modules and subworkflows in the nf-core/modules repository and
in nf-core pipelines
"""

from __future__ import print_function

import logging
import operator
import os
from pathlib import Path

import rich
from rich.markdown import Markdown
from rich.table import Table

import nf_core.modules.modules_utils
import nf_core.utils
from nf_core.components.components_command import ComponentCommand
from nf_core.components.nfcore_component import NFCoreComponent
from nf_core.lint_utils import console
from nf_core.modules.modules_json import ModulesJson
from nf_core.utils import plural_s as _s

log = logging.getLogger(__name__)

class LintException(Exception):
    """Exception raised when there was an error with module or subworkflow linting"""

    pass

class LintResult:
    """An object to hold the results of a lint test"""

    def __init__(self, component, lint_test, message, file_path):
        self.component = component
        self.lint_test = lint_test
        self.message = message
        self.file_path = file_path
        self.component_name = component.component_name

class ComponentLint(ComponentCommand):
    """
    An object for linting modules and subworkflows either in a clone of the 'nf-core/modules'
    repository or in any nf-core pipeline directory
    """

    def __init__(
        self,
        component_type,
        dir,
        fail_warned=False,
        remote_url=None,
        branch=None,
        no_pull=False,
        registry=None,
        hide_progress=False,
    ):
        super().__init__(
            component_type,
            dir=dir,
            remote_url=remote_url,
            branch=branch,
            no_pull=no_pull,
            hide_progress=hide_progress,
        )

        self.fail_warned = fail_warned
        self.passed = []
        self.warned = []
        self.failed = []
        if self.component_type == "modules":
            self.lint_tests = self.get_all_module_lint_tests(self.repo_type == "pipeline")
        else:
            self.lint_tests = self.get_all_subworkflow_lint_tests(self.repo_type == "pipeline")

        if self.repo_type == "pipeline":
            modules_json = ModulesJson(self.dir)
            modules_json.check_up_to_date()
            self.all_remote_components = []
            for repo_url, components in modules_json.get_all_components(self.component_type).items():
                if remote_url is not None and remote_url != repo_url:
                    continue
                for org, comp in components:
                    self.all_remote_components.append(
                        NFCoreComponent(
                            comp,
                            repo_url,
                            Path(self.dir, self.component_type, org, comp),
                            self.repo_type,
                            Path(self.dir),
                            self.component_type,
                        )
                    )
            if not self.all_remote_components:
                raise LookupError(
                    f"No {self.component_type} from {self.modules_repo.remote_url} installed in pipeline."
                )
            local_component_dir = Path(self.dir, self.component_type, "local")
            self.all_local_components = []
            if local_component_dir.exists():
                self.all_local_components = [
                    NFCoreComponent(
                        comp,
                        None,
                        Path(local_component_dir, comp),
                        self.repo_type,
                        Path(self.dir),
                        self.component_type,
                        remote_component=False,
                    )
                    for comp in self.get_local_components()
                ]
            self.config = nf_core.utils.fetch_wf_config(self.dir, cache_config=True)
        else:
            component_dir = Path(
                self.dir,
                self.default_modules_path if self.component_type == "modules" else self.default_subworkflows_path,
            )
            self.all_remote_components = [
                NFCoreComponent(m, None, component_dir / m, self.repo_type, Path(self.dir), self.component_type)
                for m in self.get_components_clone_modules()
            ]
            self.all_local_components = []
            if not self.all_remote_components:
                raise LookupError(f"No {self.component_type} in '{self.component_type}' directory")

            # This could be better, perhaps glob for all nextflow.config files in?
            self.config = nf_core.utils.fetch_wf_config(Path(self.dir).joinpath("tests", "config"), cache_config=True)

        if registry is None:
            self.registry = self.config.get("docker.registry", "quay.io")
        else:
            self.registry = registry
        log.debug(f"Registry set to {self.registry}")

        self.lint_config = None
        self.modules_json = None

    @staticmethod
    def get_all_module_lint_tests(is_pipeline):
        if is_pipeline:
            return [
                "module_patch",
                "module_version",
                "main_nf",
                "meta_yml",
                "module_todos",
                "module_deprecations",
                "module_changes",
            ]
        else:
            return ["main_nf", "meta_yml", "module_todos", "module_deprecations", "module_tests"]

    @staticmethod
    def get_all_subworkflow_lint_tests(is_pipeline):
        if is_pipeline:
            return ["main_nf", "meta_yml", "subworkflow_changes", "subworkflow_todos", "subworkflow_version"]
        else:
            return ["main_nf", "meta_yml", "subworkflow_todos", "subworkflow_tests"]

    def set_up_pipeline_files(self):
        self.load_lint_config()
        self.modules_json = ModulesJson(self.dir)
        self.modules_json.load()

        # Only continue if a lint config has been loaded
        if self.lint_config:
            for test_name in self.lint_tests:
                if self.lint_config.get(test_name, {}) is False:
                    log.info(f"Ignoring lint test: {test_name}")
                    self.lint_tests.remove(test_name)

    def filter_tests_by_key(self, key):
        """Filters the tests by the supplied key"""
        # Check that supplied test keys exist
        bad_keys = [k for k in key if k not in self.lint_tests]
        if len(bad_keys) > 0:
            raise AssertionError(
                "Test name{} not recognised: '{}'".format(
                    _s(bad_keys),
                    "', '".join(bad_keys),
                )
            )

        # If -k supplied, only run these tests
        self.lint_tests = [k for k in self.lint_tests if k in key]

    def _print_results(self, show_passed=False, sort_by="test"):
        """Print linting results to the command line.

        Uses the ``rich`` library to print a set of formatted tables to the command line
        summarising the linting results.
        """

        log.debug("Printing final results")

        sort_order = ["lint_test", "component_name", "message"]
        if sort_by == "module" or sort_by == "subworkflow":
            sort_order = ["component_name", "lint_test", "message"]

        # Sort the results
        self.passed.sort(key=operator.attrgetter(*sort_order))
        self.warned.sort(key=operator.attrgetter(*sort_order))
        self.failed.sort(key=operator.attrgetter(*sort_order))

        # Find maximum module name length
        max_name_len = 40
        for tests in [self.passed, self.warned, self.failed]:
            try:
                for lint_result in tests:
                    max_name_len = max(len(lint_result.component_name), max_name_len)
            except:
                pass

        # Helper function to format test links nicely
        def format_result(test_results, table):
            """
            Given an list of error message IDs and the message texts, return a nicely formatted
            string for the terminal with appropriate ASCII colours.
            """
            # TODO: Row styles don't work current as table-level style overrides.
            # Leaving it here in case there is a future fix
            last_modname = False
            even_row = False
            for lint_result in test_results:
                if last_modname and lint_result.component_name != last_modname:
                    even_row = not even_row
                last_modname = lint_result.component_name
                table.add_row(
                    Markdown(f"{lint_result.component_name}"),
                    os.path.relpath(lint_result.file_path, self.dir),
                    Markdown(f"{lint_result.message}"),
                    style="dim" if even_row else None,
                )
            return table

        # Print blank line for spacing
        console.print("")

        # Table of passed tests
        if len(self.passed) > 0 and show_passed:
            table = Table(style="green", box=rich.box.MINIMAL, pad_edge=False, border_style="dim")
            table.add_column(f"{self.component_type[:-1].title()} name", width=max_name_len)
            table.add_column("File path")
            table.add_column("Test message")
            table = format_result(self.passed, table)
            console.print(
                rich.panel.Panel(
                    table,
                    title=rf"[bold][✔] {len(self.passed)} {self.component_type[:-1].title()} Test{_s(self.passed)} Passed",
                    title_align="left",
                    style="green",
                    padding=0,
                )
            )

        # Table of warning tests
        if len(self.warned) > 0:
            table = Table(style="yellow", box=rich.box.MINIMAL, pad_edge=False, border_style="dim")
            table.add_column(f"{self.component_type[:-1].title()} name", width=max_name_len)
            table.add_column("File path")
            table.add_column("Test message")
            table = format_result(self.warned, table)
            console.print(
                rich.panel.Panel(
                    table,
                    title=rf"[bold][!] {len(self.warned)} {self.component_type[:-1].title()} Test Warning{_s(self.warned)}",
                    title_align="left",
                    style="yellow",
                    padding=0,
                )
            )

        # Table of failing tests
        if len(self.failed) > 0:
            table = Table(style="red", box=rich.box.MINIMAL, pad_edge=False, border_style="dim")
            table.add_column(f"{self.component_type[:-1].title()} name", width=max_name_len)
            table.add_column("File path")
            table.add_column("Test message")
            table = format_result(self.failed, table)
            console.print(
                rich.panel.Panel(
                    table,
                    title=rf"[bold][✗] {len(self.failed)} {self.component_type[:-1].title()} Test{_s(self.failed)} Failed",
                    title_align="left",
                    style="red",
                    padding=0,
                )
            )

    def print_summary(self):
        """Print a summary table to the console."""
        table = Table(box=rich.box.ROUNDED)
        table.add_column("[bold green]LINT RESULTS SUMMARY", no_wrap=True)
        table.add_row(
            rf"[✔] {len(self.passed):>3} Test{_s(self.passed)} Passed",
            style="green",
        )
        table.add_row(rf"[!] {len(self.warned):>3} Test Warning{_s(self.warned)}", style="yellow")
        table.add_row(rf"[✗] {len(self.failed):>3} Test{_s(self.failed)} Failed", style="red")
        console.print(table)


from __future__ import print_function, division, absolute_import

import copy
from astropy import units as u
from marvin.utils.datamodel.maskbit import get_maskbits
from .base import DAPDataModel, Template, spaxel as spaxel_unit
from .base import Model, Channel, MultiChannelProperty
from .MPL6 import SPX, HYB10, VOR10, MPL6_maps, binid_properties
from .MPL6 import MPL6_emline_channels, oii_channel, oiid_channel

# update Template for MPL-8
MILESHC_MILESHC = Template('MILESHC-MILESHC',
                           description=('Set of templates derived from the MILES library, '
                                        'used for both the stellar kinematics and emission lines.'))

# new figure of merit channels
fom_channels = [
    Channel('rms', formats={'string': 'RMS of residuals for fitted pixels',
                            'latex': r'RMS of residuals'},
            description='RMS of the residuals for all fitted pixels in the stellar continuum.',
            unit=u.erg / u.s / (u.cm ** 2) / u.Angstrom / spaxel_unit, scale=1e-17, idx=0),
    Channel('frms', formats={'string': 'RMS of the fractional residuals',
                             'latex': r'RMS of the fractional residuals'},
            description='Fractional residuals for the stellar continuum fit.', idx=1),
    Channel('rchi2', formats={'string': 'Stellar continuum reduced chi-square',
                              'latex': r'Stellar\ continuum\ reduced\ \chi^2'},
            description='Reduced chi-square of the stellar continuum fit.', idx=2),
    Channel('fresid_68th_percentile', db_name='68th_perc_frac_resid',
            formats={'string': '68th percentile',
                     'latex': r'68^{th} percentile'},
            description='68%% growth of the fractional residuals between the model and data.', idx=3),
    Channel('fresid_99th_percentile', db_name='99th_perc_frac_resid',
            formats={'string': '99th percentile Frac. Residuals',
                     'latex': r'99^{th} percentile Frac. Residuals'},
            description='99%% growth of the fractional residuals between the model and data.', idx=4),
    Channel('fresid_max', db_name='max_frac_resid',
            formats={'string': 'Max Fractional Residual',
                     'latex': r'Max Fractional Residual'},
            description='Maximum growth of the fractional residuals between the model and data.', idx=5),
    Channel('per_pix_chi_68th_percentile', db_name='68th_perc_per_pix_chi',
            formats={'string': '68th percentile',
                     'latex': r'68^{th} percentile'},
            description='68%% growth of the error-normalized residuals', idx=6),
    Channel('per_pix_chi_99th_percentile', db_name='99th_perc_per_pix_chi',
            formats={'string': '99th percentile',
                     'latex': r'99^{th} percentile'},
            description='99%% growth of the error-normalized residuals', idx=7),
    Channel('per_pix_chi_max', db_name='max_per_pix_chi',
            formats={'string': 'Max Error-Normalized Residuals',
                     'latex': r'Max Error-Normalized Residuals'},
            description='Maximum growth of the error-normalized residuals', idx=8),
]

# new emission line channels
new_channels = [
    Channel('hei_3889', formats={'string': 'HeI 3889',
                                 'latex': r'$\forb{He\,I}\;\lambda 3889$'}, idx=22),
    Channel('ni_5199', formats={'string': 'NI 5199',
                                'latex': r'$\forb{N\,I}\;\lambda 5199$'}, idx=23),
    Channel('ni_5201', formats={'string': 'NI 5201',
                                'latex': r'$\forb{N\,I}\;\lambda 5201$'}, idx=24),
]

MPL8_emline_channels = copy.deepcopy(MPL6_emline_channels) + new_channels

new_properties = [
    # New Emission Line extensions
    MultiChannelProperty('emline_ga', ivar=False, mask=False,
                         channels=[oii_channel] + MPL8_emline_channels,
                         formats={'string': 'Amplitude of Fitted Gaussians'},
                         unit=u.erg / u.s / (u.cm ** 2) / u.Angstrom / spaxel_unit, scale=1e-17,
                         binid=binid_properties[3],
                         description='Amplitude of the fitted Gaussian emission lines.'),
    MultiChannelProperty('emline_ganr', ivar=False, mask=False,
                         channels=[oii_channel] + MPL8_emline_channels,
                         formats={'string': 'Amplitude Over Noise of Fitted Gaussians'},
                         binid=binid_properties[3],
                         description='Amplitude Over Noise of the fitted Gaussian emission lines.'),
    MultiChannelProperty('emline_lfom', ivar=False, mask=False,
                         channels=[oii_channel] + MPL8_emline_channels,
                         formats={'string': 'Emission Line Reduced Chi-Square'},
                         binid=binid_properties[3],
                         description='Reduced chi-square in 15 pixel windows around each fitted emission line'),
    # Figure of Merit Extensions in MAPS
    MultiChannelProperty('emline_fom', ivar=False, mask=False,
                         channels=fom_channels,
                         formats={'string': 'Emission Line FOM'},
                         unit=u.dimensionless_unscaled,
                         description='Full spectrum figures-of-merit for emission line fitting'),
    MultiChannelProperty('stellar_fom', ivar=False, mask=False,
                         channels=fom_channels, formats={
                             'string': 'Stellar FOM'},
                         unit=u.dimensionless_unscaled,
                         description='Full spectrum figures-of-merit for stellar continuum fitting'),
    # New Stellar SigmaCorr
    MultiChannelProperty('stellar_sigmacorr', ivar=False, mask=False,
                         channels=[Channel('resolution_difference',
                                           formats={'string': 'Resolution Difference'}, idx=0),
                                   Channel('fit',
                                           formats={'string': 'Fit'}, idx=1)],
                         unit=u.km / u.s,
                         formats={'string': 'Stellar sigma correction',
                                  'latex': r'Stellar $\sigma$ correction'},
                         description='Quadrature correction for STELLAR_SIGMA to obtain the '
                                     'astrophysical velocity dispersion.'),
]

emline_props = [
    MultiChannelProperty('emline_sflux', ivar=True, mask=True,
                         channels=[oiid_channel] + MPL8_emline_channels,
                         formats={'string': 'Emission line summed flux'},
                         unit=u.erg / u.s / (u.cm ** 2) / spaxel_unit, scale=1e-17,
                         binid=binid_properties[3],
                         description='Non-parametric summed flux for emission lines.'),
    MultiChannelProperty('emline_sew', ivar=True, mask=True,
                         channels=[oiid_channel] + MPL8_emline_channels,
                         formats={'string': 'Emission line EW'},
                         unit=u.Angstrom,
                         binid=binid_properties[3],
                         description='Emission line non-parametric equivalent '
                                     'widths measurements.'),
    MultiChannelProperty('emline_gflux', ivar=True, mask=True,
                         channels=[oii_channel] + MPL8_emline_channels,
                         formats={'string': 'Emission line Gaussian flux'},
                         unit=u.erg / u.s / (u.cm ** 2) / spaxel_unit, scale=1e-17,
                         binid=binid_properties[3],
                         description='Gaussian profile integrated flux for emission lines.'),
    MultiChannelProperty('emline_gvel', ivar=True, mask=True,
                         channels=[oii_channel] + MPL8_emline_channels,
                         formats={'string': 'Emission line Gaussian velocity'},
                         unit=u.km / u.s,
                         binid=binid_properties[3],
                         description='Gaussian profile velocity for emission lines.'),
    MultiChannelProperty('emline_gew', ivar=True, mask=True,
                         channels=[oii_channel] + MPL8_emline_channels,
                         formats={'string': 'Emission line Gaussian EW'},
                         unit=u.Angstrom,
                         binid=binid_properties[3],
                         description='Gaussian-fitted equivalent widths measurements '
                                     '(based on EMLINE_GFLUX).'),
    MultiChannelProperty('emline_gsigma', ivar=True, mask=True,
                         channels=[oii_channel] + MPL8_emline_channels,
                         formats={'string': 'Emission line Gaussian sigma',
                                  'latex': r'Emission line Gaussian $\sigma$'},
                         unit=u.km / u.s,
                         binid=binid_properties[3],
                         description='Gaussian profile velocity dispersion for emission lines; '
                                     'must be corrected using EMLINE_INSTSIGMA.'),
    MultiChannelProperty('emline_instsigma', ivar=False, mask=False,
                         channels=[oii_channel] + MPL8_emline_channels,
                         formats={'string': 'Emission line instrumental sigma',
                                  'latex': r'Emission line instrumental $\sigma$'},
                         unit=u.km / u.s,
                         binid=binid_properties[3],
                         description='Instrumental dispersion at the fitted line center.'),
    MultiChannelProperty('emline_tplsigma', ivar=False, mask=False,
                         channels=[oii_channel] + MPL8_emline_channels,
                         formats={'string': 'Emission line template instrumental sigma',
                                  'latex': r'Emission line template instrumental $\sigma$'},
                         unit=u.km / u.s,
                         binid=binid_properties[3],
                         description='The dispersion of each emission line used in '
                                     'the template spectra'),
]

# Update Maps Properties
remove = ['stellar_cont_rchi2', 'stellar_sigmacorr', 'stellar_cont_fresid']
MPL8_maps = [m for m in MPL6_maps if m.name not in remove and 'emline' not in m.name]
MPL8_maps += emline_props + new_properties

# Update Model Properties
MPL8_models = [
    Model('binned_flux', 'FLUX', 'WAVE', extension_ivar='IVAR',
          extension_mask='MASK', unit=u.erg / u.s / (u.cm ** 2) / u.Angstrom / spaxel_unit,
          scale=1e-17, formats={'string': 'Binned flux'},
          description='Flux of the binned spectra',
          binid=binid_properties[0]),
    Model('full_fit', 'MODEL', 'WAVE', extension_ivar=None,
          extension_mask='MODEL_MASK', unit=u.erg / u.s / (u.cm ** 2) / u.Angstrom / spaxel_unit,
          scale=1e-17, formats={'string': 'Best fitting model'},
          description='The best fitting model spectra (sum of the fitted '
                      'continuum and emission-line models)',
          binid=binid_properties[0]),
    Model('emline_fit', 'EMLINE', 'WAVE', extension_ivar=None,
          extension_mask=None,
          unit=u.erg / u.s / (u.cm ** 2) / u.Angstrom / spaxel_unit,
          scale=1e-17, formats={'string': 'Emission line model spectrum'},
          description='The model spectrum with only the emission lines.',
          binid=binid_properties[3]),
    Model('stellar_fit', 'STELLAR', 'WAVE', extension_ivar=None,
          extension_mask='STELLAR_MASK',
          unit=u.erg / u.s / (u.cm ** 2) / u.Angstrom / spaxel_unit,
          scale=1e-17, formats={'string': 'Stellar continuum model spectrum'},
          description='The model spectrum with only the stellar continuum',
          binid=binid_properties[1])
]

# MPL-8 DapDataModel goes here
MPL8 = DAPDataModel('2.3.0', aliases=['MPL-8', 'MPL8'],
                    bintypes=[SPX, HYB10, VOR10],
                    db_only=[HYB10],
                    templates=[MILESHC_MILESHC],
                    properties=MPL8_maps,
                    models=MPL8_models,
                    bitmasks=get_maskbits('MPL-8'),
                    default_bintype='HYB10',
                    default_template='MILESHC-MILESHC',
                    property_table='SpaxelProp8',
                    default_binid=copy.deepcopy(binid_properties[0]),
                    default_mapmask=['NOCOV', 'UNRELIABLE', 'DONOTUSE'],
                    qual_flag='DAPQUAL')


import post_lib
import json

def relay(data):
    if 'type' in data.keys():
        if data['type'] == 'request':
            return json.dumps({"type":"relay","relay":[]})
        else:
            return False
    else:
        return False

if __name__=="__main__":
    server = post_lib.Server(host="127.0.0.1",port=5000,debug=True)
    server.add_response(relay)
    server.run()


from sqlalchemy.orm import Session

from app.models.roaster import Roaster
from app.schemas.roaster import RoasterCreate

def get_roasters(db: Session):
    return db.query(Roaster).all()

def create_roaster(db: Session, roaster: RoasterCreate):
    db_roaster = Roaster(
        name=roaster.name,
        url=roaster.url,
        province=roaster.province,
        country=roaster.country,
    )
    db.add(db_roaster)
    db.commit()
    db.refresh(db_roaster)
    return db_roaster


# Gui
import PySimpleGUI as sg
import sys

# Data analysis
import numpy as np

# Loading images
import cv2
import glob
import os

# Results
import pandas as pd
import matplotlib.pyplot as plt

# Report
from fpdf import FPDF

########################################################################################################################

#############
# File Path #
#############

print("Launching GUI...")

## Gui input ##
sg.theme('DarkAmber')

# GUI layout
layout = [[sg.Text('Select the data folder + operator(s)')],
          [sg.Text('Data folder:', size=(14, 1)), sg.In(key="datadir"), sg.FolderBrowse()],
          [sg.Text('')],
          [sg.Text('Operator 1:', size=(14, 1)), sg.In(key="op1", size=(5, 1)),
           sg.Text('      Operator 2:', size=(14, 1)), sg.In(key="op2", size=(5, 1))],
          [sg.Text('Comments:', size=(14, 1)), sg.In(key="comment")],
          [sg.Text('')],
          [sg.Button('Ok'), sg.Button('Cancel')]]

# Create the Window
window = sg.Window('kV Field Size Analysis', layout)

while True:
    event, values = window.read()

    if event == 'Cancel':
        sys.exit()

    # Data folder path
    path = list(values.items())[0][1]
    data_path = path.replace('/', '\\\\')
    measurement = data_path + "\\\\*.bmp"

    # Operators
    op1 = list(values.items())[-3][1]
    op2 = list(values.items())[-2][1]
    operator = [op1, op2]

    # Comments
    comment = list(values.items())[-1][1]

    if event == 'Ok':
        test = [] # dummy array to test filename
        checker = 0 # dummy value for error popup window

        for fname in glob.glob(measurement): # filling dummy array with current filename within folder
            test.append(os.path.basename(fname)[:-4])

        for i in test: # check if filename matches XXXXXX_XX_X
            if i[6] == "_" and i[9] == "_":
                continue
            else:
                checker = 1

        if checker == 0: # error popup window trigger
            break
        else:
            sg.popup("Check filename matches the following convention: YYMMDD_GX_X. Press OK after correction!")

        break

window.close()

## Save path ##
graph = data_path + "\\\\kV Field Size({}).png"

########################################################################################################################

#############
# Tolerance #
#############
# Values from XRV-4000 active script
p2mm = 1 / ((3.6090 + 3.6140) / 2)
laser_x = 799.6364
laser_y = 601.8524

# kV panel size at isocentre
AB = 2700 * np.tan(np.arctan(200 / 3700))  # unit:mm
GT = 2700 * np.tan(np.arctan(150 / 3700))

# Panel size at iso in px
PA = round(laser_x - AB/p2mm) # unit:px
PB = round(AB/p2mm + laser_x)
PG = round(laser_y - GT/p2mm)
PT = round(GT/p2mm + laser_y)

# Tolerance
limit = 27 # IPEM report 91 suggested limit at 2.7 m source to detector distance
VA = round(laser_x - (AB + limit) / p2mm)  # unit:px
VB = round((AB + limit) / p2mm + laser_x)
VG = round(laser_y - (GT + limit) / p2mm)
VT = round((GT + limit) / p2mm + laser_y)

########################################################################################################################

####################
# Threshold Method #
####################
def find_v_edge(image):
    """
    This function will take an an 8 bit grayscale image and output a
    threshold image. It will also find two pairs of arrays that define the front
    and rear vertical edges of the kV field.

    input:
        image:         input image, where image must be an 8 bit grayscale image

    output:
        thresh_img:    array for the binary threshold image created by Otsu's method

        front_v_edge:  array of the horizontal pixel index position of the front
                       vertical edge

        rear_v_edge:   array of the horizontal pixel index position of the rear
                       vertical edge

        front_pos:     array of the vertical pixel index position of the front
                       vertical edge

        rear_pos:      array of the vertical pixel index position of the rear
                       vertical edge
    """

    ###############
    # Otsu Method #
    ###############
    # Normalise image
    image = image / image.max()
    image = image * 255
    image = np.round(image)

    # Create binary image using Otsu's method
    ret, thresh_img = cv2.threshold(image.astype('uint8'), 0, 255,
                                    cv2.THRESH_BINARY + cv2.THRESH_OTSU)
    ##################
    # Edge detection #
    ##################
    front_v_edge = []  # empty array for the vertical front edge (horizontal position)
    rear_v_edge = []  # empty array for the vertical rear edge (horizontal position)

    # Finding all points on the vertical edges of the threshold image
    for i in range(thresh_img.shape[0]):  # going through the 1200 row
        cal = []
        if np.average(thresh_img[i]) == 0:  # returning zeroes if no bright pixels could be found
            front_v_edge.append(0)
            rear_v_edge.append(0)
        else:
            cal = np.nonzero(thresh_img[i])  # finding all bright pixels within the row
            front_v_edge.append(cal[0][0])  # appending the first bright pixel
            rear_v_edge.append(cal[0][len(cal[0]) - 1])  # appending the last bright pixel

    # finding the upper bound of the front vertical edge
    for i in range(len(front_v_edge)):
        if front_v_edge[i] == 0:
            continue
        else:
            upper_front = i
            break
    # finding the lower bound of the front vertical edge
    for i in reversed(range(len(front_v_edge))):
        if rear_v_edge[i] == 0:
            continue
        else:
            lower_front = i + 1
            break
    # finding the upper bound of the rear vertical edge
    for i in range(len(rear_v_edge)):
        if rear_v_edge[i] == 0:
            continue
        else:
            upper_rear = i
            break
    # finding the lower bound of the rear vertical edge
    for i in reversed(range(len(rear_v_edge))):
        if rear_v_edge[i] == 0:
            continue
        else:
            lower_rear = i + 1
            # Final arrays of the front and rear edge
            front_v_edge = front_v_edge[upper_front:lower_front]
            rear_v_edge = rear_v_edge[upper_rear:lower_rear]

            # Arrays of the position of the edges
            front_pos = np.arange(upper_front, lower_front)
            rear_pos = np.arange(upper_rear, lower_rear)
            break

    return thresh_img, front_v_edge, rear_v_edge, front_pos, rear_pos

########################################################################################################################

###################
# Gradient Method #
###################
def gradient_images(blur_img, kernel_size):
    """
    This function will take a grayscale image and output arrays of the pixel
    intensity first and second derivatives.

    input:
        blur_img:     input image, where image must be 2D numpy array

        kernel_size:  size of kernel used to calculate image gradient

    output:
        dx:         horizontal gradient array

        dy:         vertical gradient array

        grad_img:   magnitude gradient array

        ddx:        horizontal second derivative array (gradient of gradient)

        ddy:        vertical second derivative arrayd

        dgrad_img:  magnitude of second derivative array
    """

    step = kernel_size // 2
    # array subtraction for first differential
    dy = np.zeros(blur_img.shape)
    dy[:, step:-step, :] = blur_img[:, kernel_size:, :] - blur_img[:, :-kernel_size, :]
    dx = np.zeros(blur_img.shape)
    dx[:, :, step:-step] = blur_img[:, :, kernel_size:] - blur_img[:, :, :-kernel_size]
    grad_img = np.sqrt(dx ** 2 + dy ** 2)  # calculate magnitude of gradients

    # array subtraction for second differential
    ddy = np.zeros(blur_img.shape)
    ddy[:, step:-step, :] = dy[:, kernel_size:, :] - dy[:, :-kernel_size, :]
    ddx = np.zeros(blur_img.shape)
    ddx[:, :, step:-step] = dx[:, :, kernel_size:] - dx[:, :, :-kernel_size]
    dgrad_img = np.sqrt(ddx ** 2 + ddy ** 2)  # calculate magnitude of gradients

    return dx, dy, grad_img, ddx, ddy, dgrad_img

########################################################################################################################

########
# Data #
########
img = [] # raw image array
name = [] # labels for the image

# find all files and load them into img using a for loop
for fname in glob.glob(measurement):
    # labeling files
    name.append(os.path.basename(fname)[:-4])
    # reading files
    img.append(cv2.imread(fname, cv2.IMREAD_GRAYSCALE))
# convert the list into an array
img = np.array(img, dtype='float32')

# Flipping all array A images
for img_no in np.arange(len(img)):
    flip = name[img_no]
    if flip.find("A") != -1:
        img[img_no] = np.flip(img[img_no])
    else:
        continue

# Applying Gaussian blur to all image
blur_img = [] # create an empty list`
for i in range(img.shape[0]): # lopo through images
    blur_img.append(cv2.GaussianBlur(img[i, :, :], (21, 21), 0)) # blur with a 21x21 pixel kernel
blur_img = np.array(blur_img)

########################################################################################################################

####################
# Threshold Method #
####################

# pixel indices
A_pxField = []
B_pxField = []
G_pxField = []
T_pxField = []

for img_no in np.arange(len(blur_img)):

    result, x_front, x_rear, y_front, y_rear = find_v_edge(blur_img[img_no])

    ##############################################################################
    # Slicing the vertical edge array to avoid overstreching, indices were found using visual inspection of data
    x_front = x_front[20:-20]
    y_front = y_front[20:-20]
    x_rear = x_rear[20:-20]
    y_rear = y_rear[20:-20]
    ##############################################################################
    # Finding the horizontal edges
    front_h_edge = []
    rear_h_edge = []

    # Finding all points on the horizontal edges of the thershold image
    result = result.transpose()
    for i in range(result.shape[0]):  # going through the 1600 columns
        cal = []
        if np.average(result[i]) == 0:  # returning zeroes if no bright pixels could be found
            front_h_edge.append(0)
            rear_h_edge.append(0)
        else:
            cal = np.nonzero(result[i])  # finding all bright pixels within the row
            front_h_edge.append(cal[0][0])  # appending the first bright pixel
            rear_h_edge.append(cal[0][len(cal[0]) - 1])  # appending the last bright pixel
    ##############################################################################
    # Defining the bottom edge using the vertical edges
    x_bottom = np.arange(x_front[0] + 1, x_rear[0])
    y_bottom = front_h_edge[x_bottom[0] - 1:x_bottom[-1]]

    # Defining the top edge using the vertical edges
    x_top = np.arange(x_front[-1] + 1, x_rear[-1])
    y_top = rear_h_edge[x_top[0] - 1:x_top[-1]]
    ##############################################################################
    # Binning the edges
    A_pxField.append(round(np.average(x_front)))
    B_pxField.append(round(np.average(x_rear)))
    G_pxField.append(round(np.average(y_bottom)))
    T_pxField.append(round(np.average(y_top)))

# mm size
A_mmField = []
B_mmField = []
G_mmField = []
T_mmField = []

for i in np.arange(len(A_pxField)):
    A_mmField.append(round((laser_x - A_pxField[i]) * p2mm))
    B_mmField.append(round((B_pxField[i] - laser_x) * p2mm))
    G_mmField.append(round((laser_y - G_pxField[i]) * p2mm))
    T_mmField.append(round((T_pxField[i] - laser_y) * p2mm))

########################################################################################################################

###################
# Gradient Method #
###################
# field edge location datasets
a_pk = []
b_pk = []
g_pk = []
t_pk = []

# Loop through all kV images
kernel_size = 40
for i in range(blur_img.shape[0]):
    # calculate 2nd derivative gradient images
    _, _, _, ddx, ddy, _ = gradient_images(blur_img, kernel_size)

    # manually subtract unwanted image features, i.e. BBs and noisy border
    # null BB rows & cols
    ddx[:, 550:670, :] = 0  # np.nan # horizontal null
    ddx[:, :, 750:870] = 0  # np.nan # vertical null
    ddy[:, 550:670, :] = 0  # np.nan # horizontal null
    ddy[:, :, 750:870] = 0  # np.nan # vertical null
    # null FoV rows & cols
    ddx[:, :, :50] = 0  # np.nan # horizontal null
    ddx[:, :, -51:] = 0  # np.nan # vertical null
    ddx[:, :50, :] = 0  # np.nan # horizontal null
    ddx[:, -51:, :] = 0  # np.nan # vertical null
    ddy[:, :, :50] = 0  # np.nan # horizontal null
    ddy[:, :, -51:] = 0  # np.nan # vertical null
    ddy[:, :50, :] = 0  # np.nan # horizontal null
    ddy[:, -51:, :] = 0  # np.nan # vertical null

    # locate the minima in each quadrant
    x_profile = np.median(ddx[i, ...], axis=0)
    y_profile = np.median(ddy[i, ...], axis=-1)
    a_pk.append(round(np.argmin(x_profile[:801])))
    b_pk.append(round(np.argmin(x_profile[800:]) + 800))
    g_pk.append(round(np.argmin(y_profile[:601])))
    t_pk.append(round(np.argmin(y_profile[600:]) + 600))

# mm size
a_mmpk = []
b_mmpk = []
g_mmpk = []
t_mmpk = []

for i in np.arange(len(a_pk)):
    a_mmpk.append(round((laser_x - a_pk[i]) * p2mm))
    b_mmpk.append(round((b_pk[i] - laser_x) * p2mm))
    g_mmpk.append(round((laser_y - g_pk[i]) * p2mm))
    t_mmpk.append(round((t_pk[i] - laser_y) * p2mm))

########################################################################################################################

################
# Result (CSV) #
################

table = data_path + "\\\\{}_{}_Result Table.csv".format(name[0][7:9], name[0][0:6]) # table file name

# Result table layout
if np.size(A_mmField) == 1:
    data = {"Side": ["A", "B", "G", "T"],
            "Threshold ({})".format(name[0][-1]): [A_mmField[0], B_mmField[0], G_mmField[0], T_mmField[0]],
            "Gradient ({})".format(name[0][-1]): [a_mmpk[0], b_mmpk[0], g_mmpk[0], t_mmpk[0]],
            }

else:
    data = {"Side": ["A", "B", "G", "T"],
            "Threshold ({})".format(name[0][-1]): [A_mmField[0], B_mmField[0], G_mmField[0], T_mmField[0]],
            "Threshold ({})".format(name[1][-1]): [A_mmField[1], B_mmField[1], G_mmField[1], T_mmField[1]],
            "Gradient ({})".format(name[0][-1]): [a_mmpk[0], b_mmpk[0], g_mmpk[0], t_mmpk[0]],
            "Gradient ({})".format(name[1][-1]): [a_mmpk[1], b_mmpk[1], g_mmpk[1], t_mmpk[1]]
            }

# Save table as CSV
df = pd.DataFrame(data)
df.to_csv(table, index=False, header=True)

print("Results CSV Generated!")

################
# Result Graph #
################

i = 0
while i < np.size(A_pxField):
    ## FOV ##
    # AB
    PAB_y = np.arange(1200)[PG:PT + 1]
    PA_x = np.ones(np.size(PAB_y)) * PA
    PB_x = np.ones(np.size(PAB_y)) * PB

    # GT
    PGT_x = np.arange(1600)[PA:PB + 1]
    PG_y = np.ones(np.size(PGT_x)) * PG
    PT_y = np.ones(np.size(PGT_x)) * PT

    ## Tolerance ##
    # AB
    VAB_y = np.arange(1200)[VG:VT + 1]
    VA_x = np.ones(np.size(VAB_y)) * VA
    VB_x = np.ones(np.size(VAB_y)) * VB

    # GT
    VGT_x = np.arange(1600)[VA:VB + 1]
    VG_y = np.ones(np.size(VGT_x)) * VG
    VT_y = np.ones(np.size(VGT_x)) * VT

    ## Threshold ##
    # AB
    AB_thre_y = np.arange(1200)[G_pxField[i]:T_pxField[i] + 1]
    A_thre_x = np.ones(np.size(AB_thre_y)) * A_pxField[i]
    B_thre_x = np.ones(np.size(AB_thre_y)) * B_pxField[i]

    # GT
    GT_thre_x = np.arange(1600)[A_pxField[i]:B_pxField[i] + 1]
    G_thre_y = np.ones(np.size(GT_thre_x)) * G_pxField[i]
    T_thre_y = np.ones(np.size(GT_thre_x)) * T_pxField[i]

    ## Gradient ##
    # AB
    AB_grad_y = np.arange(1200)[g_pk[i]:t_pk[i] + 1]
    A_grad_x = np.ones(np.size(AB_grad_y)) * a_pk[i]
    B_grad_x = np.ones(np.size(AB_grad_y)) * b_pk[i]

    # GT
    GT_grad_x = np.arange(1600)[a_pk[i]:b_pk[i] + 1]
    G_grad_y = np.ones(np.size(GT_grad_x)) * g_pk[i]
    T_grad_y = np.ones(np.size(GT_grad_x)) * t_pk[i]

    ## plot ##
    plt.figure(figsize=(10, 5))
    # kV image
    plt.imshow(blur_img[i, :, :], cmap='gray')

    # Panel size at isocentre
    plt.plot(PA_x, PAB_y, color='r', linestyle='-.', label='Field of View')
    plt.plot(PB_x, PAB_y, color='r', linestyle='-.')
    plt.plot(PGT_x, PG_y, color='r', linestyle='-.')
    plt.plot(PGT_x, PT_y, color='r', linestyle='-.')

    # Tolerance
    plt.plot(VA_x, VAB_y, color='r', linestyle=':', label='Limit')
    plt.plot(VB_x, VAB_y, color='r', linestyle=':')
    plt.plot(VGT_x, VG_y, color='r', linestyle=':')
    plt.plot(VGT_x, VT_y, color='r', linestyle=':')

    # threshold
    plt.plot(A_thre_x, AB_thre_y, color='b', linestyle='--', label='Threshold Method')
    plt.plot(B_thre_x, AB_thre_y, color='b', linestyle='--')
    plt.plot(GT_thre_x, G_thre_y, color='b', linestyle='--')
    plt.plot(GT_thre_x, T_thre_y, color='b', linestyle='--')

    # gradient
    plt.plot(A_grad_x, AB_grad_y, color='c', linestyle='--', label='Gradient Method')
    plt.plot(B_grad_x, AB_grad_y, color='c', linestyle='--')
    plt.plot(GT_grad_x, G_grad_y, color='c', linestyle='--')
    plt.plot(GT_grad_x, T_grad_y, color='c', linestyle='--')

    # graph setting
    plt.title("kV field size for Imaging System {}".format((name[i][10:11])))
    plt.xlabel("AB axis / px")
    plt.ylabel("TG axis / px")
    plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))

    plt.savefig(graph.format((name[i][10:11])))

    i = i + 1

print("Results Images Generated!")

########################################################################################################################

##########
# Report #
##########

report = data_path + "\\\\{}_{}_Report.pdf".format(name[0][7:9], name[0][0:6]) # report file name

pdf = FPDF()
pdf.add_page()

## Title ##
pdf.set_font('Arial', 'B', 16)
pdf.cell(30, 10, 'kV Field Size Report', 'C')
pdf.ln(15)

## Session Summary ##
pdf.set_font('Arial', 'BU', 11)
pdf.cell(30, 10, 'Session Summary', 'C') # title
pdf.ln(7)

pdf.set_font('Arial', '', 10)
pdf.cell(30, 10, 'Data acquired on: {}/{}/20{}'.format(name[0][4:6], name[0][2:4], name[0][0:2]), 'C') # date
pdf.ln(7)
pdf.cell(30, 10, 'Location: Gantry {}'.format(name[0][8:9]), 'C') # gantry
pdf.ln(7)
pdf.cell(30, 10, 'Operator(s):  {} {}'.format(operator[0], operator[1]), 'C') # operator
pdf.ln(7)
pdf.cell(30, 10, 'Comments:  {}'.format(comment), 'C') # operator
pdf.ln(12)

## Tables ##
pdf.set_font('Arial', 'BU', 11)
pdf.cell(30, 10, 'Results', 'C') # title
pdf.ln(10)

# Header
if np.size(A_mmField) == 1:
    heading = ['Side', 'Threshold ({}) / mm'.format(name[0][-1]), 'Gradient ({}) / mm'.format(name[0][-1])]
    pdf.set_font('Arial', '', 10)
else:
    heading = ['Side', 'Threshold (A) / mm', 'Threshold (B) / mm', 'Gradient (A) / mm', 'Gradient (B) / mm']
    pdf.set_font('Arial', '', 10)

for i in heading:
    if i == "Side":
        pdf.cell(12, 5, str(i), align='C', border=1) # smaller cell for the first column
    else:
        pdf.cell(33, 5, str(i), align='C', border=1)
pdf.ln(5)

# Results
result_table = df.values.tolist()

for row in result_table:
    for datum in row:
        if type(datum) is str:
            pdf.cell(12, 5, str(datum), align='C', border=1) # smaller cell for the first column
        else:
            pdf.cell(33, 5, str(datum), align='C', border=1)
    pdf.ln(5)

## Graphs ##
pdf.ln(5)
for i in np.arange(len(name)):
    pdf.image(graph.format((name[i][10:11])), x=-10, h=85)

pdf.output(report, 'F')

print("Report Generated!")

input("Press Enter To Continue...")


#!/usr/bin/env python

"""
This code will enumerate the surface of the input slab model with user defined
target species and composition.
For the detailed algorithm behind the enumeration, please see the
following references below.
(1) Morgan, W. S.; Hart, G. L. W.; Forcade, R. W.
Computational Materials Science 2017, 136, 144–149.
https://doi.org/10.1016/j.commatsci.2017.04.015.
(2) Hart, G. L. W.; Nelson, L. J.; Forcade, R. W.
Computational Materials Science 2012, 59, 101–107.
https://doi.org/10.1016/j.commatsci.2012.02.015.
(3) Hart, G. L. W.; Forcade, R. W.
Phys. Rev. B 2008, 77 (22), 224115.
https://doi.org/10.1103/PhysRevB.77.224115.
(4) Hart, G. L. W.; Forcade, R. W.
Phys. Rev. B 2009, 80 (1), 014120.
https://doi.org/10.1103/PhysRevB.80.014120.

"""
__author__ = "Xinhao Li"
__email__ = "xinhao.li@columbia.edu"
__date__ = "2023-08-23"
__version__ = '0.1.0'

import os
import json
import copy
import argparse
import warnings

from itertools import product

from pymatgen.symmetry.analyzer import SpacegroupAnalyzer
from pymatgen.core.periodic_table import DummySpecies
from pymatgen.core.surface import get_slab_regions

from surface_pd.core import Slab, EnumWithComposition, PreCheck, PostCheck
from surface_pd.analysis.slab_analysis import (structure_filter,
                                               selective_dynamics_completion)
from surface_pd.util import (get_values_nested_dict,
                             all_int, replace_dummy, have_zero)
from surface_pd.error import *

warnings.filterwarnings("ignore")

def automate_surface(target_slab_path: str,
                     species: list,
                     replace: list,
                     num_layers_enumed: dict,
                     max_cell_size: int,
                     symmetric: bool,
                     to_vasp: bool = False):
    """
    This function contains the general framework to enumerate the parent
    slab model surface with different target species and compositions.
    TODO:
        1. Check the meaning of num_layers_enumed. Is this really the number of
        layers enumed or number of layers relaxed? Can the number of layers
        enumed be different than the number of layers relaxed?
        2. symmetric parameter: If turned off, is the only top surface
        enumerated?
        4. Check enumlib code surface enumeration and see how difficult it
        is to implement.
    Args:
        target_slab_path: Input slab model
        species: Target species on the surface that will be enumerated.
        replace: Species and occupancy dictionaries containing the species
            mapping in string-string pairs. E.g. {'Li': {'Li': 0.5}}
            (This dict means that only half of the original Li
            atoms in the slab model will be kept), stored in the list.
        num_layers_enumed: Dictionaries where species and number of relaxed
            layers as keys and values, respectively. E.g. {'Li': 2, 'O': 1},
            which represents 2 and 1 layers of Li and O will be relaxed
            during the DFT geometry optimization.
        max_cell_size: Maximum number of supercells of the input slab.
        symmetric: Whether the symmetry of the slab model will be kept
            after the enumeration.
        to_vasp: Whether to generate all the output slab models in VASP
        format. Defaults to False.

    Returns:
        All enumerated slab models with different composition of target
        species.

    """

    # Load parent slab model
    input_structure = Slab.from_file(target_slab_path)

    # Wrap out of the boundary fractional coordinates back into the unit cell.
    input_structure.wrap_pbc()

    # Define important parameters which will be used later, including to be
    # enumerated species, number of layers relaxed, whether to keep the
    # symmetry, direction of the slab, and the proximity tolerance for
    # adjacent atoms
    input_structure.to_be_enumerated_species = species
    input_structure.num_layers_enumed = num_layers_enumed
    input_structure.symmetric = symmetric
    direction = input_structure.direction
    tolerance = input_structure.tolerance

    # PreCheck
    pre_check = PreCheck(input_structure)

    if not pre_check.is_cuboid():
        raise SlabOrientationError

    if not pre_check.is_slab():
        raise NonSlabError

    if not pre_check.all_has_selective_dynamics():
        raise NonDefinedSelectiveDynamicsError

    if pre_check.relax_both_surfaces() != input_structure.symmetric:
        raise IncompatibleSymmError

    if symmetric:
        if not pre_check.has_inversion_symmetry():
            raise NoInversionSymmetryError

    # Check the user defined to-be-enumerated composition is valid
    if not pre_check.has_validate_composition(replace, max_cell_size):
        raise InvalidCompositionError

    # Extract the lattice parameter of the parent slab model along the slab
    # direction defined by user (will be used as criteria next)
    criteria = input_structure.lattice.abc[direction]

    # Get the indices of the to-be-enumerated atoms on the surface.
    # Get the c_frac coordinates of the lower and upper boundaries of the
    # fixed region in the central slab.
    center_bottom, center_top, relaxed_index = \
        input_structure.index_extraction(only_top=False)

    # Define the "dummy" species that will be used to substitute the target
    # species
    dummy_species = [DummySpecies(symbol='X' + str(i + 1))
                     for i in range(len(species))]

    # Replace the to-be-enumerated species from the slab top surface with
    # "dummy" species. This will facilitate the enumeration code.
    input_substituted = input_structure.surface_substitute(dummy_species)

    # Initialize a number to store total number of enumerated slab models
    num = 0

    # Begin to enumerate
    for subs_dict in replace:
        slab_substituted = copy.deepcopy(input_substituted)

        prev = 0
        supplemental_structures = []

        composition_list = list(get_values_nested_dict(subs_dict))
        if all_int(composition_list):
            structure = input_structure.supplemental_structures_gene(
                subs_dict, relaxed_index)
            supplemental_structures = [structure]
            enumerated_structures = [structure]
        else:
            if have_zero(composition_list):
                slab_substituted = slab_substituted \
                    .supplemental_structures_gene(subs_dict, relaxed_index)

            subs_dict = replace_dummy(subs_dict, dummy_species)

            ewc = EnumWithComposition(
                subs_dict=subs_dict,
                max_cell_size=max_cell_size
            )

            structures = ewc.apply_enumeration(slab_substituted)

            # Filtered out the structures which has c lattice as the
            # largest lattice (a tall cuboid)
            filtered_structures = structure_filter(structures,
                                                   direction=direction,
                                                   criteria=criteria)

            # Add selective dynamics for enumerated sites
            for filtered_structure in filtered_structures:
                selective_dynamics_completion(
                    structure=filtered_structure,
                    direction=direction,
                    dummy_species=dummy_species,
                    center_bottom=center_bottom,
                    center_top=center_top,
                    tolerance=tolerance
                )

            # Symmetrize slab models based on the top enumerated
            # surface if needed
            enumerated_structures = []
            for filtered_structure in filtered_structures:
                for i in range(len(species)):
                    filtered_structure.replace_species(
                        {dummy_species[i]: species[i]})
                filtered_structure = Slab.from_sites(filtered_structure)
                if symmetric:
                    enumerated_structures.append(
                        filtered_structure.symmetrize_top_base())
                else:
                    enumerated_structures.append(filtered_structure)

            prev = len(enumerated_structures)

        unique_index = "-{:0%dd}" % (len(str(len(enumerated_structures))))
        for k, symmetrized_structure in enumerate(enumerated_structures):
            indicator = 0
            if symmetric:
                _, origin, _ = symmetrized_structure.is_symmetry(
                    return_isc=True)
                min_c, _ = symmetrized_structure.get_max_min_c_frac()
                sga = SpacegroupAnalyzer(symmetrized_structure,
                                         symprec=0.1)
                # Create the refined structures
                refined_structure = sga.get_refined_structure()

                # Define number of sites that should be after enumeration
                total_num_sites = input_structure.calculate_num_sites(
                    composition_list=composition_list,
                    relaxed_index=relaxed_index,
                    max_cell_size=max_cell_size)

                num_sites = refined_structure.num_sites

                while ((num_sites > total_num_sites) or
                       (max(refined_structure.lattice.abc) >
                        criteria * 2 - 5) or
                       (max(refined_structure.lattice.abc) !=
                        refined_structure.lattice.abc[direction])):
                    pc = PostCheck(refined_structure)
                    indicator, refined_structure = pc.slab_size_check(
                        total_num_sites=total_num_sites,
                        enumerated_num_sites=num_sites,
                        criteria=criteria)
                    num_sites = refined_structure.num_sites

                refined_structure = Slab.from_sites(refined_structure)
                noncontiguous = (len(get_slab_regions(refined_structure)) == 2)
                # print(input_structure.layers_finder())

                if noncontiguous:
                    refined_structure = refined_structure.tune_isc(
                        origin=origin, shift_isc_back=True)
                    refined_structure = refined_structure.tune_c(
                        target_min_c=min_c)
                    # print(refined_structure.layers_finder())
                    refined_structure = \
                        refined_structure.add_selective_dynamics(
                            lower_limit=center_bottom,
                            upper_limit=center_top
                        )
                else:
                    refined_structure = refined_structure.tune_c(
                        target_min_c=min_c)
                    # print(refined_structure.layers_finder())
                    refined_structure = \
                        refined_structure.add_selective_dynamics(
                            lower_limit=center_bottom,
                            upper_limit=center_top
                        )

                refined_structure = refined_structure.tune_isc(
                    origin=origin, shift_isc_back=False)
            else:
                refined_structure = symmetrized_structure

            # Perform final check
            pc = PostCheck(refined_structure)
            pc.final_check(
                species=species,
                composition_list=composition_list,
                keep_symmetric=symmetric,
                criteria=criteria,
                index=k)

            # Generate slab models
            if to_vasp:
                # Make directories
                dirname = ''
                for n in range(len(species)):
                    dirname += str(composition_list[n]) + str(species[n])
                if not os.path.exists(dirname):
                    os.makedirs(dirname)

                if indicator == -1:
                    refined_structure.to(
                        fmt='poscar',
                        filename=os.path.join(
                            dirname,
                            "refined-prim{}.vasp".format(
                                unique_index.format(k))))
                elif indicator == 1:
                    refined_structure.to(
                        fmt='poscar',
                        filename=os.path.join(
                            dirname,
                            "refined-reduced{}.vasp".format(
                                unique_index.format(k))))
                elif indicator == 2:
                    refined_structure.to(
                        fmt='poscar',
                        filename=os.path.join(
                            dirname,
                            "refined-rotated{}.vasp".format(
                                unique_index.format(k))))
                else:
                    refined_structure.to(
                        fmt='poscar',
                        filename=os.path.join(
                            dirname,
                            "refined-structure{}.vasp".format(
                                unique_index.format(k))))

        num += len(enumerated_structures)
        print(
            'The enumeration found {}({}+{}) distinct structures for {} '
            'with {} composition.'.format(
                len(enumerated_structures), prev,
                len(supplemental_structures), species,
                composition_list))
    print(f'{num} distinct structures are found totally.')

def run(json_file_path, to_vasp):
    """
    Function to read the json file and load the parameters.

    Args:
        json_file_path: JSON file path direct to the file which includes
            all the important parameters.
        to_vasp: Whether to generate all output slab models.

    """

    # Load json input file
    try:
        with open(json_file_path) as fp:
            data = json.load(fp)
    except (json.decoder.JSONDecodeError, IsADirectoryError):
        raise InvalidInputFormatError

    # Generate all composition combo
    to_be_enumerated_species = list(data['replacements'])
    num_layers_enumed = data['num_layers_enumed']
    composition_list = list(
        get_values_nested_dict(data['replacements']))
    composition_list = [sorted(x, reverse=True) for x in
                        composition_list]

    print("target_cell_size = {}".format(data['max_cell_size']))
    print("Composition of {} on the surface will be {}, "
          "respectively.".format(to_be_enumerated_species, composition_list))

    composition_combo = []
    for combo in product(*composition_list):
        replace_dict = {}
        for key, value in zip(to_be_enumerated_species, combo):
            replace_dict[key] = {key: float(value)}
        composition_combo.append(replace_dict)

    automate_surface(target_slab_path=data['target_slab_path'],
                     species=to_be_enumerated_species,
                     replace=composition_combo,
                     num_layers_enumed=num_layers_enumed,
                     max_cell_size=data['max_cell_size'],
                     symmetric=data['symmetric'],
                     to_vasp=to_vasp)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description=__doc__ + "\n{} {}".format(__date__, __author__) +
                    "\n version: {}".format(__version__),
        formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument(
        "json_file_path",
        help="Path to a JSON file which has the necessary parameters defined.")

    parser.add_argument(
        "--generate-poscar", "-g",
        help="Generate POSCAR files of enumerated structures.",
        action="store_true")

    args = parser.parse_args()

    run(args.json_file_path,
        args.generate_poscar)


from __future__ import absolute_import, division, print_function
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
#logging errors
tf.logging.set_verbosity(tf.logging.ERROR)
# features/inputs
celsius_q = np.array([-40, -10, 0, 8, 15, 22, 38], dtype=float)
#labels/outputs
fahrenheit_a = np.array([-40, 14, 32, 46, 59, 72, 100], dtype=float)
for i, c in enumerate(celsius_q):
    print("{} degrees Celsius = {} degrees Fahrenheit".format(c, fahrenheit_a[i]))

#input_shape=[1] --> input layer in a single value
#units=1 --> number of neurons in the layer
IO = tf.keras.layers.Dense(units=1, input_shape=[1])

#assemble layer into a model
model = tf.keras.Sequential([IO])

#Loss function - A way of measuring how far off preditions are from the desired outcome.
#optimizer function - Away of adjusting internal values in order to reduce the loss.
model.compile(loss='mean_squared_error', optimizer=tf.keras.optimizers.Adam(0.1))
#train the model
history = model.fit(celsius_q, fahrenheit_a, epochs=600, verbose=False)
print('Finished training model')
# plot
plt.xlabel('Rpoch  Number')
plt.ylabel('Loss Magnitude')
plt.plot(history.history['loss'])
print(model.predict([1000.0]))
plt.show()