# -*- coding: utf-8 -*-
"""
Created on Thu Jun  1 09:35:17 2017

@author: X.X
"""

import numpy as np
from scipy import constants as C
from qcodes.instrument.base import Instrument
from pycqed.measurement.waveform_control.element import Element
from pycqed.measurement.waveform_control.pulse import Pulse, CosPulse, SquarePulse, LinearPulse#, AdiabaticCosPulse
import math
from qubit import Qubit

#%%

class AdiabaticCosPulse(Pulse):

    def __init__(self, channel, name='adiabatic cos pulse', **kw):
        Pulse.__init__(self, name)

        self.channel = channel  # this is just for convenience, internally
        self.channels.append(channel)
        # this is the part the sequencer element wants to communicate with
        self.start_frequency = kw.pop('start_frequency', 1e6)
        self.end_frequency = kw.pop('end_frequency', 1e6)
        self.amplitude = kw.pop('amplitude', 0.)
        self.length = kw.pop('length', 0.)
        self.phase = kw.pop('phase', 0.)

    def __call__(self, **kw):
        self.start_frequency = kw.pop('start_frequency', self.start_frequency)
        self.end_frequency = kw.pop('end_frequency', self.end_frequency)
        self.amplitude = kw.pop('amplitude', self.amplitude)
        self.length = kw.pop('length', self.length)
        self.phase = kw.pop('phase', self.phase)

        return self

    def chan_wf(self, chan, tvals):
        self.frequency = np.linspace(self.start_frequency, self.end_frequency, len(tvals))
        return self.amplitude * np.cos(2 * np.pi *
                                       (self.frequency * tvals +
                                        self.phase / 360.))

#%% gate

class Gate:

    def __init__(self, name, **kw):

        self.name = name                                 ## name of a gate, e.g. 'X_Q1' 'CZ_Q12'
        self.qubit = kw.pop('qubit', None)
        self.control_qubit = kw.pop('control_qubit', None)
        self.target_qubit = kw.pop('target_qubit', None)
        self.qubit_name = 'qubit'

##      Gates will be an object that consists all the pulse information in the operation,
##      but it will just store the pulses in information and the links in time domain
##      between different pulses, but not adding it into an Element

#%% single qubit gate

class Single_Qubit_Gate(Gate):

    def __init__(self, name, qubit, rotating_axis = [1, 0, 0], amplitude = None, frequency_shift = 0, refphase = 0, IQ_phase = 0):
        super().__init__(name,)

        self.qubit = qubit.name
        self.amplitude = qubit.IQ_amplitude if amplitude == None else amplitude
#        self.frequency_shift = qubit.frequency if frequency == None else frequency
        self.frequency_shift = frequency_shift
        self.channel_I = qubit.microwave_gate['channel_I']
        self.channel_Q = qubit.microwave_gate['channel_Q']
        self.channel_PM = qubit.microwave_gate['channel_PM']
        self.channel_FM = qubit.microwave_gate['channel_FM']
        self.channel_VP = qubit.plunger_gate['channel_VP']

        self.Pi_pulse_length = qubit.Pi_pulse_length
        self.halfPi_pulse_length = qubit.halfPi_pulse_length

        self.PM_before = 40e-9
        self.PM_after = 2e-9
        '''
        for cbox testing
        '''
#        self.PM_before = 500e-9
#        self.PM_after = 500e-9
        '''
        end
        '''

        self.voltage_pulse_length = 0

        self.refphase = refphase*C.pi/180
        self.IQ_phase = IQ_phase
        self.axis = np.array(rotating_axis)
#        if self.axis[0]!= 0:
#            self.axis_angle = np.arctan(self.axis[1]/self.axis[0])
#        else :
        self.axis_angle = np.arctan2(self.axis[1],self.axis[0])

        self.pulses = [None, None, None, None]            ## [microwave1_I, microwave1_Q, voltage, microwave2_I, microwave2_Q]

    def XY_rotation(self, degree = 90, length = None, waiting_time = 0, refgate = None, refpoint = 'end',
                    refqubit = None):
#        global phase
#        IQ_Modulation = self.frequency
        if length is not None:
            pulse_length = length
        else:
            pulse_length = self.halfPi_pulse_length if degree == 90 else degree*self.Pi_pulse_length/180

        pulse_amp = self.amplitude

        pulse_delay = Instrument.find_instrument(self.qubit).pulse_delay
        if self.qubit == 'qubit_2' and refqubit == 'qubit_2':
            pulse_delay = 0

        ## voltage pulse is not used here
        voltage_pulse = SquarePulse(channel = self.channel_VP, name = '%s_voltage_pulse'%self.name,
                                    amplitude = 0, length = 0)#pulse_length + waiting_time)

        PM_pulse = SquarePulse(channel = self.channel_PM, name = '%s_PM_pulse'%self.name,
                               amplitude = 2, length = pulse_length+self.PM_before+self.PM_after)

        if self.frequency_shift == 0:
            microwave_pulse_I = SquarePulse(channel = self.channel_I, name = '%s_microwave_pulse_I'%self.name,
                                            amplitude = pulse_amp*np.cos(-self.refphase + self.axis_angle),
                                            length = pulse_length)

            microwave_pulse_Q = SquarePulse(channel = self.channel_Q, name = '%s_microwave_pulse_Q'%self.name,
                                            amplitude = pulse_amp*np.sin(-self.refphase + self.axis_angle),
                                            length = pulse_length)

        elif self.frequency_shift != 0: ##here frequency and phase is not yet ready!!!!!!!!!!!
            phase = self.IQ_phase-self.refphase + self.axis_angle
            freq = self.frequency_shift
            microwave_pulse_I = CosPulse(channel = self.channel_I, name = '%s_microwave_pulse_I'%self.name, frequency = freq,
                                         amplitude = pulse_amp,
                                         length = pulse_length, phase = phase)

            microwave_pulse_Q = CosPulse(channel = self.channel_Q, name = '%s_microwave_pulse_Q'%self.name, frequency = freq,
                                         amplitude = pulse_amp,
                                         length = pulse_length, phase = phase-90)

        '''
        adiabatic sweep for test
        '''
        if type(self.frequency_shift) is list :
            phase = self.IQ_phase-self.refphase + self.axis_angle
            freq = self.frequency_shift
            start_freq = self.frequency_shift[0]
            end_freq = self.frequency_shift[1]

            microwave_pulse_I = AdiabaticCosPulse(channel = self.channel_I, name = '%s_microwave_pulse_I'%self.name,
                                                  start_frequency = start_freq, end_frequency = end_freq,
                                                  amplitude = pulse_amp,
                                                  length = pulse_length, phase = phase)

            microwave_pulse_Q = AdiabaticCosPulse(channel = self.channel_Q, name = '%s_microwave_pulse_Q'%self.name,
                                                  start_frequency = start_freq, end_frequency = end_freq,
                                                  amplitude = pulse_amp,
                                                  length = pulse_length, phase = phase-90)

        '''
        '''

        self.pulses[0] = {
                'pulse': voltage_pulse,
                'pulse_name': voltage_pulse.name,
                'refpulse': None if refgate == None else refgate[0]['pulse_name'],
                'refpoint': refpoint,
                'waiting': 0
                }

        self.pulses[1] = {
                'pulse': microwave_pulse_I,
                'pulse_name': microwave_pulse_I.name,
                'refpulse': None if refgate == None else refgate[-2]['pulse_name'],                   ## name of the refpulse
                'refpoint': refpoint,
                'waiting': waiting_time + pulse_delay
                }

        self.pulses[2] = {
                'pulse': microwave_pulse_Q,
                'pulse_name': microwave_pulse_Q.name,
                'refpulse': '%s_microwave_pulse_I'%self.name,
                'refpoint': 'start',
                'waiting': 0
                }
        ##  here you just construct a dictionary which contains all the information of pulses you use
        self.pulses[3] = {
                'pulse': PM_pulse,
                'pulse_name': PM_pulse.name,
                'refpulse': '%s_microwave_pulse_I'%self.name,
                'refpoint': 'start',
                'waiting': -self.PM_before
                }

        return True

    def Z_rotation(self, degree, name = 'Z_rotation', refphase = 0, waiting_time = 0, refrotation = None):
#        global refphase
#        refphase += degree
        return True

#    def Arbitrary_rotation(self, degree, rotating_axis = [1,0,0]):
#        theta = np.arccos(rotating_axis[2])
#        phi = np.arccos(rotating_axis[0]/np.sin(theta))
#        self.Z_rotation(degree = -phi, name = 'step1')
#        self.Y_rotation(degree = -theta, name = 'step2', refrotation = 'step1')
#        self.Z_rotation(degree = degree, name = 'step3', refrotation = 'step2')
#        self.Y_rotation(degree = theta, name = 'step4', refrotation = 'step3')
#        self.Z_rotation(degree = phi, refrotation = 'step4')
#        return True

#%% twp qubit gate

class Two_Qubit_Gate(Gate):

    def __init__(self, name, control_qubit, target_qubit, refphase = 0,):

        super().__init__(name,)

        self.control_qubit = control_qubit
        self.target_qubit = target_qubit

        """
        the three parameters below are very important,
        """
        self.frequency = target_qubit.frequency
        self.refphase = refphase
        self.Pi_pulse_length = target_qubit.Pi_pulse_length

        self.detuning_amplitude_C = 0.15
        self.detuning_amplitude_T = -0.1

        self.detuning_length_Pi = 10e-6
        self.channel_VP1 = control_qubit.plunger_gate['channel_VP']
        self.channel_VP2 = target_qubit.plunger_gate['channel_VP']
        self.channel_I = target_qubit.microwave_gate['channel_I']
        self.channel_Q = target_qubit.microwave_gate['channel_Q']

        self.channel_FM = target_qubit.microwave_gate['channel_FM']
        self.channel_PM = target_qubit.microwave_gate['channel_PM']

        self.pulses = []

    def CRotation_gate(self, exchanging_time, refgate):
        self.detuning(refpulse = refgate[-1]['pulse_name'])

        self.X_Pi_gate(refpulse = self.pulses[-1]['pulse_name'])

        return True

    def CPhase_gate(self, name = 'CPhase_gate', rotating_phase = 180, refgate = None, waiting_time = 0):

        length = rotating_phase*self.detuning_length_Pi/180

        self.detuning(name = name, length = length,
                       waiting_time = waiting_time, refpulse = None if refgate == None else refgate[-1]['pulse_name'])
        return True

    def CNot_gate(self, exchanging_time, delay_time, refgate):
        self.X_gate(refpulse = refgate[-1]['pulse_name'])
        self.CPhase_gate(refpulse = self.pulses[-1]['pulse_name'])
        self.X_gate(refpulse = self.pulses[-1]['pulse_name'])
        return True

    def detuning(self, name = 'detuning_pulse', length = 0,
                 waiting_time = 0, refpulse = None, refpoint = 'end', **kw):

        AMP_C = kw.pop('amplitude_control', self.detuning_amplitude_C)
        AMP_T = kw.pop('amplitude_target', self.detuning_amplitude_T)

        voltage_pulse_C = SquarePulse(channel = self.channel_VP1, name = '%s_detuning_pulse_C'%self.name,
                                       amplitude = AMP_C, length = length,
                                       refpulse = refpulse)

        voltage_pulse_T = SquarePulse(channel = self.channel_VP2, name = '%s_detuning_pulse_T'%self.name,
                                       amplitude = AMP_T, length = length,
                                       refpulse = refpulse)

        detuning_pulse_C = {
                'pulse': voltage_pulse_C,
                'pulse_name': voltage_pulse_C.name,
                'refpulse': None if refpulse == None else refpulse,
                'refpoint': refpoint,
                'waiting': waiting_time
                }

        detuning_pulse_T = {
                'pulse': voltage_pulse_T,
                'pulse_name': voltage_pulse_T.name,
                'refpulse': '%s_detuning_pulse_C'%self.name,
                'refpoint': 'start',
                'waiting': 0
                }

        self.pulses.append(detuning_pulse_C)
        self.pulses.append(detuning_pulse_T)

        return True

    def ramp(self, name = 'ramp_pulse', length = 0, ramp = 'up',
             waiting_time = 0, refpulse = None, refpoint = 'end', **kw):

        AMP_C = kw.pop('amplitude_control', self.detuning_amplitude_C)
        AMP_T = kw.pop('amplitude_target', self.detuning_amplitude_T)

        if ramp == 'up':
            start_C = 0
            end_C = AMP_C
            start_T = 0
            end_T = AMP_T

        elif ramp == 'down':
            start_C = AMP_C
            end_C = 0
            start_T = AMP_T
            end_T = 0

        else:
            raise NameError('ramp direction unknown: ', ramp)

        voltage_pulse_C = LinearPulse(channel = self.channel_VP1, name = '%s_detuning_pulse_C'%self.name,
                                       start_value = start_C, end_value = end_C, length = length,
                                       refpulse = refpulse)

        voltage_pulse_T = LinearPulse(channel = self.channel_VP2, name = '%s_detuning_pulse_T'%self.name,
                                       start_value = start_T, end_value = end_T, length = length,
                                       refpulse = refpulse)

        detuning_pulse_C = {
                'pulse': voltage_pulse_C,
                'pulse_name': voltage_pulse_C.name,
                'refpulse': None if refpulse == None else refpulse,
                'refpoint': refpoint,
                'waiting': waiting_time
                }

        detuning_pulse_T = {
                'pulse': voltage_pulse_T,
                'pulse_name': voltage_pulse_T.name,
                'refpulse': '%s_detuning_pulse_C'%self.name,
                'refpoint': 'start',
                'waiting': 0
                }

        self.pulses.append(detuning_pulse_C)
        self.pulses.append(detuning_pulse_T)

        return True

    def XY_rotation(self, name, degree = 90, waiting_time = 0, refpulse = None, refpoint = 'end'):

        microwave_pulse_I = SquarePulse(channel = self.channel_I, name = '%s_microwave_pulse_I'%name,
                   amplitude = np.cos(self.refphase), length = degree*self.Pi_pulse_length/180)

        microwave_pulse_Q = SquarePulse(channel = self.channel_Q, name = '%s_microwave_pulse_Q'%name,
                   amplitude = np.sin(self.refphase), length = degree*self.Pi_pulse_length/180)

        component_I = {
                'pulse': microwave_pulse_I,
                'pulse_name': microwave_pulse_I.name,
                'refpulse': refpulse,                   ## name of the refpulse
                'refpoint': refpoint,
                'waiting': waiting_time
                }

        component_Q = {
                'pulse': microwave_pulse_Q,
                'pulse_name': microwave_pulse_Q.name,
                'refpulse': '%s_microwave_pulse_I'%name,
                'refpoint': 'start',
                'waiting': 0
                }

        self.pulses.append(component_I)
        self.pulses.append(component_Q)

    def X_gate(self, name = 'X_halfPi', waiting_time = 0, refpulse = None, refpoint = 'end'):

        self.XY_rotation(name = name, degree = 90, waiting_time = waiting_time, refpulse = refpulse, refpoint = refpoint)

        return 0

    def X_Pi_gate(self, name = 'X_Pi', waiting_time = 0, refpulse = None, refpoint = 'end'):

        self.XY_rotation(name = name, degree = 180, waiting_time = waiting_time, refpulse = refpulse, refpoint = refpoint)

        return 0

    def Y_gate(self, name = 'Y_halfPi', waiting_time = 0, refpulse = None, refpoint = 'end'):

        self.XY_rotation(name = name, degree = 90, waiting_time = waiting_time, refpulse = refpulse, refpoint = refpoint)

        return 0

    def Y_Pi_gate(self, name = 'Y_Pi', waiting_time = 0, refpulse = None, refpoint = 'end'):

        self.XY_rotation(name = name, degree = 180, waiting_time = waiting_time, refpulse = refpulse, refpoint = refpoint)

        return 0

class CPhase_Gate(Two_Qubit_Gate):

    def __init__(self, name = 'CPhase_gate', control_qubit = None, target_qubit = None,
                 amplitude_control = 0, amplitude_target = 0, length = 0,
                 rotating_phase = 180, refgate = None, refpoint = 'end', waiting_time = 0):

        super().__init__(name, control_qubit, target_qubit)

        pulse_length = length if length != 0 else rotating_phase*self.detuning_length_Pi/180

        self.detuning(name = name, length = pulse_length, waiting_time = waiting_time, refpoint = refpoint,
                      amplitude_control = amplitude_control, amplitude_target = amplitude_target,
                      refpulse = None if refgate == None else refgate[-2]['pulse_name'])

    def __call__(self, **kw):

        return self

class Ramp(Two_Qubit_Gate):

    def __init__(self, name = 'Ramp', control_qubit = None, target_qubit = None,
                 amplitude_control = 0, amplitude_target = 0, length = 0, ramp = 'up',
                 rotating_phase = 180, refgate = None, refpoint = 'end', waiting_time = 0):

        super().__init__(name, control_qubit, target_qubit)

        pulse_length = length if length != 0 else rotating_phase*self.detuning_length_Pi/180

        self.ramp(name = name, length = pulse_length, waiting_time = waiting_time, refpoint = refpoint,
                  amplitude_control = amplitude_control, amplitude_target = amplitude_target, ramp = ramp,
                  refpulse = None if refgate == None else refgate[-2]['pulse_name'])

    def __call__(self, **kw):

        return self

class CNot_Gate(Two_Qubit_Gate):

    def __init__(self, name = 'CPhase_gate', control_qubit = None, target_qubit = None,
                 rotating_phase = 180, refgate = None, refpoint = 'end', waiting_time = 0):

        super().__init__(name, control_qubit, target_qubit)

        length = rotating_phase*self.detuning_length_Pi/180

        self.X_gate(name = name + '_X1', waiting_time = waiting_time, refpoint = refpoint,
                    refpulse = None if refgate == None else refgate[-2]['pulse_name'])

        self.detuning(name = name+'_detuning', length = length, waiting_time = 1e-6,
                      refpulse = self.pulses[-1]['pulse_name'])

        self.X_gate(name = name+'_X2', waiting_time = 1e-6, refpulse = self.pulses[-1]['pulse_name'])

    def __call__(self, **kw):

        return self

class CRotation_Gate(Two_Qubit_Gate):

    def __init__(self, name = 'CPhase_gate', control_qubit = None, target_qubit = None,
                 rotating_phase = 180, refgate = None, refpoint = 'end', waiting_time = 0):

        super().__init__(name, control_qubit, target_qubit)

        length = rotating_phase*self.detuning_length_Pi/180

        self.detuning(name = name+'_detuning', length = length, waiting_time = waiting_time, refpoint = refpoint,
                      refpulse = None if refgate == None else refgate[-2]['pulse_name'])

        self.X_Pi_gate(name = name+'_X', refpulse = self.pulses[-1]['pulse_name'], refpoint = 'center', waiting_time = 2e-6)


"""
Compare USGS events to events in catalogue:

1) do the big events match?
2) if they match
3) if they don't match,

I can see the future: i have to write the association script mysef

"""
import pandas as pd
import numpy as np
from obspy.geodetics import gps2dist_azimuth

def parse_station_info(input_file):
	# 'reusing functions is bad practice' yes haha

	station_info = {}

	with open(input_file, 'r') as f:
		for line in f:
			#print(line)
			try:
				sta, lon, lat = [x for x in line.strip().split("\t") if x != ""]
			except:
				sta, lon, lat = [x for x in line.strip().split(" ") if x != ""]
			station_info[sta] = {"lon": float(lon), "lat": float(lat)}
	return station_info
def dx(X1, X2):
	"""

	takes in two coordinate tuples (lon, lat), (lon, lat) returning their distance in kilometres
	gps2dist_azimuth also returns the azimuth but i guess i don't need that yet
	it also returns distances in metres so i just divide by 1000

	the order doesn't matter

	:param      X1:   The x 1
	:type       X1:   { type_description }
	:param      X2:   The x 2
	:type       X2:   { type_description }

	"""

	#print(X1, X2)
	return gps2dist_azimuth(X1[1], X1[0], X2[1], X2[0])[0] / 1000

def ip(X, Y):
	if len(X) == 3:

		# arithmetic average of in between gradients to approximate gradient at midpoint

		return 0.5 * ((Y[2] - Y[1])/(X[2] - X[1]) + (Y[1] - Y[0])/(X[1] - X[0]))

	if len(X) == 2:
		return (Y[1] - Y[0])/(X[1] - X[0])

station_info = parse_station_info('../new_station_info.dat')

# usgs catalogue
udf = pd.read_csv("usgs_aceh.csv")
udf.rename(columns = {"time": "timestamp", "latitude": "LAT", "longitude": "LON", "depth":"DEPTH", "mag": "MAG"}, inplace = True)
# print(udf.columns)
udf['timestamp'] = pd.to_datetime(udf['timestamp']).dt.tz_localize(None)

# detections
# eqt_df = pd.read_csv("../master_csv/eqtfiles/merged_customfilter_jan20-may21.csv")
eqt_df = pd.read_csv("../master_csv/eqtfiles/merged_merged_jan20-may21.csv")
eqt_df['p_arrival_time'] = pd.to_datetime(eqt_df['p_arrival_time'])
eqt_df['s_arrival_time'] = pd.to_datetime(eqt_df['s_arrival_time'])

# association file
# real_df = pd.read_csv("../master_csv/master_merged_REAL_mags.csv")
real_df = pd.read_csv("../imported_figures/all_rereal_events.csv")
real_df.drop("MAG", 1, inplace = True)
real_df['timestamp'] = pd.to_datetime(real_df['timestamp'])

# want to know rate of seismicity before the 'mainshock'
# i.e. want no. of detections 1 hr, 4 hrs, 24 hrs before + 1hr, 4 hrs, and 24 hrs after

match_counter = 0

for index, row in udf.iterrows():
	ASSOC_WINDOW = 4 # symmetric over left and right side

	real_df['dt'] = (real_df['timestamp'] - row.timestamp).astype('timedelta64[s]') # this does rounding to seconds, which is not the same as .total_seconds() in python datetime
	window_real_df = (real_df[(real_df['dt'] < ASSOC_WINDOW ) & (real_df['dt'] > -ASSOC_WINDOW)] )
	if len(window_real_df):

		#print(row.timestamp)
		match_counter += 1

		udf.at[index, 'matched'] = True

		headers = ["ID", "timestamp", "dt", "LON", "LAT", "DEPTH", "m_l", "station_gap", "residual_time", "n_picks"]

		for h in headers:
			try:
				udf.at[index, 'matched_' + h] = window_real_df[h].to_list()[0]
			except:
				pass

mudf = (udf[udf['matched'] == 1])# matched usgs dataframe

# for index, row in mudf.iterrows():
# 	mudf.at[index, 'delta_depth'] = row.matched_DEPTH - row.DEPTH
# 	mudf.at[index, 'delta_dist_km'] = dx((row.LON, row.LAT), (row.matched_LON, row.matched_LAT))
# 	mudf.at[index, 'delta_timestamp'] = (row.matched_timestamp - row.timestamp).total_seconds()

# print("mean:")
# print(mudf[["delta_depth", "delta_dist_km", "delta_timestamp"]].mean())
# print("std:")
# print(mudf[["delta_depth", "delta_dist_km", "delta_timestamp"]].std())

# mudf.to_csv("usgs_match.csv", index = False)

# load look up table lol
with open("../gridsearch/model_dlange2_451km-60km.npy", "rb") as f:
	tt = np.load(f)

TT_DX = 1
TT_DZ = 1
TT_NX = tt.shape[0]
TT_NZ = tt.shape[1]
matched_phases = []

for index, row in udf[udf["DEPTH"] < 80].iterrows(): # the travel time table goes up to 90km but uh
	print(row.timestamp)

	station_dist = {}

	# key ordering is immutable: throw into array and let numpy handle it

	dists = []
	depths = []

	# compute station distances for all stations
	for sta in station_info:
		_dist = dx((row.LON, row.LAT), (station_info[sta]["lon"], station_info[sta]["lat"]))
		dists.append(_dist)

		_depth = row.DEPTH
		depths.append(_depth)

	dists = np.array(dists)
	depths = np.array(depths)

	dist_indices = np.array([int(round(x)) for x in dists/TT_DX]) # for table interpolation
	depth_indices = np.array([int(round(x)) for x in depths/TT_DZ]) # for table interpolation

	dist_deltas = dists - dist_indices * TT_DX
	depth_deltas = depths - depth_indices * TT_DZ

	# compute theoretical travel time and do interpolation for distance and depth

	dist_grad_P = []
	dist_grad_S = []

	depth_grad_P = []
	depth_grad_S = []

	tt_P = []
	tt_S = []

	for _c, _i in enumerate(dist_indices):
		if _i + 1 > TT_NX:
			_indices = np.array([_i - 1, _i])
		elif _i - 1 < 0:
			_indices = np.array([_i, _i + 1])
		else:
			_indices = np.array([_i - 1, _i, _i + 1])

		_Y_P = [tt[_x][depth_indices[_c]][0] for _x in _indices]
		_Y_S = [tt[_x][depth_indices[_c]][1] for _x in _indices]

		tt_P.append(tt[_i][depth_indices[_c]][0])
		tt_S.append(tt[_i][depth_indices[_c]][1])

		dist_grad_P.append(ip((_indices) * TT_DX, _Y_P))
		dist_grad_S.append(ip((_indices) * TT_DX, _Y_S))

	for _c, _i in enumerate(depth_indices):
		if _i + 1 > TT_NZ:
			_indices = np.array([_i - 1, _i])
		elif _i - 1 < 0:
			_indices = np.array([_i, _i + 1])
		else:
			_indices = np.array([_i - 1, _i, _i + 1])

		depth_grad_P.append(ip((_indices) * TT_DZ, _Y_P))
		depth_grad_S.append(ip((_indices) * TT_DZ, _Y_S))

	dist_grad_P = np.array(dist_grad_P)
	dist_grad_S = np.array(dist_grad_S)
	depth_grad_P = np.array(depth_grad_P)
	depth_grad_S = np.array(depth_grad_S)

	tt_P = np.array(tt_P)
	tt_S = np.array(tt_S)

	tt_P += dist_grad_P * dist_deltas
	tt_S += dist_grad_S * dist_deltas
	tt_P += depth_grad_P * depth_deltas
	tt_S += depth_grad_S * depth_deltas

	for c, sta in enumerate(station_info):
		station_info[sta]["tt_P"] = tt_P[c]
		station_info[sta]["tt_S"] = tt_S[c]

	# given a set of theoretical arrival times, compare with the detection folder for each station

	P_WINDOW = 4 # SYMMETRIC
	S_WINDOW = 8 # SYMMETRIC
	c = 0

	for sta in station_info:
		_eqt_df = eqt_df[eqt_df['station'] == sta].copy()

		_eqt_df['dt_p'] = ((_eqt_df['p_arrival_time'] - row.timestamp).dt.total_seconds() - station_info[sta]["tt_P"])
		_eqt_df['dt_s'] = ((_eqt_df['s_arrival_time'] - row.timestamp).dt.total_seconds() - station_info[sta]["tt_S"])
		_eqt_df['absdt_p'] = ((_eqt_df['p_arrival_time'] - row.timestamp).dt.total_seconds() - station_info[sta]["tt_P"]).abs()
		_eqt_df['absdt_s'] = ((_eqt_df['s_arrival_time'] - row.timestamp).dt.total_seconds() - station_info[sta]["tt_S"]).abs()

		if (_eqt_df['absdt_p'].min()) < P_WINDOW and (_eqt_df['absdt_s'].min()) < S_WINDOW:
			if (_eqt_df['absdt_p'].idxmin()) == (_eqt_df['absdt_s'].idxmin()):
				target_index = _eqt_df['absdt_p'].idxmin()
				matched_phases.append(target_index)

				for h in ["id", "matched_ID" ]:
					eqt_df.at[target_index, h] = row[h]

				eqt_df.at[target_index, "dt_p"] = _eqt_df.at[target_index, "dt_p"]
				eqt_df.at[target_index, "dt_s"] = _eqt_df.at[target_index, "dt_s"]

print(matched_phases)

m_df = eqt_df.iloc[matched_phases]
m_df.to_csv("rereal_matched_usgs_eqt.csv", index = False)

		#_eqt_df['dt_p'] = (_eqt_df['p_arrival_time'] - (row.timestamp + tt_P[c])).abs()
		# _eqt_df['dt_p'] = ((_eqt_df['p_arrival_time'] - (row.timestamp)).dt.total_seconds() - tt_P[c]).abs()
		# _eqt_df['dt_s'] = ((_eqt_df['s_arrival_time'] - (row.timestamp)).dt.total_seconds() - tt_S[c]).abs()

		#_eqt_df['dt_s'] = (_eqt_df['s_arrival_time'] - (row.timestamp + tt_S[c])).abs()

		# if len(_eqt_df[(_eqt_df['dt_p'] < P_WINDOW)]):
		# #if len(_eqt_df[(_eqt_df['dt_p'] < P_WINDOW) & (_eqt_df['dt_s'] < S_WINDOW)]):
		# 	print("match: ", sta)

# you know, the big brain gridsearch is to simulate events (pre compute them) and compute arrival times for all stations
# and then match
# the only problem is that the stored model has like 4 dimensions, it will have 2e9 entries
# so that's probably why it's not done yet
# there's probably an efficient way to do it involving interpolation so you don't have to recalculate every single entry

# if the event is deeper than 60km, it's probably plate interface
# my travel time table goes up to

# get average difference in location, magnitude, and time

	# match by timestamp, then check
	# arrival time window? p wave velocity ~ 5km, maybe 1 minute?
	# there would be a way to calculate theoretical travel times using the look up table + interpolation


from modules.sem_tocar_config import *
from modules.toolbox import *
import time
import os
from urllib.parse import quote
import webbrowser as web
import pyautogui as pg

def notify_whatsapp (event):
    for phone_no in event["description"].split(","):
        wait_time = 60
        close_time = 15
        event_time = format_human_date(event['start'])
        message = f"Olá, você tem um evento em {COMPANY_NAME} às {event_time['time']} do dia {event_time['day']} de {event_time['month']} ({event_time['weekday']}), apresente o QR na entrada"
        #phone_no = event["description"]
        image = make_path(f"QR_{event['id']}.png", "QR")

        parsed_message = quote(message)
        web.open('https://web.whatsapp.com/send?phone=' +
                phone_no + '&text=' + parsed_message)
        print(message, phone_no)
        time.sleep(wait_time)
        width, height = pg.size()
        pg.click(width/2, height/2)
        time.sleep(wait_time/4)
        pg.press('enter')
        os.system(
        f"xclip -selection clipboard -target image/png -i {image}")
        time.sleep(wait_time/4)
        pg.hotkey("ctrl", "v")
        time.sleep(wait_time/4)
        pg.press('enter')

        time.sleep(close_time)
        pg.hotkey("ctrl", "w")


from dotenv import load_dotenv, dotenv_values
import pathlib
from pathlib import Path

load_dotenv()
config = dotenv_values("secret.env")

tk = config['OPENAI_API_KEY']
TOKEN = config['VK_TOKEN']

GROUP_ID = 220253752
FLOOD_DELAY = 10

IMAGE_KW = '/image'

HI_MESSAGE = "Добро пожаловать, {}!\nЧтобы начать работу введите запрос, и искусственный интелект вам ответит"
COMPLETE_ERROR_MSG = 'Ошибка при получении данных с https://chat.openai.com\nПовторите попытку позже'
IMAGE_ERROR_MSG = 'Невозможно сгенерировать картинку по данному запросу'
GENERATING_MSG = "Ответ генерируется, подождите..."
FLOOD_WARN = "Не флуди"
END_DIALOG_BTN = 'Завершить предыдущий диалог'
DIALOG_ENDED = 'Диалог успешно завершён, можете начинать новый!'
IMAGE_SENT = 'Вот ваше изображение!'

audio_path = Path(pathlib.Path.cwd(), 'audio')
db_path = Path(pathlib.Path.cwd(), 'db', 'conv.sqlite')
images_path = Path(pathlib.Path.cwd(), 'images')


import tensorflow as tf

from config import FLAGS

class Model(object):
    def __init__(self, batchloader, is_training=True):
        self.batchloader = batchloader
        self.is_training = is_training
        self.lr = tf.placeholder(tf.float32, shape=(), name="learning_rate")

        with tf.name_scope("Placeholders"):
            self.input_text = tf.placeholder(tf.int32,
                                             shape=(FLAGS.BATCH_SIZE, FLAGS.SEQ_LEN),
                                             name="input_text")

            self.label = tf.placeholder(tf.int32,
                                        shape=(FLAGS.BATCH_SIZE),
                                        name="label")

        with tf.variable_scope("Embedding"):
            self.embedding = tf.get_variable(name="embedding",
                                 shape=[FLAGS.VOCAB_SIZE, FLAGS.EMBED_SIZE],
                                 dtype=tf.float32,
                                 initializer=tf.truncated_normal_initializer(stddev=0.1))

        with tf.name_scope("Embedding_text"):
            # NHWC
            self.embedded_text = tf.expand_dims(tf.nn.embedding_lookup(self.embedding, self.input_text),
                                                axis=-1)
            assert self.embedded_text.shape == (FLAGS.BATCH_SIZE, FLAGS.SEQ_LEN, FLAGS.EMBED_SIZE, 1)

        with tf.variable_scope("CNN"):
            self.maxpooled_output = []
            self.cnn_W = []
            for i, filter_size in enumerate(FLAGS.FILTER_SIZE):
                layer_name = "cnn_layer_%d_%d" % (i, filter_size)
                with tf.variable_scope(layer_name):
                    self.cnn_W.append(tf.get_variable(name=layer_name+"_W",
                                                      shape=(filter_size,
                                                             FLAGS.EMBED_SIZE,
                                                             1,
                                                             FLAGS.FILTER_NUM),
                                                      dtype=tf.float32,
                                                      initializer=tf.truncated_normal_initializer(stddev=0.1)))

                    conv = tf.nn.conv2d(self.embedded_text,
                                        self.cnn_W[i],
                                        strides=[1, 1, 1, 1],
                                        padding="VALID",
                                        name=layer_name+"_conv")

                    maxpooled = tf.nn.max_pool(conv,
                                               ksize=[1, FLAGS.SEQ_LEN - filter_size + 1, 1, 1],
                                               strides=[1, 1, 1, 1],
                                               padding="VALID")

                    self.maxpooled_output.append(maxpooled)

            with tf.variable_scope("Feaure"):
                self.feature = tf.reshape(tf.concat(self.maxpooled_output, axis=3),
                                          shape=[FLAGS.BATCH_SIZE, FLAGS.FILTER_NUM * len(FLAGS.FILTER_SIZE)])

                if self.is_training:
                    self.feature = tf.nn.dropout(self.feature, FLAGS.DROPOUT_KEEP)

        with tf.variable_scope("Linear"):
            self.W = tf.get_variable(name="W",
                                     shape=(FLAGS.FILTER_NUM * len(FLAGS.FILTER_SIZE),
                                            FLAGS.CLASS_NUM),
                                     dtype=tf.float32,
                                     initializer=tf.truncated_normal_initializer(stddev=0.1))
            self.b = tf.get_variable(name="b",
                                     shape=(FLAGS.CLASS_NUM),
                                     dtype=tf.float32,
                                     initializer=tf.zeros_initializer())

        with tf.name_scope("Logits"):
            self.logits = tf.matmul(self.feature, self.W) + self.b

        with tf.name_scope("Accuracy"):
            prediction = tf.argmax(self.logits, 1, output_type=tf.int32)
            self.accuracy = tf.reduce_mean(
                                tf.cast(tf.equal(prediction, self.label),
                                        dtype=tf.float32))

        with tf.name_scope("Loss"):
            losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
                        labels=self.label,
                        logits=self.logits,
                        name="loss")
            self.loss = tf.reduce_mean(losses)

        with tf.name_scope("Summary"):
            if is_training:
                loss_summary = tf.summary.scalar("loss", self.loss, family="train_loss")
                acc_summary = tf.summary.scalar("accuracy", self.accuracy, family="train_loss")
                lr_summary = tf.summary.scalar("lr", self.lr, family="parameters")

                self.merged_summary = tf.summary.merge([loss_summary, lr_summary, acc_summary])

            else:
                loss_summary = tf.summary.scalar("valid_loss", self.loss, family="val_loss")
                acc_summary = tf.summary.scalar("valid_accuracy", self.accuracy, family="val_loss")

                self.merged_summary = tf.summary.merge([loss_summary, acc_summary])

        if self.is_training:
            tvars = tf.trainable_variables()
            with tf.name_scope("Optimizer"):
                tvars = tf.trainable_variables()
                grads, _ = tf.clip_by_global_norm(tf.gradients(self.loss, tvars),
                                                  FLAGS.MAX_GRAD)
                optimizer = tf.train.AdamOptimizer(self.lr, beta1=0.5)

                self.train_op = optimizer.apply_gradients(zip(grads, tvars))


import os
import logging
import time
import pymongo
from random import choice
MONGO_DB = pymongo.MongoClient(host='172.16.75.38',port=27017)
STATIC_IP = MONGO_DB["IP"]["STATIC_IP"]
def logger(FILE_NAME):
    """
    日志配置
    :param FILE_NAME: 日志文件名(全路径 )
    :return:日志记录生成器
    """
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
        datefmt='%Y %H:%M:%S',
        filename=FILE_NAME,
        filemode='w'
    )
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    formatter = logging.Formatter('[%(asctime)s] %(filename)s[Line:%(lineno)d] [%(levelname)s] %(message)s')
    console.setFormatter(formatter)
    logging.getLogger('').addHandler(console)
    return logging

def get_log():
    real_path = os.path.dirname(os.path.realpath(__file__)).replace('\\', '/') + "/log/"
    file_name = "{}_爬虫程序_{}.log".format(real_path,
                                             time.strftime("%Y-%m-%d",
                                                           time.localtime()))
    log = logger(file_name)
    return log

def ABY():
    try:
        IP = STATIC_IP.find()

        ip_lt = choice([_ for _ in IP])
        ip = ip_lt["ip_parmas"]
        _id = ip_lt["_id"]
        proxies = {
            "http": "http://" + ip,
            "https": "https://" + ip,
        }
        return proxies,_id

    except Exception as e:
        print(e)
        return None


#!/usr/bin/env python3

import argparse
from datetime import datetime
import sys
import pandas

def parseArgs(args):
    parser = argparse.ArgumentParser()
    parser.add_argument('file', type=str,
            help='File to process.')
    parser.add_argument('--event', type=str,
            help='File to process.')
    parser.add_argument('-b', '--begin', type=str,
            help='Start timestamp')
    parser.add_argument('-e', '--end', type=str,
            help='End timestamp')
    return parser.parse_args(args)

def main(args):
    progress_trace = pandas.read_csv(args.file,
                              converters={'TIMESTAMP': datetime.fromisoformat}
                                     )

    have_trace_id = 'TRACE ID' in progress_trace.keys()

    sync_from_events = progress_trace[
            (progress_trace['DATASTORE'] == 'running') &
            ((progress_trace['EVENT TYPE'] == 'start') |
             (progress_trace['EVENT TYPE'] == 'stop'))
    ]
    if args.event:
        sync_from_events = sync_from_events[
            sync_from_events['MESSAGE'] == args.event
        ]
    if args.begin:
        begin = datetime.fromisoformat(args.begin)
        sync_from_events = sync_from_events[
            sync_from_events['TIMESTAMP'] >= begin
        ]
    if args.end:
        end = datetime.fromisoformat(args.end)
        sync_from_events = sync_from_events[
            sync_from_events['TIMESTAMP'] <= end
        ]

    tids = []
    tids_msgs = []
    for name, fields in sync_from_events.iterrows():
        evt = fields['EVENT TYPE']
        tid = fields['TRANSACTION ID']
        msg = fields['MESSAGE']
        ann = fields['ANNOTATION']
        if evt == 'start':
            if tid not in tids:
                tids.append(tid)
                tids_msgs.append((tid, msg))
        elif evt == 'stop':
            if (tid, msg) in tids_msgs:
                tids_msgs.remove((tid, msg))
                tids.remove(tid)
                print(f"{name}  {msg:15} {tid} {ann}")

if __name__ == '__main__':
    main(parseArgs(sys.argv[1:]))


from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.keys import Keys
from locators import form_locators
import time

class Proficiency(object):

    def __init__(self, driver):
        self.driver = driver

    def get_proficiency_text(self):
        chack_box_gride = WebDriverWait(self.driver, 3).until(
            EC.presence_of_all_elements_located((By.CSS_SELECTOR, form_locators.chack_box_gride_selc))
        )
        return chack_box_gride

    def set_proficiency_check_lang(self, lang):
            reading = WebDriverWait(self.driver, 10).until(
                EC.element_to_be_clickable((By.CSS_SELECTOR, form_locators.reading_selector.format(lang)))
            )
            reading.click()

            WebDriverWait(self.driver, 10).until(
                EC.invisibility_of_element_located((By.CSS_SELECTOR, '[aria-label="Reading, response for {}"]'
                                                                     '[aria-checked="false"]'.format(lang)))
            )
            listing = WebDriverWait(self.driver, 10).until(
                EC.element_to_be_clickable((By.CSS_SELECTOR, form_locators.listing_selector.format(lang)))
            )
            listing.click()

    # def proficiency_check_french(self, fre):
    #         listing = WebDriverWait(self.driver, 10).until(
    #             EC.element_to_be_clickable((By.CSS_SELECTOR, form_locators.listing_selector.format(fre)))
    #         )
    #         listing.click()
    #
    # def proficiency_check_urdu(self, urdu):
    #         reading = WebDriverWait(self.driver, 10).until(
    #             EC.element_to_be_clickable((By.CSS_SELECTOR, form_locators.reading_selector.format(urdu)))
    #         )
    #         reading.click()
    #         writing = self.driver.find_element_by_css_selector(
    #             form_locators.writing_selector.format(urdu))
    #         writing.click()
    #         listing = WebDriverWait(self.driver, 10).until(
    #             EC.element_to_be_clickable((By.CSS_SELECTOR, form_locators.listing_selector.format(urdu)))
    #         )
    #         listing.click()


"""
This script is intended to test base implementations and should be deleted as soon as infrastructure is established
When run, this script will calculate a given amount of prime numbers, beginning at 2 for an input of 0
"""

import argparse
import copy
import logging.config

import yaml

with open('logging_conf.yaml') as conf_file:
    config = yaml.safe_load(conf_file.read())
    logging.config.dictConfig(config)

logger = logging.getLogger(__name__)

class PrimeCalculator:
    """
    Responsible for handling calls to prime calculation
    """
    def calculate_primes(self, n, i=2, primes=()):
        """
        recursively tests if each number is a prime
        n: the number of primes left to calculate
        i: the number we have reached
        primes: the primes we have already found
        """

        # base case: we have calculated the last prime
        if n == 0:
            return i - 1

        for prime in primes:
            if i % prime != 0:
                continue

            return self.calculate_primes(n, i + 1, primes)

        # no divisions have succeeded, we have found another prime
        primes = (*primes, i)
        logging.debug(f"Found a prime, current prime list is now \n{str(primes)}\n")
        return self.calculate_primes(n - 1, i + 1, primes)

def build_namespace_argument_dict(namespace):
    """
    Returns a dictionary version of a namespace for more natural parsing
    """
    # Namespace is mutable, copy it before making changes
    dict_args = vars(copy.copy(namespace))

    # Remove the function paramter if it exists, as its not an actual parameter
    dict_args.pop('func', None)
    return dict_args

def build_arg_parser():
    """
    Create an argparser configuration for command line arguments
    """
    parser = argparse.ArgumentParser(
        description="Module for getting the nth prime"
    )
    parser.add_argument('-n', '--number', type=int, help="The nth prime to retrieve")

    return build_namespace_argument_dict(parser.parse_known_args()[0])

if __name__ == "__main__":
    arguments = build_arg_parser()
    argument_str = '\n'.join([f'--- {key} = {arg}' for key, arg in arguments.items()])
    logger.info(f"Prime calculator running with the following arguments: \n{argument_str}")

    prime_calc = PrimeCalculator()
    nth_prime = prime_calc.calculate_primes(arguments['number'])
    with open('prime.txt', 'w') as prime_file:
        prime_file.write(str(nth_prime))

    logging.info(f"COMPLETED RUN, the {arguments['number']}th prime is {nth_prime}")


#!/usr/bin/env python3

import argparse
import calendar
import getpass
import happybase
import logging
import random
import sys
from collections import Counter

logging.basicConfig(level="DEBUG")

HOSTS = ["mipt-node0%d.atp-fivt.org" % i for i in range(1, 5)]

USAGE = """
Format of query:
  $ {0} --table_name (e.g. s201723Table) --period_from (e.g. 2014-09-01) --period_to (e.g. 2014-09-10)
""".format(sys.argv[0])

def connect(table_name):
    host = random.choice(HOSTS)
    conn = happybase.Connection(host)

    logging.debug("Connecting to HBase Thrift Server on %s", host)
    conn.open()

    logging.debug("Using table %s", table_name)
    return happybase.Table(table_name, conn)

def parse_date(date):
    year, month, day = date.split('-')
    return int(year), int(month), int(day)

def get_period(period_from, period_to):
    year_from, month_from, day_from = parse_date(period_from)
    year_to, month_to, day_to = parse_date(period_to)
    def get_row_name(year, month):
        return "%04d%02d" % (year, month)

    if year_from != year_to or month_from != month_to:
        start_days = ['col_day%02d' % i for i in range(day_from, 32)]
        end_days = ['col_day%02d' % i for i in range(1, day_to)]

        first_month_range = (get_row_name(year_from, month_from), get_row_name(year_from, month_from + 1), start_days)
        last_month_range = (get_row_name(year_to, month_to), get_row_name(year_to, month_to + 1), end_days)
        inner_range = (get_row_name(year_from, month_from + 1), get_row_name(year_to, month_to))
        return first_month_range, inner_range, last_month_range
    else:
        inner_range = (get_row_name(year_from, month_from),
            get_row_name(year_to, month_to + 1),
            ['col_day%02d' % i for i in range(day_from, day_to + 1)])
        return None, inner_range, None

def main():
    parser = argparse.ArgumentParser(usage=USAGE)
    parser.add_argument("--table_name", type=str, required=True)
    parser.add_argument("--period_from", type=str, required=True)
    parser.add_argument("--period_to", type=str, required=True)
    args = parser.parse_args()

    table = connect(args.table_name)

    first_month_range, inner_range, last_month_range = get_period(args.period_from, args.period_to)

    def convert_data(data):
        return Counter({col.decode("utf-8")[10:] : int.from_bytes(data[col], byteorder='big') for col in data})

    domain_hits = Counter()
    if first_month_range:
        for key, data in table.scan(row_start=first_month_range[0], row_stop=first_month_range[1], columns=first_month_range[2]):
            domain_hits += convert_data(data)
    for key, data in table.scan(row_start=inner_range[0], row_stop=inner_range[1]):
        domain_hits += convert_data(data)
    if last_month_range:
        for key, data in table.scan(row_start=last_month_range[0], row_stop=last_month_range[1], columns=last_month_range[2]):
            domain_hits += convert_data(data)

    counts_sum = sum(domain_hits.values())
    for pair in domain_hits.most_common():
        print( pair[0], pair[1] / counts_sum, sep='\t' )

if __name__ == "__main__":
    main()


import urllib.request
import json
from pprint import pprint

consYes = 0
consNo = 0
consA = 0
labYes = 0
labNo = 0
labA = 0
snpYes = 0
snpNo = 0
snpA = 0
ldYes = 0
ldNo = 0
ldA = 0
dupYes = 0
dupNo = 0
dupA = 0
othYes = 0
othNo = 0
othA = 0

Yes= {'Liberal Democrat': 0, 'Conservative': 0, 'UK Independence Party': 0, 'Scottish National': 0, 'Labour': 0, 'First Deputy Chairman of Ways and Means': 0, 'Independent': 0, 'Plaid Cymru': 0, 'Alliance': 0, 'Democratic Unionist': 0, 'Social Democratic & Labour Party': 0, 'Sinn Féin': 0, 'Respect': 0, 'Green': 0, 'Second Deputy Chairman of Ways and Means': 0, 'Speaker of the House of Commons': 0, 'Chairman of Ways and Means': 0};
No={'Liberal Democrat': 0, 'Conservative': 0, 'UK Independence Party': 0, 'Scottish National': 0, 'Labour': 0, 'First Deputy Chairman of Ways and Means': 0, 'Independent': 0, 'Plaid Cymru': 0, 'Alliance': 0, 'Democratic Unionist': 0, 'Social Democratic & Labour Party': 0, 'Sinn Féin': 0, 'Respect': 0, 'Green': 0, 'Second Deputy Chairman of Ways and Means': 0, 'Speaker of the House of Commons': 0, 'Chairman of Ways and Means': 0}
Abstain = {'Liberal Democrat': 0, 'Conservative': 0, 'UK Independence Party': 0, 'Scottish National': 0, 'Labour': 0, 'First Deputy Chairman of Ways and Means': 0, 'Independent': 0,'Plaid Cymru': 0, 'Alliance': 0, 'Democratic Unionist': 0, 'Social Democratic & Labour Party': 0, 'Sinn Féin': 0, 'Respect': 0, 'Green': 0, 'Second Deputy Chairman of Ways and Means': 0, 'Speaker of the House of Commons': 0, 'Chairman of Ways and Means': 0}
#Yes["Conservative"]=10
#Yes["UK Independence Party"]=1
#No["Conservative"]=0

print ("making request")
url = 'http://lda.data.parliament.uk/commonsdivisions/id/229689.json'
test = urllib.request.urlopen(url)

#test = open (.\testVote.json)
data = test.read()
jsonData = data.decode('utf-8')
print(len(jsonData))
count = 0
jsonObj = json.loads(jsonData)

#pprint(jsonData)
#pprint (jsonObj["result"]["primaryTopic"]["vote"][0])
votes = jsonObj["result"]["primaryTopic"]["vote"]
voteCount = len(votes)
for count in range(0, voteCount-1):
    party = jsonObj["result"]["primaryTopic"]["vote"][count]["memberParty"]
    vote = jsonObj["result"]["primaryTopic"]["vote"][count]["type"]
    name = jsonObj["result"]["primaryTopic"]["vote"][count]["memberPrinted"]["_value"]
    #print (name)
    #print (vote)
    #print (party)
    if (vote == "http://data.parliament.uk/schema/parl#AyeVote"):
        Yes[party] = Yes[party] + 1;
        #print('Yes')
    elif (vote == "http://data.parliament.uk/schema/parl#NoVote"):
        No[party]+=1
        #print ('No')
    else:
        Abstain[party]+=1
        #print ('Abstain/Other')
    #print ("\n")

consYes = Yes["Conservative"]
consNo = No["Conservative"]
consA = 331-(consYes+consNo)
print ("Con Yes " + str(consYes))
print ("Con No " + str(consNo))
print ("Con Abs " + str(consA))

labYes = Yes["Labour"]
labNo = No["Labour"]
consA = 232-(labYes+labNo)

snpYes = Yes["Scottish National"]
snpNo = No["Scottish National"]
snpA = 56-(snpYes+snpNo)

ldYes = Yes["Liberal Democrat"]
ldNo = No["Liberal Democrat"]
ldA = 8-(ldYes+ldNo)

dupYes = Yes["Democratic Unionist"]
dupNo = No["Democratic Unionist"]
dupA = 8-(dupYes+dupNo)


from django.shortcuts import render
from rest_framework import status
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework.parsers import FormParser, MultiPartParser, JSONParser
from .models import LectureVideos
from .serializers import LectureVideoSerialzer

@api_view(['GET'])
def apiOverview(request):

    api_urls = {

        'Lecture-Videos' : 'videos-list/',
        'Upload-lecture-video': 'video-upload/',
        'Delete-lecture-video' : 'delete-lecture/'
    }

    return Response(api_urls)

@api_view(['GET'])
def getLectureVideos(request):

    videos = LectureVideos.objects.all()
    serializer = LectureVideoSerialzer(videos, many=True)

    return Response(serializer.data)

@api_view(['POST'])
def uploadLectureVideo(request):

    parser_classes = (JSONParser, FormParser, MultiPartParser)
    serializer = LectureVideoSerialzer(data=request.data)

    if serializer.is_valid():
        serializer.save()
        return Response(serializer.data)

    else:
        print('Errors: ' + str(serializer.data))

    return Response(serializer.data)


# -*- coding: utf-8 -*-
"""
  Name     : c7_03_random_OLS.py
  Book     : Python for Finance (2nd ed.)
  Publisher: Packt Publishing Ltd.
  Author   : Yuxing Yan
  Date     : 6/6/2017
  email    : yany@canisius.edu
             paulyxy@hotmail.com
"""

import numpy as np
import scipy as sp
import statsmodels.regression.linear_model as sm

n=100
sp.random.seed(12345)

y=[1,2,3,4,2,3,4]
x1=range(1,8)
x2=[4,2,-1,4,2,3,5]
x3=[0,2,3,4,2,4,-1]
x=[x1,x2,x3]
x=sm.add_constant(x)
#est = sm.OLS(formula='Sales ~ TV + Radio', data=df_adv).fit()
results=sm.OLS(y,x1).fit()
print(results.summary())


# code Sample from the tutorial at https://learncodeshare.net/2015/06/02/basic-crud-operations-using-cx_oracle/
# The following resets the data to the initial state for the tutorial.

import cx_Oracle
import os
connectString = os.getenv('DB_CONNECT') # The environment variable for the connect string: DB_CONNECT=user/password@database
con = cx_Oracle.connect(connectString)
cur = con.cursor()

# Delete rows
statement = 'delete from lcs_pets'
cur.execute(statement)

# Delete rows
statement = 'delete from lcs_people'
cur.execute(statement)

# Reset Identity Coulmn
statement = 'alter table lcs_people modify id generated BY DEFAULT as identity (START WITH 3)'
cur.execute(statement)

# Insert default rows
rows = [{'id':1, 'name':'Bob', 'age':35, 'notes':'I like dogs'}, {'id':2, 'name':'Kim', 'age':27, 'notes':'I like birds'}]
cur.bindarraysize = 2
cur.executemany("insert into lcs_people(id, name, age, notes) values (:id, :name, :age, :notes)", rows)
con.commit()

# Insert default rows
rows = [{'id':1, 'name':'Duke', 'owner':1, 'type':'dog'}, {'id':2, 'name':'Pepe', 'owner':2, 'type':'bird'}]
cur.bindarraysize = 2
cur.executemany("insert into lcs_pets (id, name, owner, type) values (:id, :name, :owner, :type)", rows)
con.commit()

cur.close()


from app import Trie, cheapest_operator
import pytest

operators = Trie()
operators.insert({"1":0.92})

@pytest.mark.parametrize("test_input,expected", [("1",0.92)])
def test_trie_insert(test_input, expected):
    """test insert function """
    assert operators.lookup('1734567890') == expected

@pytest.mark.parametrize("test_input,expected", [("1",0.92)])
def test_trie_lookup(test_input, expected):
    # trie.insert({'1': 0.92})
    assert operators.lookup(test_input) == expected
    assert operators.lookup(test_input) is expected
    # assert trie.lookup('') is None

def test_cheapest_operator():
    operators_all = {
            'operatorA': operators,
        }
    number = '234567890'
    cheapest = cheapest_operator(number=number, operators=operators_all)
    assert cheapest[0] is None


from create_bot import db
from aiogram.types import InlineKeyboardMarkup, InlineKeyboardButton, ReplyKeyboardMarkup, KeyboardButton

menu_markup = InlineKeyboardMarkup(
    inline_keyboard=[
        [InlineKeyboardButton("Я подтверждаю ▶", callback_data='i_approve')]
    ]
)

form_conditional = ReplyKeyboardMarkup(
    [
        ["online", "offline"]
    ], resize_keyboard=True
)

contact = ReplyKeyboardMarkup(
    resize_keyboard=True,
    keyboard=[
        [KeyboardButton("Номер телефона",request_contact=True)]
    ]
)

admin_markup = InlineKeyboardMarkup(
    inline_keyboard=[
        [InlineKeyboardButton("ОТПРАВИТЬ СООБЩЕНИЕ", callback_data='send_message')],
        [InlineKeyboardButton("ИЗМЕНИТЬ ТЕКСТ", callback_data="change_text")],
        [InlineKeyboardButton("ДОБАВИТЬ ВОПРОС", callback_data='add_question')],
        [InlineKeyboardButton('РЕДАКТИРОВАТЬ ВОПРОС', callback_data='edit_questions')]
    ]
)

optinos = InlineKeyboardMarkup(inline_keyboard=[
    [InlineKeyboardButton("РЕДАКТИРОВАТЬ ТЕКСТ", callback_data='edit_text')],
    [InlineKeyboardButton("ИЗМЕНИТЬ ТИП ОТВЕТЫ", callback_data='edit_type_answer')],
    [InlineKeyboardButton("УДАЛИТЬ ВОПРОС", callback_data='delete')],
    [InlineKeyboardButton("ВЕРНУТЬСЯ К АДМИНАМ ПАНЕЛИ", callback_data='exit')]
])

def list_texts ():
    markup = InlineKeyboardMarkup()
    for key, in db.sqlite('SELECT key FROM Texts'):
        markup.add(InlineKeyboardButton(key, callback_data=f'change_{key}'))
    markup.add(InlineKeyboardButton("НАЗАД", callback_data='admin'))
    return markup

def list_question_markup():
    markup = InlineKeyboardMarkup()
    for key,rowid in db.sqlite('SELECT question, rowid FROM Questions'):
        markup.add(InlineKeyboardButton(key, callback_data=f'change_{rowid}'))
    markup.add(InlineKeyboardButton("НАЗАД", callback_data='exit'))
    return markup


"""
.. module:: SearchIndexWeight

Rocchio
*************

:Description: Roochio

    Performs a AND query for a list of words (--query) in the documents of an index (--index)
    You can use word^number to change the importance of a word in the match

    --nhits changes the number of documents to retrieve

:Authors:
    Diego Delgado
    Oriol Vidal

:Version:
    666
:Created on: 30/10/2019

"""

from elasticsearch import Elasticsearch
from elasticsearch.exceptions import NotFoundError

import argparse
import numpy as np
import bisect

from elasticsearch_dsl import Search
from elasticsearch_dsl.query import Q
from elasticsearch.client import CatClient

__author__ = 'DDED'
def mutableQuery2Array (mutableQuery):
    '''
    Pasa de un array de tuplas (términos, pesos) a un array de "términos^pesos"
    '''
    query = []
    for subMutableQuery in mutableQuery:
        query.append(subMutableQuery[0]+"^"+str(subMutableQuery[1]))
    return query

def getMutableQuery(query):
    '''
    Pasa de un array de "términos^pesos" a un array de de tuplas(términos, pesos)
    '''
    mutableQuery = []
    for subQuery in query:
        mutableSubQuery = subQuery.split("^")
        if len(mutableSubQuery) == 1:
            mutableQuery.append((mutableSubQuery[0], 1))
        else:
            mutableQuery.append((mutableSubQuery[0], int(mutableSubQuery[1])))
    return mutableQuery

def doc_count(client, index):
    """
    Returns the number of documents in an index
    """
    return int(CatClient(client).count(index=[index], format='json')[0]['count'])

def document_term_vector(client, index, id):
    """
    Returns the term vector of a document and its statistics a two sorted list of pairs (word, count)
    The first one is the frequency of the term in the document, the second one is the number of documents
    that contain the term
    """
    termvector = client.termvectors(index=index, id=id, fields=['text'],
                                    positions=False, term_statistics=True)
    file_td = {}
    file_df = {}
    if 'text' in termvector['term_vectors']:
        for t in termvector['term_vectors']['text']['terms']:
            file_td[t] = termvector['term_vectors']['text']['terms'][t]['term_freq']
            file_df[t] = termvector['term_vectors']['text']['terms'][t]['doc_freq']
    return sorted(file_td.items()), sorted(file_df.items())

def toTFIDF(client, index, file_id):
    """
    Returns the term weights of a document
    """
    # Get document terms frequency and overall terms document frequency
    file_tv, file_df = document_term_vector(client, index, file_id)
    max_freq = max([f for _, f in file_tv])
    dcount = doc_count(client, index)
    tfidfw = []
    for (t, w),(_, df) in zip(file_tv, file_df):
        tf = w / max_freq
        idf = np.log2(dcount/df)
        weight = tf * idf
        tfidfw.append((t,weight))
    return normalize(tfidfw)

def normalize(tw):
    """
    Normalizes the weights in t so that they form a unit-length vector
    It is assumed that not all weights are 0
    """
    sumw = 0
    for (_,w) in tw:
        sumw += w**2
    norm = np.sqrt(sumw)
    tw = [(t,w/norm) for (t,w) in tw]
    return tw

def queryAux(s, mutableQuery, k):
    '''
    Para simplificar la llamada a la query se ha dejado resumida en una función
    '''
    query = mutableQuery2Array(mutableQuery)
    q = Q('query_string',query=query[0])
    for i in range(1, len(query)):
        q &= Q('query_string',query=query[i])
    s = s.query(q)
    response = s[0:k].execute()
    relevantDocsIndex = []
    paths = []
    for r in response:  # only returns a specific number of results
        relevantDocsIndex.append(r.meta.id)
        paths.append(r.path)
    print(paths)
    return(relevantDocsIndex)

def add_weight (l,term,weight):
    '''
    Añade un término a la lista ordenada por pesos y elimina el de menos peso
    '''
    del l[0]
    lo = 0
    hi = len(l)
    while lo < hi:
        mid = (lo+hi)//2
        if weight < l[mid][1]: hi = mid
        else: lo = mid+1
    l.insert(lo, (term,weight))

def getRRelevantTerms(termsDictionary, R):
    '''
    Consigue los R términos relevantes a partir del diccionario[términos] = pesos
    '''
    size = 0
    newMutableSubQuery = []
    # se crea un array de tuplas(término, peso) de tamaño R
    for term in termsDictionary:
        if size == R:
            break
        else:
            size +=1
            newMutableSubQuery.append((term, termsDictionary[term]))
            # no se pueden los elementos de un dictionary iterando, así que modifico los valores a -1
            termsDictionary[term] = -1
    # se ordena por peso
    newMutableSubQuery = sorted(newMutableSubQuery, key = lambda x:x[1])

    for term in termsDictionary:
        if termsDictionary[term]>newMutableSubQuery[0][1]:
            add_weight(newMutableSubQuery, term, termsDictionary[term])
    return newMutableSubQuery
def rocchio(index, client, s, mutableQuery,k,nrounds,R, alpha, beta):
    for _ in range(nrounds):
        relevantDocsIndex = queryAux(s, mutableQuery, k)
        termsDictionary = {}
        for relevantDocIndex in relevantDocsIndex:
            # suma los pesos de los términos comunes del vector tfidf
            for weightTerm in toTFIDF(client, index, relevantDocIndex):
                if weightTerm[0] in termsDictionary:
                    termsDictionary[weightTerm[0]] += beta*weightTerm[1]/k
                else:
                    termsDictionary[weightTerm[0]] = beta*weightTerm[1]/k
        for mutableSubQuery in mutableQuery:
            # suma los pesos de la query actual
            if mutableSubQuery[0] in termsDictionary:
                termsDictionary[mutableSubQuery[0]] += alpha*mutableSubQuery[1]
            else:
                termsDictionary[mutableSubQuery[0]] = alpha*mutableSubQuery[1]
        mutableQuery = getRRelevantTerms(termsDictionary, R)
    # print para debugar la salida de cada iteración del bucle
        #print(mutableQuery)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--index', default=None, help='Index to search')
    parser.add_argument('--nrounds', default=15, type=int, help='Number of rounds to execute Rocchio')
    parser.add_argument('--k', default=20, type=int, help='Number of relevant documents to return')
    parser.add_argument('--R', default=500, type=int, help='Number of relevant terms to search')
    parser.add_argument('--alpha', default=0.9, type=float, help='Number of relevant terms to search')
    parser.add_argument('--beta', default=0.1, type=float, help='Number of relevant terms to search')
    parser.add_argument('--query', default=None, nargs=argparse.REMAINDER, help='List of words to search')
    args = parser.parse_args()

    index = args.index
    query = args.query
    nrounds = args.nrounds
    R = args.R
    alpha = args.alpha
    beta = args.beta
    k = args.k
    mutableQuery = getMutableQuery(query)
    try:
        client = Elasticsearch()
        s = Search(using=client, index=index)

        if mutableQuery is not None:
            rocchio(index ,client, s, mutableQuery, k, nrounds, R, alpha, beta)
        else:
            print('No query parameters passed')

    except NotFoundError:
        print(f'Index {index} does not exists')


import pywhatkit
import time
import webbrowser as web
from selenium import webdriver
from simon.accounts.pages import LoginPage
from simon.chat.pages import ChatPage
from simon.chats.pages import PanePage
from simon.header.pages import HeaderPage

from simon.accounts.pages import LoginPage
from simon.header.pages import HeaderPage
from simon.pages import BasePage
#poetry run pytest tests/test_whatsapp.py::test_send_msg -s
def test_send_msg():
    pywhatkit.sendwhatmsg_instantly("+56975858852", "Este mensaje viene del pasado y se proyecta en el futuro", 10, True, 5)

#poetry run pytest tests/test_whatsapp.py::test_read_msg  -s
def test_read_msg():
    # Creating the driver (browser)
    driver = webdriver.Chrome()
    driver.maximize_window()

    # 1. Login
    #       and uncheck the remember check box
    #       (Get your phone ready to read the QR code)
    login_page = LoginPage(driver)
    login_page.load()
    login_page.remember_me = True
    time.sleep(7)

    # 2. The base page is inherited by all pages
    #       and here you can check whether any
    #       page is available on the screen of
    #       the browser.

    # we don't need to load the pages as whatsapp
    # web works as one-page web app
    base_page = BasePage(driver)
    base_page.is_title_matches()
    base_page.is_welcome_page_available()
    base_page.is_nav_bar_page_available()
    base_page.is_search_page_available()
    base_page.is_pane_page_available()
    # chat is only available after you open one
    base_page.is_chat_page_available()

    # 3. Logout
    header_page = HeaderPage(driver)
    header_page.logout()

    # Close the browser
    driver.quit()

#https://github.com/Fantaso/whatsapp-web
#poetry run pytest tests/test_whatsapp.py::test_base -s
def test_base():
    # Creating the driver (browser)
    driver = webdriver.Chrome()
    browser = web.get('MacOSX')
    driver.maximize_window()
    # 1. Login
    #       and uncheck the remember check box
    #       (Get your phone ready to read the QR code)
    login_page = LoginPage(driver)
    login_page.load()
    login_page.remember_me = False
    time.sleep(7)


from django.urls import path # if we want to define path we need to import this

from . import views # We are going to route url's to views's function so we need to import this

urlpatterns=[

    path('',views.index,name='listings'),  # /listings i ifade eder
    path('<int:listing_id>',views.listing,name='listing'),  #/listings/23 gibi olmasını istiyoruz. listing_id parametresine erisiyoruz
    path('search',views.search,name='search'),

    # simdi ana url dosyasina dicezki eger /listings ile baslayan bir sey gorursen gel bu listings app in icindeki urls.py dosyasina bak
]


import datetime
from model.dataframe import Dataframe
from model.recommend import Recommend
from bottle import Bottle, route, run, request, response

app = Bottle()
@app.hook("after_request")
def enable_cors():
    """
    You need to add some headers to each request.
    Don"t use the wildcard "*" for Access-Control-Allow-Origin in production.
    """
    response.headers["Access-Control-Allow-Origin"] = "*"
    response.headers["Access-Control-Allow-Methods"] = "PUT, GET, POST, DELETE, OPTIONS"
    response.headers["Access-Control-Allow-Headers"] = "Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token"

@app.route("/test", method=["OPTIONS", "POST"])
def hello():
    return "O servidor está online! Data/hora atual: " + str(datetime.datetime.now())

@app.route("/smartbuy", method=["POST"])
def getRecommendation():
    try:
        clientId = int(request.params.get("clientId"))
        month = int(request.params.get("month"))

        if (month < 1 or month > 12):
            return "Erro: mês inválido. Valor deve estar entre 1 e 12."

        if (clientId < 1):
            return "Erro: categoria inválida. Valor deve ser maior que zero."

        rc = Recommend("tcc", "processed")
        return rc.get(clientId, month)

    except Exception as ex:
        print(ex)
        return str(ex)

if __name__ == "__main__":
    from optparse import OptionParser
    parser = OptionParser()
    parser.add_option("--host", dest="host", default="localhost",
                      help="hostname or ip address", metavar="host")
    parser.add_option("--port", dest="port", default=8080,
                      help="port number", metavar="port")
    (options, args) = parser.parse_args()
    run(app, host=options.host, port=int(options.port))

# run(host="localhost", port=8080, debug=True)


import torch
import torch.nn as nn
import torch.nn.functional as F
from base_net import BaseNet

# the net has a fixed architecture, but each layer's hyperparameters
# can be altered by passing a dict to this class

class CNN(BaseNet):

    def __init__(self, params, actions, input_shape=(4, 64, 64)):
        super(CNN, self).__init__()
        self.actions = actions
        self.n_frames = input_shape[0]
        self.l = {}
        self.l["conv1"] = {
            "layer": nn.Conv2d(input_shape[0], 32, [8,8]),
            "w": input_shape[1] - 8 + 1,
            "h": input_shape[2] - 8 + 1,
            "c": 32
        }
        self.l["conv2"] = {
            "layer": nn.Conv2d(self.l["conv1"]["c"], 64, [4,4]),
            "w": self.l["conv1"]["w"] - 4 + 1,
            "h": self.l["conv1"]["h"] - 4 + 1,
            "c": 64
        }
        self.l["fc1"] = {
            "layer": nn.Linear(self.l["conv2"]["w"] * self.l["conv2"]["h"] * self.l["conv2"]["c"], 128),
            "f": 128
        }
        self.l["out"] = {
            "layer": nn.Linear(self.l["fc1"]["f"], self.actions),
            "f": self.actions
        }

    def forward(self, i):
        x = F.relu(self.l["conv1"]["layer"](i))
        print(x.shape)
        x = F.relu(self.l["conv2"]["layer"](x))
        print(x.shape)
        x = x.flatten(1)
        print(x.shape)
        print(self.l["fc1"]["layer"].in_features, self.l["conv2"]["h"], self.l["conv2"]["w"], self.l["conv2"]["c"])
        x = F.relu(self.l["fc1"]["layer"](x))
        print(x.shape)
        q = self.l["out"]["layer"](x)
        return q

    def loss(self, q):
        pass

# net = CNN(None, 5)
# t = torch.rand((1, 4, 64, 64))
# print(net.forward(t))


# "coding:utf-8"
#
#***********************************************************************
#    Time           :2020年6月6日23:10:51
#    Author         :Kangruojin
#    Email          :mailbox_krj@163.com
#    Compile        :Python3.8
#    version        :1.1
#    Description    :外星人类
#    FileName       :alien.py
#***********************************************************************
import pygame
from pygame.sprite import Sprite
from random import randint

from surface import g_screen_color_dict as color

class Alien(Sprite):
    #外星人初始化
    def __init__(self, alien_attr, screen):
        super(Alien, self).__init__()
        self.screen = screen
        self.image = pygame.image.load(r"images\alien.bmp")
        self.rect = self.image.get_rect()

        #初始化坐标
        self.x = self.rect.centerx = alien_attr["pos_x"]
        self.y = self.rect.top = alien_attr["pos_y"]
        self.init_x = alien_attr["pos_x"]

        #初始化移动速率
        self.speed_x = alien_attr["speed_x"]
        self.speed_y = alien_attr["speed_y"]

        #初始化移动统计和频率
        self.move_freq = alien_attr["interval"]
        self.move_count = 0

    #绘制外星人
    def __blitme__(self):
        #print(self.rect)
        self.screen.blit(self.image, self.rect)

    def __move__(self):
        limit = self.rect.width / 2
        self.move_count += 1
        if (self.move_count % self.move_freq == 0):
            self.x = self.init_x + self.speed_x * randint(-limit, limit)
            self.y = self.y + self.speed_y
            self.rect.centerx = int(self.x)
            self.rect.top = int(self.y)


#!/usr/bin/env python3

import os
import unittest
import logging
from lofar.sas.resourceassignment.resourceassignmentestimator.service import ResourceEstimatorHandler
from lofar.sas.resourceassignment.resourceassignmentestimator.resource_estimators import ObservationResourceEstimator
from unittest import mock

class TestObservationResourceEstimator(unittest.TestCase):
    def test_raises_ValueError_on_incomplete_parset(self):
        parset = {}

        estimator = ObservationResourceEstimator()

        with self.assertRaises(ValueError):
            estimator.verify_and_estimate(parset)

    def generate_complete_parset(self):
        parset = {'Observation.sampleClock': None,
                  'Observation.startTime': "2019-03-04 13:00:00",
                  'Observation.stopTime': "2019-03-04 14:00:00",
                  'Observation.antennaSet': "LBA_INNER",
                  'Observation.nrBeams': 1,
                  'Observation.Beam[0].subbandList': [40, 41],
                  'Observation.nrBitsPerSample': 16,
                  'Observation.VirtualInstrument.stationList': ["CS001"],
                  'Observation.ObservationControl.OnlineControl.Cobalt.Correlator.nrChannelsPerSubband': 64,
                  'Observation.ObservationControl.OnlineControl.Cobalt.Correlator.integrationTime': 1,
                  'Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.flysEye': None,
                  'Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.timeIntegrationFactor': None,
                  'Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.timeIntegrationFactor': None,
                  'Observation.DataProducts.Output_Correlated.enabled': True,
                  'Observation.DataProducts.Output_Correlated.identifications': None,
                  'Observation.DataProducts.Output_Correlated.storageClusterName': None,
                  'Observation.DataProducts.Output_CoherentStokes.enabled': False,
                  'Observation.DataProducts.Output_CoherentStokes.identifications': None,
                  'Observation.DataProducts.Output_CoherentStokes.storageClusterName': None,
                  'Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.CoherentStokes.which': None,
                  'Observation.DataProducts.Output_IncoherentStokes.enabled': False,
                  'Observation.DataProducts.Output_IncoherentStokes.identifications': None,
                  'Observation.DataProducts.Output_IncoherentStokes.storageClusterName': None,
                  'Observation.ObservationControl.OnlineControl.Cobalt.BeamFormer.IncoherentStokes.which': None,
                  }

        return parset

    def test_does_not_raise_ValueError_on_complete_parset(self):
        parset = self.generate_complete_parset()

        estimator = ObservationResourceEstimator()

        try:
            estimator.verify_and_estimate(parset)
        except ValueError:
            self.fail("estimator.verify_and_estimate(parset) raised ValueError unexpectedly!")

    def test_reports_error_when_no_output_was_enabled(self):
        parset = self.generate_complete_parset()

        parset['Observation.DataProducts.Output_Correlated.enabled'] = False
        parset['Observation.DataProducts.Output_CoherentStokes.enabled'] = False
        parset['Observation.DataProducts.Output_IncoherentStokes.enabled'] = False

        estimator = ObservationResourceEstimator()

        estimate = estimator.verify_and_estimate(parset)

        self.assertEqual(
            ['Produced observation resource estimate list has no data product estimates!'],
            estimate['errors'])

    def test_raise_reports_error_when_no_subbands_are_given(self):
        parset = self.generate_complete_parset()

        parset['Observation.DataProducts.Output_Correlated.enabled'] = True
        parset['Observation.Beam[0].subbandList'] = []

        estimator = ObservationResourceEstimator()

        estimate = estimator.verify_and_estimate(parset)

        self.assertTrue("Correlated data output enabled, but empty subband list for sap 0" in
            estimate['errors'])

    def test_reports_error_when_stationlist_is_empty(self):
        parset = self.generate_complete_parset()

        parset['Observation.DataProducts.Output_Correlated.enabled'] = True
        parset['Observation.VirtualInstrument.stationList'] = []

        estimator = ObservationResourceEstimator()

        estimate = estimator.verify_and_estimate(parset)

        self.assertTrue("Observation.VirtualInstrument.stationList is empty" in
            estimate['errors'])

    def test_reports_error_when_nrbeams_less_than_one(self):
        parset = self.generate_complete_parset()

        parset['Observation.DataProducts.Output_Correlated.enabled'] = True
        parset['Observation.nrBeams'] = 0

        estimator = ObservationResourceEstimator()

        estimate = estimator.verify_and_estimate(parset)

        self.assertTrue("Correlated data output enabled, but nrBeams < 1" in
            estimate['errors'])

class TestResourceEstimatorHandler(unittest.TestCase):
    def setUp(self):
        self.unique_otdb_id = 0
        self.data_sets_dir = os.path.join(os.environ.get('srcdir', os.path.dirname(os.path.abspath(__file__))),
                                          "data_sets")

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.ObservationResourceEstimator')
    def test_ObservationResourceEstimator_gets_called_on_observation_parset(self, ObservationResourceEstimator_mock):
        specification_tree = self.generate_observation_spec()

        resource_estimator_handler = ResourceEstimatorHandler()

        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        ObservationResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_observation_spec(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_beam_observation')
        task_type = 'observation'
        task_subtype = 'bfmeasurement'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        return specification_tree

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.ReservationResourceEstimator')
    def test_ReservationResourceEstimator_gets_called_on_reservation_maintenance_parset(self, ReservationResourceEstimator_mock):
        specification_tree = self.generate_maintenance_reservation_spec()

        resource_estimator_handler = ResourceEstimatorHandler()

        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        ReservationResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_maintenance_reservation_spec(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_maintenance_reservation')
        task_type = 'reservation'
        task_subtype = 'maintenance'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        return specification_tree

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.ReservationResourceEstimator')
    def test_ReservationResourceEstimator_gets_called_on_reservation_project_parset(self, ReservationResourceEstimator_mock):
        specification_tree = self.generate_project_reservation_spec()

        resource_estimator_handler = ResourceEstimatorHandler()
        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        ReservationResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_project_reservation_spec(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_project_reservation')
        task_type = 'reservation'
        task_subtype = 'project'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        return specification_tree

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.CalibrationPipelineResourceEstimator')
    def test_CalibrationPipelineResourceEstimator_gets_called_on_calibrator_pipeline_parset(self, CalibrationPipelineResourceEstimator_mock):
        specification_tree = self.generate_calibration_pipeline_spec()

        resource_estimator_handler = ResourceEstimatorHandler()
        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        CalibrationPipelineResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_calibration_pipeline_spec(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_calibration_pipeline')
        task_type = 'pipeline'
        task_subtype = 'calibration pipeline'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        self.add_predecessor_to_specification_tree(os.path.join(self.data_sets_dir,
                                                                't_resource_estimator.in_calibration_pipeline_predecessor_558022'),
                                                   # predecessor also used for imaging pipeline test
                                                   'observation',
                                                   'bfmeasurement',
                                                   specification_tree['predecessors'])
        return specification_tree

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.CalibrationPipelineResourceEstimator')
    def test_ImagePipelineResourceEstimator_gets_called_on_averaging_pipeline_parset(self, CalibrationPipelineResourceEstimator_mock):
        specification_tree = self.generate_averaging_pipeline_spec()

        resource_estimator_handler = ResourceEstimatorHandler()
        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        CalibrationPipelineResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_averaging_pipeline_spec(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_preprocessing_pipeline')
        task_type = 'pipeline'
        task_subtype = 'averaging pipeline'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        # Pipelines need a predecessor so give it one
        predecessor_data_set_filepath = os.path.join(self.data_sets_dir,
                                                     't_resource_estimator.in_interferometer_observation')
        predecessor_task_type = 'observation'
        predecessor_task_subtype = 'interferometer'
        self.add_predecessor_to_specification_tree(predecessor_data_set_filepath,
                                                   predecessor_task_type,
                                                   predecessor_task_subtype,
                                                   specification_tree['predecessors'])
        return specification_tree

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.ImagePipelineResourceEstimator')
    def test_ImagePipelineResourceEstimator_gets_called_on_imaging_pipeline_parset(self, ImagePipelineResourceEstimator_mock):
        data_set_filepath = os.path.join(self.data_sets_dir, 't_resource_estimator.in_imaging_pipeline')
        task_type = 'pipeline'
        task_subtype = 'imaging pipeline'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)

        self.add_predecessor_to_specification_tree(os.path.join(self.data_sets_dir,
                                                   't_resource_estimator.in_calibration_pipeline_predecessor_558022'),  # predecessor also used for calibration pipeline test
                                                   'observation',
                                                   'bfmeasurements',
                                                   specification_tree['predecessors'])

        resource_estimator_handler = ResourceEstimatorHandler()
        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        ImagePipelineResourceEstimator_mock().verify_and_estimate.assert_called()

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.ImagePipelineResourceEstimator')
    def test_ImagePipelineResourceEstimator_gets_called_on_imaging_pipeline_msss_parset(self, ImagePipelineResourceEstimator_mock):
        specification_tree = self.generate_imageing_pipeline_mss()

        resource_estimator_handler = ResourceEstimatorHandler()
        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        ImagePipelineResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_imageing_pipeline_mss(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_imaging_pipeline')
        task_type = 'pipeline'
        task_subtype = 'imaging pipeline msss'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        self.add_predecessor_to_specification_tree(os.path.join(self.data_sets_dir,
                                                                't_resource_estimator.in_calibration_pipeline_predecessor_558022'),
                                                   # predecessor also used for calibration pipeline test
                                                   'observation',
                                                   'bfmeasurements',
                                                   specification_tree['predecessors'])
        return specification_tree

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.LongBaselinePipelineResourceEstimator')
    def test_LongBaselinePipelineResourceEstimator_get_called_on_long_baseline_pipeline_parset(self, LongBaselinePipelineResourceEstimator_mock):
        specification_tree = self.generate_long_baseline_pipeline_spec()

        resource_estimator_handler = ResourceEstimatorHandler()
        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        LongBaselinePipelineResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_long_baseline_pipeline_spec(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_long_baseline_pipeline')
        golden_output_filepath = os.path.join(self.data_sets_dir,
                                              't_resource_estimator.out_long_baseline_pipeline')
        task_type = 'pipeline'
        task_subtype = 'long baseline pipeline'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        tree = specification_tree
        predecessor_tree = self.get_specification_tree(
            os.path.join(
                self.data_sets_dir,
                't_resource_estimator.in_long_baseline_pipeline_predecessor_556601'),
            'pipeline',
            'averaging pipeline')
        tree['predecessors'].append(predecessor_tree)
        tree = predecessor_tree
        predecessor_tree = self.get_specification_tree(
            os.path.join(
                self.data_sets_dir,
                't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429'),
            'pipeline',
            'calibration pipeline')
        tree['predecessors'].append(predecessor_tree)
        tree = predecessor_tree
        predecessor_tree_branch_a = self.get_specification_tree(
            os.path.join(
                self.data_sets_dir,
                't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429_556373'),
            'pipeline',
            'calibration pipeline')
        tree['predecessors'].append(predecessor_tree_branch_a)
        predecessor_tree_branch_b = self.get_specification_tree(
            os.path.join(
                self.data_sets_dir,
                't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429_556375'),
            'observation',
            'bfmeasurement')
        tree['predecessors'].append(predecessor_tree_branch_b)
        predecessor_tree = self.get_specification_tree(
            os.path.join(
                self.data_sets_dir,
                't_resource_estimator.in_long_baseline_pipeline_predecessor_556601_556429_xxxxxx_556371'),
            'observation',
            'bfmeasurement')
        predecessor_tree_branch_a['predecessors'].append(predecessor_tree)
        predecessor_tree_branch_b['predecessors'].append(predecessor_tree)
        return specification_tree

    @mock.patch('lofar.sas.resourceassignment.resourceassignmentestimator.service.PulsarPipelineResourceEstimator')
    def test_PulsarPipelineResourceEstimator_get_called_on_a_pulsar_pipeline_parset(self, PulsarPipelineResourceEstimator_mock):
        specification_tree = self.generate_pulsar_pipeline_spec()

        resource_estimator_handler = ResourceEstimatorHandler()
        resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        PulsarPipelineResourceEstimator_mock().verify_and_estimate.assert_called()

    def generate_pulsar_pipeline_spec(self):
        data_set_filepath = os.path.join(self.data_sets_dir,
                                         't_resource_estimator.in_pulsar_pipeline')
        task_type = 'pipeline'
        task_subtype = 'pulsar pipeline'
        specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        # Pulsar pipelines need a beamformer observation as their predecessor
        predecessor_data_set_filepath = os.path.join(self.data_sets_dir,
                                                     't_resource_estimator.in_beam_observation')
        predecessor_task_type = 'observation'
        predecessor_task_subtype = 'bfmeasurement'
        self.add_predecessor_to_specification_tree(predecessor_data_set_filepath,
                                                   predecessor_task_type,
                                                   predecessor_task_subtype,
                                                   specification_tree['predecessors'])
        return specification_tree

    def test_add_otdb_id_to_output(self):
        specification_tree = self.generate_averaging_pipeline_spec()

        resource_estimator_handler = ResourceEstimatorHandler()

        estimate = resource_estimator_handler._get_estimated_resources(specification_tree=specification_tree)

        estimate_list = estimate['estimates']

        for est in estimate_list:
            output_files = est.get('output_files')
            for dptype in output_files:
                for dptype_dict in output_files[dptype]:
                    self.assertEqual(specification_tree['otdb_id'], dptype_dict['properties'][dptype + '_otdb_id'])

    def add_predecessor_to_specification_tree(self, data_set_filepath, task_type, task_subtype, predecessor_list):
        """ Adds a predecessor specification tree to an existing specification tree

        :param data_set_filepath: predecessor specification data set filepath
        :param task_type: predecessor task's type
        :param task_subtype: predecessor task's subtype
        :param predecessor_list: specification tree to with to add predecessor specification tree
        """
        predecessor_specification_tree = self.get_specification_tree(data_set_filepath, task_type, task_subtype)
        predecessor_list.append(predecessor_specification_tree)

    def get_specification_tree(self, data_set_filepath, task_type, task_subtype):
        """ Create a specification tree from a specification data set text file

        :param data_set_filepath: specification data set's filepath
        :param task_type: the task's type
        :param task_subtype: the task's subtype
        :return: specification tree
        """
        with open(data_set_filepath) as spec_file:
            parset = eval(spec_file.read().strip())
            return {
                'otdb_id': self.get_unique_otdb_id(),
                'task_type': task_type,
                'task_subtype': task_subtype,
                'predecessors': [],
                'specification': parset
            }

    def get_unique_otdb_id(self):
        """ Generates a unique OTDB ID (for use in parsets with predecessors)

        :return: unique OTDB ID
        """
        self.unique_otdb_id += 1
        return self.unique_otdb_id

    def get_golden_imate(self, golden_output_filepath, estimator_function=None, *estimator_args):
        """ Obtain the golden estimation from file (and create one if DO_GENERATE_GOLDEN_OUTPUTS is True)

        :param golden_output_filepath: the path to the golden estimate output file
        :param estimator_function: the estimator function to be called with estimator_args as it argument(S)
        :param estimator_args: the estimator function's arguments
        :return: the golden estimate
        """
        # Generate the golden output prior to fetching it if user requests so
        if DO_GENERATE_GOLDEN_OUTPUTS:
            estimation = estimator_function(*estimator_args)
            error_messages = self.get_uut_errors(estimation)
            # Make sure that there no errors are returned by uut
            if len(error_messages) == 0:
                self.store_datastructure(estimation, golden_output_filepath)
            else:
                raise Exception("\nThe uut reported errors:\n" + '\n- '.join(error_messages))

        # Fetch the golden output
        f = open(golden_output_filepath, "r")
        golden_output = f.read()
        f.close()

        # Remove trailing newline, and trailing and heading double-quotes
        stringified = golden_output.strip()
        stringified = stringified.strip('\n')
        stringified = stringified.strip('"')
        return stringified

    def store_datastructure(self, estimation={}, output_file=""):
        """ Stores the estimation data structure such that it can be used as golden output to verify against.

        :param estimation: resource estimator data structure
        :param output_file: file name to store the estimation to
        """
        output_filepath = os.path.join(self.data_sets_dir, output_file)
        f = open(output_filepath, 'w+')
        pprint(repr(estimation).strip(), stream=f)
        f.close()

if __name__ == '__main__':
    logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
    unittest.main()