import pandas as pd
import numpy as np
from scipy import spatial
import matplotlib.pyplot as plt
from sklearn import preprocessing
from sklearn.cluster import KMeans
import seaborn as sns

#feature vector
fv = np.load('Node_Features_Weekly.npy')
# Size 248 (weeks) x 136 (nodes) x 16 (features)

#past weeks/lags
w = 4

#average behavior
ab = np.zeros((248,136,16))
for t in range (w,248):
    ab[t,:,:] = np.mean(fv[t-w:t,:,:], axis = 0)

#dissimilarity
d = np.zeros((248,136))
#dissimilarity between fv and ab at every time
for i in range(0,136):
    for t in range (w,248):
        d[t,i] = spatial.distance.cosine(fv[t,i,:].reshape(16, 1), ab[t,i,:].reshape(16, 1))
d = np.nan_to_num(d)

#plot dissimilarity value for time (y-axis) and node (x-axis)
plt.figure(figsize = (20, 10))
a = sns.heatmap(d, vmin = 0, vmax = 1, xticklabels = 5, yticklabels = 10)
a.set(title = "Cosine Dissimilarity", xlabel = "Node Number", ylabel = "Week Number")

#k-means clustering of all values in d matrix - for finding thresholds
km = KMeans()
a = km.fit(d[w:,:].reshape(-1,1))

#number of clusters
print(a.n_clusters)
print("")

#center of clusters
print(a.cluster_centers_)

b = a.labels_.reshape(248-w,136)

k_clusters_bins = np.zeros((8,3))
for i in range(0,8):
    k_clusters_bins[i,0] = min(d[w:,:].reshape(-1,1)[a.labels_ == i])
    k_clusters_bins[i,1] = max(d[w:,:].reshape(-1,1)[a.labels_ == i])
    k_clusters_bins[i,2] = len(d[w:,:].reshape(-1,1)[a.labels_ == i])
k_clusters_bins = k_clusters_bins[k_clusters_bins[:, 0].argsort()]

ranges = [''] * 8
for i in range(0,8):
    ranges[i] = str(round(k_clusters_bins[i,0], 2))+"-"+str(round(k_clusters_bins[i,1], 2))
x = np.arange(8)
freq = k_clusters_bins[:,2]

plt.figure(figsize = (15, 5))
plt.bar(x, freq, align = 'center')
plt.xticks(x, ranges)
plt.xlabel('Cosine Dissimilarity')
plt.ylabel('Frequency')
plt.title('w = 12 weeks')
plt.show()

#location of anomalies for all combinations of w and tau
locs = [[] for i in range(4)]

#past weeks/lags
w = [4,6,8,12]
#dissimilarity threshold
tau = 0.07

for i in range(len(w)):
    #average behavior
    ab = np.zeros((248,136,16))
    for t in range (w[i],248):
        ab[t,:,:] = np.mean(fv[t-w[i]:t,:,:], axis = 0)

    #dissimilarity
    d = np.zeros((248,136))
    #dissimilarity between fv and ab at every time
    for n in range(0,136):
        for t in range (w[i],248):
            d[t,n] = spatial.distance.cosine(fv[t,n,:].reshape(16, 1), ab[t,n,:].reshape(16, 1))
            d = np.nan_to_num(d)

    locs[i] = np.argwhere(d >= tau)
    np.savetxt('Location'+str(w[i])+'.csv', locs[i], delimiter=',', fmt='%d')
    print(len(locs[i]))

#length of overlap in detected anomaly positions
print(len(np.array([x for x in set(tuple(x) for x in locs[0]) & set(tuple(x) for x in locs[1]) &\
                         set(tuple(x) for x in locs[2]) & set(tuple(x) for x in locs[3])])))


############################################################################
## This software is distributed for free, without warranties of any kind. ##
## Send bug reports or suggestions to hackbarth@gmail.com                 ##
############################################################################

import sys, os
import audioguide.util as util
import audioguide.descriptordata as descriptordata
import numpy as np

class sfsegment:
	def __init__(self, filename, startSec, endSec, descriptors, AnalInterface, envDb=+0, envAttackSec=0., envDecaySec=0., envSlope=1., envAttackenvDecayCushionSec=0.01):
		self.filename = util.verifyPath(filename, AnalInterface.searchPaths)
		self.printName = os.path.split(self.filename)[1] # short name for printing
		# filename-based descriptor info
		self.midiPitchFromFilename = descriptordata.getMidiPitchFromString(self.printName)
		self.rmsAmplitudeFromFilename, self.dynamicFromFilename = util.getDynamicFromFilename(self.printName, notFound=-1000)
		# other info
		self.soundfileExtension = os.path.splitext(self.filename)[1]
		self.soundfileTotalDuration, self.soundfileChns = AnalInterface.validateAnalResource(self.filename)
		self.segmentStartSec = startSec
		self.segmentEndSec = endSec
		self.envDb = util.getScaleDb(envDb, self)
		self.envAttack = envAttackSec
		self.envDecay = envDecaySec
		self.envSlope = envSlope
		# if startSec=None, it is begninning
		# if endSec=None, it is end of file
		if self.segmentStartSec == None: self.segmentStartSec = 0
		else: self.segmentStartSec = self.segmentStartSec
		if self.segmentEndSec == None: self.segmentEndSec = self.soundfileTotalDuration
		else: self.segmentEndSec = self.segmentEndSec
		self.segmentDurationSec = self.segmentEndSec-self.segmentStartSec
		# ensure length is at least 1 fram
		self.lengthInFrames = max(1, AnalInterface.getSegmentFrameLength(self.segmentDurationSec, self.filename))
		self.f2s = AnalInterface.f2s(1)
		##############################################################
		## check to make sure all user supplied values check out OK ##
		##############################################################
		self.testForInitErrors(AnalInterface)
		################
		## other shit ##
		################
		self.segmentHash = util.listToCheckSum([self.filename, self.segmentStartSec, self.segmentEndSec, self.envDb, self.envAttack, self.envDecay, self.envSlope])
		####################################
		## get information about envelope ##
		####################################
		self.envAttackSec = util.getDurationFromValueOrString(self.envAttack, self.segmentDurationSec)
		self.envDecaySec = util.getDurationFromValueOrString(self.envDecay, self.segmentDurationSec)
		if (self.envAttackSec+self.envDecaySec+envAttackenvDecayCushionSec) > self.segmentDurationSec:
			self.envDecaySec = self.segmentDurationSec-self.envAttackSec-envAttackenvDecayCushionSec
		self.envAttackFrames = int(round(AnalInterface.s2f(self.envAttackSec, self.filename)))
		self.envDecayFrames = int(round(AnalInterface.s2f(self.envDecaySec, self.filename)))
		if (self.envAttackFrames+self.envDecayFrames) > self.lengthInFrames:
			if self.envAttackFrames > self.envDecayFrames:
				self.envAttackFrames = self.lengthInFrames-self.envDecayFrames
			else:
				self.envDecayFrames = self.lengthInFrames-self.envAttackFrames
		if self.envAttackFrames <= 1 and self.envDecayFrames <= 1:
			self.envelopeMask = util.dbToAmp(self.envDb)
		else:
			self.envelopeMask = np.ones(self.lengthInFrames)
			if self.envAttackFrames > 0:
				self.envelopeMask[:self.envAttackFrames] = np.linspace(1/float(self.envAttackFrames), 1, num=self.envAttackFrames)
			if self.envDecayFrames > 0:
				self.envelopeMask[self.envDecayFrames*-1:] = np.linspace(1, 1/float(self.envDecayFrames), num=self.envDecayFrames)
			self.envelopeMask = np.power(self.envelopeMask, self.envSlope)
			self.envelopeMask *= util.dbToAmp(self.envDb)
		########################################
		## manage duration and time-in-frames ##
		########################################
		self.segmentStartFrame = AnalInterface.getSegmentStartInFrames(self.filename, self.segmentStartSec, self.segmentEndSec, self.lengthInFrames)

		###############################
		## initalise descriptor data ##
		###############################
		self.desc = descriptordata.container(descriptors, self) # for storing descriptor values from disk
		# tells us which descriptors are:
		#	SdifDescList - loaded from NYC analyses
		#	ComputedDescList - transformed from loaded descriptor data - delta, deltadelta, odf, etc.
		#	AveragedDescList - averaged descriptors from loaded descriptor data
		AnalDescList, ComputedDescList, AveragedDescList = self.desc.getDescriptorOrigins()

		for d in AnalDescList:
			self.desc[d.name] = AnalInterface.getDescriptorForsfsegment(self.filename, self.segmentStartFrame, self.lengthInFrames, d, self.envelopeMask)

		for dobj in ComputedDescList:
			self.desc[dobj.name] = descriptordata.DescriptorComputation(dobj, self, None, None)
		self.triggerinit = False
	###################################################
	def initThresholdTest(self, onsetDescriptorDict):
		# get min / max of trigger descriptors
		self.mins = {}
		self.maxs = {}
		for dname, weight in onsetDescriptorDict.items():
			self.mins[dname] = np.min(self.desc[dname][:])
			self.maxs[dname] = np.max(self.desc[dname][:])
		self.triggerinit = True
	###################################################
	def thresholdTest(self, time, onsetDescriptorDict):
		if not self.triggerinit: self.initThresholdTest(onsetDescriptorDict) # only once!
		trig = 0.
		weightsum = 0.
		for dname, weight in onsetDescriptorDict.items():
			value = self.desc[dname][time]
			if dname.find('power') != -1 or value < 0.:
				trigadd = value / self.maxs[dname]
			else:
				trigadd = value / self.mins[dname] # for non-power descriptors that are greater than 0.!
			trig += trigadd * weight
			weightsum += weight
		return util.ampToDb(trig)/weightsum
	#################################
	#################################
	def testForInitErrors(self, AnalInterface):
		# test self.filename
		oneframesec = AnalInterface.f2s(1)
		if not os.path.exists(self.filename):
			util.error('sfsegment init', 'file does not exist: \t%s\n'%self.filename)
		if self.soundfileExtension.lower() not in AnalInterface.validSfExtensions:
			util.error('sfsegment init', 'file is not an accepted soundfile type: \t%s\n'%self.filename)
		# test that startSec is sane
		if self.segmentStartSec < 0:
			util.error('sfsegment init', 'startSec is less than 0!!')
		if self.segmentStartSec >= self.segmentEndSec:
			util.error('sfsegment init', 'startSec is greater than its endSec!!')
		# test if requested read too long
		if self.segmentEndSec > self.soundfileTotalDuration+(oneframesec/2.):
			print('\n\nWARNING endSec (%.2f) is longer than the file\'s duration(%.2f)!!  Truncating to filelength.\n\n'%(self.segmentEndSec, self.soundfileTotalDuration))
			self.segmentEndSec = self.soundfileTotalDuration
			#util.error('sfsegment init', 'endSec is after the the file\'s duration!!')
	#################################
	#################################
	def __str__(self, length=50):
		prints = []
		for k, v in vars(self).items():
			valueStr = str(v)
			if len(valueStr) > length: valueStr = valueStr[:length-3]+'...'
			prints.append('\t%s : %s'%(k, valueStr))
		prints.sort() # alphabetize
		return '''<sfsegment object>\n'''+'\n'.join(prints)
	#################################

class corpusSegment(sfsegment):
	'''Inherits sfsegment and adds additional attributes
	used uniquely by corpus segments.'''
	def __init__(self, filename, startSec, endSec, envDb, envAttackSec, envDecaySec, envSlope, AnalInterface, concatFileName, userCpsStr, voiceID, midiPitchMethod, limitObjList, pitchfilter, scaleDistance, superimposeRule, transMethod, transQuantize, allowRepetition, restrictInTime, restrictOverlaps, restrictRepetition, postSelectAmpBool, postSelectAmpMin, postSelectAmpMax, postSelectAmpMethod, segfileData, metadata, clipDurationToTarget, instrTag, instrParams):
		# initalise the sound segment object
		sfsegment.__init__(self, filename, startSec, endSec, AnalInterface.requiredDescriptors, AnalInterface, envDb=envDb, envAttackSec=envAttackSec, envDecaySec=envDecaySec, envSlope=envSlope)
		# additional corpus-specific data
		self.userCpsStr = userCpsStr
		self.concatFileName = concatFileName
		self.voiceID = voiceID
		self.midiPitchMethod = midiPitchMethod
		self.limitObjList = limitObjList
		self.pitchfilter = pitchfilter
		self.scaleDistance = scaleDistance
		self.postSelectAmpBool = postSelectAmpBool
		self.postSelectAmpMin = postSelectAmpMin
		self.postSelectAmpMax = postSelectAmpMax
		self.postSelectAmpMethod = postSelectAmpMethod
		self.superimposeRule = superimposeRule
		self.transMethod = transMethod
		self.transQuantize = transQuantize
		self.allowRepetition = allowRepetition
		self.restrictOverlaps = restrictOverlaps
		self.restrictRepetition = restrictRepetition
		self.sim_accum = 0. # for similarity calculations
		self.segfileData = segfileData
		self.classification = 0
		self.metadata = metadata
		self.clipDurationToTarget = clipDurationToTarget
		self.instrTag = instrTag
		self.instrParams = instrParams
	###################################################
	def getValuesForSimCalc(self, tgtseg, tgtSeek, array_len, dobj, superimposeObj):
		tgtvals = tgtseg.desc[dobj.name].getnorm(tgtSeek, tgtSeek+array_len)
		if superimposeObj.calcMethod == "mixture" and dobj.is_mixable and tgtseg.has_been_mixed:
			if dobj.seg: d = dobj.parents[0]
			else:  d = dobj.name
			#############################
			## USE DESCRIPTOR MIXTURES ##
			#############################
			tgtrawvals = tgtseg.mixdesc[d][tgtSeek:tgtSeek+array_len]
			cpsrawvals = self.desc[d][:array_len]
			if dobj.describes_energy:
				mixedvals = tgtrawvals + cpsrawvals
			else:
				tgtrawpowers = tgtseg.mixdesc['power'][tgtSeek:tgtSeek+array_len]
				cpsrawpowers = self.desc['power'][:array_len]
				mixedpowers = (tgtrawpowers + cpsrawpowers)
				mixedvals = ((tgtrawvals*tgtrawpowers) + (cpsrawvals*cpsrawpowers)) / mixedpowers

			if dobj.seg: # segmented
				if dobj.describes_energy:
					averagedVal = np.average(mixedvals)
				else:
					try: averagedVal = np.average(mixedvals, weights=mixedpowers)
					except ZeroDivisionError: averagedVal = 0
				normedVal = descriptordata.normalize(averagedVal, self.desc[dobj.name].normdict['method'], self.desc[dobj.name].normdict)
				#print "mixed segmented UNNORMED", averagedVal, "NORMED", normedVal
				return tgtvals, normedVal
			else: # time-varying
				#print "mixed time-varying UNNORMED", mixedvals, "NROMED", normedvals
				normedvals = descriptordata.normalize(mixedvals, self.desc[dobj.name].normdict['method'], self.desc[dobj.name].normdict)
				return tgtvals, normedvals
		else: # not mixed
			if dobj.seg: # segmented
				return tgtvals, self.desc[dobj.name].getnorm(0, None)
			else: # time-varying
				return tgtvals, self.desc[dobj.name].getnorm(0, None)

class targetSegment(sfsegment):
	'''Inherits class of sfsegment and adds additional attributes
	used by target segments.'''
	def __init__(self, filename, segmentidx, startSec, endSec, envDb, envAttackSec, envDecaySec, envSlope, AnalInterface, midiPitchMethod):
		# initalise the sound segment object
		sfsegment.__init__(self, filename, startSec, endSec, AnalInterface.requiredDescriptors, AnalInterface, envDb=envDb, envAttackSec=envAttackSec, envDecaySec=envDecaySec, envSlope=envSlope)
		# additional target-specific data
		self.midiPitchMethod = midiPitchMethod
		self.classification = 0
		self.numberSelectedUnits = 0
		self.idx = segmentidx
	###################################################
	def initMixture(self, AnalInterface):
		self.mixdesc = descriptordata.container(AnalInterface.mixtureDescriptors, self)
		for dobj in AnalInterface.mixtureDescriptors:
			if dobj.seg: continue
			#	def setNorm(self, subtract, divide):
			self.mixdesc[dobj.name] = np.zeros(self.lengthInFrames) # array of zeros
		self.has_been_mixed = False
		self.originalPeak = self.desc['peakTime-seg'].get(0, None) # keep original, unsubtacted peak
	#################################
	def mixSelectedSamplesDescriptors(self, cpsh, cpsAmpScale, tgtsegSeek, AnalInterface, v=True):
		mix_dur = min(self.lengthInFrames-tgtsegSeek, cpsh.lengthInFrames)
		for d in AnalInterface.mixtureDescriptors:
			if not d.seg: # is time-varying
				self.mixdesc[d.name][tgtsegSeek:tgtsegSeek+mix_dur] = timeVaryingDescriptorMixture(self, tgtsegSeek, cpsh, 0, d, cpsAmpScale)
			else: # is segmented
				self.mixdesc[d.name].clear()
		self.has_been_mixed = True

def timeVaryingDescriptorMixture(tgtsegh, tgtseek, cpssegh, cpsseek, dobj, cpsAmpScale, v=False):
	mix_dur = min(tgtsegh.lengthInFrames-tgtseek, cpssegh.lengthInFrames-cpsseek)
	tgt_vals = tgtsegh.mixdesc[dobj.name][tgtseek:tgtseek+mix_dur]
	cps_vals = cpssegh.desc[dobj.name][cpsseek:cpsseek+mix_dur]
	if dobj.describes_energy: # powers are summed
		mixture = tgt_vals + (cps_vals*cpsAmpScale)
	elif dobj.name == 'zeroCross':
		mixture = np.maximum(tgt_vals, cps_vals)
	else: # power averaged
		tgt_amps = tgtsegh.mixdesc['power'][tgtseek:tgtseek+mix_dur]
		cps_amps = cpssegh.desc['power'][cpsseek:cpsseek+mix_dur]*cpsAmpScale
		mixture = ((tgt_vals*tgt_amps)+(cps_vals*cps_amps))/(tgt_amps + cps_amps)
	if v: # verbose printing
		maxxy = min(5, len(mixture))
		if dobj.describes_energy: # powers are summed
			print("SUM", dobj.name, tgtseek, mix_dur)
		else:
			print("MIXTURE", dobj.name, tgtseek, mix_dur)
		print("\tpastmix:", tgt_vals[0:maxxy])
		print("\tcorpus:", cps_vals[0:maxxy])
		print("\tnewmix:", mixture[0:maxxy])
	return mixture

class target: # the target
	def __init__(self, userOptsTargetObject, AnalInterface):
		self.filename = userOptsTargetObject.filename
		self.filename = util.verifyPath(self.filename, AnalInterface.searchPaths)
		self.filename = os.path.abspath(self.filename)
		self.startSec = userOptsTargetObject.start
		self.endSec = userOptsTargetObject.end

		# check for signal decomposition
		self.decompose = userOptsTargetObject.decompose
		if self.decompose != {}:
			if self.startSec == None: self.startSec = 0
			import audioguide.signaldecompose as signaldecompose
			newtargetpath, origtargetduration = signaldecompose.decomposeTargetSf(self.filename, self.startSec, self.endSec, self.decompose)
			self.decompose['origfilename'] = self.filename
			self.decompose['origduration'] = origtargetduration
			self.filename = newtargetpath

		self.segmentationThresh = userOptsTargetObject.thresh
		self.segmentationOffsetRise = userOptsTargetObject.offsetRise
		self.segmentationOffsetThreshAdd = userOptsTargetObject.offsetThreshAdd
		self.segmentationOffsetThreshAbs = userOptsTargetObject.offsetThreshAbs
		self.segmentationMinLenSec = userOptsTargetObject.minSegLen
		self.segmentationMaxLenSec = userOptsTargetObject.maxSegLen

		self.envDb = userOptsTargetObject.scaleDb
		self.midiPitchMethod = userOptsTargetObject.midiPitchMethod
		self.stretch = userOptsTargetObject.stretch
		self.segmentationFilepath = userOptsTargetObject.segmentationFilepath
		# multirise corpus segmentation ?
		self.multiriseBool = userOptsTargetObject.multiriseBool
		self.multirisePercentDev = userOptsTargetObject.multirisePercentDev
		self.multiriseSteps = userOptsTargetObject.multiriseSteps
	########################################
	def timeStretch(self, AnalInterface, ops, p):
		self.filename = util.initStretchedSoundfile(self.filename, self.startSec, self.endSec, self.stretch, AnalInterface.supervp_bin, p=p)
		self.startSec = 0 # this now gets reset, as starttime was considered when making the stretched file
		self.endSec = None # this now gets reset, as endtime was considered when making the stretched file
	########################################
	def initAnal(self, AnalInterface, ops, p):
		# Start by loading the entire target as an
		# sfsegment to get the whole amplitude envelope.
		# see if we need to time stretch the target file...
		if self.stretch != 1:
			self.timeStretch(AnalInterface, ops, p)
		# analise the whole target sound!
		self.whole = sfsegment(self.filename, self.startSec, self.endSec, AnalInterface.requiredDescriptors, AnalInterface, envDb=self.envDb)
		self.startSec = self.whole.segmentStartSec
		self.endSec = self.whole.segmentEndSec
		self.whole.midiPitchMethod = self.midiPitchMethod
		self.lengthInFrames = self.whole.lengthInFrames

		# SEGMENTATION DATA CONTAINERS
		self.segs = []
		self.segmentationInFrames = []
		self.segmentationInOnsetFrames = []
		self.segmentationInSec = []
		self.extraSegmentationData = []
		self.seglengths = []

		# SEGMENTATION
		power = AnalInterface.getDescriptorColumn(self.filename, 'power')
		self.segmentationMinLenFrames = AnalInterface.s2f(self.segmentationMinLenSec, self.filename)
		self.segmentationMaxLenFrames = AnalInterface.s2f(self.segmentationMaxLenSec, self.filename)
		p.log("TARGET SEGMENTATION: minimum segment length %.3f sec; maximum %.3f sec"%(self.segmentationMinLenSec, self.segmentationMaxLenSec))
		self.minPower = min(power)
		if self.minPower < util.dbToAmp(self.segmentationOffsetThreshAbs): # use absolute threshold
			self.powerOffsetValue = util.dbToAmp(self.segmentationOffsetThreshAbs)
			p.log("TARGET SEGMENTATION: using an offset amplitude value of %s"%(self.segmentationOffsetThreshAbs))
		else:
			self.powerOffsetValue = self.minPower*util.dbToAmp(self.segmentationOffsetThreshAdd)
			p.log("TARGET SEGMENTATION: the amplitude of %s never got below the offset threshold of %sdB specified in offsetThreshAbs.  So, I'm using offsetThreshAdd dB (%.2f) above the minimum found power -- a value of %.2f dB."%(self.filename, self.segmentationOffsetThreshAdd, self.segmentationOffsetThreshAdd, util.ampToDb(self.powerOffsetValue)))

		if self.segmentationFilepath == None:
			import descriptordata
			odf = descriptordata.odf(power, 7)
			# do multirise segmentation?
			if self.multiriseBool:
				multiriseDeviation = self.segmentationOffsetRise*(self.multirisePercentDev/100.)
				riseRatioList = np.linspace(max(1.05, self.segmentationOffsetRise-multiriseDeviation), self.segmentationOffsetRise+multiriseDeviation, num=self.multiriseSteps)
			else:
				# just use one rise ration
				riseRatioList = [self.segmentationOffsetRise]

			for userRiseRatio in riseRatioList: # this a list of rises if desired!
				segments, logic = segmentationAlgoV2(self.segmentationThresh, self.powerOffsetValue, userRiseRatio, power, odf, self.segmentationMinLenFrames, self.segmentationMaxLenFrames, AnalInterface)
				# ensure that each segment isn't in the list already
				for idx in range(len(segments)):
					if segments[idx] in self.segmentationInOnsetFrames: continue
					self.segmentationInOnsetFrames.append(segments[idx])
					if ops.SEGMENTATION_FILE_INFO == 'logic':
						self.extraSegmentationData.append(logic[idx])
			# end loop

			closebartxt = "Found %i segments (threshold=%.1f offsetrise=%.2f offsetthreshadd=%.2f)."%(len(self.segmentationInOnsetFrames), self.segmentationThresh, self.segmentationOffsetRise, self.segmentationOffsetThreshAdd)
		else: # load target segments from a file
			p.log("TARGET SEGMENTATION: reading segments from file %s"%(self.segmentationFilepath))
			for dataentry in util.readAudacityLabelFile(self.segmentationFilepath):
				startf = AnalInterface.s2f(dataentry[0], self.filename)
				endf = AnalInterface.s2f(dataentry[1], self.filename)
				self.segmentationInOnsetFrames.append((startf, endf))
				self.extraSegmentationData.append('from file')
			closebartxt = "Read %i segments from file %s"%(len(self.segmentationInFrames), os.path.split(self.segmentationFilepath)[1])
	########################################
	def stageSegments(self, AnalInterface, ops, p):
		###################################
		## make segment times in seconds ##
		###################################
		self.segmentationInFrames = self.segmentationInOnsetFrames
		for start, end in self.segmentationInOnsetFrames:
			self.segmentationInSec.append((AnalInterface.f2s(start), AnalInterface.f2s(end)))
			self.seglengths.append(AnalInterface.f2s(end-start))

		if ops.SEGMENTATION_FILE_INFO != 'logic':
			for start, end in self.segmentationInFrames:
				self.extraSegmentationData.append('%s=%.5f'%(ops.SEGMENTATION_FILE_INFO, self.whole.desc[ops.SEGMENTATION_FILE_INFO].get(start, end) ))
				if False:
					midilist = []
					import mu
					for f in range(start, end):
						if self.whole.desc['power'][f] < 0.0005: continue
						midi = mu.midiPitchToNoteNameString(mu.frq2midi(self.whole.desc['f0'][f]))
						midi = midi.replace(' ','')

						midilist.append((midi))
					peakidx = np.argmax(self.whole.desc['power'][start:end])
					print("\n\nSegment %.2f: -- %s"%(AnalInterface.f2s(start), str(midilist)))

		p.startPercentageBar(upperLabel="Evaluating TARGET %s from %.2f-%.2f"%(self.whole.printName, self.whole.segmentStartSec, self.whole.segmentEndSec), total=len(self.segmentationInSec))
		for sidx, (startSec, endSec) in enumerate(self.segmentationInSec):
			p.percentageBarNext(lowerLabel="@%.2f sec - %.2f sec"%(startSec, endSec))
			segment = targetSegment(self.filename, sidx, startSec, endSec, +0, 0.0001, 0.0001, 1, AnalInterface, self.midiPitchMethod)
			segment.power = segment.desc['power-seg'].get(0, None) # for sorting
			self.segs.append(segment)

		if self.segmentationFilepath == None:
			closebartxt = "Found %i segments (threshold=%.1f offsetrise=%.2f offsetthreshadd=%.2f)."%(len(self.segmentationInOnsetFrames), self.segmentationThresh, self.segmentationOffsetRise, self.segmentationOffsetThreshAdd)
		else:
			closebartxt = "Read %i segments from file %s"%(len(self.segmentationInFrames), os.path.split(self.segmentationFilepath)[1])

		p.percentageBarClose(txt=closebartxt)

		###################################
		## hack for signal decomposition ##
		###################################
		if 'origduration' in self.decompose:
			for s in self.segs:
				s.segmentStartSec = s.segmentStartSec % self.decompose['origduration']
				s.segmentEndSec = s.segmentStartSec + s.segmentDurationSec
				s.segmentStartFrame = AnalInterface.getSegmentStartInFrames(s.filename, s.segmentStartSec, s.segmentEndSec, s.lengthInFrames)

	########################################
	def writeSegmentationFile(self, filename):
		fh = open(filename, 'w')
		for sidx in range(len(self.segmentationInSec)):
			fh.write( "%f\t%f\t%s\n"%(self.segmentationInSec[sidx][0], self.segmentationInSec[sidx][1], self.extraSegmentationData[sidx]) )
		fh.close()
	########################################
	def setupConcate(self, AnalInterface):
		from userclasses import SingleDescriptor as d
		self.powerStats = getDescriptorStatistics(self.segs, d('power'))
		# initalise mixture and add norm coeffs to mixables...
		for seg in self.segs:
			seg.seek = 0
			seg.selectiondone = False
			seg.initMixture(AnalInterface)
			for dobj in AnalInterface.mixtureDescriptors:
				# copy the normalization paramaters from the target segments onto the mix descriptors
				seg.mixdesc[dobj.name].setNorm(seg.desc[dobj.name].normdict)
	########################################
	def plotMetrics(self, outputpath, AnalInterface, p, normalise=True):
		import matplotlib.pyplot as plt
		lengthTv = self.whole.lengthInFrames
		powers = self.whole.desc['power'][:]
		if normalise:
			powers -= np.min(powers)
			powers /= np.max(powers)

		for dobj in AnalInterface.requiredDescriptors:
			fig = plt.figure(figsize=(lengthTv/10., 10.))
			proot, pext = os.path.splitext(outputpath)

			savepath = proot + '-' + dobj.name + pext
			if dobj.seg or dobj.name in ['power']: continue
			vals = self.whole.desc[dobj.name][:]
			if normalise:
				vals -= np.min(vals)
				vals /= np.max(vals)

			plt.plot(range(lengthTv), vals, ls='steps-post', lw=1, label=[dobj.name])
			plt.plot(range(lengthTv), powers, ls='steps-post', lw=1, label=['power'])
			leg = plt.legend([dobj.name, 'power'],'upper right', shadow=False)
			frame  = leg.get_frame()
			frame.set_facecolor('0.90')    # set the frame face color to light gray
			p.log("TARGET: plotted descriptor %s as %s"%(dobj.name, savepath))
			plt.savefig(savepath)
			plt.close()
	########################################
	def NEWplotMetrics(self, outputpath, AnalInterface, p):
		import matplotlib.pyplot as plt
		import matplotlib.ticker as ticker

		p = {'outputdir': '/Users/ben/Desktop', 'descriptors': [('mfcc1',), ('mfcc1', 'mfcc2', 'mfcc3', 'mfcc4'), ('mfcc1', 'power')], 'diminches': (10, 4), 'ext': '.jpg', }
		assert os.path.isdir(p['outputdir'])

		timearray = [AnalInterface.f2s(i) for i in range(len(self.whole.desc[p['descriptors'][0][0]][:]))]
		for dlist in p['descriptors']:
			savepath = os.path.join(p['outputdir'], '-'.join([d for d in dlist]) + p['ext'])

			fig, ax = plt.subplots(figsize=p['diminches'])
			for d in dlist:
				ax.plot(timearray, self.whole.desc[d][:], ls='steps-post', lw=1)

			formatter = ticker.FormatStrFormatter('%1.1f')
			ax.xaxis.set_major_formatter(formatter)
			plt.savefig(savepath, bbox_inches=0, pad_inches=0.0)
			plt.close()

	########################################
	def plotSegmentation(self, outputpath, AnalInterface, p):
		import matplotlib.pyplot as plt
		powers = self.whole.desc['power'][:]
		powers -= np.min(powers)
		powers /= np.max(powers) # normalise between 0 and 1
		plotheight = 15.
		fig = plt.figure(figsize=(len(powers)/10., plotheight))
		plt.plot(range(len(powers)), powers, ls='steps-post', lw=1, label=['power'])
		for sidx in range(len(self.segmentationInFrames)): # plot segmentation
			xstart = self.segmentationInFrames[sidx][0]
			xend = self.segmentationInFrames[sidx][1]
			#print(self.segmentationInFrames[sidx], self.extraSegmentationData[sidx])
			plt.axvspan(xstart, xend, facecolor='#FFCCCC', alpha=1, ec='r', lw=1)
			plt.text(xstart, 0.5*plotheight, 'benny', fontsize=12, color='black')
		leg = plt.legend(['power'],'upper right', shadow=False)
		frame  = leg.get_frame()
		frame.set_facecolor('0.90')    # set the frame face color to light gray
		p.log("TARGET: plotted segmentation to %s"%(outputpath))
		plt.savefig(outputpath)
		plt.close()

def segmentationAlgoV2(threshold_onset, threshold_offset, riseratio, powers, odf, minlen, maxlen, AnalInterface):
	f = 0
	maxpower = np.max(powers)
	segments = []
	logic = []
	while True:
		if f+minlen > len(odf): break
		trigVal = util.ampToDb(odf[f] / maxpower)
		if trigVal < threshold_onset:
			f += 1
			continue
		# an onset because above trigger threshold
		segmentationLogicString = 'trig=%.2f'%trigVal
		s = minlen
		while True:
			# an offset if end of file is reached
			if f+s+1 >= len(odf):
				segmentationLogicString += ' eof=%.1f'%AnalInterface.f2s(len(powers))
				break
			# an offset if max seg length is reached
			if s+1 >= maxlen:
				segmentationLogicString += ' maxseglen=%.1f'%AnalInterface.f2s(maxlen)
				break
			# an offset if amplitude is below offset threshold
			if powers[f+s] < threshold_offset:
				segmentationLogicString += ' drop=%.0f'%util.ampToDb(powers[f+s])
				break
			# an offset if riseratio is too large
			riseRatioTmp = powers[f+s+1]/powers[f+s]
			if riseRatioTmp >= riseratio:
				segmentationLogicString += ' rise=%.3f'%riseRatioTmp
				break
			s += 1

		segments.append((f, f+s))
		logic.append(segmentationLogicString)
		f += s
	return segments, logic

def getDescriptorStatistics(listOfSegmentObjs, descriptorObj, takeOnlyEffDur=True, stdDeltaDegreesOfFreedom=0):
	if descriptorObj.seg:
		allDescs = np.array([sfsObj.desc[descriptorObj.name].get(0, None) for sfsObj in listOfSegmentObjs])
	else:
		cnt = 0
		for sfsObj in listOfSegmentObjs: cnt += sfsObj.desc['effDurFrames-seg'].get(0, None)
		allDescs = np.empty((cnt))
		cnt = 0
		for sfsObj in listOfSegmentObjs:
			effDur = sfsObj.desc['effDurFrames-seg'].get(0, None)
			allDescs[cnt:cnt+effDur] = sfsObj.desc[descriptorObj.name][:effDur]
			cnt += effDur
	return { 'min': np.min(allDescs), 'max': np.max(allDescs), 'mean': np.mean(allDescs),'stddev': np.std(allDescs, ddof=stdDeltaDegreesOfFreedom) }

def applyDescriptorNormalisation(listOfSegmentObjs, dobj, descStatistics):
	# this function must be unique for the corpus and target as they will have different normalization coefs
	normdatadict = {'method': dobj.normmethod}
	normdatadict.update(descStatistics)
	normdatadict['range'] = normdatadict['max']-normdatadict['min']

	for sfseg in listOfSegmentObjs:
		sfseg.desc[dobj.name].setNorm(normdatadict)


# -*- coding: utf-8 -*-
#
# Copyright (C) 2023 CERN
#
# Invenio-RDM-Records is free software; you can redistribute it and/or modify
# it under the terms of the MIT License; see LICENSE file for more details.

"""MARCXML based Schema for Invenio RDM Records."""

from copy import deepcopy

import bleach
from flask_resources.serializers import BaseSerializerSchema
from invenio_access.permissions import system_identity
from invenio_vocabularies.proxies import current_service as vocabulary_service
from marshmallow import fields, missing, post_dump

from ..schemas import CommonFieldsMixin
from ..ui.schema import current_default_locale
from ..utils import get_vocabulary_props

class MARCXMLSchema(BaseSerializerSchema, CommonFieldsMixin):
    """Schema for records in MARC."""

    doi = fields.Method("get_doi")
    oai = fields.Str(attribute="pids.oai.identifier")
    contributors = fields.Method("get_contributors")
    titles = fields.Method("get_titles")
    creators = fields.Method("get_creators")
    relations = fields.Method("get_relations")
    rights = fields.Method("get_rights")
    dates = fields.Method("get_dates")
    subjects = fields.Method("get_subjects")
    descriptions = fields.Method("get_descriptions")
    publishers = fields.Method("get_publishers")
    types = fields.Method(
        "get_types"
    )  # Corresponds to resource_type in the metadata schema
    # TODO: sources = fields.List(fields.Str(), attribute="metadata.references")
    sources = fields.Constant(
        missing
    )  # Corresponds to references in the metadata schema
    formats = fields.List(fields.Str(), attribute="metadata.formats")
    parent_id = fields.Str(attribute="parent.id")
    community_id = fields.List(fields.Str(), attribute="parent.communities.ids")
    last_updated = fields.Method("last_updated")
    sizes = fields.List(fields.Str(), attribute="metadata.sizes")
    version = fields.Str(attribute="metadata.version")
    funding = fields.Method("get_funding")

    def get_funding(self, obj):
        """Get funding information."""
        if "funding" not in obj["metadata"]:
            return missing

        funders = []
        all_funders = obj["metadata"].get("funding", [])

        for funder in all_funders:
            funder_string = ""

            award = funder.get("award", {})

            identifier = award.get("identifiers", [{}])[0]
            scheme = identifier.get("scheme", "null")
            identifier_value = identifier.get("identifier", "null")

            title = award.get("title", {})
            title = list(title.values())[0] if title else "null"

            number = award.get("number", "null")
            id = funder["funder"]["id"]
            name = funder["funder"].get("name", "null")

            funder_string += f"award_identifiers_scheme={scheme}; "
            funder_string += f"award_identifiers_identifier={identifier_value}; "
            funder_string += f"award_title={title}; "
            funder_string += f"award_number={number}; "
            funder_string += f"funder_id={id}; "
            funder_string += f"funder_name={name}; "
            funders.append(funder_string)

        return funders

    def last_updated(self, obj):
        """Get date last updated."""
        return [obj["updated"]]

    def get_relations(self, obj):
        """Get relations."""
        rels = []

        # Fundings
        # FIXME: Add after UI support is there

        # Alternate identifiers
        for a in obj["metadata"].get("alternate_identifiers", []):
            rels.append(f"{a['scheme']}:{a['identifier']}")

        # Related identifiers
        for a in obj["metadata"].get("related_identifiers", []):
            rels.append(f"{a['scheme']}:{a['identifier']}")

        return rels or missing

    def get_rights(self, obj):
        """Get rights."""
        rights = []
        access_right = obj.get("access", {}).get("status")

        if access_right:
            rights.append(f"info:eu-repo/semantics/{access_right}Access")

        if access_right == "metadata-only":
            access_right = "closed"

        ids = []
        for right in obj["metadata"].get("rights", []):
            _id = right.get("id")
            if _id:
                ids.append(_id)
            else:
                title = right.get("title").get(current_default_locale())
                if title:
                    rights.append(title)

                license_url = right.get("link")
                if license_url:
                    rights.append(license_url)

        if ids:
            vocab_rights = vocabulary_service.read_many(
                system_identity, "licenses", ids
            )
            for right in vocab_rights:
                title = right.get("title").get(current_default_locale())
                if title:
                    rights.append(title)

                license_url = right.get("props").get("url")
                if license_url:
                    rights.append(license_url)

        return rights or missing

    def get_dates(self, obj):
        """Get dates."""
        dates = [obj["metadata"]["publication_date"]]

        access_status = obj.get("access", {}).get("status", {})

        if access_status == "embargoed":
            date = obj["access"]["embargo"]["until"]
            dates.append(f"info:eu-repo/date/embargoEnd/{date}")

        return dates or missing

    def get_descriptions(self, obj):
        """Get descriptions."""
        metadata = obj["metadata"]
        descriptions = []

        description = metadata.get("description")
        if description:
            descriptions.append(description)

        additional_descriptions = metadata.get("additional_descriptions", [])
        for add_desc in additional_descriptions:
            descriptions.append(add_desc["description"])

        descriptions = [
            bleach.clean(
                desc,
                strip=True,
                strip_comments=True,
                tags=[],
                attributes=[],
            )
            for desc in descriptions
        ]

        return descriptions or missing

    def get_subjects(self, obj):
        """Get subjects."""
        metadata = obj["metadata"]
        subjects = []
        subjects.extend(
            (s["subject"] for s in metadata.get("subjects", []) if s.get("subject"))
        )
        return subjects or missing

    def get_types(self, obj):
        """Get resource type."""
        props = get_vocabulary_props(
            "resourcetypes",
            [
                "props.eurepo",
            ],
            obj["metadata"]["resource_type"]["id"],
        )
        t = props.get("eurepo")
        return [t] if t else missing

    # FIXME should this be a processor? that's somehow the post_dump place
    # otherwise we cannot guarantee order
    @post_dump()
    def keys_to_tags(self, data, many, **kwargs):
        """Changes the key name to the corresponding MARCXML tag (number)."""
        # [!] The string in the first array corresponds to the tag[0:4] + ind1[4] + ind2[5]
        # [!] The first string needs to be length *5* (this is to do with the dojson parser)
        # [!] The second string corresponds to the subfield code

        # Example: "creators" : ["100a ", "a"]

        #   <datafield tag="100" ind1="a" ind2=" ">
        #       <subfield code="a">Tarocco, Nicola</subfield>
        #   </datafield>

        # To add support for more tags, use the corresponding codes from here
        # https://scoap3.org/scoap3-repository/marcxml/

        changes = {
            "contributors": ["700a ", "u"],  # Abstract
            "titles": ["245a ", "a"],  # Title
            "creators": ["100a ", "a"],  # First author
            "oai": ["909CO", "o"],  # OAI
            "relations": ["856 2", "a"],  # Related Ressource
            "rights": ["540  ", "a"],  # License
            "dates": [
                "260c ",
                "c",
            ],  # Publication Information - Date of Publication
            "subjects": ["653  ", "a"],  # Keywords
            "descriptions": ["520  ", "a"],  # Abstract
            "publishers": [
                "260b ",
                "a",
            ],  # Publication Information - Publisher Name
            "types": ["901  ", "u"],  # Local Implementation
            "sources": ["246i ", "x"],  # Source
            "coverage": ["510  ", "a"],  # Location
            "formats": ["520 1", "a"],  # Abstract
            "parent_id": ["024 1", "a"],  # ID
            "community_id": ["024 2", "a"],  # ID
            "last_updated": [
                "260ca",
                "c",
            ],  # Publication Information - Date of Publication
            "sizes": ["520 2", "a"],  # Abstract
            "version": ["024 3", "a"],  # ID
            "funding": ["856 1", "a"],  # Related Ressource
        }

        data_copy = deepcopy(data)
        for key in data_copy:
            if key in changes:
                data[changes[key][0]] = {changes[key][1]: data.pop(key)}

        doi = data.pop("doi", None)
        if doi:
            data["024  "] = {"a": doi, "2": "doi"}

        return data


# Copyright: 2019, NLnet Labs and the Internet.nl contributors
# SPDX-License-Identifier: Apache-2.0
import os

from celery import Celery
from celery.schedules import crontab

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'internetnl.settings')

app = Celery('internetnl')

app.config_from_object('django.conf:settings')

app.autodiscover_tasks()

@app.task(bind=True)
def debug_task(self):
    print('Request: {0!r}'.format(self.request))

if app.conf.ENABLE_BATCH:
    app.conf.beat_schedule = {
        'run_batch': {
                'task': 'tasks.run_batch',
                'schedule': app.conf.BATCH_SCHEDULER_INTERVAL,
        }
    }
else:
    # Disable HoF when on batch mode, too much DB activity.
    app.conf.beat_schedule = {
        'generate_HoF': {
                'task': 'update_HoF_ranking',
                'schedule': crontab(hour='*', minute='*/10', day_of_week='*'),
        }
    }


from __future__ import unicode_literals
import frappe
import json
from frappe.model.document import Document
from erpnext.accounts.doctype.accounting_dimension.accounting_dimension import get_accounting_dimensions

from frappe.utils.data import nowdate, getdate, cint, add_days, date_diff, get_last_day, add_to_date, flt

def process(doc):
	"""
	To be called by task periodically. It checks the subscription and takes appropriate action
	as need be. It calls either of these methods depending the `Subscription` status:
	1. `process_for_active`
	2. `process_for_past_due`
	"""
	if doc.status == 'Active':
		process_for_active(doc)
	# elif doc.status in ['Past Due Date', 'Unpaid']:
	#     process_for_past_due_date(doc)

	# doc.save()

def process_for_active(doc):
	"""
	Called by `process` if the status of the `Subscription` is 'Active'.

	The possible outcomes of this method are:
	1. Generate a new invoice
	2. Change the `Subscription` status to 'Past Due Date'
	3. Change the `Subscription` status to 'Cancelled'
	"""
	if not is_current_invoice_paid(doc) and (is_postpaid_to_invoice(doc) or is_prepaid_to_invoice(doc)):
		generate_invoice(doc)
		if current_invoice_is_past_due(doc):
			doc.status = 'Past Due Date'

	if current_invoice_is_past_due(doc) and getdate(nowdate()) > getdate(doc.current_invoice_end):
		doc.status = 'Past Due Date'

	if doc.cancel_at_period_end and getdate(nowdate()) > getdate(doc.current_invoice_end):
		cancel_subscription_at_period_end(doc)

def is_current_invoice_paid(doc):
	if is_new_subscription(doc):
		return False

	last_invoice = frappe.get_doc('Sales Invoice', doc.invoices[-1].invoice)
	if getdate(last_invoice.posting_date) == getdate(doc.current_invoice_start) and last_invoice.status == 'Paid':
		return True

	return False

def is_postpaid_to_invoice(doc):
	return getdate(nowdate()) > getdate(doc.current_invoice_end) or \
		(getdate(nowdate()) >= getdate(doc.current_invoice_end) and getdate(doc.current_invoice_end) == getdate(doc.current_invoice_start)) and \
		not has_outstanding_invoice(doc)

def is_prepaid_to_invoice(doc):
	if not doc.create_inv:
		return False

	if is_new_subscription(doc):
		return True

	# Check invoice dates and make sure it doesn't have outstanding invoices
	return getdate(nowdate()) >= getdate(doc.current_invoice_start) and not has_outstanding_invoice(doc)

def generate_invoice(doc, prorate=0):
	"""
	Creates a `Sales Invoice` for the `Subscription`, updates `doc.invoices` and
	saves the `Subscription`.
	"""
	invoice = create_invoice(doc,prorate)
	doc.append('invoices', {'invoice': invoice.name})
	# doc.save()

	return invoice

def current_invoice_is_past_due(doc, current_invoice=None):
	"""
	Returns `True` if the current generated invoice is overdue
	"""
	if not current_invoice:
		current_invoice = get_current_invoice(doc)

	if not current_invoice:
		return False
	else:
		return getdate(nowdate()) > getdate(current_invoice.due_date)

def cancel_subscription_at_period_end(doc):
	"""
	Called when `Subscription.cancel_at_period_end` is truthy
	"""
	doc.status = 'Cancelled'
	if not doc.cancelation_date:
		doc.cancelation_date = nowdate()

def is_new_subscription(doc):
	"""
	Returns `True` if `Subscription` has never generated an invoice
	"""
	return len(doc.invoices) == 0

def has_outstanding_invoice(doc):
	"""
	Returns `True` if the most recent invoice for the `Subscription` is not paid
	"""
	current_invoice = get_current_invoice(doc)
	if not current_invoice:
		return False
	else:
		return not is_not_outstanding(current_invoice)

def create_invoice(doc, prorate):
	"""
	Creates a `Sales Invoice`, submits it and returns it
	"""
	invoice = frappe.new_doc('Sales Invoice')
	invoice.set_posting_time = 1
	# invoice.posting_date = doc.current_invoice_start
	invoice.customer = doc.customer
	invoice.subscription=doc.name
	invoice.project=doc.project
	invoice.agreement_start_date=doc.agreement_start_date
	invoice.agreement_end_date=doc.agreement_end_date
	invoice.property_rent=doc.property_rent
	invoice.tenure=doc.agreement_tenure
	# if doc.invoice_date and invoice.posting_date!=doc.invoice_date:
	invoice.posting_date=doc.invoice_date if doc.invoice_date else doc.current_invoice_start
	if doc.invoice_due_days:
		invoice.due_date=add_days(invoice.posting_date,doc.invoice_due_days)
	## Add dimesnions in invoice for subscription:
	accounting_dimensions = get_accounting_dimensions()

	for dimension in accounting_dimensions:
		if doc.get(dimension):
			invoice.update({
				dimension: doc.get(dimension)
			})

	# Subscription is better suited for service items. I won't update `update_stock`
	# for that reason
	items_list = get_items_from_plans(doc,doc.plans, prorate)
	for item in items_list:
		invoice.append('items',	item)

	# Taxes
	if doc.tax_template:
		invoice.taxes_and_charges = doc.tax_template
		invoice.set_taxes()

	# Due date
	invoice.append(
		'payment_schedule',
		{
			'due_date': invoice.due_date,
			'invoice_portion': 100
		}
	)

	# Discounts
	if doc.additional_discount_percentage:
		invoice.additional_discount_percentage = doc.additional_discount_percentage

	if doc.additional_discount_amount:
		invoice.discount_amount = doc.additional_discount_amount

	if doc.additional_discount_percentage or doc.additional_discount_amount:
		discount_on = doc.apply_additional_discount
		invoice.apply_discount_on = discount_on if discount_on else 'Grand Total'

	# Subscription period
	invoice.from_date = doc.current_invoice_start
	invoice.to_date = doc.current_invoice_end

	invoice.flags.ignore_mandatory = True
	invoice.save()
	invoice.submit()

	return invoice

def get_current_invoice(doc):
	"""
	Returns the most recent generated invoice.
	"""
	if len(doc.invoices):
		current = doc.invoices[-1]
		if frappe.db.exists('Sales Invoice', current.invoice):
			doc = frappe.get_doc('Sales Invoice', current.invoice)
			return doc
		else:
			frappe.throw(_('Invoice {0} no longer exists'.format(current.invoice)))

# @staticmethod
def is_not_outstanding(invoice):
	"""
	Return `True` if the given invoice is paid
	"""
	return invoice.status == 'Paid'

def get_items_from_plans(doc, plans, prorate=0):
	"""
	Returns the `Item`s linked to `Subscription Plan`
	"""
	if prorate:
		prorate_factor = get_prorata_factor(doc.current_invoice_end, doc.current_invoice_start)

	items = []
	customer = doc.customer
	for plan in plans:
		item_code = frappe.db.get_value("Subscription Plan", plan.plan, "item")
		if len(plans)<2 and doc.custom_rate>0:
			if not prorate:
				items.append({'item_code': item_code, 'qty': plan.qty, 'rate': doc.custom_rate})
			else:
				items.append({'item_code': item_code, 'qty': plan.qty, 'rate': (doc.custom_rate * prorate_factor)})
		elif frappe.db.get_value('Subscription Plan',plan.plan,'price_determination')=='Percentage of Rent':
			from homzhub_customization.homzhub_customization.doctype.sales_invoice import get_plan_rate
			rate,srv_chg=get_plan_rate(doc,plan.plan, plan.qty, customer)
			if not prorate:
				items.append({'item_code': item_code, 'qty': plan.qty, 'rate': rate,'service_charges_in_per':srv_chg})
			else:
				items.append({'item_code': item_code, 'qty': plan.qty, 'rate': (rate * prorate_factor)})
		else:
			from erpnext.accounts.doctype.subscription_plan.subscription_plan import get_plan_rate
			if not prorate:
				items.append({'item_code': item_code, 'qty': plan.qty, 'rate': get_plan_rate(plan.plan, plan.qty, customer)})
			else:
				items.append({'item_code': item_code, 'qty': plan.qty, 'rate': (get_plan_rate(plan.plan, plan.qty, customer) * prorate_factor)})

	return items

def get_prorata_factor(period_end, period_start):
	diff = flt(date_diff(nowdate(), period_start) + 1)
	plan_days = flt(date_diff(period_end, period_start) + 1)
	prorate_factor = diff / plan_days

	return prorate_factor

def validate(doc,method):
	if doc.get('project') and doc.get('invoice_date'):
		expected_date=frappe.db.get_value('Project',doc.get('project'),'expected_start_date')
		if  getdate(expected_date) > getdate(doc.get('invoice_date')):
			frappe.throw("Invoice Date must be equal or greater than project's Expected Start Date")
	if doc.get('customer') and doc.get('project'):
		project=frappe.get_doc('Project',doc.get('project'))
		customer_list=[]
		customer_list.append(project.customer)
		# customer_list.append(project.property_owner)
		for d in project.tenant_list:
			customer_list.append(d.tenant)
		for d in project.owner_list:
			customer_list.append(d.prop_owner)
		if doc.get('customer') not in customer_list:
			frappe.throw("customer in subscription should be equal to owner list/tenant list/customer")
	if doc.get('project'):
		pro_doc=frappe.get_doc('Project',doc.get('project'))
		subscriptions=[]
		for d in pro_doc.get('subscription_plans'):
			subscriptions.append(d.get('subscription'))
		if frappe.db.exists("Subscription", doc.name):
			if doc.name not in subscriptions:
				for d in doc.get('plans'):
					pro_doc.append("subscription_plans", {
					"subscription" : doc.name,
					"subscription_plan" : d.plan
					})
					pro_doc.save()

@frappe.whitelist()
def update_status(doc):
	doc=json.loads(doc)
	doc=frappe.get_doc('Subscription',doc.get('name'))
	doc.status='Cancelled'
	doc.cancelation_date = nowdate()
	doc.save()


# -*- coding:UTF-8 -*-
# Python的多线程由于全局线程锁的存在并不能实现真正的并行编程，但是Python中的多进程编程模式是可以实现这个目标的。
# 多进程模式下进行上下文切换的损耗要远远大于线程。进程间无法直接共享数据，需要通过Queue、Pipe或则Manager方式做进程间通信。

from multiprocessing import Process,Queue,Pool

import multiprocessing
import os
import time

#simple multiprocessing
# def info(title):
#     print('title:',title)
#     print('name:',__name__)
#     print('parent process:',os.getppid())
#     print('process:',os.getpid())
# def f(name):
#     info('function f')
#     print('hello',name)

#通过Queue可以实现多个进程间的数据共享，先进先出
#添加数据
# def f(q,data):
#     print('存入数据')
#     q.put(data)
#     print('存入完毕')
# #获取数据
# def out(q):
#     time.sleep(4)
#     print('取出数据')
#     print(q.get())
#     print('取出完毕')

# multiprocessing.Pipe()即管道模式，调用Pipe()返回管道的两端的Connection。
# Pipe的本质是进程之间的数据传递，而不是数据共享，这和socket有点像。
# pipe()返回两个连接对象分别表示管道的两端，每端都有send()和recv()方法。
# 如果两个进程试图在同一时间的同一端进行读取和写入那么，这可能会损坏管道中的数据。
# def consumer(pipe):
#     output_p,input_p=pipe
#     input_p.close() #消费者关闭管道的输入端
#     while True:
#         try:
#             print("writting....")
#             item=output_p.recv()
#             print(item)
#         except EOFError:
#             print("越界")
#             break
# def productor(sequence,input_p): ## sequence用于存放需要被消费的数据列表
#     for item in sequence:
#         input_p.send(item)  ##将数据发送到管道

#进程锁
# def job(v,num,l): #共享对象，新的数据，锁
#     l.acquire()
#     for _ in range(5):
#         time.sleep(0.1)
#         v.value += num
#         print(v.value)
#     l.release()
# def multicore():
#     l = multiprocessing.Lock() #创建锁对象
#     v = multiprocessing.Value('i',0) #定义共享内存
#     p1=multiprocessing.Process(target=job,args=(v,1,l))
#     p2=multiprocessing.Process(target=job,args=(v,3,l))
#     p1.start()
#     p2.start()
#     p1.join()
#     p2.join(0)

##线程池
def Foo(i):
    time.sleep(2)
    return i+100

def Bar(arg):
    print('-->exec done:',arg)

# apply() 同步执行（串行）
# apply_async() 异步执行（并行）
# terminate() 立刻关闭进程池
# join() 主进程等待所有子进程执行完毕。必须在close或terminate()之后。
# close() 等待所有进程结束后，才关闭进程池

if __name__ == '__main__':
    #simple multiprocessing
    # print('main line:',os.getpid())
    # p = Process(target=f,args=('树先生',))
    # p.start()
    # p.join()

    #通过Queue可以实现多个进程间的数据共享
    # q=Queue()#创建实例
    # p = Process(target=f,args=(q,[1,2,3]))
    # p.start()
    # p.join()
    # p1 = Process(target=out,args=(q,))
    # p1.start()
    # p1.join()

    #pipe()数据传输
    # (output_p,input_p) = multiprocessing.Pipe(True)
    # #create consumer process
    # con_s=multiprocessing.Process(target=consumer,args=((output_p,input_p),))
    # con_s.start()
    #  #关闭生产者中的输出管道
    # output_p.close()
    # print("生产者关闭")
    # #生产项目
    # sequence=[1,2,3,4]
    # productor(sequence,input_p)
    # #关闭输入管道，表示完成
    # input_p.close()
    # #等待使用者进程关闭
    # con_s.join()

    #进程锁，与线程锁类似
    #multicore()

    #线程池
    pool = Pool(5)  #允许进程池同时放入5个进程
    for i in range(10):
        pool.apply_async(func=Foo, args=(i,),callback=Bar)  #func子进程执行完后，才会执行callback，否则callback不执行（而且callback是由父进程来执行了）

    print('end')
    pool.close()
    print('线程池关闭')
    pool.join() #主进程等待所有子进程执行完毕。必须在close()或terminate()之后。
    print('子线程结束')


#I want coffee

import sys
import argparse
from torch import nn, optim
import torch
import matplotlib.pyplot as plt

from data import mnist
from model import MyAwesomeModel

class TrainOREvaluate(object):
    """ Helper class that will help launch class methods as commands
        from a single script
    """
    def __init__(self):
        parser = argparse.ArgumentParser(
            description="Script for either training or evaluating",
            usage="python main.py <command>"
        )
        parser.add_argument("command", help="Subcommand to run")
        args = parser.parse_args(sys.argv[1:2])
        if not hasattr(self, args.command):
            print('Unrecognized command')

            parser.print_help()
            exit(1)
        # use dispatch pattern to invoke method with same name
        getattr(self, args.command)()

    def validation(model, testloader, criterion):
        accuracy = 0
        test_loss = 0
        for images, labels in testloader:

            output = model.forward(images)
            test_loss += criterion(output, labels).item()

            ## Calculating the accuracy
            # Model's output is log-softmax, take exponential to get the probabilities
            ps = torch.exp(output)
            # Class with highest probability is our predicted class, compare with true label
            equality = (labels.data == ps.max(1)[1])
            # Accuracy is number of correct predictions divided by all predictions, just take the mean
            accuracy += equality.type_as(torch.FloatTensor()).mean()

        return test_loss, accuracy

    def train(self):
        print("Training day and night")
        parser = argparse.ArgumentParser(description='Training arguments')
        parser.add_argument('--lr', default=0.001)
        # add any additional argument that you want
        args = parser.parse_args(sys.argv[2:])
        print(args)

        # TODO: Implement training loop here
        model = MyAwesomeModel()
        train_set, test_set = mnist()

        criterion = nn.CrossEntropyLoss()
        optimizer = optim.Adam(model.parameters(), lr= float(args.lr))
        epochs=2
        print_every = 40
        tLoss= []

        steps = 0
        running_loss = 0
        for e in range(epochs):
            # Model in training mode, dropout is on
            model.train()
            for images, labels in train_set:
                steps += 1

                optimizer.zero_grad()

                output = model.forward(images)
                loss = criterion(output, labels)
                loss.backward()
                optimizer.step()

                running_loss += loss.item()

                if steps % print_every == 0:
                    # Model in inference mode, dropout is off
                    model.eval()

                    # Turn off gradients for validation, will speed up inference
                    with torch.no_grad():
                        test_loss, accuracy = TrainOREvaluate.validation(model, test_set, criterion)

                    print("Epoch: {}/{}.. ".format(e + 1, epochs),
                          "Training Loss: {:.3f}.. ".format(running_loss / print_every),
                          "Test Loss: {:.3f}.. ".format(test_loss / len(test_set)),
                          "Test Accuracy: {:.3f}".format(accuracy / len(test_set)))

                    tLoss.append(running_loss / print_every)
                    running_loss = 0

                    # Make sure dropout and grads are on for training
                    model.train()

        torch.save(model.state_dict(), 'checkpoint.pth')

        plt.plot(tLoss, label='Training Loss')

        plt.legend()
        plt.show()

    def evaluate(self):
        print("Evaluating until hitting the ceiling")
        parser = argparse.ArgumentParser(description='Evaluating arguments')
        parser.add_argument('--load_model_from', default="checkpoint.pth")
        # add any additional argument that you want
        args = parser.parse_args(sys.argv[2:])
        print(args)

        # TODO: Implement evaluation logic here
        if args.load_model_from:
            model = MyAwesomeModel()
            model.load_state_dict(torch.load(args.load_model_from))
            model.eval()

            accuracy = 0

            _, test_set = mnist()

           # with torch.no_grad():
            for itr, (images, labels) in enumerate(test_set):
                output = model.forward(images)

                ps = torch.exp(output)

                equality = (labels.data == ps.max(1)[1])
                # Accuracy is number of correct predictions divided by all predictions, just take the mean
                accuracy += equality.type_as(torch.FloatTensor()).mean()

            print("Test Accuracy: {:.3f}".format(accuracy / len(test_set)))

        else:
            print("There is no model to evaluate please train first...")

if __name__ == '__main__':
    TrainOREvaluate()


# -*-coding:utf-8 -*
#!/usr/bin/python3

import random
import datetime
import time
from subprocess import call
import discord

import settings.ekiller2
import settings.dellog
import utils.perms
import utils.usertools

try:
    call(['mkdir', 'tmp'])
except:
    pass

async def commandHandler(client, message, ekiller):

    try:
        if message.content.startswith("/ekiller") and (not await utils.perms.hasrole(message.author, settings.ekiller.auth)):
            await client.send_message(message.channel, message.author.mention + ", vous n'avez pas la permission d'effectuer cette commande.")
            return
    except:
        if message.content.startswith("/ekiller"):
            await client.send_message(message.channel, message.author.mention + ", Une erreur s'est produite. \n\nPS: les commandes ayant un rapport avec ekiller doivent être effectuées sur le serveur.")
            raise

    if message.content == "/ekiller start":
        await start(client, message, ekiller)

    elif message.content.startswith("/ekiller join"):
        await join(client, message, ekiller)

    elif message.content == "/ekiller quit":
        await quit(client, message, ekiller)

    elif message.content.startswith("/ekiller kick"):
        await kick(client, message, ekiller)

    elif message.content == "/ekiller players":
        await players(client, message, ekiller)

    elif message.content == "/ekiller reset":
        await reset(client, message, ekiller)

    elif message.content == "/ekiller logs":
        await logs(client, message)

    elif message.content == "/ekiller help":
        await help(client, message)

    elif message.content == "/ekiller debug":
        await debug(client, message, ekiller)

async def start(client, message, ekiller):
    await client.delete_message(message)
    game = []
    players = []
    words = []
    for (id, word) in ekiller.buffer:
        member = await utils.usertools.UserByID(client, id)
        players.append(member)
        words.append(word)
    random.shuffle(players)
    players.append(players[0])
    random.shuffle(words)
    for i in range(len(players)-1):
        dic = {"ekiller":str(players[i].id)+'#'+players[i].name, "target":str(players[i+1].id)+'#'+players[i+1].name, "word":words[i]}
        game.append(dic)
        embed = discord.Embed(title="E-KILLER", description="Votre cible est : " + players[i+1].display_name + "\nLe mot est : `" + words[i] + "`\n\nBonne chance à vous.", color=0x0000ff)
        await client.send_message(players[i], embed=embed)
    embed = discord.Embed(title="E-KILLER", description="Liste des joueurs :\n" + str([p.display_name for p in players[:-1]]) + "\n\n Bonne chance à tous !", color=0x0000ff)
    await client.send_message(message.channel, embed=embed)
    with open("tmp/ekillerLog.txt", "a") as ekillerlogfile:
        ekillerlogfile.write(str(datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')) + " [" + "\n")
        for dic in game:
            ekillerlogfile.write(str(dic) + "\n")
        ekillerlogfile.write("]\n\n")

async def join(client, message, ekiller):
    await client.delete_message(message)
    args = message.content.split(' ')
    if len(args) == 3:
        if message.author.id in dict(ekiller.buffer):
            await client.send_message(message.channel, message.author.mention + ", vous faites déjà parti des participants.")
        else:
            ekiller.buffer.append((message.author.id, args[2]))
            await client.send_message(message.channel, message.author.mention + ", vous avez bien été ajouté à la liste des participants.")
    else:
        await client.send_message(message.channel, message.author.mention + ", veuillez préciser un unique mot que vous souhaitez ajouter à la liste : `/ekiller join <word>`")

async def quit(client, message, ekiller):
    await client.delete_message(message)
    if message.author.id in dict(ekiller.buffer):
        ekiller.buffer = [(id,word) for (id,word) in ekiller.buffer if id!=message.author.id]
        await client.send_message(message.channel, message.author.mention + ", vous avez bien été retiré de la liste des participants.")
    else:
        await client.send_message(message.channel, message.author.mention + ", vous ne faites pas parti des participants.")

async def kick(client, message, ekiller):
    await client.delete_message(message)
    args = message.content.split(' ')
    if len(args) == 3:
        ids = args[2].split(',')
        for id in ids:
            member = await utils.usertools.UserByID(client, id)
            if not member:
                await client.send_message(message.channel, message.author.mention + ", le joueur `{0}` n'existe pas.".format(id))
            elif id not in dict(ekiller.buffer):
                await client.send_message(message.channel, message.author.mention + ", le joueur `{0}` ne participe pas.".format(member.display_name))
            else:
                ekiller.buffer = [(idd, word) for (idd, word) in ekiller.buffer if idd != id]
                await client.send_message(message.channel, message.author.mention + ", le joueur `{0}` a bien été retiré.".format(member.display_name))
    else:
        await client.send_message(message.channel, message.author.mention + ", veuillez préciser un unique id ou une liste d'ids séparés par une virgule : `/ekiller kick foo,bar`")

async def players(client, message, ekiller):
    await client.delete_message(message)
    players = []
    for (id, word) in ekiller.buffer:
        member = await utils.usertools.UserByID(client, id)
        players.append(member.display_name)
    await client.send_message(message.channel, "Liste des joueurs :")
    await client.send_message(message.channel, "```PYTHON\n" + str(players) + "\n```")

async def reset(client, message, ekiller):
    await client.delete_message(message)
    ekiller.buffer = []
    await client.send_message(message.channel, message.author.mention + ", le jeu a bien été réinitialisé.")

async def logs(client, message):
    if not (message.author == client.user) and await utils.perms.hasrole(message.author, settings.dellog.logsAuth):
        try:
            await client.delete_message(message)
            await client.send_file(message.author, "tmp/ekillerLog.txt")
        except:
            await client.send_message(message.author, "```FAILED```")

async def help(client, message):
    await client.delete_message(message)
    text = "Liste des commandes :\n\n"
    text += "/ekiller join <word>\nRejoindre la partie et ajouter le mot <word> à la liste de mots\n\n"
    text += "/ekiller quit\nQuitter la partie la partie\n\n"
    text += "/ekiller players\nAfficher la liste des participants\n\n"
    text += "/ekiller start\nLancer la partie\n\n"
    text += "/ekiller help\nAffiche cette aide\n\n"
    embed = discord.Embed(title="E-KILLER", description=text, color=0x0000ff)
    await client.send_message(message.channel, embed=embed)

async def debug(client, message, ekiller):
    await client.delete_message(message)
    await client.send_message(message.channel, message.author.mention + "```PYTHON\n" + str(ekiller.buffer) + "\n```")

class Ekiller:

    def __init__(self):
        self.buffer = []


import requests, time, csv, random, json
from bs4 import BeautifulSoup

def get_html(url):
    r = requests.get(url)
    return r.text

def write_csv(data):
    with open('businesscenter.csv', 'a') as f:
        writer = csv.writer(f)

        writer.writerow((data['adress'],
                         data['index1'],
                         data['index2']))

def number_of_pages(html):
    soup = BeautifulSoup(html, 'html.parser')
    str1 = soup.find('div', class_='col col-md-auto active').text.strip()
    i = 0
    num = ''
    while (str1[i]).isdigit():
        num += str1[i]
        i += 1
    num = int(num) / 30
    if (int(num) % 30) != 0:
        num += 1
    return int(num)

def transformIndex(str):
    if str:
        i = 0
        a = ''
        while str[i] != ' ':
            a += str[i]
            i += 1
        a = float(a)
        i += 1
        b = ''
        while (i < len(str)):
            b += str[i]
            i += 1
        b = float(b)
    return [b, a]

def get_page_data(html):
    soup = BeautifulSoup(html, 'html.parser')
    ads = soup.find('ul', class_='listNone').find_all_next('li', class_='cardObject combinedCard rentSaleCard')
    for ad in ads:
        try:
            adress = ad.find('span', class_='openMapCard linkDashed imitation-link').text.strip()
            print(adress)
            yandexApi = adress.replace(' ', '+')
            urlYandex = 'https://geocode-maps.yandex.ru/1.x/?apikey=5d4ae0bb-26ef-4563-b112-e98b056b6c40&geocode=' + yandexApi + '&format=json'
            response = (requests.get(urlYandex)).json()
            index = response.get('response', {}).get('GeoObjectCollection', {}).get('featureMember', {})[0].get('GeoObject', {}).get('Point', {})['pos']
            index = transformIndex(index)
            print(index)
            data = {'adress': adress,
                    'index1': index[0],
                    'index2': index[1]}
            write_csv(data)

        except:
            adress = ' '
            index = ' '

def main():
    base_url = 'https://amo.ru/bc'
    page_part = '/page-'

    num = number_of_pages(get_html(base_url))
    print(num)
    if num == 0:
        return -1

    for i in range(1, num+1):
        if i == 1:
            url_gen = base_url
        else:
            url_gen = base_url + page_part + str(i)
        html = get_html(url_gen)
        get_page_data(html)
        time.sleep(random.randint(2, 20))

if __name__ == '__main__':
    main()


import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

import tensorflow as tf

import argparse
import cv2
import numpy as np
from glob import glob
import copy

num_classes = 2
img_height, img_width = 128, 128
channel = 3
tf.set_random_seed(0)

def Res152(x, keep_prob, train=True):

    def ResBlock(x, in_f, f_1, out_f, stride=1):
        res_x = tf.layers.conv2d(x, f_1, [1, 1], strides=stride, padding='same', activation=None)
        res_x = tf.layers.batch_normalization(res_x, training=train)
        res_x = tf.nn.relu(res_x)

        res_x = tf.layers.conv2d(res_x, f_1, [3, 3], strides=1, padding='same', activation=None)
        res_x = tf.layers.batch_normalization(res_x, training=train)
        res_x = tf.nn.relu(res_x)

        res_x = tf.layers.conv2d(res_x, out_f, [1, 1], strides=1, padding='same', activation=None)
        res_x = tf.layers.batch_normalization(res_x, training=train)
        res_x = tf.nn.relu(res_x)

        if in_f != out_f:
            x = tf.layers.conv2d(x, out_f, [1, 1], strides=1, padding='same', activation=None)
            x = tf.layers.batch_normalization(x, training=train)
            x = tf.nn.relu(x)

        if stride != 1:
            x = tf.layers.max_pooling2d(x, [2, 2], strides=stride, padding='same')

        x = tf.add(res_x, x)
        x = tf.nn.relu(x)

        return x

    x = tf.layers.conv2d(x, 64, [7, 7], strides=2, padding="same", activation=None)
    x = tf.layers.batch_normalization(x, training=train)
    x = tf.nn.relu(x)

    x = tf.layers.max_pooling2d(x, [3, 3], strides=2, padding='SAME')

    # Res block 2
    x = ResBlock(x, 64, 64, 256)
    x = ResBlock(x, 256, 64, 256)
    x = ResBlock(x, 256, 64, 256)

    # Res block 3
    x = ResBlock(x, 256, 128, 512, stride=2)
    for _ in range(7):
        x = ResBlock(x, 512, 128, 512)

    # Res block 4
    x = ResBlock(x, 512, 256, 1024, stride=2)
    for _ in range(35):
        x = ResBlock(x, 1024, 256, 1024)

    # Res block 5
    x = ResBlock(x, 1024, 512, 2048, stride=2)
    x = ResBlock(x, 2048, 256, 2048)
    x = ResBlock(x, 2048, 256, 2048)

    x = tf.reduce_mean(x, axis=[1, 2])
    x = tf.layers.dense(x, num_classes)

    return x

CLS = ['akahara', 'madara']

# get train data
def data_load(path, hf=False, vf=False, rot=False):
    xs = []
    ts = []
    paths = []

    for dir_path in glob(path + '/*'):
        for path in glob(dir_path + '/*'):
            x = cv2.imread(path)
            x = cv2.resize(x, (img_width, img_height)).astype(np.float32)
            x /= 255.
            x = x[..., ::-1]
            xs.append(x)

            t = [0 for _ in range(num_classes)]
            for i, cls in enumerate(CLS):
                if cls in path:
                    t[i] = 1

            ts.append(t)

            paths.append(path)

            if hf:
                xs.append(x[:, ::-1])
                ts.append(t)
                paths.append(path)

            if vf:
                xs.append(x[::-1])
                ts.append(t)
                paths.append(path)

            if hf and vf:
                xs.append(x[::-1, ::-1])
                ts.append(t)
                paths.append(path)

            if rot != False:
                angle = rot
                scale = 1

                # show
                a_num = 360 // rot
                w_num = np.ceil(np.sqrt(a_num))
                h_num = np.ceil(a_num / w_num)
                count = 1
                #plt.subplot(h_num, w_num, count)
                #plt.axis('off')
                #plt.imshow(x)
                #plt.title("angle=0")

                while angle < 360:
                    _h, _w, _c = x.shape
                    max_side = max(_h, _w)
                    tmp = np.zeros((max_side, max_side, _c))
                    tx = int((max_side - _w) / 2)
                    ty = int((max_side - _h) / 2)
                    tmp[ty: ty+_h, tx: tx+_w] = x.copy()
                    M = cv2.getRotationMatrix2D((max_side/2, max_side/2), angle, scale)
                    _x = cv2.warpAffine(tmp, M, (max_side, max_side))
                    _x = _x[tx:tx+_w, ty:ty+_h]
                    xs.append(_x)
                    ts.append(t)
                    paths.append(path)

                    # show
                    #count += 1
                    #plt.subplot(h_num, w_num, count)
                    #plt.imshow(_x)
                    #plt.axis('off')
                    #plt.title("angle={}".format(angle))

                    angle += rot
                #plt.show()

    xs = np.array(xs, dtype=np.float32)
    ts = np.array(ts, dtype=np.int)

    return xs, ts, paths

# train
def train():
    tf.reset_default_graph()

    # place holder
    X = tf.placeholder(tf.float32, [None, img_height, img_width, channel])
    Y = tf.placeholder(tf.float32, [None, num_classes])
    keep_prob = tf.placeholder(tf.float32)

    logits = Res152(X, keep_pro, train=False)
    preds = tf.nn.softmax(logits)

    #loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(logits=logits, onehot_labels=Y))

    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=Y))
    optimizer = tf.train.MomentumOptimizer(learning_rate=0.001, momentum=0.9)
    train = optimizer.minimize(loss)

    correct_pred = tf.equal(tf.argmax(preds, 1), tf.argmax(Y, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

    xs, ts, paths = data_load('../Dataset/train/images/', hf=True, vf=True, rot=10)

    # training
    mb = 128
    mbi = 0
    train_ind = np.arange(len(xs))
    np.random.seed(0)
    np.random.shuffle(train_ind)

    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    config.gpu_options.visible_device_list="0"

    with tf.Session(config=config) as sess:
        sess.run(tf.global_variables_initializer())

        for i in range(500):
            if mbi + mb > len(xs):
                mb_ind = copy.copy(train_ind)[mbi:]
                np.random.shuffle(train_ind)
                mb_ind = np.hstack((mb_ind, train_ind[:(mb-(len(xs)-mbi))]))
                mbi = mb - (len(xs) - mbi)
            else:
                mb_ind = train_ind[mbi: mbi+mb]
                mbi += mb

            x = xs[mb_ind]
            t = ts[mb_ind]

            _, acc, los = sess.run([train, accuracy, loss], feed_dict={X: x, Y: t, keep_prob: 0.7})

            if (i + 1) % 10 == 0:
                print("iter >>", i+1, ', loss >>', los / mb, ', accuracy >>', acc)

        saver = tf.train.Saver()
        saver.save(sess, './cnn.ckpt')

# test
def test():
    tf.reset_default_graph()

    X = tf.placeholder(tf.float32, [None, img_height, img_width, 3])
    Y = tf.placeholder(tf.float32, [None, num_classes])
    keep_prob = tf.placeholder(tf.float32)

    logits = Res152(X, keep_prob, train=False)
    out = tf.nn.softmax(logits)

    xs, ts, paths = data_load("../Dataset/test/images/")

    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    config.gpu_options.visible_device_list="0"

    with tf.Session(config=config) as sess:
        saver = tf.train.Saver()
        #saver = tf.train.import_meta_graph("./cnn.ckpt.meta")
        saver.restore(sess, "./cnn.ckpt")

        for i in range(len(paths)):
            x = xs[i]
            t = ts[i]
            path = paths[i]

            x = np.expand_dims(x, axis=0)

            pred = sess.run([out], feed_dict={X:x, keep_prob:1.})[0]
            pred = pred[0]
            #pred = out.eval(feed_dict={X: x, keep_prob: 1.0})[0]

            print("in {}, predicted probabilities >> {}".format(path, pred))

def arg_parse():
    parser = argparse.ArgumentParser(description='CNN implemented with Keras')
    parser.add_argument('--train', dest='train', action='store_true')
    parser.add_argument('--test', dest='test', action='store_true')
    args = parser.parse_args()
    return args

# main
if __name__ == '__main__':
    args = arg_parse()

    if args.train:
        train()
    if args.test:
        test()

    if not (args.train or args.test):
        print("please select train or test flag")
        print("train: python main.py --train")
        print("test:  python main.py --test")
        print("both:  python main.py --train --test")


import h5py, pytz, csv, numpy
import matplotlib.pyplot as plt
from datetime import datetime
from pytz import timezone
from scipy.signal import medfilt

FILENAME = '1541962108935000000_167_838.h5'
CERN_TIMEZONE = timezone('Europe/Zurich')
UNIXTIME = FILENAME[:18]

def task_1_time():
    utc = pytz.utc
    utc_dt = utc.localize(datetime.utcfromtimestamp(int(UNIXTIME)/100000000))
    cern_dt = utc_dt.astimezone(CERN_TIMEZONE)
    print('UTC datetime: ', utc_dt, '\nCERN datetime: ', cern_dt)

def task_2_csv():
    f = h5py.File(FILENAME, 'r')

    csv_data = []
    def getData(name, obj):
        if isinstance(obj, h5py.Dataset):
            try:
                csv_data.append([name, obj.size, obj.shape, obj.dtype, 'Dataset'])
            except TypeError: # There are a few data type for which h5py does not have a type
                csv_data.append([name, obj.size, obj.shape, None, 'Dataset'])
        elif isinstance(obj, h5py.Group):
            csv_data.append([name, None, None, None, 'Group'])
    f.visititems(getData)

    with open("%s.csv" % str(UNIXTIME), "w") as csv_file:
        writer = csv.writer(csv_file)
        writer.writerow(['name', 'size', 'shape', 'dtype', 'type'])
        writer.writerows(csv_data)
    f.close()

def task_3_img():
    f = h5py.File(FILENAME, 'r')

    dataset = 'AwakeEventData/XMPP-STREAK/StreakImage/streakImageData'
    dataset_height = 'AwakeEventData/XMPP-STREAK/StreakImage/streakImageHeight'
    dataset_width = 'AwakeEventData/XMPP-STREAK/StreakImage/streakImageWidth'

    dataset_2d = f[dataset][()].reshape(f[dataset_height][0], f[dataset_width][0])
    filtered_img = medfilt(dataset_2d)
    plt.imshow(filtered_img)
    plt.savefig("%s.png" % str(UNIXTIME))
    # plt.show()
    f.close()

if __name__ == "__main__":
    task_1_time()
    task_2_csv()
    task_3_img()


import logging

class AutoLogger(object):
    def setupLogger(loggerName, logFile, level=logging.INFO):
        l = logging.getLogger(loggerName)
        formatter = logging.Formatter('%(asctime)s : %(message)s')
        fileHandler = logging.FileHandler(logFile, mode='w')
        fileHandler.setFormatter(formatter)

        l.setLevel(level)
        l.addHandler(fileHandler)


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Mon May 22 13:51:06 2017

@author: leo
"""
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import json

#with open("results_paths.json",'r') as fp:
#    p = json.load(fp)

def heat_map(plotPath, fn, thevar="DSsc"):

    filename = plotPath + fn
    data = np.loadtxt(filename, skiprows=1, delimiter=',')

    with open(filename) as f:
        lab = f.readline()
        lab = lab.replace(' ','') #old versions have spaces...
        labels = lab.split(',')
        data.shape = data.size
        data.dtype = [(l, float) for l in labels]
        data.sort(order=(labels[1], labels[3]))

    x = data['vel']
    y = data['gmax']

    sqsize = int(np.sqrt(len(data)))

    X = np.reshape(x,(-1,sqsize))
    Y = np.reshape(y,(-1,sqsize))

    SACS = fn.split('_')
    sacNames = []
    for i in range(len(SACS)):
        # if we found a number, it is a cell (TRUE), otherwise, FALSE
        sacNames.append(SACS[i].isdigit())

    sac_number = []
    for i in range(len(sacNames)):
        if(sacNames[i]):  # If true, I will save the sac number
            sac_number.append(SACS[i])

    plots = range(len(sac_number))  # how many plots? One for each SAC

    for sac in plots:
        z = data['sc%dp' % sac] * data['DSsc%d' % sac]
    #    z = data['DSsc%d' % sac]
        # z = data['sc%dp' % sac] * data['DSsc%d' % sac]
        z = data[thevar + str(sac)]
        # the following number must be concordant with the stimuli speed
        Z = np.reshape(z, (-1, sqsize))
        fig, ax = plt.subplots()
        # plt.clf()
        plt.title('SAC number: ' + str(sac_number[plots.index(sac)]))
        cax = ax.pcolor(X, Y, Z, vmin=0, vmax=.5)
        ax.set_xscale('log')
        ax.set_yscale('log')
        ax.set_ylabel('$Stimulus\/\ intensity\/\ g_{max}$', fontsize=15)
        ax.set_ylim(min(y), max(y))
        ax.set_xlim(min(x), max(x))
        ax.set_xlabel('$Velocity\/\ (mm/s)$', fontsize=15)
        cbar = fig.colorbar(cax, ticks=[0, 0.1, 0.2, 0.3, 0.4, 0.5])
        cbar.set_label('$DSI$', fontsize=15)
        plt.tight_layout()
        plt.savefig(plotPath + '/heatmap-' + thevar + str(sac) + '.png', dpi=300)


"""Class to manage a result."""
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict, List

import pandas as pd

from xplogger import utils as xplogger_utils
from xplogger.parser.experiment import ExperimentSequence  # type: ignore

# error: Module 'xplogger.parser.experiment' does not explicitly export attribute 'ExperimentSequence'; implicit reexport disabled
from xplogger.utils import to_json_serializable

@dataclass
class Result:
    id: str
    name: str
    label: str
    config_ids: List[str]
    mongo_ids: List[str]
    experiment_sequence: ExperimentSequence
    metrics: Dict[str, pd.DataFrame]
    info: Dict[str, Any]

    def _get_json_dump(self) -> str:
        return json.dumps(
            {
                "id": self.id,
                "name": self.name,
                "label": self.label,
                "config_ids": self.config_ids,
                "mongo_ids": self.mongo_ids,
                "info": self.info,
            },
            indent=4,
            sort_keys=True,
            default=to_json_serializable,
        )

    def serialize(self, dir_path: Path) -> Path:
        """Serialize the result object and save at a given path."""
        dir_path = dir_path.joinpath(self.name)
        xplogger_utils.make_dir(dir_path)
        data = self._get_json_dump()
        data_path = dir_path.joinpath("data.json")
        with open(data_path, "w") as f:
            f.write(data)
        metric_dir = dir_path.joinpath("metric")
        xplogger_utils.make_dir(metric_dir)
        for key in self.metrics:
            path_to_save = metric_dir.joinpath(key)
            if self.metrics[key].empty:
                pass
            else:
                self.metrics[key].to_feather(path=path_to_save)
        return dir_path

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, Result):
            return NotImplemented
        return (
            (
                self.id == other.id
                and self.name == other.name
                and self.label == other.label
                and self.config_ids == other.config_ids
                and self.mongo_ids == other.mongo_ids
                and self.info == other.info
            )
            and self.metrics.keys() == other.metrics.keys()
            and all(
                self.metrics[key].equals(other.metrics[key]) for key in self.metrics
            )
        )

def deserialize(dir_path: Path) -> Result:
    """Deserialize the result object, given a path."""
    data_dir = dir_path.joinpath("data.json")
    with open(data_dir, "r") as f:
        json_dump = f.read()
    data = json.loads(json_dump)
    metric_dir = dir_path.joinpath("metric")
    metrics: Dict[str, pd.DataFrame] = {}
    for path_to_load_metric in metric_dir.iterdir():
        if path_to_load_metric.is_file():
            key = path_to_load_metric.parts[-1]
            metrics[key] = pd.read_feather(path_to_load_metric)
    if not metrics:
        metrics["all"] = pd.DataFrame()
    return Result(
        **data,
        experiment_sequence=None,
        metrics=metrics,
    )


from django.db.models import signals
from django.contrib.gis.db import models
from django.contrib.auth.models import User
from localground.account.models.permissions import ObjectAuthority

class UserProfile(models.Model):
    #https://docs.djangoproject.com/en/dev/topics/auth/#creating-users
    user = models.OneToOneField('auth.User') # This field is required.
    email_announcements = models.BooleanField(default=True)
    default_location = models.PointField(null=True, blank=True,
                        help_text='Search map by address, or drag the marker to your home location')
    default_view_authority = models.ForeignKey('account.ObjectAuthority',
                                default=1, verbose_name='Share Preference',
                                help_text='Your default sharing settings for your maps and media') #default to private
    contacts = models.ManyToManyField('auth.User', related_name='%(app_label)s_%(class)s_related',
                                      null=True, blank=True,
                                      verbose_name="Users You're Following")
    objects = models.GeoManager()

    class Meta:
        app_label = "account"

def create_profile_on_insert(sender, instance, created, **kwargs):
    #When a new user is created, also create a profile_object.
    #Works just like a database trigger.
    if created:
        profile = UserProfile()
        profile.email_announcements = True
        profile.default_view_authority = ObjectAuthority.objects.get(id=1)
        profile.user = instance
        profile.save()
signals.post_save.connect(create_profile_on_insert, sender=User)


'''
To run:

    python plot.py

in this directory, and navigate to:

    http://localhost:5000

'''
#!/usr/bin/env python

from __future__ import print_function

import flask

import numpy as np
import pandas as pd
import datetime
import urllib
import ijson

from bokeh.embed import components
from bokeh.plotting import *
from bokeh.models import CustomJS,HoverTool,ColumnDataSource, PrintfTickFormatter
from bokeh.models.widgets import Slider, TextInput
from bokeh.resources import INLINE
from bokeh.util.string import encode_utf8

from collections import OrderedDict

app = flask.Flask(__name__)

url_default='http://msi.mcgill.ca/GSoC_NANOGrav/pulsar_data_test.json'

def getitem(obj, item, default):
    if item not in obj:
        return default
    else:
        return obj[item]

@app.route("/")
def polynomial():
    """ Very simple embedding of a polynomial chart

    """

    # Grab the inputs arguments from the URL
    # This is automated by the button
    args = flask.request.args

    # Get all the form arguments in the url with defaults
    _source=getitem(args,'_source',url_default)
    _size=int(getitem(args,'size','5'))
    favcolor=getitem(args,'favcolor','#1F77B4')
    bcolor=getitem(args,'bcolor','#F5F5DC')
    _shape=getitem(args,'_shape','circle')
    lcolor=getitem(args,'lcolor','#FCB938')
    toggle=bool(getitem(args,'toggle',''))

    #Parsing json

    DEFAULT_TICKERS = ['TOAs','RawProfiles', 'Period', 'PeriodDerivative', 'DM', 'RMS', 'Binary']
    TOOLS = "tap,resize,crosshair,pan,wheel_zoom,box_zoom,reset,box_select,lasso_select,previewsave,hover"

    Pulsar,TOAs,RawProfiles, Period, PeriodDerivative, DM, RMS, Binary=[],[],[],[],[],[],[],[]
    url_location=urllib.urlopen(_source)
    for item in ijson.items(url_location,"item"):
        Pulsar.append(str(item["Pulsar"]))
        TOAs.append(float(item["TOAs"]))
        RawProfiles.append(int(item["Raw Profiles"]))
        Period.append(float(item["Period"]))
        PeriodDerivative.append(float(item["Period Derivative"]))
        DM.append(str(item["DM"]))
        RMS.append(str(item["RMS"]))
        if (item["Binary"]=="Y"):
            Binary.append("Yes")
        else:
            Binary.append("No")

    #Create a plot periodvs period derivative

    s1 = ColumnDataSource(
    data=dict(
        Pulsar=Pulsar,
        TOAs=TOAs,
        RawProfiles=RawProfiles,
        Period=Period,
        PeriodDerivative=PeriodDerivative,
        DM=DM,
        RMS=RMS,
        Binary=Binary,
        )
    )

    p1 = figure(plot_width=600, plot_height=600,
               title="Period vs Period Derivative", y_axis_type="log" ,y_range=[min(PeriodDerivative)-min(PeriodDerivative)/5, max(PeriodDerivative)+max(PeriodDerivative)/5],x_range=[min(Period)-min(Period)/5, max(Period)+max(Period)/5],x_axis_label='Period[s]', y_axis_label='Period Derivative[s/s]',tools=TOOLS)
    p1.background_fill_color = bcolor
    p1.background_fill_alpha = "0.5"
    getattr(p1,_shape)('Period', 'PeriodDerivative', legend="period deri",color=favcolor,size=_size, source=s1)
    #p1.circle('Period', 'PeriodDerivative', legend="period deri",color=favcolor,size=_size, source=s1)
    p1.xaxis.axis_label_text_font_size = "15pt"
    p1.yaxis.axis_label_text_font_size = "15pt"
    #p1.xaxis[0].formatter = PrintfTickFormatter(format="s")
    #p1.yaxis[0].formatter = PrintfTickFormatter(format=" s/s")

    #Toggle Line
    #if getattr(p1,line,None):
    #   getattr(p1,line)('Period', 'PeriodDerivative',legend="period deri",line_dash=[4, 4], line_color=lcolor, line_width=1,source=s1)
    p1.line('Period', 'PeriodDerivative',legend="period deri",line_dash=[4, 4], line_color=lcolor, line_width=1,source=s1,visible=toggle)

    # Custom data source for selected points
    s2 = ColumnDataSource(
        data=dict(
            Pulsar=[],
            TOAs=[],
            RawProfiles=[],
            Period=[],
        PeriodDerivative=[],
        DM=[],
        RMS=[],
        Binary=[],
        )
    )

    p2= figure(plot_width=600, plot_height=600,
               title=" Selected points from Period vs Period Derivative", y_axis_type="log" ,y_range=[min(PeriodDerivative)-min(PeriodDerivative)/10, max(PeriodDerivative)+max(PeriodDerivative)/10],x_range=[min(Period)-min(Period)/10, max(Period)+max(Period)/10],x_axis_label='Period[s]', y_axis_label='Period Derivative[s/s]',tools=TOOLS)
    p2.xaxis.axis_label_text_font_size = "15pt"
    p2.yaxis.axis_label_text_font_size = "15pt"

    p2.circle('Period', 'PeriodDerivative', legend="period deri",alpha=1.2, source=s2)

    s1.callback = CustomJS(args=dict(s2=s2), code="""
            var inds = cb_obj.get('selected')['1d'].indices;
            var d1 = cb_obj.get('data');
            var d2 = s2.get('data');
            d2['Pulsar'] = []
            d2['TOAs'] = []
            d2['RawProfiles'] = []
            d2['Period'] = []
            d2['PeriodDerivative'] = []
            d2['DM'] = []
            d2['RMS'] = []
            d2['Binary'] = []
            for (i = 0; i < inds.length; i++) {
                d2['Pulsar'].push(d1['Pulsar'][inds[i]])
                d2['TOAs'].push(d1['TOAs'][inds[i]])
                d2['RawProfiles'].push(d1['RawProfiles'][inds[i]])
                d2['Period'].push(d1['Period'][inds[i]])
                d2['PeriodDerivative'].push(d1['PeriodDerivative'][inds[i]])
                d2['DM'].push(d1['DM'][inds[i]])
                d2['RMS'].push(d1['RMS'][inds[i]])
                d2['Binary'].push(d1['Binary'][inds[i]])

            }
            s2.trigger('change');
      """  )

    hover = p1.select(dict(type=HoverTool))
    hover.tooltips = OrderedDict([
        ("Pulsar's Name", '@Pulsar'),
        ('TOAs', '@TOAs'),
        ('RawProfiles', '@RawProfiles'),
        ('Period[s]', '@Period'),
        ('PeriodDerivative[s/s]', '@PeriodDerivative'),
        ('DM[pc/cc]', '@DM'),
        ('RMS[us]', '@RMS'),
        ('Binary', '@Binary'),
    ])

    js_resources = INLINE.render_js()
    css_resources = INLINE.render_css()

    #Setting up layout
    layout = hplot(p1, p2)

    script, div = components(layout,INLINE)
    html = flask.render_template(
        'embed.html',
        plot_script=script,
        plot_div=div,
        js_resources=js_resources,
        css_resources=css_resources,
        _source=_source,
        Pulsar=Pulsar,
        _shape=_shape,
        favcolor=favcolor,
        bcolor=bcolor,
        _size=_size,
        lcolor=lcolor,
        toggle=toggle
    )
    return encode_utf8(html)

if __name__ == "__main__":
    print(__doc__)
    app.run()


from collections import defaultdict
import math
n = list(input())
answer = 0
dict = defaultdict(int)

for i in range(len(n)):
    if n[i] == "6" or n[i] == "9":
        dict["6"] += 1
    else:
        dict[n[i]] += 1
dict["6"] = int(math.ceil(dict["6"]/2))

sorted_dict = sorted(dict.items(), key=lambda x : -x[1])
print(sorted_dict[0][1])


'''
Database
interacts with the database via files formatted from csv files
'''
# import sys
# sys.path.append("./hp_norton_inventory")
import logging
import time
import cProfile, pstats
import linear_csv_handler as csvh
from mongo_connect import *
from timeit import timeit as timer

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def import_data(directory_name, customer_file, product_file, rentals_file):
    ''' import data from csv files into database to be used in functions '''
    mongo = MongoDBConnection()
    csv_handler = csvh.CsvHandler()

    ERROR_COUNT = {
        'PRODUCT_ERROR': 0,
        'CUSTOMER_ERROR': 0,
        'RENTALS_ERROR': 0
    }

    INVENTORY_COUNT = {
        'product_totals': 0,
        'customer_totals': 0,
        'rental_totals': 0
    }

    product_file = f'{directory_name}/{product_file}'
    customer_file = f'{directory_name}/{customer_file}'
    rentals_file = f'{directory_name}/{rentals_file}'

    with mongo:
        # generate hpnorton_db
        hpnorton_db = mongo.connection.hpnorton_db

        # collections in database
        product_db = hpnorton_db['product']
        customer_db = hpnorton_db['customer']
        rentals_db = hpnorton_db['rental']

        try:
            products = csv_handler.generate_document_list(product_file, "product")
        except FileNotFoundError as error:
            logger.info(f' File not found {error}')
            ERROR_COUNT['PRODUCT_ERROR']+=1

        try:
            customers = csv_handler.generate_document_list(customer_file, "customer")
        except FileNotFoundError as error:
            logger.info(f' File not found {error}')
            ERROR_COUNT['CUSTOMER_ERROR']+=1

        try:
            rentals = csv_handler.generate_document_list(rentals_file, "rental")
        except FileNotFoundError as error:
            logger.info(f' File not found {error}')
            ERROR_COUNT['RENTALS_ERROR']+=1

        # write to database
        try:
            product_db.insert_many(products)
            product_totals = [product_id for product_id in product_db.find()]
            INVENTORY_COUNT['product_totals'] = len(product_totals)
        except UnboundLocalError as error:
            logger.info(f' {error}')
            ERROR_COUNT['PRODUCT_ERROR']+=1

        try:
            customer_db.insert_many(customers)
            customer_totals = [customer_id for customer_id in customer_db.find()]
            INVENTORY_COUNT['customer_totals'] = len(customer_totals)
        except UnboundLocalError as error:
            logger.info(f' {error}')
            logger.info(ERROR_COUNT['CUSTOMER_ERROR'])
            ERROR_COUNT['CUSTOMER_ERROR']+=1

        try:
            rentals_db.insert_many(rentals)
            rental_totals = [rental_id for rental_id in rentals_db.find()]
            INVENTORY_COUNT['rental_totals'] = len(rental_totals)
        except UnboundLocalError as error:
            logger.info(f' {error}')
            ERROR_COUNT['RENTALS_ERROR']+=1

        inventory_count = [INVENTORY_COUNT['product_totals'],
                           INVENTORY_COUNT['customer_totals'],
                           INVENTORY_COUNT['rental_totals']]

        error_count = [ERROR_COUNT['PRODUCT_ERROR'],
                       ERROR_COUNT['CUSTOMER_ERROR'],
                       ERROR_COUNT['RENTALS_ERROR']]

        return [tuple(inventory_count), tuple(error_count)]

def show_available_products():
    ''' returns a list of items available for rent '''
    mongo = MongoDBConnection()
    #available_units = {}

    with mongo:
        hpnorton_db = mongo.connection.hpnorton_db
        product_totals = [product for product in hpnorton_db.products.find()]
        available = list(filter(lambda units: int(units['quantity_available']) > 0, product_totals))

        return available

def show_rentals(product_id):
    ''' returns the matching fields information based on product id '''
    mongo = MongoDBConnection()

    with mongo:
        hpnorton_db = mongo.connection.hpnorton_db
        rental_totals = [rental for rental in hpnorton_db.rentals.find()]
        rented_unit = list(filter(lambda units: units['product_id'] == product_id, rental_totals))

        return rented_unit

def main():
    ''' main method to interact with mongodb '''
    mongo = MongoDBConnection()
    output = import_data('data', 'customer.csv', 'product.csv', 'rental.csv')

    logger.info(f' Total number of invetory and errors {output}')

    with mongo:
        database = mongo.connection.hpnorton_db
        yorn = input("Drop data?")
        if yorn.upper() == 'Y':
            database['customers'].drop()
            database['rentals'].drop()
            database['products'].drop()

if __name__ == "__main__":
    output_code = '''
output = import_data('data', 'customer.csv','product.csv','rental.csv')
    '''
    print(timer(output_code,globals=globals(),number=1))
    main()
    # profiler = cProfile.Profile()
    # profiler.run('main()')
    # profiler.print_stats()
    # pstats.Stats(profiler).sort_stats('time').print_stats()


from multiprocessing import Process
from multiprocessing import Event
import time

# 生成事件对象
e = Event()

def wait_event():
    print("wait for event setting")
    e.wait()
    print("wait_for_event01", e.is_set())

def wait_event_timeout():
    print("wait for event setting or timeout")
    e.wait(2)
    print("wait_for_event02", e.is_set())

p1 = Process(name='block', target=wait_event)
p1.start()

p2 = Process(name='non-block', target=wait_event_timeout)
p2.start()

print("main:setting the event")
time.sleep(3)
e.set()
print("event is set")


# -*- coding: utf-8 -*-
"""
Created on Mon Feb  4 17:51:31 2019

@author: Education
"""

from PIL import Image

img = Image.open("IMG20171019190155.jpg")

#print(img.mode)        to show the mode of image

img_gray = img.convert(mode = 'L')      #to convert the mode of a image
img_gray.show()                        #to show the image

img_rotate = img.rotate(90)
img_rotate.show()             #to rotate a iamge

#to crop a image
img_crop = img.crop((40,80,100,204))
img_crop.show()

#to crop a image from center
new_width = 160
new_height = 204

width,height = img.size

left = (width - new_width)/2
top = (height - new_height)/2
right = (width + new_width)/2
bottom = (height + new_height)/2

img_crop1 = img.crop((left,top,right,bottom))
img_crop1.show()

save = img.copy()              #to create copy of image
save.show()

size = (56,45)
img.thumbnail(size)
img.show()