import sys
import time
import os
import statistics as st

FILE_NAME = "atk.c"
PAUSE_TIME = 5
REPETITIONS = 10
SECRET = [20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20]
T0_LINE = 13
OFFSET = 0

W = {"name": "WAIT_TIME ", "start": 0, "end": 50, "step": 50}

I = {"name": "I ", "start": 32, "end": 256, "step": 2}
N = {"name": "N ", "start": 128, "end": 180, "step": 20}

It = {"name": "It ", "start": 2, "end": 20000, "step": 2}
Nt = {"name": "Nt ", "start": 128, "end": 50, "step": 10}

def main():

    if (str(sys.argv[1]) is '1'):
        test1()
    elif (str(sys.argv[1]) is '2'):
        test2()
    else:
        print("No test was performed!")

def test1():

    I_name = I["name"]
    N_name = N['name']

    i = N["start"]

    # Modify atk.c with the current N value
    change(FILE_NAME, N_name , i )
    # Register results with N = i
    write_results(1, "N = " + str(i))

    ii = I["start"]
    while (ii < I["end"]):

        # Modify atk.c with the current I value
        change(FILE_NAME , I_name, ii )

        n_half_bytes = []
        rep = 0
        for rep in range(REPETITIONS):

            # Pause to cool CPU
            time.sleep(PAUSE_TIME)
            # Clean, compile, run attack
            os.system("make clean; make; ./atk")
            # Pause to cool CPU
            time.sleep(PAUSE_TIME)
            # run crypto-analysis program
            os.system("python3 crypto.py")

            # get the number of half key bytes discovered by the attack
            dk = read_disc_key_file()
            n_half_bytes.append(get_n_discovered_bits(dk, SECRET))

        # write averaged half key number discovered in results file
        content = str(ii) + "\t" + str(st.mean(n_half_bytes))
        write_results(1, content )

        # Increment cicle variable
        ii *= I["step"]

def test2():

    It_name = It["name"]
    Nt_name = Nt['name']

    i = Nt["start"]

    # Modify atk.c with the current N value
    change(FILE_NAME, Nt_name , i )
    # Register results with N = i
    write_results(2, "N = " + str(i))

    ii = It["start"]
    while (ii < It["end"]):

        # Modify atk.c with the current I value
        change(FILE_NAME , It_name, ii )

        # 1 on correct tbox answer; 0 when not
        capacity = []
        rep = 0
        while(rep < REPETITIONS):
            rep += 1

            # Pause to cool CPU
            time.sleep(PAUSE_TIME)
            # Clean, compile, run attack
            os.system("make clean; make; ./atk")
            # Pause to cool CPU
            time.sleep(PAUSE_TIME)
            # run crypto-analysis program
            os.system("python3 crypto.py")
            # get the number of half key bytes discovered by the attack
            offset, t0 = read_disc_tbox_file()

            if ((t0 == T0_LINE) and (offset == OFFSET)):
                capacity.append(1)
            else:
                capacity.append(0)

        # write averaged half key number discovered in results file
        content = str(ii) + "\t" + str(st.mean(capacity))
        write_results(2, content )

        # Increment cicle variable
        ii *= It["step"]

# Auxiliar Functions

def write_results(test_number, line):

    result_file = "result" + str(test_number) + ".out"

    with open(result_file, 'a') as file:
        file.write( line + "\n")

def performance1():

    l=0
    unused_lines = [6, 31, 52, 58, 59]

    while(True):
        try:
            timings  = read_files(l, "meas")
            l+=1
        except IOError:
            break

        avg = st.mean(timings)
        dev = st.stdev(timings)

        line_score = []
        meas_score = []

        for line in unused_lines:
            if (timings[line] < (avg + 1*dev)):
                line_score.append(timings[line]/avg)
        meas_score.append(st.mean(line_score))

    attack_score = st.mean(meas_score)

    attack_score = float("{0:.4f}".format(attack_score))

    return attack_score

def get_n_discovered_bits(key1, key2):

    n_common_half_keys = 0

    if(len(key1) is not len(key2)):
        print("Key1 has not the same size as key2")

    for i in range(len(key1)):

        if((key1[i] % 16) == (key2[i] % 16)):
            n_common_half_keys += 1
        if((key1[i] >> 4) == (key2[i] >> 4)):
            n_common_half_keys += 1

    return n_common_half_keys

def change(file_name, variable_name, amount ):

    # Read file
    with open(file_name, 'r') as file:
        data = file.readlines()

    # Perform modifications
    for l, line in enumerate(data):
        if variable_name in line:
            data[l] = "#define " + str(variable_name) + str(amount) + "\n"
            break

    # Write in File Modification
    with open(file_name, 'w') as file:
        file.writelines( data )

def read_files(l, file_name):

    meas_file = open("side_channel_info/" + file_name + "#" + str(l) + ".out", "r")
    plaintext_raw = meas_file.readline()
    scores = [int(i) for i in meas_file]
    meas_file.close()

    return scores

def read_disc_key_file():

    table_file = open("discovered_key_.out", "r")
    discovered_key = [int(i) for i in table_file]
    table_file.close()

    return discovered_key

def read_disc_tbox_file():

    tbox_file = open("tbox_discovered_.out", "r")
    values = [int(i) for i in tbox_file]
    tbox_file.close()

    return values[0], values[1]

main()


import streamlit as st

st.title("201921054 임채민")

st.header("201921054 임채민")
st.subheader("과제")
st.text("폐루프 함수=G(s) = 100 / (s^2 + 5s + 6)")

import numpy as np
from scipy import signal
import matplotlib.pyplot as plt

s1 = signal.lti([100],[1,5,6])

frequencies = np.logspace(-2,2,500)

w, mag, phase= s1. bode(frequencies)

plt.figure()
plt.subplot(2,1,1)
plt.semilogx(w,mag)
plt.title('Bode plot of G(s)=100/(s+2)(s+3)')
plt.ylabel('Magnitude [dB]')

plt.subplot(2,1,2)
plt.semilogx(w,phase)
plt.ylabel('Phase [degrees]')
plt.xlabel('Frequency [Hz]')

st.pyplot(plt)


#!/usr/bin/python3

# ## {R2D2919B742E} ##
# ###########################################################################
# What if magic existed?
# What if a place existed where your every thought and dream come to life.
# There is only one catch: it has to be written down.
# Such a place exists, it is called programming.
#    - Scott Taylor Reph, RightThumb.com
# ###########################################################################
# ## {C3P0D40fAe8B} ##

##################################################
import os, sys, time
##################################################
import _rightThumb._construct as __
appDBA=__.clearFocus(__name__,__file__);__.appReg=appDBA;
def focus(parentApp='',childApp='',reg=True):
	global appDBA;f=__.appName(appDBA,parentApp,childApp);
	if reg:__.appReg=f;
	return f
import _rightThumb._base3 as _
fieldSet=_.l.vars(focus(),__name__,__file__,appDBA)
_.load()
##################################################
_v = __.imp('_rightThumb._vars')
_str = __.imp('_rightThumb._string')
##################################################

def sw():
	pass
	#b)--> examples
	_.switches.register( 'Requests', '-r' )
	_.switches.register( 'Print', '-print' )
	#e)--> examples
	# _.switches.register( 'Files', '-f,-fi,-file,-files','file.txt', isData='glob,name,data,clean', description='Files', isRequired=False )

# __.setting('require-list',['Files,Plus','File,Has']) # todo
# __.setting('require-list',['Pipe','Files'])
__.setting('receipt-log')
__.setting('receipt-file')
__.setting('myFileLocations-skip-validation',False)
__.setting('require-pipe',False)
__.setting('require-pipe||file',False)
__.setting('pre-error',False)
__.setting('switch-raw',[])

_.appInfo[focus()] = {
	# 'app': '8facG-jo0Cxk',
	'file': 'thisApp.py',
	'liveAppName': __.thisApp( __file__ ),
	'description': 'Changes the world',
		# _.ail(1,'subject')+
		# _.aib('one')+
	'categories': [
						'DEFAULT',
				],
	'usage': [
						# 'epy another',
						# 'e nmap',
						# '',
	],
	'relatedapps': [
						# 'p another -file file.txt',
						# '',
	],
	'prerequisite': [
						# 'p another -file file.txt',
						# '',
	],
	'examples': [
						_.hp('p thisApp -file file.txt'),
						_.linePrint(label='simple',p=0),
						'',
	],
	'columns': [
					# { 'name': 'name', 'abbreviation': 'n' },
					# { 'name': '{1}', 'abbreviation': '{0}', 'sort': '{2}' },
	],
	'aliases': [
					# 'this',
					# 'app',
	],
	'notes': [
					# {},
	],
}

_.appData[focus()] = {
		'start': __.startTime,
		'uuid': '',
		'audit': [],
		'pipe': False,
		'data': {
					'field': {'sent': [], 'received': [] }, # { 'label': '', 'context': [],  }
					'table': {'sent': [], 'received': [] },
		},
	}

def triggers():
	_.switches.trigger( 'Files', _.myFileLocations, vs=True )
	_.switches.trigger( 'Ago', _.timeAgo )
	_.switches.trigger( 'Folder', _.myFolderLocations )
	_.switches.trigger( 'URL', _.urlTrigger )
	_.switches.trigger( 'Duration', _.timeFuture )

_.l.conf('clean-pipe',True)
_.l.sw.register( triggers, sw )

########################################################################################
#b)--> examples
#d)--> code hints to quickly get started
	#n)--> inline examples
		# any(ele in 'scott5' for ele in list('0123456789'))
		# if _.switches.isActive('Test'): test(); return None;
		# result=[]; result=[ _.pr(line) for i, line, bi in _.numerate( _.isData(r=0) )]
		# bk=[];[  bk.append(rec['backup']) for rec in backupLog if path == rec['file']]; bk=bk[-1];
		# a=(1 if True else 0) <--#
		#!)--> m=[[row[i] for row in matrix] for i in range(4)]

	#n)--> python globals
		# for k in globals(): print(k, eval(k) )

	#n)--> webpage from url
		# for subject in _.caseUnspecific( line, needle ): line = line.replace( subject, _.colorThis( subject, 'green', p=0 ) )

	#n)--> webpage from url
		# requests=__.imp('requests.post')
		#!)--> data=str(requests.post(url,data={}).content,'iso-8859-1')

	#n)--> import and backup example
		# _bk = _.regImp( __.appReg, 'fileBackup' ); _bk.switch( 'Silent' ); _bk.switch( 'isRunOnce' ); _bk.switch( 'Flag', 'APP' ); _bk.switch( 'DoNotSchedule' )
		# _bk.switch( 'Input', path ); bkfi = _bk.action();

	#n)--> inline
		# for rel in [ subject for subject in _.isData(r=0) if _.showLine(subject) ]: print(rel)

	#n)--> banner
		# banner=_.Banner(app); goss=banner.goss;
#e)--> examples
########################################################################################
#n)--> start

def _listener_():
	global hk
	# hkr.do("beepy.note('d',3,'dotted_eigth')") # (:g e:) c,    c#, d, d#, e, f, f#, g, g#, a, a#
	# sys.exit()
	hk.beepy.note('d',3,'dotted_eigth')
	r = sr.Recognizer()
	with sr.Microphone() as source:
		if _.switches.isActive('Print'):
			print("Talk")
		# audio = r.listen(source,timeout=1)
		audio = r.listen(source)
	try:
		# beepy.note('f')
		_translate_(r.recognize_google(audio))
		# hk.beepy.note('d')
		# beepy.note('a')

	except Exception as e:
		print(e)
		pass
		if _.switches.isActive('Print'):
			print('translate failure')
		hk.beepy.note('d',4,'dotted_eigth')

def _translate_(speech):
	global hk
	speech=speech.lower()

	finagled=False
	for finagle in _.v.finagling['_replace']:
		if not finagled and finagle in speech: speech=speech.replace(finagle,_.v.finagling['_replace'][finagle]); finagled=True;
	for finagle in _.v.finagling['_startswith']:
		if not finagled and speech.startswith(finagle): speech=speech=_.v.finagling['_startswith'][finagle]; finagled=True;

	# os.system('say '+speech)
	if _.switches.isActive('Print'):
		print('said:',speech)
	_.v.prompt=speech
	subject=None
	def check(tst): return tst if not subject and len(tst.split(' ')) == sum(1 for y in tst.split(' ') if ' ' + y + ' ' in ' ' + speech + ' ') else None
	def checkAll(dic):
		subject=None
		for sub in dic:
			if not subject:
				if check(sub):
					if _.switches.isActive('Print'):
						print(0,dic[sub])
					subject = sub
					# print(99,sub,sub in _.v.local)
					if not sub in _.v.local:
						hkr.do(dic[sub])
					else:
						exec(dic[sub])
					if _.switches.isActive('Print'):
						print(1,dic[sub])
			# if subject: print(dic[sub])
		return subject

	global dic

	subject = checkAll(dic)
	if subject:
		# hkr.do("beepy.note('d',3,'dotted_eigth')")
		run='say '+subject
		if _.switches.isActive('Print'):
			print(run)
		# os.system(run)
		hk.beepy.note('d',3,'whole')
	else:
		print('no subject')
		hk.beepy.note('d',2,'half')
		# hkr.do("beepy.note('d',2,'dotted_eigth')")
		# print('no subject')

# sub='explode'
# elif check(sub):
#     subject=sub
#     Clip.explode()

# sub='implode'
# if check(sub):
#     subject=sub
#     Clip.implode()

	# print(speech)

def action():
	if _.switches.isActive('Requests'):
		global dic
		table=[]
		for k in dic:
			rec={ 'request': k, 'action': dic[k] }
			if _.showLine(str(rec)):
				table.append(rec)
		_.pt(table)
	else:
		_listener_()

# print(1)
# hkr.do('_test_()')
# print(2)
# sys.exit()
# import hotkeys as hk

def ai():

	max_tokens=1024
	if _.switches.isActive('Print'):
		print('ai: initialization')
		print(_.v.prompt)
	global interact
	# Sally paste  convert the folling python to a single line
	prompt = _.v.prompt.replace('sally','')
	if 'unlimited tokens' in _.v.prompt:
		_.v.prompt.replace('max tokens','')
		max_tokens=int(max_tokens*2)
	if 'paste' in prompt:
		prompt=prompt.replace('paste','')
		_paste = _.regImp( __.appReg, '-paste' )
		lines=[]
		lines.append(prompt.strip()+': ')
		for line in _paste.imp.paste().replace('\r','').split('\n'):
			if 'python' in prompt.lower(): line=line.split('#')[0]
			if 'javascript' in prompt.lower(): line=line.split('//')[0]
			line=line.replace('    ','\t').rstrip()
			if line: lines.append(line)
		prompt = '\n'.join(lines)
	import openai

	if _.switches.isActive('Print'):
		print('_________________')
		print(prompt)
		print('_________________')
	# Apply the API key
	# openai.api_key = _blowfish.decrypt('+WDUtsrHiXMJ2Rk2Z7QMSyoo+xzLW/BhSw/kTcUwPmZGQBDpZs7LcGpy2hm+rAWQ+ZtzKKLFVO8=', _vault.key() )
	# openai.api_key = os.environ['OPENAI_API_KEY']
	openai.api_key = _keychain.imp.key('open-ai-api')

	# Define your prompt
	# prompt = "Write a short story about a person who finds a treasure."

	# Request a response from the API
	response = openai.Completion.create(
		# https://platform.openai.com/docs/models/codex
		# engine="code-davinci-002",
		# engine="text-davinci-002",
		engine="text-davinci-003",
		prompt=prompt,
		temperature=0.9,
		top_p=1,
		frequency_penalty=0.0,
		presence_penalty=0.6,
		# max_tokens=150,
		# stop=[" Human:", " AI:"],
		max_tokens=max_tokens,
		# n=1,
		stop=None,
		# temperature=0.5,
	)

	# def process_prompt_chunk(chunk,max_tokens):
	# 	return openai.Completion.create(
	# 		engine="text-davinci-003",
	# 		prompt=chunk,
	# 		temperature=0.9,
	# 		top_p=1,
	# 		frequency_penalty=0.0,
	# 		presence_penalty=0.6,
	# 		max_tokens=max_tokens,
	# 		stop=None,
	# 	)

	# def process_large_prompt(large_prompt, chunk_size,max_tokens):
	# 	responses = []
	# 	for i in range(0, len(large_prompt), chunk_size):
	# 		chunk = large_prompt[i:i+chunk_size]
	# 		response = process_prompt_chunk(chunk,max_tokens)
	# 		responses.append(response)
	# 	return responses

	# large_prompt = prompt
	# chunk_size = 2048  # adjust based on your needs and API limitations
	# responses = process_large_prompt(large_prompt, chunk_size,max_tokens)

	# Print the response
	if not 'listen' in interact: interact['listen']=[]
	interact['listen'].append({'epoch': time.time(), 'prompt': prompt, 'response': response["choices"][0]["text"]})
	_.saveTable(interact,'ai-bot-interaction.index',p=0)
	if _.switches.isActive('Print'):
		print(response["choices"][0]["text"])
	_copy = _.regImp( __.appReg, '-copy' )
	_copy.imp.copy( response["choices"][0]["text"] )

##################################################
# https://nitratine.net/blog/post/simulate-mouse-events-in-python/

# _code = _.regImp( __.appReg, '_rightThumb._auditCodeBase' )
# _code.imp.validator.register( 'data', 'javascript' )
# _code.imp.validator.createIndex( data, 'javascript', skipLoad=True, simple=False, B=True )
# c = _code.imp.validator.identity['location']['open'][o]

# _.pv(selectors)
# sys.exit()

def cp():
	global keyboard
	with keyboard.pressed(Key.ctrl): vVv='c'; keyboard.press(vVv); keyboard.release(vVv);
def pa():
	global keyboard
	with keyboard.pressed(Key.ctrl): vVv='v'; keyboard.press(vVv); keyboard.release(vVv);

def aquire_snippets():
	print('aquire_snippets')
	global keyboard
	global ti
	snippets=[]
	i=0
	while not i==3:
		i+=1
		time.sleep(ti['min'])
		cp()
		time.sleep(ti['min'])
		snippets.append( _paste.imp.paste() )
		time.sleep(ti['min'])
		keyboard.press(Key.up)
		time.sleep(ti['tiny'])
		keyboard.release(Key.up)
		time.sleep(ti['min'])
	return snippets

# def probable_selectors(snippets):

#     _code.imp.validator.register( 'data', 'javascript' )

#     selectors=[]
#     for snip in snippets:
#         snip=snip.replace('\r','')
#         lines=snip.split('\n')
#         _code.imp.validator.createIndex( data, 'javascript', skipLoad=True, simple=False, B=True )
#         c = _code.imp.validator.identity['location']['open'][o]

def snippet_to_selector(snippets):
	print('snippet_to_selector')
	payload=[]
	profiles = {
					'tag':     {
							'definitive': [
											'pre',
											'code',
											'h3',
							],
							'common': [
											'div',
											'td',
							],
							'parent': [
							],
							'children': [
											'a',
							],
							'omit': [
											'span',
											'tbody',
											'thead',
							],
					},
					# 'attributes':     {},
	}
	proDex={}
	omit=[]
	yTag=[]
	# profiles['tag']['definitive']
	for sec in profiles['tag']:
		for tag in profiles['tag'][sec]:
			if sec == 'omit':
				omit.append(tag)
			else:
				proDex[tag]=sec
				if sec == 'children' or sec == 'parent':
					yTag.append(tag)

	_.pv(proDex)
	# sys.exit()

	results=[]
	for snip in snippets:
		code=snip.replace('\r','').split('\n')[0]
		simpin = {
					' ': 'bc05cdf4',
					'\\"': 'fa9509d9',
		}
		# code='<td class="titleColumn another yet" test="a a\\"s s\\"ds f" yy="abc 123 def 456">'
		sel={}
		selectors={}
		selectors['tag']='err'
		selectors['classes']=''
		selectors['attributes']={}
		selectors['selectors']=[]
		if not '"' in code: selectors['tag']=code.split(' ')[0][1:]
		elif '"' in code:
			code=code.replace('\\"',simpin['\\"'])
			parts=code.split('"')
			key=''
			isNext=False
			for i,x in enumerate(parts):
				if not i:
					_.pr(x,c='green')
					y=x.split(' ')[0]
					selectors['tag']=y[1:]
					_.pr(y,c='purple')
				if x.endswith('='):
					isNext=True
					# print(i,'=',x)
					if ' ' in x: key=x.split(' ')[1]
					else: key=x
					key=key.replace('=','')
				elif isNext:
					sel[key]=x.replace(simpin['\\"'],'\\"')
					isNext=False
					# print(i,'Y',x)
				# else: print(i,'N',x)
			# _.pv(sel)
			if 'class' in sel:
				_classes=[]
				for cl in sel['class'].split(' '): _classes.append('.'+cl)
				selectors['classes']=' '.join(_classes)

			selectors['attributes']=sel
			selectors['selectors'].append(selectors['tag']+' '+selectors['classes'])
			tcl=selectors['tag']+' '+selectors['classes'].split(' ')[0]
			if not tcl in selectors['selectors']: selectors['selectors'].append(tcl)
			for cl in selectors['classes'].split(' '): selectors['selectors'].append(cl)
			results.append(selectors)
	# _.saveTable2(results,'listen_snippet_to_selector.json')
	# _.pv(results)
	refined=[]
	for rec in results:
		refined.append(rec)
		if rec['tag'] in proDex and proDex[rec['tag']] == 'parent':
			break
		if rec['classes']: break
	refined.reverse()
	for rec in refined:
		if rec['tag'] in profiles['tag']['definitive']: return rec['tag']
		tag=rec['tag']
		classy=rec['classes']
		_sel_=''
		started=True
		if not tag in omit:
			if tag in proDex: started=True
			if started:
				if tag in proDex:
					if tag in yTag:
						_sel_+=tag
					if classy:
						_sel_+=' '+classy

		if _sel_: payload.append(_sel_)
	_.pv(results)
	_.pv(yTag)
	print('snippet_to_selector')
	return '  '.join(payload)

def code_scanner():
	print('code_scanner')
	global keyboard
	global ti
	def _scan_(code,r=False):
		global keyboard
		_copy.imp.copy( code )
		with keyboard.pressed(Key.ctrl): vVv='`'; keyboard.press(vVv); keyboard.release(vVv);

		time.sleep(ti['abit1'])
		pa()
		time.sleep(ti['min'])
		keyboard.press(Key.enter); keyboard.release(Key.enter);

		clip=_paste.imp.paste()
		while clip==code:
			time.sleep(ti['min'])
			clip=_paste.imp.paste()

		return clip
	snippets=aquire_snippets()
	# result=_scan_('_skimmer_=123; copy(_skimmer_);')
	# _.pv(snippets)
	# _.saveTable2(snippets,'listen_snippets.json')
	# print('_skimmer_:',result)
	# _.pv(snippets)
	selector = snippet_to_selector(snippets)
	time.sleep(ti['min'])
	keyboard.press(Key.f12)
	keyboard.release(Key.f12)
	time.sleep(ti['abit2'])
	print('selector:',selector)
	if not selector:
		_.e('unable to copy','selector is empty')
	_copy.imp.copy( selector )
	time.sleep(ti['abit1'])
	hkr.do('Clip.browser_f12_tooljs_text()')

ti={
		'tiny': .05,
		'min': .2,
		'abit1': .3,
		'abit2': .6,
}

def clean_text():
	lines = _paste.imp.paste().replace('\r','').split('\n')
	_copy.imp.copy(' '.join(lines))

def auto_scrape():

	# prompt
	# i created an algorithm where i can put my mouse over an element on a webpage and it automatically right clicks, goes to inspect, collects element information, goes up an element more than once. It then tries to identify how to extract information from that webpage.

	print('auto_scrape')
	global mouse
	global keyboard
	global ti
	# time.sleep(3)
	mouse.click(Button.right, 1)
	time.sleep(ti['min'])
	keyboard.press(Key.up)
	keyboard.release(Key.up)
	time.sleep(ti['min'])
	keyboard.press(Key.enter)
	keyboard.release(Key.enter)
	time.sleep(ti['abit1'])
	code_scanner()
	lines=[]
	count={}
	for line in _paste.imp.paste().replace('\r','').split('\n'):
		line=line.strip()
		if line:
			if not line in count:
				count[line]=0
			count[line]+=1
	for line in lines:
		line=line.strip()
		if line:
			if not count[line] > 2:
				lines.append(line)
	_copy.imp.copy('\n'.join(lines))

def ipsum():
	print('ipsum')
	import lorem
	lines=[]
	lines.append(lorem.paragraph())
	lines.append(lorem.paragraph())
	lines.append(lorem.paragraph())
	lines.append(lorem.paragraph())
	lines.append(lorem.paragraph())
	_copy.imp.copy('\n\n'.join(lines))

# time.sleep(ti['min'])
# keyboard.press(Key.f12)
# keyboard.release(Key.f12)

def randomize_text():
	_paste = _.regImp( __.appReg, '-paste' )
	text = _paste.imp.paste()
	_copy = _.regImp( __.appReg, '-copy' )

	import re
	import random
	import string
	def randomize(match):
		domains = ['domain.com', 'domain.net', 'domain.org', 'domain.quest', 'domain.xyz', 'domain.guru', 'domain.cx', 'domain.ac', 'domain.work', 'domain.app', 'domain.vip', 'example.com', 'example.net', 'example.org', 'example.quest', 'example.xyz', 'example.guru', 'example.cx', 'example.ac', 'example.work', 'example.app', 'example.vip', 'site.com', 'site.net', 'site.org', 'site.quest', 'site.xyz', 'site.guru', 'site.cx', 'site.ac', 'site.work', 'site.app', 'site.vip']
		firsts = ['Emma', 'Liam', 'Olivia', 'Ava', 'Isabella', 'Sophia', 'Mia', 'Charlotte', 'Amelia', 'Evelyn', 'Emily', 'Sofia', 'Avery', 'Ella', 'Grace', 'Madison', 'Aiden', 'Lucas', 'Ethan', 'Sebastian', 'Jack', 'Daniel', 'Samuel', 'Matthew', 'Michael', 'Andrew', 'Joshua', 'Ryan', 'Nathan', 'Adam', 'Robert', 'Nicholas', 'Anthony', 'John', 'Thomas', 'Charles', 'Zachary', 'Aaron', 'Jacob', 'Justin', 'Tyler', 'Austin', 'Jordan', 'Kyle']
		s = match.group(0)
		if re.match(r'[\'\"]\+?\d+[\'\"]', s):
			phone_number = s.strip('\'\"')
			area_code, rest = phone_number[:4], phone_number[4:]
			randomized_rest = ''.join(random.choice(string.digits) for _ in range(len(rest)))
			return f'"{area_code}{randomized_rest}"'
		elif re.match(r'[\'\"].+?@.+?[\'\"]', s):
			local, domain = s.strip('\'\"').split('@')
			domain = random.choice(domains)
			# local = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(len(local)))
			local = random.choice(firsts)

			return f'"{local}@{domain}"'
		else:
			randomized = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(len(s) - 2))
			return f'{s[0]}{randomized}{s[-1]}'

	pattern = r'[\'\"].+?[\'\"]'
	result = re.sub(pattern, randomize, text)
	_copy.imp.copy( result )
	return result

# sys.exit()

##################################################
# _var='''
# _v.stmp={home}/.rt/profile/temp
# _v.tempFile={home}/.rt/profile/temp/{8E3F33E4-86AB-AB1E-6219-801DE111D9AF}
# _v.terminal={home}/.rt/profile/vars/terminal/
# _v.terminal_variables={home}/.rt/profile/vars/terminal/16910
# _v.text_temp={home}/.rt/profile/temp/_temp.txt
# _v.thisHost={home}/.rt/profile
# _v.tmpbat={home}/.rt/profile/temp/44E28BDF-8269-EEAE-D1DC-9B05B63E5F93.bat
# _v.tmpf={home}/.rt/profile/temp/{8E3F33E4-86AB-AB1E-6219-801DE111D9AF}
# _v.tmpf0={home}/.rt/profile/temp/{B820137A-79B8-45E3-BCBD-A6CAC50892D0}
# _v.tmpf2={home}/.rt/profile/temp/{5FBF34C0-9A95-4C7E-BA53-44F84ECECCB5}
# _v.tmpf3={home}/.rt/profile/temp/{F139D191-FA1A-44D5-855C-7E5141B30E0D}
# _v.tmpf4={home}/.rt/profile/temp/{AA8EC8E1-EA9D-460D-A593-7B0FAEB9243E}
# _v.tmpf5={home}/.rt/profile/temp/{201D82D6-2DC0-4552-A598-54F5481399A1}
# _v.tmpf6={home}/.rt/profile/temp/{26B3B9C6-0A59-432A-9386-D432B53001CB}
# _v.tmpf7={home}/.rt/profile/temp/{C03C0132-CFFC-4E3A-8F0F-614BB95164C7}
# _v.tmpf8={home}/.rt/profile/temp/{4CCA3EBD-4535-42B7-9C75-05EFAACB00E0}
# _v.tmpf9={home}/.rt/profile/temp/{DF1D4EBC-838E-419C-9C58-943C1767391A}
# _v.tt={home}/.rt/profile/tables
# _v.tvy={home}/.rt/profile/vars/terminal/16910.yml
# _v.txt_temp={home}/.rt/profile/temp/_temp.txt
# _v.umlHtml={home}/.rt/profile/json-uml-tree/index.htm
# _v.umlJson={home}/.rt/profile/json-uml-tree/data.js
# _v.unixID_path={home}/.rt/profile/config/.unix_id
# _v.vault_path={home}/.rt/profile/config/.vault
# _v.wprofile={home}/.rt/profile

# _v.t={w}
# _v.ta={w}/techApps
# _v.tablesDB={w}/widgets/databank/tables
# _v.techDrive={w}
# _v.techFolder={w}
# _v.ttt={w}/widgets/databank/tables
# _v.updates={w}/widgets/project/updates
# _v.w={w}
# _v.webapp={w}/widgets/servers/web/crud
# _v.widgets={w}
# _v.ww={w}/widgets
# '''.strip().replace('\r','').replace('\n\n','\n').replace('/',os.sep).split('\n')

# _vTerm='''
# stmp={home}/.rt/profile/temp
# h={home}/.rt/profile
# pr={home}/.rt/profile/projects
# rt={home}/.rt
# tt={home}/.rt/profile/tables
# tmpf8={home}/.rt/profile/temp/{4CCA3EBD-4535-42B7-9C75-05EFAACB00E0}
# tmpf9={home}/.rt/profile/temp/{DF1D4EBC-838E-419C-9C58-943C1767391A}
# tmpf6={home}/.rt/profile/temp/{26B3B9C6-0A59-432A-9386-D432B53001CB}
# tmpf7={home}/.rt/profile/temp/{C03C0132-CFFC-4E3A-8F0F-614BB95164C7}
# tmpf4={home}/.rt/profile/temp/{AA8EC8E1-EA9D-460D-A593-7B0FAEB9243E}
# tmpf5={home}/.rt/profile/temp/{201D82D6-2DC0-4552-A598-54F5481399A1}
# tmpf2={home}/.rt/profile/temp/{5FBF34C0-9A95-4C7E-BA53-44F84ECECCB5}
# tmpf3={home}/.rt/profile/temp/{F139D191-FA1A-44D5-855C-7E5141B30E0D}
# tmpf0={home}/.rt/profile/temp/{B820137A-79B8-45E3-BCBD-A6CAC50892D0}
# tmpf1={home}/.rt/profile/temp/{C0FA8E56-8426-46BB-9CE8-4A14C51EA261}
# wprofile={home}/.rt/profile
# config={home}/.rt/profile/config

# ttt={w}/widgets/databank/tables
# db={w}/widgets/databank
# w={w}
# s={w}/widgets/batch
# bash={w}/widgets/bash
# ps={w}/widgets/powershell
# bat={w}/widgets/batch
# ww={w}/widgets
# widgets={w}
# js={w}/widgets/javascript
# '''.strip().replace('\r','').replace('\n\n','\n').replace('/',os.sep).split('\n')

# def variable():
#     global _var
#     global _vTerm
#     _paste = _.regImp( __.appReg, '-paste' )
#     data = _paste.imp.paste()
#     l=data.count('l')
#     w=data.count('\\')
#     if l > w: lw = '/'
#     else:     lw = '\\'
#     data = data.replace(lw,os.sep)
#     for line in data.split('\n'):
#         has1=[]
#         has2=[]
#         for li in _var:
#             if li in line: has1.append({'line':line,'cnt':len(li)},'li':li)
#         has1 = _.sort(has1,'cnt')
#         for li in _vTerm:
#             if li in line: has2.append({'line':line,'cnt':len(li)},'li':li)
#         has2 = _.sort(has2,'cnt')

#         for x in has1: print(x)
#         _.pr(line=1)
#         for x in has2: print(x)

##################################################
_.v.local=[
				'variable',
				'sally',
				'auto scrape',
				'ipsum',
				'lorem',
				'secure',
				'clean text',
		]

dic = {
			'variable': 'variable()',
			'sally': 'ai()',
			'auto scrape': 'auto_scrape()',
			'clean text': 'clean_text()',

			'secure': 'randomize_text()',

			'ipsum': 'ipsum()',
			'lorem': 'ipsum()',

			'center comment': 'Clip.center_to_top_comment()',

			'md5': 'Clip.md5()',
			'explode': 'Clip.explode()',
			'implode': 'Clip.implode()',
			'first': 'Clip.first()',
			'lower case': 'Clip.toLower()',
			'upper case': 'Clip.toUpper()',
			'randomize case': 'Clip.toRandomCase()',

			'scrape text': 'Clip.browser_f12_tooljs_text()',
			'extract text': 'Clip.browser_f12_tooljs_text()',

			'extract table': 'Clip.browser_f12_tooljs_table()',
			'scrape table': 'Clip.browser_f12_tooljs_table()',

			'markdown link': 'Clip.browser_f12_gen_md_link()',
			'markdown url': 'Clip.browser_f12_gen_md_link()',

			'scrape table no header': 'Clip.browser_f12_tooljs_table0()',
			'extract table no header': 'Clip.browser_f12_tooljs_table0()',

			'remove duplicate spaces': 'Clip.dup_space()',

			'convert': 'Clip.auto_yaml_json_converter()',
			# 'yaml to json': 'Clip.yaml2json()',
			# 'json to yaml': 'Clip.json2yaml()',

			'strip': 'Clip.strip1()',
			'clean up': 'Clip.strip2()',

			'space to underscore': 'Clip.space_2_underscore_text()',

			'reverse lines': 'Clip.reverse_lines()',

			'double space': 'Clip.space_double()',
			'single space': 'Clip.space_single()',

			'invert quotes': 'Clip.quote_inverter()',

			'dirty eval': 'Clip.dirty_eval()',

			'build app': 'Clip.SQL_to_crud()',

			'decrypt': 'Clip.decrypt_lines()',
			'encrypt lines': 'Clip.encrypt_lines()',
			'encrypt': 'Clip.encrypt_all()',

			'remove comments and spaces': 'Clip.remove_py_comments_spaces()',
			'remove comments': 'Clip.remove_py_comments()',

			'encode': 'Clip.base64_encode()',
			'decode': 'Clip.base64_decode()',

			'encrypt': 'Clip.encrypt_all()',
			'decrypt': 'Clip.decrypt_lines()',

			'math': 'Clip.math()',

}

_.v.finagling={
	'_replace': {
		'epsom': 'ipsum',
		'lauren': 'lorem',
		'floor m': 'lorem',
		'cobalt': 'kobold',
		'auto script': 'auto scrape',
	},
	'_startswith': {
		'auto scr': 'auto scrape',
		'autoscr': 'auto scrape',
		'andcode': 'encode',
	},
}

##################################################

# sally paste  convert the folling python to a single line

##################################################
from pynput.mouse import Button, Controller
from pynput.keyboard import Key
from pynput.keyboard import Controller as mController
mouse = Controller()
keyboard = mController()

##################################################

try: import speech_recognition as sr
except: pass

hkr = _.regImp( __.appReg, 'KeyWise' )
hk=hkr.imp

_paste = _.regImp( __.appReg, '-paste' )
_copy = _.regImp( __.appReg, '-copy' )

# ipsum()

##################################################
# import _rightThumb._vault as _vault
# import _rightThumb._encryptString as _blowfish
_keychain = _.regImp( __.appReg, 'keychain' )
interact=_.getTable('ai-bot-interaction.index')
if not 'success' in interact: interact = {'success':[],'failure':[],'chat':[]}
if not 'listen' in interact: interact['listen']=[]
########################################################################################
if __name__ == '__main__':
	#b)--> examples

	# banner.pr()
	# if len(_.switches.all())==0: banner.gossip()

	#e)--> examples
	action()
	_.isExit(__file__)

# https://www.imdb.com/chart/top/

# _.v.local


import psycopg2
import configparser
import os.path
from psycopg2.extras import Json
from utils.env_config import EnvConfig

def table_exists(con, table_str):

    exists = False
    try:
        cur = con.cursor()
        cur.execute("select exists(select relname from pg_class where relname='" + table_str + "')")
        exists = cur.fetchone()[0]
        if exists:
            columns = get_table_col_names(con, table_str)
            print("Table {} exists with columns: {}".format(table_str, columns))

        cur.close()
    except psycopg2.Error as e:
        print(e)
    return exists

def get_table_col_names(con, table_str):
    col_names = []
    try:
        cur = con.cursor()
        cur.execute("select * from " + table_str + " LIMIT 0")
        for desc in cur.description:
            col_names.append(desc[0])
        cur.close()
    except psycopg2.Error as e:
        print(e)

    return col_names

class GearMoodPS:
    def __init__(self):
        config = EnvConfig()
        try:
            self.conn = psycopg2.connect(
                "dbname='" + config.get_value("DatabaseSection","database.dbname") +
                "' user='" + config.get_value("DatabaseSection","database.user") +
                "' host='" + config.get_value("DatabaseSection","database.host") +
                "' password='" + config.get_value("DatabaseSection","database.password") +"'")
        except:
            print("Unexpected error:")
            raise
        table_exists(self.conn, "shake")
        table_exists(self.conn, "sentence")

    def save_shakedown(self, shakedown):
        cur = self.conn.cursor()
        cur.execute("INSERT INTO shake (submission_id, subreddit_id, title, data) " +
                    "VALUES (%s, %s, %s, %s) " +
                    "ON Conflict (submission_id) DO UPDATE SET data = %s",
                    (shakedown["id"], shakedown["subreddit_id"],
                    shakedown["title"], Json(shakedown), Json(shakedown)))
        #print("Shakedown {} : {}".format(shakedown["title"], "https://www.reddit.com/" + shakedown["id"]))
        self.conn.commit()
        cur.close()

    def get_shakedown(self, submission_id):
        cur = self.conn.cursor()
        sql = "SELECT * FROM shake where submission_id = %s;"
        data = (submission_id, )
        cur.execute(sql,data)
        results = cur.fetchone()
        cur.close()
        return results

    def get_all_shakedowns(self):
        cur = self.conn.cursor()
        cur.execut("SELECT * FROM test;")
        result = cur.fetchall()
        print("result: {}".format(self))
        cur.close()
        return result

    def save_sentence(self, submission_id, cut_sentences):
        cur = self.conn.cursor()
        for cut_sentence in cut_sentences:
            cur.execute("INSERT INTO cut_sentence (submission_id, words, sentence) " +
                        "VALUES (%s, %s, %s)",
                        #"ON Conflict (submission_id) DO UPDATE SET words = %s, sentence = %s",
                        (submission_id, Json(cut_sentence["words"]), cut_sentence["sentence"]))
        self.conn.commit()
        cur.close()

if __name__ == '__main__':
    GearMoodPS()


import argparse

import numpy as np
import matplotlib
import matplotlib.pyplot as plt

import util.io
import util.plot

from plot_results_lprob import get_sim, ed, get_samples_snl
from plot_results_mmd import get_true_samples

def make_plots(samples, sim, sim_name):
    """
    Makes all plots.
    """

    n_dim = samples.shape[1]
    n_bins = int(np.sqrt(samples.shape[0]))
    lims = np.asarray(sim.get_disp_lims())
    lims = np.tile(lims, [n_dim, 1]) if lims.ndim == 1 else lims
    true_ps, _ = sim.get_ground_truth()

    if sim_name == 'hh':
        ii = sim.Model().ps_order_in_plot
        true_ps = true_ps[ii]
        samples = samples[:, ii]
        lims = lims[ii]

    for i in xrange(n_dim):
        for j in xrange(i, n_dim):

            if i == j:

                fig, ax = plt.subplots(1, 1)
                ax.hist(samples[:, i], bins=n_bins, normed=True, rasterized=True)
                ax.set_xlim(lims[i])
                ax.set_ylim([0.0, ax.get_ylim()[1]])
                ax.tick_params(axis='x', which='both', top=False, bottom=False, labelbottom=False)
                ax.tick_params(axis='y', which='both', left=False, right=False, labelleft=False)
                ax.vlines(true_ps[i], 0, ax.get_ylim()[1], color='r')

            else:

                fig, ax = plt.subplots(1, 1)
                ax.scatter(samples[:, j], samples[:, i], c='k', s=3, marker='o', cmap='binary', edgecolors='none', rasterized=True)
                ax.tick_params(axis='x', which='both', top=False, bottom=False, labelbottom=False)
                ax.tick_params(axis='y', which='both', left=False, right=False, labelleft=False)
                ax.set_xlim(lims[j])
                ax.set_ylim(lims[i])
                ax.scatter(true_ps[j], true_ps[i], c='r', s=12, marker='o', edgecolors='none')

def plot_results(sim_name):
    """
    Plots posterior for a given simulator.
    """

    sim = get_sim(sim_name)

    matplotlib.rc('text', usetex=True)
    matplotlib.rc('font', size=16)

    for exp_desc in ed.parse(util.io.load_txt('exps/{0}_seq.txt'.format(sim_name))):

        if isinstance(exp_desc.inf, ed.SNL_Descriptor):

            samples = get_samples_snl(exp_desc, sim)[-1]
            make_plots(samples, sim, sim_name)

    if sim_name == 'gauss':

        samples = get_true_samples()
        make_plots(samples, sim, sim_name)

    plt.show()

def main():

    parser = argparse.ArgumentParser(description='Plotting the posteriors for SNL.')
    parser.add_argument('sim', type=str, choices=['gauss', 'mg1', 'lv', 'hh'], help='simulator')
    args = parser.parse_args()

    plot_results(args.sim)

if __name__ == '__main__':
    main()


import os
import random

import cv2
import numpy as np

root_dir = "../data/kaist/images/"

def calc_stats():
    rgb_images = []
    ir_images = []
    mean = []
    std = []
    set_list = os.listdir(root_dir)
    for s_path in [os.path.join(root_dir, s) for s in set_list]:
        vid_list = os.listdir(s_path)
        for v_path in [os.path.join(s_path, b) for b in vid_list]:
            img_list = os.listdir(os.path.join(v_path, "visible"))
            for img_name in img_list:
                if random.uniform(0, 1) > 0.95:
                    rgb_img = cv2.imread(os.path.join(v_path, "visible", img_name))
                    ir_img = cv2.imread(os.path.join(v_path, "lwir", img_name), cv2.IMREAD_GRAYSCALE)
                    rgb_images.append(rgb_img)
                    ir_images.append(ir_img)
    print(len(rgb_images))

    # calc mean
    total_red = 0
    total_green = 0
    total_blue = 0
    total_intensity = 0
    for index in range(len(rgb_images)):
        img = rgb_images[index]
        ir_img = ir_images[index]

        for i in range(img.shape[0]):
            for j in range(img.shape[1]):
                pixel = img[i, j]
                ir_pixel = ir_img[i, j]

                red = pixel[0]
                total_red += red

                green = pixel[1]
                total_green += green

                blue = pixel[2]
                total_blue += blue

                intensity = ir_pixel
                total_intensity += intensity
    denominator = len(rgb_images) * 511 * 511 * 255

    mean.append(total_red / denominator)
    mean.append(total_green / denominator)
    mean.append(total_blue / denominator)
    mean.append(total_intensity / denominator)
    print("Mean: ", mean)

    std_sum_red = 0
    std_sum_green = 0
    std_sum_blue = 0
    std_sum_ir = 0
    for index in range(len(rgb_images)):
        img = rgb_images[index]
        ir_img = ir_images[index]

        for i in range(img.shape[0]):
            for j in range(img.shape[1]):
                pixel = img[i, j]
                ir_pixel = ir_img[i, j]

                red = pixel[0] / 255
                std_sum_red += (mean[0] - red) ** 2

                green = pixel[1] / 255
                std_sum_green += (mean[1] - green) ** 2

                blue = pixel[2] / 255
                std_sum_blue += (mean[2] - blue) ** 2

                intensity = ir_pixel / 255
                std_sum_ir += (mean[3] - intensity) ** 2
    std_denom = len(rgb_images) * 511 * 511
    std.append(np.sqrt(std_sum_red / std_denom))
    std.append(np.sqrt(std_sum_green / std_denom))
    std.append(np.sqrt(std_sum_blue / std_denom))
    std.append(np.sqrt(std_sum_ir / std_denom))
    print("Std: ", std)

    all_lst = []
    for i in range(len(rgb_images)):
        res = np.reshape(rgb_images[i], (3, 511, 511))
        added = np.append(res, ir_images[i])
        added = np.reshape(added, (4, 511, 511))
        all_lst.append(added)

    # Must has (N x 4 x 511 x 511) ndarray
    print(added)

if __name__ == "__main__":
    calc_stats()


import os
import json
import time
from kafka import KafkaConsumer, KafkaProducer

KAFKA_BROKER_URL = os.environ.get('KAFKA_BROKER_URL')
FROM_TOPIC = os.environ.get('FROM_TOPIC')
TO_TOPIC = os.environ.get("TO_TOPIC")
WINDOW_SIZE = os.environ.get("WINDOW_SIZE")
THROTTLE_LIMIT = os.environ.get("THROTTLE_LIMIT_PER_MIN")
METRIC_TOPIC = os.environ.get("METRIC_TOPIC")
METRIC_CYCLE = os.environ.get("METRIC_CYCLE")

sliding_window = {}
blacklist = []

def put_sliding_window(ip_ts: dict):
    """
    Takes a dict of form {"IP": str, "TS": int}
    Puts into a sliding window dict of lists that counts
    the number of timestamps in a certain period. If that
    count exceeds the threshold, the IP address is "blacklisted"
    and added to the blacklist list and the IP is posted to a
    kafka topic called logs.blacklist

    :param ip_ts: dict of form {"IP": str, "TS": ts}
    :return: None if not new blacklisted IP or dict of form
        {
            "IP": str,
            "Req_Per_Min_Exceeded": int
        }
        if the IP address is newly blacklisted
    """
    ip = ip_ts["IP"]
    ts = ip_ts["TS"]

    # If IP is already blacklisted, ignore it
    if ip in blacklist:
        return None

    # IF IP is new, add it to sliding window keys
    if ip not in sliding_window.keys():
        sliding_window[ip] = {'ts': [ts],
                              'count': 1}

    # Otherwise, add timestamp to sliding window
    # And slide the window as necessary, keeping
    # track of count
    else:
        while sliding_window[ip]['ts'] != [] \
                and ts - (int(WINDOW_SIZE)*60) > sliding_window[ip]['ts'][0]:
            del sliding_window[ip]['ts'][0]
            sliding_window[ip]['count'] -= 1
        sliding_window[ip]['ts'].append(ts)
        sliding_window[ip]['count'] += 1

        # If the count exceeds the THROTTLE_LIMIT,
        # return the IP address to be sent to logs.blacklist
        if sliding_window[ip]['count'] \
                >= int(THROTTLE_LIMIT) * int(WINDOW_SIZE):
            blacklist.append(ip)
            return {
                "IP": ip,
                "Req_Per_Min_Exceeded": int(THROTTLE_LIMIT)
            }

    return None

if __name__ == '__main__':
    consumer = KafkaConsumer(
        FROM_TOPIC,
        bootstrap_servers=KAFKA_BROKER_URL,
        value_deserializer=lambda value: json.loads(value),
        consumer_timeout_ms=5000
    )
    producer = KafkaProducer(
        bootstrap_servers=KAFKA_BROKER_URL,
        value_serializer=lambda value: json.dumps(value).encode(),
    )

    # Metric initialization
    count = 0
    blacklisted = 0
    start = time.time()

    for message in consumer:

        # Metric Increments
        count += 1

        # Transform and Publish Data
        transaction: dict = put_sliding_window(message.value)
        if transaction:
            blacklisted += 1
            producer.send(TO_TOPIC, value=transaction)

        # Metric Aggregation
        if count >= int(METRIC_CYCLE):
            metrics = {
                "processed_per_second": count /(time.time() - start),
                "records_processed": count,
                "blacklisted_ips": blacklisted,
                "end_ts": time.time()
            }
            producer.send(METRIC_TOPIC, value=metrics)
            count = 0
            blacklisted = 0
            start = time.time()


#!/usr/bin/env python
# coding: utf-8

# In[1]:

# %pip install pandas
# %pip install gingerit
# %pip install regex
# %pip install openpyxl
#pip install Flask

import pandas as pd
import regex as re
from gingerit.gingerit import GingerIt

def remove_emoticons_hashtags_tags(text):
    # Remove emoticons
    text = re.sub(r'[\U0001F600-\U0001F64F\U0001F300-\U0001F5FF\U0001F680-\U0001F6FF\U0001F700-\U0001F77F\U0001F800-\U0001F8FF\U0001F900-\U0001F9FF\U0001FA00-\U0001FA6F\U0001FA70-\U0001FAFF\U00002702-\U000027B0\U000024C2-\U0001F251]+', '', text)

    # Remove hashtags and @tags
#     text = re.sub(r'[@#]\w+', '', text)
# The pattern [#@]\S+\s will find words that start with @ or # and continue until a space is found in the given text.
#  It will capture the entire word along with any special characters until the space is reached.
    text +=" "
    text = re.sub(r'[#@]\S+\s', '', text)

    # Remove anything inside parentheses
    text = re.sub(r'\([^)]*\)', '', text)

    # Remove any trailing punctuations
    text = text.replace("@","").replace("#","")

    return text.strip()

def check_correction_type(correction):
    excluded_correction_types = ['Accept space','Accept comma addition','None']
    for correction in ginger_cap['corrections']:
        print("\t ------------------",correction['definition'])
        if correction['definition'] in excluded_correction_types:
            return False
    return True

def fix_my_cap(text):
    parser = GingerIt()
    return parser.parse(remove_emoticons_hashtags_tags(text))

if __name__ == "__main__":
    text= """Gamers or hustlers. Corporate warriors or students. Whatever be their trip, what trips them up is a laptop that just cannot keep pace. #NewWork for @reliance_digital as part of “The Hunt for India’s Slowest Laptop” campaign by Team 21N.

Credits:
Brand: @reliance_digital
Client: Manoj Jain, Nisha Kallianpur, Anamika Tiwari, Esha Nath
Production House: @firsttakefilms
Direction: @sagnik6285
Creative Team: @bingeljell, @inde.stagram, @_kashishkakkar_
Account Management: @neerajkarkera , @divya_jain_dj, @aarsha_rathi
Founder & CEO: @the.malayali"""

    result = fix_my_cap(text)
    print(result)


import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn import metrics
import joblib
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report, accuracy_score, f1_score, precision_score, recall_score, confusion_matrix
import seaborn as sns
import matplotlib.pyplot as plt
from models import Models
from sklearn.feature_selection import SequentialFeatureSelector as SFS
import matplotlib.pyplot as plt
from sklearn.neighbors import KNeighborsClassifier
from sklearn.datasets import load_iris

pathToData = 'dataAfterProcessingCSV/'
pathToPictures = 'pictures/'
pathToModels = 'models/'
pathToRetrainModel = 'retrainModels/'
pathToRetrainPictures = 'retrainPictures/'
models = Models().models

def print_model(modelResultPath):
    modelResult = joblib.load(pathToModels + modelResultPath)
    print(modelResult)

def show_confusion_matrix(cm_test, title):
    print('Print confusion matrix for \"' + str(title) + '\"')
    plt.figure(figsize=(8,8))
    sns.heatmap(cm_test, annot=True, fmt="d", cmap="Blues")
    sns.set(font_scale=1.9)
    plt.title(title)
    plt.ylabel("Actual ")
    plt.xlabel("Predicted ")
    plt.tight_layout()
    plt.savefig(pathToPictures + title + ".png")
    #plt.show()
    plt.close()

def get_confusion_matrix(X_test, y_test, classifier):
    y_pred = classifier.predict(X_test)
    # print(y_pred)
    matrix = confusion_matrix(y_test, y_pred)
    # print(matrix)
    print("Accuracy Score:", accuracy_score(y_test, y_pred))
    print("Precision Score : ", precision_score(y_test, y_pred))
    print("Recall Score:", recall_score(y_test, y_pred))
    print("f1 Score:", f1_score(y_test, y_pred))
    return matrix

def split_data(pandas_data):
    X = pandas_data.loc[:, 'statuses_count':'description_length']
    y = pandas_data['is_bot']
    return X, y

def read_data():
    bots = pd.read_csv(pathToData + 'botsAfterProcessing.csv')
    real = pd.read_csv(pathToData + 'humansAfterProcessing.csv')

    data = pd.concat((bots, real), axis=0)
    data.to_excel('bots_and_humans.xls')
    data = data.sample(frac=1)
    data.to_excel('bots_and_humans_mixed.xls')
    return data

def build_and_evaluate_models(data):
    X, y = split_data(data)

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
    numberOfAllAccounts = np.array(y_train).size
    numberOfBots = np.count_nonzero(np.array(y_train))
    numberOfHumans = numberOfAllAccounts - numberOfBots
    print('Number of bots in train data set: ' + str(numberOfBots))
    print('Number of humans in train data set: ' + str(numberOfHumans))
    print('Bots: {0}%, humans: {1}%'.format(numberOfBots/numberOfAllAccounts*100, numberOfHumans/numberOfAllAccounts*100))
    print('')

    bestScore = 0
    bestAlgName = ''

    for name, (model, parameters) in models.items():
        print('Results for \"' + str(name) + '\"')
        gs = GridSearchCV(model, parameters, cv=10, verbose=1, n_jobs=-1, scoring='roc_auc')
        gs.fit(X_train, y_train)
        print("Best Parameters:", gs.best_params_)
        print("")
        print("Best Score:", gs.best_score_)

        joblib.dump(gs.best_estimator_, pathToModels + f"{name}.pkl", compress=1)
        joblib.dump(gs.cv_results_, pathToModels + f"{name}_results.pkl", compress=1)

        y_pred = gs.predict(X_test)
        f1 = f1_score(y_test, y_pred)
        if bestScore < f1:
            bestScore = f1
            bestAlgName = name

        print("")
        show_confusion_matrix(get_confusion_matrix(X_test, y_test, gs), name)
        print('------------------------------------')

    print('"' + bestAlgName + '" ' + 'showed best results on test dataset')
    print('f1 score: ' + str(bestScore))
    return bestAlgName

def print_bots_and_humans_count(y):
    numberOfAllAccounts = np.array(y).size
    numberOfBots = np.count_nonzero(np.array(y))
    numberOfHumans = numberOfAllAccounts - numberOfBots
    print('Number of bots in train data set: ' + str(numberOfBots))
    print('Number of humans in train data set: ' + str(numberOfHumans))
    print('------------------------------------')

def test_retrain_model_on_unseen_data(model_name, list_to_drop):
    political_bots = pd.read_csv(pathToData + 'politicalBotsAfterProcessing.csv')
    real = pd.read_csv(pathToData + 'celebritiesAfterProcessing.csv')
    pron_bots = pd.read_csv(pathToData + 'pronBotsAfterProcessing.csv')
    vendor_bots = pd.read_csv(pathToData + 'vendorBotsAfterProcessing.csv')

    real_and_political_bots = pd.concat((political_bots, real), axis=0).sample(frac=1)
    real_and_pron_bots = pd.concat((pron_bots, real), axis=0).sample(frac=1)
    real_and_vendor_bots = pd.concat((vendor_bots, real), axis=0).sample(frac=1)

    best_model = joblib.load(pathToRetrainModel + model_name + '.pkl')

    X1, y1 = split_data(real_and_political_bots)
    X1.drop(X1.columns[list_to_drop], axis=1, inplace=True)
    show_confusion_matrix(get_confusion_matrix(X1, y1, best_model), model_name + 'CelebritiesAndPoliticalBots')
    print_bots_and_humans_count(y1)

    X2, y2 = split_data(real_and_pron_bots)
    X2.drop(X2.columns[list_to_drop], axis=1, inplace=True)
    show_confusion_matrix(get_confusion_matrix(X2, y2, best_model), model_name + 'CelebritiesAndPronBots')
    print_bots_and_humans_count(y2)

    X3, y3 = split_data(real_and_vendor_bots)
    X3.drop(X3.columns[list_to_drop], axis=1, inplace=True)
    show_confusion_matrix(get_confusion_matrix(X3, y3, best_model), model_name + 'CelebritiesAndVendorBots')
    print_bots_and_humans_count(y3)

def test_model_on_unseen_data(model_name):
    political_bots = pd.read_csv(pathToData + 'politicalBotsAfterProcessing.csv')
    real = pd.read_csv(pathToData + 'celebritiesAfterProcessing.csv')
    pron_bots = pd.read_csv(pathToData + 'pronBotsAfterProcessing.csv')
    vendor_bots = pd.read_csv(pathToData + 'vendorBotsAfterProcessing.csv')

    #unseen_data = pd.concat((political_bots, real, pron_bots, vendor_bots), axis=0).sample(frac=1)
    #unseen_data.to_excel('unseen_data.xls')

    real_and_political_bots = pd.concat((political_bots, real), axis=0).sample(frac=1)
    real_and_pron_bots = pd.concat((pron_bots, real), axis=0).sample(frac=1)
    real_and_vendor_bots = pd.concat((vendor_bots, real), axis=0).sample(frac=1)

    best_model = joblib.load(pathToModels + model_name + '.pkl')

    X1, y1 = split_data(real_and_political_bots)
    show_confusion_matrix(get_confusion_matrix(X1, y1, best_model), model_name + 'CelebritiesAndPoliticalBots')
    print_bots_and_humans_count(y1)

    X2, y2 = split_data(real_and_pron_bots)
    show_confusion_matrix(get_confusion_matrix(X2, y2, best_model), model_name + 'CelebritiesAndPronBots')
    print_bots_and_humans_count(y2)

    X3, y3 = split_data(real_and_vendor_bots)
    show_confusion_matrix(get_confusion_matrix(X3, y3, best_model), model_name + 'CelebritiesAndVendorBots')
    print_bots_and_humans_count(y3)

def select_features(model_name):
    data = read_data()
    X, y = split_data(data)
    model = joblib.load(pathToModels + model_name + '.pkl')

    all_features = X.columns.values
    print('Исходный набор:' + str(all_features))
    print('--------------------------------')

    print('First')
    sfs1 = SFS(model, n_features_to_select=16, direction='backward', scoring='f1', cv=10)
    result1 = sfs1.fit(X, y)
    part_of_features1 = all_features[result1.get_support(indices=True)]
    print(result1.get_support())
    print(result1.get_support(indices=True))
    print(str(part_of_features1))
    print('--------------------------------')

    print('Second')
    sfs2 = SFS(model, n_features_to_select=16, direction='forward', scoring='f1', cv=10)
    result2 = sfs2.fit(X, y)
    part_of_features2 = all_features[result2.get_support(indices=True)]
    print(result2.get_support())
    print(result2.get_support(indices=True))
    print(str(part_of_features2))
    print('--------------------------------')

    print('Third')
    sfs3 = SFS(model, direction='forward', scoring='f1', cv=10)
    result3 = sfs3.fit(X, y)
    part_of_features3 = all_features[result3.get_support(indices=True)]
    print(result3.get_support())
    print(result3.get_support(indices=True))
    print(str(part_of_features3))
    print('--------------------------------')

    print('Fourth')
    sfs4 = SFS(model, direction='backward', scoring='f1', cv=10)
    result4 = sfs4.fit(X, y)
    part_of_features4 = all_features[result4.get_support(indices=True)]
    print(result4.get_support())
    print(result4.get_support(indices=True))
    print(str(part_of_features4))

def retrain_model(data, model_name, lists_to_drop):
    cnt = 1
    name = model_name

    for l_to_drop in lists_to_drop:
        bestScore = 0
        bestAlgName = ''

        X, y = split_data(data)
        X.drop(X.columns[l_to_drop], axis=1, inplace=True)

        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

        (model, parameters) = models[model_name]
        name = model_name + str(cnt)
        print('Results for \"' + str(name) + '\"')
        gs = GridSearchCV(model, parameters, cv=10, verbose=1, n_jobs=-1, scoring='roc_auc')
        gs.fit(X_train, y_train)
        print("Best Parameters:", gs.best_params_)
        print("")
        print("Best Score:", gs.best_score_)

        joblib.dump(gs.best_estimator_, pathToRetrainModel + f"{name}.pkl", compress=1)
        joblib.dump(gs.cv_results_, pathToRetrainModel + f"{name}_results.pkl", compress=1)

        y_pred = gs.predict(X_test)
        f1 = f1_score(y_test, y_pred)
        if bestScore < f1:
            bestScore = f1
            bestAlgName = name

        print("")
        show_confusion_matrix(get_confusion_matrix(X_test, y_test, gs), name)
        print('------------------------------------')

        cnt += 1

    print('"' + bestAlgName + '" ' + 'showed best results on retrain datasets')
    print('f1 score: ' + str(bestScore))

if __name__ == '__main__':
    first_list = [0, 5, 10, 12]
    second_list = [12, 14, 15, 19]
    third_list = [0, 2, 3, 4, 8, 10, 14, 15, 17, 19]
    fourth_list = [1, 2, 5, 6, 9, 11, 12, 15, 16, 18]
    parse_list = [3, 4, 12, 13]
    list = [first_list, second_list, third_list, fourth_list]

    data = read_data()
    #retrain_model(data, "RandomForest", list)
    #build_and_evaluate_models(data)
    #for model_name in models.keys():
    #    test_model_on_unseen_data(model_name)
    #select_features('RandomForest')

    # for i in range(1, 5):
    #     name = "RandomForest" + str(i)
    #     test_retrain_model_on_unseen_data(name, list[i - 1])

    retrain_model(data, "RandomForest", [parse_list])
    test_retrain_model_on_unseen_data("RandomForest1", parse_list)


import re
from string import Template
from typing import List

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

from roveranalyzer.tempaltes import read_tmpl_str

# fixme: are they different? (sca, vec)
_scave_cols = ["run", "type", "module", "name", "attrname", "attrvalue"]

def is_scave_df(obj: pd.DataFrame):
    return all([c in obj.columns for c in _scave_cols])

class Opp:
    """
    Handle OMNeT++ module strings and transformations
    """

    @staticmethod
    def root_mod(module, level: int):
        if not type(module) == str:
            return "notAModule"

        module_elements = module.split(".")
        if len(module_elements) >= level:
            return ".".join(module_elements[:level])
        else:
            return ""

    @staticmethod
    def subframe_match(subframe, df_match):
        run = subframe["run"].unique()[0]
        return subframe.loc[
            subframe["root_module"].isin(
                df_match.loc[df_match["run"] == run, "root_module"]
            )
        ]

    @staticmethod
    def filter_by(df, module, name, value, apply_to_module_level=1):
        """
        Each  statistic is uniquely identified by its run, module and name values.
        a module consists of point separated 'sub-modules'. eg.
        run1#mod1.mod2.mod3:name1 = 42 defines a statistic with with name 'name1' and the
        value '42'. The statistic is generated by the module mod3 which is a submodule of mod3 etc.

        This method allows to filter out modules starting with some submodule mod1 based on the
        value of a statistic hold by the submodule mod3.

        Example: select all statistics of all host which have an active app.
        run1#network.host1.app.applicationActive = 1.0
        run1#network.host1.mac.pktCount = 33.0
        run1#network.host2.app.applicationActive = 0.0
        run1#network.host2.mac.pktCount = 33.0
        run1#network.host2.ip.pktLoss = 0.03
        Result:
        run1#network.host1.app.applicationActive = 1.0
        run1#network.host1.mac.pktCount = 33.0

        module_level:
        run1#mod1.mod2.mod3.mod4:name1
          level0 ^
          level1      ^
          level2           ^
        """
        mod = df.loc[
            (df["name"] == name) & (df["value"] == value), ("run", "module")
        ].copy()
        mod["root_module"] = df.run.apply(
            lambda x: Opp.root_mod(x, apply_to_module_level)
        )
        dfg = df.copy()
        dfg["root_module"] = dfg["module"].apply(
            lambda x: Opp.root_mod(x, apply_to_module_level)
        )
        dfg = dfg.groupby("run")
        dfg = dfg.filter()
        return dfg

    @staticmethod
    def module_path(m_path, index, tuple_on_vector=False):
        """

        :param m_path: some string representing a module name. Each path element is separated by dots
        :param index: 0-Based index of path element to return
        :param tuple_on_vector: If true a vector path element is returned as a tuple. Otherwise as a string
        :return: string of module path element or
                 tuple if tuple_on_vector is true an selected path element is a vector or
                 full path as list of tuples of size 1 or 2.
        """
        if m_path is np.nan:
            return np.nan
        items = m_path.split(".")
        if index > len(items):
            raise IndexError(f"m_path has no index {index}. Path was {m_path}")
        match = re.compile("(?P<base>.*?)(?P<vector>\[\d+\])")
        path_list = []
        for i in items:
            m = match.findall(i)
            if m:
                path_list.append((m[0][0], m[0][1]))
            else:
                path_list.append((i,))
        if index == -1:
            return path_list

        ret = path_list[index]
        if tuple_on_vector:
            if len(ret) == 2:
                return ret
            else:
                return f"{ret[0]}{ret[1]}"
        else:
            if len(ret) == 2:
                return f"{ret[0]}{ret[1]}"
            else:
                return ret[0]

    @staticmethod
    def normalize_vectors(df_vec, axis=0):
        """
        def_vec data frame of opp vector values only. This will transform the values into normalized
        column vectors  ['time', 'value']
        """
        frames = []
        time_name = "time"
        value_name = "value"
        for idx in df_vec.index:
            if axis == 1:
                mod_name = f"{Opp.module_path(df_vec.loc[idx]['module'], index=1)}"
                stat_name = df_vec.loc[idx]["name"].split(":")[0]
                time_name = f"{mod_name}.{stat_name}.time"
                value_name = f"{mod_name}.{stat_name}.value"

            data = df_vec.loc[idx]["vectime"].copy()
            data = np.append(data, df_vec.loc[idx]["vecvalue"].copy())
            data = data.reshape((-1, 2), order="F")
            tmp_d = pd.DataFrame(data, columns=[time_name, value_name])
            tmp_d.reset_index(drop=True, inplace=True)
            frames.append(tmp_d)

        df_n = pd.concat(frames, axis=axis, ignore_index=True if axis == 0 else False)

        return df_n

class OppTex:
    def __init__(self, opp=None):
        self._opp = opp

    @staticmethod
    def esc_tex(val: str):
        val = val.replace("&", "\&")
        val = val.replace("%", "\%")
        val = val.replace("$", "\$")
        val = val.replace("#", "\#")
        val = val.replace("_", "\_")
        val = val.replace("{", "\{")
        val = val.replace("}", "\}")
        val = val.replace("~", "\~")
        val = val.replace("^", "\^")
        val = val.replace("\\", "\\")

        return val

    @classmethod
    def write_module_summary(cls, run_id, module_dict: dict):

        module_list = []
        tex_mod_item = read_tmpl_str("module_item.tex")
        for _, v in module_dict.items():
            module_list.append(
                Template(tex_mod_item,).substitute(
                    module=cls.esc_tex(str(v["module"])),
                    scalar=cls.esc_tex(str(v["scalar"])),
                    vector=cls.esc_tex(str(v["vector"])),
                    histogram=cls.esc_tex(str(v["histogram"])),
                )
            )

        tex_mod = read_tmpl_str("module.tex")
        return Template(tex_mod).substitute(
            run=cls.esc_tex(run_id), module_items="".join(module_list)
        )

    @classmethod
    def write_attribute_tabular(cls, run_id, runattr_dict, itervars_dict, param_dict):
        tex_tmpl = read_tmpl_str("tabular.tex")

        runattrs = "   \\\\ \n".join(
            [
                f"\t\t{cls.esc_tex(k)} & {cls.esc_tex(str(v))}"
                for (k, v) in runattr_dict.items()
            ]
        )

        itervars = "   \\\\ \n".join(
            [
                f"\t\t{cls.esc_tex(k)} & {cls.esc_tex(str(v))}"
                for (k, v) in itervars_dict.items()
            ]
        )

        params = "   \\\\ \n".join(
            [
                f"\t\t{cls.esc_tex(k)} & {cls.esc_tex(str(v))}"
                for (k, v) in param_dict.items()
            ]
        )

        return Template(tex_tmpl).substitute(
            runattrs=runattrs,
            itervars=itervars,
            params=params,
            run=cls.esc_tex(run_id),
        )

    def create_module_summary(self, run_id, output_file=None):
        """
        Create latex table for module summary.
        :param run_id:      run_id for which tex is created
        :param output_file: if not set use to stdout
        """

        modType = np.array(self._opp._obj[["module", "type"]])
        module_dict = {}
        for m in modType:
            data = module_dict.get(
                m[0], {"module": m[0], "scalar": 0, "vector": 0, "histogram": 0}
            )
            data[m[1]] = data.get(m[1], 0) + 1
            module_dict.setdefault(m[0], data)

        tmpl = self.write_module_summary(run_id=run_id, module_dict=module_dict)

        if output_file is None:
            print(tmpl)
        else:
            with open(output_file, "w") as f:
                f.write(tmpl)

    def create_attribute_tabular(self, run_id, output_file=None):
        """
        Create latex table for attribute data.
        :param run_id:      run_id for which tex is created
        :param output_file: if not set use to stdout
        """

        tmpl = self.write_attribute_tabular(
            run_id=run_id,
            runattr_dict=self._opp.attr.run_attr_dict(run_id),
            itervars_dict=self._opp.attr.run_itervar_dict(run_id),
            param_dict=self._opp.attr.run_parameter_dict(run_id),
        )

        if output_file is None:
            print(tmpl)
        else:
            with open(output_file, "w") as f:
                f.write(tmpl)

class OppFilterItem:
    """
    Filter item applicable to OMNeT++ based data frame. #name corresponds to column of df.
    """

    @classmethod
    def list_filter(cls, name, value):
        ret = cls(name, value, regex=False)
        ret.is_list = True
        return ret

    def __init__(self, name, value, regex):
        self.name = name
        self.value = value
        self.regex = regex
        self.is_list = False

    def __str__(self):
        return self.value

    def __repr__(self) -> str:
        return (
            f"FilterItem(name: {self.name}, value: {self.value}, regex: {self.regex})"
        )

class OppFilter:
    """
    Builder pattern for OMNeT based data frames. This allows simplified selection of
    rows of the dataframe. Most columns of the data frame has two methods.
    <column_name> and <column_name>_regex. The former accepts only perfect matches
    whereas the later accepts a regex matter. If allow_number_range=True of the
    regex method is True number ranges such as '.*foo\[3..5\]\.bar\.baz' will be accepted
    and replaced with '.*foo\[\[3|4|5]]\.bar\.baz'.
    Note: All special characters must be escaped if literal brackets or dots should be matched.
    """

    def __init__(self, df: pd.DataFrame = None, data_only=True):
        self._filter_dict = {}
        self._name = None
        self._name_regex = None
        self._run = None
        self._run_regex = None
        self._type = None
        self._module = None
        self._module_regex = None
        self._data_only = data_only
        self._number_range_pattern = re.compile(
            "(?P<n_range>(?P<start>\d+)\.\.(?P<end>\d+))"
        )
        self._df = df
        if data_only:
            self._add_filter(
                OppFilterItem("type", "(scalar|vector|histogram)", regex=True)
            )

    def _apply_number_range(self, val):
        matches = re.findall(self._number_range_pattern, val)
        for match in matches:
            old_range = match[0]
            start = int(match[1])
            end = int(match[2])
            if start < end:
                range_new = (
                    "[" + "|".join([str(i) for i in range(start, end + 1)]) + "]"
                )
                val = val.replace(old_range, range_new)
            else:
                raise ValueError(f"number range in regex {val} invalid")
        return val

    def _add_filter(self, f: OppFilterItem):
        self._filter_dict[f.name] = f

    def name_in(self, name_list):
        self._add_filter(OppFilterItem.list_filter("name", name_list))
        return self

    def name(self, name):
        self._add_filter(OppFilterItem("name", name, False))
        return self

    def name_regex(self, name: str, allow_number_range=True):
        if allow_number_range:
            name = self._apply_number_range(name)

        self._add_filter(OppFilterItem("name", name, True))
        return self

    def run(self, run):
        self._add_filter(OppFilterItem("run", run, False))
        return self

    def run_regex(self, run: str, allow_number_range=True):
        if allow_number_range:
            run = self._apply_number_range(run)

        self._add_filter(OppFilterItem("run", run, True))
        return self

    def module(self, module):
        self._add_filter(OppFilterItem("module", module, False))
        return self

    def module_in(self, module_list):
        self._add_filter(OppFilterItem.list_filter("module", module_list))
        return self

    def module_regex(self, module: str, allow_number_range=True):
        if allow_number_range:
            module = self._apply_number_range(module)

        self._add_filter(OppFilterItem("module", module, True))
        return self

    def scalar(self):
        self._add_filter(OppFilterItem("type", "scalar", False))
        return self

    def vector(self):
        self._add_filter(OppFilterItem("type", "vector", False))
        return self

    def apply(self, df: pd.DataFrame = None, columns=None, copy=False):

        if df is not None:
            self._df = df

        if self._df is None:
            raise ValueError("no Dataframe set")

        bool_filter = pd.Series(
            [True for _ in range(0, self._df.shape[0])], self._df.index
        )
        for key, filter_item in self._filter_dict.items():
            if filter_item.regex:
                bool_filter = bool_filter & self._df.loc[:, key].str.match(
                    filter_item.value
                )
            elif filter_item.is_list:
                bool_filter = bool_filter & (
                    self._df.loc[:, key].isin(filter_item.value)
                )
            else:
                bool_filter = bool_filter & (self._df.loc[:, key] == filter_item.value)
        if columns is not None:
            ret = self._df.loc[bool_filter, columns]
        else:
            ret = self._df.loc[bool_filter]

        return ret.copy() if copy else ret

    def normalize_vectors(self, axis=0):
        """
        transform to column based vectors.
        axis=0 creates a (n, 2)-shaped data frame with columns ['time', 'value']
        axis=1 column based data frame with two columns ['time', 'value'] for each vector.
        """
        _df = self.apply()
        return Opp.normalize_vectors(_df, axis)

    def __str__(self) -> str:
        return self._filter_dict.__str__()

    def __repr__(self) -> str:
        ret = "Filter: "
        for key, value in self._filter_dict.items():
            ret += f"\n   {value.__repr__()}"
        return ret

class OppAttributes:
    """
    Helper to access Attributes from OMNeT++ DataFrame
    """

    def __init__(self, df):
        self._df = df

    def run_parameter(self, run, deep=True):
        ret = self._df.loc[
            (self._df["run"] == run) & (self._df["type"] == "param"),
            ("attrname", "attrvalue"),
        ]
        if ret.shape[0] == 0:
            raise ValueError(f"no data for run={run}")
        return ret.copy(deep=deep)

    def run_parameter_dict(self, run):
        ret = self.run_parameter(run)
        return {r[1][0]: r[1][1] for r in ret.iterrows()}

    def run_attr(self, run, deep=True):
        ret = self._df.loc[
            (self._df["run"] == run) & (self._df["type"] == "runattr"),
            ("attrname", "attrvalue"),
        ]
        if ret.shape[0] == 0:
            raise ValueError(f"no data for run={run}")

        return ret.copy(deep=deep)

    def run_itervar(self, run, deep=True):
        ret = self._df.loc[
            (self._df["run"] == run) & (self._df["type"] == "itervar"),
            ("attrname", "attrvalue"),
        ]
        if ret.shape[0] == 0:
            raise ValueError(f"no data for run={run}")
        return ret.copy(deep=deep)

    def run_itervar_dict(self, run):
        ret = self.run_itervar(run)
        return {r[1][0]: r[1][1] for r in ret.iterrows()}

    def run_attr_dict(self, run):
        ret = self.run_attr(run)
        return {r[1][0]: r[1][1] for r in ret.iterrows()}

    def statistics_meta_data(self, run, module, stat_name):
        r = {
            "run": run,
            "module": module,
            "name": stat_name,
            "title": "???",
            "unit": "???",
        }
        ret = self._df.loc[
            (self._df["run"] == run)
            & (self._df["type"] == "attr")
            & (self._df["module"] == module)
            & (self._df["name"] == stat_name),
            ("attrname", "attrvalue"),
        ]
        if ret.shape[0] == 0:
            raise ValueError(
                f"no meta data for run={run}, module={module}, name={stat_name}"
            )
        r.update({r[1][0]: r[1][1] for r in ret.iterrows()})
        return r

    def attr_for_series(self, s: pd.Series):
        return self.statistics_meta_data(
            run=s["run"], module=s["module"], stat_name=s["name"]
        )

class OppPlot:
    """
    Plot helper functions.
    todo: All based on OMNeT++ csv file structure
    set_xlabel
    set_xlimit(left, right)
    set_xscale() [linear, log, symlog, logit]
    set_xlimit
    set_xmargin

    set_xticks
    set_xticklabels

    """

    default_plot_args = {"linewidth": 0, "markersize": 3, "marker": ".", "color": "b"}

    plot_marker = [".", "*", "o", "v", "1", "2", "3", "4"]
    plot_color = ["b", "g", "r", "c", "m", "y", "k", "w"]

    @classmethod
    def marker(cls, idx):
        return cls.plot_marker[idx % len(cls.plot_marker)]

    @classmethod
    def color(cls, idx):
        return cls.plot_color[idx % len(cls.plot_color)]

    @classmethod
    def plt_args(cls, idx=0, *args, **kwargs):
        ret = dict(cls.default_plot_args)
        if "label" not in kwargs.keys():
            ret.setdefault("label", f"{idx}-data")

        ret.update(
            {
                "marker": cls.marker(idx),
                "color": cls.color(idx),
            }
        )
        if kwargs:
            ret.update(kwargs)
        return ret

    @classmethod
    def create_label(cls, lbl_str: str, remove_filter: List[str] = None):

        if remove_filter is not None:
            for rpl in remove_filter:
                lbl_str = lbl_str.replace(rpl, "")

        return lbl_str

    def __init__(self, accessor):
        self._opp = accessor

    def _set_labels(self, ax: plt.axes, s: pd.Series, xlabel="time [s]"):
        ax.set_xlabel(xlabel)
        attr = self._opp.attr.attr_for_series(s)
        ax.set_ylabel(f"[{attr['unit']}]")
        ax.set_title(attr["title"])

    def create_time_series(
        self, ax: plt.axes, s: pd.Series, auto_labels: bool = True, *args, **kwargs
    ):
        # Note: automatic label setting currently does not work with measurement traces,
        #       therefore we allow to turn if off by the auto_labels switch
        if auto_labels:
            self._set_labels(ax, s)
        if "label" not in kwargs:
            kwargs.setdefault("label", self.create_label(s.module, []))

        if "vecvalue" in s.index and "vectime" in s.index:
            # omnetpp based Series with Index(['vectime', 'vecvalue'], dtype='object')
            return ax.plot(s.vectime, s.vecvalue, **self.plt_args(idx=0, **kwargs))
        else:
            # standardized Series with shape (XXX,)
            return ax.plot(s, **self.plt_args(idx=0, **kwargs))

    def create_histogram(
        self,
        ax: plt.axes,
        s: pd.Series,
        bins=40,
        use_path_in_title=-1,
        attr_override=None,
        auto_labels: bool = True,
        **kwargs,
    ):
        if "density" not in kwargs:
            kwargs.setdefault("density", True)

        if "vecvalue" in s.index:
            # omnetpp based Series with Index(['vectime', 'vecvalue'], dtype='object')
            ret = ax.hist(
                s.vecvalue,
                bins,
                **kwargs,
            )
        else:
            # standardized Series with shape (XXX,)
            ret = ax.hist(s, bins, **kwargs)

        if auto_labels:
            attr = self._opp.attr.attr_for_series(s)
            if attr_override is not None:
                attr.update(attr_override)
            if use_path_in_title != -1:
                attr[
                    "title"
                ] += f" - {Opp.module_path(s['module'], use_path_in_title, tuple_on_vector=False)}"

            ax.set_title(attr["title"])
            ax.set_xlabel(f"[{attr['unit']}]")

        if "cumulative" in kwargs and kwargs["cumulative"] == True:
            # cumulative histograms sometimes have a bug which shows an annoying line down to
            # zero at the end - this removes the invalid line at the end
            axpolygons = [
                poly
                for poly in ax.get_children()
                if isinstance(poly, mpl.patches.Polygon)
            ]
            for poly in axpolygons:
                poly.set_xy(poly.get_xy()[:-1])

        return ret

@pd.api.extensions.register_dataframe_accessor("opp")
class OppAccessor:
    """
    see https://pandas.pydata.org/pandas-docs/stable/development/extending.html
    opp namespace to access attribute information for scalar, vector and
    histogram types, helpers to access data via filters and helpers to create
    simple plots like time series and histogram plots.

    See classes #OppAttributes, #OppFilter, OppFilterItem and #OppPlot for more detail.
    """

    def __init__(self, pandas_obj):
        self._validate(pandas_obj)
        self._obj: pd.DataFrame = pandas_obj
        self.plot: OppPlot = OppPlot(self)
        self.tex: OppTex = OppTex(self)
        self.attr: OppAttributes = OppAttributes(self._obj)

    @staticmethod
    def _validate(obj: pd.DataFrame):
        if not is_scave_df(obj):
            raise AttributeError(
                f"Not all columns found. expected: ['{' '.join(_scave_cols)}'] column. "
                f"got: [{' '.join(obj.columns)}] "
            )

    def module_summary(self):
        ret = self._obj[
            self._obj.type.isin(["scalar", "histogram", "vector"])
        ].module.unique()
        ret.sort()
        return ret

    def name_summary(self):
        ret = self._obj[
            self._obj.type.isin(["scalar", "histogram", "vector"])
        ].name.unique()
        ret.sort()
        return ret

    def info(self):
        runs = self._obj.loc[:, "run"].unique()
        unique_scalars = (
            self._obj.loc[self._obj["type"] == "scalar", ("module", "name")]
            .drop_duplicates()
            .shape[0]
        )
        all_scalars = self._obj.loc[
            self._obj["type"] == "scalar", ("module", "name")
        ].shape[0]
        unique_vectors = (
            self._obj.loc[self._obj["type"] == "vector", ("module", "name")]
            .drop_duplicates()
            .shape[0]
        )
        all_vectors = self._obj.loc[
            self._obj["type"] == "vector", ("module", "name")
        ].shape[0]
        print(f"shape: {self._obj.shape}")
        print(f"runs: {', '.join(list(runs))}")
        print(f"unique_scalars: {unique_scalars}")
        print(f"all_scalars: {all_scalars}")
        print(f"unique_vectors: {unique_vectors}")
        print(f"all_vectors: {all_vectors}")

    def filter(self, f: OppFilter = None, data_only=True):
        if f is not None:
            return f.apply(self._obj)
        else:
            return OppFilter(self._obj, data_only)


import argparse
import os
import aiohttp
import asyncio
import time
import multiprocessing
from aiohttp import web
from retry import retry
from fake_useragent import UserAgent
from urllib import parse
from rich.console import Console
from rich.progress import (BarColumn, DownloadColumn, Progress, TextColumn,
                        TimeRemainingColumn, TransferSpeedColumn, TaskID)
console = Console()
policy = asyncio.WindowsSelectorEventLoopPolicy()
asyncio.set_event_loop_policy(policy)

async def total(url: str, session: aiohttp.ClientSession, header: dict) -> int:
    """_summary_

    Args:
        url (str): _description_
        session (aiohttp.ClientSession): _description_
        header (dict): _description_

    Returns:
        int: _description_
    """
    async with session.head(url, headers=header, timeout=aiohttp.ClientTimeout(total=60)) as req:
        print(req.headers)
        return int(req.headers.get('content-length'))

async def split(filesize: int, num_threads: int) -> list[tuple[int, int]]:
    """Split file size

    Args:
        filesize (int): The file size.
        num_threads (int): The number of threads.

    Returns:
        list[tuple[int, int]]: Chucks.
    """
    chunk_size: int = filesize // num_threads
    parts: dict[tuple[int, int]] = []
    for i in range(num_threads):
        start: int = chunk_size * i
        end: int = start + chunk_size - 1 if i < num_threads - 1 else filesize
        parts.append((start, end))
    print(parts)
    return parts

async def main(url: str, retry_nums: int, coros: int, semaphores: int) -> None:
    """_summary_

    Args:
        url (str): _description_
        retry_nums (int): _description_
    """
    main_dir: str = os.path.split(os.path.abspath(__file__))[0]
    download_dir: str = os.path.join(os.path.dirname(main_dir), "Downloaded files\\")
    download_file: str = os.path.join(download_dir, os.path.normpath(os.path.basename(parse.urlparse(url).path)))
    user_agent: str = UserAgent().random
    semaphore = asyncio.Semaphore(semaphores)
    f = open(download_file, "wb")
    @retry(tries=retry_nums)
    async def start_download(start: int, end: int, session: aiohttp.ClientSession) -> None:
        """_summary_

        Args:
            start (int): _description_
            end (int): _description_
            session (aiohttp.ClientSession): _description_
        """
        progress.start_task(task_id)
        f.seek(start)
        _headers: dict = {'User-Agent': user_agent, 'Range': f'bytes={start}-{end}'}
        Timeout: aiohttp.ClientTimeout = aiohttp.ClientTimeout(total=100000**10)
        async with semaphore:
            async with session.get(url, headers=_headers, timeout=Timeout) as req:
                async for chunk in req.content.iter_chunked((end-start)//10):
                        f.write(chunk)
                        progress.update(task_id, advance=len(chunk))
    with Progress(
        TextColumn("[bold blue]{task.fields[filename]}", justify="right"),
        BarColumn(bar_width=None),
        "[progress.percentage]{task.percentage:>3.1f}%",
        " ",
        DownloadColumn(),
        " ",
        TransferSpeedColumn(),
        " eta ",
        TimeRemainingColumn(),
) as progress:
        task_id: TaskID = progress.add_task("Download file:", filename=os.path.split(download_file)[1], start=False)
        tasks: dict = []
        async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session:
            _total = await asyncio.create_task(total(url, session, {'User-Agent': user_agent}))
            progress.update(task_id, total=_total)
            parts = await asyncio.create_task(split(_total, coros))
            print(len(parts))
            for part in parts:
                start, end = part
                tasks.append(asyncio.create_task(start_download(start, end, session)))
            tasks_a = await asyncio.gather(*tasks)
    f.close()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="A downloader.")
    parser.add_argument('-U', '--url', type=str, required=True)
    parser.add_argument('--retry-nums', type=int, required=False, default=3)
    parser.add_argument('-C', "--concurrent-nums",type=int, required=False, default=multiprocessing.cpu_count()*2+2)
    parser.add_argument('-S', "--semaphores",type=int, required=False, default=3)
    arg = parser.parse_args()
    start = time.perf_counter()
    asyncio.run(main(arg.url, arg.retry_nums, arg.concurrent_nums, arg.semaphores))
    end = time.perf_counter()
    print(end - start)


import config
import argparse
import logging
import subprocess
import os

from datetime import datetime
from lib.slackApi import SlackApi
from lib.kubeApi import KubeApi
from lib.helpers import generate_image
from lib.trello import cleanup_trello

logging.basicConfig(
    level=logging.DEBUG, format="[%(asctime)s][%(levelname)s] %(message)s"
)

class Deployorama:
    """
    Perform kubernetes deployment and all that jazz.
    """

    def __init__(self):
        self.tag = config.TAG
        self.migration = config.MIGRATION_LEVEL
        self.check_cronjobs = config.CHECK_CRONJOBS
        self.slacker = SlackApi()
        self.kuber = KubeApi(namespace=config.NAMESPACE)
        self.deployments = {
            tier: self.kuber.get_deployments(
                label_selector="project={}, tier={}".format(config.PROJECT, tier)
            )
            for tier in config.TIERS
        }
        self.cronjobs = self.kuber.get_cronjobs(label_selector="project={}".format(config.PROJECT))
        self.has_down_time = self.migration == 2
        self.has_migration = self.migration > 0
        self.migration_completed = False
        self.deploy_success = True

    def get_new_image(self, image):
        return generate_image(old_image=image, new_tag=self.tag)

    def all_deployments(self):
        return [deploy for sublist in self.deployments.values() for deploy in sublist]

    def send_release_notification(self):
        if not self.deploy_success:
            logging.debug("Skipping release notification due to deployment failure")
            return
        cleanup_trello()

    def deploy(self):
        """
        The master deployment method to process deployment from start to finish.
        """

        if config.DISABLED:
            self.slacker.send_message(text="Automated deployment is currently disabled")
            return

        error_message = None
        error_handling_message = None

        try:
            self.slacker.send_initial_message()
            self.kuber.get_cronjobs(label_selector="project={}".format(config.PROJECT))

            if self.has_down_time:
                self.scale_down_deployments()

            if self.has_migration:
                self.backup_database()
                self.run_migration()

            self.set_images()

            if (self.check_cronjobs):
                self.set_cronjob_images()

            if self.has_down_time:
                self.scale_up_deployments()

        except Exception as e:
            self.deploy_success = False
            error_message = str(e)
            logging.error(error_message)
            error_handling_message = self.handle_deploy_failure()

        self.slacker.send_completion_message(
            error_message=error_message,
            error_handling_message=error_handling_message,
            deployments=self.all_deployments(),
            requires_migration_rollback=self.has_down_time and self.migration_completed,
        )
        self.send_release_notification()

    def handle_deploy_failure(self):
        """
        Handle deployment failure by reverting all modifications.
        """
        step = "Recovering From Deployment Error"
        self.slacker.send_thread_reply(step)

        if self.has_down_time is True and self.migration_completed:
            return "Skipped Automated Recovery: Requires Manual Intervention"

        try:
            self.rollback_images()
            self.scale_up_deployments()
            error_handler_message = "Successfully Rolled Back Deployment"

        except Exception as e:
            error_handler_message = str(e)
            logging.error(error_handler_message)

        return error_handler_message

    def raise_step_error(self, error: Exception, step: str):
        """
        Handle notification for deployment step errors and raise exception with new message.
        """
        error_message = "{}\nFailed: Error={}".format(step, str(error))
        logging.error(error_message)
        self.slacker.send_thread_reply(error_message)
        raise Exception(error_message)

    def scale_down_deployments(self):
        """
        Scale down deployments.
        """
        try:
            for tier in config.TIERS:
                for deployment in self.deployments[tier]:
                    step = "Scaling Down Deployment:\ndeployment={}".format(
                        deployment["name"]
                    )
                    self.slacker.send_thread_reply(step)
                    deployment["scaled_down"] = True
                    self.kuber.set_deployment_replicas(deployment["name"], 0)
                step = "Verifying {} Deployments Scaled Down Successfully".format(tier)
                self.slacker.send_thread_reply(step)
                for deployment in self.deployments[tier]:
                    self.kuber.verify_deployment_update(deployment["name"])
        except Exception as e:
            self.raise_step_error(step=step, error=e)

    def scale_up_deployments(self):
        """
        Scale up all deployments (in reverse order) to original replica counts.
        """
        try:
            for tier in config.TIERS[::-1]:
                for deployment in self.deployments[tier]:
                    if deployment.get("scaled_down", False) is True:
                        step = "Scaling Up Deployment:\ndeployment={}\nreplicas={}".format(
                            deployment["name"], deployment["replicas"]
                        )
                        self.slacker.send_thread_reply(step)
                        self.kuber.set_deployment_replicas(
                            deployment["name"], deployment["replicas"]
                        )
                        deployment["scaled_down"] = False
                step = "Verifying {} Deployments Scaled Up Successfully".format(tier)
                self.slacker.send_thread_reply(step)
                for deployment in self.deployments[tier]:
                    self.kuber.verify_deployment_update(deployment["name"])
        except Exception as e:
            self.raise_step_error(step=step, error=e)

    def backup_database(self):
        """
        Store a cloud sql backup on google storage
        """
        backup_file = "{}-{}.sql".format(
            config.DATABASE_NAME, datetime.today().strftime("%Y-%m-%d--%H%M")
        )
        backup_uri = "{}/{}".format(config.DATABASE_BACKUP_BUCKET, backup_file)
        step = "Backing Up Database:\nbackup={}".format(backup_uri)
        try:
            self.slacker.send_thread_reply(step)
            backup_command = [
                "gcloud",
                "sql",
                "export",
                "sql",
                config.DATABASE_INSTANCE_NAME,
                backup_uri,
                "--database={}".format(config.DATABASE_NAME),
                "--verbosity=debug",
            ]
            subprocess.run(backup_command, check=True)
        except Exception as e:
            self.raise_step_error(step=step, error=e)

    def run_migration(self):
        """
        Perform database migration via k8 app-migrator job.
        """
        step = "Migrating Database"
        try:
            self.slacker.send_thread_reply(step)
            self.kuber.run_migration(tag=self.tag, source=config.APP_MIGRATOR_SOURCE)
            self.migration_completed = True
        except Exception as e:
            self.raise_step_error(step=step, error=e)

    def set_images(self):
        """
        Update images for all deployments.
        """
        try:
            for deployment in self.all_deployments():
                new_image = self.get_new_image(deployment["image"])
                step = "Setting Deployment Image:\ndeployment={}\nold_image={}\nnew_image={}".format(
                    deployment["name"], deployment["image"], new_image
                )
                if deployment["image"] == new_image:
                    self.slacker.send_thread_reply(
                        "Deployment Doesn't Require Image Update: deployment={} image={}".format(
                            deployment["name"], new_image
                        )
                    )
                    continue
                self.slacker.send_thread_reply(step)
                self.kuber.set_deployment_image(deployment["name"], new_image)
                deployment["updated_image"] = True
            step = "Verifying Deployment Updates Completed Successfully"
            self.slacker.send_thread_reply(step)
            for deployment in self.all_deployments():
                self.kuber.verify_deployment_update(deployment["name"])
        except Exception as e:
            self.raise_step_error(step=step, error=e)

    def set_cronjob_images(self):
        """
        Update images for all cronjobs.
        """
        try:
            for cronjob in self.cronjobs:
                new_image = self.get_new_image(cronjob["image"])
                step = "Setting Cronjob Image:\ncronjob={}\nold_image={}\nnew_image={}".format(
                    cronjob["name"], cronjob["image"], new_image
                )
                if (cronjob["image"] == new_image):
                    self.slacker.send_thread_reply(
                        "Cronjob Doesn't Require Image Update: cronjob={} image={}".format(
                            cronjob["name"], new_image
                        )
                    )
                    continue
                self.slacker.send_thread_reply(step)
                self.kuber.set_cronjob_image(cronjob["name"], new_image)
            step = "Cronjob Updates Completed"
            self.slacker.send_thread_reply(step)

        except Exception as e:
            self.raise_step_error(step=step, error=e)

    def rollback_images(self):
        """
        Rollback all deployment images to their original state prior to deployment.
        """
        for deployment in self.all_deployments():
            if deployment.get("updated_image", False) is False:
                continue
            step = "Rolling Back Deployment Image:\ndeployment={}\nattempted_image={}\nrollback_image={}".format(
                deployment["name"],
                self.get_new_image(deployment["image"]),
                deployment["image"],
            )
            try:
                self.slacker.send_thread_reply(step)
                self.kuber.set_deployment_image(
                    deployment["name"], deployment["image"], verify_update=True
                )
                deployment["updated_image"] = False
            except Exception as e:
                self.raise_step_error(step=step, error=e)

if __name__ == "__main__":
    parser = argparse.ArgumentParser("deploy")
    parser.add_argument(
        "-t", "--tag", help="New image tag to be rolled out.", type=str, required=True
    )
    parser.add_argument(
        "-m",
        "--migration",
        help="Migration level: 0=None, 1=Hot, 2=Cold",
        type=int,
        required=True,
        choices=[0, 1, 2]
    )
    parser.add_argument(
        "-cj",
        "--cronjob",
        help=f"Set to True to update cronjobs available for PROJECT:{config.PROJECT}.",
        type=bool,
        default=True,
        required=False,
        choices=[True, False]
    )
    args = parser.parse_args()
    config.TAG = args.tag.strip()
    config.MIGRATION_LEVEL = args.migration
    config.CHECK_CRONJOBS = args.cronjob

    deployer = Deployorama()
    deployer.deploy()
    os._exit(os.EX_OK)


import pandas as pd
from core.models import City
from django.contrib.gis.geos import Point
from django.core.management.base import BaseCommand

tmp_data_city = pd.read_csv(
    'core/management/commands/city.csv', sep=',', encoding="utf-8"
)
tmp_data_city['city'] = tmp_data_city['city'].fillna(tmp_data_city['region'])

class Command(BaseCommand):
    help = 'The Zen of Python'

    def handle(self, **options):
        cities = [
            City(
                name=row['city'],
                location=Point(row['geo_lon'], row['geo_lat']),
            )
            for _, row in tmp_data_city.iterrows()
        ]

        City.objects.bulk_create(cities, ignore_conflicts=True)