import requests, random, time, json, re, scipy.stats
import matplotlib.pyplot as plt

def send_aes_plaintext(plaintext, url):
    headers = {
        "Content-Type": "text/plain"  # Assuming plaintext is just text
    }

    start = time.monotonic()
    response = requests.post(url, headers=headers, data=plaintext)
    end = time.monotonic()

    # Ensure the request was successful
    response.raise_for_status()

    # Return the output from the response body
    return response.text, end - start

def time_aes_plaintext(ptext, url="http://localhost:8080/hex"):
    resp, outer_time = send_aes_plaintext(ptext, url)
    return extract_time(resp), outer_time

def time_aes_get_ctext(ptext, url="http://localhost:8080/hex"):
    resp, outer_time = send_aes_plaintext(ptext, url)
    return extract_ctext(resp), extract_time(resp), outer_time

## return a random 16 byte plaintext
def random_plaintext():
    return random.getrandbits(128).to_bytes(16, 'little').hex()

## normalize times
def normalize_times(times, counts):
    normal = [[0 for x in range(256)] for y in range(16)]
    for i in range(16):
        for b in range(256):
            if counts[i][b] != 0:
                normal[i][b] = times[i][b] / counts[i][b]
    return normal

## tally result
def tally(ptext, encrypt_time, times, counts):
    for (i,b) in enumerate(bytes.fromhex(ptext)):
        try:
            t = times[i][b]
            cnt = counts[i][b]
        except KeyError:
            t = 0
            cnt = 0

        times[i][b] = t + encrypt_time
        counts[i][b] = cnt + 1

def extract_time(s):
    print(s)
    m = re.match('".*Elapsed time: (.*) cputicks"', s)
    return float(m.group(1))

def extract_ctext(s):
    m = re.match('"Ciphertext: (.*),.*"', s)
    return m.group(1)

def sweep_tally_time(n):
    times = [[0 for x in range(256)] for y in range(16)]
    counts = [[0 for x in range(256)] for y in range(16)]

    for i in range(n):
        if i % 1000 == 0:
            print(i)
        ptext = random_plaintext()
        inner_time, outer_time = time_aes_plaintext(ptext)
        if inner_time > 3000:
            print("time: {0}, i: {1}, ptext: {2}".format(inner_time, i, ptext))
        tally(ptext, inner_time, times, counts)

    return normalize_times(times, counts), counts

def save(timing, counts):
    records = []
    for i in range(16):
        for b in range(256):
            record = {'n': i, 'b': b.to_bytes(1, 'little').hex(), 't': timing[i][b], 'cnt': counts[i][b]}
            records.append(record)
    out = json.dumps(records)
    print(out)

def hit(n, ptext):
    times = []
    for i in range(n):
        if i % 1000 == 0:
            print(i)
        inner_time, _ = time_aes_plaintext(ptext)
        times.append(encrypt_time)

    print("avg encrypt time: {0}".format(sum(times) / len(times)))

def infinite_encrypt(ptext):
    while True:
        time_aes_plaintext(ptext)

# explore n random plaintext encryptions m times each
# print out summary statistics of drawn distributions
def explore(n, m):
    dists = {}
    ptexts = [ random_plaintext() for i in range(n) ]
    for i in range(m):
        if i % 100 == 0:
            print(i)
        for ptext in ptexts:
            t, _ = time_aes_plaintext(ptext)
            if t > 3000:
                continue
            if not ptext in dists:
                dists[ptext] = []
            dists[ptext].append(t)
    tester = dists[ptexts[0]]
    for ptext in dists:
        data = dists[ptext]
        print("{0}: {1}, ks test: {2}".format(ptext, scipy.stats.describe(data), scipy.stats.ks_2samp(data, tester, alternative='two-sided', mode='auto').pvalue))

def sweep_score_ctext(n, lastkeyhex):
    '''
    Sweep random plaintexts, score output ciphertexts by count of Sbox duplicate hits.
    Sbox duplicate hits are measured by checking if ciphertext byte combinations include
    any lastkeyhex ciphertext byte combinations.  lastkeyhex is the last 4 words of the expanded
    round key bytes -- this function is validating against a known key.

    Sbox duplicates necessitate a cache hit when doing the final Sbox lookup in the T table
    implementation and therefore higher ciphertext score should roughly correlate to lower time.
    Noise comes from cache timing noise + the existence of cache hits unseen by duplicate
    Sbox hits -- ~8-16 Sbox entries are loaded in a cache line and will be hit without duplication
    '''
    lastkey_combinations = set(byte_combinations(lastkeyhex))
    scores = {}
    for i in range(n):
        ptext = random_plaintext()
        print(ptext)
        ctext, inner_time, outer_time = time_aes_get_ctext(ptext)
        # Filter outliers -- usually runtime artifacts
        if inner_time > 5000:
            print("time: {0}, i: {1}, ptext: {2}".format(inner_time, i, ptext))
            continue
        score = score_ctext(ctext, lastkey_combinations)
        if score not in scores:
            scores[score] = []
        scores[score].append(inner_time)
    tester = None
    for score in scores:
        data = scores[score]
        if tester is None:
            tester = data
        print("score {0}: {1}, ks test: {2}".format(score, scipy.stats.describe(data), scipy.stats.ks_2samp(data, tester, alternative='two-sided', mode='auto').pvalue))

def score_ctext(ctexthex, match_set):
    score = 0
    ctext_combos = byte_combinations(ctexthex)
    for c in ctext_combos:
        if c in match_set:
            score += 1
    return score

def byte_combinations(hexstr):
    '''
    Return the set of the xor of each input byte with each other input byte
    excluding same bytes and repeats
    '''
    bs = enumerate(bytes.fromhex(hexstr))
    return [a ^ b for (i,a) in bs for (j, b) in bs if i < j]

def compare(m, ptext1, ptext2):
    dists = {}
    ptexts = [ptext1, ptext2]

    for i in range(m):
        if i % 100 == 0:
            print(i)
        for ptext in ptexts:
            t, _ = time_aes_plaintext(ptext)
            # filter outliers, (for clojure likely JVM artifact)
            if t > 3000:
                continue
            if not ptext in dists:
                dists[ptext] = []
            dists[ptext].append(t)
    tester = dists[ptexts[0]]
    for ptext in dists:
        data = dists[ptext]
        print("{0}: {1}, ks test: {2}".format(ptext, scipy.stats.describe(data), scipy.stats.ks_2samp(data, tester, alternative='two-sided', mode='auto').pvalue))
        plt.hist(data)

if __name__ == "__main__":
    sweep_score_ctext(10000, "13111d7fe3944a17f307a78b4d2b30c5")
#    explore(10, 500)
#    compare(2000, "0b474a7d75289451531748dc1a221ff9", "d93f6152518ddec1d05c2a83e679340e")
#    timing, counts = sweep_tally_time(10000)
#    save(timing, counts)
#    infinite_encrypt("440ed4556d87501082ab5285c33960c0")

## Send lots of requests -- ranomd
## take the time of each request in like ns or something
## we accumulate total time in a record t[j][b]  j 0..15, byte position in plaintext, b 0.255 byte value at position

## Each request generates 16 different bytes

## 0000xx0000000000000


from matplotlib import pyplot as plt; import numpy as np; import os
import RV.library.spectrum_processing_functions as spf
from astropy.time import Time
from astropy.coordinates import EarthLocation
from barycorrpy import utc_tdb

target = "kic8430105"
observatory_name = 'Roque de los Muchachos'
observatory_location = EarthLocation.of_site(observatory_name)
stellar_target = "kic8430105"

spectra_folder_path = '../Data/unprocessed/NOT/KIC8430105/'
filename_identifier = 'merge.fits'
reduced_filename_identifier = '_reduced_set.dat'

filename_list = []
date_array = []
RA_array = np.array([])
DEC_array = np.array([])
sn_array = np.array([])

period = 63.3271045716

eclipse_primary = 0.0
eclipse_secondary = 0.6589
approximate_eclipse_hwidth= 0.02

wavelength_normalization_limit = (4450, 7000)

for filename in os.listdir(spectra_folder_path):
    if filename_identifier in filename and '.lowSN' not in filename:
        wavelength, flux, date, ra, dec = spf.load_program_spectrum(spectra_folder_path + filename)

        filename_bulk = filename[:filename.rfind(".fits")]

        date_array = np.append(date_array, date)
        RA_array = np.append(RA_array, ra * 15.0)  # converts unit
        DEC_array = np.append(DEC_array, dec)
        filename_list.append(filename[:filename.rfind("_step011_merge.fits")])

        # Prepare for continuum fit
        selection_mask = (wavelength > wavelength_normalization_limit[0]) & \
                         (wavelength < wavelength_normalization_limit[1])
        wavelength = wavelength[selection_mask]
        flux = flux[selection_mask]

        # Remove values under 0
        selection_mask = (flux >= 0.0)
        flux = flux[selection_mask]
        wavelength = wavelength[selection_mask]

        # Performs continuum fit and reduces emission lines (by removing above 2.5 std from fitted continuum)
        wavelength, flux = spf.simple_normalizer(wavelength, flux, reduce_em_lines=True, plot=False)

        # Calculate S/N
        mask = (wavelength > 5605) & (wavelength < 5613)
        rms = np.sqrt(np.sum((1-flux[mask]) ** 2)/flux[mask].size)
        signal_noise = 1/rms
        # print(rms)
        # print(signal_noise)

        plt.figure(figsize=(16, 9))
        plt.plot(wavelength, flux)
        plt.plot(wavelength[mask], flux[mask])
        plt.xlim([5602, 5617])
        sn_array = np.append(sn_array, signal_noise)

plt.show()

# # Calculate bjdtdb
RA, DEC = np.mean(RA_array), np.mean(DEC_array)
times = Time(date_array, scale='utc', location=observatory_location)
times.format = 'jd'
times.out_subfmt = 'long'
bjdtdb, _, _ = utc_tdb.JDUTC_to_BJDTDB(times, ra=RA, dec=DEC, starname=stellar_target, obsname=observatory_name)

# # Extra observation date
time_extra = Time('2021-08-02T23:03:19', scale='utc', location=observatory_location)
time_extra.format = 'jd'
time_extra.out_subfmt = 'long'
bjd_extra, _, _ = utc_tdb.JDUTC_to_BJDTDB(time_extra, ra=RA, dec=DEC, starname=stellar_target, obsname=observatory_name)

# Phases
model_filename = '../../Binary_Analysis/JKTEBOP/NOT/kepler_pdcsap/model.out'
rvA_filename = '../../Binary_Analysis/JKTEBOP/NOT/kepler_pdcsap/rvA.out'
rvB_filename = '../../Binary_Analysis/JKTEBOP/NOT/kepler_pdcsap/rvB.out'
bjdtdb -= 2400000 + 54998.2336737188  # 54976.6348
phase_model, rv_Am, rv_Bm = np.loadtxt(model_filename, usecols=(0, 6, 7), unpack=True)
phase_spectra = np.mod(bjdtdb, period) / period
time_rvA, rvA, error_rvA, phase_rvA = np.loadtxt(rvA_filename, usecols=(0, 1, 2, 3), unpack=True)
time_rvB, rvB, error_rvB, phase_rvB = np.loadtxt(rvB_filename, usecols=(0, 1, 2, 3), unpack=True)
sort_idx = np.argsort(phase_rvA)
time_rvA, rvA, error_rvA, phase_rvA = time_rvA[sort_idx], rvA[sort_idx], error_rvA[sort_idx], phase_rvA[sort_idx]
sort_idx = np.argsort(phase_rvB)
time_rvB, rvB, error_rvB, phase_rvB = time_rvB[sort_idx], rvB[sort_idx], error_rvB[sort_idx], phase_rvB[sort_idx]

# Sort
sort_idx = np.argsort(phase_spectra)
filenames_sorted = np.array(filename_list)[sort_idx]
phase_spectra_sorted = phase_spectra[sort_idx]
sn_sorted = sn_array[sort_idx]

print('Phase\tSpectra\tS/N')
for i in range(0, len(phase_spectra_sorted)):
    print(phase_spectra_sorted[i], filenames_sorted[i], sn_sorted[i])
print('')
print('Processed rv A:')
print('Phase\tTime\tRV\tError')
for i in range(0, len(rvA)):
    print(phase_rvA[i], time_rvA[i]-50000, rvA[i], error_rvA[i])
print('')
print('Processed rv B:')
print('Phase\tTime\tRV\tError')
for i in range(0, len(rvB)):
    print(phase_rvB[i], time_rvB[i]-50000, rvB[i], error_rvB[i])
print('')
print('Phase\tSpectra')
for i in range(0, len(phase_spectra_sorted)):
    print(phase_spectra_sorted[i], filenames_sorted[i])
# RV plot
fig_rv = plt.figure(figsize=(16, 9))
gs_rv = fig_rv.add_gridspec(1, 1)
ax_rv = fig_rv.add_subplot(gs_rv[:, :])

system_rvA = 11.6144
system_rvB = 12.0293

ax_rv.plot(phase_model, rv_Am-system_rvA, 'r')
ax_rv.plot(phase_model, rv_Bm-system_rvB, 'b')
ax_rv.plot(phase_rvA, rvA-system_rvA, 'r*')
ax_rv.plot(phase_rvB, rvB-system_rvB, 'b*')
ax_rv.plot(phase_spectra, np.zeros(phase_spectra.shape), 'k*', markersize=5)
for i in range(0, filenames_sorted.size):
    ax_rv.annotate(filenames_sorted[i], (phase_spectra_sorted[i], 0),
                   (phase_spectra_sorted[i], (np.mod(i, filenames_sorted.size//4)/(filenames_sorted.size//4) - 0.5)*30),
                   arrowprops={'arrowstyle': '->'})
ax_rv.set_xlabel('Orbital Phase')

ax_rv.plot([eclipse_primary, eclipse_primary], [-40, 50], color='gray')
ax_rv.plot([1.0-approximate_eclipse_hwidth, 1.0-approximate_eclipse_hwidth], [-40, 50], '--',
           color='gray')

ax_rv.plot([eclipse_primary+approximate_eclipse_hwidth, eclipse_primary+approximate_eclipse_hwidth], [-40, 50], '--',
           color='gray')
ax_rv.plot([eclipse_secondary, eclipse_secondary], [-40, 50], color='gray')
ax_rv.plot([eclipse_secondary-approximate_eclipse_hwidth, eclipse_secondary-approximate_eclipse_hwidth], [-40, 50],
           '--', color='gray')
ax_rv.plot([eclipse_secondary+approximate_eclipse_hwidth, eclipse_secondary+approximate_eclipse_hwidth], [-40, 50],
           '--', color='gray')
ax_rv.plot(np.mod(bjd_extra - 2454998.2336737188, period)/period, 0, 'g*')

plt.show(block=True)


from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render_to_response
from django.contrib.auth.decorators import login_required
from django.template import Context, Template, RequestContext
from django.template.loader import get_template
from django.core.exceptions import ObjectDoesNotExist
from django.utils import simplejson
from models import *
from common import *
from views import *
import pickle
from time import time
import exchtran.settings
import os

"""

To add widgets to a component:
1) Put this code in the component's Python edit function
widgetCount = request.POST.get("widgetCount")
widgets = []
for i in range(int(widgetCount)):
	widgets.append(int(request.POST.get("widgetSelect_" + str(i+1))))

(Be sure to add the widgets list to the component object)

2) Put this code in the component's HTML edit template
{% include "widgets/addWidgets.html" %}

3) Put this code in the component's Python display function
widgets = prepareWidgets(request.session['exchangeParameters'].widgets)

4) Put this code in the component's HTML display template in the Right Col div
{{widgets}}

"""

#
# Prepare Widgets
#
def prepareWidgets(widgetIDList):
	"""Goes through each widget in widgetIDList an renders the templates.
	All the template text is concated and returned"""
	counter = 1
	renderedWidgets = ""
	for widgetID in widgetIDList:
		widget = Component.objects.get(id=widgetID)
		widgetParams = pickle.loads(widget.parameters)
		widgetTemplate = get_template(widget.componentType.kickoffTemplate)
		widgetContext = Context({"widgetParams": widgetParams, "counter": counter})
		renderedWidgets += widgetTemplate.render(widgetContext)
		counter = counter + 1

	return renderedWidgets

#
# Timer Widget
#
class widgetTimerObj(object):
	"""A Data structure holding a timer widget object"""
	def __init__(	self,
					mins = 3,
					secs = 30,
					label = "Time remaining for this round"
				):
		self.mins = mins
		self.secs = secs
		self.label = label

@login_required
def timerDisplay(request):
	"""Displays the timer widget"""

	return render_to_response("widgets/timer_display.html",
							{
							},
						  	context_instance=RequestContext(request))

@login_required
def timerEdit(request):
	"""Saves the contents of the timer widget component form"""

	comID = request.POST.get("comIM")

	componentParams = widgetTimerObj(
									mins = request.POST.get("mins"),
									secs = request.POST.get("secs"),
									label = request.POST.get("label")
									)

	c = Component.objects.get(id=comID)
	c.name = request.POST.get("componentName")
	c.description = request.POST.get("componentDescription")
	c.displayName = request.POST.get("displayName")
	c.parameters = pickle.dumps(componentParams)
	c.save()

	response = "Component Saved"
	return render_to_response('api.html',
						  {'response': response},
						  context_instance=RequestContext(request))

#
# Image Widget
#
class widgetImageObj(object):
	"""A Data structure holding an image widget object"""
	def __init__(	self,
					fileName = None,
					filePath = None,
					fileType = None,
					webPath = None,
					label = None
				):
		self.fileName = fileName
		self.filePath = filePath
		self.fileType = fileType
		self.webPath = webPath
		self.label = label

@login_required
def imageDisplay(request):
	"""Displays the image widget"""

	return render_to_response("widgets/timer_display.html",
							{
							},
						  	context_instance=RequestContext(request))

@login_required
def imageEdit(request):
	"""Saves the contents of the image widget component form"""

	comID = request.POST.get("comIM")
	c = Component.objects.get(id=comID)
	oldParameters = pickle.loads(c.parameters)

	# Handle file upload
	try:
		fileName = request.FILES['imageUpload']['filename']
		filePath = exchtran.settings.UPLOAD_DIR + request.FILES['imageUpload']['filename']
		fileType = request.FILES['imageUpload']['content-type']
		webPath = "/site_media/uploads/" + fileName
	except:
		# There isn't a new file to upload, so just update the label field
		oldParameters.label = request.POST.get("label")
		c.parameters = pickle.dumps(oldParameters)
	else:

		# Check for previously uploaded images, delete if found
		if(oldParameters.filePath != None):
			os.remove(oldParameters.filePath)

		validFileTypes = ['.png', '.jpg', '.gif']
		if (filePath[-4:] in validFileTypes):
			f = open(filePath,"w")
			f.write(request.FILES['imageUpload']['content'])
			f.close()

		componentParams = widgetImageObj(
									fileName = fileName,
									filePath = filePath,
									fileType = fileType,
									webPath = webPath,
									label = request.POST.get("label")
									)
		c.parameters = pickle.dumps(componentParams)

	# Handle the name and description fields
	c.name = request.POST.get("componentName")
	c.description = request.POST.get("componentDescription")
	c.displayName = request.POST.get("displayName")
	c.save()

	return HttpResponseRedirect('/components/edit/?id=' + comID)

#
# Bank Widget
#
class widgetBankObj(object):
	"""A Data structure holding an bank widget object"""
	def __init__(	self,
					topLabel = ""
				):
		self.topLabel = topLabel

@login_required
def bankDisplay(request):
	"""Displays the bank widget"""

	return render_to_response("widgets/timer_display.html",
							{
							},
						  	context_instance=RequestContext(request))

@login_required
def bankEdit(request):
	"""Saves the contents of the bank widget component form"""

	comID = request.POST.get("comIM")

	componentParams = widgetBankObj(
									topLabel = request.POST.get("topLabel")
									)

	c = Component.objects.get(id=comID)
	c.name = request.POST.get("componentName")
	c.description = request.POST.get("componentDescription")
	c.displayName = request.POST.get("displayName")
	c.parameters = pickle.dumps(componentParams)
	c.save()

	response = "Component Saved"
	return render_to_response('api.html',
						  {'response': response},
						  context_instance=RequestContext(request))


#!/usr/bin/env python
# coding: utf-8

# # Variable correlation analysisnotebook
#
#

# ## Data set
#
#
#
# ### Node info
#
# APs are stationary and colocated. Clients are mobile and colocated. Data flows in the AP -> client direction, so clients are also referred to as receivers.
#
# | 802.11 type | AP lat | AP lon | channel | cntr. freq (MHz) | bw (MHz) |
# |-------------|--------|--------|---------|------------------|----------|
# | n | 41.111879 | -8.631146 | 6 | 2437 | 20 |
# | ac (wave 1) | 41.111879 | -8.631146 | 40 | 5200 | 40 |
# | ad | 41.111879 | -8.631146 | 1 | 60480 | 2160 |
#
# Main clients, all positioned in the moving vehicle's roof, a VW Golf Mk 3:
#
# | 802.11 type | radio | nr. antennas |
# |-------------|-----------------------------------|---|
# | n           | csl usb 2.0 wlan Adapter 300 Mbps | 2 |
# | ac          | tp-link archer t4uh               | 2 |
# | ad          | tp-link talon ad7200 (tp-03)      | ? |
#
# Monitor nodes, used to collect frame parameters unavailable from main clients. They were colocated with the main clients. Details are:
#
# | 802.11 type | radio | nr. antennas |
# |-------------|-------|--------------|
# | n | csl usb 2.0 wlan adapter 300 Mbps | 2 |
# | ac | tp-link talon ad7200 (tp-02) | 8 |
# | ad | tp-link talon ad7200 (tp-02) | ? |
#
# Background clients, used to induce channel utilization diversity. They were static, and place near the AP. Details:
#
# | 802.11 type | radio | nr. antennas | position |
# |-------------|-------|--------------|----------|
# | n | tp-link wn722n | 1 | ~2m away from AP |
# | n | tp-link wn722n | 1 | ~2m away from AP |
# | ac | csl usb 2.0 wlan Adapter 300 Mbps | 2 | ~2m away from AP |
# | ac | csl usb 2.0 wlan Adapter 300 Mbps | 2 | ~2m away from AP |
# | ad | tp-link talon ad7200 (tp-04) | ? | stopped vehicle's roof (so very near AP) |
#
#
# ### Trace info
#
# The data is divided into traces. Each trace represents a uninterrupted experimental period.
# Traces vari
#
# | trace nr | date | start time | #samples w/ successful reception (802.11n) | #samples w/ successful reception (802.11ac) | #samples w/ successful reception (802.11ad) | AP vehicle | nr. clients |
# |-----|------------|----------|------|------|-----| --- | --- |
# | 302 | 2019-08-20 | 10:28:45 | 3262 | 2787 | 423 | Honda Civic sedan (2001) | 1 |
# | 303 | 2019-08-20 | 11:26:23 | 3374 | 3027 | 312 | Honda Civic sedan (2001) | 2 |
# | 304 | 2019-08-20 | 12:39:46 | 1711 | 216  | 14  | Honda Civic sedan (2001) | 3 (n & ac) 2 (ad) |
# | 401 | 2019-08-22 | 10:19:24 | 1685 | 1681 | 545 | Peugeot Partner van (2002) | 1 |
# | 402 | 2019-08-22 | 10:48:26 | 2859 | 2827 | 764 | Peugeot Partner van (2002) | 2 |
# | 403 | 2019-08-22 | 11:39:36 | 135 | 135 | 116 | Peugeot Partner van (2002) | 2 |
# | 404 | 2019-08-22 | 11:42:50 | 114 | 114 | 53 | Peugeot Partner van (2002) | 2 |
# | 405 | 2019-08-22 | 11:45:07 | 2019 | 2019 | 507 | Peugeot Partner van (2002) | 2 |
#
#
# ### Feature info
#
# Each row represents 1-second worth of data, and is comprised of many features (columns):
#
# * **systime** : system time (1 Hz resolution) that this row refers to. All node clocks were synchronized through NTP.
# * **traceNr** : nr. of the trace the row belongs to.
# * **lon** : longitude (in decimal degrees) reported by the client's GPS at `systime`.
# * **lat** : latitude (in decimal degrees) reported by the client's GPS at `systime`.
# * **receiverAlt** : altitude (in meters) reported by the client's GPS at `systime`.
# * **receiverX** : x coordinate of the client's position when space is discretized as a Cartesian plane and the AP is set to be the origin of the coordinate system. The x axis corresponds to east-west (positive values are east, negative values are west). Unit is meters.
# * **receiverY** : y coordinate of the client's position when space is discretized as a Cartesian plane, in meters.
# * **receiverDist** : great-circle distance (in meters) between AP and client, computed by the haversine formula.
# * **receiverSpeed** : speed (in m/s) reported by the client's GPS at `systime`.
# * **receiverId** : system-specific id for the client (in the vehicle).
# * **senderId** : system-specific id for the AP serving the client.
# * **isIperfOn** : 1 if the row's `systime` corresponds to a period where iperf (the app used to send and receive data) is known to have been running on the receiver side, and 0 otherwise. Should always be 1.
# * **isInLap** : 1 if this row's systime has been marked as being part of a time period where clients were doing laps around the APs, and 0 otherwise. **This feature is unreliable and should be ignored.**
# * **rssiMean** : the mean of the RSSI (Received Signal Strength Indicator) values of AP data frames received by the client during the 1-second period the row refers to. Measured in dBm. **For 802.11ad, we retrieve RSSI from sector sweep feedback frames.**
# * **snrMean** : the mean of the SNR (Signal-to-Noise Ratio) values of AP data frames received by the client during the 1-second period the row refers to. Only valid for 802.11ad rows.
# * **channelFreq** : center frequency of the WiFi channel used, in MHz.
# * **channelBw** : channel width of the WiFi channel used, in MHz.
# * **channelUtil** : percentage of time the wireless medium was sensed to be busy during the 1-second period the row refers to. A value of -1 indicates an absense of data. **In traces numbered 40x, the 802.11n and ac routers didn't log channel busy time, and as such we had to approximate channel util. based on x,y coordinates and nr. of active clients.**
# * **dataRateMedian** : the median of the data rate values of AP data frames received by client second period the row refers to, in Mbps.
# * **dataRateMean** : the mean of the data rate values of AP data frames received by the client during the 1-second period the row refers to, in Mbps.
# * **nBytesReceived** : total number of bytes received by the client from the ap during the 1-second period systime period the row refers to, in Bytes.
# * **tghptConsumer**: throughput reported by the receiving end of iperf, during the 1-second period the row refers to, in Mbps.
# * **wifiType** : IEEE 802.11 network type, i.e. n, ac or ad.
# * **nrClients** : total number of clients operating on the same channel and bandwidth as `receiverId` during the 1-second period the row refers to.
# * **nRetries** : total number of frames marked as being retries (i.e. retransmissions due to loss) during the 1-second period the row refers to.
# * **meanBeaconRssi** : the mean of the RSSI (Received Signal Strength Indicator) values of AP beacon frames received by the client during the 1-second period the row refers to. Measured in dBm.
# * **meanInterBeaconTime** : the mean of the interval between AP beacon frames received by the client during the 1-second period the row refers to. Measured in seconds.
# * **nBeacons** : the total number of beacons received during the 1-second period the row refers to.
#

# ### Dataset loading
#

# In[1]:

import warnings
import pandas
import sys
import numpy as np

warnings.filterwarnings('ignore')

dframe = pandas.read_csv("./wifi-exp-log-summary.csv")

dframe = dframe.reset_index(drop = True) # creates index column, numbered from 0 to n-1

print("Loaded data set (%d rows)" % (len(dframe)))

# let's now filter out stuff we don't want
dframe = dframe.loc[dframe['isIperfOn'] == 1] # filter out periods when iperf was off
dframe.drop(['isIperfOn'], axis = 1, inplace = True)

# Split up days
dfday1 = dframe[:26130]
dfday2 = dframe[26130:]
dfday1['systime'] = dfday1['systime'] - dfday1.iloc[0]['systime']
dfday2['systime'] = dfday2['systime'] - dfday2.iloc[0]['systime']
#print(dfday1['systime'])
# print(dfday2['systime'])

dframe = dframe.reset_index(drop = True) # creates index column, numbered from 0 to n-1

dframes = []
for df in [dfday1, dfday2]:
    for wifi in ['n', 'ac', 'ad']:
        dframeNew = df[df['wifiType'] == wifi]  # focus on a particular type of wifi
        dframes.append(dframeNew)

#print("After filtering, left with %d rows\n" % (len(dframe)))

#print("Sample summary:")
#with pandas.option_context('display.max_rows', None, 'display.max_columns', None):
  #print(dframe.groupby(['wifiType']).size().reset_index(drop = False, name = 'nr-samples'))
  #print(dframe['systime'])

# In[16]:

# Plots of different wifi types vs datarate over time
import matplotlib.pyplot as plt

for wifi in dframes:
 for xval in ['systime', 'receiverDist', 'receiverSpeed']:
    fig = plt.figure(figsize=(15, 5))
    plt.title(f"802.11{wifi['wifiType'].iloc[0]} experiment for 2019/08/20")
    plt.xlabel(xval)
    plt.ylabel("Mean data rate (Mbps)")
    plt.grid(True, ls="dotted", lw=0.75)
    plt.xlim([min(wifi[xval]), max(wifi[xval])])
    plt.ylim([min(wifi["dataRateMean"]), max(wifi["dataRateMean"]) + 10])
    #line graph if data is temporal, otherwise scatter plot
    if(xval == "systime"):
        plt.plot(wifi[xval], wifi["dataRateMean"])
    else:
        plt.plot(wifi[xval], wifi["dataRateMean"],'o')
    plt.show()

# In[17]:

for df in dframes:
    corMat= df.corr(method="kendall")
    display(corMat.style.background_gradient(cmap="coolwarm", axis=None).set_precision(2))

# In[25]:

import sklearn

dframeNew = dframe[dframe['wifiType'] == 'n']
#print(dframeNew)
dframeNew = dframeNew.drop(['wifiType','senderId', 'receiverId'], axis = 1)

#print(dframeNew)

x = dframeNew.values # the values in the data frame
from sklearn.preprocessing import StandardScaler
x = StandardScaler().fit_transform(x) # subtract mean and
# divide by standard deviation (yields mean=0, variance=1)

from sklearn import decomposition
pca = decomposition.PCA(n_components=0.95) # enough comps to explain 95% var
pca.fit(x)

print("Found", pca.n_components_, "PCA components:", pca.explained_variance_)
print("Explained variance ratios:", pca.explained_variance_ratio_)
print("Total:", sum(pca.explained_variance_ratio_))
print("Components:", pca.components_)

xtrans = pca.transform(x) # apply dimensionality reduction to x
pcaDf = pandas.DataFrame(data = xtrans) # create a data frame out of x

# In[8]:

from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error, r2_score
import matplotlib.pyplot as plt

target = 'dataRateMean' # variable we're trying to predict
for wifiType in ('n', 'ac', 'ad'): # separate data by wifi type
    # filter out unwanted rows
    fdf = dfday1.loc[dfday1['wifiType'] == wifiType]
    featureList = ['receiverDist', 'rssiMean', 'receiverSpeed'] # features of interest
    # replace rssi with snr for ad, if needed
    if wifiType == 'ad' and 'rssiMean' in featureList:
        featureList.remove('rssiMean')
        featureList.append('snrMean')

    featDropList = [] # drop the features we won't be using
    for feature in fdf.columns:
        if feature not in featureList and feature != target:
            featDropList.append(feature)
    fdf.drop(featDropList, axis=1, inplace=True)
    fdf.reset_index(inplace=True, drop=True)
    print("Predicting 802.11%s %s using features: %s" %
    (wifiType, target, featureList))
    # split data set into features and target labels
    x = fdf.drop(target, axis=1) # x contains all the features of interest
    y = fdf[target] # y contains only the target label
    # split data into training and test subsets:
    # - xTrain and yTrain contain features and labels for training
    # - xTest and yTest contain features and labels for testing
    # test_size=0.3 means 30% data for testing
    # random_state=1, is the seed value used by the random number generator
    xTrain, xTest, yTrain, yTest = train_test_split(x, y, test_size=0.3,
    random_state=1)

    regLinr = LinearRegression() # create linear regression model
    regLinr.fit(xTrain, yTrain) # train model
    print("Linear regression, coefficients: \n", regLinr.coef_)
    yPred = regLinr.predict(xTest) # predict on test data
    yPred = yPred.clip(min=0) # data rate can't be negative, so make it >= 0
    print("Mean-Square Error: %.2f" % mean_squared_error(yTest, yPred))
    print("Coefficient of determination: %.2f" %     r2_score(yTest, yPred)) # 1 is perfect prediction
    #plt.scatter(xTest, yTest, color='black')
    #plt.plot(xTest, yPred, color='blue', linewidth=3)
   # plt.xticks(())
    #plt.yticks(())
  #  plt.show()

# In[51]:

from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sn

def classify(fdf):
    # split data set into features and target labels
    x = fdf.drop(target, axis=1) # x contains all the features of interest
    y = fdf[target] # y contains only the target label
    # split data into training and test subsets:
    # - xTrain and yTrain contain features and labels for training
    # - xTest and yTest contain features and labels for testing
    # test_size=0.3 means 30% data for testing
    # random_state=1, is the seed value used by the random number generator
    xTrain, xTest, yTrain, yTest = train_test_split(x, y, test_size=0.3,
    random_state=1)
    print(f"yTrain: {yTrain}")
    clfKnn = KNeighborsClassifier()
    clfKnn.fit(xTrain, yTrain) # train classifier
    yPred = clfKnn.predict(xTest) # predict on the unseen data
    print("Accuracy of k-nearest neighbors:", accuracy_score(yTest, yPred))
    print(classification_report(yTest, yPred))
    # display a confusion matrix
    print("Confusion matrix:")
    cmat = confusion_matrix(yTest, yPred)
    plt.figure(figsize=(6.5, 6.5)) # new figure
    sn.set(font_scale=1.4) # for label size
    sn.heatmap(cmat, annot=True, annot_kws={"size": 13}, fmt='g')
    plt.show()

def exp_binulate(factor, nbins):
    return [0] + [factor**n for n in range(0, nbins - 1)]

target = 'dataRateMean' # variable we're trying to predict
for wifiType in ('n', 'ac', 'ad'): # separate data by wifi type
    fdf = dframe.loc[dframe['wifiType'] == wifiType] # filter out unwanted rows
    featureList = ['rssiMean', 'receiverDist', 'receiverSpeed', 'meanBeaconRssi'] # features of interest
    # replace rssi with snr for ad, if needed
    if wifiType == 'ad' and 'rssiMean' in featureList:
        featureList.remove('rssiMean')
        featureList.append('snrMean')
    featDropList = [] # drop the features we won't be using
    for feature in fdf.columns:
        if feature not in featureList and feature != target:
            featDropList.append(feature)
    fdf.drop(featDropList, axis=1, inplace=True)
    fdf.reset_index(inplace=True, drop=True)
    print("Classifying 802.11%s %s using features: %s\n" %
    (wifiType, target, featureList))
    # discretize target variable
    # start by printing some information about the bins
    nquantiles = 4
    uniBins = pandas.qcut(fdf[target], q=nquantiles, duplicates='drop').unique()
    uniBins = uniBins.ravel() # convert data to an array for sorting
#     uniBins.sort()
    print("Discretizing", target, "using", nquantiles,
    "quantiles, yielding bins:\n", uniBins, "\n")
    target_orig = fdf[target]

    print("TG ", target_orig[1610])

    fdf[target] = pandas.qcut(fdf[target], q=nquantiles, labels=False,
    duplicates='drop')

#     classify(fdf)

    fdf[target] = pandas.cut(target_orig, bins=exp_binulate(2, 20), duplicates='drop')

    print(f"NEW: {fdf[target]}")

    # Error here, negative numbers in target_orig? Some columns NAN after cut?
    classify(fdf)

#     # split data set into features and target labels
#     x = fdf.drop(target, axis=1) # x contains all the features of interest
#     y = fdf[target] # y contains only the target label
#     # split data into training and test subsets:
#     # - xTrain and yTrain contain features and labels for training
#     # - xTest and yTest contain features and labels for testing
#     # test_size=0.3 means 30% data for testing
#     # random_state=1, is the seed value used by the random number generator
#     xTrain, xTest, yTrain, yTest = train_test_split(x, y, test_size=0.3,
#     random_state=1)
#     clfKnn = KNeighborsClassifier()
#     clfKnn.fit(xTrain, yTrain) # train classifier
#     yPred = clfKnn.predict(xTest) # predict on the unseen data
#     print("Accuracy of k-nearest neighbors:", accuracy_score(yTest, yPred))
#     print(classification_report(yTest, yPred))
#     # display a confusion matrix
#     print("Confusion matrix:")
#     cmat = confusion_matrix(yTest, yPred)
#     plt.figure(figsize=(6.5, 6.5)) # new figure
#     sn.set(font_scale=1.4) # for label size
#     sn.heatmap(cmat, annot=True, annot_kws={"size": 13}, fmt='g')
#     plt.show()

# ## NOTES on the Heat Map
# When we run classify over just RSSI mean, the precision, recall, and f1-score are all lower than when we run over distance and speed as well, telling us that these are strong indicators for data rate mean as well.
#
# Without meanBeaconRssi
#            0       0.93      0.93      0.93      1146
#            1       0.74      0.79      0.77      1158
#            2       0.61      0.59      0.60      1179
#            3       0.75      0.73      0.74      1174
#
# The final section requires some correction for NaN values, cleaning the data set/pruning prior to runtime.

# In[ ]:


# __author__: Mai feng
# __file_name__: profile.py
# __time__: 2019:05:01:20:57

from medicine.api_1_0 import api
from flask import jsonify, session, current_app, request, g
from medicine.utils.response_code import RET
from medicine.utils.common import login_required
from medicine.models import User, Expert
from medicine import db, constants

@api.route('users')
@login_required
def get_user_info():
    '''获取用户信息
        前提是已经登陆
    :return: json
    '''
    user_id = g.user_id
    try:
        user = User.query.filter(User.id == user_id).first()
    except Exception as e:
        current_app.logger.debug(e)
        return jsonify(re_code=RET.DBERR, msg='数据库查询错误')

    if not user:
        return jsonify(re_code=RET.NODATA, msg='用户不存在')

    # 查询用户信息
    user_info = user.to_dict()
    return jsonify(re_code=RET.OK, msg='查询成功', user=user_info)

@api.route('experts')
@login_required
def get_expert_info():
    '''获取专家信息
    :return: json
    '''
    user_id = g.user_id
    try:
        user = Expert.query.filter(Expert.id == user_id).first()
    except Exception as e:
        current_app.logger.debug(e)
        return jsonify(re_code=RET.DBERR, msg='数据库查询错误')

    if not user:
        return jsonify(re_code=RET.NODATA, msg='用户不存在')

    # 查询用户信息
    user_info = user.to_dict()
    return jsonify(re_code=RET.OK, msg='查询成功', expert=user_info)


import argparse
import jinja2
import ProtoParser

def main():

    arg_parser = argparse.ArgumentParser(description = 'PacketGenerator')
    arg_parser.add_argument('--path', type=str, default='C:/Users/user/Desktop/Education/Portfolio/GameServer/Server/Common/Protobuf/bin/Protocol.proto', help='proto path')
    arg_parser.add_argument('--output', type=str, default='TestPacketHandler', help='output file')
    arg_parser.add_argument('--recv', type=str, default='CS_', help='recv convention')
    arg_parser.add_argument('--send', type=str, default='SC_', help='send convention')
    args = arg_parser.parse_args()

    parser = ProtoParser.ProtoParser(1000, args.recv, args.send)
    parser.parse_proto(args.path)

    #jinja2
    file_loader = jinja2.FileSystemLoader("Template")
    env = jinja2.Environment(loader = file_loader)

    template = env.get_template('PacketHandler.template')
    output = template.render(parser=parser, output=args.output)

    f = open(args.output+'.h', 'w+')
    f.write(output)
    f.close()

    print(output)

    return

if __name__ == '__main__':
    main()


import datetime

import pandas as pd

class QuandlFetcher(object):
    """
    Modified version of the Quandl Python API that builds the
    url query string given a dataset code or list of dataset codes.

    url is retrieved via the attribute self.url

    parameters

    dataset: string or list
        A single dataset code or list of codes
        Dataset codes are available on the Quandl website

    Optional keyword args

    auth_token: string
        quandl api auth token, calls will be limited without a token

    trim_start: string
        first date in dataset

    trim_end: string
        last date in dataset

    collapse: string
        Options are daily, weekly, monthly, quarterly, annual

    transformation: string
        options are diff, rdiff, cumul, and normalize

    rows: string
        Number of rows which will be returned

    sort_order: string
        options are asc, desc. Default: `asc`

    Other `kwargs` are added to the url string with no interference.

    """

    API_URL = 'http://www.quandl.com/api/v1/'

    def __init__(self, dataset, **kwargs):
        self.url = self.build_url(dataset, **kwargs)

    def _append_query_fields(self, url, **kwargs):
        field_values = ['{0}={1}'.format(key, val)
                        for key, val in kwargs.items() if val]
        return url + 'request_source=python&request_version=2&' +'&'.join(field_values)

    def _parse_dates(self, date):
        if date is None:
            return date
        if isinstance(date, datetime.datetime):
            return date.date().isoformat()
        if isinstance(date, datetime.date):
            return date.isoformat()
        try:
            date = pd.to_datetime(date)
        except ValueError:
            raise ValueError("{} is not recognised a date.".format(date))
        return date.date().isoformat()

    def build_url(self, dataset, **kwargs):

        kwargs.setdefault('sort_order', 'asc')
        auth_token = kwargs.pop('auth_token', None)
        trim_start = self._parse_dates(kwargs.pop('trim_start', None))
        trim_end = self._parse_dates(kwargs.pop('trim_end', None))

        # Check whether dataset is given as a string (for a single dataset) or
        # an array (for a multiset call)

        #Unicode String
        if type(dataset) == unicode or type(dataset) == str:
            url = self.API_URL + 'datasets/{}.csv?'.format(dataset)

        #Array
        elif type(dataset) == list:
            url = self.API_URL + 'multisets.csv?columns='
            #Format for multisets call
            dataset = [d.replace('/', '.') for d in dataset]
            for i in dataset:
                url += i + ','
            #remove trailing ,
            url = url[:-1] + '&'

        #If wrong format
        else:
            error = "Your dataset must either be specified as a string (containing a Quandl code) or an array (of Quandl codes) for multisets"
            raise Exception(error)

        url = self._append_query_fields(
            url,
            auth_token=auth_token,
            trim_start=trim_start,
            trim_end=trim_end,
            **kwargs
        )
        return url


import json

import argparse
from yahoo import Yahoo

from utils import utils, data

def run():
    argparser = argparse.ArgumentParser(description='Yahoo Finance Historical Data')
    argparser.add_argument("-s", "--symbol", help="stock symbol", required=True)
    argparser.add_argument("--save", action='store_true', help="save data")

    args = argparser.parse_args()
    symbol = args.symbol

    yahoo = Yahoo()
    df = yahoo.request_historical_data(symbol, 'max')

    json = df.to_json(orient='index')
    print(json)

    if args.save:
        path = utils.file_path(__file__)
        name = utils.file_name(__file__)

        csv = True
        extension = f'.{"csv" if csv else "pkl"}'

        file = f'{path}/{name}/{symbol}{extension}'
        data.df_write(df, file, verbose=True)

run()


tags = {}

line = None

import requests
import urllib.parse
import time
import difflib
import json
import string
from os import listdir
from unidecode import unidecode
from common import dump_paper_data_to_file
from common import load_paper_data_from_file
from common import Paper
import scholarly
from scholarly import ProxyGenerator

pg = ProxyGenerator()
pg.ScraperAPI("ae3a578dd2aa8db929728c05130c7b8a")
scholarly.scholarly.use_proxy(pg)

total_querys = 0

def to_ascii(s):
	if s == None:
		return s
	return unidecode(s, errors="replace")

def query_paper_by_id(paperID):
	global total_querys
	authors, abstract, venue, year = None, None, None, None

	query = "https://api.semanticscholar.org/graph/v1/paper/{}?fields=title,abstract,authors,venue,year".format(paperID)
	response = requests.get(query)
	total_querys += 1
	# Handle
	if response.status_code != 200:
		raise Exception("Error: Wrong Response Code ({}), Raw: {}\n".format(response.status_code, response.text))

	data = response.json()
	if 'title' not in data:
		raise Exception("Error: Response Format Exception, Raw: {}\n".format(response.text))

	if "venue" in data:
		venue = to_ascii(data['venue'])

	if "abstract" in data:
		abstract = to_ascii(data['abstract'])

	if "authors" in data:
		authors = [to_ascii(author['name']) for author in data['authors']]

	if "year" in data:
		year = data['year']

	time.sleep(2)
	return authors, abstract, venue, year

def query_papers_matching_title(title, tags = None, venue = None, backup_year = None):
	global total_querys
	print("Paper: {}".format(title))
	paper_obj = Paper(title)
	paper_obj.add_tags(tags)
	paper_obj.venue = venue

	title_formatted = urllib.parse.quote(title.strip())
	query = "http://api.semanticscholar.org/graph/v1/paper/search?query={}".format(title_formatted)

	response = requests.get(query)
	total_querys += 1

	# Handle
	if response.status_code != 200:
		print("Error: Wrong Response Code ({}), Raw: {}\n".format(response.status_code, response.text))
		return None

	data = response.json()
	if 'total' not in data:
		print("Error: Response Format Exception, Raw: {}\n".format(response.text))
		return None

	if int(data['total']) == 0:
		print("Error: No paper found, Raw: {}\n".format(response.text))
		return None

	for paper_data in data['data']:
		paperID = paper_data['paperId']
		paperTitle = to_ascii(paper_data['title'])
		if difflib.SequenceMatcher(None,paperTitle,title).ratio() > 0.7:
			try:
				authors, abstract, venue, year = query_paper_by_id(paperID)
			except Exception as e:
				print(e.message)
				return None

			paper_obj.authors = authors
			paper_obj.abstract = abstract
			paper_obj.year = year if year != None else backup_year
			# Venue from scholar is not accurate
			if paper_obj.venue == None or paper_obj.venue == "SSRN":
				paper_obj.venue = venue

			break
	else:
		print("Error: No matching paper found, Raw: {}\n".format(response.text))
		return None

	return paper_obj

def clean_paper_title(line):
	# Paper title
	line = unidecode(line)
	line = line.strip()

	# Formatted with conference info
	bracket_start = line.rindex("(")
	title = line[:bracket_start].strip()
	venue = line[bracket_start:].strip('()').strip().split("'")[0].strip()
	year = int("20"+line[bracket_start:].strip('()').strip().split("'")[-1].strip())
	return title, venue, year

def query_gscholar(title,  tags = None, venue = None, backup_year = None, paper_obj = None):
	try:
		print("Trying Google Scholar")
		pubs = scholarly.scholarly.search_pubs(title)
		best_match = next(pubs)
		if 'bib' in best_match and 'title' in best_match['bib']:
			paperTitle = to_ascii(best_match['bib']['title'])
			if difflib.SequenceMatcher(None,paperTitle,title).ratio() > 0.7:
				bib = best_match['bib']
				authors, abstract, _, year = [to_ascii(author) for author in bib['author']], to_ascii(bib['abstract']),  None, bib['pub_year']
				if paper_obj == None:
					paper_obj = Paper(title)
					paper_obj.add_tags(tags)
					paper_obj.venue = venue
					paper_obj.authors = authors
					paper_obj.abstract = abstract
					paper_obj.year = int(year) if year != "NA" else backup_year
				else:
					paper_obj.abstract = abstract
		else:
			print("Trying Google Scholar No Valid Title, Raw: {}".format(best_match))
			time.sleep(60)
			return None

	except StopIteration as e:
		print("Error trying Google Scholar: {}".format(e))
		time.sleep(60)
		return None

	except Exception as e:
		print("Error trying Google Scholar: {}".format(e.message))
		time.sleep(60)
		return None

	time.sleep(60)
	return paper_obj

def main():
	existing_papers = load_paper_data_from_file()
	existing_papers = set(paper.title for paper in existing_papers)

	with open("../data/raw/doc.md") as f:
		for line in f:
			line = unidecode(line.strip()).strip()

			# Empty
			if len(line) == 0:
				continue

			# Tags
			if line.startswith("#"):
				lvl = line.count('#')
				# print(lvl)
				if int(lvl) >= 3:
					# Clear tags
					if int(lvl) in tags:
						del tags[int(lvl)]
					if int(lvl) + 1 in tags:
						del tags[int(lvl) + 1]

					tags[int(lvl)] = line.strip("#").strip()
					# print(tags)
				continue

			# Unknown
			if not ("(" in line and line.endswith(")") and "'" in line):
				continue

			# Paper
			title, venue, backup_year = clean_paper_title(line)
			#print(title, venue)
			if title in existing_papers:
				print("Paper: {}".format(title))
				print('Data Exist!\n')
				continue

			query_result = query_papers_matching_title(title, list(tags.values()), venue, backup_year)
			if query_result == None or query_result.abstract == None:
				query_result = query_gscholar(title, list(tags.values()), venue, backup_year, query_result)

			# Dump
			if query_result != None:
				dump_paper_data_to_file(query_result)
				print("Success!\n")
			else:
				print('Fail trying both approaches')

			time.sleep(2)

			# Don't overload the server
			if total_querys % 80 == 0:
				time.sleep(600)

if __name__ == "__main__":
	main()


from colorama import Fore, init, Style
init(autoreset = True)

# Kelas untuk membuat Node / simpul
class Node:
	def __init__(self, data):
		self.data = data
		self.next = None

# Kelas untuk membuat linked list (6 fungsi)
class linked_list:
	def __init__(self):
		self.head = None
		self.last = self.head

	# Masukkan data baru di awal linked list
	def add_beg(self):
		Error = True
		while Error is True:
			try:
				jumlah = int(input("Jumlah data yang akan dimasukkan : "))
			except ValueError:
				print(f"{Fore.RED}Hanya masukkan bilangan bulat\n")
			else:
				Error = False
				for x in range(jumlah):
					data = input(f"\nMasukkan data ke - {x+1} untuk disisipkan di awal linked list : ")
					if self.head is None:
						self.head = Node(data)
						self.last = self.head
					else:
						new = Node(data)
						new.next = self.head
						self.head = new
					print(f"{Fore.GREEN}Data '{data}' berhasil dimasukkan")
		menu()

	# Masukkan data baru di akhir linked list
	def add_last(self):
		Error = True
		while Error is True:
			try:
				jumlah = int(input("Jumlah data yang akan dimasukkan : "))
			except ValueError:
				print(f"{Fore.RED}Hanya masukkan bilangan bulat\n")
			else:
				Error = False
				for x in range(jumlah):
					data = input(f"\nMasukkan data ke - {x+1} untuk disisipkan di akhir linked list : ")
					if self.head is None:
						self.head = Node(data)
						self.last = self.head
					else:
						new = Node(data)
						self.last.next = new
						# while search.next is not None:
						# 	search = search.next
						# last_node.pointer = new
						self.last = new
					print(f"{Fore.GREEN}Data '{data}' berhasil dimasukkan")
		menu()

	# Tampilkan linked list
	def print_list(self):
		jumlah = 0
		temp = self.head
		if temp is not None:
			while temp is not None:
				if temp.next is not None:
					print(temp.data, end=" -> ")
				else:
					print(temp.data)
				temp = temp.next
				jumlah += 1
			print(f"\nJumlah elemen = {jumlah}")
		else:
			print(f"{Fore.RED}Linked list kosong")
		menu()

	# Cari data tertentu dalam linked list
	def find_node(self, data):
		temp = self.head
		index = 1
		found = False
		while temp is not None:
			# if temp.data.lower().strip() == data.lower().strip():
			if temp.data == data:
				found = True
				break
			else:
				temp = temp.next
				index += 1
		if found is True:
			# print(f"{Fore.GREEN}data '{temp.data.strip()}' ditemukan pada urutan ke - {index}")
			print(f"{Fore.GREEN}data '{temp.data}' ditemukan pada urutan ke - {index}")
		else:
			# print(f"{Fore.RED}data '{data.strip()}' tidak ditemukan")
			print(f"{Fore.RED}data '{data}' tidak ditemukan")
		return(found)

	# Menghapus Node atau data dalam linked list
	def delete_node(self):
		if self.head is None:
			print(f"{Fore.RED}Linked list kosong")
		else:
			data = input("Masukkan data yang ingin dihapus : ")
			if linked_list.find_node(self, data) is True:
				temp = self.head
				Found = False
				while temp is not None:
					# if temp.data.lower().strip() == data.lower().strip():
					if temp.data == data:
						self.head = temp.next
						found = True
						break
					# elif temp.next.data.lower().strip() == data.lower().strip():
					elif temp.next.data == data:
						if temp.next == self.last:
							self.last = temp
						temp.next = temp.next.next
						found = True
						break
					else:
						temp = temp.next
				if found is True:
					print(f"{Fore.GREEN}Data '{data}' berhasil dihapus")
				# else:
				# 	print(f"{Fore.RED}Data '{data}' tidak ditemukan")
		menu()

	def DELETE(self):
		temp = self.head
		while(temp is not None):
			temp = temp.next
			del self.head
			self.head = temp

# Fungsi untuk print menu
def menu():
	print("\n^^^ Pilihan Menu ^^^")
	print("1. Tambahkan data di akhir")
	print("2. Tambahkan data di awal")
	print("3. Print linked list")
	print("4. Cari data / elemen")
	print("5. Hapus data / elemen")
	print("6. Kosongkan linked list")
	print("7. Keluar")


import boto3,time

client = boto3.client('ec2')

response = client.describe_instances()
GroupIds = []
for Dict in response['Reservations']:
    for Instances in Dict['Instances']:
        for SecurityGroups in Instances['SecurityGroups']:
            GroupIds.append(SecurityGroups['GroupId'])

for item in GroupIds:
    print(f"Checking the security group {item} for open port 22")
    time.sleep(2)
    response = client.describe_security_groups(
        GroupIds=[
            item
        ]
    )
    for SGs in response['SecurityGroups']:
        if SGs['IpPermissions']:
            for IpInfo in SGs['IpPermissions']:
                if IpInfo['FromPort'] == 22:
                    for IP in IpInfo['IpRanges']:
                        if IP['CidrIp'] == '0.0.0.0/0':
                            print(f"{item}'s port 22 is open to the internet!")
                            print("Revoking port 22 access")
                            response = client.revoke_security_group_ingress(
                                CidrIp='0.0.0.0/0',
                                FromPort=22,
                                GroupId=item,
                                IpProtocol='tcp',
                                ToPort=22,
                            )
                            print()
                        else:
                            print(f"{item} has a port 22 rule set up, but it is not open to the internet")
                            print()
                else:
                    print(f"{item} does not have any port 22 rules set up")
                    print()
        else:
            print(f"{item} does not have any ingress rules")
            print()


import ast
import datetime
import json
import os
import random
import time
import re
from getpass import getpass
import openai

import numpy as np
import streamlit
import torch as torch
from omegaconf import DictConfig, OmegaConf
from transformers import AutoTokenizer, AutoModelForCausalLM
from openelm.codegen.codegen_utilities import set_seed
from openelm.mutation_model import PromptModel

from architext_genotype import ArchitextGenotype
from concurrent.futures import ThreadPoolExecutor
from util import base_folder

max_json = re.compile(r"\{[\d\D]*\}")

class ArchitextPromptMutation(PromptModel):
    """
    Generating hf outputs on the local machine.
    """

    room_labels = ['bedroom1', 'kitchen', 'living_room', 'corridor', 'bathroom1']

    def __init__(self, cfg, prompts: list[str], default_height=2.0):
        """
        Args:
            cfg: the config dict.
            prompts: a list of default prompts
        """
        if isinstance(cfg, str):
            self.cfg = OmegaConf.load(cfg)
        elif isinstance(cfg, (dict, DictConfig)):
            self.cfg = DictConfig(cfg)
        else:
            raise ValueError

        set_seed(self.cfg.seed)
        # Use RNG to rotate random seeds during inference.
        self.rng = np.random.default_rng(seed=self.cfg.seed)
        self.prompts = prompts
        self.default_height = default_height

        self.batch_size = self.cfg.batch_size
        self.device = torch.device("cuda" if torch.cuda.is_available() and self.cfg.cuda else "cpu")

        # Set up the tokenizer
        self.tokenizer = AutoTokenizer.from_pretrained(self.cfg.model)
        self.tokenizer.padding_side = "left"
        self.tokenizer.pad_token = self.cfg.pad_token

        # Set up the model
        # TODO: fix data parallel
        if self.cfg.gpus > 1:
            self.model = torch.nn.DataParallel(
                AutoModelForCausalLM.from_pretrained(self.cfg.model),
                device_ids=list(range(self.cfg.gpus))
            ).to(self.device)
            self.model.generate = self.model.module.generate
        else:
            self.model = AutoModelForCausalLM.from_pretrained(self.cfg.model).to(self.device)

    def mutate_genotypes(self, inputs: list[ArchitextGenotype], **kwargs) -> list[ArchitextGenotype]:
        """
        This class does not use codes as intermediate representation. To fit into the genotype format, we output
        a dict with `program_str` (== `result_obj`) being the string describing a floor plan using coordinates.

        Args:
            inputs: the list of input genotypes.
        Returns:
            a list of mutated prompts.
        """
        prompts = []
        for pd in inputs:
            if pd is None:
                # Random generation
                mutated_prompt = random.choice(self.prompts)
            else:
                # Mutate the design string
                lines = pd.to_design_string().split(', ')
                random_prompt = random.choice(self.prompts)
                cut_off = np.random.randint(1, 3, size=1)[0]
                cut_off = min(cut_off, len(lines) - 1)
                mutated_prompt = random_prompt + ' ' + ', '.join(lines[1:cut_off + 1]) + ", " + random.choice(
                    self.room_labels) + ":"
            prompts.append(mutated_prompt)

        completion = self.model.generate(**self.tokenizer(prompts,
                                                          return_tensors="pt",
                                                          padding=True,
                                                          truncation=True).to(self.device),
                                         num_beams=self.cfg.num_beams,
                                         num_return_sequences=1,
                                         max_length=self.cfg.gen_max_len,
                                         pad_token_id=50256,
                                         do_sample=True,
                                         **kwargs)

        mutated_design_str = self.tokenizer.batch_decode(completion)
        mutated_designs = [
            ArchitextGenotype(design_string=elem,
                              height=self.default_height if parent is None else parent.height,
                              parent=parent) for elem, parent in zip(mutated_design_str, inputs)]
        return mutated_designs

class ArchitextChatGPTMutation(PromptModel):
    """
    This prompt mutation calls GPT-3.5 API to generate designs
    """

    def __init__(self, cfg, prompts: list[str],
                 seed_designs: list[dict] | None = None, default_height=2.0,
                 count_tokens: bool = False, **kwargs):
        """
        Args:
            cfg: the config dict.
            prompts: a list of default prompts
        """
        if isinstance(cfg, str):
            self.cfg = OmegaConf.load(cfg)
        elif isinstance(cfg, (dict, DictConfig)):
            self.cfg = DictConfig(cfg)
        else:
            raise ValueError

        self.prompts = prompts
        self.default_height = default_height
        if seed_designs is None:
            try:
                with open("seed_designs.json", "r") as f:
                    seed_designs = ast.literal_eval(f.read())
            except:
                raise ValueError("Please provide a list of seed designs.")
        self.seed_designs = seed_designs

        if 'OPENAI_API_KEY' in os.environ:
            openai.api_key = os.environ['OPENAI_API_KEY']
        else:
            openai.api_key = getpass('Enter your OpenAI API key:')

        self.count_tokens = count_tokens
        if self.count_tokens:
            import streamlit

    def mutate_genotypes(self, genotypes: list[ArchitextGenotype], num_threads=10, **kwargs) -> list[ArchitextGenotype]:
        args = []
        for x in genotypes:
            example = random.choice(self.seed_designs) if x is None else x.to_dict()
            prompt = random.choice(self.prompts)
            args.append((example, prompt))

        with ThreadPoolExecutor(num_threads) as pool:
            results = list(pool.map(self._get_completion, args))
        mutated_json, usages = [r[0] for r in results], [r[1] for r in results]
        if self.count_tokens:
            for usage in usages:
                streamlit.session_state["prompt_tokens"] = \
                    streamlit.session_state.get("prompt_tokens", 0) + usage.get("prompt_tokens", 0)
                streamlit.session_state["tokens"] = \
                    streamlit.session_state.get("tokens", 0) + usage.get("total_tokens", 0)

        self._log_returns(mutated_json)

        mutated_genotypes = []
        for x, parent in zip(mutated_json, genotypes):
            if x is None:
                mutated_genotypes.append(None)
            else:
                assert isinstance(x, str)
                try:
                    new_dict = max_json.search(x).group(0)
                    new_dict = ast.literal_eval(new_dict)
                    mutated_genotypes.append(
                        ArchitextGenotype.from_dict(new_dict,
                                                    height=self.default_height if parent is None else parent.height,
                                                    parent=parent)
                    )
                except Exception as e:
                    with open("logs.txt", "a") as f:
                        timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                        f.write(f"[{timestamp} ERROR]: Parsing error in the return of GPT-3.5.\n"
                                f"Returned string: {str(x)}\n"
                                f"Exception: {str(e)}\n"
                                )
                    mutated_genotypes.append(None)

        return mutated_genotypes

    @staticmethod
    def _get_completion(args) -> tuple[str | None, dict]:
        """
        The inner loop of getting new design json from GPT-3.5 API.
        args: a tuple of (example, prompt)
            where example: dict, prompt: str
        """
        example, prompt = args

        requirements = "1. The prompt is all the info you have. The design detailed in `layout` follows the prompt as "\
                       "closely as possible.\n" \
                       "2. Different rooms cannot overlap.\n" \
                       "3. The room names should start with one of 'living_room', 'kitchen', 'bedroom', 'bathroom', " \
                       "'corridor'.\n" \
                       "4. Number of bathroom <= Number of bedroom <= 4.\n" \
                       "5. The return must be a valid JSON document.\n"
        sys_msg = "You are a REST API server receiving prompts describing a floor plan. You only return JSON " \
                  "documents describing your design. The format is the following:\n"\
                  "1. `prompt`: the original input prompt, \n"\
                  "2. `layout`: the room-by-room details of the floor plan in terms of the coordinates.\n"\
                  "An example is the following:\n" \
                  "```\n" + str(example) + "\n```\n" \
                  "In the `layout` field, each room is represented as a list of coordinates defining a polygon.\n" \
                  "The returned JSON document must satisfy " \
                  "the following requirements:\n" + requirements
        for i in range(5):
            try:
                completion = openai.ChatCompletion.create(
                    model="gpt-3.5-turbo",
                    messages=[{"role": "system", "content": sys_msg}, {"role": "user", "content": prompt}]
                )

                return completion["choices"][0]["message"]["content"], completion["usage"]
            except:
                time.sleep(60)
                if i == 4:
                    print("Warning: Calling OpenAI API failed for 5 consecutive times. Returning None. "
                          "See `logs.txt` for details.")
                    with open("logs.txt", "a") as f:
                        timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                        f.write(f"[{timestamp} ERROR]: Calling OAI API failed for 5 consecutive times.\n"
                                "Prompt: " + prompt + "\n"
                                                      "Example: " + str(example) + "\n")
                return None, {}

    @staticmethod
    def _log_returns(mutated_json):
        """
        Log the returned jsons to a file.
        """
        with open("returned.txt", "a") as f:
            timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            f.write(f"[{timestamp} INFO]: Returned JSONs: " + str(mutated_json) + "\n")

def build_default_mutation_model(key: str, cfg: DictConfig, count_tokens: bool = True):
    """
    A utility function that builds a default version of each mutation model class.

    Args:
        key (str): A key that defines the mutation model.
        cfg (DictConfig): A configuration object.
        count_tokens (bool): Whether to count the number of tokens in the generated text.
            Note: if True, it will import streamlit and count OpenAI API tokens using
            streamlit.session_state['tokens'].

    Returns:
        MutationModel: A mutation model.
    """
    with open(str(base_folder / 'prompts.txt'), 'r') as f:
        prompts = [p.strip() for p in f.read().split('\n') if p.strip()]
    aug_prompts = ['[prompt] ' + prompt.rstrip() + ' [layout]' for prompt in prompts]

    with open(str(base_folder / "seed_designs.json"), "r") as f:
        seed_designs = json.load(f)

    if key == "gpt-j":
        return ArchitextPromptMutation(cfg,
                                       prompts=aug_prompts,
                                       default_height=cfg.height)

    elif key == "chatgpt":
        return ArchitextChatGPTMutation(cfg,
                                        prompts=prompts,
                                        seed_designs=seed_designs,
                                        default_height=cfg.height,
                                        count_tokens=count_tokens)


import requests
import json
import time
from dataclasses import dataclass
from kafka import KafkaProducer
from datetime import datetime

@dataclass
class BikeSensor5Min:
    ts: str
    id: int
    name: str
    count: int
    time: str
    coordinateType: str
    coordinates: list

# class BikeSensor5Min:

def get_batch_data_5_min():
    # while True:
    resp = requests.get("https://iot.hamburg.de/v1.1/Things?$filter=Datastreams/properties/serviceName%20eq%20%27HH_STA_HamburgerRadzaehlnetz%27%20and%20Datastreams/properties/layerName%20eq%20%27Anzahl_Fahrraeder_Zaehlfeld_5-Min%27&$count=true&$expand=Datastreams($filter=properties/layerName%20eq%20%27Anzahl_Fahrraeder_Zaehlfeld_5-Min%27;$expand=Observations($orderby=phenomenonTime%20desc))")
    # resp = requests.get("https://iot.hamburg.de/v1.1/Things?$filter=Datastreams/properties/serviceName%20eq%20%27HH_STA_HamburgerRadzaehlnetz%27%20and%20Datastreams/properties/layerName%20eq%20%27Anzahl_Fahrraeder_Zaehlstelle_15-Min%27&$count=true&$expand=Datastreams($filter=properties/layerName%20eq%20%27Anzahl_Fahrraeder_Zaehlstelle_15-Min%27;$expand=Observations($top=10;$orderby=phenomenonTime%20desc))")
    response = json.loads(resp.content)
    done = False
    cnt = 0
    sensors = []
    phenomenaTime = ""
    epoch_time = datetime(1970, 1, 1)
    while(not done):
        if resp.status_code == 200:
            for feature in response['value']:
                for stream in feature['Datastreams']:
                    batch_link = 'https://iot.hamburg.de/v1.1/Datastreams(' + str(stream['@iot.id']) + ')/Observations?$skip=0&$orderby=phenomenonTime+desc'

                    f = open("safestate.txt", "r")
                    if str(stream['@iot.id']) in f.read():
                        print("skip")
                        continue

                    batch_resp = requests.get(batch_link, timeout=30)
                    batch_response = json.loads(batch_resp.content)

                    batch_done = False
                    if batch_resp.status_code == 200:
                        while not batch_done:
                            for result_value in batch_response['value']:
                                date = datetime.strptime(result_value['phenomenonTime'].split('/')[0].split('Z')[0], "%Y-%m-%dT%H:%M:%S.%f")
                                bikesensor = BikeSensor5Min((date - epoch_time).total_seconds, feature['@iot.id'], feature['name'], result_value['result'], result_value['phenomenonTime'], stream['observedArea']['type'], stream['observedArea']['coordinates'])
                                sensors.append(bikesensor)
                                date = datetime.strptime(result_value['phenomenonTime'].split('/')[0].split('Z')[0], "%Y-%m-%dT%H:%M:%S.%f")
                                cnt += 1
                            if cnt %  10000 == 0:
                                print(cnt)
                            if '@iot.nextLink' not in batch_response:
                                batch_done = True
                            else:
                                try:
                                    batch_done = True
                                    batch_resp = requests.get(batch_response['@iot.nextLink'], timeout=30)
                                except requests.exceptions.Timeout:
                                    print("TIMEOUT")

                                batch_response = json.loads(batch_resp.content)
                    else:
                        print("BATCHREQUEST NOT OK")

                    data_dict = [sensor.__dict__ for sensor in sensors]
                    sensors = []
                    # print(data_dict)
                    producer = KafkaProducer(bootstrap_servers='localhost:9092')
                    topic = 'bikes_5_min'

                    for bike_data in data_dict:
                        # pass
                        # send every hour data as single message to kafka
                        json_data = json.dumps(bike_data)
                        # print(json_data)
                        future = producer.send(topic, bike_data)

                    f = open("safestate.txt", "a")
                    f.write(str(stream['@iot.id']) + " \n")
                    f.close()
        else:
            print("REQUEST NOT OK")
        if '@iot.nextLink' not in response:
            done = True
        else:
            done = True
            resp = requests.get(response['@iot.nextLink'])
            response = json.loads(resp.content)

    print("cnt ",cnt)
    # data_dict = [sensor.__dict__ for sensor in sensors]
    # # print(data_dict)
    # producer = KafkaProducer(bootstrap_servers='localhost:9092')
    # topic = 'bikes_5_min'

    # for bike_data in data_dict:
    #     # pass
    #     # send every hour data as single message to kafka
    #     json_data = json.dumps(bike_data)
    #     # print(json_data)
    #     future = producer.send(topic, bike_data)

if __name__ == "__main__":
    get_batch_data_5_min()
        # time.sleep(60 * 5)


# LabelFrame is the combination of label and frame

from tkinter import *
from tkinter import filedialog # -> filedialog is used here to ask user where to download & store the video
from tkinter.ttk import *
from tkinter.messagebox import * # -> messagebox is used to display info,error messages or warnings
from pytube import YouTube   # -> pytube is the module which helps in downloading Youtube video

class finalDownloader:
    def __init__(self):
        self.root = Tk()
        self.root.geometry('600x600')
        self.root.title('YouTube Video Downloader')
        self.lb = LabelFrame(self.root, text='Choose Video', height=550, width=550)
        self.f = Frame(self.lb)
        self.f.pack(pady=10)
        Label(self.f, text='Enter URL', font=('arial',12)).grid(row=0, column=0, padx=5, pady=20)
        self.searchBar = Entry(self.f, width=45)
        self.searchBar.grid(row=0, column=1, padx=5, pady=20)
        Button(self.f, text='Search', command=self.stepTwo).grid(row=0, column=2, padx=5, pady=20)
        self.lb.pack(pady=20, padx=20, fill=BOTH)
        self.root.mainloop()

    def stepTwo(self):
        self.yt = YouTube(self.searchBar.get(), on_progress_callback=self.makeProgress)
        avail_res = self.get_resolutions()
        self.label = Label(self.lb, text=f"Title :- {self.yt.title}", font=('arial',20))
        self.label.pack(pady=10)
        self.f1 = Frame(self.lb)
        Label(self.f1, text='Select Resolution', font=('arial',12)).grid(row=0, column=0, padx=5, pady=20)
        self.selected_res = Combobox(self.f1, width=42, values=avail_res, state='readonly')
        self.selected_res.grid(row=0, column=1, padx=5, pady=20)
        Button(self.f1, text='Download', command=self.downloadVideo).grid(row=0, column=2, padx=5, pady=20)
        self.f1.pack(pady=10)

    def get_resolutions(self):
        # yt = YouTube(self.searchBar.get())
        all_res = ['144p','240p','360p','480p','720p','1080p','2160p']
        x = []
        for i in all_res:
            a = self.yt.streams.filter(res=i, type='video', progressive=True)
            print(a)
            if len(a)>0:
                x.append(i)
        return x

    def downloadVideo(self):
        location = filedialog.askdirectory()
        self.progressbar = Progressbar(self.lb, orient = HORIZONTAL,length = 300, mode = 'determinate')
        self.progressbar.pack(pady=20)
        # self.label1 = Label()
        self.selected_stream = self.yt.streams.filter(res=self.selected_res.get(), type='video', progressive=True).first()
        self.selected_stream.download(output_path=location)
        showinfo('','Video Downloaded Successfully')
        self.f1.destroy()
        self.label.destroy()
        self.searchBar.delete(0,END)
        self.progressbar.destroy()

    def makeProgress(self, chunk,file_handle, bytes_remaining):
        size = self.selected_stream.filesize

        progress = float(abs(bytes_remaining - size)/size) * float(100)
        print(progress)
        self.progressbar['value'] = int(progress)
        # self.label1.config(text=f'{progress}%')
        self.root.update_idletasks()

finalDownloader()


###Q3 final
# sc.stop()
from pyspark import SparkContext
sc = SparkContext(appName="exercise")

# read csv file
temperature_file=sc.textFile("temperature-readings-tiny.csv")
lines = temperature_file.map(lambda line:line.split(";"))

# map necessary keys and values
# (year-month, temperature), stationNumber
year_temperature = lines.map(lambda x:(x[1][0:7],float(x[3]),x[0]))
year_temperature = year_temperature.filter(lambda x: int(x[0][0:4])>=1960 and int(x[0][0:4])<=2014)

# (year-month, stationNumber), #readings
countSource = year_temperature.map(lambda x:((x[0],x[2]),1))
counts=countSource.reduceByKey(lambda x,y: x+y)
# print(*unique.take(10), sep="\n")

# (year-month, stationNumber), tempSum
temperature = year_temperature.map(lambda x:((x[0],x[2]),x[1]))
tempSum=temperature.reduceByKey(lambda v1,v2:v1+v2)
# print(*tempSum.take(10), sep="\n")

# (year, month, stationNumber, TempAve)
aveSource=tempSum.join(counts)
aveTemp=aveSource.map(lambda x:(x[0][0][0:4],x[0][0][5:7],x[0][1],round(x[1][0]/x[1][1],2)))
# print(*aveTemp.take(30),sep="\n")

averagetemp.saveAsTextFile("aveTemp")


from PyQt5 import Qt, QtCore
from PyQt5.QtGui import QColor, QStandardItemModel, QStandardItem
from PyQt5.QtSql import QSqlQueryModel, QSqlQuery, QSql
from PyQt5.QtWidgets import QTableView, QAbstractSlider
from PyQt5.QtCore import QItemSelectionModel, pyqtSignal, QEvent
import time
import math

from PyQt5.QtCore import QCoreApplication as QC

class GenericTableModel(QStandardItemModel):
    rowCountChanged = pyqtSignal()

    db = None
    tableName = ""
    # total row count which must de displayed in the view
    totalRowCount = 0
    #
    lastColumnCount = 0

    # original query string before we modify it
    origQueryStr = QSqlQuery()
    # previous original query string; used to check if the query has changed
    prevQueryStr = ''
    # modified query object
    realQuery = QSqlQuery()

    items = []
    lastItems = []

    def __init__(self, tableName, headerLabels):
        self.tableName = tableName
        self.headerLabels = headerLabels
        self.lastColumnCount = len(self.headerLabels)
        QStandardItemModel.__init__(self, 0, self.lastColumnCount)
        self.setHorizontalHeaderLabels(self.headerLabels)

    #Some QSqlQueryModel methods must be mimiced so that this class can serve as a drop-in replacement
    #mimic QSqlQueryModel.query()
    def query(self):
        return self

    #mimic QSqlQueryModel.query().lastQuery()
    def lastQuery(self):
        return self.origQueryStr

    #mimic QSqlQueryModel.query().lastError()
    def lastError(self):
        return self.realQuery.lastError()

    #mimic QSqlQueryModel.clear()
    def clear(self):
        pass

    def data(self, index, role=QtCore.Qt.DisplayRole):
        if role == QtCore.Qt.DisplayRole:
            items_count = len(self.items)
            if index.isValid() and items_count > 0 and index.row() < items_count:
                return self.items[index.row()][index.column()]
        return QStandardItemModel.data(self, index, role)

    # set columns based on query's fields
    def setModelColumns(self, newColumns):
        self.headerLabels = []
        self.removeColumns(0, self.lastColumnCount)
        self.setHorizontalHeaderLabels(self.headerLabels)
        for col in range(0, newColumns):
            self.headerLabels.append(self.realQuery.record().fieldName(col))
        self.lastColumnCount = newColumns
        self.setHorizontalHeaderLabels(self.headerLabels)
        self.setColumnCount(len(self.headerLabels))

    def setQuery(self, q, db):
        self.origQueryStr = q
        self.db = db
        #print("q:", q)

        if self.prevQueryStr != self.origQueryStr:
            self.realQuery = QSqlQuery(q, db)

        self.realQuery.exec_()
        self.realQuery.last()

        queryRows = max(0, self.realQuery.at()+1)
        self.totalRowCount = queryRows
        self.setRowCount(self.totalRowCount)

        # update view's columns
        queryColumns = self.realQuery.record().count()
        if queryColumns != self.lastColumnCount:
            self.setModelColumns(queryColumns)

        self.prevQueryStr = self.origQueryStr
        self.rowCountChanged.emit()

    def nextRecord(self, offset):
        cur_pos = self.realQuery.at()
        q.seek(max(cur_pos, cur_pos+offset))

    def prevRecord(self, offset):
        cur_pos = self.realQuery.at()
        q.seek(min(cur_pos, cur_pos-offset))

    # refresh the viewport with data from the db.
    def refreshViewport(self, scrollValue, maxRowsInViewport, force=False):
        # set records position to last, in order to get correctly the number of
        # rows.
        self.realQuery.last()
        rowsFound = max(0, self.realQuery.at()+1)
        if scrollValue == 0 or self.realQuery.at() == QSql.BeforeFirstRow:
            self.realQuery.seek(QSql.BeforeFirstRow)
        elif self.realQuery.at() == QSql.AfterLastRow:
            self.realQuery.seek(rowsFound - maxRowsInViewport)
        else:
            self.realQuery.seek(min(scrollValue-1, self.realQuery.at()))

        upperBound = min(maxRowsInViewport, rowsFound)
        self.setRowCount(self.totalRowCount)

        # only visible rows will be filled with data, and only if we're not
        # updating the viewport already.
        if upperBound > 0 or self.realQuery.at() < 0:
            self.fillRows(self.realQuery, upperBound, force)

    def fillRows(self, q, upperBound, force=False):
        rowsLabels = []
        self.setVerticalHeaderLabels(rowsLabels)

        self.items = []
        cols = []
        self.blockSignals(True)
        #don't trigger setItem's signals for each cell, instead emit dataChanged for all cells
        for x in range(0, upperBound):
            q.next()
            if q.at() < 0:
                # if we don't set query to a valid record here, it gets stucked
                # forever at -2/-1.
                q.seek(upperBound)
                break
            rowsLabels.append(str(q.at()+1))
            cols = []
            for col in range(0, len(self.headerLabels)):
                cols.append(str(q.value(col)))

            self.items.append(cols)
        self.blockSignals(False)

        self.setVerticalHeaderLabels(rowsLabels)
        if self.lastItems != self.items or force == True:
            self.dataChanged.emit(self.createIndex(0,0), self.createIndex(upperBound, len(self.headerLabels)))
        self.lastItems = self.items
        del cols

    def dumpRows(self):
        rows = []
        q = QSqlQuery(self.db)
        q.exec(self.origQueryStr)
        q.seek(QSql.BeforeFirstRow)
        while True:
            q.next()
            if q.at() == QSql.AfterLastRow:
                break
            row = []
            for col in range(0, len(self.headerLabels)):
                row.append(q.value(col))
            rows.append(row)
        return rows

class GenericTableView(QTableView):
    # how many rows can potentially be displayed in viewport
    # the actual number of rows currently displayed may be less than this
    maxRowsInViewport = 0
    vScrollBar = None

    def __init__(self, parent):
        QTableView.__init__(self, parent)
        #eventFilter to catch key up/down events and wheel events
        self.installEventFilter(self)
        self.verticalHeader().setVisible(True)
        self.horizontalHeader().setDefaultAlignment(QtCore.Qt.AlignCenter)
        self.horizontalHeader().setStretchLastSection(True)
        #the built-in vertical scrollBar of this view is always off
        self.setVerticalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff)

    def setVerticalScrollBar(self, vScrollBar):
        self.vScrollBar = vScrollBar
        self.vScrollBar.valueChanged.connect(self.onValueChanged)
        self.vScrollBar.setVisible(False)

    def setModel(self, model):
        super().setModel(model)
        model.rowCountChanged.connect(self.onRowCountChanged)
        model.rowsInserted.connect(self.onRowsInsertedOrRemoved)
        model.rowsRemoved.connect(self.onRowsInsertedOrRemoved)
        self.horizontalHeader().sortIndicatorChanged.disconnect()
        self.setSortingEnabled(False)

    # model().rowCount() is always <= self.maxRowsInViewport
    # stretch the bottom row; we don't want partial-height rows at the bottom
    # this will only trigger if rowCount value was changed
    def onRowsInsertedOrRemoved(self, parent, start, end):
        if self.model().rowCount() == self.maxRowsInViewport:
            self.verticalHeader().setStretchLastSection(True)
        else:
            self.verticalHeader().setStretchLastSection(False)

    def resizeEvent(self, event):
        super().resizeEvent(event)
        #refresh the viewport data based on new geometry
        self.refresh()

    def refresh(self):
        self.calculateRowsInViewport()
        self.model().setRowCount(min(self.maxRowsInViewport, self.model().totalRowCount))
        self.model().refreshViewport(self.vScrollBar.value(), self.maxRowsInViewport, force=True)

    def calculateRowsInViewport(self):
        rowHeight = self.verticalHeader().defaultSectionSize()
        columnSize = self.horizontalHeader().defaultSectionSize()
        # we don't want partial-height rows in viewport, hence .floor()
        self.maxRowsInViewport = math.floor(self.viewport().height() / rowHeight)+1

    def onValueChanged(self, vSBNewValue):
        self.model().refreshViewport(vSBNewValue, self.maxRowsInViewport, force=True)

    def onRowCountChanged(self):
        scrollBar = self.vScrollBar

        totalCount = self.model().totalRowCount
        scrollBar.setVisible(True if totalCount > self.maxRowsInViewport else False)

        scrollBar.setMinimum(0)
        # we need to substract the displayed rows to the total rows, to scroll
        # down correctly.
        scrollBar.setMaximum(max(0, totalCount - self.maxRowsInViewport+1))

        self.model().refreshViewport(scrollBar.value(), self.maxRowsInViewport)

    def getCurrentIndex(self):
        return self.selectionModel().currentIndex().internalId()

    def selectItem(self, _data, _column):
        """Select a row based on the data displayed on the given column.
        """
        items = self.model().findItems(_data, column=_column)
        if len(items) > 0:
            self.selectionModel().setCurrentIndex(
                items[0].index(),
                QItemSelectionModel.Rows | QItemSelectionModel.SelectCurrent
            )

    def _selectLastRow(self):
        internalId = self.getCurrentIndex()
        idx = self.model().createIndex(self.maxRowsInViewport-2, 0, internalId)
        self.selectionModel().setCurrentIndex(
            idx, QItemSelectionModel.Rows | QItemSelectionModel.SelectCurrent
        )

    def _selectRow(self, pos):
        internalId = self.getCurrentIndex()
        self.selectionModel().setCurrentIndex(
            self.model().createIndex(pos, 1, internalId), QItemSelectionModel.Rows | QItemSelectionModel.SelectCurrent
        )

    def onKeyUp(self):
        if self.selectionModel().currentIndex().row() == 0:
            self.vScrollBar.setValue(max(0, self.vScrollBar.value() - 1))

    def onKeyDown(self):
        if self.vScrollBar.isVisible() == False:
            return

        if self.selectionModel().currentIndex().row() >= self.maxRowsInViewport-2:
            self.vScrollBar.setValue(self.vScrollBar.value() + 1)
            self._selectLastRow()

    def onKeyHome(self):
        self.vScrollBar.setValue(0)
        self.selectionModel().setCurrentIndex(self.model().createIndex(0, 0), QItemSelectionModel.Rows | QItemSelectionModel.SelectCurrent)

    def onKeyEnd(self):
        self.vScrollBar.setValue(self.vScrollBar.maximum())
        self._selectLastRow()

    def onKeyPageUp(self):
        newValue = max(0, self.vScrollBar.value() - self.maxRowsInViewport)
        self.vScrollBar.setValue(newValue)

    def onKeyPageDown(self):
        if self.vScrollBar.isVisible() == False:
            return

        newValue = self.vScrollBar.value() + (self.maxRowsInViewport-2)
        if newValue >= self.model().rowCount():
            self._selectLastRow()
            return

        if newValue < self.model().rowCount():
            self.vScrollBar.setValue(newValue)
            self._selectRow(0)

    def eventFilter(self, obj, event):
        if event.type() == QEvent.KeyPress:
            # FIXME: setValue() does not update the scrollbars correctly in
            # some pyqt versions.
            if event.key() == QtCore.Qt.Key_Up:
                self.onKeyUp()
            elif event.key() == QtCore.Qt.Key_Down:
                self.onKeyDown()
            elif event.key() == QtCore.Qt.Key_Home:
                self.onKeyHome()
            elif event.key() == QtCore.Qt.Key_End:
                self.onKeyEnd()
            elif event.key() == QtCore.Qt.Key_PageUp:
                self.onKeyPageUp()
            elif event.key() == QtCore.Qt.Key_PageDown:
                self.onKeyPageDown()
        elif event.type() == QEvent.Wheel:
            self.vScrollBar.wheelEvent(event)
            return True
        #return False
        return super(GenericTableView, self).eventFilter(obj, event)


import base64
import hashlib
import hmac
import struct

import thirdparty.dns.exception
import thirdparty.dns.rdataclass
import thirdparty.dns.name
import thirdparty.dns.rcode

class BadTime(thirdparty.dns.exception.DNSException):

    """The current time is not within the TSIG's validity time."""

class BadSignature(thirdparty.dns.exception.DNSException):

    """The TSIG signature fails to verify."""

class BadKey(thirdparty.dns.exception.DNSException):

    """The TSIG record owner name does not match the key."""

class BadAlgorithm(thirdparty.dns.exception.DNSException):

    """The TSIG algorithm does not match the key."""

class PeerError(thirdparty.dns.exception.DNSException):

    """Base class for all TSIG errors generated by the remote peer"""

class PeerBadKey(PeerError):

    """The peer didn't know the key we used"""

class PeerBadSignature(PeerError):

    """The peer didn't like the signature we sent"""

class PeerBadTime(PeerError):

    """The peer didn't like the time we sent"""

class PeerBadTruncation(PeerError):

    """The peer didn't like amount of truncation in the TSIG we sent"""

# TSIG Algorithms

HMAC_MD5 = thirdparty.dns.name.from_text("HMAC-MD5.SIG-ALG.REG.INT")
HMAC_SHA1 = thirdparty.dns.name.from_text("hmac-sha1")
HMAC_SHA224 = thirdparty.dns.name.from_text("hmac-sha224")
HMAC_SHA256 = thirdparty.dns.name.from_text("hmac-sha256")
HMAC_SHA384 = thirdparty.dns.name.from_text("hmac-sha384")
HMAC_SHA512 = thirdparty.dns.name.from_text("hmac-sha512")

_hashes = {
    HMAC_SHA224: hashlib.sha224,
    HMAC_SHA256: hashlib.sha256,
    HMAC_SHA384: hashlib.sha384,
    HMAC_SHA512: hashlib.sha512,
    HMAC_SHA1: hashlib.sha1,
    HMAC_MD5: hashlib.md5,
}

default_algorithm = HMAC_SHA256

def _digest(wire, key, rdata, time=None, request_mac=None, ctx=None,
            multi=None):
    """Return a context containing the TSIG rdata for the input parameters
    @rtype: hmac.HMAC object
    @raises ValueError: I{other_data} is too long
    @raises NotImplementedError: I{algorithm} is not supported
    """

    first = not (ctx and multi)
    if first:
        ctx = get_context(key)
        if request_mac:
            ctx.update(struct.pack('!H', len(request_mac)))
            ctx.update(request_mac)
    ctx.update(struct.pack('!H', rdata.original_id))
    ctx.update(wire[2:])
    if first:
        ctx.update(key.name.to_digestable())
        ctx.update(struct.pack('!H', thirdparty.dns.rdataclass.ANY))
        ctx.update(struct.pack('!I', 0))
    if time is None:
        time = rdata.time_signed
    upper_time = (time >> 32) & 0xffff
    lower_time = time & 0xffffffff
    time_encoded = struct.pack('!HIH', upper_time, lower_time, rdata.fudge)
    other_len = len(rdata.other)
    if other_len > 65535:
        raise ValueError('TSIG Other Data is > 65535 bytes')
    if first:
        ctx.update(key.algorithm.to_digestable() + time_encoded)
        ctx.update(struct.pack('!HH', rdata.error, other_len) + rdata.other)
    else:
        ctx.update(time_encoded)
    return ctx

def _maybe_start_digest(key, mac, multi):
    """If this is the first message in a multi-message sequence,
    start a new context.
    @rtype: hmac.HMAC object
    """
    if multi:
        ctx = get_context(key)
        ctx.update(struct.pack('!H', len(mac)))
        ctx.update(mac)
        return ctx
    else:
        return None

def sign(wire, key, rdata, time=None, request_mac=None, ctx=None, multi=False):
    """Return a (tsig_rdata, mac, ctx) tuple containing the HMAC TSIG rdata
    for the input parameters, the HMAC MAC calculated by applying the
    TSIG signature algorithm, and the TSIG digest context.
    @rtype: (string, hmac.HMAC object)
    @raises ValueError: I{other_data} is too long
    @raises NotImplementedError: I{algorithm} is not supported
    """

    ctx = _digest(wire, key, rdata, time, request_mac, ctx, multi)
    mac = ctx.digest()
    tsig = thirdparty.dns.rdtypes.ANY.TSIG.TSIG(thirdparty.dns.rdataclass.ANY, thirdparty.dns.rdatatype.TSIG,
                                     key.algorithm, time, rdata.fudge, mac,
                                     rdata.original_id, rdata.error,
                                     rdata.other)

    return (tsig, _maybe_start_digest(key, mac, multi))

def validate(wire, key, owner, rdata, now, request_mac, tsig_start, ctx=None,
             multi=False):
    """Validate the specified TSIG rdata against the other input parameters.

    @raises FormError: The TSIG is badly formed.
    @raises BadTime: There is too much time skew between the client and the
    server.
    @raises BadSignature: The TSIG signature did not validate
    @rtype: hmac.HMAC object"""

    (adcount,) = struct.unpack("!H", wire[10:12])
    if adcount == 0:
        raise thirdparty.dns.exception.FormError
    adcount -= 1
    new_wire = wire[0:10] + struct.pack("!H", adcount) + wire[12:tsig_start]
    if rdata.error != 0:
        if rdata.error == thirdparty.dns.rcode.BADSIG:
            raise PeerBadSignature
        elif rdata.error == thirdparty.dns.rcode.BADKEY:
            raise PeerBadKey
        elif rdata.error == thirdparty.dns.rcode.BADTIME:
            raise PeerBadTime
        elif rdata.error == thirdparty.dns.rcode.BADTRUNC:
            raise PeerBadTruncation
        else:
            raise PeerError('unknown TSIG error code %d' % rdata.error)
    if abs(rdata.time_signed - now) > rdata.fudge:
        raise BadTime
    if key.name != owner:
        raise BadKey
    if key.algorithm != rdata.algorithm:
        raise BadAlgorithm
    ctx = _digest(new_wire, key, rdata, None, request_mac, ctx, multi)
    mac = ctx.digest()
    if not hmac.compare_digest(mac, rdata.mac):
        raise BadSignature
    return _maybe_start_digest(key, mac, multi)

def get_context(key):
    """Returns an HMAC context foe the specified key.

    @rtype: HMAC context
    @raises NotImplementedError: I{algorithm} is not supported
    """

    try:
        digestmod = _hashes[key.algorithm]
    except KeyError:
        raise NotImplementedError(f"TSIG algorithm {key.algorithm} " +
                                  "is not supported")
    return hmac.new(key.secret, digestmod=digestmod)

class Key:
    def __init__(self, name, secret, algorithm=default_algorithm):
        if isinstance(name, str):
            name = thirdparty.dns.name.from_text(name)
        self.name = name
        if isinstance(secret, str):
            secret = base64.decodebytes(secret.encode())
        self.secret = secret
        if isinstance(algorithm, str):
            algorithm = thirdparty.dns.name.from_text(algorithm)
        self.algorithm = algorithm

    def __eq__(self, other):
        return (isinstance(other, Key) and
                self.name == other.name and
                self.secret == other.secret and
                self.algorithm == other.algorithm)