import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.pyplot as plt
import matplotlib as mpl

plt.style.use('seaborn-paper')

# mpl.use('pdf')

#plt.rc('font', family='Avenir', serif='Computer Modern')
plt.rc('text', usetex=True)
plt.rc('xtick', labelsize=16)
plt.rc('ytick', labelsize=16)
plt.rc('axes', labelsize=19)
plt.rc('axes', titlesize=19)
plt.rc('legend',fontsize=15) # using a size in points
plt.rcParams['axes.linewidth'] = 2.3
plt.rcParams["figure.figsize"] = (6.,4.1631189606246317)

rc = {"axes.spines.left" : True,
      "axes.spines.right" : True,
      "axes.spines.bottom" : True,
      "axes.spines.top" : True,
      "xtick.bottom" : True,
      "xtick.labelbottom" : True,
      "ytick.labelleft" : True,
      "ytick.left" : True}
plt.rcParams.update(rc)

data = np.load("scores.npy", allow_pickle=True).item()

# print('Percentage of images where the RBM scores better:')
fig, axs = plt.subplots(1, 3, figsize=(15, 5))

# fig = plt.figure(figsize=(10, 10))
#
# ax1 = plt.subplot(331)  # (0, 0)
# ax2 = plt.subplot(332)  # (0, 1)
# ax3 = plt.subplot(333)  # (0, 2)
#
# ax4 = plt.subplot(334)  # (1, 0)
# ax5 = plt.subplot(335)  # (1, 1)
# ax6 = plt.subplot(336)  # (1, 2)
#
# ax7 = plt.subplot(337)  # (2, 0)
# ax8 = plt.subplot(338)  # (2, 1)
# ax9 = plt.subplot(339)  # (2, 2)

for i, score in enumerate(["delete"]):

    # ax = axs[i]
    ax = axs

    # get the data
    df = data[score]
    lime_1 = np.array(df["lime_1"])
    lime_2 = np.array(df["lime_2"])
    lime_3 = np.array(df["lime_3"])
    mean = np.array(df["mean"])
    rmb_flip_detection = np.array(df["rbm_flip_detection"])

    # plot the distribution
    for j, lime in enumerate([lime_1, lime_2, lime_3]):
        if score == "delete":
            b = lime - rmb_flip_detection
        else:
            b = rmb_flip_detection - lime
        b += 1
        threshold = 1

        percentage = sum(b >= threshold) / len(b) * 100
        pos = b[b >= threshold]
        neg = b[b < threshold]
        sns.distplot(pos, kde=False, ax=ax[j], color='green')
        sns.distplot(neg, kde=False, ax=ax[j], color='red')

        if i % 3 == 0:
            t = "LIME-" + str(j)
            ax[j].set_title(t)
        if j % 3 == 0:
            if score == "irof":
                score = score.upper()
            ax[j].set(ylabel=score)
        # remove y-ticks
        ax[j].set(yticklabels=[])
        ax[j].legend(["{:.1f}\%".format(percentage), "{:.1f}\%".format(100-percentage)])

    # # add the title
    # if score == "irof":
    #     score = score.upper()
    # ax[0].set_title("RBM vs. LIME-1 ({})".format(score))
    # ax[1].set_title("RBM vs. LIME-2 ({})".format(score))
    # ax[2].set_title("RBM vs. LIME-3 ({})".format(score))

plt.tight_layout()
# plt.show()

    # if score == "delete":
    #     lime = np.min([lime_1, lime_2, lime_3])
    #     x_1 = rmb_flip_detection <= lime_1
    #     x_2 = rmb_flip_detection <= lime_2
    #     x_3 = rmb_flip_detection <= lime_3
    #     x_lime = rmb_flip_detection <= lime
    #     x_mean = rmb_flip_detection <= mean
    # else:
    #     lime = np.max([lime_1, lime_2, lime_3])
    #     x_1 = rmb_flip_detection >= lime_1
    #     x_2 = rmb_flip_detection >= lime_2
    #     x_3 = rmb_flip_detection >= lime_3
    #     x_lime = rmb_flip_detection >= lime
    #     x_mean = rmb_flip_detection >= mean
    #
    # print("=== {} ===".format(score))
    # print("lime1: \t", sum(x_1) / len(x_1) * 100, "%")
    # print("lime2: \t", sum(x_2) / len(x_2) * 100, "%")
    # print("lime3: \t", sum(x_3) / len(x_3) * 100, "%")
    # print("best of lime: \t", sum(x_lime) / len(x_lime) * 100, "%")
    # print("mean:  \t", sum(x_mean) / len(x_mean) * 100, "%")

plt.savefig('multiple_lime.pdf', bbox_inches="tight")


#!/usr/bin/env python3
import genrndtbl

class HashTable:
    def __init__(self, table):
        self.table = table
        self.size = len(table)

    def hash(self, x):
        return self.table[x % self.size]

    @staticmethod
    def Create(length, seed=None):
        t = genrndtbl.generate_random_table(length, seed)
        return HashTable(t)

if __name__ == "__main__":
    import sys
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("-l", "--length", type=int,
                        default=256, help="length of table")
    parser.add_argument("-r", "--random_seed", type=int, help="random seed")
    args = parser.parse_args()

    ht = HashTable.Create(args.length, args.random_seed)

    for line in sys.stdin:
        s = line.strip()
        if not s:
            break

        v = s.split(' ')

        xi = int(v[0])
        yi = int(v[1])
        xj = (int(v[0]) + 1)
        yj = (int(v[1]) + 1)

        aa = ht.hash(ht.hash(xi) + yi)
        ab = ht.hash(ht.hash(xi) + yj)
        ba = ht.hash(ht.hash(xj) + yi)
        bb = ht.hash(ht.hash(xj) + yj)
        print('{0} {1} {2} {3} {4[2]} {4[3]}'.format(aa, ab, ba, bb, v))


import Quandl
import pandas as pd
import pickle
import matplotlib.pyplot as plt
from matplotlib import style
style.use('fivethirtyeight')

def grab_initial_state_data():
    states = state_list()

    main_df = pd.DataFrame()

    for abbv in states:
        query = "FMAC/HPI_"+str(abbv)
        df = Quandl.get(query, authtoken=api_key)
        print(query)
        df = df.pct_change()
        print(df.head())
        if main_df.empty:
            main_df = df
        else:
            main_df = main_df.join(df)

    pickle_out = open('fiddy_states2.pickle','wb')
    pickle.dump(main_df, pickle_out)
    pickle_out.close()

grab_initial_state_data()

HPI_data = pd.read_pickle('fiddy_states2.pickle')

HPI_data.plot()
plt.legend().remove()
plt.show()


from constants import *
from datetime import datetime, timedelta
import random
import inspect
from modes.chase import Chase
from modes.pulse import Pulse
from modes.rainbow import Rainbow

class Randomizer:
    pixelmapper = None
    pixelcontroller = None
    tick_ms = None
    active_mode = None
    seconds_per_mode = None
    change_mode_dt = None

    def __init__(self, pixelmapper, pixelcontroller, tick_ms=100, seconds_per_mode=10):
        self.pixelmapper = pixelmapper
        self.pixelcontroller = pixelcontroller
        self.tick_ms = tick_ms
        self.seconds_per_mode = seconds_per_mode

    def start(self):
        self.set_random_mode()

        print("%s started" % __name__)

    def stop(self):
        self.pixelcontroller.set_color()

        print("%s stopped" % __name__)

    def input(self, event, axis, buttons):
        pass

    def tick(self):
        if self.change_mode_dt and datetime.now() >= self.change_mode_dt:
            self.set_random_mode()

        # if isinstance(self.active_mode, Chase):

        if self.active_mode:
            self.active_mode.tick()

    def set_random_mode(self):
        if self.active_mode:
            self.active_mode.stop()

        mode_idx = random.randint(0, 3)

        if mode_idx == 0:
            self.pixelmapper.base_color = self.pixelmapper.get_random_color(min_brightness=64)
            self.active_mode = Chase(self.pixelmapper, self.pixelcontroller)
        elif mode_idx == 1:
            self.active_mode = Pulse(self.pixelmapper, self.pixelcontroller)
        elif mode_idx == 2:
            self.active_mode = Pulse(self.pixelmapper, self.pixelcontroller,
                                     explicit_color_sequence=self.pixelmapper.get_rainbow(5))
        elif mode_idx == 3:
            self.active_mode = Rainbow(self.pixelmapper, self.pixelcontroller)

        self.change_mode_dt = datetime.now() + timedelta(seconds=self.seconds_per_mode)
        self.active_mode.start()


import cv2
import numpy as np
import classes.system_utilities.image_utilities.ImageUtilities as IU
from classes.system_utilities.helper_utilities.Enums import YoloModel

# Global variable declarations
yolo_net = 0
yolo_class_names = 0
yolo_output_layer_names = 0
yolo_net_input_size = 0

def OnLoad(model=YoloModel.YOLOV3):
    # All models and internal/external dependencies should be both loaded and initialized here

    # Initialize YOLOv3 model
    global yolo_net
    global yolo_class_names
    global yolo_output_layer_names
    global yolo_net_input_size

    if model == YoloModel.YOLOV4:
        model_config = 'config\\yolov4\\yolov4-tiny.cfg'
        model_weights = 'config\\yolov4\\yolov4-tiny.weights'
        classes_file = 'config\\yolov3\\coco.names'
        yolo_net_input_size = 416
    elif model == YoloModel.YOLOV3:
        model_config = 'config\\yolov3\\yolov3-320.cfg'
        model_weights = 'config\\yolov3\\yolov3-320.weights'
        classes_file = 'config\\yolov3\\coco.names'
        yolo_net_input_size = 320
    elif model == YoloModel.LICENSE_DETECTOR:
        model_config = 'config\\yolov4\\license_detector.cfg'
        model_weights = 'config\\yolov4\\license_detector.weights'
        classes_file = "config\\yolov4\\license_detector.names"
        yolo_net_input_size = 416

    with open(classes_file, 'rt') as f:
        yolo_class_names = f.read().rstrip('\n').split('\n')

    yolo_net = cv2.dnn.readNetFromDarknet(model_config, model_weights)
    # Set the target device for computation
    yolo_net.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA)
    yolo_net.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA)

    yolo_layer_names = yolo_net.getLayerNames()
    yolo_output_layer_names = [yolo_layer_names[i[0] - 1] for i in yolo_net.getUnconnectedOutLayers()]
    from classes.system_utilities.helper_utilities import Constants
    DetectObjectsInImage(image=np.zeros(shape=(Constants.default_camera_shape[1], Constants.default_camera_shape[0], Constants.default_camera_shape[2]), dtype=np.uint8))

def DetectObjectsInImage(image):
    # The function takes an input image and outputs all of the objects it detects in the image.
    # The bounding box is output in the format of [TL, BR] with points [x, y]

    height, width, _ = image.shape

    # Convert image to blob for the yolo network
    blob = IU.ImageToBlob(image=image,
                          input_size=yolo_net_input_size)

    yolo_net.setInput(blob)

    yolo_outputs = yolo_net.forward(yolo_output_layer_names)

    # Declare variables
    bounding_boxes = []
    class_ids = []
    confidence_scores = []
    # Higher confidence threshold means that the detections with confidence above threshold will be shown
    # Lower nms means that the threshold for overlapping bounding boxes is lowering meaning they filter out more
    confidence_threshold = 0.8
    nms_threshold = 0.3

    # Obtain bounding boxes of the detections that are over the threshold value
    is_one_detection_above_threshold = False
    for output in yolo_outputs:
        for detection in output:
            scores = detection[5:]
            class_id = np.argmax(scores)
            confidence = scores[class_id]

            if confidence > confidence_threshold:
                is_one_detection_above_threshold = True

                w, h = int(detection[2] * width), int(detection[3] * height)
                x, y = int((detection[0] * width) - w/2), int((detection[1] * height) - h/2)

                bounding_boxes.append([x, y, w, h])
                class_ids.append(class_id)
                confidence_scores.append(float(confidence))

    # Determines indices to keep by filtering overlapping bounding boxes with the same object against the threshold
    indices_to_keep = cv2.dnn.NMSBoxes(bounding_boxes, confidence_scores, confidence_threshold, nms_threshold)

    # Filter detections based on indices to keep
    temp_bounding_boxes = []
    class_names = []
    temp_confidence_scores = []
    for index in indices_to_keep:
        index = index[0]

        # Filter detections by classes
        if yolo_class_names[class_ids[index]] != 'car' and yolo_class_names[class_ids[index]] != 'truck':
            continue

        box = bounding_boxes[index]

        x, y, w, h = box[0], box[1], box[2], box[3]

        temp_bounding_boxes.append([x, y, w, h])

        # Get class ids corresponding class name
        temp_class_name = yolo_class_names[class_ids[index]].upper()
        class_names.append(temp_class_name)

        # Convert confidence score to percentage out of 100
        temp_confidence_scores.append(confidence_scores[index] * 100)

    bounding_boxes = temp_bounding_boxes
    confidence_scores = temp_confidence_scores

    # Convert bounding boxes to [TL, BR] format
    for i in range(len(bounding_boxes)):
        bounding_boxes[i][2] = bounding_boxes[i][0] + bounding_boxes[i][2]
        bounding_boxes[i][3] = bounding_boxes[i][1] + bounding_boxes[i][3]

        bounding_boxes[i] = [[bounding_boxes[i][0], bounding_boxes[i][1]], [bounding_boxes[i][2], bounding_boxes[i][3]]]

    return is_one_detection_above_threshold, class_names, bounding_boxes, confidence_scores


from tkinter import *
from timeit import default_timer
from time import sleep
import signal
import sys
import time
from tkinter import filedialog
from PIL import Image
from PIL import ImageTk
import cv2
import imutils
import numpy as np
import glob
import os
import serial
from matplotlib.backends.backend_tkagg import (FigureCanvasTkAgg, NavigationToolbar2Tk)
import matplotlib.pyplot as plt
from matplotlib.figure import Figure
plt.style.use('ggplot')
from PIL import ImageTk,Image
import serial, time, csv, ast,datetime
import numpy as np
import pandas as pd
from scipy import signal
from scipy.interpolate import lagrange
from scipy.fft import fft, fftfreq
from scipy import interpolate
import matplotlib
import datetime
from datetime import timedelta
import time
from os import remove

root = Tk()
root.geometry("1000x1000")
root.title("Medidor SER")

esp32 = serial.Serial('/dev/ttyUSB0', 230400, timeout=0.5)
esp32.flushInput()

if not os.path.exists('images/'):
    os.makedirs('images/')
"""
if not os.path.exists('images/fft'):
    os.makedirs('images/fft')

if not os.path.exists('images/fft/voltage'):
    os.makedirs('images/fft/voltage')

if not os.path.exists('images/fft/current'):
    os.makedirs('images/fft/current')

if not os.path.exists('images/señal'):
    os.makedirs('images/señal')

if not os.path.exists('images/señal/voltage'):
    os.makedirs('images/señal/voltage')
"""

font = {'family': 'serif',
        'color':  'darkred',
        'weight': 'normal',
        'size': 8,
        }

"""
folder = "images/señal/voltage/"
def BorrarArchivos():
      count1=0
      count2=0
      count3=0
      for f in os.listdir("images/señal/voltage/"):
                 #print(f)
                 count1=count1+1
      for f in os.listdir("images/fft/current/"):
                 #print(f)
                 count2=count2+1
      for f in os.listdir("images/fft/voltage/"):
                 #print(f)
                 count3=count3+1
        #print(count)
      try:
          if(count2>5):
               for i in os.listdir("images/señal/voltage"):
                    print(i)
                    os.remove(f'images/señal/voltage/{i}')
          if(count2>10):
               for i in os.listdir("images/fft/current"):
                    print(i)
                    os.remove(f'images/fft/current/{i}')
          if(count3>10):
               for i in os.listdir("images/fft/voltage"):
                    print(i)
                    os.remove(f'images/fft/voltage/{i}')

      except OSError:
               print("Imagen does not exists or is inaccessible")
               pass
"""

def BorrarArchivos():
      count1=0
      for f in os.listdir("images/"):
                 #print(f)
                 #print(count1)
                 count1=count1+1

      try:
          if(count1>10):
               for i in os.listdir("images/"):
                    #print(i)
                    os.remove(f'images/{i}')

      except OSError:
               print("Imagen does not exists or is inaccessible")
               pass

def graphVoltageCurrent(listVoltage,listCurrent,samplings): ##Grafica corriente y Voltaje
        global labelsfp
        global valuesvoltage
        global valuescurrent
        global maxvoltaje
        global minvoltaje
        #global maxcorriente
        #global mincorriente
        tiempo = 1/(samplings*(0.001/4200))
        tiempoms = np.arange(0,tiempo,tiempo/4096)
        my_formatted_list = [ '%.2f' % elem for elem in tiempoms ]

        #f = interpolate.interp1d(tiempoms, listVoltage)
        #xnew = np.arange(0, 4096, 5)  # 2550
        # print(f'largo xnew : {len(xnew)}')
        #ynew = f(xnew)
        valores = listVoltage#[200:4000]
        valores2 = listCurrent#[200:4000]
        #valores=round(val,1)
        valuesvoltage = [ i for i in valores ]
        valuescurrent = [ i for i in valores2 ]
        #labelsfp = [ i for i in range(0,len(valuesvoltage))]
        labelsfp =  [ i for i in my_formatted_list]

        maxvoltaje = max(valores)+300
        minvoltaje = min(valores)-300

        #plt.savefig(imagenVoltaje)
        #plt.close(fig)

def graphCurrent(list_fftVoltage, i):
       # y = np.linspace(0,len(list_fftVoltage),len(list_fftVoltage))
        i = str(i)
        x = list_fftVoltage*10
        print(f'corriente fase : {i}')
        plt.figure(figsize=(15,5))
        plt.plot(x)
        oldepoch = time.time()
        st = datetime.datetime.fromtimestamp(oldepoch).strftime('%Y-%m-%d-%H:%M:%S')
        plt.title("Corriente Fase"+i+".",fontdict=font)
        plt.ylabel("Corriente (mA-Peak-Peak)",fontdict=font)
        plt.xlabel("Tiempo(s)",fontdict=font)
        #print("images/señal/current"+i+"/"+st+"Current"+i+".png")
        plt.savefig("images/señal/current"+i+"/"+st+"Current.png")
        #plt.close(fig)

def graphVoltageFFT(list_fftVoltages,samplings):
    N = 4096
    T = 1 / samplings
    yf = fft(list_fftVoltages)
    Label(root, text=round(samplings,2),font=('Arial', 16)).grid(row=10, column=1)
    print(f'largo yf : {len(yf)}')
    xf = fftfreq(N, T)[:N] #tiene un largo de 4096
    print(f'largo xf : {len(xf)}')
    fig = plt.figure(figsize=(35,15))
    ejey = 20*np.log10(yf[:N])
    if (len(xf)>2550):
           f = interpolate.interp1d(xf, ejey)
           xnew = np.arange(0, 2550, 1) #2550
           print(f'largo xnew : {len(xnew)}')
           ynew = f(xnew)
           n = 0
           ax = fig.add_subplot(111)
           ax.plot(xnew,ynew)
           rangex = np.zeros(56)
           for h in range(0, 2600, 50):
             rangex[n]=h
             n = n+1
           ax.xaxis.set_ticks(rangex)
           ax.grid(True)
           plt.title("FFT Voltaje Fase 1",fontdict=font)
           ax.set_xlabel('Frecuencia (Hz)',fontdict=font)
           ax.set_ylabel('|dB|',fontdict=font)
           oldepoch = time.time()
           st = datetime.datetime.fromtimestamp(oldepoch).strftime('%Y-%m-%d-%H:%M:%S')
           plt.savefig("images/fft/voltage/"+st+"Voltage1.png")

           thd_array = []
           for i in range(46, 2596,50):
                a = np.amax(np.abs(ynew[i:i+8]))
                thd_array.append(a)
                #print(i)
                #print(a)
           sum = np.sum([thd_array[1:51]])
           raiz = np.sqrt(sum)
           thd_final = (raiz/thd_array[0])
           print(f'largo thd_array : {len(thd_array)}')
           print(f'thd_final : {thd_final}'+' %')
           Label(root, text=round(thd_final,2),font=('Arial', 16)).grid(row=6, column=1)

def graphCurrentFFT(list_fftVoltages,samplings):
    N = 4096
    T = 1 / samplings
    yf = fft(list_fftVoltages)
    xf = fftfreq(N, T)[:N]
    #print(f'largo xf : {len(xf)}')
    fig = plt.figure(figsize=(35,15))
    ejey = 20*np.log10(yf[:N])
    if (len(xf)>2550):
         f = interpolate.interp1d(xf, ejey)
         xnew = np.arange(0, 2550, 1)
         #print(f'largo xnew : {len(xnew)}')
         ynew = f(xnew)

         n = 0
         ax = fig.add_subplot(111)
         ax.plot(xnew,ejeyabsolut)
         rangex = np.zeros(56)
         for h in range(0, 2600, 50):
           rangex[n]=h
           n = n+1
         ax.xaxis.set_ticks(rangex)
         ax.grid(True)
         plt.title("FFT Corriente Fase"+i+".",fontdict=font)
         ax.set_xlabel('Frecuencia (Hz)',fontdict=font)
         ax.set_ylabel('|dB|',fontdict=font)
         oldepoch = time.time()
         st = datetime.datetime.fromtimestamp(oldepoch).strftime('%Y-%m-%d-%H:%M:%S')
         plt.savefig("images/fft/current"+i+"/"+st+"Current"+i+".png")

         thd_array = []
         for i in range(46, 2596,50):
              a = np.amax(np.abs(ynew[i:i+8]))
              thd_array.append(a)
              #print(i)
              #print(a)
         sum = np.sum([thd_array[1:51]])
         raiz = np.sqrt(sum)
         thd_final = (raiz/thd_array[0])

def getMaxValues(myList, quantity):
        return(sorted(list(set(myList)), reverse=True)[:quantity])
        #print(f'max : {max(myList)}')

def getMinValues(myList, quantity):
        return(sorted(list(set(myList)))[:quantity])
        #print(f'max : {max(myList)}')

vrms=0.0
def VoltRms(maximovoltaje2):
    if(maximovoltaje2<13):
        vrms=0
        print(f'vrms : {vrms}')
        return vrms
    vrms=maximovoltaje2*0.65
    return vrms

irms=0.0
def CurrentRms(maximocorriente2):
    if(maximocorriente2>430):
         irms = maximocorriente2*0.0133
    else:
         irms=(0.0046 + 0.000282*maximocorriente2 - 0.00000328*(maximocorriente2**2) + 0.0000000167*(maximocorriente2**3) - 0.0000000000382*(maximocorriente2**4) + 0.0000000000000322*(maximocorriente2**5))*maximocorriente2

    return irms

vrms0=0.0
def VoltajeRms(listVoltage):
    global vrms0
    N = len(listVoltage)
    Squares = []

    for i in range(0,N,1):    #elevamos al cuadrado cada termino y lo amacenamos
         listsquare = listVoltage[i]*listVoltage[i]
         Squares.append(listsquare)

    SumSquares=0
    for i in range(0,N,1):    #Sumatoria de todos los terminos al cuadrado
         SumSquares = SumSquares + Squares[i]

    MeanSquares = (1/N)*SumSquares #Dividimos por N la sumatoria

    vrms0=np.sqrt(MeanSquares)

    #print(f'Voltaje RMS : {vrms0}')
    return vrms0

irms0=0.0
def CorrienteRms(listCurrent):
    global irms0

    N = len(listCurrent)
    Squares = []

    for i in range(0,N,1):    #elevamos al cuadrado cada termino y lo amacenamos
         listsquare = listCurrent[i]*listCurrent[i]
         Squares.append(listsquare)

    SumSquares=0
    for i in range(0,N,1):    #Sumatoria de todos los terminos al cuadrado
         SumSquares = SumSquares + Squares[i]

    MeanSquares = (1/N)*SumSquares #Dividimos por N la sumatoria

    irms0=np.sqrt(MeanSquares)

    #print(f'Corriente RMS : {irms0}')
    return irms0

def CalculoDesfase(NoVoltageoffset,NoCurrentoffset,samplings):
    try:
        tiempo = 1/(samplings*(0.001/4200))
        tiempo2=(tiempo/4200)
        peakind = signal.find_peaks_cwt(NoVoltageoffset,np.arange(1,100))
        peakind2 = signal.find_peaks_cwt(NoCurrentoffset,np.arange(1,100))
        peakindvoltajems = peakind* tiempo2
        peakindcorrientems = peakind2* tiempo2
        dif=(peakindvoltajems-peakindcorrientems)
        print(np.mean(dif[1:10]/8.333))
    except:
        pass

DATVoltajeCGE=0.0
phasevoltajeCGE=0.0
FDVoltajeCGE=0.0

def VoltageFFT(list_fftVoltages, samplings,i):
    global j
    j = str(i)
    global DATVoltajeCGE1
    global phasevoltajeCGE
    global FDVoltajeCGE
    global ejeyabsolutv
    global xnewv
    #global FaseArmonicoFundamentalVoltaje
    N = len(list_fftVoltages)
    T = 1 / samplings
    list_fftVoltages -= np.mean(list_fftVoltages)
    datosfft = list_fftVoltages * np.hamming(4096)

    yf = np.fft.rfft(datosfft)
    #yf = fft(list_fftVoltages)
    #ejeyfase =  2.0/N * np.abs(yf[:50])
    #index_max2 = np.argmax(ejeyfase[:50])
    #a2 = yf[index_max2]/2048
    xf = fftfreq(N, T)[:N//2]  # tiene un largo de 4096
    #ejey = 2.0/N * np.abs(yf[:N//2])
    if (samplings > 5100):
           #f = interpolate.interp1d(xf, ejey)
           f = interpolate.interp1d(xf, yf[:N//2] )
           xnewv = np.arange(0, 2575, 1)  # 2550
           # print(f'largo xnew : {len(xnew)}')
           ynew = f(xnewv)
           ejeyabsolutv =  2.0/4096 * np.abs(ynew)#ynew
           #n = 0
           #fig = plt.figure(figsize=(7,2))
           #ax = fig.add_subplot(3,1,3)
           #ax.plot(xnew,ejeyabsolut)
           #rangex = np.zeros(28)
           #for h in range(50, 2600, 100):
           #  rangex[n]=h
           #  n = n+1
           #ax.xaxis.set_ticks(rangex)
           #ax.grid(True)
           #plt.title("FFT Voltaje",fontdict=font)
           #ax.set_xlabel('Frecuencia (Hz)',fontdict=font)
           #ax.set_ylabel('|dB|',fontdict=font)
           #oldepoch = time.time()
           #st = datetime.datetime.fromtimestamp(oldepoch).strftime('%Y-%m-%d-%H:%M:%S')
           #imagenVoltajeFFT = f'images/fft/voltage/{st}-Voltagefft.png'
           #plt.savefig(imagenVoltajeFFT)
           #load = Image.open(imagenVoltajeFFT)
           #renderfftVoltage = ImageTk.PhotoImage(load)
           #Label(root, image=renderfftVoltage).grid(column=2, row=13, rowspan=6)
           FD = []
           complejo = []
           real=[]
           imag=[]
           #dccomponent = max(ynew[0:10])
           z=0
           for i in range(45, 2575, 50):
                 a2 = max(ynew[i:i+10])
                 arra = max(ejeyabsolutv[i:i+10])
                 complejo.append(a2)
                 #index_max = np.argmax(ejeyabsolut[i-10:i+20])
                 #print(f'a : {a}')
                 #a = ynew[i+index_max]
                 real1 = a2.real
                 real.append(real1)
                 imag1 = a2.imag
                 imag.append(imag1)
                 #radiani = np.arctan(real1/imag1)
                 #degrees = math.degrees(radians)
                 #print(f'index max2 : {i+index_max}')
                 z=z+1
                 FD.append(arra)
                 #print(f'Armonico numero:{z} {i+index_max} + magnitud de {arra} + magnitud2 {abs(ynew[i+index_max])} + forma rectangular de {a} o {a*2/N} y radianes : {np.angle(a)}')
                 #print(f'Armonico corriente numero: {z} =>  {round(arra,3)} + {a2} + .. + {round(radiani,4)})')
                 #print(f'Armonico corriente numero: {z} =>  {round(arra,3)}')

           FD2=[]
           for i in range(0,len(FD)):
               if(FD[i]>(FD[0]/10)):
                   FD2.append(FD[i])

           SumMagnitudEficaz = (np.sum([FD[0:len(FD)]]))
           #print(f'Vrms total: {round(SumMagnitudEficaz,2)}')
           Magnitud1 = FD[0]
           if(j=="1"):
                 global sincvoltaje1
                 phasevoltajeCGE = np.arctan(real[0]/(imag[0]))
                 #FaseArmonicoFundamentalVoltaje1=round(np.angle(complejo[0]),2)
                 FDVoltajeCGE = round(Magnitud1/SumMagnitudEficaz,2)
                 DATVoltajeCGE1= round(np.sqrt(((SumMagnitudEficaz**2)-(Magnitud1**2))/(Magnitud1**2)),2)
                 print(f'DAT Voltaje CGE: {round(DATVoltajeCGE1,2)}')
                 sincvoltaje1 = 1

DATCorrienteCGE = 0.0
FDCorrienteCGE= 0.0
FDCorrienteCGE= 0.0
phasecorrienteCGE= 0.0
FPCGE= 0.99
FPCGE0= 0.99
cosphiCGE= 0.0
FDCorrienteCGE1=0.0
DATCorrienteCGE1=0.0

def CurrentFFT(list_fftVoltages, samplings, i,irms):
    global DATCorrienteCGE
    global DATCorrienteCGE1
    global a2
    global FDCorrienteCGE
    global FDCorrienteCGE1
    global phasecorrienteCGE
    global FPCGE
    global FPCGE0
    global cosphiCGE
    global q
    global renderfftcurrent
    global xnew
    global ejeyabsolut

    q = str(i)
    N = len(list_fftVoltages)
    T = 1 / samplings
    list_fftVoltages -= np.mean(list_fftVoltages)
    datosfft = list_fftVoltages * np.hamming(4096)#np.kaiser(N,100)
    yf = np.fft.rfft(datosfft)
    #yf=fft(list_fftVoltages)
    #if (g == 1):
    #     print(f'Sampling corriente 1: {round(samplings,2)}')

    xf = fftfreq(N, T)[:N//2]
    if (samplings > 5100):
         f = interpolate.interp1d(xf,yf[:N//2])
         xnew = np.arange(0, 2575, 1)
         ynew = f(xnew)
         ejeyabsolut =  2.0/N * np.abs(ynew)
         #n = 0
         #fig = plt.figure(figsize=(7,2))
         #ax = fig.add_subplot(3,1,2)
         #ax.plot(xnew,ejeyabsolut)
         #rangex = np.zeros(28)
         #for h in range(50, 2600, 100):
         #  rangex[n]=h
         #  n = n+1
         #ax.xaxis.set_ticks(rangex)
         #ax.grid(True)
         #plt.title(f'FFT Corriente | DAT: {DATCorrienteCGE1}, FD: {FDCorrienteCGE1}',fontdict=font)
         #ax.set_xlabel('Frecuencia (Hz)',fontdict=font)
         #ax.set_ylabel('|dB|',fontdict=font)
         #oldepoch = time.time()
         #st = datetime.datetime.fromtimestamp(oldepoch).strftime('%Y-%m-%d-%H:%M:%S')
         #imagenCurrentFFT = f'images/fft/current/{st}-Currentfft.png'
         #plt.savefig(imagenCurrentFFT)
         #load = Image.open(imagenCurrentFFT)
         #renderfftcurrent = ImageTk.PhotoImage(load)
         #Label(root, image=renderfftcurrent).grid(column=2, row=7, rowspan=6)
         #p = int(i)
         #z=0
         FD= []
         #dccomponent = max(ynew[0:10])
         complejo = []
         real=[]
         imag=[]
         #dccomponent = max(ynew[0:10])
         for i in range(45, 2575, 50):
               a2 = max(ynew[i:i+10])
               arra = max(ejeyabsolut[i:i+10])
               complejo.append(a2)
               #index_max = np.argmax(ejeyabsolut[i-10:i+20])
               #print(f'a : {a}')
               #a = ynew[i+index_max]
               real1 = a2.real
               real.append(real1)
               imag1 = a2.imag
               imag.append(imag1)
               #radiani = np.arctan(real1/imag1)
               #degrees = math.degrees(radians)
               #print(f'index max2 : {i+index_max}')
               #z=z+1
               FD.append(arra)
               #print(f'Armonico numero:{z} {i+index_max} + magnitud de {arra} + magnitud2 {abs(ynew[i+index_max])} + forma rectangular de {a} o {a*2/N} y radianes : {np.angle(a)}')
               #print(f'Armonico corriente numero: {z} =>  {round(arra,3)} + {a2} + .. + {round(radiani,4)})')
         #FD2=[]
         #for i in range(0,len(FD)):
         #    if(FD[i]>(FD[0]/10)):
         #        FD2.append(FD[i])

         #print(f'FD2: {FD2}')
         #print(f'FD largo: {len(FD)}')
         SumMagnitudEficaz = (np.sum([FD[0:len(FD)]]))*0.01

         Magnitud1 = FD[0]*0.01
         ArmonicosRestantes=SumMagnitudEficaz-Magnitud1
         #print(f'Irms armonico 1 {q}: {round(Magnitud1,2)}')
         proporcion = irms/(np.sqrt(Magnitud1**2+ArmonicosRestantes**2))
         irmsarmonico1prop=Magnitud1*proporcion
         #irmstotalproporcionado=np.sqrt((irmsarmonico1prop**2)+(ArmonicosRestantes*proporcion)**2)
         #MagnitudArmonicoFundamentalCorriente=round(thd_array[0],3)
         #fp2=round((armonico1corriente*np.cos(phasevoltaje-phasen))/valor1,2)
         #FaseArmonicoFundamentalCorriente=round(np.angle(complejo[0]),2)

         #GradoArmonicoFundamentalCorriente=round(Grados,2)
         if(q=="1"):
             global sincvoltaje1
             FDCorrienteCGE1 = round(irmsarmonico1prop/irms,2)
             print(f'FDCorrienteCGE : {round(FDCorrienteCGE1,2)}')
             DATCorrienteCGE1 = np.sqrt((SumMagnitudEficaz**2-Magnitud1**2)/(Magnitud1**2))
             print(f'DAT corriente CGE: {round(DATCorrienteCGE1,2)}')
             phasecorrienteCGE = np.arctan(real[0]/(imag[0]))
             if (sincvoltaje1 == 1):
                 FPCGE0=np.cos(phasevoltajeCGE-phasecorrienteCGE)*FDCorrienteCGE1
                 print(f'Desfase voltaje: {phasevoltajeCGE}')
                 print(f'Desfase corriente: {phasecorrienteCGE}')
                 print(f'Desfase: {phasevoltajeCGE-phasecorrienteCGE}')
                 cosphiCGE=np.cos(phasevoltajeCGE-phasecorrienteCGE)
                 #if(FPCGE0>0.0):
                 #    FPCGE0=FPCGE0+0.05
                 #else:
                 #    FPCGE0=FPCGE0-0.05
                 #if(FPCGE0>=1.0):
                 #    FPCGE0=0.99
                 #if(FPCGE0<=-1.0):
                 #    FPCGE0=-0.99

                 #FP=np.cos(FaseArmonicoFundamentalVoltaje-FaseArmonicoFundamentalCorriente)
                 print(f'FP1 cge: {round(FPCGE0,2)}')
                 print(f'cos(phi) cge : {round(cosphiCGE,2)}')
                 sincvoltaje1=0
                 #return FPCGE
         #sincvolaje1=0

def graphVoltage(list_fftVoltage,list_FPCurrent,samplings):
        global ax
        #y = np.linspace(0,len(list_fftVoltage),1000)
        #x = len(list_fftVoltage)
        #print(f'largo : {x}')
        global render
        #plt.figure(figsize=(12,2))
        #plt.plot(list_fftVoltage)
        fig=plt.figure(figsize=(12,8))
        #plt.plot(list_fftVoltage, color="blue", label="Voltaje")
        #plt.plot(list_FPCurrent, color="green", label="Corriente")
        #plt.legend(loc='upper left')
        tiempo = 1/(samplings*(0.001/4200))
        tiempoms = np.arange(0,tiempo,tiempo/4096)

        ax = fig.add_subplot(5,1,1)
        ax.plot(tiempoms,list_fftVoltage,color="blue", label="Voltaje")
        ax.plot(tiempoms,list_FPCurrent,color="green", label="Corriente")
        ax.legend(loc='upper left')
        ax.set_xlabel('Tiempo (mS)',fontdict=font)
        plt.title(f'Voltaje y Corriente | V: {round(vrms1,2)}    |    I: {round(irms1,2)}     |     P-Activa: {round(ActivaCGEFase11,2)}   |    P-Aparente: {round(AparenteCGEFase11,2)}    |     P-Reactiva:{round(ReactivaCGEFase11,2)}    |    FP: {round(FPCGE0,2)}',fontdict=font)

        ax = fig.add_subplot(3,1,3)
        plt.title(f'FFT Corriente | DAT: {round(DATCorrienteCGE1,2)}, FD: {round(FDCorrienteCGE1,2)}',fontdict=font)
        ax.plot(xnew,ejeyabsolut)
        rangex = np.zeros(28)
        n=0
        for h in range(50, 2600, 100):
           rangex[n]=h
           n = n+1
        ax.xaxis.set_ticks(rangex)
        ax.grid(True)
        ax.set_xlabel('Frecuencia (Hz)',fontdict=font)
        #ax.set_ylabel('|dB|',fontdict=font)

        ax = fig.add_subplot(5,1,5)
        ax.plot(xnewv,ejeyabsolutv)
        rangex = np.zeros(28)
        n=0
        for h in range(50, 2600, 100):
           rangex[n]=h
           n = n+1
        ax.xaxis.set_ticks(rangex)
        ax.grid(True)
        ax.set_xlabel('Frecuencia (Hz)',fontdict=font)
        #ax.set_ylabel('|dB|',fontdict=font)

        oldepoch = time.time()
        st = datetime.datetime.fromtimestamp(oldepoch).strftime('%Y-%m-%d-%H:%M:%S')
        #plt.xlabel("Tiempo(ms)",fontdict=font)
        plt.title(f'FFT Voltaje |    DAT: {DATVoltajeCGE1}     |     FD: {FDVoltajeCGE}    |   cos phi: {round(cosphiCGE,2)}',fontdict=font)
        ax.set_xlabel('Frecuencia (Hz)',fontdict=font)
        #ax.set_ylabel('Pk-Pk',fontdict=font)
        imagenVoltaje = f'images/{st}.png'
        plt.savefig(imagenVoltaje)
        load = Image.open(imagenVoltaje)
        render = ImageTk.PhotoImage(load)
        Label(root, image=render).grid(column=2, row=1, rowspan=6)

a = datetime.datetime.now()
#energyCGEFase11 = 0.0
energyCGEFase11 = 0.0
energyCGEFase11Hour = 0.0
AparenteCGEFase11 = 0.0
ActivaCGEFase11 = 0.0
ReactivaCGEFase11 = 0.0
energyCGEFase1 = 0.0
AparenteCGEFase1 = 0.0
ActivaCGEFase1 = 0.0
ReactivaCGEFase1 = 0.0

def Potencias(irms,vrms):
          global a
          global energyCGEFase1
          global ActivaCGEFase1
          global AparenteCGEFase1
          global ReactivaCGEFase1
          global energyCGEFase11
          global energyCGEFase11Hour
          global AparenteCGEFase11
          global ActivaCGEFase11
          global ReactivaCGEFase11
          AparenteCGEFase11 = vrms*irms
          print(f'Energia CGE: {AparenteCGEFase11}')
          ActivaCGEFase11 = np.abs(vrms*irms*cosphiCGE)
          print(f'Activa CGE: {ActivaCGEFase11}')
          ReactivaCGEFase11 = vrms*irms*np.sin(phasevoltajeCGE-phasecorrienteCGE)
          print(f'Reactiva CGE: {ReactivaCGEFase11}')
          a2 = datetime.datetime.now()
          delta=(((a2 - a).microseconds)/1000+((a2 - a).seconds)*1000)/10000000000
          energyCGEFase11 += ActivaCGEFase11*delta*2.9
          energyCGEFase11Hour += ActivaCGEFase11*delta*2.9
          a = datetime.datetime.now()
          if(a2.minute==0 or a2.minute==1):
              energyCGEFase11Hour=0
          if(a2.hour==0 and a2.minute==0):
              energyCGEFase11=0
          if(a2.hour==0 and a2.minute==1):
              energyCGEFase11=0
          print(f'Energia CGE: {energyCGEFase11}')

vrms1=0.0
vrms2=0.0
vrms3=0.0
irms1=0.0
irms2=0.0
irms3=0.0
vrms11=0.0
vrms22=0.0
vrms33=0.0
irms11=0.0
irms22=0.0
irms33=0.0
modamaximovoltaje11=[]
modamaximocorriente11=[]
modamaximovoltaje22=[]
modamaximocorriente22=[]
modamaximovoltaje33=[]
modamaximocorriente33=[]

def sin_wave(A, fs, N,phi):

    x = np.arange(N)
    y = A*np.sin(2*np.pi*50*x/fs + phi)
    #plt.plot(x, y)
    #plt.xlabel('sample(n)')
    #plt.ylabel('voltage(V)')
    #plt.show()
    return y

def received():
    while True:
            try:
                esp32_bytes = esp32.readline()
                decoded_bytes = str(esp32_bytes[0:len(esp32_bytes)-2].decode("utf-8"))#utf-8
            except:
                print("Error en la codificación")
                continue

            np_array = np.fromstring(decoded_bytes, dtype=float, sep=',')
            #print(f'{len(np_array)}')
            if (len(np_array) == 8402):
                       #print("..")
                       if (np_array[0] == 11):
                            global modamaximovoltaje11
                            global modamaximocorriente11
                            global vrms1
                            global vrms11
                            global irms1
                            global irms11
                            global modavoltaje
                            global modacorriente
                            global samplings
                            #samplings=30000
                            samplings = np_array[-1]
                            list_FPVoltage3 = np_array[0:4200]
                            list_FPCurrent3 = np_array[4201:8400]
                            #list_FPVoltage3 = sin_wave(A=320, fs=samplings, N=4200, phi=0)#np_array[0:4200]
                            #list_FPCurrent3 = sin_wave(A=320, fs=samplings,N=4200, phi=4.18) #np_array[4201:8400]
                            #print(f'max inicio: {max(list_FPVoltage3)}')
                            sos = signal.butter(10, 2500, 'low', fs=samplings, output='sos')
                            list_FPVoltage2 = signal.sosfilt(sos, list_FPVoltage3)
                            #list_FPVoltage2 = savgol_filter(list_FPVoltage2,len(list_FPVoltage2)-1,))
                            #sos = signal.butter(4, 50, 'low', fs=samplings, output='sos')
                            list_FPCurrent2 = signal.sosfilt(sos, list_FPCurrent3)
                            #print(f'max inicio con filtro: {max(list_FPVoltage2)}')
                            list_FPVoltage = list_FPVoltage2[104:4200]
                            list_FPCurrent = list_FPCurrent2[103:4200]
                            #Valor dc de Voltaje

                            valoresmaximovoltajesinmedia=getMaxValues(list_FPVoltage, 50)
                            valoresminimovoltajesinmedia=getMinValues(list_FPVoltage, 50)
                            maximovoltaje = np.median(valoresmaximovoltajesinmedia)
                            minimovoltaje = np.median(valoresminimovoltajesinmedia)
                            mediadcvoltaje = (maximovoltaje+minimovoltaje)/2
                            # Valores maximo y minimos de voltaje sin componente continua
                            NoVoltageoffset=list_FPVoltage-mediadcvoltaje

                            vrms1=VoltajeRms(NoVoltageoffset)

                            if (len(modamaximovoltaje11)>=5):
                                modavoltaje=np.median(modamaximovoltaje11)
                                vrms1=VoltRms(modavoltaje)
                                print(f'Vrms CGE: {vrms1}')
                                VoltageFFT(NoVoltageoffset,samplings,1)

                                modamaximovoltaje11=[]
                            else:
                                modamaximovoltaje11.append(vrms1)
                                #print(f'array voltaje: {modamaximovoltaje2
                            #Valor dc de corriente

                            valoresmaxcorriente=getMaxValues(list_FPCurrent, 50)
                            valoresmincorriente=getMinValues(list_FPCurrent, 50)
                            maximocorriente = np.median(valoresmaxcorriente)
                            minimocorriente = np.median(valoresmincorriente)
                            mediadccorriente = (maximocorriente+minimocorriente)/2

                            # Valores maximo y minimos de corriente
                            NoCurrentoffset=list_FPCurrent-mediadccorriente

                            irms1=CorrienteRms(NoCurrentoffset)

                            if (len(modamaximocorriente11)>=5):
                                modacorriente=np.median(modamaximocorriente11)
                                irms1=CurrentRms(modacorriente)
                                print(f'Irms CGE: {irms1}')
                                CurrentFFT(NoCurrentoffset,samplings,1,irms1)
                                #print(f'MODA CORRIENTE CGE: {modacorriente}')
                                Potencias(irms1,vrms1)
                                modamaximocorriente11=[]
                                graphVoltage(NoVoltageoffset,NoCurrentoffset,samplings)
                                CalculoDesfase(NoVoltageoffset,NoCurrentoffset,samplings)
                                #BorrarArchivos()
                            else:
                                modamaximocorriente11.append(irms1)
                            #    print(f'array corriente: {modamaximocorriente2}')
            #Label(root, text=round(vrms1,2),font=('Arial', 16)).grid(row=1, column=1)
            #Label(root, text=round(irms1,2),font=('Arial', 16)).grid(row=2, column=1)
            #Label(root, text=round(energyCGEFase11,2),font=('Arial', 16)).grid(row=3, column=1)
            #Label(root, text=round(ActivaCGEFase11,4),font=('Arial', 16)).grid(row=4, column=1)
            #Label(root, text=round(AparenteCGEFase11,2),font=('Arial', 16)).grid(row=5, column=1)
            #Label(root, text=round(ReactivaCGEFase11,2),font=('Arial', 16)).grid(row=6, column=1)
            #Label(root, text=round(FPCGE0,2),font=('Arial', 16)).grid(row=7, column=1)
            #Label(root, text=round(cosphiCGE,2),font=('Arial', 16)).grid(row=8, column=1)

                            BorrarArchivos()
                            root.after(500, received)

#show_imagen_voltaje()
#show_imagen_voltaje_fft()
#lblInputImage.image = ""
#lblInputImage2.image = ""

"""
lblInputImage = Label(root)
lblInputImage.grid(column=2, row=1, rowspan=6)
lblInputImage2 = Label(root)
lblInputImage2.grid(column=2, row=8, rowspan=6)
"""
selected = IntVar()

#my_buttonCurrentFFT = Button(root, text="FFT Corriente", command=graphCurrentFFT)
#my_buttonCurrentFFT.grid(row=4,column=2)

#Label(root, text="Voltaje",font=('Arial', 16)).grid(row=1, column=0)
#Label(root, text="Corriente",font=('Arial', 16)).grid(row=2, column=0)
#Label(root, text="Energia",font=('Arial', 16)).grid(row=3, column=0)
#Label(root, text="Activa",font=('Arial', 16)).grid(row=4, column=0)
#Label(root, text="Aparente",font=('Arial', 16)).grid(row=5, column=0)
#Label(root, text="Reactiva",font=('Arial', 16)).grid(row=6, column=0)
#Label(root, text="FP",font=('Arial', 16)).grid(row=7, column=0)
#Label(root, text="cos phi",font=('Arial', 16)).grid(row=8, column=0)

root.after(500, received)
root.mainloop()


'''
Created on Nov 10, 2012

@author: peterb
'''

from sqlalchemy.types import Integer, String
from sqlalchemy.schema import Column
from model.base import Base, Common

ADMIN_ROLE = u'admin'
USER_ROLE = u'user'
DEV_ROLE = u'dev'
AUTOMATIC_ROLE = 'automatic'
REMOTE_ROLE = "remote"

class Permission(Base, Common):

    ROLES = [ADMIN_ROLE, USER_ROLE, DEV_ROLE, AUTOMATIC_ROLE, REMOTE_ROLE]

    id = Column(Integer, primary_key=True)
    name =  Column(String(80), nullable=False, unique=True)

    @classmethod
    def find_or_create(cls, session, name):
        result = session.query(cls).filter(cls.name==name).first()
        if result is None:
            result = cls(name=name)
            session.add(result)
            session.flush()
        return result


from functions import*
import numpy as np
import time
import matplotlib
matplotlib.use("TkAgg")
from pathlib import Path
from matplotlib import cm

# path = os.path.realpath(Path("C:/Users/ermu8317/Downloads/Sphere_10242"))
# path = "Sphere_10242"
# sizes = [10242, 20480]
path = "Sphere_2964"
sizes = [2964, 5924]
# path = "Sphere_642"
# sizes = [642, 1280]
#
n = 101
phi1 = np.linspace(0, np.pi, n)
theta1 = np.linspace(0, 2 * np.pi, n)
phi2, theta2 = np.meshgrid(phi1, theta1)
phi, theta = phi2.T, theta2.T
scaling_factor = 1
r = 0.9
direction = np.array([0, 0, 1])
d_norm = direction/np.linalg.norm(direction)
r0 = 1.05 * d_norm
# m, m_pos, transformation_matrix, sigmas = read_mesh_from_hdf5(fn2, mode="coil")
# m = d_norm
m = np.array([0, 0, 1])
# omega = 3e3
omega = 18.85956e3

xyz_grid = xyz_grid(r, phi, theta)
# xyz_grid, n = sphere_mgrid(70, r)

# omega = 100
r_target = sphere_to_carthesian(r=r, phi=phi.flatten(), theta=theta.flatten())
# r_target = r_target[np.where(r_target[:, 2] > (0.8*r))]

start = time.time()
time_0 = start
# calculate analytic solution
res1 = reciprocity_three_D(r, theta, r0_v=r0, m=m, phi=phi, omega=omega, projection='sphere_surface')
# res1 = reciprocity_sphere(n=n, grid=xyz_grid, r0_v=r0, m=m, omega=omega)[1]
end = time.time()
t = t_format(end - start)
print(f"{t[0]:.2f}" + t[1] + " receprocity")

start = time.time()
tc, areas, tri_points, n_v, avg_len = read_sphere_mesh_from_txt(sizes=None, path="spheres/", n=400)
# tc, areas, tri_points, n_v, avg_len = read_sphere_mesh_from_txt(sizes, path)
# tc, areas, tri_points, n_v, avg_len = read_sphere_mesh_from_txt_locations_only(sizes, path, scaling=1)
# tc, areas, tri_points, n_v, avg_len = read_sphere_mesh_from_txt_locations_only(sizes, path, scaling=1)
# tc, areas, tri_points, n_v, avg_len = sphere_mesh(samples=10000, scaling=scaling_factor)
print(f"average length: {avg_len}")
end = time.time()
t = t_format(end - start)
print(f"{t[0]:.2f}" + t[1] + " triangulation")
n_elements = tc.shape[0]
print(f"elements: {n_elements}")

start = time.time()
b_im = jacobi_vectors_numpy(tc, n_v, r0, m, omega=omega)
# Q = q_jac_cu(tc, areas, n_v, b_im, tol=1e-19, n_iter=20)
Q = SCSM_jacobi_iter_cupy(tc, areas, n_v, b_im, tol=5e-16, n_iter=20, omega=omega, complex_values=True, verbose=True)
# Q = q_jac_vec(tc, areas, n_v, b_im, tol=5e-16, n_iter=20)
# Q = SCSM_jacobi_iter_vec_numpy(tc, areas, n_v, b_im, tol=5e-16, n_iter=20, omega=omega, complex_values=False, verbose=True)
# Q = SCSM_jacobi_iter_numpy(tc, areas, n_v, b_im, tol=5e-16, n_iter=20, omega=omega, verbose=True)
# Q = SCSM_matrix(tc, areas, n=n_v, b_im=b_im, omega=omega)
# Q = SCSM_tri_sphere_numba(tc, tri_points, areas, r0=r0, m=m, omega=omega)[0]

rs = tc
end = time.time()
t = t_format(end - start)
print(f"{t[0]:.2f}" + t[1] + " Q jacobi")
b_im_ = vector_potential_for_E_single_m(rs=r_target, m=m, m_pos=r0, omega=omega)
start = time.time()

res_flat = SCSM_FMM_E2(Q=Q, r_source=tc, r_target=r_target, eps=1e-15, b_im=b_im_)
res = array_unflatten(res_flat, n_rows=n)
# res3 = functions.parallel_SCSM_E_sphere(man, Q, rs, r, theta=theta, phi=phi, r0=r0, m=m, projection="sphere_surface")
# res = functions.SCSM_E_sphere(Q, rs, r, theta, r0=r0, m=m)
# res = SCSM_E_sphere_numba_surf(Q, rs, r, theta, r0=r0, m=m, phi=phi)

end = time.time()
t = t_format(end - start)
print(f"{t[0]:.2f}" + t[1] + "  E calculation")

# start = time.time()
# res3 = E_near_correction(E=res, Q=Q, r_q=rs, r_sphere=r, tri_points=tri_points, theta=theta, phi=phi, n=n_elements,
#                          r_near=3*avg_len)
# end = time.time()
# t = t_format(end - start)
# print(f"{t[0]:.2f}" + t[1] + "  E correction")

time_last = end
t = t_format(end - time_0)
print(f"{t[0]:.2f}" + t[1] + "  complete simulation")

res2 = res.copy()

diff = np.abs(res1 - res2)
relative_diff = diff / np.linalg.norm(res1)
max_analytic = np.max(res1)
min_analytic = np.min(res1)
max_numeric = np.max(res)
min_numeric = np.min(res)

print(f"max_analytic = {max_analytic}, min_analytic = {min_analytic}, max_numeric = {max_numeric}, min_numeric = {min_numeric}")

# diff2 = np.abs(res1 - res3)
# relative_diff2 = diff2 / np.linalg.norm(res1)
#
error_imag = nrmse(res1, res2) * 100
# rerror_imag2 = np.linalg.norm(diff2) * 100 / np.linalg.norm(res1)

print(f"relative error: {error_imag:.7f}%")
# print(f"relative error with near field: {rerror_imag2:.7f}%")

# plot_E_sphere_surf(res, phi, theta, r)
plot_E_sphere_surf_diff(res1, res2, xyz_grid=xyz_grid, c_map=cm.bwr, title=False, plot_difference=False)
# functions.plot_E_sphere_surf(res2, phi, theta, r)
# functions.plot_E_sphere_surf(diff, phi, theta, r)
# functions.plot_E_sphere_surf(relative_diff, phi, theta, r)


#!/usr/bin/env python3
import os
import json
import jieba
import pandas as pd

def isChinese(string):
    """
    判断一段字符串是否全为汉字
    """
    for char in string:
        if not (0x4e00<=ord(char)<0x9fa5):
            return False

    return True

def count_frequency(text: str, stats: dict = {'char': {}, 'word': {}}):
    """
    统计一段文本的字频(逐字)和词频(通过jieba分词)
    """
    # 统计单字频次
    for char in text:
        if isChinese(char):
            stats['char'][char] = stats['char'].get(char, 0) + 1

    # 统计词语频次
    words = jieba.lcut(text, cut_all=False)
    for word in words:
        if isChinese(word) and len(word)>1:  # 要求word是二字以上词语且全为汉字
            stats['word'][word] = stats['word'].get(word, 0) + 1

    return stats

def process_char_frequency(stats: dict):
    """
    处理字频, 输出覆盖80%总字频的汉字、字频、累计字频到HF_SingleHZ.txt (按字频从高到低排序)
    """
    char_freq = list(stats.items())
    char_freq = sorted(char_freq, key=lambda x: x[1], reverse=True)  # 按freq从大到小排序
    char, freq = zip(*char_freq)
    total_freq = sum(freq)
    result = {"汉字": [], "字频": [], "累计字频": []}
    for i in range(len(freq)):
        result['汉字'].append(char[i])
        result['字频'].append(freq[i])
        sum_freq = sum(freq[:i + 1])
        result['累计字频'].append(sum_freq)
        if sum_freq > 0.8 * total_freq:
            break

    df = pd.DataFrame(result)
    df.to_csv('HF_SingleHZ.txt', sep='\t', index=True, header=True, encoding='utf-8')
    return result

def process_word_frequency(stats: dict):
    """
    处理词频, 输出前一万字频的词语、词频到HF_SingleHZ.txt (按词频从高到低排序)
    """
    word_freq = list(stats.items())
    word_freq = sorted(word_freq, key=lambda x: x[1], reverse=True)  # 按freq从大到小排序
    word, freq = zip(*word_freq)
    result = {"词条": [], "频次": []}
    for i in range(10000):
        result['词条'].append(word[i])
        result['频次'].append(freq[i])
        if i>=len(word_freq):
            break

    df = pd.DataFrame(result)
    df.to_csv('HF_Word.txt', sep='\t', index=True, header=True, encoding='utf-8')
    return result

def main(file: str):
    """
    file代表用于统计频次的文件路径, 统计完成后把数据存入'HF_SingleHZ.txt'和'HF_Word.txt'
    """
    if not (os.path.exists('HF_SingleHZ.txt') and os.path.exists('HF_Word.txt')):
        print("不存在已经统计好的频次数据, 开始统计")
        texts = []
        with open(file, 'r') as f:
            for line in f:
                texts.append(json.loads(line)['content'])

        stats = {'char': {}, 'word': {}}
        for i in range(len(texts)):
            print(f"==> 正在统计第{i+1}/{len(texts)}段文本")
            stats = count_frequency(texts[i], stats)

        process_char_frequency(stats=stats['char'])
        process_word_frequency(stats=stats['word'])

    print("==> 频次数据已统计并储存, 请查看")

if __name__=="__main__":
    main(file='new2016zh/news2016zh_train.json')


import re
import json
import numpy
import collections
import openpyxl as xl

runners_wb = xl.load_workbook('Leaderboard.xlsx')
runners_sheet = runners_wb.active

with open('test.npy', 'rb') as f:
    runsArray = numpy.load(f, allow_pickle=True)

pop1anyDOS = []
pop1anyDOSNMG = []
pop1anyNES = []
pop1anyAppleII = []
pop1anyTG16 = []
pop1anySMS = []
pop1anyGBC = []
pop1anyAmiga = []
pop1anyMD = []
pop1anyGenesis = []
pop1anyLS = []
pop1anyLSNMG = []
pop1hundo = []
pop1hundoNMG = []
pop1pacifist = []
pop1istaria = []
pop1RPR = []
pop1SNESanyNW = []
pop1SNESany = []
pop1SNESanyNGS = []
pop1SNESSR = []
pop1SNES30AP = []
pop1SNESTQoL = []
pop1SNESEvol = []
pop1SNESTDC = []
pop2any = []
pop3dany = []
pop3danyAT = []
popanany = []
popsotACZip = []
popsotACZipl = []
popsotACNMG = []
popsotanyZip = []
popsotanyZipl = []
popsotanyNMG = []
popsothundoZip = []
popsothundoNMG = []
popsotGBAany = []
popwwanyZip = []
popwwanyZipl = []
popwwanyNMG = []
popwwTEZip = []
popwwTEZipl = []
popwwTENMG = []
popwwACNMG = []
popwwAChZipl = []
popwwAChNMG = []
popwwAAZip = []
popt2tanyZip = []
popt2tanyZipl = []
popt2tanyNMG = []
popt2tAPZip = []
popt2tAPZipl = []
popt2tAPNMG = []
popt2tPSPUSNMG = []
popclassicany = []
pop2k8anySt = []
pop2k8anyNMG = []
pop2k8AS = []
pop2k8Epilougeany = []
pop2k8Hacked = []
pop2k8anyDupe = []
poptfkanyCon = []
poptfkanyEmu = []
poptfsanySt = []
poptfsanyNEG = []
poptfsanyNMG = []
poptfsPSPany = []
poptfsWiianyNG = []
poptfsWiianyNGP = []
poptfsDSanyCon = []
poptfsDSanyEmu = []
mobileHA = []
mobileSnFR = []
mobileSoT = []
mobileWW = []
mobileT2T = []
mobileClassic = []
mobile2k8 = []
mobileTFS = []
multiSTanyZip = []
multiSTanyZipl = []
multiSTanyNMG = []
multiSTCompZip = []
multiSTCompZipl = []
multiSTCompNMG = []
runnersArray = {"":0}

for i in runsArray:
    if(i['category'] == 'rklx8pnk'):
        if(i['subcat'] == 'rqvvyeyq'):
            pop1anyDOS.append(i)
        if(i['subcat'] == '4qyxp34l'):
            pop1anyDOSNMG.append(i)
        if(i['subcat'] == '5le2xv6l'):
            pop1anyNES.append(i)
        if(i['subcat'] == '81w2ve51'):
            pop1anyAppleII.append(i)
        if(i['subcat'] == '81wm0j5q'):
            pop1anyTG16.append(i)
        if(i['subcat'] == 'zqo50dpl'):
            pop1anyGBC.append(i)
        if(i['subcat'] == 'p12z2m41'):
            pop1anySMS.append(i)
        if(i['subcat'] == '81pdo5v1'):
            pop1anyAmiga.append(i)
        if(i['subcat'] == '810y2o5l'):
            pop1anyMD.append(i)
        if(i['subcat'] == 'klrv7woq'):
            pop1anyGenesis.append(i)

    if(i['category'] == 'ndx9xq5d'):
        if(i['subcat'] == 'klrnw4w1'):
            pop1anyLS.append(i)
        if(i['subcat'] == '21dr7ggq'):
            pop1anyLSNMG.append(i)

    if(i['category'] == 'wkpq95gd'):
        if(i['subcat'] == '21ggx3m1'):
            pop1hundo.append(i)
        if(i['subcat'] == '21ggx3m1'):
            pop1hundoNMG.append(i)

    if(i['category'] == 'mkeloznd'):
        pop1pacifist.append(i)

    if(i['category'] == 'wdmvjyod'):
        pop1istaria.append(i)

    if(i['category'] == '9kv8ww8d'):
        pop1RPR.append(i)

    if(i['category'] == 'vdomz5y2'):
        pop1SNESanyNW.append(i)

    if(i['category'] == '8241l8m2'):
        pop1SNESany.append(i)

    if(i['category'] == 'wkpm95gk'):
        pop1SNESanyNGS.append(i)

    if(i['category'] == 'w20go3zk'):
        pop1SNESSR.append(i)

    if(i['category'] == 'mkev0xx2'):
        pop1SNES30AP.append(i)

    if(i['category'] == '5dw9l1n2'):
        pop1SNESTQoL.append(i)

    if(i['category'] == 'wdmxp4ek'):
        pop1SNESEvol.append(i)

    if(i['category'] == 'vdo8eo6k'):
        pop1SNESTDC.append(i)

    if(i['category'] == 'jdrpw3x2'):
        pop2any.append(i)

    if(i['category'] == 'jdz7lxvk'):
        pop3dany.append(i)

    if(i['category'] == 'z27zlok0'):
        pop3danyAT.append(i)

    if(i['category'] == 'rklv09qd'):
        popanany.append(i)

    if(i['category'] == 'zd3xnrdn'):
        if(i['subcat'] == '21d5m3le'):
            popsotACZip.append(i)
        if(i['subcat'] == '5q8pmrld'):
            popsotACZipl.append(i)
        if(i['subcat'] == '5leygm1o'):
            popsotACNMG.append(i)

    if(i['category'] == 'xd1v9wd8'):
        if(i['subcat'] == '4qy963l7'):
            popsotanyZip.append(i)
        if(i['subcat'] == 'mln926qp'):
            popsotanyZipl.append(i)
        if(i['subcat'] == '810prjlv'):
            popsotanyNMG.append(i)

    if(i['category'] == 'zd3lmz8d'):
        if(i['subcat'] == '013w9krq'):
            popsothundoZip.append(i)
        if(i['subcat'] == 'rqv6v2rl'):
            popsothundoNMG.append(i)

    if(i['category'] == '02q0gjky'):
        popsotGBAany.append(i)

    if(i['category'] == '02qnm92y'):
        if(i['subcat'] == '0q5mvvlp'):
            popwwanyZip.append(i)
        if(i['subcat'] == '4lxk5g12'):
            popwwanyZipl.append(i)
        if(i['subcat'] == 'zqo7k51y'):
            popwwanyNMG.append(i)

    if(i['category'] == 'wkp550dr'):
        if(i['subcat'] == '013ydrl5'):
            popwwTEZip.append(i)
        if(i['subcat'] == '5lexv6qo'):
            popwwTEZipl.append(i)
        if(i['subcat'] == 'rqvngr16'):
            popwwTENMG.append(i)

    if(i['category'] == 'mke9qpjd'):
        if(i['subcat'] == '21gy9d61'):
            popwwACNMG.append(i)

    if(i['category'] == '9kvoge82'):
        if(i['subcat'] == 'jqzkge2q'):
            popwwAChZipl.append(i)
        if(i['subcat'] == 'xqk56vnq'):
            popwwAChNMG.append(i)

    if(i['category'] == 'wdmx13ok'):
        if(i['subcat'] == 'jq63v07q'):
            popwwAAZip.append(i)

    if(i['category'] == '5dw11nkg'):
        if(i['subcat'] == 'p12p0vqx'):
            popt2tanyZip.append(i)
        if(i['subcat'] == '81w2wm14'):
            popt2tanyZipl.append(i)
        if(i['subcat'] == '81p25el7'):
            popt2tanyNMG.append(i)

    if(i['category'] == 'n2yvvm2o'):
        if(i['subcat'] == '810y85lv'):
            popt2tAPZip.append(i)
        if(i['subcat'] == '9qjorgl4'):
            popt2tAPZipl.append(i)
        if(i['subcat'] == 'rqvyeyq6'):
            popt2tAPNMG.append(i)

    if(i['category'] == 'zd3xvzrd'):
        if(i['subcat'] == '81wwzv51'):
            popt2tPSPUSNMG.append(i)

    if(i['category'] == '9d8pn3wk'):
        popclassicany.append(i)

    if(i['category'] == 'xd1rv6zk'):
        if(i['subcat'] == 'jqzvpd4l'):
            pop2k8anySt.append(i)
        if(i['subcat'] == 'gq793ndl'):
            pop2k8anyNMG.append(i)

    if(i['category'] == 'z276g30d'):
        pop2k8AS.append(i)

    if(i['category'] == 'zd30xped'):
        pop2k8Epilougeany.append(i)

    if(i['category'] == 'zd366lyd'):
        pop2k8Hacked.append(i)

    if(i['category'] == 'xd170vwd'):
        pop2k8anyDupe.append(i)

    if(i['category'] == '9kv74gjk'):
        if(i['subcat'] == '21go0xmq'):
            poptfkanyCon.append(i)
        if(i['subcat'] == 'gq7z37y1'):
            poptfkanyEmu.append(i)

    if(i['category'] == 'wdm4o54d'):
        if(i['subcat'] == 'zqomxg51'):
            poptfsanySt.append(i)
        if(i['subcat'] == 'jqzxo941'):
            poptfsanyNEG.append(i)
        if(i['subcat'] == '013w9jrq'):
            poptfsanyNMG.append(i)

    if(i['category'] == 'w201mqzk'):
        poptfsPSPany.append(i)

    if(i['category'] == 'zd3l4vnd'):
        if(i['subcat'] == 'xqkr6ey1'):
            poptfsWiianyNG.append(i)
        if(i['subcat'] == 'gq7nz5y1'):
            poptfsWiianyNGP.append(i)

    if(i['category'] == '9kvj3q0k'):
        if(i['subcat'] == '81wr08ml'):
            poptfsDSanyCon.append(i)
        if(i['subcat'] == 'zqo2vn5l'):
            poptfsDSanyEmu.append(i)

    if(i['category'] == 'wk67pnxd'):
        mobileHA.append(i)

    if(i['category'] == 'mkerym6d'):
        mobileSnFR.append(i)

    if(i['category'] == '9kvmp78k'):
        mobileSoT.append(i)

    if(i['category'] == 'rklqxvn2'):
        mobileWW.append(i)

    if(i['category'] == 'ndxmj7r2'):
        mobileT2T.append(i)

    if(i['category'] == 'vdoo0mvd'):
        mobileClassic.append(i)

    if(i['category'] == 'w20wpv5d'):
        mobile2k8.append(i)

    if(i['category'] == 'wdmwzlo2'):
        mobileTFS.append(i)

    if(i['category'] == 'rklxvvqk'):
        if(i['subcat'] == 'jq68rvlm0q503m1p'):
            multiSTanyZip.append(i)
        if(i['subcat'] == 'jq68rvlm4lxj62q2'):
            multiSTanyZipl.append(i)
        if(i['subcat'] == 'jq68rvlm814nw0ld'):
            multiSTanyNMG.append(i)
        if(i['subcat'] == '81w6oml40q503m1p'):
            multiSTCompZip.append(i)
        if(i['subcat'] == '81w6oml44lxj62q2'):
            multiSTCompZipl.append(i)
        if(i['subcat'] == '81w6oml4814nw0ld'):
            multiSTCompNMG.append(i)

def allotPoints(catArray):
    catArray = sorted(catArray, key=lambda d: d['time'])
    points = len(catArray)
    for j in catArray:
        if j['runner'] in runnersArray.keys():
            runnersArray[j['runner']] += points
            points -= 1
        else:
            runnersArray[j['runner']] = points
            points -= 1

def calcPoints(catArray, runner):
    catArray = sorted(catArray, key=lambda d: d['time'])
    points = len(catArray)
    runnerpoints = 0
    for j in catArray:
        if j['runner'] == runner:
            runnerpoints+=points
        points-=1
    return runnerpoints

allotPoints(pop1anyDOS)
allotPoints(pop1anyDOSNMG)
allotPoints(pop1anyNES)
allotPoints(pop1anyAppleII)
allotPoints(pop1anyTG16)
allotPoints(pop1anySMS)
allotPoints(pop1anyGBC)
allotPoints(pop1anyAmiga)
allotPoints(pop1anyMD)
allotPoints(pop1anyGenesis)
allotPoints(pop1anyLS)
allotPoints(pop1anyLSNMG)
allotPoints(pop1hundo)
allotPoints(pop1hundoNMG)
allotPoints(pop1pacifist)
allotPoints(pop1istaria)
allotPoints(pop1RPR)
allotPoints(pop1SNESanyNW)
allotPoints(pop1SNESany)
allotPoints(pop1SNESanyNGS)
allotPoints(pop1SNESSR)
allotPoints(pop1SNES30AP)
allotPoints(pop1SNESTQoL)
allotPoints(pop1SNESEvol)
allotPoints(pop1SNESTDC)
allotPoints(pop2any)
allotPoints(pop3dany)
allotPoints(pop3danyAT)
allotPoints(popanany)
allotPoints(popsotACZip)
allotPoints(popsotACZipl)
allotPoints(popsotACNMG)
allotPoints(popsotanyZip)
allotPoints(popsotanyZipl)
allotPoints(popsotanyNMG)
allotPoints(popsothundoZip)
allotPoints(popsothundoNMG)
allotPoints(popsotGBAany)
allotPoints(popwwanyZip)
allotPoints(popwwanyZipl)
allotPoints(popwwanyNMG)
allotPoints(popwwTEZip)
allotPoints(popwwTEZipl)
allotPoints(popwwTENMG)
allotPoints(popwwACNMG)
allotPoints(popwwAChZipl)
allotPoints(popwwAChNMG)
allotPoints(popwwAAZip)
allotPoints(popt2tanyZip)
allotPoints(popt2tanyZipl)
allotPoints(popt2tanyNMG)
allotPoints(popt2tAPZip)
allotPoints(popt2tAPZipl)
allotPoints(popt2tAPNMG)
allotPoints(popt2tPSPUSNMG)
allotPoints(popclassicany)
allotPoints(pop2k8anySt)
allotPoints(pop2k8anyNMG)
allotPoints(pop2k8AS)
allotPoints(pop2k8Epilougeany)
allotPoints(pop2k8Hacked)
allotPoints(pop2k8anyDupe)
allotPoints(poptfkanyCon)
allotPoints(poptfkanyEmu)
allotPoints(poptfsanySt)
allotPoints(poptfsanyNEG)
allotPoints(poptfsanyNMG)
allotPoints(poptfsPSPany)
allotPoints(poptfsWiianyNG)
allotPoints(poptfsWiianyNGP)
allotPoints(poptfsDSanyCon)
allotPoints(poptfsDSanyEmu)
allotPoints(mobileHA)
allotPoints(mobileSnFR)
allotPoints(mobileSoT)
allotPoints(mobileWW)
allotPoints(mobileT2T)
allotPoints(mobileClassic)
allotPoints(mobile2k8)
allotPoints(mobileTFS)
allotPoints(multiSTanyZip)
allotPoints(multiSTanyZipl)
allotPoints(multiSTanyNMG)
allotPoints(multiSTCompZip)
allotPoints(multiSTCompZipl)
allotPoints(multiSTCompNMG)

sorted_array = sorted(runnersArray.items(), key=lambda kv: kv[1], reverse=True)

runnercount = 0;
for i in sorted_array:
    runners_sheet.cell(row=runnercount+4, column=1).value = i[0]
    runners_sheet.cell(row=runnercount+4, column=2).value = i[1]
    runners_sheet.cell(row=runnercount+4, column=3).value = calcPoints(pop1anyDOS, i[0])
    runners_sheet.cell(row=runnercount+4, column=4).value = calcPoints(pop1anyDOSNMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=5).value = calcPoints(pop1anyNES, i[0])
    runners_sheet.cell(row=runnercount+4, column=6).value = calcPoints(pop1anyAppleII, i[0])
    runners_sheet.cell(row=runnercount+4, column=7).value = calcPoints(pop1anyTG16, i[0])
    runners_sheet.cell(row=runnercount+4, column=8).value = calcPoints(pop1anyGBC, i[0])
    runners_sheet.cell(row=runnercount+4, column=9).value = calcPoints(pop1anySMS, i[0])
    runners_sheet.cell(row=runnercount+4, column=10).value = calcPoints(pop1anyAmiga, i[0])
    runners_sheet.cell(row=runnercount+4, column=11).value = calcPoints(pop1anyMD, i[0])
    runners_sheet.cell(row=runnercount+4, column=12).value = calcPoints(pop1anyGenesis, i[0])
    runners_sheet.cell(row=runnercount+4, column=13).value = calcPoints(pop1anyLS, i[0])
    runners_sheet.cell(row=runnercount+4, column=14).value = calcPoints(pop1anyLSNMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=15).value = calcPoints(pop1hundo, i[0])
    runners_sheet.cell(row=runnercount+4, column=16).value = calcPoints(pop1hundoNMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=17).value = calcPoints(pop1pacifist, i[0])
    runners_sheet.cell(row=runnercount+4, column=18).value = calcPoints(pop1istaria, i[0])
    runners_sheet.cell(row=runnercount+4, column=19).value = calcPoints(pop1RPR, i[0])
    runners_sheet.cell(row=runnercount+4, column=20).value = calcPoints(pop1SNESanyNW, i[0])
    runners_sheet.cell(row=runnercount+4, column=21).value = calcPoints(pop1SNESany, i[0])
    runners_sheet.cell(row=runnercount+4, column=22).value = calcPoints(pop1SNESanyNGS, i[0])
    runners_sheet.cell(row=runnercount+4, column=23).value = calcPoints(pop1SNESSR, i[0])
    runners_sheet.cell(row=runnercount+4, column=24).value = calcPoints(pop1SNES30AP, i[0])
    runners_sheet.cell(row=runnercount+4, column=25).value = calcPoints(pop1SNESTQoL, i[0])
    runners_sheet.cell(row=runnercount+4, column=26).value = calcPoints(pop1SNESEvol, i[0])
    runners_sheet.cell(row=runnercount+4, column=27).value = calcPoints(pop1SNESTDC, i[0])
    runners_sheet.cell(row=runnercount+4, column=28).value = calcPoints(pop2any, i[0])
    runners_sheet.cell(row=runnercount+4, column=29).value = calcPoints(pop3dany, i[0])
    runners_sheet.cell(row=runnercount+4, column=30).value = calcPoints(pop3danyAT, i[0])
    runners_sheet.cell(row=runnercount+4, column=31).value = calcPoints(popanany, i[0])
    runners_sheet.cell(row=runnercount+4, column=32).value = calcPoints(popsotanyZip, i[0])
    runners_sheet.cell(row=runnercount+4, column=33).value = calcPoints(popsotanyZipl, i[0])
    runners_sheet.cell(row=runnercount+4, column=34).value = calcPoints(popsotanyNMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=35).value = calcPoints(popsotACZip, i[0])
    runners_sheet.cell(row=runnercount+4, column=36).value = calcPoints(popsotACZipl, i[0])
    runners_sheet.cell(row=runnercount+4, column=37).value = calcPoints(popsotACNMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=38).value = calcPoints(popsothundoZip, i[0])
    runners_sheet.cell(row=runnercount+4, column=39).value = calcPoints(popsothundoNMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=40).value = calcPoints(popsotGBAany, i[0])
    runners_sheet.cell(row=runnercount+4, column=41).value = calcPoints(popwwanyZip, i[0])
    runners_sheet.cell(row=runnercount+4, column=42).value = calcPoints(popwwanyZipl, i[0])
    runners_sheet.cell(row=runnercount+4, column=43).value = calcPoints(popwwanyNMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=44).value = calcPoints(popwwTEZip, i[0])
    runners_sheet.cell(row=runnercount+4, column=45).value = calcPoints(popwwTEZipl, i[0])
    runners_sheet.cell(row=runnercount+4, column=46).value = calcPoints(popwwTENMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=47).value = calcPoints(popwwACNMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=48).value = calcPoints(popwwAChZipl, i[0])
    runners_sheet.cell(row=runnercount+4, column=49).value = calcPoints(popwwAChNMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=50).value = calcPoints(popwwAAZip, i[0])
    runners_sheet.cell(row=runnercount+4, column=51).value = calcPoints(popt2tanyZip, i[0])
    runners_sheet.cell(row=runnercount+4, column=52).value = calcPoints(popt2tanyZipl, i[0])
    runners_sheet.cell(row=runnercount+4, column=53).value = calcPoints(popt2tanyNMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=54).value = calcPoints(popt2tAPZip, i[0])
    runners_sheet.cell(row=runnercount+4, column=55).value = calcPoints(popt2tAPZipl, i[0])
    runners_sheet.cell(row=runnercount+4, column=56).value = calcPoints(popt2tAPNMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=57).value = calcPoints(popt2tPSPUSNMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=58).value = calcPoints(popclassicany, i[0])
    runners_sheet.cell(row=runnercount+4, column=59).value = calcPoints(pop2k8anySt, i[0])
    runners_sheet.cell(row=runnercount+4, column=60).value = calcPoints(pop2k8anyNMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=61).value = calcPoints(pop2k8AS, i[0])
    runners_sheet.cell(row=runnercount+4, column=62).value = calcPoints(pop2k8Epilougeany, i[0])
    runners_sheet.cell(row=runnercount+4, column=63).value = calcPoints(pop2k8Hacked, i[0])
    runners_sheet.cell(row=runnercount+4, column=64).value = calcPoints(pop2k8anyDupe, i[0])
    runners_sheet.cell(row=runnercount+4, column=65).value = calcPoints(poptfkanyCon, i[0])
    runners_sheet.cell(row=runnercount+4, column=66).value = calcPoints(poptfkanyEmu, i[0])
    runners_sheet.cell(row=runnercount+4, column=67).value = calcPoints(poptfsanySt, i[0])
    runners_sheet.cell(row=runnercount+4, column=68).value = calcPoints(poptfsanyNEG, i[0])
    runners_sheet.cell(row=runnercount+4, column=69).value = calcPoints(poptfsanyNMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=70).value = calcPoints(poptfsPSPany, i[0])
    runners_sheet.cell(row=runnercount+4, column=71).value = calcPoints(poptfsWiianyNG, i[0])
    runners_sheet.cell(row=runnercount+4, column=72).value = calcPoints(poptfsWiianyNGP, i[0])
    runners_sheet.cell(row=runnercount+4, column=73).value = calcPoints(poptfsDSanyCon, i[0])
    runners_sheet.cell(row=runnercount+4, column=74).value = calcPoints(poptfsDSanyEmu, i[0])
    runners_sheet.cell(row=runnercount+4, column=75).value = calcPoints(mobileHA, i[0])
    runners_sheet.cell(row=runnercount+4, column=76).value = calcPoints(mobileSnFR, i[0])
    runners_sheet.cell(row=runnercount+4, column=77).value = calcPoints(mobileSoT, i[0])
    runners_sheet.cell(row=runnercount+4, column=78).value = calcPoints(mobileWW, i[0])
    runners_sheet.cell(row=runnercount+4, column=79).value = calcPoints(mobileT2T, i[0])
    runners_sheet.cell(row=runnercount+4, column=80).value = calcPoints(mobileClassic, i[0])
    runners_sheet.cell(row=runnercount+4, column=81).value = calcPoints(mobile2k8, i[0])
    runners_sheet.cell(row=runnercount+4, column=82).value = calcPoints(mobileTFS, i[0])
    runners_sheet.cell(row=runnercount+4, column=83).value = calcPoints(multiSTanyZip, i[0])
    runners_sheet.cell(row=runnercount+4, column=84).value = calcPoints(multiSTanyZipl, i[0])
    runners_sheet.cell(row=runnercount+4, column=85).value = calcPoints(multiSTanyNMG, i[0])
    runners_sheet.cell(row=runnercount+4, column=86).value = calcPoints(multiSTCompZip, i[0])
    runners_sheet.cell(row=runnercount+4, column=87).value = calcPoints(multiSTCompZipl, i[0])
    runners_sheet.cell(row=runnercount+4, column=88).value = calcPoints(multiSTCompNMG, i[0])
    runnercount+=1;

runners_wb.save('Leaderboard.xlsx')


import os
from collections import defaultdict

MAX_N = 3
CORPORA_PATH = './corpora/'

def make_freq():
	freqs = []
	for i in range(MAX_N):
		freqs.append(defaultdict(int))

	for file in os.listdir(CORPORA_PATH):
		if file.endswith(".txt"):
			with open(CORPORA_PATH + file, 'r') as f:
				contents = f.read()
				for i in range(len(contents)):
					for j in range(1, MAX_N+1):
						if i + j <= len(contents) and contents[i:i+j].isalpha():
							freqs[j-1][contents[i:i+j]] += 1
	return freqs


import pandas as pd
import time
import spacy
import pickle

nlp = spacy.load("en_core_web_md")
sub_ids = pd.read_csv("../imdb_sub_ids.tsv", sep="\t")

docs_tokens = []
docs_lemmatized = []
docs_pos_tags = []
srts_cleaned = []

srts = []

count = 0
chunk_size=100
chunk_count = 0

doc_index = 0

for index, row in sub_ids.iterrows():
    count += 1

    srt = parse_srt(gunzip(os.path.join("subtitles",str(row["sub_id"]) + ".gz")))
    srt = clean_srt(srt)

    srt = get_continous_text(srt)

    srts.append(srt)

    if count == chunk_size or index == len(sub_ids)-1:
      t0 = time.time()
      count = 0

      for doc in nlp.pipe(srts, batch_size=10000, n_threads=2):
        doc_tokens = []
        doc_lemmatized = []
        doc_pos_tags = []

        for t in doc:
          if t.is_sent_start:
            doc_pos_tags.append("#")
          if t.is_alpha and not t.is_punct and not t.is_stop:
            if t.ent_type not in [378,381,386]:
              doc_tokens.append(t.lower)
              doc_lemmatized.append(t.lemma_)
            else:
              doc_lemmatized.append("-NE-")
          doc_pos_tags.append(t.tag_)

        docs_tokens.append(doc_tokens)
        docs_lemmatized.append(doc_lemmatized)
        docs_pos_tags.append(doc_pos_tags)

        doc_index += 1
        return

      pickle.dump(docs_tokens, open("../pickles/docs_tokens/docs_tokens_{:04d}.pkl".format(chunk_count), "wb"))
      pickle.dump(docs_lemmatized, open("../pickles/docs_lemmatized/docs_lemmatized_{:04d}.pkl".format(chunk_count), "wb"))
      pickle.dump(docs_pos_tags, open("../pickles/docs_pos_tags/docs_pos_tags_{:04d}.pkl".format(chunk_count), "wb"))
      pickle.dump(srts_cleaned, open("../pickles/srts_cleaned/srts_cleaned_{:04d}.pkl".format(chunk_count), "wb"))

      print(time.time()-t0, end=",", flush=True)
      srts_cleaned = []
      docs_tokens = []
      docs_lemmatized = []
      docs_pos_tags = []
      srts = []

      chunk_count += 1


import logging
import cv2
import numpy as np
from deepface.commons.functions import load_image
import tensorflow as tf

tf_version = tf.__version__
tf_major_version = int(tf_version.split(".")[0])
tf_minor_version = int(tf_version.split(".")[1])

if tf_major_version == 1:
    from keras.preprocessing import image
elif tf_major_version == 2:
    from tensorflow.keras.preprocessing import image

log = logging.getLogger("redis")

def preprocess_face(img, target_size=(224, 224), grayscale=False, enforce_detection=True):

    img = load_image(img)
    base_img = img.copy()

    # img, region = detect_face(img=img, detector_backend=detector_backend, grayscale=grayscale,
    #                           enforce_detection=enforce_detection, align=align)

    # --------------------------

    if img.shape[0] == 0 or img.shape[1] == 0:
        if enforce_detection == True:
            raise ValueError("Detected face shape is ", img.shape,
                             ". Consider to set enforce_detection argument to False.")
        else:  # restore base image
            img = base_img.copy()

    # --------------------------

    # post-processing
    if grayscale == True:
        img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    # ---------------------------------------------------
    # resize image to expected shape

    # img = cv2.resize(img, target_size) #resize causes transformation on base image, adding black pixels to resize will not deform the base image

    if img.shape[0] > 0 and img.shape[1] > 0:
        factor_0 = target_size[0] / img.shape[0]
        factor_1 = target_size[1] / img.shape[1]
        factor = min(factor_0, factor_1)

        dsize = (int(img.shape[1] * factor), int(img.shape[0] * factor))
        img = cv2.resize(img, dsize)

        # Then pad the other side to the target size by adding black pixels
        diff_0 = target_size[0] - img.shape[0]
        diff_1 = target_size[1] - img.shape[1]
        if grayscale == False:
            # Put the base image in the middle of the padded image
            img = np.pad(img, ((diff_0 // 2, diff_0 - diff_0 // 2), (diff_1 // 2, diff_1 - diff_1 // 2), (0, 0)),
                         'constant')
        else:
            img = np.pad(img, ((diff_0 // 2, diff_0 - diff_0 // 2), (diff_1 // 2, diff_1 - diff_1 // 2)), 'constant')

    # ------------------------------------------

    # double check: if target image is not still the same size with target.
    if img.shape[0:2] != target_size:
        img = cv2.resize(img, target_size)

    # ---------------------------------------------------

    # cv2.imwrite("C:\\Users\\Kaustav\\Downloads\\istockphoto-1364222624-170667a__preprocessed___{face_count}.jpg",img)

    # normalizing the image pixels

    img_pixels = image.img_to_array(img)  # what this line doing? must?
    img_pixels = np.expand_dims(img_pixels, axis=0)
    img_pixels /= 255  # normalize input in [0, 1]

    # ---------------------------------------------------

    return img_pixels


from abc import ABC, abstractmethod
import numpy as np
import pandas as pd
from typing import Dict, List, Optional

from .utils import KeyHashable
from .utils.rules import Rule, make_single_feature_rule

class RuleSeeder(ABC, KeyHashable):
    """Base class for seed rule/LF generators."""

    @abstractmethod
    def get_seed_rules(self,
                       classes: np.array,
                       train_binary_features: pd.DataFrame,
                       train_y: pd.Series,
                       rngseed: int) -> List[Rule]:
        """Generates a set of seed rules/LFs."""
        pass

class BlankRS(RuleSeeder):
    """Generates no seed rules"""

    def get_seed_rules(self,
                       classes: np.array,
                       train_binary_features: pd.DataFrame,
                       train_y: pd.Series,
                       rngseed: int) -> List[Rule]:
        return []

class AccRS(RuleSeeder):
    """Generates seed rules for random features within a range of
    acceptable accuracies:

    min_cov_ratio: The minimum coverage for each seed rule.
    min_accuracy_gap: The minimum accuracy gap above a random classifier of seed rules.
    max_accuracy_gap: The maximum accuracy gap above a random classifier of seed rules.
    rpc: The number of seed rules to generate for each class.
    """

    def __init__(self,
                 min_cov_ratio: float = 0.02,
                 min_acc_gap: float = 0.2,
                 max_acc_gap: float = 0.35,
                 rpc: int = 2):
        self.min_cov_ratio = min_cov_ratio
        self.min_accuracy_gap = self.min_acc_gap = min_acc_gap
        self.max_accuracy_gap = self.max_acc_gap = max_acc_gap
        self.rules_per_class = self.rpc = rpc

    def check_accuracy(self, accuracy: float, *, class_count: int) -> bool:
        """Check whether the given accuracy is within the acceptable range."""
        chance_accuracy = 1 / class_count
        if (self.min_accuracy_gap is not None
                and accuracy < (chance_accuracy + self.min_accuracy_gap)):
            return False
        if (self.max_accuracy_gap is not None
                and accuracy > (chance_accuracy + self.max_accuracy_gap)):
            return False
        return True

    def get_potential_class_rules(self,
                                  classes: np.array,
                                  train_binary_features: pd.DataFrame,
                                  train_y: pd.Series) -> Dict[str, List[Rule]]:
        """Return all potential rules for each class that meet requirements."""
        column_mask = train_binary_features.mean() >= self.min_cov_ratio
        train_binary_features = train_binary_features.loc[:, column_mask]

        potential_class_rules: Dict[str, List[Rule]] = {
            target_class: [] for target_class in classes
        }
        for feature_name in train_binary_features.columns:
            covered_mask = train_binary_features[feature_name]
            covered_y = train_y[covered_mask]
            class_proportions = covered_y.value_counts(normalize=True, sort=True)
            most_frequent_class = class_proportions.index[0]
            if self.check_accuracy(class_proportions[most_frequent_class],
                                   class_count=classes.shape[0]):
                rule = make_single_feature_rule(most_frequent_class, feature_name)
                potential_class_rules[most_frequent_class].append(rule)
        return potential_class_rules

    def get_seed_rules(self,
                       classes: np.array,
                       train_binary_features: pd.DataFrame,
                       train_y: pd.Series,
                       rngseed: int) -> List[Rule]:
        potential_class_rules = self.get_potential_class_rules(
            classes=classes,
            train_binary_features=train_binary_features,
            train_y=train_y,
        )

        rng = np.random.RandomState(rngseed)
        seed_rules = []
        for target_class in classes:
            if len(potential_class_rules[target_class]) < self.rules_per_class:
                raise ValueError(f'Not enough potential seed rules for class: {target_class}')
            seed_rules += rng.choice(potential_class_rules[target_class],
                                     self.rules_per_class,
                                     replace=False).tolist()
        return seed_rules

class ClassSubsetAccRS(AccRS):
    """Generates seed rules for random features within a range of
    acceptable accuracies:

    min_cov_ratio: The minimum coverage for each seed rule.
    min_accuracy_gap: The minimum accuracy gap above a random classifier of seed rules.
    max_accuracy_gap: The maximum accuracy gap above a random classifier of seed rules.
    rpc: The number of seed rules to generate for each class.
    c: Number of random classes to generate seed rules for (all classes if None).
    """
    def __init__(self,
                 min_cov_ratio: float = 0.02,
                 min_acc_gap: float = 0.2,
                 max_acc_gap: float = 0.35,
                 rpc: int = 2,
                 c: Optional[int] = None):
        super().__init__(
            min_cov_ratio=min_cov_ratio,
            min_acc_gap=min_acc_gap,
            max_acc_gap=max_acc_gap,
            rpc=rpc,
        )
        self.class_count = self.c = c

    def get_seed_rules(self,
                       classes: np.array,
                       train_binary_features: pd.DataFrame,
                       train_y: pd.Series,
                       rngseed: int) -> List[Rule]:
        potential_class_rules = self.get_potential_class_rules(
            classes=classes,
            train_binary_features=train_binary_features,
            train_y=train_y,
        )
        potential_classes = [
            target_class for target_class in classes
            if len(potential_class_rules[target_class]) >= self.rules_per_class
        ]
        seed_class_count = self.c if self.c is not None else len(classes)
        if len(potential_classes) < seed_class_count:
            raise ValueError(f'Not enough potential rules for {seed_class_count} classes.')

        rng = np.random.RandomState(rngseed)
        seed_classes = (rng.choice(potential_classes, self.c, replace=False)
                        if self.c is not None else classes)
        seed_rules = []
        for target_class in seed_classes:
            seed_rules += rng.choice(potential_class_rules[target_class],
                                     self.rules_per_class,
                                     replace=False).tolist()
        return seed_rules


import numpy as np
import torch
from torch import nn
from torch.nn import functional as F
import torch.utils.data as data

class PairwiseInteractionsMLP(data.Dataset):
    """
    Sample data from an interactions matrix in a pairwise fashion. The row is
    treated as the main dimension, and the columns are sampled pairwise.
    """

    def __init__(self, data, n_items, n_challenges_per_user,
                 use_item_attr, seen_items_dct_all, item_attr_dct,
                 seed=1):
        self.df = data
        self.seen_items_dct_all = seen_items_dct_all
        self.item_attr_dct = item_attr_dct
        self.seed = seed
        self.n_items = n_items
        self.n_challenges_per_user = n_challenges_per_user
        self.use_item_attr = use_item_attr
        self.user_cols = ['user_id']
        self.item_cols = ['programming_language', 'challenge_series_ID',
                          'author_ID', 'author_gender',
                          'author_org_ID', 'category_id']
        self.joint_cols = ['challenge_sequence']
        if self.use_item_attr:
            self.cat_cols = self.user_cols + ['challenge'] + self.item_cols + self.joint_cols
        else:
            self.cat_cols = self.user_cols + ['challenge']
        self.num_cols = ['total_submissions']

    def __getitem__(self, index):
        row = self.df.iloc[index]
        user_id = row['user_id']
        user_values = [user_id]

        pos_cat_values = np.array([row[col] for col in self.cat_cols])
        pos_num_values = np.array([row[col] for col in self.num_cols])

        np.random.seed(self.seed)
        neg_item = np.random.permutation(
            [x for x in range(self.n_items) if x not in
             self.seen_items_dct_all[user_id]])[0]
        neg_item_values = [self.item_attr_dct[neg_item][i] for i, col in
                           enumerate(self.item_cols)]
        neg_num_values = np.array([self.item_attr_dct[neg_item][6]])

        if self.use_item_attr:
            neg_joint_values = [self.n_challenges_per_user]
            neg_cat_values = np.array(user_values + [neg_item] + neg_item_values +
                                      neg_joint_values)
        else:
            neg_cat_values = np.array(user_values + [neg_item])

        return pos_cat_values, pos_num_values, neg_cat_values, neg_num_values

    def __len__(self):
        return self.df.shape[0]

class BPRModuleMLP(nn.Module):
    """
    Defines the neural network for challenge recommendation
    """

    def __init__(self, embedding_sizes, n_cont):
        super().__init__()
        self.embeddings = nn.ModuleList([nn.Embedding(categories, size) for
                                         categories, size in embedding_sizes])
        n_emb = sum(e.embedding_dim for e in self.embeddings)
        self.n_emb, self.n_cont = n_emb, n_cont
        self.lin1 = nn.Linear(self.n_emb + self.n_cont, 1)
        self.bn1 = nn.BatchNorm1d(self.n_cont)
        self.emb_drop = nn.Dropout(0.6)

    def calc_pred(self, x_cat, x_cont):
        x = [e(x_cat[:, i]) for i, e in enumerate(self.embeddings)]
        x = torch.cat(x, 1)
        x = self.emb_drop(x)
        x2 = self.bn1(x_cont)
        x = torch.cat([x, x2], 1)
        x = self.lin1(x)
        return x

    def forward(self, pos_cat, pos_num, neg_cat, neg_num):
        pos_preds = self.calc_pred(pos_cat, pos_num)
        neg_preds = self.calc_pred(neg_cat, neg_num)
        return pos_preds - neg_preds


# -*- coding: utf-8 -*-
"""
Created on Wed Aug  7 09:39:48 2019

@author: Edward Carney
"""

import math
import matplotlib.pyplot as plt

class MaskHandler():
    """
    A class to handle creation of azimuth-elevation files for STK. Intended
    to be used for the BridgeSat program to generate azimuth-files for partial
    roof opening/closure.
    """
    def __init__(self, pivotAz, shutterEl_r, shutterEl_l):

        # define mask file fidelity (assumed 360 for now)
        self._fidelity = 360

        # define pivot azimuth in degrees CW from North (e.g. 0 => pivot aligns
        # with NS line, 90 => pivot aligns with EW line)
        self.pivotAz = pivotAz

        # define left and right shutter elevation
        self.shutterEl_r = shutterEl_r
        self.shutterEl_l = shutterEl_l

        # define azimuth and elevation lists; will use 1-deg fidelity; need to
        # have both 0 and 360 points (total of fidelity + 1 points)
        self.az = list(range(self._fidelity + 1))
        self.el = [0] * (self._fidelity + 1)

        # define azimuths for start, final, and max elevation of left and right
        # shutter
        self._shutterStartAz_r = int(math.floor(self.pivotAz))
        self._shutterMaxElAz_r = int(math.floor(self.pivotAz + (self._fidelity / 4)))
        self._shutterFinalAz_r = int(math.floor(self.pivotAz + (self._fidelity / 2)))

        self._shutterStartAz_l = int(math.floor(self.pivotAz + (self._fidelity / 2)))
        self._shutterMaxElAz_l = int(math.floor(self.pivotAz + (3 * self._fidelity / 4)))
        self._shutterFinalAz_l = int(math.floor(self.pivotAz + self._fidelity))

        # generate mask values
        self._generateMask()

    def _setElevation(self, startAz, finalAz, maxEl):
        """
        Sets elevation values for a defined range of azimuth. Will do a sinusoidal progression
        from zero to the max elevation and back to zero.
        """

        # define the elevation function based on the max elevation to be used
        if maxEl == 90.0:
            elFunc = lambda az: 90.0
        elif maxEl > 90.0:
            elFunc = lambda az: 180.0 - abs(math.sin(math.pi / 180.0 * (az - self.pivotAz)) * (180.0 - maxEl))
        else:
            elFunc = lambda az: abs(math.sin(math.pi / 180.0 * (az - self.pivotAz)) * maxEl)

        # iterate through all azimuth values and set elevation
        for az in range(startAz, finalAz):
            self.el[az % self._fidelity] = elFunc(az)

    def _generateMask(self):
        """
        Sets object mask list based on left and right shutter settings.
        """

        # set elevation for the left and right shutters
        self._setElevation(self._shutterStartAz_r, self._shutterFinalAz_r, self.shutterEl_r)
        self._setElevation(self._shutterStartAz_l, self._shutterFinalAz_l, self.shutterEl_l)

        # make sure the first and last values are the same
        self.el[-1] = self.el[0]

    def plotData(self):
        """
        Plots the elevation vs azimuth for easy visualization.
        """
        plt.plot(self.az[:-1], self.el[:-1])
        plt.xlabel("Azimuth (deg)")
        plt.ylabel("Elevation (deg)")
        plt.title("Elevation vs. Azimuth")

        print("INFO: Plotting data. Close out window to continue.")

        plt.show()

    def printToFile(self, fileName):
        """
        Prints azimuth-elevation data to a provided file in STK *.aem file format.
        """

        with open(fileName, 'w') as outFile:
            # write out default file header data
            outFile.write("stk.v.11.2.1\nBEGIN AzElMask\nNumberOfPoints {0}\nBEGIN AzElMaskData\n\n".format(int(self._fidelity + 1)))

            # write out actual data
            for i in range(self._fidelity + 1):
                outFile.write("{0} {1}\n".format(self.az[i], self.el[i]))

            # note do NOT add these lines to the end or else things break
            # outFile.write("\nEND AzElMaskData\nEND AzElMask")


import logging
import sys
from redis import Redis
from botocore.exceptions import ClientError
import json
import time
from threading import Timer
import boto3
import uuid
import os
# Setup logging to stdout
logger = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)

DYNAMO_DB = "Raspberry_Car_Object"
S3_BUCKET = "iot-raspberry-car"
S3_KEY = "camera_images/"

def s3_upload(img_local_path):
    try:
        logging.debug("s3_upload: entry")
        file_name = os.path.basename(img_local_path)
        s3 =boto3.resource('s3')
        logging.debug("s3_upload: client loaded.")
        s3.meta.client.upload_file(img_local_path, S3_BUCKET, S3_KEY+file_name)
        #s3_client.upload_file(img_local_path, S3_BUCKET, S3_KEY+file_name)
        logging.debug("s3_upload: success: ")
        logging.debug("s3_upload: %s"%img_local_path)
        logging.debug("s3_upload: %s"%S3_BUCKET)
        logging.debug("s3_upload: %s"%file_name)
    except ClientError as e:
        logging.error(e)
        return False
    return True

def upload_img_to_cloud():
    try:
        cli = Redis('localhost')
        while True:
            item = cli.lpop('car')
            if not item:
                logger.info("car queue is empty")
                break
            dt = json.loads(item)
            img_local_path = str(dt['img_local_path'])
            detection_time = str(dt['detection_time'])
            position = str(dt['position'])
            rfid_pos = str(dt['rfid_pos'])
            logging.debug("redis data is below: ")
            logging.debug(dt)
            if s3_upload(img_local_path):
                uniqueid = uuid.uuid4().hex
                file_name = os.path.basename(img_local_path)
                try:
                    logging.debug("dynamodb: entry")
                    dynamodb_client = boto3.client('dynamodb')
                    dynamodb_client.put_item(TableName=DYNAMO_DB,
                                Item={'detection_time': {'S':detection_time},
                                    'position':{'S': position},
                                    'rfid_pos':{'S': rfid_pos},
                                    's3_path':{'S': S3_KEY+file_name},
                                    'hash_tag':{'S': uniqueid} })
                    logging.debug("dynamodb: success")
                except ClientError as e:
                    logger.error(e)

    except Exception as e:
        logger.error("upload_img_to_cloud error: " + repr(e))

    # Asynchronously schedule this function to be run again in 5 seconds
    Timer(10, upload_img_to_cloud).start()

# Start executing the function above
upload_img_to_cloud()

# This is a dummy handler and will not be invoked
# Instead the code above will be executed in an infinite loop for our example
def function_handler(event, context):
    return


import cv2
import numpy as np
import os

pixel_norm_factor = 255

def read_and_transform_img(fpath):
    '''
    Reads image and transforms it for modeling. Inverse of normalize_pred_img_array fn.
    '''
    return tf_to_th_dim_order(cv2.imread(fpath)).astype(float) / pixel_norm_factor

def normalize_pred_img_array(img):
    '''
    Convert predicted values from generated model to properly formatted image
    Largely the inverse function of read_and_transform_img
    '''
    img_with_index = img[0,:,:,:]
    out = th_to_tf_dim_order(img_with_index) * pixel_norm_factor
    out = out.clip(0,255).astype('uint8')
    return out

def tf_to_th_dim_order(img):
    '''Moves channel from index position 2 to index position 0'''
    return img.swapaxes(1, 2).swapaxes(0, 1)

def th_to_tf_dim_order(img):
    '''Moves channel from index position 0 to index position 2'''
    return img.swapaxes(0, 1).swapaxes(1, 2)

def train_val_split(raw_vids_dir, num_training_images, images_per_video):
    vid_fnames = os.listdir(raw_vids_dir)
    train_fnames = []
    val_fnames = []
    for vid_fname_with_extension in vid_fnames:
        vid_name = vid_fname_with_extension.split('.')[0]
        train_fnames += [vid_name + '_' + str(i) + '.jpg' for i in range(num_training_images)]
        val_fnames += [vid_name + '_' + str(i) + '.jpg' for i in range(num_training_images, images_per_video)]
    return train_fnames, val_fnames

def save_predicted_images(gen_model, val_fnames):
    '''
    Creates predicted versions of clear image and saves it in tmp/predicted.
    '''
    for fname in val_fnames:
        blurred_img = read_and_transform_img('./tmp/blur/' + fname)
        blurred_img = blurred_img[np.newaxis, :, :, :]  # Image number is new index 0
        img = gen_model.predict(blurred_img)
        img = normalize_pred_img_array(img)
        out_fname = './tmp/predicted/' + fname
        cv2.imwrite(out_fname, img)