import streamlit as st
import datetime
import json

st.markdown("# Bestellen ðŸ›’")
st.sidebar.markdown("# Bestellen ðŸ›’")
#test
# Datenbank-Datei fÃ¼r Werkzeugnisinformationen im JSON-Format
database_filename = "bestellungen_database.json"

# Laden der bestehenden Werkzeugnisdaten aus der JSON-Datei
def load_existing_data(filename):
    try:
        with open(filename, "r") as file:
            data = [json.loads(line) for line in file]
        return data
    except (FileNotFoundError, json.JSONDecodeError):
        return []

existing_data = load_existing_data(database_filename)

# Seitentitel
#st.title("Bestellung aufgeben")

# Kunde
kunde = st.text_input("Kundenname")

# Automatisches EinfÃ¼gen des aktuellen Datums und der Uhrzeit
current_datetime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

# Generiere die Auftragsnummer aus den ersten Buchstaben des Kunden und dem Datum
if kunde:
    auftragsnummer = kunde[:3].upper() + current_datetime.replace("-", "").replace(" ", "").replace(":", "")
    st.write(f"Auftragsnummer: {auftragsnummer}")

st.write(f"Bestellung vom: {current_datetime}")
# Auswahl der Bestellvarianten
st.write("WÃ¤hlen Sie Ihre Farben aus:")

varianten_farben = {
    "FÃ¼hrerhaus": ["Blau", "Rot", "Gelb"],
    "Sidepipes": ["Rot", "Blau"],
    "Container 1": ["GrÃ¼n", "Gelb", "Blau"],
    "Container 2": ["GrÃ¼n", "Gelb", "Blau"],
    "Container 3": ["GrÃ¼n", "Gelb", "Blau"],
    "Container 4": ["GrÃ¼n", "Gelb", "Blau"]

}

selected_variants = {}

# Erstelle 6 Spalten
columns = st.columns(6)

for variante, farben in varianten_farben.items():
    #with columns[0]:
      #  st.write(variante)
    with columns[1]:
        selected_color = st.radio(f"Auswahl {variante}", farben)
        if selected_color:
            selected_variants[variante] = selected_color

# Sonderwunsch
sonderwunsch = st.text_input("Sonderwunsch", "")

# Kundentakt
Kundentakt = st.text_input("Kundentakt", "")

# SchaltflÃ¤che, um Bestellung abzuschicken
if st.button("Bestellung abschicken"):

    # Speichern der Bestellinformationen in der Datenbank als separates JSON-Objekt pro Zeile
    bestellungen_info = {
        "Bestelldatum und Uhrzeit": current_datetime,
        "Kunde": kunde,
        "Auftragsnummer": auftragsnummer,
        "Sonderwunsch": sonderwunsch,
        "Variante nach Bestellung": selected_variants,
        "Kundentakt": Kundentakt,
    }
    existing_data.append(bestellungen_info)  # HinzufÃ¼gen der neuen Daten zu den vorhandenen Daten

    with open(database_filename, "w") as db:
        for entry in existing_data:
            db.write(json.dumps(entry) + "\n")

    st.write("Die Bestellung wurde abgeschickt")
    # Laden der bestehenden Werkzeugnisdaten aus der JSON-Datei
    st.experimental_rerun()


import pandas
import numpy as np
from scipy.stats import linregress
import matplotlib.pyplot as plt

def linear_regression(x, y):
    x = np.array(x)
    y = np.array(y)
    slope, intercept, r_value, p_value, std_err = linregress(x, y)
    return slope, intercept, r_value

table = pandas.read_csv("./data.csv")
x_data = table["Assault"]
y_data = table["Murder"]

slope, intercept, correlation_coefficient = linear_regression(x_data, y_data)

print("Slope (m):", slope)
print("Intercept (c):", intercept)
print("Correlation Coefficient (r):", correlation_coefficient)

plt.figure(figsize=(16, 10))

regression_line = x_data*slope + intercept
plt.scatter(x_data, y_data.where(y_data > regression_line), color='red')
plt.scatter(x_data, y_data.where(y_data < regression_line), color='green')
plt.plot(x_data, regression_line, color='red', label='Linear Regression Line')
plt.title(label="Murder / Assault", fontsize=40, color="green")
plt.savefig("output.jpg")


import os
import threading

import rospy

from sound_play.msg import SoundRequest

try:
    import gi
    gi.require_version('Gst', '1.0')
    from gi.repository import Gst as Gst
except Exception:
    str = """
**************************************************************
Error opening pygst. Is gstreamer installed?
**************************************************************
"""
    rospy.logfatal(str)
    # print str
    exit(1)

class SoundType(object):
    STOPPED = 0
    LOOPING = 1
    COUNTING = 2

    def __init__(self, file, device, volume=1.0):
        self.lock = threading.RLock()
        self.state = self.STOPPED
        self.sound = Gst.ElementFactory.make("playbin", None)
        if self.sound is None:
            raise Exception("Could not create sound player")

        if device:
            self.sink = Gst.ElementFactory.make("alsasink", "sink")
            self.sink.set_property("device", device)
            self.sound.set_property("audio-sink", self.sink)

        if (":" in file):
            uri = file
        elif os.path.isfile(file):
            uri = "file://" + os.path.abspath(file)
        else:
            rospy.logerr('Error: URI is invalid: %s' % file)

        self.uri = uri
        self.volume = volume
        self.sound.set_property('uri', uri)
        self.sound.set_property("volume", volume)
        self.staleness = 1
        self.file = file

        self.bus = self.sound.get_bus()
        self.bus.add_signal_watch()
        self.bus_conn_id = self.bus.connect("message", self.on_stream_end)

    def on_stream_end(self, bus, message):
        if message.type == Gst.MessageType.EOS:
            if (self.state == self.LOOPING):
                self.sound.seek_simple(Gst.Format.TIME, Gst.SeekFlags.FLUSH, 0)
            else:
                self.stop()

    def __del__(self):
        # stop our GST object so that it gets garbage-collected
        self.dispose()

    def update(self):
        if self.bus is not None:
            self.bus.poll(Gst.MessageType.ERROR, 10)

    def loop(self):
        self.lock.acquire()
        try:
            self.staleness = 0
            if self.state == self.COUNTING:
                self.stop()

            if self.state == self.STOPPED:
                self.sound.seek_simple(Gst.Format.TIME, Gst.SeekFlags.FLUSH, 0)
                self.sound.set_state(Gst.State.PLAYING)
            self.state = self.LOOPING
        finally:
            self.lock.release()

    def dispose(self):
        self.lock.acquire()
        try:
            if self.bus is not None:
                self.sound.set_state(Gst.State.NULL)
                self.bus.disconnect(self.bus_conn_id)
                self.bus.remove_signal_watch()
                self.bus = None
                self.sound = None
                self.sink = None
                self.state = self.STOPPED
        except Exception as e:
            rospy.logerr('Exception in dispose: %s' % str(e))
        finally:
            self.lock.release()

    def stop(self):
        if self.state != self.STOPPED:
            self.lock.acquire()
            try:
                self.sound.set_state(Gst.State.NULL)
                self.state = self.STOPPED
            finally:
                self.lock.release()

    def single(self):
        self.lock.acquire()
        try:
            rospy.logdebug("Playing %s" % self.uri)
            self.staleness = 0
            if self.state == self.LOOPING:
                self.stop()

            self.sound.seek_simple(Gst.Format.TIME, Gst.SeekFlags.FLUSH, 0)
            self.sound.set_state(Gst.State.PLAYING)
            self.state = self.COUNTING
        finally:
            self.lock.release()

    def command(self, cmd):
        if cmd == SoundRequest.PLAY_STOP:
            self.stop()
        elif cmd == SoundRequest.PLAY_ONCE:
            self.single()
        elif cmd == SoundRequest.PLAY_START:
            self.loop()

    def get_staleness(self):
        self.lock.acquire()
        position = 0
        duration = 0
        try:
            position = self.sound.query_position(Gst.Format.TIME)[1]
            duration = self.sound.query_duration(Gst.Format.TIME)[1]
        except Exception:
            position = 0
            duration = 0
        finally:
            self.lock.release()

        if position != duration:
            self.staleness = 0
        else:
            self.staleness = self.staleness + 1
        return self.staleness

    def get_playing(self):
        return self.state == self.COUNTING


import re
import os
import glob
import json
import time
import requests
import threading
import fcntl
import socket
import struct
import mycommon as com
from math import sqrt

global first_time_set
global last_time_set
global header
global server_url
global sd_keys
global nfps
global people_counting_enabled
global aforo_enabled
global social_distance_enabled
global social_distance_url
global people_counting_url
global plate_detection_url

header = None

first_time_set = set()
last_time_set = set()

# #### GENERIC FUNCTIONS

def log_error(msg, _quit = True):
    print("-- PARAMETER ERROR --\n"*5)
    print(" %s \n" % msg)
    print("-- PARAMETER ERROR --\n"*5)
    if _quit:
        quit()
    else:
        return False

def api_get_number_of_frames_per_second():
    '''
    TODO: function not yet defined
    '''
    return None

def file_exists(file_name):
    try:
        with open(file_name) as f:
            return file_name
    except OSError as e:
        return False

def open_file(file_name, option='a+'):
    if file_exists(file_name):
        return open(file_name, option)
    return False

def create_file(file_name, content=None):
    if file_exists(file_name):
        os.remove(file_name)
        if file_exists(file_name):
            raise Exception('unable to delete file: %s' % file_name)
    if content:
        with open(file_name, 'w+') as f:
            f.write(content)
    else:
        with open(file_name, 'w+') as f:
            f.close()

    return True

def get_number_of_frames_per_second():
    global nfps

    nfps = api_get_number_of_frames_per_second()

    if nfps is None:
        return 16

    return nfps

def get_supported_actions():
    valid_values = ('GET', 'POST', 'PUT', 'DELETE')
    return valid_values

def get_timestamp():
    return int(time.time() * 1000)

def set_header(token_file = None):
    if token_file is None:
        token_file = '.token'

    global header

    if header is None:
        if isinstance(token_file, str):
            token_handler = open_file(token_file, 'r+')
            if token_handler:
                header = {'Content-type': 'application/json', 'X-KAIROS-TOKEN': token_handler.read().split('\n')[0]}
                print('Header correctly set')
                return True

    return False

def getHwAddr(ifname):
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    info = fcntl.ioctl(s.fileno(), 0x8927,  struct.pack('256s', bytes(ifname, 'utf-8')[:15]))
    return ':'.join('%02x' % b for b in info[18:24])

def get_ip_address(ifname):
    return [l for l in ([ip for ip in socket.gethostbyname_ex(socket.gethostname())[2] if not ip.startswith("127.")][:1], [[(s.connect(("8.8.8.8", 53)), s.getsockname()[0], s.close()) for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]][0][1]]) if l][0][0]

def get_machine_mac_addresses():
    list_of_interfaces = [item for item in os.listdir('/sys/class/net/') if item != 'lo']
    macaddress_list = []

    for iface_name in list_of_interfaces:
        ip = get_ip_address(iface_name)
        if ip:
            macaddress_list.append(getHwAddr(iface_name))
            return macaddress_list

def get_server_info_from_local_file(filename, _quit=True):
    if file_exists(filename):
        with open(filename) as f:
            data = json.load(f)
            if isinstance(data, dict):
                return data
            return log_error("data unknown error, data is not a dictionary: {}")
    else:
        return log_error("Unable to read the device configuration from local file: {}".format(filename), _quit)

def parse_parameters_and_values_from_config(config_data):
    # filter config and get only data for this server using the mac to match
    config_data = get_config_filtered_by_local_mac(config_data)

    # filter config and get only data of active services
    return get_config_filtered_by_active_service(config_data)

def get_config_filtered_by_local_mac(config_data):
    '''
    By now we only support one nano server and one interface
    but it can be a big server with multiple interfaces so I
    leave the logic with to handle this option
    '''
    services_data = {}
    for key in config_data.keys():
        if mac_address_in_config(key):
            services_data[key] = config_data[key]
    if services_data:
        return services_data

    log_error("The provided configuration does not match any of server interfaces mac address")

def get_config_filtered_by_active_service(config_data):
    if not isinstance(config_data, dict):
        log_error("Configuration error - Config data must be a dictionary - type: {} / content: {}".
                  format(type(config_data), config_data))
    active_services = {}

    # at this point there should be only one server mac, but we loop in case we have many multiple networks interfaces
    for server_mac in config_data.keys():
        #  we loop over all the different cameras attach to this server
        for camera_mac in config_data[server_mac]:
            # we loop over all the services assigned to the camera
            for service in config_data[server_mac][camera_mac]:
                # if the service is enabled we add it to the active services
                if 'enabled' in config_data[server_mac][camera_mac][service] and \
                        config_data[server_mac][camera_mac][service]['enabled'] is True:
                    if 'source' not in config_data[server_mac][camera_mac][service]:
                        log_error("Service {} must have a source (video or live streaming)".format(service))

                    # Create new key only for each of the active services
                    new_key_name = 'srv_' + server_mac + "_camera_" + camera_mac + '_' + service
                    active_services[new_key_name] = {service: config_data[server_mac][camera_mac][service]}

    if len(active_services) < 1:
        com.log_error("\nConfiguration does not contain any active service for this server: \n\n{}".format(config_data))

    return active_services

def mac_address_in_config(mac_config):
    for machine_id in com.get_machine_mac_addresses():
        if mac_config == machine_id:
            return True
    return False

def send_json(payload, action, url=None, **options):
    set_header()
    global header

    if action not in get_supported_actions() or url is None:
        raise Exception('Requested action: ({}) not supported. valid options are:'.format(action, get_supported_actions()))

    retries = options.get('retries', 2)
    sleep_time = options.get('sleep_time', 1)
    expected_response = options.get('expected_response', 200)
    abort_if_exception = options.get('abort_if_exception', True)

    data = json.dumps(payload)

    # emilio comenta esto para insertar en MongoDB
    # return True

    for retry in range(retries):
        try:
            if action == 'GET':
                r = requests.get(url, data=data, headers=header)
            elif action == 'POST':
                r = requests.post(url, data=data, headers=header)
            elif action == 'PUT':
                r = requests.put(url, data=data, headers=header)
            else:
                r = requests.delete(url, data=data, headers=header)
            return r
        except requests.exceptions.ConnectionError as e:
            time.sleep(sleep_time)
            if retry == retries - 1 and abort_if_exception:
                raise Exception("Unable to Connect to the server after {} retries\n. Original exception: {}".
                                format(retry, str(e)))
        except requests.exceptions.HTTPError as e:
            time.sleep(sleep_time)
            if retry == retries - 1 and abort_if_exception:
                raise Exception("Invalid HTTP response in {} retries\n. Original exception: {}".format(retry, str(e)))
        except requests.exceptions.Timeout as e:
            time.sleep(sleep_time)
            if retry == retries - 1 and abort_if_exception:
                raise Exception("Timeout reach in {} retries\n. Original exception: {}".format(retry, str(e)))
        except requests.exceptions.TooManyRedirects as e:
            time.sleep(sleep_time)
            if retry == retries - 1 and abort_if_exception:
                raise Exception("Too many redirection in {} retries\n. Original exception: {}".format(retry, str(e)))

def check_if_object_is_in_area2(object_coordinates, reference_line, m, b):
    '''
    * returns True if object is in Area2
    * returns False if object is in Area1
    * si m es None la pendiente es infinita es decir recta vertical a 90 grados con respecto de la horizontal
    * si m es 0 recta a horizontal
    '''
    if m is None:
        '''
        # object_coordinates[0] ->  x
        # reference_line[0][0]  ->  x1
        # if x > x1 then is in Area2, else in Area1
        '''
        if object_coordinates[0] > reference_line[0][0]:
            return True
        return False
    elif m == 0:
        '''
        # object_coordinates[1] -  y
        # reference_line[0][1]  -  y1
        # if y > y1 then is in Area2, else in Area1
        '''
        if object_coordinates[1] > reference_line[0][1]:
            return True
        return False
    else:
        '''
        # x1 = reference_line[0][0]
        # y1 = reference_line[0][1]
        # x = object_coordinates[0]
        # m = ((y2 - y1) * 1.0) / (x2 - x1) ... multiply by 1.0 to force float division
        # y_over_the_line = (m * (x - x1)) + y1  ... general line equation

        *** in the video the y's values increases while going down
        *** Are2 is always the higher values of y's after the reference line

        -------------      ------------
        | A1        /      |  \
        |          /       |   \
        | p=x,y   /        |    \   A1
        |        /  A2     | A   \
        |       /          |      \

        Given a point p=x,y the "x" is evaluated in the line equation, if the
        calculated "y_over_the_line" is lower than the "y" of
        the point "p", then the point is down the line and so in A2
        '''
        y_over_the_line = (m * object_coordinates[0]) + b
        '''
        # if y > y_overtheline point is in Area2
        '''
        if object_coordinates[1] > y_over_the_line:
            return True
        else:
            return False

def is_point_insde_polygon(x, y, polygon_length, polygon):

    p1x, p1y = polygon[0]
    for i in range(polygon_length+1):
        p2x, p2y = polygon[i % polygon_length]
        if y > min(p1y, p2y):
            if y <= max(p1y, p2y):
                if x <= max(p1x, p2x):
                    if p1y != p2y:
                        xinters = (y-p1y)*(p2x-p1x)/(p2y-p1y)+p1x
                    if p1x == p2x or x <= xinters:
                        # returns True if x,y are inside
                        return True
        p1x, p1y = p2x, p2y

    # returns False if x,y are not inside
    return False

##### PEOPLE COUNTING

def set_service_people_counting_url(server_url):
    global people_counting_url
    people_counting_url = server_url + 'SERVICE_NOT_DEFINED_'

def people_counting(camera_id, total_objects):
    '''
    Sending only the total of detected objects
    '''
    global people_counting_url

    date = get_timestamp()
    alert_id = str(date) + '_' + str(camera_id) + '_' + str(date)
    data = {
            'id': alert_id,
            'camera-id': camera_id,
            '#total_updated_at': date,
            'object_id': total_objects,
            }
    #print('People_counting first time..POST', data, people_counting_url)
    #x = threading.Thread(target=send_json, args=(data, 'POST', srv_url))
    #x.start()

##### AFORO

#def set_aforo_url():
#    global aforo_url, srv_url
#aforo_url = srv_url + 'tx/video-people.endpoint'

def aforo(aforo_url, box, object_id, camera_id, initial, last, entradas, salidas, outside_area=None,
          reference_line=None, m=None, b=None, rectangle=None):
    '''
    A1 is the closest to the origin (0,0) and A2 is the area after the reference line
    A1 is by default the outside
    A2 is by default the inside
    This can be changed by modifying the configuration variable "outside_area" to 2 (by default 1)
    x = box[0]
    y = box[1]

    initial -  must be a dictionary, and will be used to store the first position (Area1 or Area2) of a given ID
    last -     must be a dictionary, and will be used to store the last position (Area1 or Area2) of a given ID
    '''
    if rectangle:
        # si el punto esta afuera del area de interes no evaluamos
        if box[0] < rectangle[0] or box[0] > rectangle[4] or box[1] > rectangle[5] and box[1] < rectangle[1]:
            if reference_line:
                return entradas, salidas
            else:
                outside_area = 1
                area = 1

    if reference_line:
        if check_if_object_is_in_area2(box, reference_line, m, b):
            area = 2
        else:
            area = 1
    else:
        outside_area = 1
        area = 2

    if outside_area == 1:
        direction_1_to_2 = 1
        direction_2_to_1 = 0
    else:
        direction_1_to_2 = 0
        direction_2_to_1 = 1

    if object_id not in initial:
        initial.update({object_id: area})
        if object_id not in last:
            return entradas, salidas
    else:
        last.update({object_id: area})

    # De igual forma si los elementos continen las misma areas en el estado
    # actual que en el previo, entonces no tiene caso evaluar mas
    if initial[object_id] == last[object_id]:
        return entradas, salidas

    for item in last.keys():
        if initial[item] == 1 and last[item] == 2:
            time_in_epoc = get_timestamp()
            data_id = str(time_in_epoc) + '_' + str(object_id)
            data = {
                    'id': data_id,
                    'direction': direction_1_to_2,
                    'camera-id': camera_id,
                    '#date-start': time_in_epoc,
                    '#date-end': time_in_epoc,
                }
            initial.update({item: 2})

            print('Sending Json of camera_id: ', camera_id, 'ID: ', item, 'Sal:0,Ent:1 = ',
                  direction_1_to_2, "tiempo =", time_in_epoc)
            x = threading.Thread(target=send_json, args=(data, 'PUT', aforo_url,))
            x.start()

            if direction_1_to_2 == 1:
                entradas += 1
            else:
                salidas += 1

        elif initial[item] == 2 and last[item] == 1:
            time_in_epoc = get_timestamp()
            data_id = str(time_in_epoc) + '_' + str(object_id)
            data = {
                    'id': data_id,
                    'direction': direction_2_to_1,
                    'camera-id': camera_id,
                    '#date-start': time_in_epoc,
                    '#date-end': time_in_epoc,
                }
            initial.update({item: 1})

            print('Sending Json of camera_id: ', camera_id, 'ID: ', item, 'Sal:0,Ent:1 = ', direction_2_to_1,
                  "tiempo =", time_in_epoc)
            x = threading.Thread(target=send_json, args=(data, 'PUT', aforo_url,))
            x.start()

            if direction_2_to_1 == 1:
                entradas += 1
            else:
                salidas += 1

    return entradas, salidas

##### SOCIAL DISTANCE

def set_social_distance_url(srv_url):
    global social_distance_url
    social_distance_url = srv_url + 'tx/video-socialDistancing.endpoint'

def social_distance2(camera_id, ids_and_boxes, tolerated_distance, persistence_time, max_side_plus_side, detected_ids):
    '''
    social distance is perform in pairs of not repeated pairs
    Being (A, B, C, D, E, F) the set of detected objects

    The possible permutation are:

       AB AC AD AE AF
          BC BD BE BF
             CD CE CF
                DE DF
                   Ef

    We are going to start compararing the first element (index=0 or i=0)
    '''
    # TODO: diccionario puede crecer mucho depurarlo comparando los elementos que dejen
    #  de existir o no sean detectados despues de 5seg')

    # sorting elements to always have the same evaluation order
    ids = [item for item in ids_and_boxes.keys()]
    ids.sort()
    # creating the list
    i = 1
    for pivot in ids[:-1]:
        for inner in ids[i:]:
            if pivot not in detected_ids:
                Ax = ids_and_boxes[pivot][0]
                x = ids_and_boxes[inner][0]

                if Ax > x:
                    dx = Ax -x
                else:
                    dx = x - Ax

                if dx < tolerated_distance:
                    Ay = ids_and_boxes[pivot][1]
                    y = ids_and_boxes[inner][1]

                    if Ay > y:
                        dy = Ay - y
                    else:
                        dy = y - Ay

                    if (dx + dy) < max_side_plus_side and sqrt((dx*dx) + (dy*dy)) < tolerated_distance:
                        # first time detection for pivot A and associated B
                        pivot_time = get_timestamp()
                        detected_ids.update({
                            pivot: {
                                inner:{
                                    '#detected_at': pivot_time,
                                    '#reported_at': None,
                                    'reported': False,
                                    }
                                }
                            })
            else:
                if inner not in detected_ids[pivot]:
                    Ax = ids_and_boxes[pivot][0]
                    x = ids_and_boxes[inner][0]

                    if Ax > x:
                        dx = Ax -x
                    else:
                        dx = x - Ax

                    if dx < tolerated_distance:
                        Ay = ids_and_boxes[pivot][1]
                        y = ids_and_boxes[inner][1]

                        if Ay > y:
                            dy = Ay - y
                        else:
                            dy = y - Ay

                        if (dx + dy) < max_side_plus_side and sqrt((dx*dx) + (dy*dy)) < tolerated_distance:
                            # first time detection for associated C is registered
                            detected_at_inner = get_timestamp()
                            detected_ids[pivot].update({
                                inner: {
                                    '#detected_at': detected_at_inner,
                                    '#reported_at': None,
                                    'reported': False,
                                    }
                                })
                else:
                    Ax = ids_and_boxes[pivot][0]
                    x = ids_and_boxes[inner][0]

                    if Ax > x:
                        dx = Ax -x
                    else:
                        dx = x - Ax

                    if dx > tolerated_distance:
                        if not detected_ids[pivot][inner]['reported']:
                            del detected_ids[pivot][inner]
                    else:
                        Ay = ids_and_boxes[pivot][1]
                        y = ids_and_boxes[inner][1]

                        if Ay > y:
                            dy = Ay - y
                        else:
                            dy = y - Ay

                        if (dx + dy) >= max_side_plus_side or sqrt((dx*dx) + (dy*dy)) >= tolerated_distance:
                            del detected_ids[pivot][inner]
                        else:
                            current_time = get_timestamp()
                            initial_time = detected_ids[pivot][inner]['#detected_at']
                            if not detected_ids[pivot][inner]['reported'] and \
                                    (current_time - initial_time) >= persistence_time:
                                detected_ids[pivot][inner].update({'#reported_at': current_time})
                                detected_ids[pivot][inner].update({'reported': True})
                                alert_id = str(current_time) + '_'+str(pivot) + '_and_'+str(inner)
                                data = {
                                    'id': alert_id,
                                    'camera-id': camera_id,
                                    '#date': current_time,
                                    }
                                print('Social distance', data, social_distance_url, 'PUT', 'distance=',
                                      sqrt((dx*dx) + (dy*dy)), 'tolerada:', tolerated_distance)
                                x = threading.Thread(target=send_json, args=(data, 'PUT', social_distance_url,))
                                x.start()
            i += 1

#### MASK DETECTION

def set_mask_detection_url(server_url):
    global mask_detection_url
    mask_detection_url = server_url + 'tx/video-maskDetection.endpoint'

def mask_detection(mask_id, no_mask_ids, camera_id, reported_class=0):
    time_in_epoc = get_timestamp()
    data_id = str(time_in_epoc) + '_' + str(mask_id)
    data = {
        'id': data_id,
        'mask': reported_class,
        'camera-id': camera_id,
        '#date-start': time_in_epoc,
        '#date-end': time_in_epoc
        }

    print('Mask detection', data, mask_detection_url, 'PUT')
    x = threading.Thread(target=send_json, args=(data, 'PUT', mask_detection_url,))
    x.start()


import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.autograd as autograd
import torch.optim as optim
import numpy as np
from torch.autograd import Variable

from hybrid_model import VAEDualEncoder
from data import UDCv2
from evaluation import recall_at_k, eval_model_hybrid_v1
from util import save_model

import argparse
from tqdm import tqdm

parser = argparse.ArgumentParser(
    description='UDC Experiment Runner'
)

parser.add_argument('--gpu', default=False, action='store_true',
                    help='whether to run in the GPU')
parser.add_argument('--emb_dim', type=int, default=100, metavar='',
                    help='embedding dimension (default: 100)')
parser.add_argument('--h_dim', type=int, default=100, metavar='',
                    help='hidden dimension (default: 100)')
parser.add_argument('--z_dim', type=int, default=100, metavar='',
                    help='latent dimension (default: 100)')
parser.add_argument('--lr', type=float, default=1e-3, metavar='',
                    help='learning rate (default: 1e-3)')
parser.add_argument('--mb_size', type=int, default=128, metavar='',
                    help='size of minibatch (default: 128)')
parser.add_argument('--n_epoch', type=int, default=500, metavar='',
                    help='number of iterations (default: 500)')
parser.add_argument('--randseed', type=int, default=123, metavar='',
                    help='random seed (default: 123)')

args = parser.parse_args()

max_seq_len = 160

# Set random seed
np.random.seed(args.randseed)
torch.manual_seed(args.randseed)

if args.gpu:
    torch.cuda.manual_seed(args.randseed)

udc = UDCv2('/home/DebanjanChaudhuri/UDC/ubuntu_data', batch_size=args.mb_size, use_mask=True,
            max_seq_len=max_seq_len, gpu=args.gpu, use_fasttext=True)

eos_idx = 63346
pad_idx = 0
model = VAEDualEncoder(udc.emb_dim, udc.vocab_size, args.h_dim, args.z_dim, udc.vectors, eos_idx, pad_idx, 0.5, args.gpu)

solver = optim.Adam(model.parameters(), lr=1e-3)

for epoch in range(args.n_epoch):
    print('\n\n-------------------------------------------')
    print('Epoch-{}'.format(epoch))
    print('-------------------------------------------')

    train_iter = enumerate(udc.get_iter('train'))
    train_iter = tqdm(train_iter)
    train_iter.set_description_str('Training')
    train_iter.total = udc.n_train // udc.batch_size

    for it, mb in train_iter:
        context, response, y, _, _ = mb

        outputs_retrieval, recon_loss_c, kl_loss_c, recon_loss_r, kl_loss_r = model(context, response)

        loss_ret = F.binary_cross_entropy_with_logits(outputs_retrieval, y)
        loss_vae_c = recon_loss_c + kl_loss_c
        loss_vae_r = recon_loss_r + kl_loss_r

        loss = loss_ret + loss_vae_c + loss_vae_r

        loss.backward()
        grad_norm = nn.utils.clip_grad_norm(model.parameters(), 10)
        solver.step()
        solver.zero_grad()

        if it != 0 and it % 2000 == 0:
            # Validation
            recall_at_ks = eval_model_hybrid_v1(model, udc, 'valid', gpu=args.gpu)

            print('\nLoss: {:.3f}; recall@1: {:.3f}; recall@2: {:.3f}; recall@5: {:.3f}'
                .format(loss.data[0], recall_at_ks[0], recall_at_ks[1], recall_at_ks[4]))

    save_model(model, 'hybrid_variational')


import docx
import time
from src.common import defines as dv
from src.common import configVars as cv
from src.common import stringHelpers as strHlprs
from src.cmdInterface import cmdProgressBar

class DocxExporter:
    """Class to generate the output acronym document"""
    def __init__(self, acro_dict_handler):
        self.acro_dict_handler = acro_dict_handler
        self.document = None
        self.obj_normal_style = None
        self.obj_header_style = None
        self.time_paragraph = 0

    def generate_output_docx(self, time_begin_acronymate):
        """Generates a docx document with the user accepted acronyms

        :param time_begin_acronymate: time.monotonic() of the program start. Used in the stats print
        :return: python-docx document object
        """
        # --- Initialize ---
        print(_("Generando Word con tabla de acrónimos"))
        obj_progress_bar = cmdProgressBar.CmdProgressBar(len(self.acro_dict_handler.acros_output.keys()), _("Acrónimos"))

        self.document = docx.Document()

        # --- Define styles ---
        self._define_styles()

        # --- Footer ---
        self._add_footer()

        # --- Text before table (Statistics) ---
        self._add_summary_paragraphs()

        # --- Acronym Table ---
        self._add_acronym_table(obj_progress_bar)

        # --- Fill elapsed time to statistics ---
        self._complete_time_paragraph(time_begin_acronymate)

        return self.document

    def _define_styles(self):
        """Defines the styles used in the document"""
        self.obj_normal_style = self.document.styles['Normal']
        font = self.obj_normal_style.font
        font.name = cv.config_output_font
        font.size = docx.shared.Pt(cv.config_output_font_size)
        paragraph_format = self.obj_normal_style.paragraph_format
        paragraph_format.space_after = docx.shared.Pt(2)
        paragraph_format.space_before = docx.shared.Pt(2)

        self.obj_header_style = self.document.styles.add_style('HeaderStyle', docx.enum.style.WD_STYLE_TYPE.PARAGRAPH)
        hdr_font = self.obj_header_style.font
        hdr_font.size = docx.shared.Pt(cv.config_output_font_size_header)
        hdr_font.name = cv.config_output_font
        paragraph_format = self.obj_header_style.paragraph_format
        paragraph_format.space_after = docx.shared.Pt(2)
        paragraph_format.space_before = docx.shared.Pt(2)

    def _add_footer(self):
        """Adds the footer of the document"""
        # Only one section in document
        section_footer = self.document.sections[0].footer
        section_footer.paragraphs[0].text = "ACRONYMATE " + dv.define_acronymate_version + " - SAHR Projects 2023"
        section_footer.paragraphs[0].alignment = docx.enum.text.WD_ALIGN_PARAGRAPH.RIGHT

    def _add_summary_paragraphs(self):
        """Adds a brief summary before the acronyms table"""
        self.document.add_paragraph(_('Archivo procesado: %s') % self.acro_dict_handler.str_file)
        self.document.add_paragraph(_('Acrónimos extraídos: %d') % len(self.acro_dict_handler.acros_output.keys()))
        self.document.add_paragraph('Tiempo')
        self.time_paragraph = len(self.document.paragraphs) - 1  # Save paragraph to fill later with a precise time
        self.document.add_paragraph('')  # Spacer

    def _complete_time_paragraph(self, time_begin_acronymate):
        """Completes the elapsed time paragraph with hours minutes and seconds since the beginning of the program"""
        elapsed_f_sec = time.monotonic() - time_begin_acronymate
        hours, minutes = 0, 0
        while elapsed_f_sec >= 3600:
            elapsed_f_sec -= 3600
            hours += 1
        while elapsed_f_sec >= 60:
            elapsed_f_sec -= 60
            minutes += 1
        seconds = int(elapsed_f_sec)
        msec = int((elapsed_f_sec - seconds)*100)

        self.document.paragraphs[self.time_paragraph].text = \
            _('Tiempo transcurrido: %s:%s:%s.%d') % \
            (str(hours).zfill(2), str(minutes).zfill(2), str(seconds).zfill(2), msec)

    def _add_acronym_table(self, obj_progress_bar):
        """Adds the acronym table"""
        table = self.document.add_table(rows=1, cols=2)
        # In case we would like to use some styling: table.obj_normal_style = 'LightShading-Accent1'
        table.autofit = True
        hdr_cells = table.rows[0].cells
        hdr_cells[0].text = _('Acrónimo')
        hdr_cells[1].text = _('Definición')
        for i in range(len(hdr_cells)):
            hdr_cells[i].paragraphs[0].runs[0].font.bold = True
            hdr_cells[i].paragraphs[0].style = self.obj_header_style
        # Set parameter in cells for auto width
        for cell in table.rows[0].cells:
            cell._tc.tcPr.tcW.type = 'auto'

        for n_acro, acro in enumerate(sorted(self.acro_dict_handler.acros_output.keys(), key=strHlprs.acro_ordering)):
            row_cells = table.add_row().cells
            row_cells[0].text = acro
            row_cells[0].paragraphs[0].runs[0].font.bold = True
            for i in range(len(self.acro_dict_handler.acros_output[acro]['Def'])):
                if i > 0:
                    row_cells[1].add_paragraph()
                run_main_def = row_cells[1].paragraphs[i].add_run(
                    self.acro_dict_handler.acros_output[acro]['Def'][i]['Main'])
                if 'Translation' in self.acro_dict_handler.acros_output[acro]['Def'][i]:
                    run_main_def.italic = True
                    row_cells[1].paragraphs[i].add_run(
                        " (" + self.acro_dict_handler.acros_output[acro]['Def'][i]['Translation'] + ")")
            # Set parameter in cells for auto width
            for cell in row_cells:
                cell._tc.tcPr.tcW.type = 'auto'

            obj_progress_bar.update(n_acro+1)

def save_document(filepath_output, document):
    """Static function that saves an python-docx object

    :param filepath_output: Desired path
    :param document: python-docx document object
    """
    document.save(filepath_output)


"""
gs_utils are common utlity functions for the app, such as file io.
"""
import json
import re
import io
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.ticker import FormatStrFormatter
from os.path import join
from os.path import exists, isfile, splitext
from os import listdir
from gs_datadict import OUTDIR, MODELDIR, LYRICDIR, ARTISTDIR, GENRES, CORESTOPS, GS_CONTRACT, \
    STOP_LYRICS, GS_STOP, CHAR_CONVERSION, QUOTEDASH_TABLE
import matplotlib.pyplot as plt
from wordcloud import WordCloud

trans_tmptbl = str.maketrans(CHAR_CONVERSION)

def clean_name(rawstr: str):
    """
    small wrangling utility to clean up artist names prior to searching on genius
    """
    cleanstr: str = rawstr.translate(trans_tmptbl)
    finalstr: str = cleanstr.translate(QUOTEDASH_TABLE).replace(" ", "")

    return finalstr

def get_files_list(ldir: str=LYRICDIR, incl_pre: list=None, excl_lst: list=None):
    """
    this is a simple file selector, creating a list that can be used by several tasks
    :param ldir: folder where lyrics files reside, my convention is LYRICDIR,
        usually STR (folder name), BUT can specify LIST of folders if necessary
    :param incl_pre: specify one or more genres for selecting artists, for example
        'rap' -> 'rap_KendrickLamar.lyr', 'rap_OutKast.lyr'..., or 'rock' or 'punk' etc.
    :param excl_lst: identify genre prefix(es) NOT to include, can specify prefix to
        exclude or specific artists to exclude
    :return: list of artist lyric files or None if error occurred
    """
    selected: list = []
    selectednum: int = 0
    if isinstance(ldir, str):
        # easier to deal with as a list
        dirs: list = [ldir]
    elif isinstance(ldir, list):
        dirs: list = ldir

    if isinstance(incl_pre, str):
        if incl_pre in GENRES:
            prefixes: list = [incl_pre]
        elif incl_pre == 'all':
            prefixes: list = GENRES
            print("get_files: including all genres")
        else:
            print("get_files: prefix %s not a recognized genre, Exiting" % incl_pre)
            return None
    elif isinstance(incl_pre, list):
        prefixes: list = []
        for pre in incl_pre:
            if pre in GENRES:
                prefixes.append(pre)
            elif pre == 'all':
                prefixes = GENRES
                print("get_files: including all genres")
            else:
                print("get_files: %s not a recognized genre, exiting" % pre)
                return None
    else:
        prefixes: list = GENRES

    if excl_lst:
        if isinstance(excl_lst, str):
            if excl_lst in prefixes:
                prefixes.remove(excl_lst)
                print("get_files: excluded %s genre" % excl_lst)
        else:
            for excl in excl_lst:
                if excl in prefixes:
                    prefixes.remove(excl)
                    print("get_files: excluded %s genre" % excl)

    numfil = len(prefixes)
    print("get_files: searching %d folders for %d genres --" % (len(dirs), numfil))

    for dirx in dirs:
        print("get_files: searching files in %s" % dirx)
        for fname in listdir(dirx):
            sep_loc = fname.find("_")
            dot_loc: int = fname.find(".")
            if sep_loc != -1:
                genre = fname[:sep_loc]
                artist: str = fname[sep_loc + 1: dot_loc]
            else:
                genre = "none"
                artist = "none"

            if genre not in prefixes:
                print("  %s genre not included, artist: %s, file: %s" % (genre, artist, fname))
                continue
            if excl_lst:
                if artist in excl_lst:
                    print("  %s - excluded artist, file: %s" % (artist, fname))
                    continue
            fqfil = join(dirx, fname)
            if isfile(fqfil):
                file_splits = splitext(fqfil)
                try:
                    file_splits.index('.lyr')
                except ValueError:
                    print("  get_files: not a lyrics file, file: %s" % fname)
                    continue
                else:
                    selected.append({"file": True, "name": fname, "path": fqfil})
                    selectednum += 1
                    # print("get_files: added %s to filter" % artist)
            else:
                print("  %s is not a file, continuing with folder" % fqfil)

    print("  -- get_files: COMPLETED, %d files selected --\n" % selectednum)

    return selected

def filter_for_end_of_track(lyf: str, ldir: str=LYRICDIR, eot: str="\nSONGBREAK\n"):
    """
    standardizes end-of-track indicator for lyrics files, can be used for other filters
    but this is a DESIGN DECISION with DOWNSTREAM IMPLICATIONS.
      Alter corpus source files ONLY if ALL downstream processes, known or planned, have
      the need for it! (i.e. you can't unring this bell)
        I scrub text 'just-in-time' (JIT), in-line with generator that streams lyrics
        Exception- this Fx adds a delimiter between songs and removes blank lines
    :param lyf: str filename for lyrics file
    :param ldir: str folder where lyf resides, expected to be with other lyrics
    :param eot: str to embed in .lyr file as marker for end of track
    :return: 0 if success, 13 if fail or known error
    """

    fq_lyf = join(ldir, lyf)
    if exists(fq_lyf):
        f_h = open(fq_lyf, mode="r")
        contents: str = f_h.read()
        f_h.close()
        sizein: int = len(contents)
        seppos: int = lyf.find("_")
        extpos: int = lyf.find(".")
        artst: str = lyf[seppos + 1: extpos]
        mark: str = "EmbedShare URLCopyEmbedCopy"
        mtch = re.split(mark, string=contents)

        if len(mtch) < 2:
            mark = "songbreak"
            mtch = re.split(mark, string=contents)
            if len(mtch) > 1:
                contents = re.sub(mark, repl=eot, string=contents)
                print("converted %d songbreak to eot in %s lyrics" % (len(mtch), artst))
            else:
                mark = re.sub(r"\n", repl="", string=eot)
                mtch = re.split(mark, string=contents)
                if len(mtch) > 1:
                    print("%s has %d %s delimiters, u good bro, skipping this one"
                          % (artst, len(mtch), mark))
                else:
                    mark = r"(?m)^\n\n"
                    mtch = re.split(mark, string=contents)
                    if len(mtch) > 1:
                        contents = re.sub(mark, repl=eot, string=contents)
                        print("converted %d double-blank-line to eot in %s lyrics" % (len(mtch), artst))
                    else:
                        print("\nError: no known song breaks in %s lyrics, EDIT MANUALLY" % artst)
                        return 13
        else:
            contents = re.sub(mark, repl=eot, string=contents)
            eotx = re.sub(r"\n", repl="", string=eot)
            print("converted %d %s to %s in %s lyrics" % (len(mtch), mark, eotx, artst))

        # set re MULTILINE mode to match caret (^) at start of each 'line'
        mtch = re.split(r"(?m)^\n", string=contents)
        if len(mtch) > 1:
            contents = re.sub(r"(?m)^\n", repl="", string=contents)
            print("filter_lyrics: removed %d blank lines in %s lyrics" % (len(mtch), artst))
        else:
            print("no blank lines for %s" % artst)

        f_h = open(fq_lyf, mode="w", newline="")
        sizeout = f_h.write(contents)
        f_h.close()
        print("filter_lyrics for %s - %d chars read in, %d written out" % (artst, sizein, sizeout))
        return 0

    else:
        print("filter_lyrics ERROR: could not locate %s file" % fq_lyf)
        return 13

def clean_lyrics_file(lyf: str):
    """
    read lyrics corpus (*.lyr file), scrub it, and write it back to a new file (append a
    version number to end of file).  perform the following edits on each line of file:
        1. remove punctuation and convert text to lower case
        2. removes words that match CORESTOPS (my appended list based on gensim eng stoplist)
        3. expands contractions, as per my contraction dictionary GS_CONTRACT
    :param lyf: str name of lyrics corpus
    :return: list of str for each track in lyrics corpus
    """
    ext_pos = lyf.find(".")
    lyfpre = lyf[:ext_pos]
    lyfpost = lyf[ext_pos:]
    fq_file = join(OUTDIR, lyf)
    fq2 = OUTDIR + lyfpre + "_cln" + lyfpost
    re_punc = re.compile(r'[.?!;:<>,(){}]')
    if exists(fq_file):
        f2_h = open(fq2, mode='w', newline="")
        for line in open(fq_file, mode='r', newline=""):
            if len(line) < 5:
                continue
            else:
                tmp = line.lower()
                for wrd, expand in GS_CONTRACT.items():
                    tmp = re.sub(wrd, repl=expand, string=tmp)
                for wrd, cnvrt in CHAR_CONVERSION.items():
                    tmp = re.sub(wrd, repl=cnvrt, string=tmp)
                line = re_punc.sub(repl="", string=line).split()
                line = " ".join(w for w in line if w not in CORESTOPS)
                f2_h.write(line)
        f2_h.close()
    print("\ncompleted cleaning corpus file: %s" % lyfpre)

    return

def read_tokens_to_list(lyf: str, wrdtokens: bool=True):
    """
    reads lines from a lyrics corpus file into python list- splits words to separate tokens
    -- memory-intensive! better alternative is to use an iterator-generator object
    -- this simple utility also assumes that the lyrics have already been scrubbed
    :param lyf: str name of lyrics file, assumed to be located in app OUTDIR
    :param wrdtokens: if True, splits each line into list of words
    :return: list of lyrics lines
    """
    lyriclst: list = []
    fqf = join(OUTDIR, lyf)
    f_h = open(fqf, mode="r", newline="")
    for line in f_h:
        if len(line) > 5:
            if wrdtokens:
                lyriclst.append(line.split())
            else:
                lyriclst.append(line)
    f_h.close()

    return lyriclst

def extract_words(sentstr, stops):
    """
    simple utility to word tokenize a text string and remove stop words
    :param sentstr: input string of text
    :param stops: list of stop words to remove
    :return:
    """
    words = re.sub(r"[^\w]", " ", sentstr).split()
    cleaned_text = [w.lower() for w in words if w.lower() not in stops]
    new_s: str = ""
    for x in cleaned_text:
        new_s = new_s + " " + x
    return new_s

def do_stops(twlst: list, stop1: list=GS_STOP, stop2: list=STOP_LYRICS):
    """
    do_stops is preprocessing function to remove word tokens based on a stop list
    :param twlst: list of list, list of dict, or list of str for Tweets
    :param stop1: list of stop words, defaults to GS_STOP
    :param stop2: list of stop words, defaults to STOP_ADD
    :return: list of tweets with word tokens and stop words removed
    """
    clean_list: list = []
    for twis in twlst:
        if isinstance(twis, list):
            tmp_wrds: list = [cw for cw in twis if cw not in stop1]
            if stop2 is not None:
                clean_list.append([cw for cw in tmp_wrds if cw not in stop2])
            else:
                clean_list.append(tmp_wrds)
        else:
            if isinstance(twis, dict):
                twemp: list = str(twis['text']).split()
            else:  # assume isinstance(twis, str)
                twemp: list = twis.split()

            tmp_wrds: list = [cw for cw in twemp if cw not in stop1]
            if stop2 is not None:
                clean_list.append(' '.join([str(cw) for cw in tmp_wrds if cw not in stop2]))
            else:
                clean_list.append(' '.join([str(cw) for cw in tmp_wrds]))

    return clean_list

def get_stop_file(stf: str):
    """
    this reads stopfile from the nlp models folder and returns it as a set
    :param stf: file name, to be prepended with MODELDIR
    :return: set of stop words
    """
    tmpset: set = set()
    fqfil = MODELDIR + stf
    f_h = open(fqfil, mode="r")
    for line in f_h:
        tmpset.add(line)

    adhoc: list = ['ha', 'oh', 'get', 'got', 'one', 'say', 'like', 'la', 'go', 'say',
                   'said', 'yeah', 'ya', 'u', 'da', 'give', 'thing', 'way', 'really',
                   'let', 'whoa', 'hey', 'na', 'put', 'doo', 'doo', 'let', 'fa', 'cata',
                   'hi', 'so', 'little', 'dit', 'songbreak', 'could']
    for wrd in adhoc:
        tmpset.add(wrd)

    return tmpset

def append_genre_registry(albdct: list, genre: str, outd: str=OUTDIR):
    """
    save a unique track ID (artist name + 3-digit counter) plus name of track for each song
    in the lyrics corpus, one artist file for each genre.
    entries are appended when this is called, so there is logic to verify that the artist's
    tracks have not already been added.
    PURPOSE: this allows TaggedDocument tags used in topic modeling to be mapped back to
        an artist and song- for example, if providing test lyrics and looking for the most
        similar lyrics, it is desirable to be able to return similar artists-songs!
    :param albdct: all artist's albums as list of dict
        'albumid': int, 'title': str, and for each track- song ID: song URL
    :param genre: name of genre registry to create or append
    :param outd: output directory, defaults to app OUTDIR
    :return 0 if save OK
    """

    if not genre in GENRES:
        print("save_artist_file: use out file= '<genre>_registry.json'! exiting")
        return 1
    else:
        outf: str = genre + "_registry.json"

    fqf = join(outd, outf)
    with open(fqf, mode='a') as f_h:
        if isinstance(albdct, list):
            for alb in albdct:
                json.dump(alb, f_h)
                f_h.write("\n")
        elif isinstance(albdct, dict):
            # typical course when processing after get_lyrics is run in build_corpus
            json.dump(albdct, f_h)
            f_h.write("\n")
            for ad, i in zip(albdct, range(1)):
                print("append_genre_registry: added tracks for %s" % ad[:-3])

    return 0

def retrieve_genre_registry(genre: str, indir: str=ARTISTDIR):
    """
    retrieve track lists for artists, useful in matching up document ID's when doing
    topic modeling and other doc2vec tasks with song lyrics
    :param genre: each genre has a registry file made up of track dicts for each artist
    :param indir: folder where json registry files reside, default is ARTISTDIR
    :return:
    """
    infile = genre + "_registry.json"
    fqf = join(indir, infile)
    if isfile(fqf):
        reg_lst: list = []
        with open(fqf, mode="r") as f_h:
            for line in f_h:
                reg_lst.append(json.loads(line))

        print("  retrieved %s registry with %d artist track lists" % (genre, len(reg_lst)))
        return reg_lst
    else:
        print("retrieve_genre_registry ERROR: %s not valid!" % fqf)
        return None

def get_artists_in_genre(incl_genres, lydir: str=LYRICDIR):
    """
    creates and displays a list of all artists for genres selected via incl_genres parm.
    selects all genres if incl_genres is left blank or if 'all' is passed as str or list
    :param incl_genres:
    :param lydir: folder for lyric files
    :return:
    """
    print("  -- create_genre_lists finds all artists with lyrics in genre --")
    if not incl_genres:
        search_gen = GENRES
    else:
        if incl_genres == "all":
            search_gen = GENRES
        else:
            if isinstance(incl_genres, str):
                if incl_genres in GENRES:
                    search_gen = [incl_genres]
            elif isinstance(incl_genres, list):
                search_gen: list = []
                for genx in incl_genres:
                    if genx in GENRES:
                        search_gen.append(genx)

    total_gen: int = len(search_gen)
    art_added: int = 0
    genre_artists: dict = {}
    genre_artists = genre_artists.fromkeys(search_gen)

    def add_artist(gn: str, ar: str, g_a: dict):
        """
        inner function to insert artist in appropriate dict for genre
        :param gn: str with genre
        :param ar: str with artist
        :param g_a: dict of genre-artists
        :return: 0 if artist inserted, 1 if not
        """
        found: bool = False
        for k, v in g_a.items():
            if k == gn:
                found = True
                if not v:
                    artsts: set = {ar}
                elif isinstance(v, list):
                    artsts: set = set(v)
                elif isinstance(v, set):
                    artsts: set = v

                if not ar in artsts:
                    artsts.add(ar)
        if found:
            g_a[gn] = artsts
        return g_a, found

    for fname in listdir(lydir):
        fqfil = join(lydir, fname)
        if isfile(fqfil):
            sep_pos = fname.find("_")
            ext_pos = fname.find(".")
            gen_pre = fname[:sep_pos]
            art_nam = fname[sep_pos + 1: ext_pos]
            if gen_pre in search_gen:
                genre_artists, was_added = add_artist(gen_pre, art_nam, genre_artists)
                if was_added:
                    art_added += 1

    print("      create_genre added %d artists across %d genres" % (art_added, total_gen))
    return genre_artists

def save_artist_info(artdct: dict, genre: str, outd: str = OUTDIR):
    """
    this is an optional method to save the artist info dict that is simply created as a
    step towards pulling albums and lyric tracks, it may allow recreation of a corpus
    from scratch easier.
    :param artdct: artist name, genius id, img file, lyric file, etc.
    :param genre: name of genre for this artist
    :param outd: output directory, defaults to app OUTDIR
    :return 0 if save OK
    """
    outf = genre + "_artistdb.info"
    fqf = join(outd, outf)
    with open(fqf, mode='a') as f_h:
        if isinstance(artdct, dict):
            json.dump(artdct, f_h)
            f_h.write("\n")

    return 0

def scrub_cloud(twl: list):
    """
    scrub_text can perform numerous text removal or modification tasks on tweets,
    there is tweet-specific content handled here which can be optionally commented out
    if resulting corpus loses too much detail for downstream tasks like sentiment analysis

    :param twl: list of tweets
    :return: list of words OR str of words if rtn_list= False
    """
    from gs_datadict import GS_STOP, STOP_LYRICS, CHAR_CONVERSION
    import re

    for tweetxt in twl:
        if isinstance(tweetxt, str):
            # remove newlines in tweets, they cause a mess with many tasks
            tweetxt = tweetxt.replace("\n", " ")
            # remove standalone period, no need in a tweet
            tweetxt = re.sub(r"\.", " ", tweetxt)
            # expand contractions using custom dict of contractions
            tweetxt = re.sub(GS_STOP, "", tweetxt)
            # convert ucs-2 chars appearing in english tweets
            tweetxt = re.sub(STOP_LYRICS, "", tweetxt)
            # parsing tweet punc: don't want to lose sentiment or emotion
            tweetxt = re.sub(CHAR_CONVERSION, "", tweetxt)
            # parameter "utf-8" escapes ucs-2 (\uxxxx), "ascii" removes multi-byte
            binstr = tweetxt.encode("ascii", "ignore")
            tweetxt = binstr.decode()
            # now, tokenize and run words through cleaning:
            splitstr = tweetxt.split()
            stokens: list = []
            for w in splitstr:
                if len(w) < 2:
                    continue
                # lower case all alpha words
                if str(w).isalpha():
                    w: str = str(w).lower()
                    stokens.append(w)
            splitstr: list = []
            for wrd in stokens:
                wrd = wrd.strip()
                splitstr.append(wrd)

    return splitstr

def get_cloud_input(batch_words):
    """
    get lyrics token lists from object and process to prep for cloud
    :param batch_words:
    :return:
    """
    wrdct: dict = {}
    cur_artist: str = ""
    for wrds, artst in batch_words:
        for word in wrds:
            if wrdct.get(word):
                wrdct[word] += 1
            else:
                wrdct[word] = 1
        if not cur_artist.startswith(artst):
            if not cur_artist == "":
                print("streaming done for %s" % artst)
            cur_artist = artst

    return wrdct

def do_cloud(cloud_words: list, opt_stops: list = None, maxwrd: int = 120):
    """
    wordcloud package options can be explored via '?wordcloud' (python- show docstring)
    background_color="white" - lighter background makes smaller words more legible,
    max_words= this can prevent over clutter, mask=shape the cloud to an image,
    stopwords=ad-hoc removal of unwanted words, contour_width=3,
    :param cloud_words: list of list of word tokens
    :param opt_stops: list of words (str) to remove prior to creating wordcloud
    :param maxwrd: int typically from 80 to 120 for total words to appear in cloud
    :return:
    """
    from wordcloud import WordCloud
    import io
    import matplotlib.pyplot as plt

    cloud_text = io.StringIO(newline="")
    for tok in cloud_words:
        if isinstance(tok, str):
            cloud_text.write(tok + " ")
        else:
            for a_tw in tok:
                if isinstance(a_tw, list):
                    cloud_text.write(" ".join([str(x) for x in a_tw]) + " ")
                if isinstance(a_tw, str):
                    # if simple list of text for each tweet:
                    cloud_text.write(a_tw + " ")

    wordcld = WordCloud(width=800, height=800, max_words=maxwrd,
                        background_color='white',
                        stopwords=opt_stops, min_word_length=4,
                        min_font_size=10).generate(cloud_text.getvalue())

    # plot the WordCloud image
    plt.figure(figsize=(8, 8), facecolor=None)
    plt.imshow(wordcld)
    plt.axis("off")
    plt.tight_layout(pad=0)
    plt.show()

    return

def check_outliers(lobj):
    """
    goes through a lyrics object and checks for few extreme values for
    calculated attributes such as tf and idf
    """
    artvals: list = []
    artistlst: list = list(lobj.artists.copy())
    artistct: int = len(artistlst)
    for i in range(artistct):
        if lobj.tfidf_artist.get(artistlst[i]):
            artvals.append(list(lobj.tfidf_artist[artistlst[i]].values()))

    for i in range(artistct):
        hicut = np.percentile(a=artvals[i], q=98)
        locut = np.percentile(a=artvals[i], q=2)
        voc_size = len(artvals[i])
        removed: int = 0
        for_deletion: list = []
        tmpdct: dict = lobj.tfidf_artist[artistlst[i]]
        for wrd, tfival in tmpdct.items():
            if tfival > hicut or tfival < locut:
                for_deletion.append(wrd)
        if for_deletion:
            for wrd in for_deletion:
                tmpdct.pop(wrd)
                removed += 1

        lobj.tfidf_artist[artistlst[i]] = tmpdct
        pct_del = round((removed / voc_size), ndigits=3)
        print("outliers removed %d tfidf_artist, %.3f percent, for %s" % (removed, pct_del, artistlst[i]))

    return artistlst, artvals

def plot_artist_tfidf(lobj, artst: str):

    artvals: list = list(lobj.tfidf_artist[artst].values())
    fig, ax1 = plt.subplots()
    ax1.xaxis.set_major_formatter(FormatStrFormatter('%.4f'))

    finalmin: float = 0.0006
    tfimed = np.median(artvals)
    tfistd = np.std(artvals)
    finalmax = tfimed + (3 * tfistd)
    xrang = finalmax - finalmin
    num_bins = int(round(xrang / 0.0002, ndigits=0) + 2)

    ax1.set_xlim(finalmin, finalmax)
    plt.xlim(finalmin, finalmax)

    ax1.hist(artvals, bins=num_bins, density=False, range=(finalmin, finalmax))

    plt.ylabel('Occurences')
    plt.xlabel('tfidf value')
    plt.suptitle("tfidf for words of %s" % artst)
    plt.show()

    return

def plot_artist_tf(lobj, artst: str):
    artvals: list = list(lobj.tf_by_artist[artst].values())
    tot_wrds: int = len(artvals)

    fig, ax1 = plt.subplots()
    ax1.xaxis.set_major_formatter(FormatStrFormatter('%.4f'))

    finalmin: float = 0.002
    tfimed = np.median(artvals)
    tfistd = np.std(artvals)
    finalmax = tfimed + (3 * tfistd)
    xrang = finalmax - finalmin
    num_bins = int(round(xrang / 0.0002, ndigits=0) + 2)

    ax1.set_xlim(finalmin, finalmax)
    plt.xlim(finalmin, finalmax)
    ax1.hist(artvals, bins=num_bins, density=False, range=(finalmin, finalmax))
    plt.ylabel('Occurences')
    plt.xlabel('tfidf value')
    plt.suptitle("tf for %d words for artist: %s" % (tot_wrds, artst))
    plt.show()

    return

def compare_artist_tfidf(lobj, artistlst: list = None):
    """
    within a lyrics object, compare high-tf words between artists
    ASSUMES TWO ARTISTS WILL ALWAYS BE NAMED!
    """
    num_bins: int = 200
    artvals: list = []
    plots: int = 0
    for artist in artistlst:
        artvals.append(list(lobj.tfidf_artist[artist].values()))
        plots += 1

    fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=plots)
    ax1.xaxis.set_major_formatter(FormatStrFormatter('%.3f'))
    ax2.xaxis.set_major_formatter(FormatStrFormatter('%.3f'))

    totmx: float = 0.09
    totmn: float = 0.002
    for i in range(plots):
        tfimx = np.max(artvals[i])
        tfimn = np.min(artvals[i])
        if i == 0:
            ax1.set_xlim(tfimn, tfimx)
            ax1.hist(artvals[i], num_bins, density=False, range=(tfimn, tfimx))
        else:
            ax2.set_xlim(tfimn, tfimx)
            ax1.hist(artvals[i], num_bins, density=False, range=(tfimn, tfimx))

    # color1 = np.random.uniform(15, 80, len(art1_vals))
    # the histogram of the data
    # n, bins, patches = ax1.hist(artvals[0], num_bins, density=False, range=(totmn, totmx))
    # n2, bins2, patches2 = ax2.hist(artvals[1], num_bins, density=False, range=(totmn, totmx))

    # add a 'best fit' line
    # mu = np.mean(artvals[0])  # mean of distribution
    # sigma = np.std(artvals[0])  # standard deviation of distribution
    # y = ((1 / (np.sqrt(2 * np.pi) * sigma)) *
    #     np.exp(-0.5 * (1 / sigma * (bins - mu)) ** 2))
    # ax1.plot(bins, y, '--')

    # fig, ax = plt.subplots()
    # ax.hist(art1_vals, bins=30, linewidth=0.01, edgecolor="white")
    # ax.xaxis.set_major_formatter(FormatStrFormatter('%.3f'))
    plt.ylabel('Occurences')
    plt.xlabel('tfidf value')
    plt.suptitle("tfidf for words of %s vs %s" % (artistlst[0], artistlst[1]))
    plt.show()

    return


from multiprocessing import Pool
from sys import argv

import pandas as pd
from numpy import average
from sklearn.metrics import accuracy_score
from sklearn.model_selection import KFold
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import StandardScaler

fileName = argv[1]

dataSet = pd.read_csv(fileName, index_col=0)

y = dataSet['class']
spectro = dataSet['spectrometric_redshift'].values

for column in ('Unnamed: 0', 'Unnamed: 0.1', 'Unnamed: 0.1.1', 'Unnamed: 0.1.1.1', 'galex_objid', 'sdss_objid', 'class', 'spectrometric_redshift', 'pred'):
	try: dataSet.drop(columns=column, inplace=True)
	except: pass

X = StandardScaler().fit_transform(dataSet)

def score(indices):
	train_index, test_index = indices

	X_train, X_test = X[train_index], X[test_index]
	y_train, y_test = y[train_index], y[test_index]
	spectro_test = spectro[test_index]

	knn = KNeighborsClassifier(5)
	knn.fit(X_train, y_train)
	y_pred = knn.predict(X_test)

	spectro_pred = [0 if z <= 0.0033 else (1 if z >= 0.004 else 2) for z in spectro_test]

	return accuracy_score(y_test, y_pred), accuracy_score(spectro_pred, y_pred)

scores = Pool(10).map(score, KFold(n_splits=10, random_state=29, shuffle=True).split(X))

print(round(average([i[0] * 100 for i in scores]), 2),
      round(average([i[1] * 100 for i in scores]), 2))


import argparse
import sys
import yaml
import ast

def parse_args():
    parser = argparse.ArgumentParser(prog=sys.argv[0],
                                     formatter_class=argparse.RawDescriptionHelpFormatter,
                                     description="""
customize_s3_conf.py is a tool to generating customized config file for S3 server.
Script uses some predefined file as a base for changing some parameters.
Using this base file script generates new one that contains updated values
specified by user.

Example of usage:
$ ./customize_s3_conf.py -s /opt/seagate/cortx/s3/conf/s3config.yaml
                         -d /opt/seagate/cortx/s3/conf/s3config.yaml
                         --s3-srv-param "S3_DI_DISABLE_DATA_CORRUPTION_IEM=False"
                         --s3-auth-param "S3_AUTH_IP_ADDR='ipv4:127.0.0.1'"
                         --s3-motr-param "S3_MOTR_LOCAL_ADDR='<ipaddress>@tcp:12345:33:100'"
                         --s3-thirdparty-param "S3_LIBEVENT_POOL_BUFFER_SIZE=16384"
""")

    parser.add_argument("-s", "--src-conf-file", type=str, required=True,
                        help="path to config file used as base")

    parser.add_argument("-d", "--dst-conf-file", type=str, required=True,
                        help="path to store result")

    parser.add_argument("--s3-srv-param", action='append', required=False,
                        help="""parameter of S3_SERVER_CONFIG section
                        to be overrided in a form 'PARAM_NAME=PARAM_VALUE'""")

    parser.add_argument("--s3-auth-param", action='append', required=False,
                        help="""parameter of S3_AUTH_CONFIG section
                        to be overrided in a form 'PARAM_NAME=PARAM_VALUE'""")

    parser.add_argument("--s3-motr-param", action='append', required=False,
                        help="""parameter of S3_MOTR_CONFIG section
                        to be overrided in a form 'PARAM_NAME=PARAM_VALUE'""")

    parser.add_argument("--s3-thirdparty-param", action='append', required=False,
                        help="""parameter of S3_THIRDPARTY_CONFIG section
                        to be overrided in a form 'PARAM_NAME=PARAM_VALUE'""")

    return parser.parse_args()

def read_src_config(src_config_path):
    with open(src_config_path) as f:
        return yaml.safe_load(f.read())

def write_dst_config(cdf_data, dst_config_path):
    with open(dst_config_path, 'wt') as f:
        f.write(yaml.dump(cdf_data))

def override_param(conf_data, section_name, param_name, param_val_str):
    param_val = ast.literal_eval(param_val_str)
    print(f'overriding {section_name}:{param_name} to {param_val_str}')
    conf_data[section_name][param_name] = param_val

def main():
    args = parse_args()
    conf_data = read_src_config(args.src_conf_file)

    if args.s3_srv_param is not None:
        for key_val in args.s3_srv_param:
            key, val = key_val.split('=')
            override_param(conf_data, 'S3_SERVER_CONFIG', key, val)

    if args.s3_auth_param is not None:
        for key_val in args.s3_auth_param:
            key, val = key_val.split('=')
            override_param(conf_data, 'S3_AUTH_CONFIG', key, val)

    if args.s3_motr_param is not None:
        for key_val in args.s3_motr_param:
            key, val = key_val.split('=')
            override_param(conf_data, 'S3_MOTR_CONFIG', key, val)

    if args.s3_thirdparty_param is not None:
        for key_val in args.s3_thirdparty_param:
            key, val = key_val.split('=')
            override_param(conf_data, 'S3_THIRDPARTY_CONFIG', key, val)

    write_dst_config(conf_data, args.dst_conf_file)

if __name__ == '__main__':
    main()


from collections import defaultdict
import json
import time
import threading
import tempfile
import os

from webob.dec import wsgify
from webob import exc

from webtestplus.recorder import get_record

__all__ = ['ClientTesterMiddleware']

DISABLED = 'disabled'
RECORD = 'recording'
REPLAY = 'playing'

def _int2status(status):
    if status == 200:
        return '200 OK'
    if status == 400:
        return '400 Bad Request'
    if status == '401':
        return '400 Unauthorized'

    return '%d Explanation' % status

class ClientTesterMiddleware(object):
    """Middleware that let a client drive failures for testing purposes.
    """
    def __init__(self, app, mock_path='/__testing__',
                 filter_path='/__filter__',
                 rec_path='/__record__',
                 secret='CHANGEME',
                 requires_secret=True,
                 rec_file=None):
        self.custom_paths = mock_path, filter_path, rec_path
        self.app = app
        self.mock_path = mock_path
        self.filter_path = filter_path
        self.rec_path = rec_path
        self.replays = defaultdict(list)
        self.filters = defaultdict(dict)
        self.is_recording = defaultdict(lambda: DISABLED)
        self.lock = threading.RLock()
        if rec_file is None:
            fd, rec_file = tempfile.mkstemp()
            os.close(fd)

        self.rec_file = rec_file
        self.secret = secret
        self.requires_secret = requires_secret

    def _auth(self, request):
        if not self.requires_secret:
            return
        provided = request.headers.get('X-Secret')
        if provided != self.secret:
            raise exc.HTTPUnauthorized()

    def _get_client_ip(self, environ):
        if 'HTTP_X_FORWARDED_FOR' in environ:
            return environ['HTTP_X_FORWARDED_FOR'].split(',')[0].strip()

        if 'REMOTE_ADDR' in environ:
            return environ['REMOTE_ADDR']

        return None

    def _resp(self, req, status='200 OK', body='', headers=None):
        resp = req.response
        resp.status = status
        resp.body = body
        resp.content_type = 'application/json'

        if headers is not None:
            resp.headers.update(dict(headers))

        return resp

    def _apply_filters(self, resp, filters):
        status = resp.status
        intst = int(status.split()[0])
        if intst in filters:
            time.sleep(filters[intst])
        elif '*' in filters:
            time.sleep(filters['*'])

        # XXX maybe we will have filters that change the resp
        return resp

    @wsgify
    def __call__(self, request):
        if request.path_info in self.custom_paths:
            self._auth(request)

        environ = request.environ
        path = request.path_info

        environ['_ip'] = ip = self._get_client_ip(environ)
        environ['_replays'] = replays = self.replays[ip]
        environ['_filters'] = filters = self.filters[ip]
        environ['_is_recording'] = rec = self.is_recording[ip]

        # routing
        if path.startswith(self.mock_path):
            return self._mock(request)
        elif path.startswith(self.filter_path):
            return self._filter(request)
        elif path.startswith(self.rec_path):
            return self._rec_state(request)

        # classical call, do we have something to replay ?
        if len(replays) > 0:
            # yes
            replay = replays.pop()
            status = _int2status(replay.get('status', 0))
            body = replay.get('body', u'').encode('utf8')
            headers = replay.get('headers')
            delay = replay.get('delay', 0)
            passthrough = replay.get('passthrough', False)

            # repeat it, always
            if replay.get('repeat') == -1:
                replays.insert(0, replay)

            # build the response
            if not passthrough:
                resp = request.response
            else:
                resp = request.get_response(self.app)
            if status:
                resp.status = status
            if body:
                resp.body = body
            if headers:
                resp.headers.update(dict(headers))

            # apply filters
            resp = self._apply_filters(resp, filters)

            # extra delay
            time.sleep(delay)

            return resp
        else:
            # no, regular app
            # do we record or play or just call the app ?
            if rec in (DISABLED, RECORD):
                resp = request.get_response(self.app)
                if rec == RECORD:
                    self._record(request, resp)
            else:  # REPLAY:
                resp = self._replay(request)

            return self._apply_filters(resp, filters)

    def _checkmeth(self, method, allowed=None):
        if allowed is None:
            allowed = ('POST', 'DELETE')

        if method not in allowed:
            raise exc.HTTPMethodNotAllowed(
                              allow=','.join(allowed))

    def _rec_state(self, request):
        ip = request.environ['_ip']
        method = request.method
        self._checkmeth(method, ('POST', 'GET'))

        if method == 'POST':
            # define the toggle
            try:
                st = json.loads(request.body)
            except ValueError:
                raise exc.HTTPBadRequest()

            self.is_recording[ip] = st
            return self._resp(request)

        status = json.dumps(self.is_recording[ip])
        return self._resp(request, body=status)

    def _mock(self, request):
        # what's the method ?
        method = request.method
        self._checkmeth(method)
        replays = request.environ['_replays']

        if method == 'DELETE':
            # wipe out
            replays[:] = []
            return self._resp(request)

        # that's something to add to the pile
        try:
            resp = json.loads(request.body)
        except ValueError:
            raise exc.HTTPBadRequest()

        repeat = resp.get('repeat', 1)
        if repeat == -1:
            # will repeat indefinitely
            replays.insert(0, resp)
        else:
            for i in range(repeat):
                replays.insert(0, resp)

        return self._resp(request)

    def _filter(self, request):
        # what's the method ?
        method = request.method
        self._checkmeth(method)
        filters = request.environ['_filters']

        if method == 'DELETE':
            # wipe out
            filters.clear()
            return self._resp(request)

        # that's something to set
        try:
            new = json.loads(request.body)
        except ValueError:
            raise exc.HTTPBadRequest()

        filters.clear()

        for status, delay in new.items():
            if status != '*':
                status = int(status)
            filters[status] = delay

        return self._resp(request)

    def _replay(self, request):
        ip = request.environ['_ip']
        resp = get_record(self.rec_file, request)
        if resp is None:
            # failed to find a matching record
            # by-passing
            return request.get_response(self.app)

        return resp

    def _record(self, request, resp):
        data = []

        data.append('--Request:\n')
        data.append(str(request))
        if not request.content_length:
            data.append('\n')

        data.append('\n--Response:\n')
        data.append(str(resp))
        if not resp.body:
            data.append('\n')
        data.append('\n')

        self.lock.acquire()
        try:
            with open(self.rec_file, 'a+') as f:
                f.write(''.join(data))
        finally:
            self.lock.release()


from django import forms
from energize_andover.models import *
class SchoolForm(forms.ModelForm):
    class Meta:
        model = School
        fields = ['Name']

class ClosetForm(forms.ModelForm):
    class Meta:
        model = Closet
        fields = ['Name', 'Old_Name']

class PanelForm(forms.ModelForm):
    class Meta:
        model = Panel
        fields = ['Name', 'Voltage']

class RoomForm(forms.ModelForm):
    class Meta:
        model = Room
        fields = ['Name', 'OldName', 'Type']

class CircuitForm(forms.ModelForm):
    class Meta:
        model = Circuit
        fields = ['Name', 'Number', 'Panel']
        #widgets = {'Rooms': forms.CheckboxSelectMultiple, 'Devices': forms.CheckboxSelectMultiple}

class DeviceForm(forms.ModelForm):
    class Meta:
        model = Device
        fields = ['Name']

class AdderTypeForm(forms.Form):
    type = forms.ChoiceField(
        label='Select object type to add',
        choices=[('', ''),
                 ('Closet', 'Closet'),
                 #('School', 'School'),
                 ('Panel', 'Panel'),
                 ('Room', 'Room'),
                 ('Device', 'Device')
                 ],
        required=True
    )

def get_all_users():
    users = []
    usrs = User.objects.all()
    for i in usrs:
        users.append((i.username, i.username))
    return users

class PopulationForm(forms.Form):

    New_School = forms.CharField(
        label='New School: ',
        required=True
    )

    Room_File = forms.FileField(
        label='Select a file for Room Mapping'
    )

    Panel_File = forms.FileField(
        label='Select a file for Panel and Circuit Mapping'
    )

    Device_File = forms.FileField(
        label='Select a file for Device Mapping'
    )

    Device_Association_File = forms.FileField(
        label='Select a file for Additional Device Info',
        required = False
    )

    all_users = forms.MultipleChoiceField(choices=get_all_users(),
                                          widget=forms.CheckboxSelectMultiple(),
                                          label="Which Users Can Access: ")


from tortoise.contrib.fastapi import register_tortoise
from data.model import *
from fastapi import FastAPI, Depends

TORTOISE_ORM = {
    "connections": {"default": 'sqlite://notesdb.sqlite'},
    "apps": {
        "models": {
            "models": ["data.model.user", "data.model.note", "aerich.models"],
            "default_connection": "default",
        },
    },
}

def init_db(app: FastAPI):
    register_tortoise(
        app,
        db_url='sqlite://notesdb.sqlite',
        modules={"models": ["data.model.user", "data.model.note"]},
        generate_schemas=False,
        add_exception_handlers=False,
    )


from collections import Counter

def solution(str1, str2):
    # 알파벳 대문자 아스키 코드 값들 리스트
    alpha = [k for k in range(65, 91)]

    # 두글자씩 끊어서 다중집합 생성, 알파벳 이외의 문자가 삽입된 경우 제외(아스키 코드로 알파벳인지 판별)
    strlist = [str1, str2]
    strcombi = [[] for _ in range(2)]
    for i in range(len(strlist)):
        strlist[i] = strlist[i].upper()
        for j in range(len(strlist[i])-1):
            if ord(strlist[i][j]) in alpha and ord(strlist[i][j+1]) in alpha:
                strcombi[i].append((strlist[i][j], strlist[i][j+1]))

    # 다중집합 내의 원소 값들의 갯수 계산(원소(key)-갯수(value) 딕셔너리)
    str1info = Counter(strcombi[0])
    str2info = Counter(strcombi[1])

    # 키 값인 원소 값만 추출 후 교집합, 합집합 처리
    str1key = list(str1info.keys())
    str2key = list(str2info.keys())

    gyolist = list(set(str1key).intersection(set(str2key)))
    haplist = list(set(str1key).union(set(str2key)))

    # 집합 A와 집합 B가 모두 공집합일 경우 J(A, B) = 1로 정의한다는 예외 설정
    if len(str1info)==0 and len(str2info)==0:
        return 65536

    # 자카드 유사도 계산
    gyo = 0
    for x in gyolist:
        gyo += min(str1info[x], str2info[x])

    hap = 0
    for y in haplist:
        if y in str1key and y in str2key:
            hap += max(str1info[y], str2info[y])
        elif y in str1key:
            hap += str1info[y]
        else:
            hap += str2info[y]

    answer = int((gyo / hap) * 65536)

    return answer

str1 = 'FRANCE'
str2 = 'french'
# str1 = 'A+C'
# str2 = 'DEF'
# result: 0
print(solution(str1, str2))


from django.contrib import admin
from tinymce.widgets import TinyMCE
from django.db import models
# Register your models here.
from .models import Cat, Category

class TutorialAdmin(admin.ModelAdmin):
    fieldsets = [
        ("Title/date", {'fields': ["title", "published"]}),
        ("URL", {'fields': ["slug"]}),
        ("Content", {"fields": ["content","cat_category","birth"]}),

        ("Icon",{"fields": ["icon"]})
    ]
    formfield_overrides = {
        models.TextField: {'widget': TinyMCE()},
    }

admin.site.register(Category)
admin.site.register(Cat, TutorialAdmin)


import os

from sqlalchemy import (
    create_engine,
    Integer,
    Column,
    String,
    ForeignKey)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, sessionmaker

BASE_DIR = os.path.dirname(os.path.realpath(__file__))
conn = 'sqlite:///' + os.path.join(BASE_DIR, 'blog.db')

e = create_engine(conn, echo=True)

Base = declarative_base()

"""
class User(Base):
        id: int
        username: str
        email: str

class Post:
        id: int
        title: str
        content: str
        user_id: int foreignkey
"""

class User(Base):
    __tablename__ = 'users'

    id = Column(Integer, primary_key=True)
    username = Column(String(40), nullable=False)
    email = Column(String(40), nullable=False)
    posts = relationship('Posts', backref='author')

    def __repr__(self):
        return f'<User {self.username}>'

class Post(Base):
    __tablename__ = 'posts'
    id = Column(Integer(), primary_key=True)
    title = Column(String(45), nullable=False)
    content = Column(String(255), nullable=False)
    user_id = Column(Integer, ForeignKey('users.id'))

    def __repr__(self):
        return f'<User{self.title}>'

Base.metadata.create_all(e)
session = sessionmaker()(bind=e)


# !/usr/bin/env python
# -*- coding: utf-8 -*-
# from logconfig import LogConfig
# logger = LogConfig.get_logger()
import os
import logging
logger = logging.getLogger('mylogger')

# import MySQLdb
class Bloom(object):

    # filter ids that have been crawled
    @classmethod
    def getIds(self):
        # f = open(r'../files/w_id_high.csv','r')
        # f = open(r'bloom/id_high_un.csv','r')
        # f = open(r'bloom/id_low_un.csv','r')
        f = open(r'bloom/id_root_30_un.csv','r')
        bloom = []
        # index = 0 # 每个爬虫各负责一端Ip
        # end = 50000
        index = 0 # 10万以内好了吗？
        end = 5000
        num = 0
        # for line in f.readlines():
        #     if(line!="\n"):
        #         num += 1
        #         if num>index and num<=end:
        #             lines = line.split(",")
        #             bloom.append(lines[0])
        for line in f.readlines():
            num += 1
            if num>index and num<=end:
                id = line.strip()
                bloom.append(id)
        ids = bloom
        return ids

    # 获取已经爬过的Ids
    # def get_already_ids(self):
    def getdb(self):
        # db = MySQLdb.connect('166.111.134.51','root','fit4-305','weibo',charset = 'utf8')
        db = MySQLdb.connect("localhost","Tiger","123456","weibo",charset='utf8')
        return db

    def getIdsDataBase(self):
        #从数据库获取id
        ids = {}
        db = self.getdb()
        cursor = db.cursor()
        sql = "select id from weibo.user_profile"
        cursor.execute(sql)
        data = cursor.fetchall()
        for each in data:
            # print(each)
            # ids.append(each[0])
            ids[each[0]] = ""
        return ids

from pybloom import ScalableBloomFilter
class FileBloom(object):
    def __init__(self):
        self.file_path = "bloom/bloom_weibo.txt";
        self.bloom_filter = ScalableBloomFilter(initial_capacity=10000,error_rate=0.001)

    def read_bloom(self):
        if os.path.exists(self.file_path):
            f = open(self.file_path,"r")
            ids = f.readlines()
            for id in ids:
                id_s = id.strip()
                self.bloom_filter.add(id_s)
            f.close()
        else:
            f = open(self.file_path,"w")
            f.close()

    def to_file(self):
        pass

    def update_bloom_file(self,m_id):
        f =open(self.file_path,"a")
        f.write(str(m_id)+"\n")
        f.close()

    def update_bloom(self,m_id):
        self.bloom_filter.add(m_id)

    def has_id(self,m_id):
        if m_id in self.bloom_filter:
            return True
        else:
            return False

if __name__ == "__main__":
    fb = FileBloom()
    fb.read_bloom()
    fb.update_bloom_file("23920323")
    fb.update_bloom("23920323")
# if __name__ == "__main__":
#     bl = Bloom()
#     ids = bl.getIdsDataBase()
#     f = open(r'id_root_30.csv','r')
#     uids = []
#     for line in f.readlines():
#         id = line.strip()
#         uids.append(id)
#     f.close()
#     print(len(uids))
#     uids = list(set(uids))
#     print(len(uids))
#     f = open(r'id_root_30_un.csv','w')
#     for uid in uids:
#         if uid not in ids:
#             print(uid)
#             f.write(uid+"\n")
#     f.close()

# ids = Bloom.getIds()
# ids = Bloom.getIdsDataBase()
# logger.info(ids)
# print(len(ids))


import math
import matplotlib.pyplot as plt
import matplotlib
plt.style.use('ggplot')
x = xrange(2,100)
birthday = [1-(math.factorial(365)/math.factorial(365 - k))/float(365**k) for k in x]

plt.plot(x, birthday, 'g-')
plt.ylim(0,1.05)
plt.xlabel('# of people')
plt.ylabel('Probability')
plt.show()


# Always prefer setuptools over distutils
# To use a consistent encoding
from codecs import open
from os import path

from setuptools import setup

# The directory containing this file
HERE = path.abspath(path.dirname(__file__))

# Get the long description from the README file
with open(path.join(HERE, 'README.md')) as f:
    long_description = f.read()

# This call to setup() does all the work
setup(
    name="stocksent",
    version="0.4.5",
    description="A Python library for sentiment analysis of various tickers from the latest news by trusted sources, "
                "and tools to plot results.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://stocksent.readthedocs.io/",
    author="Arya Manjaramkar",
    author_email="aryagm01@email.com",
    license="Mozilla Public License 2.0",
    classifiers=[
        "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
        "Natural Language :: English",
        "Topic :: Software Development :: Libraries :: Python Modules",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
        "Operating System :: OS Independent"
    ],
    packages=["stocksent"],
    include_package_data=True,
    install_requires=["numpy", "matplotlib",
                      "pandas", "nltk", "wordcloud", "bs4"]
)


from django.shortcuts import render
from django.contrib.auth.decorators import login_required
from django.http import HttpResponse, JsonResponse
from django.conf import settings
from .models import *
from .forms import *
import stripe
import json
from django.http import HttpResponse
import json

# Create your views here.
stripe.api_key = settings.STRIPE_API_KEY

def index(request):

    context = {}

    return render(request, "front-office/index.html", context)

@login_required
def dashboard(request):
    context = {}

    return render(request, "backoffice/base.html", context)

@login_required
def fixes(request):
    context = {}

    return render(request, "backoffice/fixes.html", context)

@login_required
def customerProfile(request):
    form = CustomerProfileForm()
    context = {
        'form': form,
    }

    return render(request, "backoffice/profile.html", context)

@login_required
def addFixes(request):

    context = {}

    return render(request, "backoffice/fixes.html", context)

@login_required
def capsulesView(request):

    context = {}

    return render(request, "backoffice/capsules.html", context)

@login_required
def addFixAttachement(request):
    if request.method == "POST":
        fix_pk = request.POST.get("fix_id")
        fix = SmallFix.objects.get(pk=fix_pk)
        for file in request.FILES:
            attachment = FixAttachment(file=file)
            attachment.save()
            fix.files.add(attachment)
        fix.save()

        return JsonResponse({'succes':True})

    return HttpResponse('test')

@login_required
def getPublishableKey(request):

    key = settings.STRIPE_PUBLISHABLE_KEY

    return JsonResponse({'key': key})

@login_required
def chargePaymentToken(request):
    if request.method == "POST":
        transaction = StripeTransaction()
        transaction.save()
        data = request.body
        data = json.loads(request.body)
        token = data["token"]
        amount = data["amount"]
        if amount > 50:
            try:
                res = stripe.Charge.create(
                  amount=str(amount),
                  currency="eur",
                  description="BMDG Partners",
                  source=token,  # obtained with Stripe.js
                  idempotency_key=str(transaction.idempotency_key),
                )
                print(res)
                return JsonResponse({'success': True})
            except:
                return JsonResponse({'success': False})
        else:
            return JsonResponse({'success': False})


import speech_recognition as sr
import os
from gensim import summarization as summary
import nltk.tokenize
from nltk.stem.snowball import SnowballStemmer
import json

import sys
from pathlib import Path

isFilePresent = True
POS_REWARD = 1.0
NEG_REWARD = 1.0
THRESHOLD = 0.3
DIR = 'Outputs\\Transcriptions'

def getKeywords(text):

    nltk.download('wordnet')
    tokenized = nltk.tokenize.word_tokenize(text)
    stemmer = SnowballStemmer("english", ignore_stopwords=False)
    rawkws = summary.keywords(text)
    keyws = rawkws.split()

    return keyws

def writeData(text, params):
    final_keywords = ""
    for word in params:
        if params.get(word) > THRESHOLD:
            final_keywords += (word+",")

    final_keywords = final_keywords[:-1]
    print(final_keywords)

if __name__ == "__main__":

    params = {}
    receivedKeywords = getKeywords(sys.argv[1])
    # updateConfidence(receivedKeywords, params)
    writeData(receivedKeywords, params)