import sys
import pygame
from settings import *
from gmanager import GManager

class Game:
    def __init__(self):
        pygame.init()
        pygame.display.set_caption('Checkers Game')
        # general setup
        self.screen = pygame.display.set_mode((WIDTH, HEIGTH))
        self.background = pygame.Surface((WIDTH, HEIGTH))
        self.background.fill(pygame.Color('#000000'))
        self.clock = pygame.time.Clock()
        self.gm = GManager()
        self.gm.game_init("human", "human")

    def run(self):
        while True:
            for event in pygame.event.get():
                # quit game
                if event.type == pygame.QUIT:
                    pygame.quit()
                    sys.exit()
                # game events
                self.gm.game_play(event)

            self.gm.update()
            pygame.display.update()
            self.screen.blit(self.background, (0, 0))
            self.clock.tick(FPS)

if __name__ == '__main__':
    game = Game()
    game.run()


import os
import json
import subprocess

from PyQt5 import QtWidgets
from PyQt5 import uic

import utils

class EvaluatorGui(QtWidgets.QWidget):

    def __init__(self, x, y, h):

        super().__init__()

        if os.path.isfile(utils.cfg_filename):
            with open(utils.cfg_filename, 'r') as cfg:
                self.paths = json.load(cfg)
        else:
            self.paths = dict()

        uic.loadUi(utils.join('ui', 'Evaluation.ui'), self)

        folder = self.paths.get('evaluation_data', '')
        self.data_path = folder if os.path.isdir(folder) else ''
        self.data_folder_line.setText(self.data_path)
        folder = self.paths.get('evaluation_results', '')
        self.result_path = folder if os.path.isdir(folder) else ''
        self.result_folder_line.setText(self.result_path)

        self.data_folder_select_button.clicked.connect(self.open_data_path)
        self.result_folder_select_button.clicked.connect(self.open_result_path)
        self.evaluate_button.clicked.connect(self.evaluate)
        self.show_button.clicked.connect(self.show_points)

        self.eval_bat_file = utils.join(utils.root_folder, 'ins_gps_project',
                                        'sources', 'movement_model', 'eval_results.cmd')
        self.show_bat_file = utils.join(utils.root_folder, 'ins_gps_project',
                                        'sources', 'movement_model', 'show_user.cmd')

        self.setFixedSize(self.geometry().width(), self.geometry().height())
        self.move(x, y + h + 2)
        self.show()

        self.pid = None

    def evaluate(self):

        out_string = ''
        if self.position_checkbox.isChecked():
            out_string += 'position,'
        if self.velocity_checkbox.isChecked():
            out_string += 'velocity,'
        if self.course_checkbox.isChecked():
            out_string += 'course,'
        if self.pitch_checkbox.isChecked():
            out_string += 'pitch,'
        if self.roll_checkbox.isChecked():
            out_string += 'roll,'
        if self.sat_num_checkbox.isChecked():
            out_string += 'sat_num,'
        if self.sat_mean_checkbox.isChecked():
            out_string += 'sat_mean,'
        if self.sphere_checkbox.isChecked():
            out_string += 'sphere,'
        if self.pos_ratio_checkbox.isChecked():
            out_string += 'pos_ratio,'
        if self.vel_ratio_checkbox.isChecked():
            out_string += 'vel_ratio,'
        if self.course_ratio_checkbox.isChecked():
            out_string += 'course_ratio,'

        utils.save_cfg(self.paths)

        if self.pid is None:
            with open(self.eval_bat_file, 'w') as f:
                f.write('@echo off\n')
                f.write(''.join(['cd ', os.path.split(self.eval_bat_file)[0], '\n']))
                f.write(''.join(['"', self.paths['matlab'], '"',
                                 ' -nosplash -nodesktop -minimize -r \"addpath(\'',
                                 os.path.split(self.eval_bat_file)[0],
                                 '\'); display_results_func(\'',
                                 self.data_path,
                                 '\', \'',
                                 self.result_path,
                                 '\', {\'sel\', \'',
                                 out_string,
                                 '\'});\"\n']))

            self.pid = subprocess.Popen([self.eval_bat_file])
            self.evaluate_button.setText('CLOSE')
        else:
            utils.close_matlab()
            self.evaluate_button.setText('EVALUATE')
            self.pid = None

    def show_points(self):

        utils.save_cfg(self.paths)

        if self.pid is None:
            with open(self.show_bat_file, 'w') as f:
                f.write('@echo off\n')
                f.write(''.join(['cd ', os.path.split(self.show_bat_file)[0], '\n']))
                f.write(''.join(['"', self.paths['matlab'], '"',
                                 ' -nosplash -nodesktop -minimize -r \"addpath(\'',
                                 os.path.split(self.show_bat_file)[0],
                                 '\'); display_user_points_func(\'',
                                 self.result_path,
                                 '\');\"\n']))

            self.pid = subprocess.Popen([self.show_bat_file])
            self.show_button.setText('CLOSE')
        else:
            utils.close_matlab()
            self.show_button.setText('SHOW POINTS')
            self.pid = None

    def open_data_path(self):
        options = QtWidgets.QFileDialog.Options()
        options |= QtWidgets.QFileDialog.DontUseNativeDialog
        open_path = self.paths.get('evaluation_data', '')
        if not os.path.isdir(open_path):
            open_path = ''
        dirname = QtWidgets.QFileDialog.getExistingDirectory(self,
                                                             'Select Folder',
                                                             os.path.split(open_path)[0],
                                                             options=options)
        if dirname:
            self.paths['evaluation_data'] = dirname
            self.data_folder_line.setText(self.paths['evaluation_data'])
            self.data_path = self.paths['evaluation_data']

    def open_result_path(self):
        options = QtWidgets.QFileDialog.Options()
        options |= QtWidgets.QFileDialog.DontUseNativeDialog
        open_path = self.paths.get('evaluation_results', '')
        if not os.path.isdir(open_path):
            open_path = ''
        dirname = QtWidgets.QFileDialog.getExistingDirectory(self,
                                                             'Select Folder',
                                                             os.path.split(open_path)[0],
                                                             options=options)
        if dirname:
            self.paths['evaluation_results'] = dirname
            self.result_folder_line.setText(self.paths['evaluation_results'])
            self.result_path = self.paths['evaluation_results']

    def closeEvent(self, event):

        utils.save_cfg(self.paths)


import requests

all_pokemon_dict = {}
all_pokemon_info_dict = {}

def get_all_pokemon_dict():
    global all_pokemon_dict
    if not all_pokemon_dict:
        api_url = "https://pokeapi.co/api/v2/pokemon?limit=100000&offset=0"
        response = requests.get(api_url).json()
        for pokemon in response['results']:
            all_pokemon_dict[pokemon['name']] = pokemon['url']
    return all_pokemon_dict

def get_dict_info_pokemon(name):
    global all_pokemon_info_dict
    if name not in all_pokemon_info_dict:
        api_url = 'https://pokeapi.co/api/v2/pokemon/' + name
        response = requests.get(api_url).json()
        all_pokemon_info_dict[response['name']] = {
            'name': response['name'],
            'pokemon_id': response['id'],
            'height': response['height'],
            'weight': response['weight'],
            'base_exp': response['base_experience'],
            'sprite': response['sprites']['front_default'],
            'moves': {move['move']['name']: move['move']['url'] for move in response['moves']},
            'types': [type['type']['name'] for type in response['types']]
        }
    return all_pokemon_info_dict[name]


# -*- coding: utf-8 -*-
"""
Created on Wed Aug  4 09:40:30 2021

@author: Larissa Lima
"""

import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Dropout, GlobalAveragePooling3D
from tensorflow.keras.layers import Conv3D, MaxPooling3D, BatchNormalization
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
from keras_layer_normalization import LayerNormalization
from sklearn.model_selection import StratifiedKFold, train_test_split
from time import time
from sklearn.utils import class_weight
from tensorflow.python.keras import backend, regularizers
from tensorflow.keras.optimizers import Adam
from sklearn.utils import shuffle
from sklearn.metrics import balanced_accuracy_score, classification_report, confusion_matrix
import seaborn as sn
import matplotlib.pyplot as plt
import datetime
import tweaked_ImageGenerator_v2 as imggen

def data_aug(entrada_x, entrada_y, n_dados):
  new_imgs = []
  new_label = []

  for paciente in range(len(entrada_x)):
    img = entrada_x[paciente].reshape(20,img_size,img_size)
    label = entrada_y[paciente]
    for rotacoes in range(n_dados):
      image = imggen.random_rotation(img, 360, fill_mode='nearest')
      new_imgs.append(np.array(image))
      new_label.append(label)
  new_imgs = np.array(new_imgs)
  new_imgs = new_imgs.reshape(tuple([new_imgs.shape[0]] + list(entrada_x.shape[1:])))
  new_label = np.array(new_label)
  all_CTs = np.append(entrada_x, new_imgs, axis = 0)
  all_labels = np.append(entrada_y, new_label, axis = 0)
  all_CTs, all_labels = shuffle(all_CTs, all_labels, random_state=42) #shuffle/mistura dos dados
  all_CTs = all_CTs.reshape(all_CTs.shape[0], 20, img_size, img_size, 1) #fazendo channels_last
  return all_CTs, all_labels

def rotaciona(img, label, n_dados, new_imgs, new_label, angulo = 360, fill_mode='nearest'):
    for rotacoes in range(n_dados):
      image = imggen.random_rotation(img, 360, fill_mode='nearest')
      new_imgs.append(np.array(image))
      new_label.append(label)

def data_aug_v2(entrada_x, entrada_y, n_dados, classe_aug = -1):
  new_imgs = []
  new_label = []
  for paciente in range(len(entrada_x)):
    img = entrada_x[paciente].reshape(20,img_size,img_size)
    label = entrada_y[paciente]
    if classe_aug != -1:
        if label == classe_aug:
            rotaciona(img, label, n_dados, new_imgs, new_label, 360, 'nearest')
    else:
        rotaciona(img, label, n_dados, new_imgs, new_label, 360, 'nearest')
  new_imgs = np.array(new_imgs)
  new_imgs = new_imgs.reshape(tuple([new_imgs.shape[0]] + list(entrada_x.shape[1:])))
  new_label = np.array(new_label)
  all_CTs = np.append(entrada_x, new_imgs, axis = 0)
  all_labels = np.append(entrada_y, new_label, axis = 0)
  all_CTs, all_labels = shuffle(all_CTs, all_labels, random_state=42) #shuffle/mistura dos dados
  return all_CTs, all_labels

def model(n_neuronios1, n_neuronios2, drop, regul):
    classificador = Sequential()

    classificador.add(Conv3D(filters=16, kernel_size =(3,16,16), strides=(1, 4, 4), input_shape = (20,img_size,img_size,1), activation = 'relu', padding='same'))
    classificador.add(LayerNormalization())
    classificador.add(MaxPooling3D(pool_size = (2,2,2), strides=(2, 2, 2)))

    classificador.add(Conv3D(filters=8, kernel_size =(3,16,16), strides=(1, 4, 4), activation = 'relu', padding='same'))
    classificador.add(LayerNormalization())
    classificador.add(MaxPooling3D(pool_size = (2,2,2), strides=(2, 2, 2)))

    classificador.add(Flatten())

    classificador.add(Dense(units = n_neuronios1, activation = 'relu', kernel_regularizer=regularizers.l2(regul)))
    classificador.add(Dropout(drop))

    classificador.add(Dense(units = n_neuronios2, activation = 'relu', kernel_regularizer=regularizers.l2(regul)))
    classificador.add(Dropout(drop))

    classificador.add(Dense(units = 1, activation = 'sigmoid')) #2 opcoes de saida 1 ou 0

    opt = Adam(learning_rate=0.00001) #learning rate de 10-5
    classificador.compile(loss = 'binary_crossentropy', optimizer = opt,
                        metrics = ['accuracy', tf.keras.metrics.AUC(), tf.keras.metrics.Precision(), tf.keras.metrics.Recall(), tf.keras.metrics.TruePositives(), tf.keras.metrics.TrueNegatives(), tf.keras.metrics.FalsePositives(), tf.keras.metrics.FalseNegatives()],
                        weighted_metrics=['accuracy']
                        )
    return classificador

img_size = 128
all_CT = np.load(f'C:\\Users\\Larissa\\.spyder-py3\\all_CT_20_{img_size}x{img_size}.npy')
all_label = np.load(f'C:\\Users\\Larissa\\.spyder-py3\\all_label_20_{img_size}x{img_size}.npy')

print(pd.Series(list(all_label)).value_counts())

seed = 5
kfold = StratifiedKFold(n_splits = 3, shuffle = True, random_state = seed)
resultados = []
str_today = datetime.date.today().strftime('%Y-%m-%d')

n_neuronios1 = 8
n_neuronios2 = 8
drop = 0.6
regularizacao = 0.0001
epocas = 400
n_dados1 = 20 #numero que imagens geradas a partir de cada original para classe menor (1)
n_dados0 = 10 #numero que imagens geradas a partir de cada original para classe maior (0)
paciencia = 50 #por quantas epocas vamos esperar ate parar a execucao
delta = 0.01 #qual min mudanca que consideramos pra dizer que melhorou a metrica

acc_balanceada = []
i = 1
for indice_treino, indice_val in kfold.split(all_CT, all_label):
  start_time = time()
  print()
  print(f'Validacao {i}')
  print(indice_val)
  temp = pd.DataFrame(all_label, columns = ['label'])
  temp1 = pd.DataFrame(all_label[indice_treino], columns = ['label'])
  temp2 = pd.DataFrame(all_label[indice_val], columns = ['label'])
  print(f"doentes no total: {temp['label'].sum()} ({round(temp['label'].sum()/temp['label'].count()*100, 2)})%")
  print(f"doentes no treino: {temp1['label'].sum()} ({round(temp1['label'].sum()/temp1['label'].count()*100, 2)})%")
  print(f"doentes na val: {temp2['label'].sum()} ({round(temp2['label'].sum()/temp2['label'].count()*100, 2)})%")
  print()
  #data augmentation dos dados
  train_aug_CTs, train_aug_labels = data_aug_v2(all_CT[indice_treino], all_label[indice_treino], n_dados1, 1)
  train_aug_CTs, train_aug_labels = data_aug_v2(train_aug_CTs, train_aug_labels, n_dados0, 0)

  #channels last
  all_CT = all_CT.reshape(all_CT.shape[0], 20, img_size, img_size, 1)
  train_aug_CTs = train_aug_CTs.reshape(train_aug_CTs.shape[0], 20, img_size, img_size, 1)
  print("data aug realizado")

  X_val = all_CT[indice_val]
  y_val = all_label[indice_val]

  modelo = model(n_neuronios1, n_neuronios2, drop, regularizacao)
  # Salva o modelo focando em maximizar o auc da validacao.
#  checkpoint = ModelCheckpoint(f"C:\\Users\\Larissa\\.spyder-py3\\lari_model_{i}_{str_today}.h5", monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='min')
  # Pausa o modelo com base no erro da validação e salva os melhores pesos.
  early = EarlyStopping(monitor='val_loss', min_delta=delta, patience=paciencia, verbose=1, mode='min', restore_best_weights = True)

  peso_classes = class_weight.compute_class_weight(class_weight = 'balanced', classes = np.unique(train_aug_labels), y = np.ravel(train_aug_labels))  #calcula peso pra equilibrar proporcao entre classes
  history = modelo.fit(train_aug_CTs, train_aug_labels,
                    batch_size = 36, epochs = epocas, class_weight = {0: peso_classes[0], 1: peso_classes[1]},
                    validation_data = (X_val, y_val), callbacks=[early])

  # vamos avaliar as metricas com base na validação
  y_pred = (modelo.predict(X_val) > 0.5).astype("int32")
  acc_balanceada.append(balanced_accuracy_score(y_val, y_pred))

  metricas = modelo.evaluate(X_val, y_val)
  resultados.append(metricas)
  print(metricas)

  backend.clear_session()

  plt.plot(history.history['accuracy'], 'b')
  plt.plot(history.history['val_accuracy'], 'g')
  plt.xlabel('Epochs')
  plt.ylabel('accuracy')
  plt.legend(['train', 'validation'], loc='upper left')
  plt.show()

  plt.plot(history.history['loss'], 'b')
  plt.plot(history.history['val_loss'], 'g')
  plt.xlabel('Epochs')
  plt.ylabel('Loss')
  plt.legend(['train', 'validation'], loc='upper right')
  plt.show()

  end_time = time()
  tempo_total = end_time - start_time

  print(f'tempo da validacao {i}: {tempo_total/60} minutos')

  print(classification_report(y_val, y_pred))

  print(f'balanced accuracy: {balanced_accuracy_score(y_val, y_pred)}')

  print(f'Total tests: {len(y_val)}')

  cm = confusion_matrix(y_val, y_pred)
  plt.figure(figsize = (5,5))
  sn.heatmap(cm, annot = True)
  plt.xlabel('Predicted')
  plt.ylabel('Truth')
  plt.show()

  linhas = int(np.ceil(len(X_val)/5))
  fig, ax = plt.subplots(nrows=linhas, ncols=5, figsize=(15,10))
  fig.subplots_adjust(hspace=0.6, wspace=0.5)
  j = 0
  for linha in ax:
    for coluna in linha:
      if j == X_val.shape[0]:
        break
      coluna.imshow(X_val.reshape(X_val.shape[0], 1, 20, img_size, img_size)[j][0][10], cmap = plt.cm.bone)
      coluna.set_title(f'Previsto:{y_pred[j]}, Correto:{y_val[j]}')
      j += 1
      if j == len(X_val):
          break
  plt.show()

  i += 1

loss = []
acuracia = []
auc = []
precisao = []
recall = []
sensib = []
espec = []
acuracia_balanceada = []

for i in range(len(resultados)):
  VP = resultados[i][5]
  VN = resultados[i][6]
  FP = resultados[i][7]
  FN = resultados[i][8]
  loss.append(resultados[i][0])
  acuracia.append(resultados[i][1])
  auc.append(resultados[i][2])
  precisao.append(resultados[i][3])
  recall.append(resultados[i][4])
  sensib.append(VP/(VP+FN))
  espec.append(VN/(VN+FP))
  acuracia_balanceada.append(resultados[i][9])

print(f'Loss medio: {np.mean(loss)}')
print(f'Acuracia media: {round(np.mean(acuracia)*100, 2)}%')
print(f'AUC medio: {round(np.mean(auc)*100, 2)}%')
print(f'Precisao media: {round(np.mean(precisao)*100, 2)}%')
print(f'Recall medio: {round(np.mean(recall)*100, 2)}%')
print(f'Sensibilidade media: {round(np.mean(sensib)*100, 2)}%')
print(f'Especificidade media: {round(np.mean(espec)*100, 2)}%')
print(f'Acuracia balanceada media: {round(np.mean(acuracia_balanceada)*100, 2)}%')
print(f'acur. balanceada scikit: {round(np.mean(acc_balanceada)*100, 2)}%')

print()
print(f'loss: {loss}')
print(f'acuracia: {acuracia}')
print(f'AUC: {auc}')
print(f'precisao: {precisao}')
print(f'recall: {recall}')
print(f'sensibilidade: {sensib}')
print(f'especificidade: {espec}')
print(f'acur. balanceada: {acuracia_balanceada}')
print(f'acur. balanceada do scikit: {acc_balanceada}')


import torch
from torch import nn

class PolarIoULoss(nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self,y_pred, y_gt):
        batchsize = y_pred.shape[0]
        dim = y_pred.shape[1]
        loss = 0.0
        for i in range(batchsize):
            sum_max = torch.zeros(1)
            sum_min = torch.zeros(1)
            for j in range(dim):
                sum_max += torch.max(y_pred[i][j], y_gt[i][j])
                sum_min += torch.min(y_pred[i][j], y_gt[i][j])
            loss += torch.log(sum_max/sum_min)
        loss /= batchsize
        return loss


from django.urls import path

from .views import *

urlpatterns = [
    path('', first, name='first'),
    path('contact/', contact, name ='contact'),
path('logout/', logout_view, name ='logout'),
path('buildings/', KidsKindergarden.as_view(), name ='buildings'),
path('search/', Search.as_view(), name ='search'),
path('search/<int:sad_id>/', KindergardenInfo.as_view(), name ='sad'),
path('school/', school, name = 'school')

]


#! /usr/bin/python
# Author: Kai Chen

# Make the Figure 3 for paper: Relationship between SC and Residual.

# %%
from fcpy.core import *
import numpy as np
import matplotlib.pyplot as plt
from MakeFigure1 import axis_log_formater, gen_sc_fc_figure_new
from matplotlib.ticker import MultipleLocator, AutoMinorLocator
# %%
path = 'image/'
data_tdmi = EcogTDMI()
data_tdmi.init_data(path, 'snr_th_kmean_tdmi.pkl')
sc_tdmi, fc_tdmi = data_tdmi.get_sc_fc('ch')
snr_mask_tdmi = data_tdmi.get_snr_mask(path, 'snr_th_kmean_tdmi.pkl')
roi_mask = data_tdmi.roi_mask.copy()

data_package = np.load('data/preprocessed_data.npz', allow_pickle=True)
dist_mat = data_package['d_matrix']
d_mat = {band: dist_mat[roi_mask] for band in data_tdmi.filters}

data_gc = EcogGC()
data_gc.init_data()
sc_gc, fc_gc = data_gc.get_sc_fc('ch')
# %%

fig, ax = plt.subplots(1,2,figsize=(9,4), dpi=400)

band = 'raw'
# new_mask = np.ones_like(snr_mask_tdmi[band])
new_mask = snr_mask_tdmi[band].copy()
new_mask[sc_tdmi[band]==0] = False
new_mask[sc_tdmi[band]==1.5] = False

d_metric = 1./d_mat[band]
# regression with distance to get residue
fc_tdmi_res = fc_tdmi[band].copy()
nan_mask = ~np.isnan(np.log10(fc_tdmi_res))
pval = np.polyfit(d_metric[nan_mask*new_mask], np.log10(fc_tdmi_res[nan_mask*new_mask]), deg=1)
fc_tdmi_res[nan_mask*new_mask] = np.abs(fc_tdmi_res[nan_mask*new_mask] - 10**np.polyval(pval, d_metric[nan_mask*new_mask]))

gen_sc_fc_figure_new(ax[0], fc_tdmi_res, sc_tdmi[band], new_mask,)

# new_mask = np.ones_like(new_mask, dtype=bool)
# new_mask[sc_gc[band]==0] = False
# new_mask[sc_gc[band]==1.5] = False
fc_gc_res = fc_gc[band].copy()
nan_mask = ~np.isnan(np.log10(fc_gc_res))
pval = np.polyfit(d_metric[nan_mask], np.log10(fc_gc_res[nan_mask]), deg=1)
fc_gc_res[nan_mask] = np.abs(fc_gc_res[nan_mask] - 10**np.polyval(pval, d_metric[nan_mask]))
gen_sc_fc_figure_new(ax[1], fc_gc_res, sc_gc[band], new_mask,)

for axi, labeli in zip(ax, ('TDMI', 'GC')):
    axi.set_title(axi.get_title().replace(band, labeli))
    axi.set_ylabel('Residual')
fig.suptitle(band)

fig.savefig(path+'Figure_3.png')
# %%
def Linear_R2(x:np.ndarray, y:np.ndarray, z:np.ndarray, pval:np.ndarray)->float:
    """Compute R-square value for linear fitting.

    Args:
        x (np.ndarray): variable 1 of function
        y (np.ndarray): variable 2 of function
        y (np.ndarray): image of function
        pval (np.ndarray): parameter of linear fitting

    Returns:
        float: R square value
    """
    mask = ~np.isnan(x)*~np.isnan(y)*~np.isnan(z)*~np.isinf(x)*~np.isinf(y)*~np.isinf(z)# filter out nan
    z_predict = x[mask]*pval[0]+y[mask]*pval[1] + pval[2]
    R = np.corrcoef(z[mask], z_predict)[0,1]
    return R**2

@axis_log_formater(axis='both')
def gen_sc_fc_figure_3d(ax, fc:np.ndarray,
                        sc:np.ndarray,
                        dist:np.ndarray,
                        snr_mask:np.ndarray=None,
                        is_log:bool=True,
)->plt.Figure:
    """Generated figure for analysis of causal distributions.

    Args:
        tdmi_flatten (np.ndarray): flattened data for target tdmi statistics.
        sc (np.ndarray): flattened data for true connectome.

    Returns:
        plt.Figure: matplotlib.figure.Figure
    """
    if snr_mask is None:
        snr_mask = np.ones_like(fc).astype(bool)
    if is_log:
        log_fc = np.log10(fc)
    else:
        log_fc = fc.copy()

    log_fc[~snr_mask] = np.nan
    # pval = np.polyfit(np.log10(sc+1e-6)[snr_mask], log_fc[snr_mask], deg=1)
    coef_mat = np.vstack((np.log10(sc+1e-6)[snr_mask], np.log10(dist)[snr_mask])).T
    coef_mat = np.hstack((coef_mat, np.ones((coef_mat.shape[0], 1)))) # adding bias
    b = log_fc[snr_mask]
    pval,_,_,_ = np.linalg.lstsq( coef_mat, b )
    # ax.plot3D(np.log10(sc+1e-6), np.log10(dist), log_fc.flatten(), 'k.', label='TDMI samples')
    ax.scatter3D(np.log10(sc+1e-6), np.log10(dist), log_fc, c='k', s=2, label='TDMI (above SNR th)', zorder=0)
    x_range = (ax.get_xticks()[0], ax.get_xticks()[-1])
    y_range = (ax.get_yticks()[0], ax.get_yticks()[-1])
    z_range = (ax.get_zticks()[0], ax.get_zticks()[-1])
    xx, yy = np.meshgrid(np.linspace(*x_range, 30),np.linspace(*y_range, 30))
    ax.plot_surface(xx, yy, xx * pval[0] + yy * pval[1] + pval[2],
        alpha=.5, cmap=plt.cm.coolwarm,
        edgecolors='royalblue', lw=0.,antialiased=False,
        zorder=0,
    )
    # ax.set_xlabel(r'$log_{10}$(Connectivity Strength)')
    # sc_buffer = sc.copy()
    # sc_buffer[sc_buffer==0] = np.nan
    # ax.set_xlim(*x_range)
    # ax.set_ylim(*y_range)
    # ax.set_zlim(*z_range)
    ax.set_title('r = %5.3f' %
        Linear_R2(np.log10(sc+1e-6), np.log10(dist), log_fc, pval)**0.5,
        fontsize=16,
    )
    return ax

# %%
fig = plt.figure(figsize=(14,6), dpi=400)
gs = fig.add_gridspec(nrows=1, ncols=2,
                        left=0.05, right=0.96, top=0.96, bottom=0.05,
                        wspace=0.15, hspace=0.20)
ax = [fig.add_subplot(i, projection='3d', azim=-20, elev=30) for i in gs]

band = 'raw'
# new_mask = np.ones_like(snr_mask_tdmi[band])
new_mask = snr_mask_tdmi[band].copy()
new_mask[sc_tdmi[band]==0] = False
new_mask[sc_tdmi[band]==1.5] = False

d_metric = 1./d_mat[band]
# regression with distance to get residue

gen_sc_fc_figure_3d(ax[0], fc_tdmi[band], sc_tdmi[band], d_metric, new_mask,)

# new_mask = np.ones_like(new_mask, dtype=bool)
# new_mask[sc_gc[band]==0] = False
# new_mask[sc_gc[band]==1.5] = False
gen_sc_fc_figure_3d(ax[1], fc_gc[band], sc_gc[band], d_metric, new_mask,)

for axi, labeli in zip(ax, ('TDMI', 'GC')):
    axi.set_title(labeli + ' : ' + axi.get_title())
    axi.set_xlabel('SC')
    axi.set_ylabel('1/Distance')
    axi.set_zlabel(r'$\log_{10}$('+labeli+')', rotation=270)
    axi.xaxis.set_major_locator(MultipleLocator(4))
    axi.xaxis.set_minor_locator(MultipleLocator(2))
    axi.yaxis.set_major_locator(MultipleLocator(1))
    axi.yaxis.set_minor_locator(MultipleLocator(0.5))
    axi.zaxis.set_major_locator(MultipleLocator(1))
fig.suptitle(band)

fig.savefig(path+'Figure_3-1.png')
# %%

# %%


import abc
import discord
import asyncio
import urllib.parse
import operator

from redbot.core import Config
from redbot.core import commands
from redbot.core import checks
from redbot.core.utils.predicates import ReactionPredicate
from redbot.core.utils.menus import start_adding_reactions

defaults = {"RoleRanges": {}, "LogChannel": None, "Players": {}, "Results": []}
verify_timeout = 30
k_factor = 30

class SixMansElo(commands.Cog):
    """Manages aspects of Ballchasing Integrations with RSC"""

    def __init__(self, bot):
        self.bot = bot
        self.config = Config.get_conf(self, identifier=1234567893, force_registration=True)
        self.config.register_guild(**defaults)
        self.players = []
        self.six_mans_cog = bot.get_cog("SixMans")

        try:
            self.observe_six_mans()
        except:
            pass

    @commands.guild_only()
    @commands.command()
    @checks.admin_or_permissions(manage_guild=True)
    async def setLogChannel(self, ctx, log_channel: discord.TextChannel):
        """Sets the channel where all transaction messages will be posted"""
        await self._save_log_channel(ctx.guild, log_channel.id)
        await ctx.send("Done")

    @commands.guild_only()
    @commands.command()
    @checks.admin_or_permissions(manage_guild=True)
    async def unsetLogChannel(self, ctx):
        """Sets the channel where all transaction messages will be posted"""
        await self._save_log_channel(ctx.guild, None)
        await ctx.send("Done")

    @commands.command()
    @commands.guild_only()
    @checks.admin_or_permissions(manage_guild=True)
    async def addPlayers(self, ctx, *players_to_add):
        """Add the players provided to the player list.

        Arguments:

        players_to_add -- One or more players in the following format:
        ```
        "['<player_id>','<wins>', '<losses>', '<elo_rating>']"
        ```
        Each player should be separated by a space.

        Examples:
        ```
        [p]addPlayers "['123456789','2', '1', '1000']"
        [p]addPlayers "['123456789','2', '1', '1000']" "['987654321','1', '2', '950']"
        ```
        """
        addedCount = 0
        try:
            for playerStr in players_to_add:
                player = ast.literal_eval(playerStr)
                playerAdded = await self._add_player(ctx, *player)
                if playerAdded:
                    addedCount += 1
                else:
                    await ctx.send("Error adding player: {0}".format(repr(player)))
        finally:
            await ctx.send("Added {0} players(s).".format(addedCount))
        await ctx.send("Done.")

    @commands.command()
    @commands.guild_only()
    @checks.admin_or_permissions(manage_guild=True)
    async def addPlayer(self, ctx, member: discord.Member, wins: int, losses: int, elo_rating: int):
        """Add a single player and their info to the file system."""
        playerAdded = await self._add_player(ctx, member, wins, losses, elo_rating)
        if(playerAdded):
            await ctx.send("Done.")
        else:
            await ctx.send("Error adding player: {0}".format(member.name))

    @commands.command()
    @commands.guild_only()
    @checks.admin_or_permissions(manage_guild=True)
    async def clearEloRoles(self, ctx):
        await self._save_role_ranges(ctx.guild, {})
        await ctx.send("Done")

    @commands.command()
    @commands.guild_only()
    @checks.admin_or_permissions(manage_guild=True)
    async def addEloRole(self, ctx, role: discord.Role, min_elo: float, max_elo: float):
        # Save Elo Role
        await self._register_role_range(role, min_elo, max_elo)

        # Log Role addition
        log_channel = await self._get_log_channel(role.guild)

        # Add roles to appropriate existing players
        for player in self.players:
            if player.elo_rating >= min_elo and player.elo_rating <= max_elo:
                await player.member.add_roles(role)
                if log_channel:
                    message = "{} has had their roles updated!".format(player.member.mention)
                    message += "\n\t - **Added:** {}".format(role.name)
        await ctx.send("Done.")

    @commands.command(aliases=['getEloRoles'])
    @commands.guild_only()
    @checks.admin_or_permissions(manage_guild=True)
    async def getEloRanges(self, ctx):
        guild = ctx.guild
        elos = await self._get_role_ranges(guild)

        elo_roles = []
        for elo_id in elos.keys():
            elo_roles.append(guild.get_role(int(elo_id)))
        elo_roles = sorted(elo_roles, key=operator.attrgetter('position'))
        elo_roles.reverse()

        out = "Elo Ranges"
        for role in elo_roles:
            elo_range = elos[str(role.id)]
            role = ctx.guild.get_role(int(role.id))
            out += "\n- {}: [{}-{}]".format(role.mention, elo_range[0], elo_range[1])

        await ctx.send(out)

    @commands.guild_only()
    @commands.Cog.listener("on_guild_role_delete")
    async def on_guild_role_delete(self, role):
        await self._unregister_elo_role(role)

## OBSERVER PATTERN IMPLEMENTATION ########################

    @commands.command()
    @commands.guild_only()
    @checks.admin_or_permissions(manage_guild=True)
    async def observeForElo(self, ctx):
        self.observe_six_mans()
        await ctx.send("Observing.")

    @commands.guild_only()
    @commands.Cog.listener("on_ready")
    async def on_ready(self):
        pass
        # self.observe_six_mans()

    @commands.guild_only()
    @commands.Cog.listener("on_resumed")
    async def on_resumed(self):
        pass
        # self.observe_six_mans()

    def observe_six_mans(self):
        self.six_mans_cog = self.bot.get_cog("SixMans")
        self.six_mans_cog.add_observer(self)

    async def update(self, game):
        await game.textChannel.send("State: {}".format(game.game_state))
        if game.game_state == "game over":
            await self._finish_series(game)

###########################################################

    async def _finish_series(self, game, update_roles=True):
        if not game.winner or game.winner.lower() not in ['blue', 'orange']:
            return False

        # get blue avg elo
        blue_players = [self._get_player_by_id(blue_player) for blue_player in game.blue]
        orange_players = [self._get_player_by_id(orange_player) for orange_player in game.orange]

        # remove/ignore players who aren't found
        while None in blue_players:
            blue_players.remove(None)
        while None in orange_players:
            orange_players.remove(None)

        avg_blue_elo = self._get_avg_team_rating(blue_players)
        avg_orange_elo = self._get_avg_team_rating(orange_players)

        # determine winner
        blue_wins = 0
        orange_wins = 0
        if game.winner.lower() == 'blue':
            blue_wins = 1
        else:
            orange_wins = 1

        # calculate new team elo
        new_blue_elo, new_orange_elo, = self._update_elo(elo1, elo2, blue_wins/(blue_wins + orange_wins))

        blue_diff = new_blue_elo - avg_blue_elo
        orange_diff = new_orange_elo - avg_orange_elo

        # apply new elo on players
        for player in blue_players:
            self._update_player_info(player, blue_wins, orange_wins, player.elo_rating + blue_diff)
        for player in orange_players:
            self._update_player_info(player, orange_wins, blue_wins, player.elo_rating + orange_diff)

        await self._save_players(blue_players + orange_players)

        if update_roles:
            for player in blue_players + orange_players:
                await self._reassign_elo_roles(player.member, player.elo_rating)

#region Adjust Player Ratings
    async def _reassign_elo_roles(self, player: discord.Member, player_elo: int=None):
        if player_elo == None:
            elo = await self._get_player_rating(player)

        log_channel = await self._get_log_channel(player.guild)

        guild = player.guild
        all_elo_roles = await self._get_sm_roles(guild)
        remove_roles = list(set(all_elo_roles) & set(player.roles))
        add_elo_roles = await self._get_elo_roles(elo)

        keep_roles = list(set(add_elo_roles) & set(remove_roles))
        for role in keep_roles:
            add_elo_roles.remove(role)
            remove_roles.remove(role)

        if add_elo_roles or remove_roles:
            try:
                await player.remove_roles(*remove_roles)
                await player.add_roles(*add_elo_roles)

                # return add_elo_roles, remove_roles
                if log_channel:
                    message = "{} has had their roles updated!".format(player.mention)
                    if add_elo_roles:
                        message += "\n\t - **Added:** {}".format(', '.join(role.name for role in add_elo_roles))
                    if remove_roles:
                        message += "\n\t - **Removed:** {}".format(', '.join(role.name for role in remove_roles))
            except:
                return False

    def _get_avg_team_rating(self, players):
        total = 0
        for member in players:
            total += player.elo_rating

        return total/len(players)

    async def _get_player_rating(self, player: discord.Member):
        pass

    def _update_player_info(self, player, new_wins, new_losses, new_elo_rating):
        player.wins += new_wins
        player.losses += new_losses
        player.elo_rating = new_elo_rating

    async def _get_elo_roles(self, guild: discord.Guild, elo):
        elo_roles = []
        for role, elo_range in (await self._get_role_ranges(guild)):
            min_elo = elo_range[0]
            max_elo = elo_range[1]
            if elo >= min_elo and elo <= max_elo:
                elo_roles.append(role)
        return elo_roles

    async def _get_player_sm_roles(self, player: discord.Member):
        return list(set(await self._get_sm_roles(member.guild)) & set(player.roles))

    def _update_elo(self, elo1: int, elo2: int, result: float):
        """Calculates and returns the new Elo ratings for the two elo ratings based on their match results and the K-factor.
        Result param should be a decimal between 0 and 1 relating to the match results for elo1, i.e. a result of 1
        means player 1 won all the games in the match, a result of .25 means player 1 won 25% of the games in the match, etc."""
        elo_dif = int(player_1_elo) - int(player_2_elo)
        exponent = -1 * (elo_dif / 100)
        expectation = 1 / (1 + pow(10, exponent))
        new_elo1 = round(int(elo1) + (k_factor * (result - expectation)))
        new_elo2 = round(int(elo2) + (k_factor * ((1 - result) - (1 - expectation))))
        return new_elo1, new_elo2

#endRegion Adjust Player Ratings

#region SixMansRoles settings
    async def _register_role_range(self, role: discord.Role, min_elo: float, max_elo: float):
        if min_elo > max_elo:
            return False
        guild = role.guild
        role_ranges = await self._get_role_ranges(guild)
        role_ranges[str(role.id)] = [min_elo, max_elo]
        await self._save_role_ranges(guild, role_ranges)

    async def _unregister_elo_role(self, role: discord.Role):
        guild = role.guild
        role_ranges = await self._get_role_ranges(guild)
        if str(role.id) in role_ranges:
            del role_ranges[str(role.id)]
            await self._save_role_ranges(role_ranges)
            if log_channel:
                await log_channel.send("The Six Mans role **{}** has been removed.".format(role.name))

    async def _get_role_range(self, role: discord.Role):
        guild = role.guild
        role_ranges = await self._get_role_ranges(guild)
        for sm_role_id, role_range in role_ranges.items():
            sm_role = guild.get_role(int(sm_role_id))
            if role == sm_role:
                return role_range[0], rolerange[1]
        return None

    async def _get_sm_roles(self, guild: discord.Guild):
        return [guild.get_role(int(role_id)) for role_id in (await self._get_role_ranges(guild)).keys()]

    async def _get_role_ranges(self, guild: discord.Guild):
        return await self.config.guild(guild).RoleRanges()

    async def _save_role_ranges(self, guild: discord.Guild, role_ranges):
        await self.config.guild(guild).RoleRanges.set(role_ranges)

#endregion SixMansRoles settings

#region uncategoriezed helpers
    def _get_player_by_id(self, member_id: discord.Member, players=None):
        if not players:
            players = self.players
        for player in players:
            if player.member.id == member_id:
                return player
        return None

    async def _add_player(self, ctx, member, wins, losses, elo_rating, assign_elo_roles=True):
        await self.load_players(ctx)
        players = self.players

        wins = int(wins)
        losses = int(losses)
        elo_rating = int(elo_rating)

        # Validation of input
        # There are other validations we could do, but don't
        #     - that there aren't extra args for example
        errors = []
        if not isinstance(member, discord.Member):
            try:
                member = await commands.MemberConverter().convert(ctx, member)
            except:
                errors.append("Member {} not found.".format(member))
        if wins < 0:
            errors.append("Wins cannot be a negative number.")
        if losses < 0:
            errors.append("Losses cannot be a negative number.")
        if not elo_rating:
            errors.append("Elo rating not found.")
        if errors:
            await ctx.send(":x: Errors with input:\n\n  "
                               "* {0}\n".format("\n  * ".join(errors)))
            return

        try:
            player = Player(member, wins, losses, elo_rating, -1)
            players.append(player)
            if add_elo_roles:
                await self._reassign_elo_roles(member, elo_rating)
        except:
            return False
        await self._save_players(ctx, players)
        return True

    async def _remove_player(self, ctx, member: discord.Member):
        await self.load_players(ctx)
        players = self.players

        try:
            player = await self.get_player_by_id(self.players, member.id)
            if not player:
                await ctx.send("{0} does not seem to be a current player.".format(member.name))
                return False
            players.remove(player)
        except ValueError:
            await ctx.send("{0} does not seem to be a current player.".format(member.name))
            return False
        await self._save_players(ctx, players)
        return True

#endregion uncategorized helpers

#region load/save methods
    async def _save_log_channel(self, guild, channel_id):
        await self.config.guild(guild).LogChannel.set(channel_id)

    async def _get_log_channel(self, guild: discord.Guild):
        guild.get_channel(await self.config.guild(guild).LogChannel())

    async def load_players(self, ctx, force_load = False):
        players = await self._players(ctx)
        player_list = []
        remove_player = False
        for value in players.values():
            member = ctx.guild.get_member(value["Id"])
            if not member:
                # Member not found in server, don't add to list of players and
                # re-save list at the end to ensure they get removed
                remove_player = True
                continue
            wins = value["Wins"]
            losses = value["Losses"]
            elo_rating = value["EloRating"]
            # temp_rating = value["TempRating"]
            player = Player(member, wins, losses, elo_rating)
            player_list.append(player)

        self.players = player_list
        if remove_player:
            await self._save_players(ctx, self.players)

    async def _players(self, ctx):
        return await self.config.guild(ctx.guild).Players()

    async def _save_players(self, ctx, players):
        player_dict = {}
        for player in players:
            player_dict[player.member.id] = player._to_dict()
        await self.config.guild(ctx.guild).Players.set(player_dict)

#endregion

class Player:
    def __init__(self, member: discord.Member, wins: int, losses: int, elo_rating: int):
        self.member = member
        self.wins = wins
        self.losses = losses
        self.elo_rating = elo_rating

    def _to_dict(self):
        return {
            "Id": self.member.id,
            "Wins": self.wins,
            "Losses": self.losses,
            "EloRating": self.elo_rating,
            "TempRating": self.temp_rating
        }


import csv
import urllib.request
import xml.etree.ElementTree as ET
import pprint
import time

with open("trainingdata/spamnodes.csv") as f:
    nodes = f.read().splitlines()

changesetDb = []
with open('trainingdata/changesets.csv', newline='',encoding='utf-8') as csvfile:
    spamreader = csv.reader(csvfile, delimiter=',')
    next(spamreader) # skip header.
    for row in spamreader:
        changesetDb.append(row)

osmApiBase =  "https://api.openstreetmap.org"
nodeCount = 0
for node in nodes:
    url = "{}/api/0.6/node/{}/history".format(osmApiBase,node)
    try:
        with urllib.request.urlopen(url) as response:
            bodyStr = response.read()
            #print(bodyStr)
            body = ET.fromstring(bodyStr)

            #for revision in body:
            #    print(revision.tag, revision.attrib)
            #    for tags in revision:
            #        print("  {} {}".format(tags.tag, tags.attrib))

            #exit(0)

            changeset = ''
            for revision in body:
                for tag in revision:
                    if ( tag.tag == 'tag' and tag.attrib['k'] == 'description') :
                        if ( len(changeset) == 0 ):
                            changeset = revision.attrib['changeset']

            print("node {} changeset {}".format( node, changeset))

            alreadyIn = False
            for row in changesetDb:
                if ( row[0] == changeset) :
                    alreadyIn = True

            if (alreadyIn == False):
                changesetDb.append( [changeset,'','','Y'] )

            if ( nodeCount % 5) == 0 :
                time.sleep(2)

            nodeCount = nodeCount + 1
    except:
        print("error fetching {}".format(url))

#with open('data/changesets.csv', 'w', encoding='utf-8') as csvfile:
#    csvfile.write("changeset,Notes,Validated,SPAM\n")
#    for row in changesetDb:
#        csvfile.write(",".join(row) + "\n")


import pygame as pg
from math import sqrt

class Bird(pg.sprite.Sprite):
    def __init__(self, board, alive=False):
        pg.sprite.Sprite.__init__(self)
        self.x, self.y = board.mid
        self.board = board
        self.score = 0

        self.size = (int(26*0.1*board.board_w/18),
                     int(17*0.1*board.board_w/18))
        self.IMGS = [pg.transform.scale(pg.image.load('data/Bird.png'), self.size),
                     pg.transform.scale(pg.image.load('data/Bird_jumping.png'), self.size)]
        self.IMGS = [pg.transform.flip(i, True, False) for i in self.IMGS]
        self.image = self.IMGS[0]

        self.alive = alive
        self.jumping = False
        self.tick = 0
        self.vel = 5

        self.best_gap = (self.board.right, self.board.mid[1])
        self.reset_rect()

    def collision(self, spikes):
        coll = pg.sprite.spritecollideany(self, spikes, False)
        if coll:
            if pg.sprite.spritecollide(self, spikes, False, pg.sprite.collide_mask):
                return True
            return False

    def jump(self):
        self.tick = -20
        self.jumping = True
        self.alive = True

    def move(self):
        self.tick += 1
        parabola = -0.001*self.tick**2+0.5*self.tick
        # 60fps    -0.001*self.tick**2+0.5*self.tick
        self.y += parabola
        self.x += self.vel

        self.reset_rect()

    def collision_check(self, spikes, AI=False):
        # COLISION WITH RIGHT SITE
        if self.x+self.image.get_width()/2 >= self.board.right:
            self.IMGS = [pg.transform.flip(i, True, False) for i in self.IMGS]
            self.vel *= -1
            self.score += 1
            spikes.random_spikes(self.board.left)
        # COLISION WITH LEFT SITE
        elif self.x-self.image.get_width()/2 <= self.board.left:
            self.IMGS = [pg.transform.flip(i, True, False) for i in self.IMGS]
            self.vel *= -1
            self.score += 1
            spikes.random_spikes(self.board.right)
        # COLISION WITH BOTTOM SITE
        elif self.y+self.image.get_height() > self.board.bottom:
            self.reset_bird(spikes)
        # COLISION WITH TOP SITE
        elif self.y < self.board.top:
            self.reset_bird(spikes)
        # COLISION WITH SPIKES
        if self.collision(spikes.spikes):
            self.reset_bird(spikes)

        if AI:
            return self.alive

    def find_best_gap(self, spikes):
        distances = []
        for g in spikes.gaps:
            distances.append(sqrt((self.x - g[0]) ** 2 + (self.y - g[1]) ** 2))
        if len(spikes.gaps) > 0:
            self.best_gap = spikes.gaps[distances.index(min(distances))]

    def draw(self, screen):
        self.image = self.IMGS[0]
        if self.jumping:
            self.image = self.IMGS[1]
            if self.tick > 3:
                self.jumping = False
        screen.blit(self.image, self.rect)

    def reset_bird(self, spikes):
        self.image = self.IMGS[0]
        self.jumping = False
        self.alive = False
        self.score = 0
        self.x, self.y = self.board.mid
        spikes = spikes.initial_spikes()
        self.reset_rect()

    def reset_rect(self):
        self.rect = self.image.get_rect()
        self.rect.center = (self.x, int(self.y+self.image.get_height()/2))
        self.mask = pg.mask.from_surface(self.image)


from torch.utils.data import Dataset, DataLoader
import torch
import matplotlib.pyplot as plt

class Data(Dataset):
    def __init__(self):
        self.x = torch.arange(-3, 3, 0.1).view(-1, 1)
        self.f = 2 * self.x - 1
        self.y = self.f + 0.5*torch.randn((self.x.shape[0],1))

    def __len__(self):
        return self.x.shape[0]

    def __getitem__(self, index):
        return self.x[index], self.y[index]

dataset = Data()
data = DataLoader(dataset, batch_size = 16, shuffle = True)

class LinearRegression1(torch.nn.Module):
    def __init__(self, input_size, output_size):
        torch.nn.Module.__init__(self)
        self.linear = torch.nn.Linear(input_size, output_size)

    def forward(self, x):
        return self.linear(x)

    def train(self, train_data, epochs, optimizer, loss_function):
        for epoch in range(epochs):
            for x, y in train_data:
                yhat = self(x)
                loss = loss_function(yhat, y)
                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

class LinearRegression2(torch.nn.Module):
    def __init__(self, input_size, output_size):
        torch.nn.Module.__init__(self)
        self.w = torch.nn.Parameter(torch.randn(()))
        self.bias = torch.nn.Parameter(torch.randn(()))

    def forward(self, x):
        return self.w*x + self.bias

    def train(self, train_data, epochs, lr):
        for epoch in range(epochs):
            for x, y in train_data:
                yhat = self(x)
                loss = torch.sum((yhat - y)**2)/x.shape[0]
                loss.backward()
                self.w.data -= lr*self.w.grad
                self.bias.data -= lr*self.bias.grad
                self.w.grad.data.zero_()
                self.bias.grad.data.zero_()

class LinearRegression3(torch.nn.Module):
    def __init__(self, input_size, output_size):
        torch.nn.Module.__init__(self)
        self.w = torch.randn(())
        self.bias = torch.randn(())

    def forward(self, x):
        return self.w*x + self.bias

    def train(self, train_data, epochs, lr):
        for epoch in range(epochs):
            for x, y in train_data:
                yhat = self(x)
                loss = torch.sum((yhat - y)**2)/x.shape[0]
                grad_w = (2/x.shape[0]) * torch.sum((y - self.w*x - self.bias)*-x)
                grad_b = (2/x.shape[0]) * torch.sum((y - self.w*x - self.bias)*-1)

                self.w -= lr*grad_w
                self.bias -= lr*grad_b

model1 = LinearRegression1(1, 1)
model2 = LinearRegression2(1, 1)
model3 = LinearRegression3(1, 1)

model1.train(data, 50, torch.optim.SGD(model1.parameters(), lr = 0.01), torch.nn.MSELoss())
model2.train(data, 50, 0.01)
model3.train(data, 50, 0.01)

models = [(model1.linear.weight.item(), model1.linear.bias.item()),
          (model2.w.item(), model2.bias.item()),
          (model3.w.item(), model3.bias.item())]

x = dataset.x.numpy()
fig, axs = plt.subplots(3)
for i in range(3):
        w, b = models[i]
        axs[i].plot(x, dataset.y.numpy(), 'rx', label = 'Data points')
        axs[i].plot(x, dataset.f.numpy(), label = 'Original function (y = 2x - 1)' ,alpha = 0.8)
        axs[i].plot(x, [w*j + b for j in x],
                    label = f"Learned function (y = {round(w, 4)}x + {round(b, 4)})")
        axs[i].legend()
for ax in axs.flat:
    ax.set(xlabel='x', ylabel='y')
    ax.label_outer()
plt.show()


import paho.mqtt.client as mqtt, time, json
from Tubbercar.MovingS import MovingService
from ..Controller.SensorController import SensorController

class MqttService:
    client = mqtt.Client()

    broker_address = "10.0.0.113"

    sensors = SensorController()
    drive = MovingService()

    domoticzTopic = "domoticz/in"
    sensorUltraTopic = "home/vehicle/tubbercar/sensor"

    def __init__(self):
        print("MQTT initializing")

    def connectandsubscribe(self, subscribeTopic, vehicle):
        try:
            self.client.connect(self.broker_address)
            self.client.subscribe(subscribeTopic)

            if (vehicle == "TubberCar"):
                self.client.on_message = self.on_message_tc
            elif (vehicle == "Hexapod"):
                self.client.on_message = self.on_message_hp
            time.sleep(2)
            print("MQTT is ready.")
            self.client.loop_forever()
        except:
            print("ERROR! Cannot connect to the Mqtt broker! ")

    def sendmqttmsg(self, topic, payload):
        self.client.connect(self.broker_address)
        self.client.publish(topic, json.dumps(payload))

    def on_message_tc(self, mosq, obj, msg):
        try:
            print(msg.topic)
            print(str(msg.payload))

            data = json.loads(msg.payload.decode())
            value = str(data["Value"])

            if (value == "Up"):
                self.drive.forward()
            elif (value == "Down"):
                self.drive.backward()
            elif (value == "Left"):
                self.drive.turn_left()
            elif (value == "Right"):
                self.drive.turn_right()
            elif (str(data["Value"]) == 'MeasureUltra'):
                measuring = self.sensors.measureultra()
                self.sendmqttmsg(self.domoticzTopic, measuring)
                self.sendmqttmsg(self.sensorUltraTopic, measuring)
        except:
            print("ERROR! Cannot handle mqtt msg...")

    def on_message_hp(self, mosq, obj, msg):
        print(msg.topic)
        print(str(msg.payload))

        data = json.loads(msg.payload.decode())
        value = str(data["Value"])


# -*- coding: utf-8 -*-
from dipy.io.stateful_tractogram import Space, Origin
import numpy as np

from scilpy.tracking.seed import SeedGenerator

def test_seed_generation():

    mask = np.zeros((5, 5, 5))

    # Two seeds:
    mask[1, 1, 1] = 1
    mask[4, 3, 2] = 1

    # With n_repeats: it is our role to get seeds sequentially.
    generator = SeedGenerator(mask, voxres=[1, 1, 1], space=Space('vox'),
                              origin=Origin('corner'), n_repeats=3)

    rng_generator, shuffled_indices = generator.init_generator(
        rng_seed=1, numbers_to_skip=0)

    # Generating 3 seeds, should be the same
    seed0 = generator.get_next_pos(rng_generator, shuffled_indices, 0)
    seed1 = generator.get_next_pos(rng_generator, shuffled_indices, 1)
    seed2 = generator.get_next_pos(rng_generator, shuffled_indices, 2)

    assert np.array_equal(seed0, seed1)
    assert np.array_equal(seed0, seed2)
    assert np.array_equal(np.floor(seed0), [1, 1, 1])

    # Generating more seeds, should be the same, in the other voxel
    seed3 = generator.get_next_pos(rng_generator, shuffled_indices, 3)
    seed4 = generator.get_next_pos(rng_generator, shuffled_indices, 4)
    _ = generator.get_next_pos(rng_generator, shuffled_indices, 5)
    assert np.array_equal(seed3, seed4)
    assert np.array_equal(np.floor(seed3), [4, 3, 2])

    # Generating n=4 seeds at once, back to voxel 1
    seeds = generator.get_next_n_pos(rng_generator, shuffled_indices, 6, 4)
    assert np.array_equal(seeds[0], seeds[1])
    assert np.array_equal(seeds[0], seeds[2])
    assert np.array_equal(np.floor(seeds[0]), [1, 1, 1])
    assert np.array_equal(np.floor(seeds[3]), [4, 3, 2])


import pandas as pd
import seaborn as sn
import scipy
import numpy as np
import plotly.express as px
from statsmodels.distributions.empirical_distribution import ECDF
import streamlit as st

@st.cache
def cargamos_base_datos():

    esi = pd.read_csv("/Users/pipe/Documents/repos/esi2019-modelos/datos/esi-2019.csv",
                      sep=';', encoding='latin')
    region_comuna = pd.read_csv("/Users/pipe/Documents/repos/esi2019-modelos/datos/region_comuna.csv",
                                sep=';')

    esi = pd.merge(esi, region_comuna, left_on='r_p_c', right_on='Código Comuna 2018', how='left')
    esi.drop(['region'], axis=1, inplace=True)

    columns_map = {
    'id_identificacion': 'id_hogar',
    'idrph': 'id_persona',
    'tipo': 'tipo_estrato',
    't_muestra': 'tipo_muestra',
    'd1_monto': 'sueldo_neto',
    'Nombre Región': 'region',
    'Nombre Provincia': 'provincia',
    'Nombre Comuna': 'comuna',
    'b1': 'grupo_ocupacion',
    'cise': 'categoria_empleo',
    'b8': 'tiene_contrato',
    'b12': 'est_subcontratado',
    'b13_rev4cl_cae': 'rubro',
    'c1': 'tipo_jornada',
    }

    esi = esi.rename(columns=columns_map)

    values_map = {
    'parentesco': {
        1: 'Jefe de hogar',
        2: 'Cónyuge',
        3: 'Conviviente',
        4: 'Hijo(a) / Hijastro(a)',
        5: 'Yerno / Nuera',
        6: 'Nieto(a)',
        7: 'Hermano(a) / Cuñado(a)',
        8: 'Padres / Suegros',
        9: 'Otro pariente',
        10: 'No pariente',
        11: 'Servicio doméstico',
        12: 'Fuera del hogar',
        13: 'Fallecido',
    }
    ,

    'nivel': {
        0: 'Nunca estudió',
        1: 'Sala cuna',
        2: 'Kínder',
        3: 'Básica o primaria 4 Media común',
        5: 'Media técnico profesional 6 Humanidades',
        7: 'Centro formación técnica 8 Instituto profesional',
        9: 'Universitario 10 Postítulo',
        11: 'Magíster',
        12: 'Doctorado',
        14: 'Normalista',
        99: 'Nivel ignorado',
        999: 'Nivel ignorado',
    }
    ,

    'tipo_estrato': {
        1: 'Ciudad',
        2: 'Resto area urbana',
        3: 'Rural'
    }
    ,

    'sexo': {
        1: 'Hombre',
        2: 'Mujer'
    }
    ,

    'termino_nivel': {
        1: "Si",
        2: "No"
    }
    ,

    'cine': {
        1: "Nunca estudió",
        2: "Educación preescolar",
        3: 'Educación primaria (nivel 1)',
        4: 'Educación primaria (nivel 2)',
        5: 'Educación secundaria',
        6: 'Educación técnica',
        7: 'Educación universitaria',
        8: 'Postítulo y maestría',
        9: 'Doctorado',
        999: None
    }
    ,

    'est_conyugal': {
        0: None,
        1: "Casade",
        2: "Coviviente",
        3: "Soltere",
        4: "Viude",
        5: "Separade",
        6: "Divorciade"
    }
    ,

    #'nacionalidad': {}

    }

    for variable, mapping in values_map.items():
        esi[variable] = esi[variable].map(mapping)

    def procesar_monto(x):
        try:
            if ',' in x:
                x = x.replace(',', '.')
            return float(x)
        except:
            return x

    esi['sueldo_neto'] = esi['sueldo_neto'].apply(procesar_monto)
    return esi


# Run this app with `python app.py` and
# visit http://127.0.0.1:8050/ in your web browser.
import dash_bootstrap_components as dbc
import dash_daq as daq
import dash
import dash_core_components as dcc
import dash_html_components as html
import plotly.express as px
import pandas as pd
from dash.dependencies import State
from dash_bootstrap_components import Input
from dash_html_components import Output

from data_preparation import generate_figures

FONT_AWESOME = "https://use.fontawesome.com/releases/v5.10.2/css/all.css"

app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP, FONT_AWESOME])
# app = dash.Dash(__name__, external_stylesheets=["./assets/custom.css"])

colors = {
    'background': '#111111',
    'text': '#ffffff'
}

time_fig = generate_figures(colors)

card_group = dbc.CardGroup([
    dbc.Card(
        dbc.CardBody([
                html.H5("Defected images 24h", className="card-title"),
                html.P("12%", className="card-text"),
            ])
    ),
    dbc.Card(
        dbc.CardBody([
                html.H5("7 day rolling average", className="card-title"),
                html.P("5%", className="card-text"),
        ])
    )],className="mt-4 shadow",
)

app.layout = html.Div(style={'backgroundColor': colors['background']}, children=[
    dbc.NavbarSimple(
        brand="United Jackets Dashboard",
        brand_href="#",
        color="primary",
        dark=True,
        style={'margin-bottom': '25px', 'text-transform': "uppercase"},
        className="g-0"
    ),
    dbc.Col([
        #   "rec_garching00048.png"
        html.Img(src=app.get_asset_url("rec_test.jfif"),
                 style={'width': '500px', 'display': 'inline-block', 'vertical-align': 'top'}),

    ], style={'width': '39%', 'display': 'inline-block', 'vertical-align': 'top'}),
    dbc.Col(
        [
            # dbc.Row(
            #     html.Div(children='Dash: A web application framework for your data.', style={
            #         'textAlign': 'center',
            #         'color': colors['text']
            #     })
            # ),
            dbc.Row(
                dcc.Graph(
                    id='example-graph-2',
                    figure=time_fig,
                    animate=True
                )
            ),
            dbc.Col([card_group]),
        ], style={'width': '59%', 'display': 'inline-block', 'vertical-align': 'top'}
    )
])

if __name__ == '__main__':
    app.run_server(debug=True)


from multiprocessing import Pool
def Start(tasks):
    print(f'Tasks Running : {tasks}')

#main methods called here
if __name__ == '__main__':
    tasks =['Task1','Task2','Task3']

    with Pool(len(tasks)) as pool:
        pool.map(Start,tasks)


import os
import numpy as np
import torch
from sklearn.cluster import KMeans
from collections import Counter
from matplotlib import pyplot as plt

from scripts.helper import write_to_json
from extract_rank_subscenes import map_file_names_to_idx

def calculate_WSS(points, kmax):
    sse = []
    for k in range(1, kmax + 1):
        kmeans = KMeans(n_clusters=k).fit(points)
        centroids = kmeans.cluster_centers_
        pred_clusters = kmeans.predict(points)
        curr_sse = 0

        # calculate square of Euclidean distance of each point from its cluster center and add to current WSS
        for i in range(len(points)):
            curr_center = centroids[pred_clusters[i]]
            curr_sse += (points[i, 0] - curr_center[0]) ** 2 + (points[i, 1] - curr_center[1]) ** 2

        sse.append(curr_sse)
    return sse

def kmeans_embeddings(features, file_names_to_idx, k_max):
    # find the within cluster sum of squared errors to choose k
    sse = calculate_WSS(features, k_max)
    plt.plot(range(1, len(sse) + 1), sse)
    plt.show()

    # determine the number of clusters using the elbow method.
    diffs = []
    # compute first derivative
    for i in range(1, len(sse)):
        diffs.append(abs(sse[i] - sse[i-1]))
    # compute second derivative
    k = 2
    for i in range(1, len(diffs)):
        if abs(diffs[i] - diffs[i-1]) / diffs[i-1] > 0.8:
            k = i + 3
            break

    # cluster using kmeans
    kmeans = KMeans(n_clusters=k).fit(features)
    pred_clusters = kmeans.predict(features)

    # record the cluster_id for each file name
    file_name_to_cluster = {}
    for file_name, idx in file_names_to_idx.items():
        file_name_to_cluster[file_name] = int(pred_clusters[idx])

    return file_name_to_cluster

def adjust_paths(args, exceptions):
    base_dir = os.path.dirname(os.path.abspath(__file__))
    for k, v in vars(args).items():
        if (type(v) is str) and ('/' in v) and k not in exceptions:
            v = v.format(args.dataset)
            vars(args)[k] = os.path.join(base_dir, v)

def prepare_embedding_features(args):
    # load the features and file names.
    features_dir = os.path.join(args.cp_dir, args.results_folder_name, args.features_dir_name_thresholding)
    features = torch.load(os.path.join(features_dir, "trainfeat.pth"))
    print('Loaded features of shape {}'.format(features.shape))

    # map the file names to feature indices.
    file_indices = torch.load(os.path.join(features_dir, "test_file_names.pth"))
    file_name_to_idx = map_file_names_to_idx(args, file_indices, args.metadata_path_queries)

    return features, file_name_to_idx

def cluster(args):
    # load features and file_name mappings.
    features, file_name_to_idx = prepare_embedding_features(args)

    # find the thresholds by clustering the similarity scores.
    file_name_to_cluster = kmeans_embeddings(features, file_name_to_idx, args.k_max)

    # save the clustering.
    write_to_json(file_name_to_cluster, os.path.join(args.cp_dir, args.results_folder_name,
                                                     args.predicted_clusters_file_name))

def find_sim_threshold(args):
    # load features and file_name mappings.
    features, file_name_to_idx = prepare_embedding_features(args)

    # sample from the features; Otherwise, it Kmeans will crash.
    sample_size = len(features)
    if sample_size > 13000:
        sample_size = 13000
    random_indices = np.random.choice(range(len(features)), sample_size, replace=False)
    features = features[random_indices, ...]

    # compute similarities between all train features
    sims = torch.matmul(features, features.T)
    sims = sims.view(-1, 1)

    # cluster the similarities into two clusters
    kmeans = KMeans(n_clusters=2).fit(sims)
    centroids = kmeans.cluster_centers_

    # find the thresholds and sort them to send the highest one
    thresholds = [centroids[0, 0], centroids[1, 0]]

    return sorted(thresholds)[1]


from PyQt5.QtWidgets import QWidget, QTableWidgetItem as QItem, QHeaderView, \
    QMessageBox, QTableWidgetSelectionRange
from PyQt5.QtGui import QIcon
from forms.table_checkbox import TableCheckbox
from PyQt5 import uic
from models.edge import Rib
from settings import ARE_YOU_SURE, NOT_NUMBER, EMPTY, NOT_DIF_NODES, \
    NOT_DIF_EDGES
from functions import get_graph_by_name, str_is_float
from models import db_session
from typing import Union

class EdgeList(QWidget):
    """Класс для окна со списком ребер"""
    def __init__(self, graph_name: str, parent=None) -> None:
        super().__init__()
        uic.loadUi("UI/edge_list.ui", self)
        self.parent = parent
        self.graph_name = graph_name
        self.modified = {}
        self.new_ribs = 0
        self.initUI()
        self.loadTable()

    def initUI(self) -> None:
        """Метод для установки UI и привязки событий"""
        self.setWindowIcon(QIcon('images/icon2.png'))
        self.setLayout(self.vl)
        self.addEdge.clicked.connect(self.addRow)
        self.deleteEdge.clicked.connect(self.deleteRow)
        self.saveChanges.clicked.connect(self.save)
        header = self.table.horizontalHeader()
        for i in range(self.table.columnCount()):
            header.setSectionResizeMode(i, QHeaderView.Stretch)
        self.table.itemChanged.connect(self.changeItem)
        self.table.selectionModel().selectionChanged.connect(
            self.checkUnselected)

    def changeGraph(self, graph_name):
        """Меняет граф"""
        self.graph_name = graph_name
        self.loadTable()

    def checkUnselected(self, _, unselected) -> bool:
        """Обработчик валидности невыделенных ячеек"""
        if len(unselected.indexes()) != 1:
            return True
        last_cell = unselected.indexes()[0]
        return self.validCell(last_cell.row(), last_cell.column())

    def validCell(self, row: int, col: int) -> bool:
        """Проверка валидности значения ячейки таблицы"""
        if col != self.get_last_col() and not self.validEmpty(row, col):
            return False
        if col == self.get_last_col() - 1:
            return self.validNumber(row, col)
        return self.validDifNodes(row, col)

    def validEmpty(self, row: int, col: int) -> bool:
        """Проверка наличия значения в ячейке"""
        try:
            if row > self.get_last_row():
                return True
            if not self.table.item(row, col).text():
                raise AttributeError
            return True
        except AttributeError:
            QMessageBox.warning(self, "Error", EMPTY)
            return False

    def validNumber(self, row: int, col: int) -> bool:
        """Проверка числового значения в ячейке"""
        if str_is_float(self.get_item(row, col)):
            return True
        QMessageBox.critical(self, "Error", NOT_NUMBER)
        return False

    def validDifNodes(self, row: int, col: int) -> bool:
        """Проверка на отсутствие петли (ребро между одинаковыми вершинами"""
        item1 = self.get_item(row, col)
        item2 = self.get_item(row, (col + 1) % 2)
        if item1 == item2 and item1 is not None and item2 is not None:
            QMessageBox.critical(self, 'Error', NOT_DIF_NODES)
            return False
        return True

    def validDifEdges(self) -> bool:
        """Проверка отсутствия одинааковых ребер в списке"""
        dif = {f'{self.get_item(i, 0)}{self.get_item(i, 1)}' for i in range(
            self.table.rowCount())}
        if len(dif) != self.table.rowCount():
            QMessageBox.critical(self, "Error", NOT_DIF_EDGES)
            return False
        return True

    def changeItem(self, item) -> None:
        """Метод для сохранения изменений в таблице"""
        if str_is_float(item.text()) and item.column() == 2:
            self.modified[item.row(), item.column()] = float(item.text())
        else:
            self.modified[item.row(), item.column()] = item.text()

    def changeCheckbox(self) -> None:
        """Метод для сохранения изменения состояния флажков"""
        sender = self.sender()
        self.modified[sender.index, self.get_last_col()] = sender.isChecked()

    def loadTable(self) -> None:
        """Метод для загрузки данных в таблицу"""
        session = db_session.create_session()
        graph = get_graph_by_name(session, self.graph_name)
        self.table.setRowCount(len(graph.ribs))
        for i, rib in enumerate(graph.get_ordered_ribs()):
            self.ribPresentation(i, rib)
        self.table.resizeRowsToContents()
        self.modified = {}
        session.close()

    def ribPresentation(self, row: int, rib: Rib) -> None:
        """"Метод для занесения ребра в таблицу"""
        self.table.setItem(row, 0, QItem(str(rib.nodes[0])))
        self.table.setItem(row, 1, QItem(str(rib.nodes[1])))
        self.table.setItem(row, 2, QItem(str(rib.weight)))
        item = self.get_table_checkbox(row, rib.is_directed)
        self.table.setCellWidget(row, 3, item)

    def addRow(self) -> None:
        """Метод для добавления ребра в таблицу"""
        if not self.checkComplete():
            return
        last_row, last_col = self.get_last_indexes()
        last_row += 1
        self.table.insertRow(last_row)
        self.table.setRangeSelected(QTableWidgetSelectionRange(
            last_row, 0, last_row, last_col), True
        )
        item = self.get_table_checkbox(last_row, False)
        self.table.setCellWidget(last_row, last_col, item)
        self.new_ribs += 1
        [self.modified.update({(last_row, i): ''}) for i in range(3)]

    def deleteRow(self) -> None:
        """Метод для удаления ребра из таблицы"""
        idx = sorted({i.row() for i in self.table.selectedIndexes()},
                     reverse=True)
        if not idx:
            return
        flag = QMessageBox.question(
            self, "Delete ribs",
            f"{ARE_YOU_SURE} selected ribs"
        )
        if flag == QMessageBox.No:
            return
        [self.table.removeRow(i) for i in idx]
        self.shiftModifies(idx)
        self.deleteEdges(idx)

    def deleteEdges(self, idx) -> None:
        """Удаляет выделенные в списке ребра из графа, если
        они в нем существуют"""
        session = db_session.create_session()
        g_ribs = get_graph_by_name(session, self.graph_name).get_ordered_ribs()
        ribs_to_del = []
        for i in idx:
            if i < len(g_ribs):
                ribs_to_del.append(g_ribs[i])
            else:
                self.new_ribs -= 1
        [session.delete(rib) for rib in ribs_to_del]
        session.commit()
        session.close()

    def save(self) -> None:
        """Метод сохранения изменений в БД"""
        if not self.checkComplete():
            return
        # print(self.modified)
        session = db_session.create_session()
        graph = get_graph_by_name(session, self.graph_name)
        ribs = graph.get_ordered_ribs()
        new_ribs = [Rib() for _ in range(self.new_ribs)]
        session.add_all(new_ribs)
        session.commit()
        graph.add_ribs(*new_ribs)
        ribs.extend(new_ribs)
        [ribs[i].change_attrs(j, self.modified[i, j]) for i, j in self.modified]
        session.commit()
        session.close()
        self.modified = {}
        self.new_ribs = 0
        self.parent.definer.define_all()
        self.parent.showTreeOfElements()
        self.parent.canvas.loadGraph(self.graph_name)
        self.parent.canvas.repaint()

    def checkComplete(self) -> bool:
        """Метод проверки корректности измененных данных"""
        if self.get_last_row() < 0:
            return True
        return all(self.validCell(i, j) for i, j in self.modified) and \
            self.validDifEdges()

    def get_last_row(self):
        """Метод, возвращающий индекс последней строки таблицы"""
        return self.table.rowCount() - 1

    def get_last_col(self):
        """Метод, возвращающий индекс последнего столбца таблицы"""
        return self.table.columnCount() - 1

    def get_last_indexes(self):
        """Метод, возвращающий индекс правого нижнего элемента таблицы"""
        return self.get_last_row(), self.get_last_col()

    def get_table_checkbox(self, row: int, value: bool) -> TableCheckbox:
        """Метод, возвращающий флажок для ячейки таблицы"""
        item = TableCheckbox(row)
        item.setState(value)
        item.checkbox.clicked.connect(self.changeCheckbox)
        return item

    def get_item(self, row: int, col: int) -> Union[str, None]:
        """Возвращает значение из таблицы"""
        item = self.table.item(row, col)
        if item is None:
            return None
        return item.text()

    def shiftModifies(self, idx) -> None:
        """Сдвигает строки в измененных данных при удалении строк в таблице"""
        [self.modified.pop((i, j), None) for i in idx for j in
         range(self.table.columnCount())]
        new_mod = {}
        for i, j in self.modified:
            new_mod[i - self.calcShift(i, idx), j] = self.modified[i, j]
        self.modified = new_mod

    def calcShift(self, row: int, idx: list[int]) -> int:
        """Считает смещение для строки таблицы"""
        return sum([1 for i in idx if i < row])

    def setModified(self, val_dict: dict) -> None:
        """Устновка нового словаря изменений"""
        self.modified = val_dict
        self.new_ribs = len(val_dict) // self.table.columnCount()


"""
Input: An ontology file in xml or ttl format.
Output: The concept list of the ontology

A concept of an ontology consist two part: 1. prefix; 2. name
For easy usage for later steps, every concept is split to prefix and name.
Two types of concept lists are output: 1. only contain the name; 2. contain both the prefix and the name (the original format)
A couple of function are provided to parse the ontology.
The main issue is each ontology may have its own splits and delimiters.
The user may need to develop customized functions to parse specific ontology.
"""
import rdflib

base_query = '''
select ?s where {?s a owl:Class.}
'''

def load_ontology(file, format='xml'):
    g = rdflib.Graph()
    if type(file) is type(list()):
        for f in file:
            g.parse(f, format=format)
    else:
        g.parse(file, format=format)
    return g

def camel_case_split(str):
    words = [[str[0]]]

    for c in str[1:]:
        if words[-1][-1].islower() and c.isupper():
            words.append(list(c))
        else:
            words[-1].append(c)

    return [''.join(word) for word in words]

def extract_concept_list(g: rdflib.Graph, split='camel', delimiter='#'):
    extract_query = '''select ?s where {?s a/rdfs:subClassOf* owl:Class.}'''
    res = g.query(extract_query)
    concept_list = []
    original_list = []
    if split == 'camel':
        for term in res:
            if '#' in term[0]:
                temp = term[0].split(delimiter)[-1]
                concept_list.append(' '.join(camel_case_split(temp)))
                original_list.append(term[0])
    if split == '_':
        for term in res:
            if '#' in term[0]:
                temp = term[0].split(delimiter)[-1].split('_')
                if 'Setpoint' in temp:
                    temp.remove('Setpoint')
                    temp.extend(['Set', 'Point'])
                concept_list.append(' '.join(temp))
                original_list.append(term[0])
    return concept_list, original_list


import csv
from typing import Sequence

class Logger:
    def __init__(self, filename: str, params: Sequence[str]):
        """
        :param filename: The name of the log file to save
        :param params: A list of first row titles"""
        self.file = open(filename, 'w')
        self.writer = csv.writer(self.file, dialect='excel')
        self.params = params
        self.writer.writerow(params)

    def __del__(self):
        self.file.close()

    def writeline(self, arguments: Sequence):
        """Enter a new line of elements to the log.

        Must have the same number of elements
        as this element was initialized with.
        """
        if len(arguments) == len(self.params):
            self.writer.writerow(arguments)
        else:
            print("Exception: improper number of arguments!")


import itertools
import time
import os.path
import re
import sys
import traceback

try:
    from StringIO import StringIO
except (ImportError, ModuleNotFoundError):
    from io import StringIO

try:
    from PySide2 import QtWidgets, QtGui, QtCore
except ImportError:
    from PySide import QtGui, QtCore
    QtWidgets = QtGui

class NumberBar(QtWidgets.QWidget):
    def __init__(self, editor):
        super(NumberBar, self).__init__(editor)
        self.editor = editor
        self.setStyleSheet("text-align: center;")

    def sizeHint(self):
        return QtCore.QSize(self.editor.lineNumberAreaWidth(), 0)

    def paintEvent(self, event):
        self.editor.lineNumberAreaPaintEvent(event)

class CodeEditor(QtWidgets.QPlainTextEdit):

    clearOutputRequested = QtCore.Signal()
    runCodeRequested = QtCore.Signal()

    codeSaveName = 'customAction.py'
    escapeCharacters = [b'\n', b'\t', b'\r', b'\b', b'\f', b'\'', b'\"', b'\\', b'\v']

    def __init__(self, **kwargs):
        super(CodeEditor, self).__init__()

        self.saveDir = kwargs.get('saveDir', None)
        self.saveLog = kwargs.get('saveLog', False)
        self.saveCode = kwargs.get('saveCode', False)
        self.defaultCode = kwargs.get('defaultCode', None)

        self.lineNumberArea = NumberBar(self)
        self.updateRequest.connect(self.updateLineNumberArea)
        self.updateLineNumberAreaWidth()
        self.highlightCurrentLine()

        self.blockCountChanged.connect(self.updateLineNumberAreaWidth)
        self.cursorPositionChanged.connect(self.highlightCurrentLine)

        self.initialized = False

    @property
    def logSaveName(self):
        """
        Returns:
            str: Dated and times name for saving out the log of the run code
                 ie: log/%m.%d.%Y-%H.%M.%S.log
        """
        return time.strftime('log/%m.%d.%Y-%H.%M.%S.log')

    def lineNumberAreaPaintEvent(self, event):

        if self.isReadOnly():
            return

        painter = QtGui.QPainter(self.lineNumberArea)
        painter.fillRect(event.rect(), QtGui.QColor(36, 36, 36))

        block = self.firstVisibleBlock()
        blockNumber = block.blockNumber()
        top = int(self.blockBoundingGeometry(block).translated(self.contentOffset()).top())
        bottom = top + int(self.blockBoundingRect(block).height())

        currentLine = self.document().findBlock(self.textCursor().position()).blockNumber()
        painter.setPen(self.palette().color(QtGui.QPalette.Text))

        painterFont = QtGui.QFont()
        painterFont.setStyleHint(QtGui.QFont.Monospace)
        painterFont.setFixedPitch(True)

        while block.isValid() and top <= event.rect().bottom():

            textColour = QtGui.QColor(110, 110, 110)
            if blockNumber == currentLine and self.hasFocus():
                textColour = QtGui.QColor(255, 170, 0)

            painter.setPen(textColour)

            number = str(blockNumber + 1)
            painter.drawText(-3, top, self.lineNumberArea.width(),
                             self.fontMetrics().height(),
                             QtCore.Qt.AlignRight, number)

            block = block.next()
            top = bottom
            bottom = top + int(self.blockBoundingRect(block).height())
            blockNumber += 1

    def lineNumberAreaWidth(self):
        """
        Determines the width to use for the line numbers sidebar
        Returns:
            int: width of the line number sidebar
        """
        digits = 1
        maxNum = max(1, self.blockCount())
        while maxNum >= 10:
            maxNum /= 10
            digits += 1

        space = 3 + self.fontMetrics().width('9') * digits
        return space

    def resizeEvent(self, event):
        """
        Update the line number area so that it matches the size of the editor
        """
        super(CodeEditor, self).resizeEvent(event)
        cr = self.contentsRect()
        self.lineNumberArea.setGeometry(
            QtCore.QRect(cr.left(), cr.top(), self.lineNumberAreaWidth(), cr.height()))

    def updateLineNumberAreaWidth(self, *args):
        self.setViewportMargins(self.lineNumberAreaWidth(), 0, 0, 0)

    def highlightCurrentLine(self):
        """
        Highlights the current line under the cursor
        """
        extraSelections = []
        if not self.isReadOnly():
            selection = QtWidgets.QTextEdit.ExtraSelection()
            lineColor = QtGui.QColor(QtCore.Qt.gray)
            lineColor.setAlpha(50)
            selection.format.setBackground(lineColor)
            selection.format.setProperty(QtGui.QTextFormat.FullWidthSelection, True)
            selection.cursor = self.textCursor()
            selection.cursor.clearSelection()
            extraSelections.append(selection)
        self.setExtraSelections(extraSelections)

    def highlightErrorLine(self, line):
        """
        Highlights the given line number in red
        Args:
            line (int): Line number that is to be highlighted
        """
        cursor = QtGui.QTextCursor(self.document().findBlockByLineNumber(line))
        self.setTextCursor(cursor)
        extraSelections = []
        if not self.isReadOnly():
            selection = QtWidgets.QTextEdit.ExtraSelection()
            lineColor = QtGui.QColor(QtCore.Qt.red)
            lineColor.setAlpha(50)
            selection.format.setBackground(lineColor)
            selection.format.setProperty(QtGui.QTextFormat.FullWidthSelection, True)
            selection.cursor = self.textCursor()
            selection.cursor.clearSelection()
            extraSelections.append(selection)
        self.setExtraSelections(extraSelections)

    def updateLineNumberArea(self, rect, dy):
        if dy:
            self.lineNumberArea.scroll(0, dy)
        else:
            self.lineNumberArea.update(0, rect.y(), self.lineNumberArea.width(), rect.height())
        if rect.contains(self.viewport().rect()):
            self.updateLineNumberAreaWidth(0)

    def selectedText(self):
        """
        Returns:
            str: The text currently selected in the editor
        """
        selectedText = self.textCursor().selectedText()
        if not selectedText:
            return ''
        lines = selectedText.splitlines()
        text = '\n'.join(lines)

        # Fix syntax error if last line is a comment with no new line
        if not text.endswith('\n'):
            text = text + '\n'

        return text

    def getSelectedLineNumber(self):
        """
        If there is selected text this will collect and return the line number for the first line
        of the selected text.  If there is no selected text then 0 will be returned
        Returns:
            int:  The first line for the currently selected text or 0
        """
        if not self.selectedText():
            return 0
        preBlock, _ = self.toPlainText().split(self.selectedText(), 1)
        return len(preBlock.split('\n')) - 1

    def runCode(self, selected=False, globalAttrs=None):
        """
        Runs the code in the current editor,  This will run it in a try/except and save out the
        output if set to do so.

        Either the error log or the output will be returned.  In the event of an error that line
        will be highlighted in the editor
        Args:
            selected (bool|optional): True or False if only the selected code should be executed
            globalAttrs (dict|optional): Dictionary of globals to have included in the environment
                                         when the code is executed

        Returns:
            str: The output of the code execution
        """
        globalAttrs = globalAttrs or globals()
        if selected:
            text = self.selectedText() or self.toPlainText()
            lineNumber = self.getSelectedLineNumber()
        else:
            text = self.toPlainText()
            lineNumber = 0

        print('line:', lineNumber)
        error = False
        currentOutput = sys.stdout

        try:
            executionMode = 'exec'
            if text.count('\n') == 1:
                executionMode = 'single'

            compiled = compile(text, 'CustomAction', executionMode)

            stdout = StringIO()
            sys.stdout = stdout
            exec(compiled, globalAttrs)
            output = 'Result:\n{0}'.format(stdout.getvalue())

        except Exception:
            error = True
            errors = traceback.format_exc().split('\n')
            del errors[1:3]

            formattedLines = list()
            foundLine = False
            for line in reversed(errors):
                if not foundLine:
                    search = re.search('File "CustomAction", line (\d+)', line)
                    if search:

                        lineNumber += int(search.groups()[0]) - 1
                        line = re.sub('File "CustomAction", line (\d+)',
                                      'File "CustomAction", line {0}'.format(lineNumber + 1),
                                      line)
                formattedLines.append(line)

            output = '\n'.join(reversed(formattedLines))

        finally:
            sys.stdout = currentOutput

        if self.saveLog:
            self.saveData(self.logSaveName, output)

        if error and isinstance(lineNumber, int):
            self.highlightErrorLine(lineNumber)

        return output

    def saveData(self, basename, data):
        """
        Saves out the given data to the specified save dir for this instance.  If no save dir has
        been set then nothing is saved
        Args:
            basename (str): basename for the file to save the data to
            data (str): Data to be saved out
        """
        if not self.saveDir:
            return

        savePath = os.path.join(self.saveDir, basename).replace('\\', '/')

        if not os.path.exists(os.path.dirname(savePath)):
            os.makedirs(os.path.dirname(savePath))

        with open(savePath, 'w') as saveFile:
            saveFile.write(data)

    def loadData(self, basename):
        """
        Loads data for the given basename.  This will collect this from the save dir specified for
        the current instance
        Args:
            basename (str): basename for the file to load the data from

        Returns:
            str: Data that has been loaded
        """
        loadPath = os.path.join(self.saveDir, basename).replace('\\', '/')
        if not os.path.exists(loadPath):
            return

        with open(loadPath, 'r') as saveFile:
            data = saveFile.read()

        if data and isinstance(data, str):
            self.setPlainText(data)

        return data

    def focusOutEvent(self, event):
        """
        Triggers the save of the current text in the editor
        """
        if self.saveCode:
            self.saveData(self.codeSaveName, self.toPlainText())
        super(CodeEditor, self).focusOutEvent(event)

    def keyPressEvent(self, event):
        """
        Add in some keyboard shortcuts for the editor

        Ctrl+Backspace: emits signal to clear output
        Ctrl+Enter: Emits signal to run code
        Ctrl+L: Open the log dir
        """
        ctrlToggled = event.modifiers() == QtCore.Qt.ControlModifier
        # altToggled = event.modifiers() == QtCore.Qt.AltModifier
        # shiftToggled = event.modifiers() == QtCore.Qt.ShiftModifier
        key = event.key()

        if ctrlToggled:
            if key in [QtCore.Qt.Key_Backspace]:
                self.clearOutputRequested.emit()
                return
            elif key in [QtCore.Qt.Key_Enter, QtCore.Qt.Key_Return]:
                self.runCodeRequested.emit()

            elif key in [QtCore.Qt.Key_L]:
                self.openLogDir()

        super(CodeEditor, self).keyPressEvent(event)

    def setToolTip(self, toolTip):
        """
        Override for setTooltip to ensure that we add in our shortcuts to the tooltip no matter what
        the user has set it to
        Args:
            toolTip (str): Tooltip to set
        """
        toolTip += ('\n\nShortcuts:\n'
                    'Ctrl+Backspace: emits signal to clear output\n'
                    'Ctrl+Enter: Emits signal to run code\n'
                    'Ctrl+L: Open the log dir')
        super(CodeEditor, self).setToolTip(toolTip)

    def openLogDir(self):
        """
        Opens the log dir for the current instance
        """
        logPath = os.path.dirname(os.path.join(self.saveDir, self.logSaveName)).replace('\\', '/')
        os.startfile(logPath, 'open')

    def showEvent(self, event):
        """
        Override to ensure that the initialization is run on startup.  This will also load any
        previous code if there is any present
        """
        if not self.initialized:
            if self.saveDir:
                self.loadData(self.codeSaveName)
            if not self.toPlainText() and self.defaultCode:
                self.setPlainText(self.defaultCode)
            self.initialized = True

        super(CodeEditor, self).showEvent(event)


from typing import Callable, Dict, List, Tuple

import numpy as np
import pandas as pd
import skimage
import torch
from torchvision.io import read_image

from src.image_segmentation_ray.src.image_segmentation.v1.core.postprocessing import (
    post_process_np_array,
)

def predict_masks(
    metadata: pd.DataFrame,
    model: torch.nn.Module,
    threshold: float,
    output_layer_activ_function: Callable,
    post_process_functions: List[Tuple[Callable, Dict]] = None,
    split_col_name: str = None,
    splits_to_predict: List[str] = None,
    use_gpu: bool = False,
) -> bool:
    """Predicts masks and then applies post-processing (if needed).

    The function writes the resulting masks in the locations specified
    in metadata table.

    Args:
        metadata: table containing images and mask paths and potentially
            other relevant information specific to the use case
        model: model to be used
        threshold: lower threshold to trigger pixel classification (between 0 and 1)
        output_layer_activ_function: output layer activation function
        post_process_functions: (Optional) list of tuples including functions
            (and their arguments) to apply to image (in array format). One can
            add pre-built functions (e.g. scipy.morphology.remove_small_objects
            or any other scipy.morphology function) or their own custom functions
        split_col_name: (Optional) split column name. If not provided, a
            prediction will be made for every image listed in the metadata.
        splits_to_predict: (Optional) list of split categories to predict.
            The default is to make a prediction for all images listed in the
            metadata.
        use_gpu: (Optional) boolean to indicate whether to use GPU for inference
            or not. If not provided, CPUs will be used.

    Returns:
        dummy boolean
    """
    if splits_to_predict is not None and split_col_name is not None:
        metadata = metadata[metadata[split_col_name].isin(splits_to_predict)]

    # Switch model to evaluation mode
    model.eval()

    for img_path in metadata["img_path"]:
        # Prediction
        image = read_image(img_path)
        image = image.unsqueeze(0)
        # pylint: disable=no-member
        if torch.cuda.is_available() and use_gpu:
            image = image.type(torch.cuda.FloatTensor)
            model.to("cuda")
            image.to("cuda")
            with torch.no_grad():
                predicted_mask = output_layer_activ_function(model(image))
        else:
            image = image.type(torch.FloatTensor)
            predicted_mask = output_layer_activ_function(model(image))
        predicted_mask = predicted_mask >= threshold
        numpy_predicted_mask = predicted_mask.cpu().detach().numpy()

        # Post-processing
        if post_process_np_array is not None:
            numpy_predicted_mask = post_process_np_array(
                numpy_predicted_mask, post_process_functions,
            )

        # Write
        skimage.io.imsave(
            img_path.rsplit("/", 1)[0] + "/predicted_mask.png",
            skimage.img_as_ubyte(np.squeeze(numpy_predicted_mask)),
        )
    return True