""" Simple demonstration of the new 3D visualization tool for Madagascar.
The program displays x-, y-, z- slices of a numpy array in 3D, allows user
to interactively drag the slices, includes useful features such as colorbar
and axis legend, and can output the figure to .png file with various resolution.

python seamai_data_demo.py --filepath seamai_data/data_train.npz
python seamai_data_demo.py --filepath seamai_data/data_test_1.npz

"""
import argparse

import numpy as np
from vispy.color import get_colormap, Colormap, Color
from vispy.gloo.util import _screenshot

from seismic_canvas import (SeismicCanvas, volume_slices, XYZAxis, Colorbar)
# Create the parser
my_parser = argparse.ArgumentParser(description='file path to viz')

# Add the arguments
my_parser.add_argument('--filepath',
                      # metavar='path',
                       type=str,
                       help='the path to file')

if __name__ == '__main__':

  # Execute the parse_args() method
  args = my_parser.parse_args()
 #volume = np.load('./data_train.npz')["data"]
  volume = np.load(args.filepath)["data"] #
  volume = volume.transpose(1,2,0)
  #print(volume.min(),volume.max())
  print(volume.shape) #(1006, 782, 590) z,x,y ==> (782,590,1006)
  axis_scales = (0.7, 0.5, 1) # anisotropic axes (stretch z-axis)

  # Colormaps.
  cmap='seismic'; clim=(-1500, 1500)   #-5195.5234 5151.7188
  # Get visual nodes ready.
  visual_nodes = volume_slices(volume,
    cmaps=cmap, clims=clim,
    # x_pos=32, y_pos=25, z_pos=93)
    x_pos=volume.shape[0]-1, y_pos=volume.shape[1]-1, z_pos=0) #(782, 590, 1006)
  xyz_axis = XYZAxis()
  colorbar = Colorbar(cmap=cmap, clim=clim, label_str='Data',
                      label_size=8, tick_size=6)

  # Run the canvas.
  canvas = SeismicCanvas(title='Data Interactive Demo',
                         visual_nodes=visual_nodes,
                         xyz_axis=xyz_axis,
                         colorbar=colorbar,
                         # Set the option below=0 will hide the colorbar region
                         # colorbar_region_ratio=0,
                         axis_scales=axis_scales,
                         # Manual camera setting below.
                         # auto_range=False,
                         # scale_factor=972.794,
                         # center=(434.46, 545.63, 10.26),
                         fov=30,
                         elevation=36,
                         azimuth=45,
                         zoom_factor=1.2 # >1: zoom in; <1: zoom out
                         )
  canvas.measure_fps()

  canvas.app.run()


import os
import numpy as np
import pandas as pd
import torch
# requires: pip install PySoundFile to avoid RuntimeErr: No audio I/O backend is available.
# https://stackoverflow.com/questions/62543843/cannot-import-torch-audio-no-audio-backend-is-available
import torchaudio
from torch.utils.data import Dataset, DataLoader
from torchvision.io import read_image
from tqdm import tqdm

torch.set_default_dtype(torch.float32)

def loadCSV(filepath: str) -> tuple[torch.Tensor, torch.Tensor, int]:
    """
    :param str filepath:
    :rtype: tuple[torch.Tensor, torch.Tensor, int]
    """
    features_df = pd.read_csv(filepath)

    # Remove broken jazz.00054.wav
    features_df.drop(features_df[features_df['filename'] == 'jazz.00054.wav'].index, inplace=True)

    # Get the number of items in the features
    length = len(features_df.index)

    # First drop the file name, it is not needed.
    del features_df['filename']

    # Save the names of the categories
    cats = ["blues", "classical", "country", "disco", "hiphop", "jazz", "metal", "pop", "reggae", "rock"]

    # First save the label column to a separate variable, which can then be turned into onehot Tensor.
    labels_df = features_df['label']
    labels = makeOnehotTensorFromDataframe(cats, labels_df)

    # Then drop the label column and turn the features into a Tensor.
    del features_df['label']
    # normalization
    for col in features_df.columns:
        features_df[col] = (features_df[col] - min(features_df[col])) / (max(features_df[col]) - min(features_df[col]))
    features = torch.as_tensor(features_df.values, dtype=torch.float32)
    return features, labels, length

def makeOnehotTensorFromNdarray(cats: list[str], keys: np.ndarray) -> torch.Tensor:
    labels_list = []
    n = len(keys)
    for i in range(0, n):
        key = keys[n]
        result = makeOnehot(cats, key, i)
        labels_list.append(result)
    labels = torch.as_tensor(labels_list, dtype=torch.float32)
    return labels

def makeOnehotTensorFromDataframe(cats: list[str], keys: pd.DataFrame) -> torch.Tensor:
    labels_list = []
    total_cats = len(cats)
    index = 0
    for key in keys:
        result = makeOnehot(cats, total_cats, key, index)
        labels_list.append(result)
        index += 1
    labels = torch.as_tensor(labels_list, dtype=torch.float32)
    return labels

def makeOnehot(cats: list[str], total_cats: int, key: str, index: int) -> list[int]:
    try:
        i = cats.index(key)
    except IndexError as e:
        raise IndexError(f'The genre ({key}) I got is not in the list ({cats}).')
    else:
        result = [0] * total_cats
        result[i] = 1
        return result

def torch_onehot(cats: list[str], key: str) -> torch.Tensor:
    """
    Generate the onehot tensor of given categories and key.
    Example
    -------
        cats: ['apple', ''banana', 'orange']
        key: ['banana']
    Return:
        torch.Tensor([[0., 1., 0.]])
    :param cats: list of categories
    :param key: key
    :return: tensor of onehot representation
    """
    try:
        i = cats.index(key)
    except IndexError:
        raise IndexError(f'Error reading genres. The genre ({key}) I got is not in the list ({cats}). ')
    else:
        result = torch.zeros(1, len(cats))
        result[0, i] = 1
        return result

def model_test(net, test_set):
    net.eval()
    with torch.no_grad():
        correct_count = 0
        count = 0
        x_test, y_test = next(iter(DataLoader(test_set, batch_size=20, shuffle=True)))
        pred_test = net(x_test)

        for i in range(len(pred_test)):
            pred_label = torch.argmax(pred_test[i])
            true_label = torch.argmax(y_test[i])
            if pred_label.item() == true_label.item():
                correct_count += 1
            count += 1
    return correct_count / count

class WavData(Dataset):
    def __init__(self, path: str, device: str = 'cpu'):
        """
        Assume directory structure be like:
        data
        |- genre_1
            |- genre_1.00000.wav
            |- genre_2.00000.wav
            ...
        |- genre_2
            |- ...
        |- genre_3
            |- ...
        ...

        Then path would be 'data'
        :param path: Path to the root of all genres
        :param device: Where the data is going to be sent to ('cpu' or 'cuda')
        """
        cats = sorted(os.listdir(path))  # cats = categories

        self.x = []
        self.y = []
        # Assume sample rates of all the files are the same
        _, self.sample_rate = torchaudio.load(open(f'{path}/{os.listdir(path)[0]}/{os.listdir(path + "/" + os.listdir(path)[0])[0]}', 'rb'))
        resampler = torchaudio.transforms.Resample(self.sample_rate, self.sample_rate // 4)
        with tqdm(total=1000) as pbar:
            for genre in os.listdir(path):
                current_onehot = torch_onehot(cats, genre)
                for file in os.listdir(f'{path}/{genre}'):
                    if file == 'jazz.00054.wav':
                        continue
                    current_x, _ = torchaudio.load(open(f'{path}/{genre}/{file}', 'rb'))

                    current_x = resampler(current_x)[:, 0:22050].swapaxes(0, 1)
                    self.x.append(current_x)
                    self.y.append(current_onehot[0])
                    pbar.update(1)

        self.device = device

    def __getitem__(self, item):
        return self.x[item].to(self.device), self.y[item].to(self.device)

    def __len__(self):
        return len(self.x)

class PngData(Dataset):
    def __init__(self, path: str, device: str = 'cpu'):
        """
        Assume directory structure be like:
        data
        |- genre_1
            |- genre_1.00000.png
            |- genre_2.00000.png
            ...
        |- genre_2
            |- ...
        |- genre_3
            |- ...
        ...

        Then path would be 'data'
        :param path: Path to the root of all genres
        :param device: Where the data is going to be sent to ('cpu' or 'cuda')
        """
        cats = sorted(os.listdir(path))

        shape = read_image(f'{path}/{cats[0]}/{os.listdir(f"{path}/{cats[0]}")[0]}')[None, :].shape

        self.x = []
        self.y = torch.zeros((1, len(cats)))
        for genre in os.listdir(path):
            current_onehot = torch_onehot(cats, genre)
            for file in os.listdir(f'{path}/{genre}'):
                if file == 'jazz00054.png':
                    continue
                self.x.append(f'{path}/{genre}/{file}')
                self.y = torch.concat((self.y, current_onehot))

        self.x = self.x[1:]
        self.y = self.y[1:]
        self.device = device

    def __getitem__(self, item):
        x = read_image(self.x[item]).type(torch.FloatTensor).to(self.device)
        y = self.y[item].to(self.device)
        x = x / 255.
        return x, y

    def __len__(self):
        return len(self.x)

class MultiSourceData(Dataset):
    def __init__(self, path: str, device: str = 'cpu'):
        self.png = PngData(f'{path}/images_original', device=device)
        self.wav = WavData(f'{path}/genres_original', device=device)
        self.csv = loadCSV(f'{path}/features_30_sec.csv')

    def __getitem__(self, item):
        png, label = self.png[item]
        wav = self.wav[item][0]
        csv = self.csv[0][item]

        return png, wav, csv, label

    def __len__(self):
        return len(self.png)


import fastentrypoints
from setuptools import setup, find_packages
from setuptools.command.install import install

install_requirements = ['psutil']

version = '0.8.0'

try:
    import importlib
except ImportError:
    install_requirements.append('importlib')

try:
    import PyQt5
except ImportError:
    raise RuntimeError("Please install PyQt5 via your system package manager")

class InstallSystemdServiceFileInstall(install):
    """
    Installs systemd service user file during install.
    """

    def install_systemd_service(self):
        import os

        if os.path.isdir('/etc/systemd/user/'):
            from pkg_resources import Requirement, resource_filename
            from shutil import copy2
            service_path = resource_filename(Requirement.parse("actor"),
                                             "actor/static/actor.service")
            copy2(service_path, '/etc/systemd/user/')

    def run(self):
        install.run(self)
        self.install_systemd_service()

setup(
    name='actor',
    version=version,
    description='Actor the activity enforcer',
    long_description=open('README.md').read(),
    author='Tomas Babej',
    author_email='tomasbabej@gmail.com',
    license='AGPLv3',
    url='https://github.com/tbabej/actor',
    download_url='https://github.com/tbabej/actor/downloads',
    packages=find_packages(exclude=['tests']),
    include_package_data=True,
    install_requires=install_requirements,
    classifiers=[
        'Development Status :: 4 - Beta',
    ],
    entry_points={
        'console_scripts': [
            'actor = actor.core.client:main',
            'actor-daemon = actor.core.actord:main',
            'actor-debug = actor.core.actord:main_debug',
        ]
    },
    cmdclass={
        'install': InstallSystemdServiceFileInstall,
    }
)


########################################################################
# import python-library
########################################################################
# from import
import keras.models
from keras import backend as K
from keras import regularizers
from keras.layers import Input, Dense, BatchNormalization, Activation, Conv2D, PReLU, Flatten, MaxPool2D, Dropout
from keras.models import Model
import tensorflow as tf
import tensorflow_addons as tfa

########################################################################
# keras model
########################################################################
def get_model(input_dim, lr):
    """
    define the keras model
    the model based on the simple dense auto encoder
    (128*128*128*128*8*128*128*128*128)
    """

def get_model(input_dim, lr):
    """
    define the keras model
    the model based on the simple dense auto encoder
    (128*128*128*128*8*128*128*128*128)
    """
    regularizer = regularizers.L2(1e-4)
    x = Input(shape=(input_dim,))

    h = Dense(128, kernel_regularizer=regularizer)(x)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(128, kernel_regularizer=regularizer)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(128, kernel_regularizer=regularizer)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(128, kernel_regularizer=regularizer)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(8)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(128, kernel_regularizer=regularizer)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(128, kernel_regularizer=regularizer)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(128, kernel_regularizer=regularizer)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(128, kernel_regularizer=regularizer)(h)
    h = BatchNormalization()(h)
    h = PReLU()(h)

    h = Dense(input_dim)(h)

    model = Model(inputs=x, outputs=h)

    model.compile(optimizer=tfa.optimizers.AdamW(weight_decay=lr),
                  loss='mean_squared_error')

    return model

#########################################################################

def load_model(file_path):
    return keras.models.load_model(file_path, compile=False)

def clear_session():
    K.clear_session()


from django.db import models
from rest_framework import serializers
from eventos.models import Evento, Tipo_Evento, SubTipo_Evento, Periodo, Convocatoria
from authentication.serializers import UsuarioSerializer
from authentication.models import Usuario
from datetime import datetime
import json

# class EventoSerializer(serializers.PrimaryKeyRelatedField, serializers.ModelSerializer):
class EventoSerializer(serializers.ModelSerializer):
    #autor = UsuarioSerializer()
    convocatoria = models.ForeignKey(Tipo_Evento)
    created_at = serializers.DateTimeField(format="%Y-%m-%d %H:%M:%S", read_only=True)

    class Meta:
        model = Evento
        fields = ('id', 'contenido', 'convocatoria', 'autor', 'titulo', 'created_at', 'updated_at')
        read_only_fields = ('created_at', 'updated_at')

    def create_evento(self, validated_data):
        return Evento.objects.create_evento(**validated_data)

    def update(self, evento, validated_data):
        try:
            # comprobando contenido
            if 'contenido' in validated_data.keys():
                new_contenido = validated_data.get('contenido')
                if not isinstance(new_contenido, basestring):
                    raise NameError("Error en el contenido")
                else:
                    evento.contenido = new_contenido

            # comprobando convocatoria
            # if 'convocatoria' in validated_data.keys():
            #     new_convocatoria = validated_data.get('convocatoria')
            #     try:
            #         convocatoria = Convocatoria.objects.get(codigo=new_convocatoria)
            #     except Tipo_Evento.DoesNotExist:
            #         raise NameError("Convocatoria incorrecto")
            #     evento.convocatoria = convocatoria

            # comprobando tipo
            if 'tipo' in validated_data.keys():
                new_tipo = validated_data.get('tipo')
                try:
                    tipo = SubTipo_Evento.objects.get(codigo=new_tipo)
                except SubTipo_Evento.DoesNotExist:
                    raise NameError("Tipo Evento incorrecto")
                evento.tipo = tipo

            # comprobando autor
            # if 'autor' in validated_data.keys():
            #     new_autor = validated_data.get('autor')
            #     autor = Usuario.objects.get(email=new_autor)
            #     if not isinstance(autor, Usuario):
            #         raise NameError("Autor incorrecto")
            #     else:
            #         evento.autor = autor

            # comprobando titulo
            if 'titulo' in validated_data.keys():
                new_titulo = validated_data.get('titulo')
                if not isinstance(new_titulo, basestring):
                    raise NameError("El titulo del evento no tiene formato correcto")
                else:
                    evento.titulo = new_titulo

            # comprobando desde
            if 'desde' in validated_data.keys():
                try:
                    new_desde = datetime.strptime(validated_data.get('desde')[:19], '%Y-%m-%dT%H:%M:%S')
                except:
                    raise NameError("Error en formato fecha desde")

            # comprobando hasta
            if 'hasta' in validated_data.keys():
                try:
                    new_hasta = datetime.strptime(validated_data.get('hasta')[:19], '%Y-%m-%dT%H:%M:%S')
                except:
                    raise NameError("Error en formato fecha hasta")

            # Actualizo el periodo
            if 'hasta' in validated_data.keys() or 'desde' in validated_data.keys():
                periodo = Periodo.objects.get(evento=evento)
                if 'hasta' in validated_data.keys():
                    periodo.end = new_hasta
                if 'desde' in validated_data.keys():
                    periodo.start = new_desde
                periodo.save()
            evento.save()
            return dict(status=True, data=EventoSerializer(Evento.objects.get(contenido=evento.contenido)).data)

        except NameError as e:
            return dict(status=False, message=e.message)
        except:
            return dict(status=False, message="Error en los parametros")

    def delete(self, evento):
        evento.delete()
        return dict(status=True)

class Tipo_EventoSerializer(serializers.ModelSerializer):

    class Meta:
        model = Tipo_Evento
        fields = ('id', 'codigo', 'nombre',)

    def create(self, validated_data):
        return Tipo_Evento.objects.create_tipo_evento(**validated_data)

    def update(self, tipo_evento, validated_data):
        try:
            # comprobando codigo
            if 'codigo' in validated_data.keys():
                new_codigo = validated_data.get('codigo')
                res = Tipo_Evento.objects.filter(codigo=new_codigo)
                if res.count() != 0:
                    raise NameError("El tipo de evento ya existe")
                elif not isinstance(new_codigo, basestring):
                    raise NameError("El codigo del tipo de evento no tiene formato correcto")
                else:
                    tipo_evento.codigo = new_codigo

            # comprobando nombre
            if 'nombre' in validated_data.keys():
                new_nombre = validated_data.get('nombre')
                if not isinstance(new_nombre, basestring):
                    raise NameError("El nombre del tipo de evento no tiene formato correcto")
                else:
                    tipo_evento.nombre = new_nombre

            tipo_evento.save()

            return dict(status=True, data=tipo_evento)

        except NameError as e:
            return dict(status=False, message=e.message)
        except:
            return dict(status=False, message="Error en los parametros")

    def delete(self, tipo_evento):
        tipo_evento.delete()
        return dict(status=True)

class SubTipo_EventoSerializer(serializers.ModelSerializer):

    class Meta:
        model = SubTipo_Evento
        fields = ('id', 'codigo', 'nombre',)

    def update(self, subtipo, validated_data):
        try:
            # comprobando codigo
            if 'codigo' in validated_data.keys():
                new_codigo = validated_data.get('codigo')
                res = SubTipo_Evento.objects.filter(codigo=new_codigo)
                if res.count() != 0:
                    raise NameError("El departamento ya existe")
                elif not isinstance(new_codigo, basestring):
                    raise NameError("El codigo del subtipo no tiene formato correcto")
                else:
                    subtipo.codigo = new_codigo

            # comprobando nombre
            if 'nombre' in validated_data.keys():
                new_nombre = validated_data.get('nombre')
                if not isinstance(new_nombre, basestring):
                    raise NameError("El nombre del subtipo no tiene formato correcto")
                else:
                    subtipo.nombre = new_nombre

            subtipo.save()

            return dict(status=True, data=subtipo)

        except NameError as e:
            return dict(status=False, message=e.message)
        except:
            return dict(status=False, message="Error en los parametros")

    def delete(self, subtipo):
        subtipo.delete()
        return dict(status=True)


from functools import reduce
import sys
from typing import (
    Dict,
    List,
    Set,
    Tuple,
)

Cell = Tuple[int, int]
Path = Dict[Cell, Set[Cell]]

class HeightMap:
    def __init__(self, raw_values: List[str]) -> None:
        self.values: List[List[int]] = []
        self.low_points: List[Tuple[int, int]] = []
        self.risks: List[int] = []
        self.rows: int = 0
        self.cols: int = 0
        self.basins: Path = {}
        self.basin_sizes: List[int] = []

        self._init_values(raw_values)
        self._init_lows()
        self._init_risks()

    def _init_values(self, raw: List[str]) -> None:
        for l in raw:
            self.values.append([int(raw) for raw in l.strip()])

    def _init_lows(self):
        vals = self.values
        self.rows, self.cols = len(vals), len(vals[0])

        for r in range(self.rows):
            for c in range(self.cols):
                cur = vals[r][c]

                left = float("inf") if c == 0 else vals[r][c - 1]
                right = float("inf") if c == self.cols - 1 else vals[r][c + 1]
                up = float("inf") if r == 0 else vals[r - 1][c]
                down = float("inf") if r == self.rows - 1 else vals[r + 1][c]

                if cur < left and cur < right and cur < up and cur < down:
                    self.low_points.append((r, c))

    def _init_risks(self):
        for r, c in self.low_points:
            self.risks.append(self.values[r][c] + 1)

    def calc_basins(self):
        def dfs(r: int, c: int, val: int, path: Set[Cell]):
            cur = self.values[r][c]
            if cur == -1:
                return

            path.add((r, c))
            dirs = [(1, 0), (0, 1), (-1, 0), (0, -1)]
            self.values[r][c] = -1
            for dr, dc in dirs:
                nr, nc = r + dr, c + dc
                if not (0 <= nr < self.rows and 0 <= nc < self.cols):
                    continue
                nxt = self.values[nr][nc]
                if nxt <= val or nxt == 9:
                    continue
                dfs(nr, nc, self.values[nr][nc], path)
            self.values[r][c] = cur

        for r, c in self.low_points:
            self.basins[(r, c)] = set()
            path = self.basins[(r, c)]
            dfs(r, c, self.values[r][c], path)
            self.basin_sizes.append(len(path))

        self.basin_sizes.sort(reverse=True)

    def print_basins(self):
        for (r, c), path in self.basins.items():
            print(f"for {self.values[r][c]} @ ({r},{c}) -> total of {len(path)}")
            for pr, pc in path:
                print(f"    {self.values[pr][pc]} @ ({pr},{pc})")

def read_input(input_file: str) -> List[str]:
    lines: List[str] = []
    with open(input_file) as f:
        lines = f.readlines()
    return lines

def main(input_file: str) -> None:
    raw_vals = read_input(input_file)
    h_map = HeightMap(raw_vals)
    #  print(sum(h_map.risks))
    h_map.calc_basins()
    #  h_map.print_basins()
    print(reduce(lambda a, b: a * b, h_map.basin_sizes[:3]))

if __name__ == "__main__":
    main(sys.argv[1])


import nanoutils as nano
from interface_client import InterfaceClient
import tornado.httpserver
import tornado.websocket
import tornado.ioloop
import tornado.web
from tornado import gen
import logging
import socket
import requests
import json
import time
import datetime
import argparse
import random
import rethinkdb
import hashlib
import uuid
from enum import Enum

# from tornado.concurrent import Future, chain_future

TIMEOUT_ON_DEMAND = 6.0

parser = argparse.ArgumentParser()
parser.add_argument("--rai_node_uri", help='rai_nodes uri, usually 127.0.0.1', default='127.0.0.1')
parser.add_argument("--rai_node_port", help='rai_node port, usually 7076', default='7076')
parser.add_argument("--internal_port", help='internal port which nginx proxys', default='5000')
parser.add_argument("--interface", help='send data to interface (configured in interface.cfg', action='store_true')
parser.add_argument("-v", "--verbose", help='more prints to help debugging', action='store_true')

args = parser.parse_args()

rai_node_address = 'http://{uri}:{port}'.format(uri=args.rai_node_uri, port=args.rai_node_port)

wss_demand = []
wss_precache = []
wss_work = []
wss_timeout = []
hash_to_precache = []
work_tracker = {}
precache_work_tracker = {}
old_hashes = []
blacklist = {}

worker_counter = 0

rethinkdb.set_loop_type("tornado")
connection = rethinkdb.connect("localhost", 28015)

timezone = rethinkdb.make_timezone('+00:00')

def print_time_debug(message):
    if args.verbose:
        print("(DEBUG)", end=' ')
        print_time(message)

def print_time(message):
    print(time.strftime("%d/%m/%Y %H:%M:%S") + " " + str(message))

def print_lists(work=False, demand=False, precache=False, timeout=False):
    if not (work or demand or precache):
        return
    s = "State of lists:"
    if work:
        s += "\n\t\t\twss_work: {}".format(wss_work)
    if demand:
        s += "\n\t\t\twss_demand: {}".format(wss_demand)
    if precache:
        s += "\n\t\t\twss_precache: {}".format(wss_precache)
    if timeout:
        s += "\n\t\t\twss_timeout: {}".format(wss_timeout)
    print_time(s)

def remove_from_timeout(client):
    print_lists(timeout=True)
    print_time("Removing client {} IP from timeout".format(client))
    if client.ip in wss_timeout:
        wss_timeout.remove(client.ip)

def get_all_clients():
    clients = set(wss_demand + wss_precache)
    for client in clients:
        if not client.ws_connection.stream.socket:
            print_time("Removing client, socket is not active: {}".format(client))
            client.close()
            client.remove_from_lists()
            clients.remove(client)
            del client
    return clients

def build_blacklist(filepath='blacklist.txt'):
    try:
        with open(filepath, 'r') as f:
            lines = f.readlines()
    except:
        print('Blacklist file does not exist: {}'.format(filepath))
        return {}

    blacklist = { line.split('\n')[0] for line in lines }
    print('Built blacklist:')
    { print('\t{}'.format(acc)) for acc in blacklist }

    return blacklist

# TODO we would like to not have to use this at all, once all services provide the account
def get_account_from_hash(hash_hex):
    """Returns tuple (account,is_open_block)"""

    get_account = '{ "action" : "block_account", "hash" : "%s"}' % hash_hex
    print_time_debug("action block_account request:\n{}".format(get_account))
    r = requests.post(rai_node_address, data=get_account)
    print_time_debug("action block_account response:\n{}".format(r.text))

    resulting_data = r.json()
    if 'account' in resulting_data:
        account = resulting_data['account'].replace('nano_','xrb_')
        return (account, False)
    elif 'error' in resulting_data:
        if resulting_data['error'] == 'Block not found':
            # This maybe a public key for an Open Block, convert to xrb address
            print_time("Open Block")
            account = nano.hex_to_account(hash_hex)
            print_time("The account is {}".format(account))
            return (account, True)
        else:  # other errors, for instance invalid hash
            return ('Error', None)
    else:
        return ('Error', None)

class WorkState(Enum):
    needs = "0"

class Work(tornado.web.RequestHandler):
    def data_received(self, chunk):
        pass

    @gen.coroutine
    def ws_demand(self, message):
        print_time("Sending via WS Demand")
        # randomise our wss list then cycle through until we manage to send a message
        random.shuffle(wss_demand)
        print_lists(work=1, demand=1, timeout=1)

        # First pass: prioritize on_demand clients only
        for ws in wss_demand:
            if not ws.ws_connection.stream.socket:
                print_time("Web socket does not exist anymore!!!")
                ws.close()
                ws.remove_from_lists()
                del ws
            else:
                if ws not in wss_work and ws.ip not in wss_timeout and ws not in wss_precache:
                    ws.write_message(message)
                    wss_work.append(ws)
                    return ws

        # Second pass: go for on_demand and both
        for ws in wss_demand:
            if ws not in wss_work and ws.ip not in wss_timeout:
                ws.write_message(message)
                wss_work.append(ws)
                return ws

        # no clients
        return None

    @gen.coroutine
    def get_work_via_ws(self, hash_hex, account):
        conn = yield connection

        # Get appropriate threshold value
        # TODO after prioritization PoW is implemented, calculate here an appropriate multiplier
        multiplier = 1.0
        threshold = nano.from_multiplier(nano.NANO_DIFFICULTY, multiplier)
        threshold_str = nano.threshold_to_str(threshold)

        # Try up to 2 times, sending work to another client if the first fails
        error = None
        max_tries = 2
        try_count = 0
        while try_count < max_tries:
            try_count += 1

            # Send request to websocket clients to process
            send_result = yield self.ws_demand('{"hash" : "%s", "type" : "urgent", "threshold" : "%s"}' % (hash_hex,threshold_str))

            # If no clients, mark this account in DB
            if not send_result:
                print_time("No clients to to work for account %s" % account)
                if wss_timeout and try_count != max_tries:
                    print_lists(timeout=1)
                    removed = wss_timeout.pop(0)
                    print_time("Trying again after removing the oldest IP in wss_timeout: {}".format(removed))
                    continue
                else:
                    error = 'no_clients'
                    break

            # Place this hash in the work tracker so it can be updated in the client handler
            client_id = None
            work_tracker[hash_hex] = -1
            t_start = time.time()
            print_time("Waiting for work...")
            while time.time() - t_start < TIMEOUT_ON_DEMAND:
                try:
                    if work_tracker.get(hash_hex) != -1:
                        work_output, client_id = work_tracker.pop(hash_hex)
                        print_time_debug("Hash handled in {} seconds: {}".format(time.time()-t_start, hash_hex))
                        raise gen.Return((work_output, client_id, multiplier))
                except KeyError:
                    print_time_debug("Hash was removed from work_tracker due to an error: {}".format(hash_hex))
                    error = 'failed_clients'
                    break
                else:
                    yield gen.sleep(0.01)

            # Took too long, add to timeout list
            client_to_timeout = send_result
            print_time("Placing {} in timeout for 2 hours".format(client_to_timeout))
            wss_timeout.append(client_to_timeout.ip)
            tornado.ioloop.IOLoop.current().add_timeout(time.time() + 120*60, lambda: remove_from_timeout(client_to_timeout))

        if error:
            if error == 'no_clients':
                raise gen.Return(('no clients', None, None))

            elif error == 'failed_clients':
                raise gen.Return(('error', None, None))

        else:
            # No specific error by client handler, simply a timeout reached
            raise gen.Return(('timeout', None, None))

    @gen.coroutine
    def post(self):
        receive_time = datetime.datetime.now(timezone)
        post_data = json.loads(self.request.body.decode('utf-8'))
        if 'hash' in post_data:
            hash_hex = post_data['hash'].upper()
        elif 'accounts' not in post_data:
            return_json = '{"status" : "no hash"}'
            print_time(return_json)
            self.write(return_json)
            return

        conn = yield connection

        # 1 Check key is valid
        if 'key' in post_data:
            print_time("found API key")
            key = post_data['key']
            key_hashed = hashlib.sha512(key.encode('utf-8')).hexdigest()
            service_data = yield rethinkdb.db("pow").table("api_keys").filter({"api_key": key_hashed}).nth(0).default(False).run(conn)
            if not service_data:
                print_time("incorrect API key")
                return_json = '{"status" : "incorrect key"}'
                print_time(return_json)
                self.write(return_json)
                return
            else:
                print_time("Correct API key from %s - continue" % service_data['username'])
        else:
            print_time("no API key")
            return_json = '{"status" : "no key"}'
            print_time(return_json)
            self.write(return_json)
            return

        # Are they sending a precache list?
        if 'accounts' in post_data:
            accounts_precache = set(post_data['accounts'])
            print_time("Request to precache for {} accounts".format(len(accounts_precache)))

            # Take out those that are not in DB
            accounts_in_db = set()
            accounts_expr = rethinkdb.expr(accounts_precache)
            accounts_data = yield rethinkdb.db("pow").table("hashes").filter(lambda doc: accounts_expr.contains(doc['account'])).run(conn)
            while (yield accounts_data.fetch_next()):
                item = yield accounts_data.next()
                accounts_in_db.add(item["account"])

            not_in_db = accounts_precache.difference(accounts_in_db)
            print_time("{} accounts already in DB, adding the other {}".format(len(accounts_in_db), len(not_in_db)))

            # Get the public keys
            bad_accounts = set()
            public_keys = dict()
            for account in not_in_db:
                get_key = '{ "action" : "account_key", "account" : "%s"}' % account
                r = requests.post(rai_node_address, data=get_key)
                r = r.json()
                if 'error' in r:
                    # invalid account
                    bad_accounts.add(account)
                else:
                    public_keys[account] = r['key']

            # Filter out the bad accounts
            filtered_accounts = not_in_db.difference(bad_accounts)

            if bad_accounts:
                print_time("{} accounts given to precache are not valid:\n{}".format(len(bad_accounts), bad_accounts))
                print_time("{} were valid".format(len(filtered_accounts)))

            # Add to DB
            results = yield rethinkdb.db("pow").table("hashes").insert(
                [{"work": WorkState.needs.value, "hash": public_keys[account], "account": account, "unopened_account": True} for account in filtered_accounts]
            ).run(conn)

            # Send for precaching
            hash_to_precache.extend(public_keys.values())

            # Send feedback to service
            return_json = json.dumps({"added": list(filtered_accounts), "already_tracked": list(accounts_in_db), "bad_accounts": list(bad_accounts)})
            print_time_debug(return_json)
            self.write(return_json)
            return

        # 2 Do they provide the account besides the hash?
        account = post_data.get('account') or None
        if account:
            print_time('Account provided: {}'.format(account))
            account = account.replace('nano_','xrb_')
            if account.find('xrb_') == -1:
                print_time_debug('NOTIFY SERVICE! Invalid account provided {}'.format(account))
                account = None
        else:
            print_time_debug('NOTIFY SERVICE! Ask to provide accounts')

            # Get account to setup DB entries and check if invalid hash
            account, is_open_block = get_account_from_hash(hash_hex)
            if account == 'Error':
                return_json = '{"status" : "bad hash"}'
                print_time(return_json)
                self.write(return_json)
                return

        # Update entry
        new_count = int(service_data['count']) + 1
        yield rethinkdb.db("pow").table("api_keys").filter(rethinkdb.row['api_key'] == key_hashed).update(
            {"count": new_count, "provides_accounts": post_data.get('account')!=None}).run(conn)

        # 3 Do we have hash in db? If not (or work is not provided) handle on demand
        hash_data = yield rethinkdb.db("pow").table("hashes").filter({"account": account}).nth(0).default(False).run(conn)
        if hash_data and hash_data['hash'] == hash_hex:
            print_time('Found cached work value %s' % hash_data['work'])
            work_output = hash_data['work']
            new_entry = False
            if work_output == WorkState.needs.value:
                print_time("Empty work, get new on demand")
                work_output, client_id, multiplier = yield self.get_work_via_ws(hash_hex, account)
                work_type = 'O'
            else:
                client_id = hash_data.get('last_worker') or None
                if 'threshold' in hash_data:
                    multiplier = nano.to_multiplier(nano.NANO_DIFFICULTY, nano.threshold_from_str(hash_data['threshold']))
                else:
                    multiplier = 1.0
                work_type = 'P'
        elif not hash_data:
            print_time('Account not in DB, creating entry and getting on demand...')
            new_entry = True
            work_output, client_id, multiplier = yield self.get_work_via_ws(hash_hex, account)
            work_type = 'O'
        else: # hash_data but hash is old
            print_time('Old hash, getting on demand...')
            new_entry = False
            work_output, client_id, multiplier = yield self.get_work_via_ws(hash_hex, account)
            work_type = 'O'

        complete_time = datetime.datetime.now(timezone)

        # 4 Return work
        if work_output in ['timeout', 'no clients', 'error']:
            return_json = '{"status" : "%s"}' % work_output
            work_ok = False
        else:
            return_json = '{"work" : "%s"}' % work_output
            work_ok = True

        print_time(return_json)
        self.write(return_json)

        # DB update
        if work_ok:
            threshold = nano.from_multiplier(nano.NANO_DIFFICULTY, multiplier)
            threshold_str = nano.threshold_to_str(threshold)

            if new_entry:
                yield rethinkdb.db("pow").table("hashes").insert(
                    {"account": account, "last_worker": client_id, "hash": hash_hex, "work": work_output, "threshold": threshold_str, "unopened_account": False}).run(conn)
            else:
                yield rethinkdb.db("pow").table("hashes").filter({"account": account}).update(
                    {"hash": hash_hex, "last_worker": client_id, "work": work_output, "threshold": threshold_str, "unopened_account": False}).run(conn)
        else:
            if new_entry:
                yield rethinkdb.db("pow").table("hashes").insert(
                    {"account": account, "hash": hash_hex, "work": WorkState.needs.value, "unopened_account": False}).run(conn)
            else:
                yield rethinkdb.db("pow").table("hashes").filter({"account": account}).update(
                    {"hash": hash_hex, "work": WorkState.needs.value, "unopened_account": False}).run(conn)

        # Interface update
        if work_ok and interface:

            # Random request id
            request_id = str(uuid.UUID(int=random.SystemRandom().getrandbits(128)))

            # Send to interface
            service_id = service_data['id']
            interface.pow_update(request_id, service_id, client_id, work_type, multiplier, receive_time, complete_time, new_entry)

class WSHandler(tornado.websocket.WebSocketHandler):

    worker_counter = 0

    def __init__(self, *args, **kwargs):
        WSHandler.worker_counter += 1
        self.id = WSHandler.worker_counter
        self.type = ''
        self.address = ''
        self.ip = ''
        super().__init__(*args, **kwargs)

    def __repr__(self):
        return '{id} ({ip}, {type}, {state})'.format(id=self.id or 'unknown id', ip=self.ip or 'unknown ip', type=self.type or 'unknown type',
                                       state='timeout' if self.ip in wss_timeout else 'busy' if self in wss_work else 'free')

    @staticmethod
    def validate_work(hash_hex, work, threshold):
        get_validation = '{ "action" : "work_validate", "hash" : "%s", "work": "%s", "threshold": "%s" }' % (hash_hex, work, threshold)
        r = requests.post(rai_node_address, data=get_validation)
        resulting_validation = r.json()
        if 'error' in resulting_validation:
            print_time_debug(get_validation)
            raise gen.Return('Error in validation: {}'.format(resulting_validation))
        return int(resulting_validation['valid'])

    def open(self):
        print_time('New worker connected - {}'.format(self.id))
        self.ip = self.request.remote_ip
        print_time('IP: {}'.format(self.ip))

    @gen.coroutine
    def on_message(self, message):
        print_time('Message from worker {}: {}'.format(self.id, message))
        try:
            ws_data = json.loads(message)
            if 'address' not in ws_data:
                raise Exception('Incorrect data from client: {}'.format(ws_data))

            self.address = ws_data['address']

            if 'work_type' in ws_data:
                # Setup message handling

                # remove from any lists
                self.remove_from_lists()

                # restrict clients per IP
                connected_clients = get_all_clients()
                same_ip = list(filter(lambda c: c.ip == self.ip, connected_clients))
                if len(same_ip) >= 2:  # this client is not yet in the lists
                    print_time("Client attempted to connect more than 2 clients: IP: {} , already in list: {} , new client: {}".format(self.ip, same_ip, self))
                    self.write_message('{"status": "error", "description": "Maximum of 2 clients allowed"}')
                    self.close()
                    return

                # handle setup message for work type
                if ws_data['address'] in blacklist:
                    print_time("Blacklisted: {}".format(ws_data['address']))
                else:
                    work_type = ws_data['work_type']
                    print_time("Found work_type -> {}".format(work_type))
                    try:
                        self.update_work_type(work_type)
                        self.write_message('{"status": "success"}')
                    except Exception as e:
                        print_time(e)
                        self.write_message('{"status": "error", "description": "%s"}' % e)
            else:
                if 'hash' not in ws_data or 'work' not in ws_data:
                    raise Exception('Incorrect data from client: {}'.format(ws_data))

                if self not in wss_precache and self not in wss_demand:
                    self.write_message('{"status": "error", "description": "Must setup first"}')
                    return

                # handle work message
                hash_hex = ws_data['hash'].upper()
                work = ws_data['work']
                payout_account = ws_data['address'].lower()

                if work == 'error':
                    raise Exception("'Something wrong with the client, work returned as error'")

                if hash_hex in work_tracker:
                    this_work_type = 'urgent'
                elif hash_hex in precache_work_tracker:
                    this_work_type = 'precache'
                else:
                    print_time("Work was given for hash {} but this hash was not in any of the work trackers".format(hash_hex))
                    if hash_hex in old_hashes:
                        conn = yield connection
                        print_time("Hash was an old hash given to this client, we should still increase client count but not update the account entry")
                        clients_data = yield rethinkdb.db("pow").table("clients").filter(
                            rethinkdb.row['account'] == payout_account).nth(0).default(False).run(conn)
                        if clients_data:
                            client = clients_data
                            count = int(client['count'])
                            yield rethinkdb.db("pow").table("clients").filter(rethinkdb.row['account'] == payout_account).update(
                                {"count": count + 1}).run(conn)
                    else:
                        print_time("Hash not in old_hashes, so could be a client trying to do fake work. Rejected and return here")
                        return

                # check the threshold at which this was computed
                # defaults to NANO_DIFFICULTY if not found (e.g. early entries in DB)
                conn = yield connection
                data = yield rethinkdb.db("pow").table("hashes").filter(rethinkdb.row['hash'] == hash_hex).nth(0).default(False).run(conn)
                if data:
                    threshold_str = data.get("threshold") or nano.threshold_to_str(nano.NANO_DIFFICULTY)
                else:
                    threshold_str = nano.threshold_to_str(nano.NANO_DIFFICULTY)

                # validate the work from client
                valid = self.validate_work(hash_hex, work, threshold_str)
                if valid:
                    if hash_hex in hash_to_precache:
                        hash_to_precache.remove(hash_hex)

                    if this_work_type == 'urgent':
                        # Update the work tracker so that the service wait loop knows it is done
                        work_tracker[hash_hex] = (work, payout_account)
                    else:
                        if hash_hex in precache_work_tracker:
                            precache_work_tracker.pop(hash_hex)

                        # Update DB here directly
                        yield rethinkdb.db("pow").table("hashes").filter({"hash": hash_hex}).update(
                            {"work": work, "threshold": threshold_str, "last_worker": payout_account, "unopened_account": False}).run(conn)

                    # Add work record to client database to allow payouts
                    clients_data = yield rethinkdb.db("pow").table("clients").filter(
                        rethinkdb.row['account'] == payout_account).nth(0).default(False).run(conn)
                    if clients_data:
                        client = clients_data
                        count = int(client['count'])
                        count_type = int(client['{}_count'.format(this_work_type)])
                        yield rethinkdb.db("pow").table("clients").filter(
                            rethinkdb.row['account'] == payout_account).update(
                                {"count": count + 1, "{}_count".format(this_work_type): count_type + 1, "time": time.time()}).run(conn)
                    else:
                        other_work_type = 'urgent' if this_work_type == 'precache' else 'precache'
                        yield rethinkdb.db("pow").table("clients").insert(
                               {"account": payout_account, "count": 1, "{}_count".format(this_work_type): 1,
                                       "{}_count".format(other_work_type): 0, "time": time.time()}).run(conn)
                    # Remove from work list
                    if self in wss_work:
                        print_time("Removing {} from wss_work".format(self))
                        wss_work.remove(self)

                else:
                    raise Exception("Failed to validate work - {} for worker {}".format(valid, self))

        except Exception as e:
            print_time("Error {}".format(e))
            if self in wss_work:
                print_time("Removing {} from wss_work after exception".format(self))
                wss_work.remove(self)

            try:
                # Remove from work tracker, the service wait loop can decide what to do (for instance give work to someone else)
                if hash_hex in work_tracker:
                    work_tracker.pop(hash_hex)

                if hash_hex in precache_work_tracker:
                    precache_work_tracker.pop(hash_hex)
                    if hash_hex not in hash_to_precache:
                        hash_to_precache.append(hash_hex)
            except:
                pass

    def update_work_type(self, work_type):

        if work_type == 'any':
            # Add to both demand and precache
            wss_demand.append(self)
            wss_precache.append(self)
            self.type = 'B'
        elif work_type == 'precache_only':
            # Add to precache
            wss_precache.append(self)
            self.type = 'P'
        elif work_type == 'urgent_only':
            # Add to demand
            wss_demand.append(self)
            self.type = 'O'
        else:
            raise Exception('Invalid work type {}'.format(work_type))

    def on_close(self):
        print_time('Worker disconnected - {}'.format(self.id))
        self.remove_from_lists()

    def remove_from_lists(self):
        for l in [wss_work, wss_demand, wss_precache]:
            if self in l:
                l.remove(self)

    def check_origin(self, origin):
        return True

application = tornado.web.Application([
    (r'/group/', WSHandler),
    (r"/work", Work),
])

@gen.coroutine
def push_precache():
    hash_count = 0
    work_count = 0

    # Get appropriate threshold value
    # TODO what is a good threshold value for precaching?
    multiplier = 1.0
    threshold = nano.from_multiplier(nano.NANO_DIFFICULTY, multiplier)
    threshold_str = nano.threshold_to_str(threshold)

    if hash_to_precache:
        print_time("Got precache work to push")
        random.shuffle(wss_precache)
        print_lists(work=1, precache=1)
        # if all precachers are doing work... no free workers
        if set(wss_precache).issubset(set(wss_work)):
            print_time("No clients available to push precache")
            return

    for hash_hex in hash_to_precache:
        hash_count = hash_count + 1
        hash_handled = False
        for work_clients in wss_precache:  # type: WSHandler
            print_time("Sending via WS Precache")
            print_lists(work=1, precache=1)
            try:
                if not work_clients.ws_connection.stream.socket:
                    print_time("Web socket does not exist anymore!!!")
                    work_clients.close()
                    work_clients.remove_from_lists()
                    del work_clients
                else:
                    if work_clients not in wss_work:
                        work_count = work_count + 1
                        message = '{"hash" : "%s", "type" : "precache", "threshold" : "%s"}' % (hash_hex, threshold_str)
                        work_clients.write_message(message)
                        wss_work.append(work_clients)
                        hash_to_precache.remove(hash_hex)
                        precache_work_tracker[hash_hex] = time.time()
                        hash_handled = True
                        break

            except Exception as e:
                print_time('Error when sending via WS precache: {}'.format(e))
                if work_clients in wss_precache:
                    wss_precache.remove(work_clients)
                    print_time('Client {} removed from precache work'.format(work_clients.id))
                if work_clients in wss_work:
                    wss_work.remove(work_clients)

        if hash_handled:
            print_time("Hash was given to a precache client")
        else:
            print_time("Precache not handled - no free workers?")
            break

@gen.coroutine
def precache_update():
    t_start = time.time()
    count_updates = 0
    up_to_date = 0
    too_old = 0
    not_up_to_date = 0
    conn = yield connection
    #   print_time("precache_update")
#    precache_data = yield rethinkdb.db("pow").table("hashes")['account'].distinct().run(conn)
    precache_data = yield rethinkdb.db("pow").table("hashes").run(conn)
    if not precache_data:
        print_time("Failed to retrieve data from DB in precache_update")
        return

    count_unique = 0
    count_all = 0
    account_to_hash = dict()
    while (yield precache_data.fetch_next()):
        item = yield precache_data.next()
        account = item['account']
        if account not in account_to_hash:
            account_to_hash[account] = (item['hash'], item['work'])
            count_unique += 1
        count_all += 1

    accounts = list(account_to_hash.keys())
    print('Total {}, unique {}, duplicate: {}'.format(count_all, count_unique, count_all-count_unique))

    '''
    Get frontiers for all accounts in a single RPC call
    - if any account is not correct (wrong checksum), this will return an error - so it should be absolutely avoided
    - if an account is valid but no frontier is found by the node, it is likely an open block
    - since it could be simply that the "open account" block was not processed by the service, we will not be deleting open blocks here anymore
    '''
    # TODO if needed, then do a cleanup of wrong blocks every now and then, e.g. every 24h
    for i in range(0, len(accounts), 100):
        get_frontiers = json.dumps({"action": "accounts_frontiers", "accounts": accounts[i:i+99]})

        r = requests.post(rai_node_address, data=get_frontiers)
        frontiers = r.json()
        if 'frontiers' not in frontiers:
            print_time("Error getting account frontiers in precache_update: {}".format(frontiers))
            return

        frontiers = frontiers['frontiers']
        if type(frontiers) == str:    continue

        # this code would get the open block mistakes
        # open_block_mistakes = [a for a in accounts if a not in frontiers]

        for account, frontier in frontiers.items():
            count_updates += 1

            old_hash, work = account_to_hash[account]
            if old_hash == frontier:
                up_to_date += 1
                if old_hash in precache_work_tracker:
                    time_sent = precache_work_tracker[old_hash]
                    if time.time() - time_sent > 1000.0:
                        # probably client disconnected while doing precache work, we need to queue it again
                        too_old += 1
                        precache_work_tracker.pop(old_hash)
                        hash_to_precache.append(old_hash)
                else:
                    if work == WorkState.needs.value:
                        # this was precache work left undone since the last server restart
                        if old_hash not in hash_to_precache:
                            hash_to_precache.append(old_hash)

            else:
                not_up_to_date += 1
                if frontier not in hash_to_precache:
                    hash_to_precache.append(frontier)
                    if old_hash in precache_work_tracker:
                        # even if returned, the hash will not be up-to-date, so better remove from tracking and reject it later
                        precache_work_tracker.pop(old_hash)

                        # add to a list of old hashes. since we asked the clients, we should update their count later
                        old_hashes.append(old_hash)

                    yield rethinkdb.db("pow").table("hashes").filter(rethinkdb.row['account'] == account).update(
                        {"work": WorkState.needs.value, "hash": frontier, "unopened_account": False}).run(conn)

    print_time("Count: {:d}, Up to date: {:d}, Too old: {:d}, Not up to date: {:d}".format(
                count_updates, up_to_date, too_old, not_up_to_date))

    print_time_debug("precache_update took: {:.4f} seconds".format(time.time()-t_start))

@gen.coroutine
def update_interface_clients():
    conn = yield connection
    connected_clients = get_all_clients()
    # seen = set()
    # unique_clients_by_account = [c for c in connected_clients if c.address not in seen and not seen.add(c.address)]
    accounts = [c.address for c in connected_clients]
    accounts_expr = rethinkdb.expr(accounts)
    counts_by_type = dict()

    data = yield rethinkdb.db("pow").table("clients").filter(lambda row: accounts_expr.contains(row["account"])).run(conn)
    while (yield data.fetch_next()):
        client_data = yield data.next()
        counts_by_type[client_data["account"]] = {"precache": client_data["precache_count"], "urgent": client_data["urgent_count"]}

    clients = list(map(lambda c: {
        'client_id': c.address+'_'+c.type,
        'client_address': c.address,
        'client_type': c.type,
        'client_demand_count': counts_by_type[c.address]["urgent"] if c.address in counts_by_type else 0,
        'client_precache_count': counts_by_type[c.address]["precache"] if c.address in counts_by_type else 0
    }, connected_clients))

    interface.clients_update(clients)
    print_time_debug("Updated interface clients")

@gen.coroutine
def update_interface_services():
    conn = yield connection
    registered_services = yield rethinkdb.db("pow").table("api_keys").run(conn)
    if registered_services:
        services = list(map(lambda s: {
            'service_id': s.get('id'),
            'service_name': (s.get('display_name') or s.get('username')) if s.get('public') else None,
            'service_web': (s.get('website') or '') if s.get('public') else ''
        }, registered_services.items))

        interface.services_update(services)
        print_time_debug("Updated interface services")
    else:
        print_time_debug("Could not update interface services")

@gen.coroutine
def initial_db_update():
    conn = yield connection
    unopened_data = yield rethinkdb.db("pow").table("hashes").filter({"unopened_account": True, "work": WorkState.needs.value}).run(conn)
    unopened_accounts = set()
    while (yield unopened_data.fetch_next()):
        item = yield unopened_data.next()
        unopened_accounts.add(item['hash'])
    if unopened_accounts:
        print_time("Adding {} unopened accounts to precache".format(len(unopened_accounts)))
        hash_to_precache.extend(unopened_accounts)
    print_time("Initial DB update done")

if __name__ == "__main__":
    http_server = tornado.httpserver.HTTPServer(application)
    http_server.listen(int(args.internal_port))
    myIP = socket.gethostbyname(socket.gethostname())

    # setup interface
    if args.interface:
        interface = InterfaceClient()
        if not interface.read_config('interface.cfg'):
            print_time('INTERFACE DISABLED')
            interface = None
        else:
            print_time('Interface is setup -> {}'.format(interface.server))
    else:
        interface = None

    # initial DB check
    tornado.ioloop.IOLoop.current().run_sync(initial_db_update)

    blacklist = build_blacklist('blacklist.txt')

    print_time('*** Websocket Server Started at %s***' % myIP)
    pc = tornado.ioloop.PeriodicCallback(precache_update, 15000)
    pc.start()
    push = tornado.ioloop.PeriodicCallback(push_precache, 3000)
    push.start()
    if interface:
        # clients update every 10 seconds
        icli = tornado.ioloop.PeriodicCallback(update_interface_clients, 10000)
        icli.start()

        # services update every 60 seconds
        iserv = tornado.ioloop.PeriodicCallback(update_interface_services, 60000)
        iserv.start()

    main_loop = tornado.ioloop.IOLoop.instance()

    try:
        main_loop.start()
    except KeyboardInterrupt:
        pass


from flask import Flask, render_template, jsonify, request
from name import find_name, is_name
from nlp_fin import rest

import nltk.classify.util
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import names
from nltk.corpus import stopwords
import json
import math
import re
from nltk.tokenize import sent_tokenize, word_tokenize

app = Flask(__name__)
app.debug = True

@app.route('/')
def index():
    return render_template("home.html")

@app.route('/change_name', methods=['POST'])
def requester():
	if request.json:
		json = request.get_json(silent=True)
		print(json)
		text = json["text"]
		name = find_name(text)
		data = jsonify({'name': name})
		print(text)
		return data

@app.route('/api/check_name', methods=['POST', 'GET'])
def checker():
	if request.json:
		json = request.get_json(silent=True)
		print(json)
		text = json["name"]
		result = is_name(text)
		print(result)
		if(result):
			return("1")
		else:
			return("0")

@app.route('/result', methods=['POST', 'GET'])
def result() :
	if request.method == 'POST' :
		word = str(request.form['text']).lower()
		print(word)
		fin = rest(word)
		print(str(fin))
		return str(fin)
		# rest(word)"ram with 4gb and budget with 12000rs."

if __name__ == '__main__':
    app.run()


from sympy.parsing.mathematica import mathematica

import sympy as sp
from sympy.functions.special.bessel import besselj
from sympy.parsing.sym_expr import SymPyExpression
from sympy import latex

def get_ylm_FTs():
    results = []
    with open("SphericalHarmonicsResults_10.txt", 'r') as file:
        for line in file:
            # Split the line into columns based on tabs
            columns = line.strip().split('\t')
            l = int(columns[0])
            m = int(columns[1])
            result = mathematica(columns[2], {'BesselJ[n, rho]':'besselj(n, rho)'})
            results.append([l,m,result])
    replacements = {'pi':'jnp.pi',
                'cos':'jnp.cos',
                'sin':'jnp.sin',
                'sqrt':'jnp.sqrt',
                'I':'1j'}
    def replace_all(text, replacements):
        for i in replacements:
            text = text.replace(str(i), str(replacements[i]))
        return text
    jax_exprs = {}
    for l, m, res in results:
        jax_exprs[(l,m)]=replace_all(str(res), replacements)
    return jax_exprs


from django.core.management.base import BaseCommand, CommandError
from newsroom.models import Article, Author
from django.utils import timezone
from django.db.models import Count

def process(year, author_no=1):
    date_from = timezone.datetime(year=year, month=1, day=1)
    date_to = timezone.datetime(year=year, month=12, day=31)
    articles = Article.objects.published().filter(published__gte=date_from). \
               filter(published__lte=date_to)
    authors = Author.objects.all()
    author_field = "author_01"
    if author_no == 2:
        author_field = "author_02"
    elif author_no == 3:
        author_field = "author_03"
    elif author_no == 4:
        author_field = "author_04"
    elif author_no == 5:
        author_field = "author_04"

    count = articles.values(author_field).annotate(total=Count('author_01')).\
        order_by('-total')
    result = []
    for item in count:
        pk = pk=item[author_field]
        if pk:
            author = str(Author.objects.get(pk=item[author_field]))
        else:
            author = "NONE"
        num = item["total"]
        result.append([author, num])
    return result

class Command(BaseCommand):
    help = 'Count first authorships by author over a specified period of time'

    def add_arguments(self, parser):
        parser.add_argument('year', type=int,
                            help="Year to count for.")

        parser.add_argument('author_no', nargs='?', type=int,
                            help="Author number", default=1)

    def handle(self, *args, **options):
        year = options["year"]
        author_no = options["author_no"]
        print("CountAuthors for year {0}. Author # {1}.". \
              format(str(year),str(author_no)))
        author_list = process(year, author_no)
        for item in author_list:
            print(item[0], item[1])


try:
    import json
except ImportError:
    import simplejson as json

from pyhole import plugin
from pyhole import utils

class Redmine(plugin.Plugin):
    """Provide access to the Redmine API"""

    def __init__(self, irc):
        self.irc = irc
        self.name = self.__class__.__name__
        self.disabled = False

        try:
            self.redmine = utils.get_config("Redmine")
            self.redmine_domain = self.redmine.get("domain")
            self.redmine_key = self.redmine.get("key")
            self.redmine_url = "https://%s:password@%s" % (
                    self.redmine_key, self.redmine_domain)
        except Exception:
            self.disabled = True

    @plugin.hook_add_command("rbugs")
    @utils.spawn
    def rbugs(self, params=None, **kwargs):
        """Redmine bugs for a user (ex: .rbugs <login>)"""
        if params and not self.disabled:
            login = params.split(" ", 1)[0]
            user_id = self._find_user(login)

            i = 0
            issues = self._find_issues(user_id)
            for i, issue in enumerate(issues):
                if i <= 4:
                    self._find_issue(issue["id"])
                else:
                    self.irc.reply("[...] truncated last %d bugs" % (
                            len(issues) - i))
                    break
            else:
                if i <= 0:
                    self.irc.reply("No Redmine bugs found for '%s'" % login)
        else:
            self.irc.reply(self.rbugs.__doc__)

    @plugin.hook_add_keyword("rm")
    @utils.spawn
    def keyword_rm(self, params=None, **kwargs):
        """Retrieve Redmine bug information (ex: RM12345)"""
        if params and not self.disabled:
            params = utils.ensure_int(params)
            if params:
                self._find_issue(params)

    @plugin.hook_add_msg_regex("https?:\/\/redmine\..*/issues")
    def _watch_for_rm_bug_url(self, params=None, **kwargs):
        """Watch for Redmine bug URLs"""
        try:
            line = kwargs["full_message"].split("/")
            for i, word in enumerate(line):
                if word == "issues":
                    bug_id = line[i + 1].split(" ", 1)[0]
                    self.keyword_rm(bug_id)
        except TypeError:
            return

    def _find_issues(self, user_id):
        """Find all issues for a Redmine user"""
        url = "%s/issues.json?assigned_to_id=%s" % (
                self.redmine_url, user_id)
        response = self.irc.fetch_url(url, self.name)
        if not response:
            return

        return json.loads(response.read())["issues"]

    def _find_user(self, login):
        """Find a specific Redmine user"""
        for user in self._find_users():
            if login == user["login"]:
                return user["id"]

        for user in self._find_users(100):
            if login == user["login"]:
                return user["id"]

    def _find_users(self, offset=None):
        """Find all Redmine users"""
        if offset:
            url = "%s/users.json?limit=100&offset=%d" % (
                    self.redmine_url, offset)
        else:
            url = "%s/users.json?limit=100" % self.redmine_url
        response = self.irc.fetch_url(url, self.name)
        if not response:
            return

        return json.loads(response.read())["users"]

    def _find_issue(self, issue_id):
        """Find and display a Redmine issue"""
        url = "%s/issues/%s.json" % (self.redmine_url, issue_id)
        response = self.irc.fetch_url(url, self.name)
        if not response:
            return

        try:
            issue = json.loads(response.read())["issue"]
        except Exception:
            return

        self.irc.reply("RM %s #%s: %s [Status: %s, Assignee: %s] "
                "https://%s/issues/show/%s" % (
                issue["tracker"]["name"], issue["id"], issue["subject"],
                issue["status"]["name"],
                issue.get("assigned_to", {}).get("name", "N/A"),
                self.redmine_domain, issue["id"]))


"""pets

Revision ID: 98cf562f08e2
Revises: e07cd90eaa03
Create Date: 2022-10-08 22:16:41.157524
"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = "98cf562f08e2"
down_revision = "e07cd90eaa03"
branch_labels = None
depends_on = None

def upgrade():
    op.create_table(
        "pets",
        sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False, server_default=sa.text("gen_random_uuid()")),
        sa.Column("name", sa.Text(), nullable=False),
        sa.Column("notes", sa.Text(), nullable=True),
        sa.Column("status", sa.Text(), nullable=False),
        sa.Column("registration_id", sa.Text(), nullable=True),  # registration document id
        sa.Column("owner_id", postgresql.UUID(as_uuid=True), nullable=True),
        sa.Column("project_id", postgresql.UUID(as_uuid=True), nullable=True),
        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=True),
        sa.PrimaryKeyConstraint("id"),
        sa.ForeignKeyConstraint(
            ("owner_id",),
            ["people.id"],
            ondelete="SET NULL",
        ),
        sa.ForeignKeyConstraint(
            ("project_id",),
            ["projects.id"],
            ondelete="SET NULL",
        ),
    )

    op.create_index(op.f("ix_pets_name"), "pets", [sa.text("lower(name)")])
    op.create_index(op.f("ix_pets_status"), "pets", ["status"])
    op.create_index(op.f("ix_pets_registration_id"), "pets", ["registration_id"])
    op.create_index(op.f("ix_pets_owner_id"), "pets", ["owner_id"])
    op.create_index(op.f("ix_pets_project_id"), "pets", ["project_id"])

def downgrade():
    op.drop_table("pets")


import glob
import os
import sys
import pdb
import os.path as osp

sys.path.append(os.getcwd())

import math
import joblib
import numpy as np
from isaacgym import gymapi, gymutil, gymtorch
import torch
from poselib.poselib.skeleton.skeleton3d import SkeletonMotion, SkeletonState
from poselib.poselib.visualization.common import plot_skeleton_motion_interactive, plot_skeleton_state
from pacer.utils.motion_lib import MotionLib
from pacer.utils.motion_lib_smpl import MotionLib as MotionLibSMPL
from uhc.smpllib.smpl_local_robot import Robot
from mujoco_py import load_model_from_path, load_model_from_xml, MjSim, MjViewer
from poselib.poselib.skeleton.skeleton3d import SkeletonTree
from pacer.utils import torch_utils

def clamp(x, min_value, max_value):
    return max(min(x, max_value), min_value)

# simple asset descriptor for selecting from a list

class AssetDesc:
    def __init__(self, file_name, flip_visual_attachments=False):
        self.file_name = file_name
        self.flip_visual_attachments = flip_visual_attachments

robot_cfg = {
                "mesh": False,
                "rel_joint_lm": False,
                "upright_start": True,
                "remove_toe": False,
                "real_weight_porpotion": True,
                "model": "smpl",
                "body_params": {},
                "joint_params": {},
                "geom_params": {},
                "actuator_params": {},
}
smpl_robot = Robot(
    robot_cfg,
    data_dir="data/smpl",
    masterfoot=False,
)
test_good = f"test_good.xml"
smpl_robot.write_xml(test_good)
sk_tree = SkeletonTree.from_mjcf(test_good)

asset_descriptors = [
    AssetDesc(test_good, False),
]

# parse arguments
args = gymutil.parse_arguments(
    description="Joint monkey: Animate degree-of-freedom ranges",
    custom_parameters=[{
        "name":
        "--asset_id",
        "type":
        int,
        "default":
        0,
        "help":
        "Asset id (0 - %d)" % (len(asset_descriptors) - 1)
    }, {
        "name": "--speed_scale",
        "type": float,
        "default": 1.0,
        "help": "Animation speed scale"
    }, {
        "name": "--show_axis",
        "action": "store_true",
        "help": "Visualize DOF axis"
    }])

if args.asset_id < 0 or args.asset_id >= len(asset_descriptors):
    print("*** Invalid asset_id specified.  Valid range is 0 to %d" %
          (len(asset_descriptors) - 1))
    quit()

# initialize gym
gym = gymapi.acquire_gym()

# configure sim
sim_params = gymapi.SimParams()
sim_params.dt = dt = 1.0 / 60.0
sim_params.up_axis = gymapi.UP_AXIS_Z
sim_params.gravity = gymapi.Vec3(0.0, 0.0, -9.81)
if args.physics_engine == gymapi.SIM_FLEX:
    pass
elif args.physics_engine == gymapi.SIM_PHYSX:
    sim_params.physx.solver_type = 1
    sim_params.physx.num_position_iterations = 6
    sim_params.physx.num_velocity_iterations = 0
    sim_params.physx.num_threads = args.num_threads
    sim_params.physx.use_gpu = args.use_gpu

sim_params.use_gpu_pipeline = False
if args.use_gpu_pipeline:
    print("WARNING: Forcing CPU pipeline.")

sim = gym.create_sim(args.compute_device_id, args.graphics_device_id,
                     args.physics_engine, sim_params)
if sim is None:
    print("*** Failed to create sim")
    quit()

# add ground plane
plane_params = gymapi.PlaneParams()
plane_params.normal = gymapi.Vec3(0.0, 0.0, 1.0)
gym.add_ground(sim, plane_params)

# create viewer
viewer = gym.create_viewer(sim, gymapi.CameraProperties())
if viewer is None:
    print("*** Failed to create viewer")
    quit()

# load asset
# asset_root = "pacer/data/assets"
asset_root = "./"
asset_file = asset_descriptors[args.asset_id].file_name

asset_options = gymapi.AssetOptions()
asset_options.fix_base_link = True
# asset_options.flip_visual_attachments = asset_descriptors[
#     args.asset_id].flip_visual_attachments
asset_options.use_mesh_materials = True

print("Loading asset '%s' from '%s'" % (asset_file, asset_root))
asset = gym.load_asset(sim, asset_root, asset_file, asset_options)

# set up the env grid
num_envs = 1
num_per_row = 6
spacing = 0
env_lower = gymapi.Vec3(-spacing, spacing, 0)
env_upper = gymapi.Vec3(spacing, spacing, spacing)

# position the camera
cam_pos = gymapi.Vec3(0, -10.0, 3)
cam_target = gymapi.Vec3(0, 0, 0)
gym.viewer_camera_look_at(viewer, None, cam_pos, cam_target)

# cache useful handles
envs = []
actor_handles = []

num_dofs = gym.get_asset_dof_count(asset)
print("Creating %d environments" % num_envs)
for i in range(num_envs):
    # create env
    env = gym.create_env(sim, env_lower, env_upper, num_per_row)
    envs.append(env)

    # add actor
    pose = gymapi.Transform()
    pose.p = gymapi.Vec3(0.0, 0, 0.0)
    pose.r = gymapi.Quat(0, 0.0, 0.0, 1)

    actor_handle = gym.create_actor(env, asset, pose, "actor", i, 1)
    actor_handles.append(actor_handle)

    # set default DOF positions
    dof_states = np.zeros(num_dofs, dtype=gymapi.DofState.dtype)
    gym.set_actor_dof_states(env, actor_handle, dof_states, gymapi.STATE_ALL)

# Setup Motion
body_ids = []
key_body_names = ["R_Ankle", "L_Ankle", "R_Wrist", "L_Wrist"]
for body_name in key_body_names:
    body_id = gym.find_actor_rigid_body_handle(envs[0], actor_handles[0],
                                               body_name)
    assert (body_id != -1)
    body_ids.append(body_id)

body_ids = np.array(body_ids)
_dof_body_ids = [
    1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
    22, 23
]
_dof_offsets = np.linspace(0, 23 * 3, 24).astype(int)
_dof_obs_size = 138
_num_actions = 69

# motion_file = "data/amass/pkls/amass_isaac_locomotion_upright.pkl"
# motion_file = "data/amass/pkls/amass_isaac_slowalk_upright.pkl"
# motion_file = "data/amass/pkls/amass_isaac_slowalk_upright_slim.pkl"
# motion_file = "data/amass/pkls/amass_isaac_slowalk_upright_slim_double.pkl"
# motion_file = "data/amass/pkls/amass_isaac_run_upright_slim_double.pkl"
# motion_file = "data/amass/pkls/amass_isaac_walk_upright_test_slim.pkl"

import ipdb; ipdb.set_trace()
motion_file = "output/renderings/rough_terrain.pkl"
motion_data = joblib.load(motion_file)
joblib.dump(motion_data, motion_file)

motion_keys = list(joblib.load(motion_file).keys())
motion_lib = MotionLib(motion_file=motion_file,
                       dof_body_ids=_dof_body_ids,
                       dof_offsets=_dof_offsets,
                       key_body_ids=body_ids,
                       device=args.compute_device_id)

current_dof = 0
speeds = np.zeros(num_dofs)
fps = 60
time_step = 0
dt = 1 / fps
rigidbody_state = gym.acquire_rigid_body_state_tensor(sim)
rigidbody_state = gymtorch.wrap_tensor(rigidbody_state)
rigidbody_state = rigidbody_state.reshape(num_envs, -1, 13)

gym.subscribe_viewer_keyboard_event(viewer, gymapi.KEY_LEFT, "previous")
gym.subscribe_viewer_keyboard_event(viewer, gymapi.KEY_RIGHT, "next")
gym.subscribe_viewer_keyboard_event(viewer, gymapi.KEY_G, "add")
gym.subscribe_viewer_keyboard_event(viewer, gymapi.KEY_P, "print")
motion_id = 0
motion_acc = set()
while not gym.query_viewer_has_closed(viewer):
    # step the physics

    motion_len = motion_lib.get_motion_length(motion_id).item()
    motion_time = time_step % motion_len

    motion_res = motion_lib.get_motion_state_smpl(
        torch.tensor([motion_id]).to(args.compute_device_id),
        torch.tensor([motion_time]).to(args.compute_device_id))

    root_pos, root_rot, dof_pos, root_vel, root_ang_vel, dof_vel, key_pos, smpl_params, limb_weights, pose_aa, rb_pos, rb_rot, body_vel, body_ang_vel = \
                motion_res["root_pos"], motion_res["root_rot"], motion_res["dof_pos"], motion_res["root_vel"], motion_res["root_ang_vel"], motion_res["dof_vel"], \
                motion_res["key_pos"], motion_res["motion_bodies"], motion_res["motion_limb_weights"], motion_res["motion_aa"], motion_res["rg_pos"], motion_res["rb_rot"], motion_res["body_vel"], motion_res["body_ang_vel"]

    if args.show_axis:
        gym.clear_lines(viewer)
    root_states = torch.cat([root_pos, root_rot, root_vel, root_ang_vel],
                        dim=-1).cpu().repeat(num_envs, 1)
    gym.set_actor_root_state_tensor(sim, gymtorch.unwrap_tensor(root_states))

    gym.simulate(sim)
    gym.refresh_rigid_body_state_tensor(sim)
    gym.fetch_results(sim, True)

    dof_pos = dof_pos.cpu().numpy()

    dof_states['pos'] = dof_pos
    speed = speeds[current_dof]

    # clone actor state in all of the environments
    for i in range(num_envs):
        gym.set_actor_dof_states(envs[i], actor_handles[i], dof_states,
                                 gymapi.STATE_POS)

        if args.show_axis:
            # get the DOF frame (origin and axis)
            dof_handle = gym.get_actor_dof_handle(envs[i], actor_handles[i], current_dof)
            frame = gym.get_dof_frame(envs[i], dof_handle)

            # draw a line from DOF origin along the DOF axis
            p1 = frame.origin
            p2 = frame.origin + frame.axis * 0.7
            color = gymapi.Vec3(1.0, 0.0, 0.0)
            gymutil.draw_line(p1, p2, color, gym, viewer, envs[i])

    # update the viewer
    gym.step_graphics(sim)
    gym.draw_viewer(viewer, sim, True)

    # Wait for dt to elapse in real time.
    # This synchronizes the physics simulation with the rendering rate.
    gym.sync_frame_time(sim)
    time_step += dt
    for evt in gym.query_viewer_action_events(viewer):
        if evt.action == "previous" and evt.value > 0:
            motion_id = (motion_id - 1) % len(motion_keys)
            print(
                f"Motion ID: {motion_id}. Motion length: {motion_len:.3f}. Motion Name: {motion_keys[motion_id]}"
            )
        elif evt.action == "next" and evt.value > 0:
            motion_id = (motion_id + 1) % len(motion_keys)
            print(
                f"Motion ID: {motion_id}. Motion length: {motion_len:.3f}. Motion Name: {motion_keys[motion_id]}"
            )
        elif evt.action == "add" and evt.value > 0:
            motion_acc.add(motion_keys[motion_id])
            print(f"Adding motion {motion_keys[motion_id]}")
        elif evt.action == "print" and evt.value > 0:
            print(motion_acc)
            time_step = 0

gym.destroy_viewer(viewer)
gym.destroy_sim(sim)


"""
Update KKTIX's data if attendee has been refunded
"""
from datetime import datetime, timedelta

from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from ods.kktix_ticket_orders.udfs import kktix_refund

DEFAULT_ARGS = {
    "owner": "henry410213028@gmail.com",
    "depends_on_past": False,
    "start_date": datetime(2023, 4, 10, 4),
    "retries": 2,
    "retry_delay": timedelta(minutes=5),
    "on_failure_callback": lambda x: "Need to send notification to Discord!",
}
dag = DAG(
    "KKTIX_TICKET_REFUND_V2",
    default_args=DEFAULT_ARGS,
    schedule_interval="50 23 * * *",  # At 23:50 (everyday)
    max_active_runs=1,
    catchup=True,
)
with dag:
    UPDATE_REFUNDED_ATTENDEE_IDS = PythonOperator(
        task_id="UPDATE_REFUNDED_ATTENDEE_IDS", python_callable=kktix_refund.main,
    )

    UPDATE_REFUNDED_ATTENDEE_IDS

if __name__ == "__main__":
    dag.cli()


import asyncio
import base64
import json
import pathlib
import sys

import requests
from terra_sdk.client.lcd import LCDClient, Wallet
from terra_sdk.core import Coins
from terra_sdk.core.fee import Fee as StdFee
from terra_sdk.client.lcd.api.tx import CreateTxOptions

from terra_sdk.core.wasm import (MsgExecuteContract, MsgInstantiateContract,
                                 MsgStoreCode)
from terra_sdk.core.wasm.data import AccessConfig
from terra_proto.cosmwasm.wasm.v1 import AccessType

from terra_sdk.key.mnemonic import MnemonicKey
from terra_sdk.util.contract import (get_code_id, get_contract_address,
                                     read_file_as_b64)

from capsule.abstractions.ADeployer import ADeployer
from capsule.lib.credential_handler import get_mnemonic
from capsule.lib.logging_handler import LOG

sys.path.append(pathlib.Path(__file__).parent.resolve())

import enum
from enum import auto
class SupportedChains(enum.Enum):
   TERRA = auto()
   JUNO = auto()
   OSMOSIS = auto()

class Deployer(ADeployer):
    """Deployer is a simple facade object
    providing an interface towards general deployment
    actions such as sending messages, getting addresses,
    uploading code objects, instantiating them into contracts
    and also executing or querying those contracts
    """

    def __init__(self, client, target_chain: enum.Enum = SupportedChains.TERRA) -> None:
        """__init__ takes only a client which is expected to be an already instantiated LCDClient for a network of your choice.
        By default it is expected you will provide a LCDClient configured for use with the Terra Network as this is the original target network.
        In the event you want to use this deployer in a multi-chain sense for any other CosmWasm enabled chain you should also provide a different value for the
        `target_chain` param. Provided the target_chain is a SupportedChain, a relevant chain specific implementation of each of
        the abstractmethods defined in ADeployer should be provided.

        Args:
            client (oneOf terra_sdk.client.lcd.LCDClient | OtherClient): An instantiated LCDClient class for the chain you want to use.
            target_chain (enum.Enum, optional): Optional, used only when you want to target a chain other than Terra such as Juno. Defaults to SupportedChains.TERRA.
        """
        self.target_chain = target_chain
        self.client = client
        self.mnemonic = asyncio.run(get_mnemonic())
        self.deployer = Wallet(lcd=self.client, key=MnemonicKey(self.mnemonic))
        # TODO: Use constants here
        self.std_fee = StdFee(3969390, Coins.from_str("700000uusd"))

    async def send_msg(self, msg):
        """send_msg attempts to create
        and sign a transaction with the provided
        msg and then broadcasts the tx

        """
        tx = self.deployer.create_and_sign_tx(CreateTxOptions(
            msgs=[msg])
        )
        return self.client.tx.broadcast(tx)

    async def store_contract(self, contract_name:str, contract_path:str="") -> str:
        """store_contract attempts to
        gather a given wasm artifact file
        and upload it to the given chain as a StoredCodeObject
        The storage operation is done using the send_msg helper

        Args:
            contract_name (str): The name of the contract to deploy
            contract_path (str, optional): The path to a wasm contract artifact to deploy. Defaults to "".

        Returns:
            str: The contract storage result is parsed and just the code id of the stored code object is returned
        """
        # If the full path was provided, use it else assume its located in artifacts
        LOG.info(contract_path if contract_path else f"artifacts/{contract_name}.wasm")
        bytes = read_file_as_b64(contract_path if contract_path else f"artifacts/{contract_name}.wasm")
        msg = MsgStoreCode(self.deployer.key.acc_address, bytes, instantiate_permission=AccessConfig(permission=AccessType.ACCESS_TYPE_ONLY_ADDRESS, address=self.deployer.key.acc_address))
        contract_storage_result = await self.send_msg(msg)
        LOG.info(contract_storage_result)
        return get_code_id(contract_storage_result)

    async def instantiate_contract(self, code_id: str, init_msg:dict, label: str = "Contract deployed with Capsule") -> str:
        """instantiate_contract attempts to
        instantiate a code object with an init msg
        into a live contract on the network.

        Args:
            code_id (str): The code_id of the stored wasm contract artifact
            init_msg (dict): The init msg to send to setup the contract

        Returns:
            str: The contracts address
        """
        msg = MsgInstantiateContract(
            sender=self.deployer.key.acc_address,
            admin=self.deployer.key.acc_address,
            code_id=code_id,
            msg=init_msg,
            label=label
        )

        instantiation_result = await self.send_msg(msg)

        LOG.info(instantiation_result)
        return get_contract_address(instantiation_result)

    async def execute_contract(self, contract_addr: str, execute_msg, coins = []):
        """Execute a message to perform an action on a given contract, returning the result

        Args:
            contract_addr (str): The contract to execute a msg on
            execute_msg (dict): The msg to execute on the contract
            coins (list, optional): Coins which may be needed for the execution tx. Defaults to [].

        Returns:
            dict: execution results
        """
        msg = MsgExecuteContract(
            sender=self.deployer.key.acc_address,
            contract=contract_addr,
            execute_msg=execute_msg,
            coins=coins
        )
        exe_result = await self.send_msg(msg)
        LOG.debug(exe_result)
        return exe_result

    async def query_contract(self, contract_addr: str, query_msg: dict):
        """Perform a query on a given contract, returning the result

        Args:
            contract_addr (str): The contract to perform the query on
            query_msg (dict): The query to perform

        Returns:
            dict: Query Result
        """
        LOG.info(f"Query to be ran {query_msg}")
        query_result = self.client.wasm.contract_query(contract_addr, query_msg)

        LOG.info(query_result)
        return query_result

    async def query_code_id(self, chain_url: str, code_id: int, target_chain=SupportedChains.TERRA):
        """query_code_id is used to make a REST request to the relevant chain_url
        to specifically query the `code_details` of a given code_id

        Args:
            chain_url (str): The Chain Host of the chain to interact with e.g
            code_id (int): The code_id to query
            target_chain (SupportedChains Enum, optional): Only used when you wish to target a chain other than Terra.
                Each chain can have different methods or URLs to request the same info and this option decides which way to perform queries  Defaults to SupportedChains.TERRA.

        Returns:
            Dict: `code_details` structure containing a hash with a certain encoding.
        """
        LOG.info(f"Query to be ran {code_id}")
        # query_result = self.client.wasm.code_info(code_id)
        # query_two = self.client.wasm._c._get(f"/wasm/codes/{code_id}")
        if target_chain == SupportedChains.TERRA:
            query_raw = requests.get(f"{chain_url}/wasm/codes/{code_id}").json()

        elif target_chain in SupportedChains:
            LOG.info("Chain which should be support")
            if target_chain == SupportedChains.JUNO:
                LOG.info(f"Running query against the Juno chain with Chain URL of {chain_url}")
                query_raw = requests.get(f"{chain_url}/cosmwasm/wasm/v1/code/{code_id}").json()
        else:
            LOG.info("Chain not supported")
        return query_raw

    async def query_code_bytecode(self, chain_url: str, code_id: int, target_chain=SupportedChains.TERRA):
        """query_code_bytecode is used to make a REST request to the relevant chain_url
        to query the actualy bytecode of the stored code object for the given code_id

        Args:
            chain_url (str): The Chain Host of the chain to interact with e.g
            code_id (int): The code_id to query
            target_chain (SupportedChains Enum, optional): Only used when you wish to target a chain other than Terra.
                Each chain can have different methods or URLs to request the same info and this option decides which way to perform queries  Defaults to SupportedChains.TERRA.

        Returns:
            dict: Dictionary containing the byte_code in base64
        """
        LOG.info(f"Query to be ran {code_id}")

        if target_chain == SupportedChains.TERRA:

            # TODO: This query is not cross-chain capable
            query_raw = requests.get(f"{chain_url}/terra/wasm/v1beta1/codes/{code_id}/byte_code").json()
            LOG.info(query_raw)
        elif target_chain in SupportedChains:
            LOG.info("Chain which should be support")
            if target_chain == SupportedChains.JUNO:
                LOG.info(f"Running query against the Juno chain with Chain URL of {chain_url}")
                query_raw = requests.get(f"{chain_url}/cosmwasm/wasm/v1/code/{code_id}").json()
                # In order to maintain similar outputs, cut off everything but the datahere
                query_raw = {
                    "byte_code": query_raw["data"]
                }
        else:
            LOG.info("Chain not supported")
        return query_raw


import json
import pandas as pd
import numpy as np
import copy
import os
import geopandas as gpd

def get_pre_ripa_data(cols_cfgs, **cfgs):

    base_url, description_url, race_url = [
      "http://seshat.datasd.org/pd/vehicle_stops_2014_datasd_v1.csv",
      "http://seshat.datasd.org/pd/vehicle_stops_search_details_2014_datasd.csv",
      "http://seshat.datasd.org/pd/vehicle_stops_race_codes.csv"
    ]

    years, races, cols, outdir = copy.deepcopy(cfgs['years']), cfgs['races'], cfgs['pre-RIPA_cols'], cfgs['outdir']
    if not os.path.exists(outdir):
        os.makedirs(outdir)

    stop_df = pd.DataFrame()

    # if years includes only 2019, no pre-ripa data needed
    if min(years) > 2018:
        return

    # 2018 data is located in the 2017 csv file, handle case when 2017 is not in years but 2018 is
    if 2018 in years:
        years.remove(2018)
        if 2017 not in years:
            years.append(2017)

    if 2019 in years:
        years.remove(2019)

    # map how race is indicated in config file to how it is indicated in dataset (e.g., white -> W)
    race_mapper = {'white':'W','black':'B','hispanic':'H'}
    if races[0] == 'all':
        race_filter = []
    else:
        race_filter = [race_mapper[r] for r in races]

    # indicate which columns belong to the basic details dataset, and which belong to search details dataset
    main_cols = list(set(cols_cfgs['pre-ripa'].keys()).intersection(set(cols)))
    detail_cols = list(set(cols_cfgs['pre-ripa_details'].keys()).intersection(set(cols)))

    for year in set(years):

        # Get urls for stop data from year
        url = base_url.replace('2014',str(year))
        description_url_c = description_url.replace('2014',str(year))
        df = pd.read_csv(url,usecols = main_cols)

        default_date = '{}-12-31 23:39:00'.format(year)
        df['date_time'].fillna(default_date)

        # handle case when either 2017 or 2018 is in dataset, but not both
        if year == 2017:
            if 2017 not in cfgs['years']:
                df = df[df['date_time'].apply(lambda x: pd.to_datetime(x).year) == 2018]
            elif 2018 not in cfgs['years']:
                df = df[df['date_time'].apply(lambda x: pd.to_datetime(x).year) == 2017]

        # if race is to be included and not all races are needed, filter by neede races
        # join race description to basic details dataset
        if 'subject_race' in df.columns:
            if races[0] != 'all':
                df = df[df.subject_race.isin(race_filter)]
            df_race = pd.read_csv(race_url)
            df = df.merge(df_race,left_on = 'subject_race',right_on='Race Code')

        # handle when no columns from details dataset are specified in config file
        if len(detail_cols) > 1:
            df_details = pd.read_csv(description_url_c,usecols=detail_cols)
            df = df.merge(df_details,on='stop_id')
        stop_df = pd.concat([stop_df,df])

    stop_df.reset_index(inplace=True,drop=True)
    file_path = os.path.join(outdir,'pre_ripa.csv')
    stop_df.to_csv(file_path,index=False)

def get_ripa_data(cols_cfgs, **cfgs):

    years, races, r_cols, outdir = copy.deepcopy(cfgs['years']), cfgs['races'], cfgs['RIPA_cols'], cfgs['outdir']
    stop_df = pd.DataFrame()

    if not os.path.exists(outdir):
        os.makedirs(outdir)

    # If no data from 2018 or 2019 is needed, no data from ripa datasets are needed
    if max(years) < 2018:
        return

    filter_years = list(set(years).intersection(set([2018,2019])))

    RIPA_urls = [
     "http://seshat.datasd.org/pd/ripa_stops_datasd_v1.csv",
     "http://seshat.datasd.org/pd/ripa_actions_taken_datasd.csv",
     "http://seshat.datasd.org/pd/ripa_contraband_evid_datasd.csv",
     "http://seshat.datasd.org/pd/ripa_prop_seize_type_datasd.csv",
     "http://seshat.datasd.org/pd/ripa_race_datasd.csv",
     "http://seshat.datasd.org/pd/ripa_stop_reason_datasd.csv",
     "http://seshat.datasd.org/pd/ripa_stop_result_datasd.csv"
    ]

    # map how race is indicated in config file to how it is indicated in dataset (e.g., white -> W)
    race_mapper = {'white':'White','black':'Black/African American',
    'hispanic':'Hispanic/Latino/a','other':['Native American','Pacific Islander'],
    'asian':['Middle Eastern or South Asian','Asian']}
    # create list of races to filter by
    race_filter = []
    if races[0] != 'all':
        for r in races:
            if r in ['asian','other']:
                race_filter.append(race_mapper[r.lower()][0])
                race_filter.append(race_mapper[r.lower()][1])
            else:
                race_filter.append(race_mapper[r.lower()])

    for url, cols in zip(RIPA_urls,cols_cfgs.items()):
        # specify which cols are needed from each dataset
        usecols  = list(set(r_cols).intersection(set(cols[1].keys())))
        df = pd.read_csv(url,usecols=usecols)
        # If years don't include both 2018 and 2019, filter out unneeded year
        if ('date_stop' in df.columns) and (len(filter_years) != 2):
            df = df[df.date_stop.apply(lambda x: pd.to_datetime(x).year).isin(filter_years)]
        # filter out unneeded races
        if 'race' in df.columns:
            if races[0] != 'all':
                df = df[df.race.isin(race_filter)]
        if stop_df.empty:
            stop_df = df.copy()
        else:
            stop_df = stop_df.merge(df,on=['stop_id','pid'])

    stop_df.reset_index(inplace=True,drop=True)

    file_path = os.path.join(outdir,'ripa.csv')

    stop_df.to_csv(file_path,index=False)

# Read in service area data and write to file
def get_service_area_info(**cfgs):

    pd_beats_url = 'http://seshat.datasd.org/sde/pd/pd_beats_datasd.geojson'
    sa = gpd.read_file(pd_beats_url)
    outdir = cfgs['outdir']
    if not os.path.exists(outdir):
        os.makedirs(outdir)
    file_path = os.path.join(outdir,'service_area.geojson')
    sa.to_file(file_path,driver="GeoJSON")

def get_stops_data(cols_cfgs, **cfgs):
    get_pre_ripa_data(cols_cfgs, **cfgs)
    get_ripa_data(cols_cfgs, **cfgs)
    get_service_area_info(**cfgs)