from inspect import getmembers, isclass

import torch
from pytorch_lightning import Trainer
from torch.utils.data import DataLoader

import kedro_image_classification.pytorch.models as models
from kedro_image_classification.pytorch.tasks import ClassificationTask

def create_model(model_architectures: dict, model_selection: dict) -> torch.nn.Module:
    """Function creating model from predefined architectures
    and types based on the selection. Checks whether the selected model
    is in the predefined model architectures and if the model type is defined.

    Args:
        model_architectures (dict): Dictionary with all defined model architectures.
        model_selection (dict): Dictionary with the model selection.

    Raises:
        KeyError: Raised when the selected model is not in the model_architectures.
        AttributeError: Raised when type of the model from the architecture
        is not predefined.

    Returns:
        torch.nn.Module: Created model object.
    """
    try:
        model_config = model_architectures[model_selection["model_name"]]
    except KeyError:
        predefined_model_names = list(model_architectures.keys())
        raise KeyError(
            f"Selected non existing model name: '{model_selection['model_name']}'. "
            f"Predefined model names are: {predefined_model_names}."
            f"You can add your own custom model by adding its configuration"
            f"to the model_architectures.yml file."
        )

    try:
        model_constructor = getattr(models, model_config["model_type"])
    except AttributeError:
        predefined_model_types = [
            name for name, obj in getmembers(models) if isclass(obj)
        ]
        raise AttributeError(
            f"Specified unsupported model type. "
            f"Predefined model types are: {predefined_model_types}. "
            f"You can add your own custom pytorch model by adding its code to the "
            f"kedro_image_classification.pytorch.models"
        )

    return model_constructor(model_config)

def create_trainer(trainer_params: dict) -> Trainer:
    """Function creating pytorch_lightning trainer with given parameters.

    Args:
        trainer_params (dict): Dictionary with parameters for the trainer object.

    Returns:
        Trainer: Created pytorch_lightning trainer object.
    """
    trainer = Trainer(
        default_root_dir="data/06_models",
        accelerator=trainer_params["accelerator"],
        # logger=wandb_logger,
        log_every_n_steps=trainer_params["log_every_n_steps"],
        val_check_interval=trainer_params["val_check_interval"],
        devices=trainer_params["devices"],
        max_epochs=trainer_params["max_epochs"],
        # plugins=DDPPlugin(find_unused_parameters=False),
    )
    return trainer

def create_task(model: torch.nn.Module, optimizer_params: dict) -> ClassificationTask:
    """Function creating classification task (pytorch_lightning lightning module)
    from model (created in the previous pipeline part) and provided parameters
    of the optimizer.

    Args:
        model (torch.nn.Module): Model created in the create_model node.
        optimizer_params (dict): Dictionary with parameters for the optimizer.

    Returns:
        ClassificationTask: Lightning module with functions required for model training.
    """
    return ClassificationTask(model, optimizer_params)

def train(
    trainer: Trainer,
    task: ClassificationTask,
    train_loader: DataLoader,
    test_loader: DataLoader,
) -> Trainer:
    """Function performing training of the model.

    Args:
        trainer (Trainer): PyTorch Lightning trainer which will perform
        the task of training.
        task (ClassificationTask): The model to train wrapped in Classification task.
        train_loader (DataLoader): Loader of the train set.
        test_loader (DataLoader): Loader of the test set.

    Returns:
        Trainer: Given trainer after the training process
    """
    trainer.fit(task, train_loader, test_loader)
    return trainer


from typing import Tuple, List, TypeVar

T = TypeVar("T")

def orderForLeastMotorMovement(positions: List[Tuple[T, Tuple[float, float]]], startposition: Tuple[float, float]) -> List[T]:
    xpositions = set([p[1][0] for p in positions] + [startposition[0]])
    ypositions = set([p[1][1] for p in positions] + [startposition[1]])

    if len(xpositions) < len(ypositions):
        # there are more unique Y coordinates than X coordinates: go by X coordinates first
        slow = 0
        fast = 1
    else:
        slow = 1
        fast = 0
    # put the slowest moving sample (not starting position!) coordinates first in increasing order
    slow_ordered = sorted(
        set([p[1][slow] for p in positions if p[1][slow] != startposition[slow]]))
    if not slow_ordered:
        # only one position, which is the start position:
        slow_ordered = []
    else:
        # see which end we must start. Start from that end which is nearest to the empty beam measurement
        if abs(slow_ordered[-1] - startposition[slow]) < abs(slow_ordered[0] - startposition[slow]):
            slow_ordered = reversed(slow_ordered)
        slow_ordered = list(slow_ordered)
    lastfastcoord = startposition[fast]
    objects_ordered = []
    for slowpos in [startposition[slow]] + slow_ordered:
        # sort those samples which have this X coordinate first by increasing Y coordinate
        objects = sorted([p for p in positions if p[1][slow] == slowpos],
                         key = lambda p:p[1][fast])
        if not objects:
            # no samples with this slow coordinate
            continue
        # see which end of the fastest coordinate is nearest to the last fast coordinate position
        if abs(objects[0][1][fast] - lastfastcoord) > abs(objects[-1][1][fast] - lastfastcoord):
            objects = reversed(objects)
        objects = list(objects)
        objects_ordered.extend(objects)
        lastfastcoord = objects_ordered[-1][1][fast]
    assert len(objects_ordered) == len(positions)
    return [o[0] for o in objects_ordered]


import json

class Rate:
    """
    Class used to determine the correct taxation of a product.
    """
    class __Rate:
        def __init__(self):
            rate_file="../data/rate.json"
            with open(rate_file) as f:
                self.rate_data = json.load(f)

    instance = None

    def __init__(self):
        if not Rate.instance:
            Rate.instance = Rate.__Rate()

    def get_rate(self, group=None, imported=False):
        """
        Returns the right rate based on the group to which it belongs
        and whether it is imported or not.

        The correct rate is made up of a fixed part and one dependent
        on the possible import.

        For both the fixed part and the part dependent on the import,
        the quota is decided on the basis of the group.

        Returns
        -------

        the percentage resulting from the sum of the two rates.
        """
        if group in self.instance.rate_data['fix']:
            fix_rate = self.instance.rate_data['fix'][group]
        else:
            fix_rate = self.instance.rate_data['fix']['default']

        if imported:
            if group in self.instance.rate_data['imported']:
                imported_rate = self.instance.rate_data['imported'][group]
            else:
                imported_rate = self.instance.rate_data['imported']['default']
        else:
            imported_rate = 0

        total_rate = fix_rate + imported_rate

        return total_rate

class Product:
    """
    A class used to represent a Product of Cart.

    Attributes
    ----------
    label : str
        product name
    group : str
        the group to which the product belongs
    price : float
        product price without taxes
    imported : bool
        is true if the product is imported
    count : int
        quantity in the cart

    """

    def __init__(self, label=None, group=None, price=0, imported=False, count=0):
        self.label = label
        self.group = group
        self.price = price
        self.imported = imported
        self.count = count

    def get_tax(self):
        """
        Calculates the amount of tax resulting from the rate applied.

        The tax is rounded up to the nearest 0.5.
        """
        rate = Rate()
        rate_calculated = rate.get_rate(group=self.group, imported=self.imported)
        if rate_calculated > 0:
            n_tax =  self.price * rate_calculated

            if n_tax % 5 > 0:
                # This step allows me to apply the desired rounding
                # by working on integer values.
                n_tax = n_tax + (5 - (n_tax % 5))

            rounded_tax = n_tax / 100
            return rounded_tax
        else:
            return 0

    def __str__(self):
        """
        Allows the Product to be printed as formatted in the exercise description.
        """
        return '{count} {imported}{label}: {price:0.2f}'.format(
            count = self.count,
            imported = 'imported ' if self.imported else '',
            label = self.label,
            price = self.price + self.get_tax()
        )

class Cart:
    """
    A class used to represent a Cart.

    Attributes
    ----------
    products : list
        list of products in the cart
    sales_taxes : float
        sum of taxes
    total : float
        sum of taxed prices
    """
    def __init__(self):
        self.products = list()
        self.sales_taxes = 0
        self.total = 0

    def add_product(self, product):
        """
        Adds a given product to your cart and updates tax and total summaries.
        """
        self.sales_taxes = self.sales_taxes + product.get_tax() * product.count
        self.total = self.total + (product.price + product.get_tax()) * product.count
        self.products.append(product)

    def __str__(self):
        """
        Allows the Cart to be printed as formatted in the exercise description.
        """
        p_str_list = list()
        for p in self.products:
            p_str_list.append(str(p))

        p_str_list.append('Sales Taxes: {sales_taxes:0.2f}'.format(sales_taxes = self.sales_taxes))
        p_str_list.append('Total: {total:0.2f}'.format(total = self.total))
        return '\n'.join(p_str_list)


import json
import hashlib
import logging
import os
import sys
import tempfile
from subprocess import check_output

log = logging.getLogger(__name__)

BASE_URL = "https://pypi.python.org"

def get_hash(url):
    # the file name is at the end after the last slash
    file_name = url.rsplit("/", 1)[-1]
    for fl in files:
        if fl == file_name:
            with open(path + "/" + fl, 'rb') as r_f:
                return hashlib.sha256(r_f.read()).hexdigest()

def construct_json(pkg_name, rel_url):
    url_pkg = BASE_URL + relative_url
    data_json = {'name': pkg_name, 'build-options': {}, 'sources': [{}, {}, {}]}
    data_json['build-options']['env'] = {}
    data_json['build-options']['env']['PYTHONUSERBASE'] = '/app'
    data_json['sources'][0]['type'] = 'archive'
    data_json['sources'][0]['url'] = url_pkg
    file_hash = get_hash(rel_url)
    data_json['sources'][0]['sha256'] = file_hash
    data_json['sources'][1]['type'] = 'script'
    data_json['sources'][1]['dest-filename'] = 'configure'
    data_json['sources'][1]['commands'] = ["echo just so that a configure file exists"]
    data_json['sources'][2]['type'] = "file"
    data_json['sources'][2]['path'] = "pip-Makefile"
    data_json['sources'][2]['dest-filename'] = "Makefile"
    return data_json

if __name__ == "__main__":
    path = tempfile.TemporaryDirectory().name
    print("Downloading and checking the required dependencies.\n"
          "Depending on your Internet connection and CPU this operation may take a while.\n"
          "Please wait.")
    data_log = check_output("pip3 -v --no-cache-dir download " + " ".join(sys.argv[1:]) +
                            " -d " + path + " --no-binary :all:", shell=True)
    data_log = data_log.decode()

    files = os.listdir(path)
    for file in files:
        log.info("Downloaded %s", file)

    packages = []
    packages_ordered = []

    for line in data_log.splitlines():
        if "Collecting" in line:
            # Save only the package name
            package = line.split()[1]
            packages.append(package)
            # Remove the Collecting word
            package_full = line.rsplit(" ", 1)[1]
            packages_ordered.append(package_full)

    json_output = []

    for line in data_log.splitlines():
        if "GET /packages" in line:
            for package in packages:
                # Simply remove any extra information.
                # The packages versions that we have were already
                # checked with pip
                pkg, _, _ = package.partition("!")
                pkg, _, _ = pkg.partition("=")
                pkg, _, _ = pkg.partition(">")
                pkg, _, _ = pkg.partition("<")
                pkg, _, _ = pkg.partition("[")
                if pkg.lower() in line.lower():
                    relative_url = line.split()[1]
                    pkg_json = construct_json(pkg, relative_url)
                    json_output.append(pkg_json)
                    # remove the found package
                    packages.remove(package)
                    break

    # this is not enough because the same package may be a dependency
    # for multiple other packages, and the installing order is important
    json_output.reverse()

    with open("output.json", "w") as output:
        json.dump(json_output, output, indent=4)


# -*- coding: utf-8 -*-
import numpy as np
import pandas as pd
import seaborn
import csv
import matplotlib
matplotlib.use('agg')
### size and font of the writtings in the plots
font = {'family' : 'normal',
        'weight' : 'bold',
        'size'   : 15}

matplotlib.rc('font', **font)
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA

def load_csv_no_header(filePath):
    """
    Load data of csv file into a matrix. The csv file is only composed of data, no header.
    Input:
        filePath : file path to the .csv file
    Outputs:
        mergeAll: matrix of float with the data in the file path.
        nrRows: int, number of rows of the matrix
        nrCols: int, number of columns of the matrix
    """
    with open(filePath, 'r') as f:
        reader = csv.reader(f, delimiter = ',', quotechar = "'")
        rows = [row for row in reader]
        nrRows = len(rows)
        nrCols = len(rows[0])
        mergeAll = np.ndarray((nrRows, nrCols), dtype=np.float32)
        for r in range(nrRows):
            mergeAll[r,:] = [word for word in rows[r]]
    return mergeAll, nrRows, nrCols

path_dataset_matrix ="gcn_u_train_emb128_synthnoteasy_nonorm.csv"#"gcn_u_train_emb217_tadpole_nonorm.csv"
M_str, nrRows, nrCols = load_csv_no_header(path_dataset_matrix)
gender_file_path = "gender_column_train_synthnoteasy.csv"#"gender_column_train_tadpole217.csv"
gender_column, nrRowsg, nrColsg = load_csv_no_header(gender_file_path)
gender_column=gender_column.reshape(-1)

age_file_path ="age_column_train_synthnoteasy.csv"#"age_column_train_tadpole217.csv"
age_column, nrRowsa, nrColsa = load_csv_no_header(age_file_path)
age_column=age_column.reshape(-1)

path_labels="labels_column_train_synthnoteasy.csv"#"labels_column_train_tadpole217.csv"
labels, _, _ = load_csv_no_header(path_labels)
labels= labels.reshape(-1)

hidden_a=61 #99
X = np.concatenate((age_column[:, np.newaxis], gender_column[:, np.newaxis]), axis=1) #covariates

age_max_int = 92
age_min_int = 84

for i in range(M_str.shape[1]//hidden_a):
    # i different support matrix

    #first step : construct the PCA decomposition
    y=[]
    print(hidden_a*i)
    print(hidden_a*i + hidden_a)
    for j in range(M_str.shape[0]):
        y.append(M_str[j][i*hidden_a:hidden_a*i+hidden_a])
    y =np.array(y)
    print(y.shape)
    pca = PCA(n_components=2)
    X_new = pca.fit_transform(y)

    #split between subjects that are MCIc and MCInc
    labels_zeros= np.where(labels==0)
    labels_ones= np.where(labels==1)

    plt.figure()
    plt.title('PCA support %i' %i)

    if (i-2)%3==0 and i!=2:
        #After 3 support, except for the 2 first one, the age range changes
        age_max_int = age_min_int
        age_min_int = age_min_int-5

    if i == 0:
        #Women support
        zero_class = np.where(gender_column==1)
        zero_class_label0=np.intersect1d(zero_class, labels_zeros)
        zero_class_label1=np.intersect1d(zero_class, labels_ones)
        one_class = np.where(gender_column==0)
    elif i==1:
        #Men support
        zero_class = np.where(gender_column==0)
        zero_class_label0=np.intersect1d(zero_class, labels_zeros)
        zero_class_label1=np.intersect1d(zero_class, labels_ones)
        one_class = np.where(gender_column==1)
    elif (i-2)%3==0:
        #Women and age
        gender = np.where(gender_column==1)
        age_min = np.where(age_column>=age_min_int)
        age_max = np.where(age_column<=age_max_int)
        age_f = np.intersect1d(age_min, age_max)
        zero_class = np.intersect1d(gender, age_f)
        zero_class_label0=np.intersect1d(zero_class, labels_zeros)
        zero_class_label1=np.intersect1d(zero_class, labels_ones)
        new_ind = np.arange(len(gender_column))
        one_class=np.delete(new_ind, zero_class)
        print(len(zero_class))
    elif (i%3)==0:
        #Men and age
        gender = np.where(gender_column==0)
        age_min = np.where(age_column>=age_min_int)
        age_max = np.where(age_column<=age_max_int)
        age_f = np.intersect1d(age_min, age_max)
        zero_class = np.intersect1d(gender, age_f)
        zero_class_label0=np.intersect1d(zero_class, labels_zeros)
        zero_class_label1=np.intersect1d(zero_class, labels_ones)
        new_ind = np.arange(len(gender_column))
        one_class=np.delete(new_ind, zero_class)
    elif (i-4)%3==0:
        # age
        age_min = np.where(age_column>=age_min_int)
        age_max = np.where(age_column<=age_max_int)
        zero_class = np.intersect1d(age_min, age_max)
        zero_class_label0=np.intersect1d(zero_class, labels_zeros)
        zero_class_label1=np.intersect1d(zero_class, labels_ones)
        new_ind = np.arange(len(gender_column))
        one_class=np.delete(new_ind, zero_class)

    plt.scatter(X_new[zero_class_label0, 0], X_new[zero_class_label0, 1],s=160, edgecolors='b',
               facecolors='none', linewidths=2, label='Support 1 label 0')
    plt.scatter(X_new[zero_class_label1, 0], X_new[zero_class_label1, 1],s=160, edgecolors='r',
               facecolors='none', linewidths=2, label='Support 1 label 1')
    plt.scatter(X_new[one_class, 0], X_new[one_class, 1], s=80, edgecolors='orange',
               facecolors='none', linewidths=2, label='Support 0')
    plt.legend()
    plt.savefig('embeddings_tadpole_test/pca_embedding%i_features_labels.pdf' %i )
    plt.show()


import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"]="0,1,2,3"
import logging
logging.getLogger('tensorflow').setLevel(logging.ERROR)

import argparse
import pandas as pd
import numpy as np
from tqdm.auto import tqdm as tq
import tensorflow as tf
from tensorflow.keras import mixed_precision
import tensorflow_model_optimization as tfmot # pip install 필요!
import wandb

# private
from modules.model import A2IModel
from cfg import configs
import functions

if __name__ == '__main__':
    # argument parsing : [ cluster, backbone type (densenet, convnext), head ]
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--cluster', action='store', default='akmu')
    parser.add_argument('-f', '--backbone', action='store', default='densenet')
    parser.add_argument('-a', '--add_expert', action='store_true')
    parser.add_argument('-e', '--epochs', action ='store', type=int, default=5)
    parser.add_argument('-b', '--batch', action='store', type=int, default=16)
    parser.add_argument('-p', '--progress_bar', action ='store_false')

    # for debugging
    parser.add_argument('-t', '--test', action='store_true')
    parser.add_argument('-s', '--single_gpu', action ='store_true')
    parser.add_argument('-w', '--wandb_off', action='store_false')

    args = parser.parse_args()
    configs.general.progress_bar = 1 if args.progress_bar==True else 2 # one line per epoch
    cluster = args.cluster
    configs.dataset.data_dir = "."
    # if cluster == 'gsds-ab':
    #     configs.dataset.data_dir = '/home/n1/gyuseonglee/workspace/datasets/chexpert-resized'
    # if cluster == 'gsds-c':
    #     configs.dataset.data_dir = '/home/gyuseonglee/workspace/dataset/chexpert-resized'
    # if cluster == 'akmu':
    #     configs.dataset.data_dir = '/data/s1/gyuseong/chexpert-resized'
    configs.model.backbone = args.backbone
    configs.model.classifier.add_expert = bool(args.add_expert)
    configs.dataset.cutoff = 500 if args.test == True else None
    configs.wandb.use_wandb = False
    configs.wandb.run_name = f'a2i-{configs.model.backbone}-{configs.dataset.image_size[0]}'
    configs.general.distributed = True if args.single_gpu == False else False
    configs.general.epochs = int(args.epochs)
    configs.general.batch = int(args.batch)
    configs.saved_model_path = "./" + configs.model.backbone + "_best_model_{epoch:02d}-{val_loss:.2f}.h5"

    # wandb initialization
    if configs.wandb.use_wandb == True :
        wandb.login()
        run = wandb.init(project=configs.wandb.project_name, name=configs.wandb.run_name, config=configs)

    # multi-gpu training strategy
    strategy = tf.distribute.MirroredStrategy()
    num_devices = strategy.num_replicas_in_sync
    configs.general.batch_size = configs.general.batch_size*num_devices # global batch size

    # mixed precision policy
    policy = mixed_precision.Policy('mixed_float16' if configs.general.precision == 16 else 'float32')
    mixed_precision.set_global_policy(policy)

    # load datasets
    train_dataset, valid_dataset = functions.load_datasets(configs)
    configs.general.steps_per_epoch = train_dataset.steps_per_epoch

    # settings
    if args.single_gpu == False:
        with strategy.scope():
            model, callbacks = functions.set_model_callbacks(
                model_class=A2IModel,
                configs=configs
            )
    else:
        model, callbacks = functions.set_model_callbacks(
            model_class=A2IModel,
            configs=configs
        )

    # pruning
    with strategy.scope():
        model.load_weights('final_densenet_320.h5') # base model weight 불러오기
        layers = [layer for layer in model.layers]
        model = tf.keras.Sequential(layers)

        print("MODEL LOAD SUCCESS!")

        # model.summary()

        prune_low_magnitude = tfmot.sparsity.keras.prune_low_magnitude
        target_sparsity = 0.80 # !!!!!!!!!!!!!!!!!! 0.80 /0.90 / 0.95 세팅 바꿔서 돌리기 configuration 반영 플리즈!

        end_step = np.ceil(len(train_dataset) / configs.general.batch_size).astype(np.int32) * configs.general.epochs

        pruning_params = {
                'pruning_schedule': tfmot.sparsity.keras.PolynomialDecay(initial_sparsity=0.50,
                                                                        final_sparsity=target_sparsity,
                                                                        begin_step=0,
                                                                        end_step=end_step)
        }

        def apply_pruning_to_dense(layer):
            if isinstance(layer, tf.keras.layers.Dense):
                return tfmot.sparsity.keras.prune_low_magnitude(layer)
            if isinstance(layer, tf.keras.layers.Conv2D):
                return tfmot.sparsity.keras.prune_low_magnitude(layer)
            return layer

        model_for_pruning = tf.keras.models.clone_model(
                model,
                clone_function=apply_pruning_to_dense,
            )

        # recompile
        model_for_pruning, callbacks = functions.set_model_callbacks(model_class=A2IModel,
                                                                    configs=configs
                                                                    )
        # model_for_pruning.summary()
        print("PRUNING RECOMPILE DONE!")
        print("START TRAINING...")
        model_for_pruning.fit(
            train_dataset,
            epochs=configs.general.epochs,
            validation_data=valid_dataset,
            callbacks=callbacks,
            workers=configs.general.num_workers,
            verbose=configs.general.progress_bar,
            shuffle=True,
        )

        model.summary()

        model.save("pruned_{target_sparsity}.h5", save_format='h5')


import pandas as pd
import umap
import numpy as np
import copy
import matplotlib.pyplot as plt
from pylab import rcParams
rcParams['figure.figsize'] = 16, 10
from PIL import Image,ImageChops,ImageDraw
from sklearn.cluster import KMeans,SpectralClustering
from mpl_toolkits.axes_grid1 import make_axes_locatable
from matplotlib.colors import ListedColormap,from_levels_and_colors
from mpl_toolkits.axes_grid1 import make_axes_locatable
from matplotlib.patches import Patch
from matplotlib.pyplot import imshow
from multiprocessing import Pool
import seaborn as sns
N_cluster=8
cmap_k=sns.color_palette('muted', N_cluster)

# IMG_PATH='/home/swei20/cancerHH/AceCanZ/cancer/lbl_data/M21_1_[46163,12853]_label.png'

IMG_PATH='/home/swei20/cancerHH/AceCanZ/cancer/lbl_data/M21_1_[46163,13653]_label.png'
lbl = np.array(Image.open(IMG_PATH))
lbl[lbl==192]=128
l128=1*((lbl==128)| (lbl==192))
l64=1*(lbl==64)

def get_kmean_cluster_plot(HH_pdh,pred_k,N_cluster,cmap_k):
    f, (ax0,ax1)=plt.subplots(1,2,figsize=((16,5)))
    sns.scatterplot('u1','u2',data=HH_pdh,hue=f'k{N_cluster}', marker='x',s=5, palette=cmap_k,legend="full",ax = ax0)
    cmap, norm =from_levels_and_colors(list(range(N_cluster+2)),[[0.99,0.99,0.99,1]]+cmap_k)
    im=ax1.imshow(pred_k, cmap=cmap, norm=norm)
    divider = make_axes_locatable(ax1)
    cax = divider.append_axes('right', size='5%', pad=0.05)
    f.colorbar(im, cax=cax)

def process_kmean(exact_pdh,  N_clusters = [3,10], u1 = 'u1', u2 = 'u2', k_cluster = 'kmean', weight=None):
    k_names = []
    umap_result = exact_pdh.loc[:,[u1, u2 ]].values
    l_cluster = len(N_clusters)
    f, axes = plt.subplots(1,l_cluster,figsize= (16,5) )
    for i in range(l_cluster):
        N_cluster = N_clusters[i]
        k_name = f'k{N_cluster}'
        k_names += [k_name]
        kmeans = KMeans(n_clusters=N_cluster,n_init=30, algorithm='elkan')
        kmeans.fit(umap_result, sample_weight = weight)
        exact_pdh[k_name] = kmeans.labels_ +1
        sns.scatterplot(
            x=u1, y=u2,
            hue= k_name , marker='x',s=5,
            palette=sns.color_palette("muted", N_cluster),
            data=exact_pdh,
            legend="full",
            ax = axes[i]
            # alpha=0.3
            )
    # print(exact_pdh.loc[[0,1]])
    return k_names,kmeans

def get_intensity_hist(intensity,N_bins = 100 ,N_sigma = 3,ax=None):
    if ax is None: ax=plt.gca()
    para = np.log(intensity[intensity > 0])
    histdata = plt.hist(para, bins = N_bins, color = 'gold')
    x, y = histdata[0], histdata[1][:-1]
    y = (y[1]-y[0])/2 + y
    assert len(x) == len(y)
    x_max =  np.max(x)
    x_half = x_max//2
    mu = y[x == x_max]
    sigma = abs(y[abs(x - x_half).argmin()] -mu)
    cutoff_log = N_sigma* sigma + mu
    cutoff = int(np.exp(cutoff_log))
    peak_val = int(np.exp(mu))
    lower_sig = int(np.exp(mu - sigma))
    upper_sig = int(np.exp(mu + sigma))
    ax.axvline(mu, color = 'b', label = f'Peak = {peak_val}')
    ax.axvline(mu - sigma, color = 'cyan', ls = ':', label = f'-sigma = {lower_sig}')
    ax.axvline(mu+sigma, color = 'cyan', ls = ':', label = f'+sigma = {upper_sig}')
    ax.axvline(cutoff_log, color = 'r', label = f'{N_sigma}sigma = exp{np.round(cutoff_log,1)} =  {cutoff}' )
    ax.axhline(x_half, color = 'cyan', ls = ':')
    ax.set_title('Histogram of Intensity Cutoff ')
    ax.set_xlabel('log(intensity)')
    ax.legend()
    return cutoff

def plot_normal(ax=None,lbl=l64,c='deepskyblue',a=1):
    if ax is None: ax=plt.gca()
    ax.contour(lbl, colors=c, alpha=a, linewidths=0.5)
    return None
def plot_cancer(ax=None,lbl=l128,c='crimson',a=1):
    if ax is None: ax=plt.gca()
    ax.contour(lbl, colors=c, alpha=a,linewidths=0.5)
    return None

def plot_CN_all(pred,idxs, l128,l64, cmaps):
    l=len(idxs)
    f, axs=plt.subplots(l,2,figsize=(20,8*l))
    for ii, (ax0,ax1) in enumerate(axs):
        mat= (pred==idxs[ii])
        ax0.matshow(mat,cmap=ListedColormap([[0.95,0.95,0.95,1], cmaps[ii]]))
        plot_normal(ax=ax0,lbl=l64,c='deepskyblue',a=1)
        ax1.matshow(mat,cmap=ListedColormap([[0.95,0.95,0.95,1], cmaps[ii]]))
        plot_cancer(ax=ax1,lbl=l128,c='crimson',a=1)
    return None
def plot_2_clusters_1(final_c10, idx1 =1, idx2 = 7):
    s1 = (final_c10 == idx1)*(1)
    s2 = (final_c10 == idx2)* 1
    masked = s1 - s2
    plt.matshow(np.reshape(masked, (1004,1344)))

def get_rank(HH_pd):
    HH_pd['rk']=HH_pd['freq'].cumsum()
    plt.figure(figsize=(8,6))
    plt.plot(HH_pd['rk']/HH_pd['rk'].values[-1])
    plt.xlabel('rank')
    plt.ylabel('ratio')
    plt.ylim(0,1)
    plt.grid()

def get_umap_pd(ftr_pd, ftr, n_comp=2):
    umapT = umap.UMAP(n_components=n_comp,min_dist=0.0,n_neighbors=50,metric='euclidean',random_state=1178)
    try: df_umap=ftr_pd[ftr]
    except: df_umap=ftr_pd[list(map(str,ftr))]
    umap_result = umapT.fit_transform(df_umap.values)
    for ii in range(n_comp):
        ftr_pd[f'u{ii+1}'] = umap_result[:,ii]
    sns.scatterplot('u1','u2', data=ftr_pd, marker='x',s=10, color='k')
    return umapT

def plot_clusters(stream_1D, mask, pd,  k_cluster, val = 'val', bg = -1, color = 0, sgn = 1 ):
    mask=mask.astype('bool')
    HHvals = np.array(pd[val])
    HHcluster = np.array(pd[k_cluster])
    masked_streams = np.zeros(np.shape(stream_1D))
    for idx, val in enumerate( HHvals):
        label = HHcluster[idx]
        masked_streams = np.where(stream_1D != val, masked_streams, color +sgn * label)
    final_umap = np.ones(mask.shape) * bg
    final_umap[mask] = masked_streams
    return final_umap, masked_streams

def get_conf_mat(pred, target):
    pred=pred.reshape((1004,1344))
    conf_mat = pd.DataFrame(index=np.unique(pred), columns=np.unique(target))
    conf_mat[:] = 0
    for i in np.unique(pred):
        for j in np.unique(target):
            conf_mat.loc[i,j] = np.sum((pred == i)&(target==j))
    print(conf_mat)
#     conf_mat['ratio']=conf_mat.apply(lambda x: x[[64]]/x[[128]])
    return conf_mat

def get_freq_aug(df):
    freq=df['freq'].values
    df['FN']=freq/freq[-1]
    df['FR']=df['FN'].apply(lambda x: 1+np.floor(np.log2(x)))
#     df['RR']=1+np.floor(np.log2(freq/freq[-1]))
    return freq

def get_aug_pd(df,ftr,mode='FR'):
    data=df[ftr].values
    freqN =df[mode].values
    freqInt=freqN.astype('int')
    plt.figure(figsize=(5,5))
    _=plt.hist(freqInt,bins=freqInt[0])
    data_aug=data[0]
    freq_list=[]
    np.random.seed(112)
    for ii, da in enumerate(data[1:]):
        freqInt_ii=freqInt[ii]
        freq_list+=[freqInt_ii]*freqInt_ii
        randmat=np.random.rand(freqInt_ii,pca_comp)-0.5
        da_aug = da+0.25*randmat
        assert np.sum(np.round(da_aug)-da)<0.001
        data_aug=np.vstack((data_aug,da_aug))
    data_aug=data_aug[1:]
    print(data_aug.shape,len(freq_list))
    aug_pd=pd.DataFrame(data_aug, columns=list(range(pca_comp)))
    aug_pd['freqInt']=freq_list
    aug_pd['freq']=aug_pd['freqInt'].apply(lambda x: float(x))
    return aug_pd

# umapT=get_umap_pd(HH_pdc,list(range(pca_comp)))
# plt.figure(figsize=(20,5))
# sns.scatterplot('u1','u2',data=HH_pdc, hue=np.log(HH_pdc['freq']), size=np.log(HH_pdc['freq']))

# g = sns.PairGrid(norm_pd1k,hue='h3')
# g.map_diag(sns.kdeplot)
# g.map_offdiag(sns.kdeplot, n_levels=4);
# pcag = sns.PairGrid(HH_pdc,hue='m4',palette="Set2",
#                  hue_kws={"marker": ["o", "s", "D"]})
# pcag = sns.PairGrid(HH_plot,hue='m4',palette="Set2")
# pcag.map_diag(plt.hist,histtype="step", linewidth=3)
# pcag.map_offdiag(sns.kdeplot, n_levels=3);
# g = sns.PairGrid(color_pct,hue="lbl")
# g = g.map_upper(sns.scatterplot)
# g = g.map_lower(sns.kdeplot,n_levels=3)
# g = g.map_diag(sns.kdeplot, lw=2 )
# g.add_legend()

# f, (ax0,ax1,ax2)=plt.subplots(1,3,figsize=(20,8))
# ax0.scatter(HH_pdc['u1'],HH_pdc['u2'],s=lw**2*100, c='k')
# ax1.scatter(HH_pdc['u1'],HH_pdc['u2'],s=lw**2*100, c=HH_pdc['k8'])
# # sns.scatterplot('u1','u2',data=HH_pdc, hue='k3',ax=ax2,palette="viridis")
# ax2.scatter(HH_pdc['u1'],HH_pdc['u2'],s=lw**2*100, c=HH_pdc['m4'])

# f,(ax0,ax1)=plt.subplots(1,2,figsize=(16,5))
# ax0.scatter(HH_pdc['u1'],HH_pdc['u2'],s=lw**2*100, c=HH_pdc['m4'])
# ax1 = sns.kdeplot(HH_pdc[HH_pdc['m4']==2]['u1'],HH_pdc[HH_pdc['m4']==2]['u2'],
#                  cmap="Reds", shade=True, shade_lowest=False,label='TN')
# ax1 = sns.kdeplot(HH_pdc[HH_pdc['m4']==4]['u1'],HH_pdc[HH_pdc['m4']==4]['u2'],
#                  cmap="Blues", shade=True, shade_lowest=False,label='NTN')
# ax1 = sns.kdeplot(HH_pdc[HH_pdc['m4']==3]['u1'],HH_pdc[HH_pdc['m4']==3]['u2'],
#                  cmap="Greens", shade=True, shade_lowest=False,label='MPH')
# ax1 = sns.kdeplot(HH_pdc[HH_pdc['m4']==1]['u1'],HH_pdc[HH_pdc['m4']==1]['u2'],
#                  cmap="Purples", shade=True, shade_lowest=False,label='Interface')
# ax1.legend()

def get_kmean_cluster_plot(HH_pdh,pred_k,N_cluster,cmap_k):
    f, (ax0,ax1)=plt.subplots(1,2,figsize=((16,5)))
    sns.scatterplot('u1','u2',data=HH_pdh,hue=f'k{N_cluster}', marker='x',s=5, palette=cmap_k,legend="full",ax = ax0)
    cmap, norm =from_levels_and_colors(list(range(N_cluster+2)),[[0.95,0.95,0.95,1]]+cmap_k)
    im=ax1.imshow(pred_k, cmap=cmap, norm=norm)
    divider = make_axes_locatable(ax1)
    cax = divider.append_axes('right', size='5%', pad=0.05)
    f.colorbar(im, cax=cax)


from typing import List
from collections import Counter
import heapq

class Solution:
    def medianSlidingWindow(self, nums: List[int], k: int) -> List[float]:
        # Approach 2 : Two Heaps (Lazy Removal)
        if k == 1 or len(nums) <= 1:
            return nums  # special case

        # init two heap
        init = sorted(nums[:k])
        max_heap = []
        for i in init[: (1 + k) // 2][::-1]:
            max_heap.append(-i)
        min_heap = init[(1 + k) // 2 :]

        if k % 2 == 0:
            ISEVEN = True
        else:
            ISEVEN = False

        medians = []
        if ISEVEN:
            medians.append((min_heap[0] - max_heap[0]) / 2)
        else:
            medians.append(-max_heap[0])

        pop_nums = Counter()  # hash map to store invalid nums
        for i in range(len(nums) - k):
            out = nums[i]
            in_ = nums[i + k]

            if out <= -max_heap[0]:
                balance = -1
            else:
                balance = 1

            pop_nums[out] += 1

            if in_ <= -max_heap[0]:
                heapq.heappush(max_heap, -in_)
                balance += 1
            else:
                heapq.heappush(min_heap, in_)
                balance -= 1

            if balance < 0:
                heapq.heappush(max_heap, -heapq.heappop(min_heap))
            elif balance > 0:
                heapq.heappush(min_heap, -heapq.heappop(max_heap))

            while pop_nums[-max_heap[0]]:
                pop_nums[-max_heap[0]] -= 1
                heapq.heappop(max_heap)
            while pop_nums[min_heap[0]]:
                pop_nums[min_heap[0]] -= 1
                heapq.heappop(min_heap)

            if ISEVEN:
                medians.append((min_heap[0] - max_heap[0]) / 2)
            else:
                medians.append(-max_heap[0])

        return medians

if __name__ == "__main__":
    # test case
    nums = [1, 3, -2, -3, 5, 3, 6, 7]
    k = 3

    sol = Solution()
    print(sol.medianSlidingWindow(nums, k))


"""
K-Means. S
"""

from collections import Counter
from collections import defaultdict
import gzip
import math

import numpy as np

class KMeans(object):

    def __init__(self, k=2):
        """ Initialize a k-means clusterer. Should not have to change this."""
        self.k = k

    def cluster(self, documents, iters=10):
        """
        Cluster a list of unlabeled documents, using iters iterations of k-means.
        Initialize the k mean vectors to be the first k documents provided.
        After each iteration, print:
        - the number of documents in each cluster
        - the error rate (the total Euclidean distance between each document and its assigned mean vector), rounded to 2 decimal places.
        See Log.txt for expected output.
        The order of operations is:
        1) initialize means
        2) Loop
          2a) compute_clusters
          2b) compute_means
          2c) print sizes and error
        """
        self.doc_norm=defaultdict(lambda:0)
        self.docs=documents
        self.mean_vectors=[]
        count=0

        #1) initialize means
        for i in range(self.k):
            self.mean_vectors.append(documents[i])
        self.mean_norms=[]
        for doc in documents:
            n_count=0
            for term in doc:
                n_count = n_count+self.sqnorm(doc[term])
            self.doc_norm[count]=n_count
            count=count+1
        for doc in self.mean_vectors:
            norm_val=0
            for term in doc:
                val=math.pow(doc[term],2)
                norm_val = norm_val+val
            self.mean_norms.append(norm_val)
        #2 Loop
        for j in range(iters):
            #2a) compute_clusters
            self.compute_clusters(documents)
            #2b) compute_means
            self.compute_means()
            #2c) print sizes and error
            no_of_docs=[]
            for i in self.cluster_doc:
                no_of_docs.append(len(self.cluster_doc[i]))
            print(no_of_docs)
            print(self.error(documents))

    def compute_means(self):
        """ Compute the mean vectors for each cluster (results stored in an
        instance variable of your choosing)."""
        del self.mean_vectors[:]
        for i in range(self.k):
            count=Counter()
            len=0
            for doc_id in self.cluster_doc[i]:
                count.update(self.docs[doc_id])
                len=len+1
            if (len>0):
                for doc in count:
                    count[doc]=count[doc]/len
            self.mean_vectors.append(count)
        self.mean_norms=[]

        for doc in self.mean_vectors:
            norm=0
            for term in doc:
                norm = norm + self.sqnorm(doc[term])
            self.mean_norms.append(norm)
        pass

    def compute_clusters(self, documents):
        """ Assign each document to a cluster. (Results stored in an instance
        variable of your choosing). """
        self.cluster_doc=defaultdict(list)
        doc_id=0
        for doc in documents:
            for i in range(self.k):
                mean_norm = self.mean_norms[i]+self.doc_norm[doc_id]
                dist=self.distance(doc,self.mean_vectors[i],mean_norm)
                if (i==0):
                   min=i
                   min_dist=dist
                else:
                    if (dist<min_dist):
                        min=i
                        min_dist=dist
            self.cluster_doc[min].append(doc_id)
            doc_id+=1
        pass

    def sqnorm(self, d):
        """ Return the vector length of a dictionary d, defined as the sum of
        the squared values in this dict. """
        val=math.pow(d,2)
        return val
        pass

    def distance(self, doc, mean, mean_norm):
        """ Return the Euclidean distance between a document and a mean vector.
        See here for a more efficient way to compute:
        http://en.wikipedia.org/wiki/Cosine_similarity#Properties"""
        dist=mean_norm
        for term in doc:
            val=-2.0*doc[term]*mean[term]
            dist = dist +val
        return math.sqrt(dist)
        pass

    def error(self, documents):
        """ Return the error of the current clustering, defined as the total
        Euclidean distance between each document and its assigned mean vector."""
        self.cluster_doc_dist=defaultdict(list)
        err_val=0

        for cluster in self.cluster_doc:
            for doc_id in self.cluster_doc[cluster]:
                doc=documents[doc_id]
                mean_norm =self.mean_norms[cluster]+self.doc_norm[doc_id]
                di=self.distance(doc,self.mean_vectors[cluster],mean_norm)
                err_val=err_val+di
                self.cluster_doc_dist[cluster].append((doc,di))
        return err_val
        #return ("%0.2f"%err_val)
        #pass

    def print_top_docs(self, n=10):
        """ Print the top n documents from each cluster. These are the
        documents that are the closest to the mean vector of each cluster.
        Since we store each document as a Counter object, just print the keys
        for each Counter (sorted alphabetically).
        Note: To make the output more interesting, only print documents with more than 3 distinct terms.
        See Log.txt for an example."""
        for i in self.cluster_doc:
            print ("CLUSTER %d" %i)
            top=sorted(self.cluster_doc_dist[i],key=lambda x:x[1])

            k=0
            j=0
            while (j<n and k<len(top)):
                if(len(top[k][0])>3):
                    print(' '.join(str(v) for v in sorted(top[k][0].keys())))
                    j+=1
                k+=1

def prune_terms(docs, min_df=3):
    """ Remove terms that don't occur in at least min_df different
    documents. Return a list of Counters. Omit documents that are empty after
    pruning words.
    >>> prune_terms([{'a': 1, 'b': 10}, {'a': 1}, {'c': 1}], min_df=2)
    [Counter({'a': 1}), Counter({'a': 1})]
    """
    doc_count = Counter()
    result = []
    for n in docs:
        for term in n:
            doc_count[term] += 1

    for n in docs:
        result_doc = Counter()
        for term in n:
            if doc_count[term] >= min_df:
                result_doc[term] = n[term]
        if len(result_doc) > 0:
            result.append(result_doc)
    return result
    pass

def read_profiles(filename):
    """ Read profiles into a list of Counter objects.
    """
    profiles = []
    with gzip.open(filename, mode='rt', encoding='utf8') as infile:
        for line in infile:
            profiles.append(Counter(line.split()))
    return profiles

def main():
    profiles = read_profiles('profiles.txt.gz')
    print('read', len(profiles), 'profiles.')
    profiles = prune_terms(profiles, min_df=2)
    km = KMeans(k=10)
    km.cluster(profiles, iters=20)
    km.print_top_docs()

if __name__ == '__main__':
    main()


import re
import xml.etree.ElementTree as et
from pathlib import Path

class Item:
    Id = 0
    Text = ''

    def __init__(self, id, text):
        self.Id = id
        self.Text = text

class ArabicSoundex:
    _asoundexCodes = []
    _aphonixCode = []
    _transliteration = []
    _map = []
    _len = 4
    _code = 'soundex'
    _lang = "en"

    # constructor
    def __init__(self, xml_file):
        xtree = et.parse(xml_file)
        xroot = xtree.getroot()
        for node in xroot.iter('asoundexCode'):
            for x in node:
                myobj = Item(x.attrib['id'], x.text)
                self._asoundexCodes.append(myobj)
        for node in xroot.iter('aphonixCode'):
            for x in node:
                myobj = Item(x.attrib['id'], x.text)
                self._aphonixCode.append(myobj)
        for node in xroot.iter('transliteration'):
            for x in node:
                myobj = Item(x.attrib['id'], x.text)
                self._transliteration.append(myobj)
        self._map = self._asoundexCodes

    # Set the length of soundex key (default value is 4)
    # @param intValue $integer Soundex key length

    def setLen(self, intValue):
        self._len = intValue

    # Set the language of the soundex key (default value is "en")
    # @param string strValue Soundex key language [ar|en]

    def setLang(self, strValue):
        self._lang = strValue

    # Set the mapping code of the soundex key (default value is "soundex")
    # @param string strValue Soundex key mapping code [soundex|phonix]

    def setCode(self, strValue):
        xx = strValue.lower()
        if (xx == 'soundex' or xx == 'phonix'):
            self._code = xx
            if (xx == 'phonix'):
                self._map = self._aphonixCode
            else:
                self._map = self._asoundexCodes

    def getLen(self):
        return self._len

    def getLang(self):
        return self._lang

    def getCode(self):
        return self._code

    def mapCode(self, word):
        encodedWord = ''
        for elem in word:
            matchingVals = [x for x in self._map if x.Text == elem]
            if (not matchingVals):
                encodedWord = encodedWord + "0"
            else:
                encodedWord = encodedWord + matchingVals[0].Id
        return encodedWord

    def trimRep(self, word):
        lastChar = ''
        cleanWord = ''
        # max = len(word)
        for elem in word:
            if (elem != lastChar):
                cleanWord += elem
            else:
                lastChar = elem
        return cleanWord

    def soundex(self, word):
        soundex = word[0]
        if (self._lang == 'en'):
            matchingVals = [x for x in self._transliteration if x.Id == soundex]
            if (not matchingVals):
                soundex = ''
            else:
                soundex = matchingVals[0].Text
        # rest = word[1:len(word)]
        encodedRest = self.mapCode(word)
        cleanEncodedRest = self.trimRep(encodedRest)
        soundex += cleanEncodedRest
        soundex = soundex.replace("0", "")
        totalLen = len(soundex)
        if (totalLen > self._len):
            soundex = soundex[0:self._len]
        else:
            soundex += '0' * (self._len - totalLen)
        return soundex

    def arabicFixConfusingLettersKey(self, word):
        result = ''
        result = word.replace(" ", "")
        result = result.replace("أ", "ا")
        result = result.replace("آ", "ا")
        result = result.replace("ى", "ا")
        result = result.replace("ؤ", "ا")
        result = result.replace("إ", "ا")
        result = result.replace("ئ", "ا")
        result = result.replace("ة", "ه")
        result = result.replace("ص", "س")
        result = result.replace("ض", "د")
        result = result.replace("ث", "ت")
        result = result.replace("ذ", "ز")
        result = result.replace("ق", "ك")
        result = result.replace("ح", "ه")
        result = re.sub('[^A-Za-zا-ي]+', '', result)
        return result

absPath = Path(__file__).parent / 'ArSoundex.xml'

arabicSoundex = ArabicSoundex(absPath)


"""Module for common plotting functions used to visualise SPIIR pipeline results.

Note that all functions in this module are currently placeholders and will be subject
to significant changes in the near future.
"""

import logging
from os import PathLike
from typing import Optional, Union, Tuple, Dict, Sequence, Any

import astropy
import astropy_healpix as ah
import ligo.skymap.io
import ligo.skymap.plot
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from astropy.coordinates import SkyCoord
from ligo.skymap.postprocess import find_greedy_credible_levels
from matplotlib.figure import Figure
from scipy.stats import halfnorm, poisson

logger = logging.getLogger(__name__)

def save_background_analysis_plots(
    data: Dict[str, pd.DataFrame],
    path: Union[str, PathLike],
):
    # this function produces two comparison plots across two runs
    # i.e. a inj and noninj run for two different runs, comparing cohsnr/cmbchisq
    # TODO: the input columns should be customisable
    # TODO: more thought should be put into generalising this function
    nrows, ncols = 2, 2
    fig, axes = plt.subplots(
        nrows=nrows, ncols=ncols, figsize=(12, 8), sharex=True, sharey=True
    )

    for i, key in enumerate(data):
        ax = axes[i // ncols, i % ncols]
        ax.scatter(
            data[key]["cohsnr"], data[key]["cmbchisq"], s=20, alpha=1.0, marker="."
        )
        ax.grid(False)
        ax.set(
            xscale="log",
            yscale="log",
            xlabel="Coherent SNR",
            ylabel=r"Combined $\chi^{2}$",
            title=key,
        )

    fig.suptitle("Background Analysis", fontsize=14)
    fig.tight_layout()
    fig.savefig(path)

def save_foreground_analysis_plots(
    data: Dict[str, pd.DataFrame],
    path: Union[str, PathLike],
):

    fig, ax = plt.subplots(figsize=(12, 8))
    for i in range(2):
        key = list(data.keys())[i + 2]
        data[key]["cmbchisq"].hist(
            bins=100, histtype="step", lw=2 - i, label=key, ax=ax
        )  # should probably replace column here with an input argument
    ax.set_xlim(right=10)
    ax.set_title("Foreground Determinism Comparison for Combined ChiSq")
    ax.legend(loc="upper right")
    fig.tight_layout()
    fig.savefig(path)

def save_zerolag_trigger_plots(
    data: Dict[str, pd.DataFrame],
    path: Union[str, PathLike],
):
    nrows, ncols = 1, 2
    fig, axes = plt.subplots(nrows=nrows, ncols=ncols, figsize=(16, 8))

    # calculate event counts at each ifar tick label
    for i, key in enumerate(data):
        # plot observed data
        data[key]["ifar"] = [None if y == 0 else 1 / y for y in data[key]["far"]]
        ticks = np.geomspace(
            min(data[key]["ifar"].dropna()), max(data[key]["ifar"].dropna())
        )
        counts = [
            int(data[key]["ifar"][data[key]["ifar"] > tick].count()) for tick in ticks
        ]
        ax = axes[i // ncols]
        ax.plot(ticks, counts, label=key, linewidth=2)

        # set axis details for every odd (last) iteration
        if i % ncols == 1:
            # estimate theoretical iFAR candidate frequencies over n_seconds duration
            # n_seconds = 7*24*(60**2)  # we don't have duration metadata
            # TODO: Should n_seconds be a user input arg if we don't have this metadata?
            n_seconds = data[key]["end_time"].max() - data[key]["end_time"].min()
            n_seconds = int(np.ceil(n_seconds))
            r = np.geomspace(1, n_seconds)
            ax.plot(r, n_seconds / r, label="Theoretical", c="k")

            # plot expected variance in iFAR
            for nsigma in reversed(range(1, 4)):
                ifar_grid = np.geomspace(1, 2e6)
                prob = halfnorm.cdf(nsigma)
                lo, hi = np.asarray(poisson(n_seconds / ifar_grid).interval(prob))
                ax.fill_between(
                    ifar_grid,
                    lo,
                    hi,
                    color=str(1 - 0.5 / nsigma),
                    label=r"{}$\sigma$".format(nsigma),
                )

            # format axes
            ax.grid(True, which="major", color="k", linestyle="-", alpha=0.4)
            ax.grid(True, which="minor", color="k", linestyle="--", alpha=0.2)
            ax.legend()
            ax.set(
                title=" ".join(key.split(" ")[:2]),
                xlabel="iFAR",
                xscale="log",
                xlim=(1, n_seconds),
                ylabel="Cumulative Counts",
                yscale="log",
            )

    fig.suptitle("Cumulative Counts of Zerolag Triggers vs. iFAR", fontsize=14)
    fig.tight_layout()
    fig.savefig(path)

def plot_skymap_from_fits(
    path: Union[str, bytes, PathLike],
    contours: Optional[Union[float, Sequence[float]]]=None,
    inset_args: Optional[Union[Dict[str, Any], Sequence[Dict[str, Any]]]]=None,
    ground_truth: Optional[SkyCoord]=None,
    annotate: bool=True,
    figsize: Tuple[float, float]=(16, 7),
    title: Optional[str]=None,
) -> Figure:
    """Plots a detailed probability skymap from a FITS file.

    TODO:
        - Implement automatic inset axis sizing and ordering based on ra / dec.
        - Ensure logic works for greater than two inset axes.
        - Test various figsizes to ensure figure is consistently formatted.

    Parameters
    ----------
    path: str | bytes | os.PathLike
        A path to a valid FITS file, typically generated from ligo.skymap.io.fits.
    contours: float | Sequence[float] | None
        A sequence of probability contour levels (credible intervals) used to draw on
        the skymap. For example, (0.5, 0.9) would correspond to the 50% and 90% levels.
    inset_kwargs: dict | Sequence[dict] | None
        A set of keyword arguments passed used to define one or more inset skymap axes.
        If a sequence of argument dicts are provided, one inset axis is made for each
        element of the list. At a minimum, each must have a 'center' key with a valid
        astropy.coordinates.SkyCoord to center the inset axis on the skymap canvas.
    ground_truth: astropy.coordinates.Skycoord | None
        An astropy SkyCoord input for a "ground truth" marker for the true location.
    annotate: bool
        Whether to annotate the plot with information about additional metadata such as
        the LIGOLW event id and the probability per square degree.
    figsize: tuple[float, float] | None
        An optional tuple specifying the figure size. Default is (16, 7).
    title: str | None
        An optional string specifying the figure title.

    Returns
    -------
    matplotlib.figure.Figure
        The probability skymap as a matplotlib Figure object.

    """
    skymap, metadata = ligo.skymap.io.fits.read_sky_map(path, nest=None)
    nside = ah.npix_to_nside(len(skymap))
    deg2perpix = ah.nside_to_pixel_area(nside).to_value(astropy.units.deg**2)
    probperdeg2 = skymap / deg2perpix

    # get sky position of max point of probability once, if required later
    healpix = ah.HEALPix(nside=nside, order='nested')
    max_longitude, max_latitude = healpix.healpix_to_lonlat(probperdeg2.argmax())

    fig = plt.figure(figsize=figsize, facecolor="white")
    ax = plt.axes(projection='astro mollweide')
    ax.grid()

    if title:
        if not isinstance(title, str): raise TypeError("title argument must be a str.")
        ax.set_title(title, fontsize=16, pad=15)

    ax.imshow_hpx(
        (probperdeg2, "ICRS"),
        nested=metadata["nest"],
        vmin=0.,
        vmax=probperdeg2.max(),
        cmap="cylon",
    )

    if ground_truth:
        marker = "X"
        ax.plot_coord(
            ground_truth,
            marker,
            markerfacecolor='white',
            markeredgecolor='black',
            markersize=8,
            linewidth=1,
        )

    if contours:
        contours = [contours] if isinstance(contours, float) else contours
        credible_levels = 100 * find_greedy_credible_levels(skymap)
        contour = ax.contour_hpx(
            (credible_levels, 'ICRS'),
            nested=metadata['nest'],
            colors='k',
            linewidths=0.5,
            levels=contours,
        )
        # fmt = r'%g\%%' if mpl.rcParams['text.usetex'] else '%g%%'
        contour.clabel(fmt=r'%g\%%', fontsize=6, inline=True)

    if annotate:
        text = []
        try:
            objid = metadata['objid']
        except KeyError:
            pass
        else:
            text.append('event ID: {}'.format(objid))
        if contours:
            pp = np.round(contours).astype(int)
            ii = np.searchsorted(np.sort(credible_levels), contours) * deg2perpix
            ii = np.round(ii).astype(int)
            for i, p in zip(ii, pp):
                text.append(r'{:d}% area: {:,d} deg$^2$'.format(p, i))
        ax.text(.05, 0.95, '\n'.join(text), transform=ax.transAxes, ha='right')

    if inset_args is not None:
        if isinstance(inset_args, dict):
            inset_args = [inset_args]

        for i, inset_arg in enumerate(inset_args):
            if not isinstance(inset_arg, dict):
                raise TypeError(f"inset_args must be a dict or sequence of dicts.")

            if "center" not in inset_arg:
                raise KeyError(f"inset_arg must contain a valid 'center' SkyCoord.")

            # define inset axis size
            inset_width = inset_arg.get("width", 0.3)
            inset_height = inset_arg.get("height", 0.3)
            inset_left = inset_arg.get("left", 0.8)
            inset_bottom = inset_arg.get("bottom", None)
            if inset_bottom is None:
                if len(inset_args) == 1:
                    inset_bottom = 0.5 - inset_height/2
                else:
                    # FIXME: Handle more than 2 inset axes
                    inset_bottom = 0.85 - ((i%2)+1)*inset_height - (i%2)*0.1

            # center point as astropy.coordinates.SkyCord
            # FIXME: implement more robust input argument logic and exception handling
            inset_center = inset_arg.get("center", "max")
            if isinstance(inset_center, str) and inset_center.lower() == "max":
                inset_center = SkyCoord(max_longitude, max_latitude)
            if not isinstance(inset_center, SkyCoord):
                try:
                    inset_center = SkyCoord(inset_center)
                except Exception as exc:
                    print(exc)
                    try:
                        inset_center = SkyCoord(**inset_center)
                    except Exception as exc:
                        raise ValueError(f"inset_arg['center'] is an invalid SkyCoord.")

            ax_inset = plt.axes(
                [inset_left, inset_bottom, inset_width, inset_height],
                projection='astro zoom',
                center=inset_center,
                radius=(inset_arg.get("radius", None) or 10)*astropy.units.deg,
            )

            for coords in ax_inset.coords:
                coords.set_ticklabel(exclude_overlapping=True)
                coords.set_ticklabel(size=8)

            ax.mark_inset_axes(ax_inset)
            ax.connect_inset_axes(ax_inset, 'upper left')
            ax.connect_inset_axes(ax_inset, 'lower left')
            ax_inset.scalebar((0.1, 0.1), 5 * astropy.units.deg).label()
            ax_inset.compass(0.9, 0.1, 0.2)
            ax_inset.set_xlabel(inset_arg.get("xlabel", " "))  # empty string required
            ax_inset.set_ylabel(inset_arg.get("ylabel", " "))  # None breaks x/y labels
            ax_inset.set_title(inset_arg.get("title", None))

            ax_inset.imshow_hpx(
                (probperdeg2, "ICRS"),
                nested=metadata["nest"],
                vmin=0.,
                vmax=probperdeg2.max(),
                cmap="cylon",
            )

            if ground_truth:
                ax_inset.plot_coord(
                    ground_truth,
                    marker,
                    markerfacecolor='white',
                    markeredgecolor='black',
                    markersize=8,
                    linewidth=1,
                )

            if contours:
                contour_inset = ax_inset.contour_hpx(
                    (credible_levels, 'ICRS'),
                    nested=metadata['nest'],
                    colors='k',
                    linewidths=0.5,
                    levels=contours,
                )
                # fmt = r'%g\%%' if mpl.rcParams['text.usetex'] else '%g%%'
                contour_inset.clabel(fmt=r'%g\%%', fontsize=8, inline=True)

    return fig


import scrapy
from ..items import BeatstartsItem
from selenium import webdriver
from scrapy_selenium import SeleniumRequest
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import time

class BeatstartsScrapSpider(scrapy.Spider):
    name = 'beatstarts_scrap'
    allowed_domains = ['beatstarts.com', 'www.beatstarts.com']
    start_urls = [
        'https://www.beatstars.com/explore-tracks'
    ]

    def start_requests(self):
        DRIVER_PATH = '/home/rifat/Projects/python/scrap/scrapy/beatstarts/chromedriver'
        driver = webdriver.Chrome(executable_path=DRIVER_PATH)
        driver.get('https://www.beatstars.com/explore-tracks')
        for i in range(1, 4):
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(10)
            elements = driver.find_elements(By.CSS_SELECTOR, ".fit-parent.ng-star-inserted .ng-star-inserted a")
            for element in elements:
                profile_link = element.get_attribute('href')
                yield scrapy.Request(url=profile_link, callback=self.parse_detail, dont_filter=True)
        driver.quit()

    def parse_detail(self, response):
        items = BeatstartsItem()
        name = response.css(".heading-s .name").css("::text").extract()
        bio = response.css(".vb-margin-b-xl .ng-star-inserted").css("::text").extract()
        followers = response.css(".social-interaction-line:nth-child(1) .ng-star-inserted").css("::text").extract()
        items['name'] = name
        items['bio'] = bio
        items['followers'] = followers
        yield items


# Author: Miguel Martinez Lopez

from rpyc.utils.server import ThreadedServer
from rpyc.utils.classic import DEFAULT_SERVER_PORT
from rpyc.core.service import Service, ModuleNamespace

from rpyc.lib.compat import execute, is_py3k

import sys
import os
import threading
import argparse

EXECUTED_PYTHON_FILE = False

def exec_python(filepath, namespace):
    global EXECUTED_PYTHON_FILE

    if EXECUTED_PYTHON_FILE:
        raise Exception("exec_python can be used only one time")

    EXECUTED_PYTHON_FILE = True

    filepath = os.path.abspath(filepath)
    sys.path = [os.path.dirname(filepath)] + sys.path[1:]

    namespace["__file__"] = filepath
    namespace["__name__"] = "__main__"

    with open(filepath, 'rb') as file:
        exec(compile(file.read(), filepath, 'exec'), namespace)

class PublicService(Service):
    exposed_namespace = {}

    def on_connect(self):
        self._conn._config.update(dict(
            allow_all_attrs = True,
            allow_pickle = True,
            allow_getattr = True,
            allow_setattr = True,
            allow_delattr = True,
            import_custom_exceptions = True,
            instantiate_custom_exceptions = True,
            instantiate_oldstyle_exceptions = True,
        ))

        # shortcuts
        self._conn.modules = ModuleNamespace(self._conn.root.getmodule)
        self._conn.eval = self._conn.root.eval
        self._conn.execute = self._conn.root.execute
        self._conn.namespace = self._conn.root.namespace
        if is_py3k:
            self._conn.builtin = self._conn.modules.builtins
        else:
            self._conn.builtin = self._conn.modules.__builtin__
        self._conn.builtins = self._conn.builtin

    def exposed_execute(self, text):
        """execute arbitrary code (using ``exec``)"""
        execute(text, self.exposed_namespace)
    def exposed_eval(self, text):
        """evaluate arbitrary code (using ``eval``)"""
        return eval(text, self.exposed_namespace)
    def exposed_getmodule(self, name):
        """imports an arbitrary module"""
        return __import__(name, None, None, "*")
    def exposed_getconn(self):
        """returns the local connection instance to the other side"""
        return self._conn

parser = argparse.ArgumentParser(description='Remote debugging and testing')
parser.add_argument('filename', help="Path to script")
parser.add_argument('-p', '--port', action="store", dest="port", default=DEFAULT_SERVER_PORT, help="Remote interpreter port", type=int)

args = parser.parse_args()
thread = threading.Thread(target=lambda: ThreadedServer(PublicService, hostname = "localhost", port=args.port).start())
thread.daemon=True
thread.start()

exec_python(args.filename, PublicService.exposed_namespace)


import os
import argparse

parser = argparse.ArgumentParser(description="A python script to locate all git repositories on a filesystem and create a shell script to clone them all")
parser.add_argument("searchdir", help="Directory to search for git repositories for")
parser.add_argument("sshhostname", help="Hostname or SSH config name of the host you are cloning the repositories from")
parser.add_argument("--mr-register", action='store_true', default=False, help="If provided this will also output the script for mr register")
parser.add_argument("--create-dirs", action='store_true', default=False, help="If provided this will add a mkdir -p command in front of each git clone")
parser.add_argument("--git-clone-command",
                    help="Command used in place of git clone, will be the prefix to the repository",
                    default="git clone"
                    )

args = parser.parse_args()

clone_dir = "."

def find_all_git_repos(directory):
    for d in os.listdir(directory):
        if is_git_bare_repo(os.path.join(directory, d)):
            print_git(os.path.join(directory, d))
        elif os.path.isdir(os.path.join(directory, d)):
            find_all_git_repos(os.path.join(directory, d))

def is_git_bare_repo(directory):
    dirs_to_check = ["branches", "hooks", "info", "objects", "refs"]
    files_to_check = ["config", "description", "HEAD"]

    for dir in dirs_to_check:
        if not os.path.isdir(os.path.join(directory, dir)):
            return False

    for filename in files_to_check:
        if not os.path.isfile(os.path.join(directory, filename)):
            return False

    return True

def print_git(directory):
    dir_to_place_git_repo = directory[len(args.searchdir):].lstrip("/")
    if args.create_dirs:
        print("mkdir -p", os.path.dirname(dir_to_place_git_repo))
    print(args.git_clone_command + " " + args.sshhostname + ":" + directory + " " + dir_to_place_git_repo)
    if args.mr_register:
        print("mr register " + directory[len(args.searchdir):])

find_all_git_repos(args.searchdir)


import carla
import open3d
import time
import numpy as np
import math
import os
import random
from matplotlib import cm

# Global variables
import debugVisualizer

global client
global world
global bp_lib
global lidar_bp
global car_lidar
global lidar
global vehicle
global test_rectangle_flag
global vehicle_bp
global all_vehicle_bp
global vehicle_points
global spectator
global carlaClientApp
global box_semantic_tag
global place_box
global box_points

"""main methods, in order used during training step"""

def setup_carla_environment(no_render=False, host_ip='127.0.0.1', host_port=2000, macos=False):
    global place_box
    place_box = True
    global carlaClientApp
    if macos:
        os.system("open /Users/jordan/carla/Dist/CARLA_Shipping_9607eca-dirty/MacNoEditor/CarlaUE4-Mac-Shipping.app --args -quality-level=Low")
        time.sleep(4)
    # Connect the client and set up bp library and spawn point
    global client
    client = carla.Client(host_ip, host_port)
    global world
    world = client.get_world()
    world.unload_map_layer(carla.MapLayer.All)
    world.load_map_layer(carla.MapLayer.Ground)
    settings = world.get_settings()
    if no_render:
        settings.no_rendering_mode = True
    world.apply_settings(settings)
    global bp_lib
    bp_lib = world.get_blueprint_library()
    global spectator
    spectator = world.get_spectator()
    transform = carla.Transform(carla.Location(x=-4, z=2.5), carla.Rotation())
    spectator.set_transform(transform)
    global lidar_bp
    lidar_bp = bp_lib.find('sensor.lidar.ray_cast_semantic')
    lidar_bp.set_attribute('points_per_second', '4000000')
    lidar_bp.set_attribute('rotation_frequency', '60.0')
    lidar_bp.set_attribute('range', '15.0')
    lidar_bp.set_attribute('channels', '128')
    lidar_bp.set_attribute('upper_fov', '20')
    world.tick()
    return client

def place_cylinder_and_car(distance=10.0):
    global vehicle
    global bp_lib
    global vehicle_bp
    # Add vehicle
    vehicle_bp = bp_lib.find('vehicle.audi.etron')
    vehicle = world.try_spawn_actor(vehicle_bp, carla.Transform(carla.Location(x=0, y=0, z=0.2)))
    # Spawn test cylinder
    box_bp = bp_lib.filter('box02*')[0]
    for theta in range(0, 360, 3):
        for height in np.arange(0, 4, 0.6):
            world.try_spawn_actor(box_bp,carla.Transform(carla.Location(x=distance * math.cos(theta*math.pi/180),
                                                                        y=distance * math.sin(theta*math.pi/180),
                                                                        z=height*0.65),
                                                         carla.Rotation(yaw=theta)))

def findCarMesh(numOfPoints = 6000, setNumber = False, downsample=False):
    distance = 8.5
    # spawn sensor
    global world
    global bp_lib
    global lidar_bp
    global vehicle
    global vehicle_points
    global test_rectangle_flag
    test_rectangle_flag = False
    # place semantic sensor
    car_lidar_bp = bp_lib.find('sensor.lidar.ray_cast_semantic')
    car_lidar_bp.set_attribute('points_per_second', '4000000')
    car_lidar_bp.set_attribute('rotation_frequency', '60.0')
    car_lidar_bp.set_attribute('channels', '64')
    global car_lidar
    total_points = open3d.geometry.PointCloud()
    car_lidar = world.spawn_actor(car_lidar_bp, carla.Transform(carla.Location(x=distance, z=4)))
    # measure while rotating around car
    theta = 0
    temp_array = np.zeros((0, 3))
    collect_initial_points_flag = True
    while True:
        theta += 75
        car_lidar.set_transform(carla.Transform(carla.Location(x=distance * math.cos(theta*math.pi/180),
                                                               y=distance * math.sin(theta*math.pi/180),
                                                               z=4)))
        single_detection = open3d.geometry.PointCloud()
        car_lidar.listen(lambda data: lidar_car_callback(data, single_detection, car_lidar.get_transform(), vehicle.semantic_tags[0]))
        time.sleep(0.1)
        # if points were collected, add them to the output
        if single_detection.has_points():
            if collect_initial_points_flag:
                total_points = single_detection
                collect_initial_points_flag = False
            else:
                temp_array = np.vstack((np.asarray(single_detection.points), np.asarray(open3d.geometry.PointCloud(total_points).points)))
                total_points = open3d.utility.Vector3dVector(temp_array)
        if (setNumber and temp_array.shape[0] > numOfPoints) or (not setNumber and theta > 720):
            break
        car_lidar.stop()
    car_lidar.destroy()
    # format output point cloud and move to open-3d origin
    vehicle_points = open3d.geometry.PointCloud(total_points).translate([0.0, 0.0, -4.0])
    if downsample:
        vehicle_points = vehicle_points.voxel_down_sample(voxel_size=0.05)
    temp_array = np.asarray(vehicle_points.points)
    if setNumber:
        if temp_array.shape[0] >= numOfPoints:
            temp_array = temp_array[:numOfPoints]
    # debugVisualizer.debugVisualizer(temp_array)
    return temp_array

def testBoxMesh(numOfPoints=6000, setNumber=False, downsample=False):
    # destroy the vehicle and create the box:
    global place_box
    global box_points
    global world
    global test_rectangle_flag
    test_rectangle_flag = True
    if place_box:
        place_box = False
        global vehicle
        vehicle.destroy()
        global bp_lib
        for x in np.arange(-3,3.1,0.5):
            for y in np.arange(-1.5,1.6,0.5):
                for z in np.arange(0,1.1,0.5):
                    box_bp = bp_lib.filter('*box02*')[0]
                    world.try_spawn_actor(box_bp,
                                          carla.Transform(carla.Location(x=x, y=y, z=z), carla.Rotation()))
    distance = 8.5
    # spawn sensor
    global vehicle_points
    # place semantic sensor
    car_lidar_bp = bp_lib.find('sensor.lidar.ray_cast_semantic')
    car_lidar_bp.set_attribute('points_per_second', '1000000')
    car_lidar_bp.set_attribute('rotation_frequency', '60.0')
    car_lidar_bp.set_attribute('channels', '64')
    global car_lidar
    total_points = open3d.geometry.PointCloud()
    car_lidar = world.spawn_actor(car_lidar_bp, carla.Transform(carla.Location(x=distance, z=4)))
    # measure while rotating around car
    theta = 0
    temp_array = np.zeros((0, 3))
    collect_initial_points_flag = True
    while True:
        theta += 105
        car_lidar.set_transform(carla.Transform(carla.Location(x=distance * math.cos(theta*math.pi/180),
                                                               y=distance * math.sin(theta*math.pi/180),
                                                               z=4)))
        single_detection = open3d.geometry.PointCloud()
        car_lidar.listen(lambda data: lidar_car_callback(data, single_detection, car_lidar.get_transform(), 20))
        time.sleep(0.04)
        # if points were collected, add them to the output
        if single_detection.has_points():
            if collect_initial_points_flag:
                total_points = single_detection
                collect_initial_points_flag = False
            else:
                temp_array = np.vstack((np.asarray(single_detection.points), np.asarray(open3d.geometry.PointCloud(total_points).points)))
                filter_array = []
                for point in temp_array:
                    if -5 < point[0] < 5 and -5 < point[1] < 5 and -2 < point[2] < 8.0:
                        filter_array.append(True)
                    else:
                        filter_array.append(False)
                temp_array = temp_array[filter_array]
                total_points = open3d.utility.Vector3dVector(temp_array)
        if (setNumber and temp_array.shape[0] > numOfPoints) or (not setNumber and theta > 720):
            break
        car_lidar.stop()
    car_lidar.destroy()
    # format output point cloud and move to open-3d origin
    vehicle_points = open3d.geometry.PointCloud(total_points).translate([0.0, 0.0, -4.0])
    if downsample:
        vehicle_points = vehicle_points.voxel_down_sample(voxel_size=0.05)
    # bbox = open3d.geometry.AxisAlignedBoundingBox(min_bound=(-3, -3, 0), max_bound=(3, 3, 3))
    temp_array = np.asarray(vehicle_points.points)
    if setNumber:
        if temp_array.shape[0] >= numOfPoints:
            temp_array = temp_array[:numOfPoints]
    # debugVisualizer.debugVisualizer(temp_array)
    return temp_array

def lidar_car_callback(point_cloud, point_list, sensor_transform, vehicle_index):
    points = []
    # filter for points touching car
    for detection in point_cloud:
        if detection.object_tag == vehicle_index:
            # for some reason, transforming the y coordinate causes smearing of the point cloud
            points.append([sensor_transform.transform(detection.point).x,
                           detection.point.y,
                           sensor_transform.transform(detection.point).z])
    # shape point array into open 3d point cloud
    data = np.array(points, float)
    data = data.reshape((int(data.size/3), 3))
    point_list.points = open3d.utility.Vector3dVector(data)

def correct_input_points_to_point_on_vehicle(input_sensor_coords):
    global vehicle_points
    output = input_sensor_coords.reshape((-1, 3))
    # find the closest point on car to desired location
    for i in range(output.shape[0]):
        distance_map = vehicle_points.compute_point_cloud_distance(
                       open3d.geometry.PointCloud(open3d.utility.Vector3dVector(output[i].reshape((1, 3)))))
        closest_point_index = np.argmin(distance_map)
        # use the normal map to offset the sensor a bit off the car
        vehicle_points.estimate_normals()
        vehicle_points.orient_normals_to_align_with_direction(vehicle_points.points[closest_point_index])
        output[i] = np.copy(np.array([[vehicle_points.points[closest_point_index][0] + 0.2*vehicle_points.normals[closest_point_index][0],
                                       vehicle_points.points[closest_point_index][1] + 0.2*vehicle_points.normals[closest_point_index][1],
                                       vehicle_points.points[closest_point_index][2] + 0.2*vehicle_points.normals[closest_point_index][2]]]))
    return output

def placeSensor(coords=np.array, place_sensor_outside_of_vehicle=True):
    global vehicle_points
    global lidar
    global lidar_bp
    global world
    if coords is None:
        coords = np.array([0.0, 0.0, 0.0])
    internal_coords = np.copy(coords.reshape((1, 3)))
    if place_sensor_outside_of_vehicle:
        bounding_box = vehicle.bounding_box.extent
        if abs(internal_coords[0, 0]) > bounding_box.x:
            internal_coords[0, 0] = bounding_box.x if internal_coords[0, 0] > 0 else -bounding_box.x
        if abs(internal_coords[0, 1]) > bounding_box.y:
            internal_coords[0, 1] = bounding_box.y if internal_coords[0, 1] > 0 else -bounding_box.y
        if abs(internal_coords[0, 2]) > bounding_box.z:
            internal_coords[0, 2] = bounding_box.z if internal_coords[0, 2] > 0 else -bounding_box.z
            internal_coords[0, 2] = 0.01 if internal_coords[0, 2] < 0.01 else internal_coords[0, 2]
    # spawn the sensor
    lidar = world.try_spawn_actor(lidar_bp, carla.Transform(carla.Location(x=float(internal_coords[0, 0]),
                                                                           y=float(internal_coords[0, 1]),
                                                                           z=float(internal_coords[0, 2]))))
    return np.asarray([lidar.get_transform().location.x, lidar.get_transform().location.y, lidar.get_transform().location.z])

def lidar_callback(point_cloud, point_list, vehicle_index):
    points = []
    # filter for points touching car
    for detection in point_cloud:
        if detection.object_tag == vehicle_index:
            # for some reason, transforming the y coordinate causes smearing of the point cloud
            points.append([detection.point.x,
                           detection.point.y,
                           detection.point.z])
    # shape point array into open 3d point cloud
    data = np.array(points, float)
    data = data.reshape((int(data.size/3), 3))
    point_list.points = open3d.utility.Vector3dVector(data)

def fetchLiDARMeasurement():
    global vehicle
    global lidar
    global world
    # start listening to the sensors
    total_points = np.zeros(shape=(0, 3))
    single_detection = open3d.geometry.PointCloud()
    lidar.listen(lambda data: lidar_callback(data, single_detection, 20))
    # if points were collected, add them to the output
    j = 0
    start_time = time.thread_time()
    while j < 40:
        if single_detection.has_points():
            j = j + 1
            if total_points.shape == (0, 3):
                total_points = np.asarray(single_detection.points)
            else:
                total_points = np.vstack((np.asarray(single_detection.points), total_points))
            time.sleep(0.02)
        if time.thread_time() - start_time > 10:
            break
    lidar.stop()
    total_points = np.asarray(open3d.geometry.PointCloud(open3d.utility.Vector3dVector(total_points)).translate((lidar.get_transform().location.x, lidar.get_transform().location.y, lidar.get_transform().location.z)).points)
    lidar.destroy()
    # debugVisualizer.debugVisualizer(total_points)
    return total_points

def calculate_test_cylinder_surface(captured_points, cylinder_shape=(3600, 40), down_sample=True):
    if captured_points.shape[0] == 0:
        return np.zeros(shape=cylinder_shape)
    cylinder_face = np.zeros(shape=cylinder_shape)
    if down_sample:
        captured_points = np.asarray(open3d.geometry.PointCloud(open3d.utility.Vector3dVector(captured_points)).voxel_down_sample(voxel_size=0.05).points)

    def calc_func(_p, tlo_surface):
        # do conversion to cylindrical on the fly to prevent looping through the array twice
        if math.sqrt(_p[0] ** 2 + _p[1] ** 2) > 9.0 and 0 <= _p[2] < 4:
            tlo_surface[int((tlo_surface.shape[0]) * (
                math.atan2(_p[1], _p[0]) if _p[1] > 0 else math.atan2(_p[1], _p[0]) + 2 * math.pi) / (2 * math.pi)),
                        int((tlo_surface.shape[1]) * (_p[2] / 4.0))] = 1

    np.apply_along_axis(calc_func, -1, captured_points, cylinder_face)
    return cylinder_face

def cartesian_to_cylindrical_coords(cartesian_points):
    def conv_func(input_points=np.array):
        return np.array([math.sqrt(input_points[0]**2 + input_points[1]**2),
                         math.atan2(input_points[1], input_points[0]) if input_points[1] > 0 else math.atan2(input_points[1], input_points[0]) + 2*math.pi,
                         input_points[2]])
    cylindrical_points = np.apply_along_axis(conv_func, -1, cartesian_points)
    return cylindrical_points

def calculate_sensor_occlusion(input_points, coordinates, index):
    for i in range(coordinates.shape[0]):
        if index == i:
            continue
        distance = math.sqrt((coordinates[i, 0] - coordinates[index, 0])**2 + (coordinates[i, 1] - coordinates[index, 1])**2)
        if distance <= 0.1:
            return np.zeros(shape=input_points.shape)
        top = (coordinates[index, 2] + 10/distance*(coordinates[i, 2] - coordinates[index, 2] + 0.2))
        bottom = (coordinates[index, 2] + 10/distance*(coordinates[i, 2] - coordinates[index, 2] - 0.2))
        left = (math.atan2(coordinates[i, 1] - coordinates[index, 1], coordinates[i, 0] - coordinates[index, 0]) + math.sin(0.4/distance))
        right = (math.atan2(coordinates[i, 1] - coordinates[index, 1], coordinates[i, 0] - coordinates[index, 0]) - math.sin(0.4/distance))
        left = left + 2*math.pi if left < 0 else left
        right = right + 2*math.pi if right < 0 else right
        if top > 4.0:
            top = 4.0
        if bottom < 0:
            bottom = 0
        difference = np.copy(input_points)
        if left > right:
            input_points[int((input_points.shape[0])*right/(2*math.pi)):int((input_points-1).shape[0]*left/(2*math.pi)),
                         int((input_points.shape[1]-1)*bottom/4.0):int((input_points.shape[1]-1)*top/4.0)] = 0
        else:
            input_points[0:int((input_points.shape[0]) * left / (2 * math.pi)),
                         int((input_points.shape[1]) * bottom / 4.0):int((input_points.shape[1] - 1) * top / 4.0)] = 0
            input_points[int(input_points.shape[0]*right/(2*math.pi)):input_points.shape[0],
                         int((input_points.shape[1])*bottom/4.0):int((input_points.shape[1]-1)*top/4.0)] = 0
    return input_points

def calculate_tlo_with_occlusion(coords, cylinder_shape=(360, 40), downsample=True, front_and_back_scaling=True):
    global bp_lib
    global vehicle
    global vehicle_bp
    global world
    global lidar
    if coords is None:
        coords = np.asarray([[1, 1, 2], [-1, 1, 2], [1, -1, 2], [-1, -1, 2]], dtype=float)
    np_coords = np.asarray(coords)
    np_coords = np_coords.reshape((-1, 3))
    total_surface = np.zeros(shape=cylinder_shape)
    for index in range(np_coords.shape[0]):
        occluded_surface = np.zeros(shape=cylinder_shape)
        placeSensor(np_coords[index], place_sensor_outside_of_vehicle=False)
        measure_points = fetchLiDARMeasurement()
        if downsample:
            measure_points = np.asarray(
                open3d.geometry.PointCloud(open3d.utility.Vector3dVector(measure_points)).voxel_down_sample(
                    voxel_size=0.005).points)
        # debugVisualizer.debugVisualizer(measure_points)
        if measure_points.shape == (0, 3):
            continue

        def calc_func(_p, tlo_surface):
            # do conversion to cylindrical on the fly to prevent looping through the array twice
            if math.sqrt(_p[0] ** 2 + _p[1] ** 2) > 9.0 and 0 <= _p[2] < 4:
                tlo_surface[int((tlo_surface.shape[0] - 1) *
                                (math.atan2(_p[1], _p[0]) if _p[1] >= 0 else math.atan2(_p[1], _p[0]) + 2 * math.pi)
                                / (2 * math.pi)),
                            min(tlo_surface.shape[1]-1, int((tlo_surface.shape[1] - 1) * (_p[2] / 3.0)))] = 1

        np.apply_along_axis(calc_func, -1, measure_points, occluded_surface)
        if front_and_back_scaling:
            front_angle = 40
            back_angle = 30
            occluded_surface[int(occluded_surface.shape[0]*(90-front_angle)/360):int(occluded_surface.shape[0]*(90+front_angle)/360), :]\
                = occluded_surface[int(occluded_surface.shape[0]*(90-front_angle)/360):int(occluded_surface.shape[0]*(90+front_angle)/360), :] * 2
            occluded_surface[int(occluded_surface.shape[0]*(270-back_angle)/360):int(occluded_surface.shape[0]*(270+back_angle)/360), :]\
                = occluded_surface[int(occluded_surface.shape[0]*(270-back_angle)/360):int(occluded_surface.shape[0]*(270+back_angle)/360), :] * 2
        if np_coords.shape[0] > 1:
            occluded_surface = calculate_sensor_occlusion(occluded_surface, np_coords, index)
        total_surface = total_surface + occluded_surface
    if np_coords.shape[0] > 1:
        def redundancy_scaling(x, num_sensors=np_coords.shape[0]):
            # return 1 if x > 0 else 0
            return (2-(0.5)**(x-1))/(2-(0.5)**(num_sensors-1))
        total_surface = np.vectorize(redundancy_scaling)(total_surface)
    # debugVisualizer.debugVisualizer(measure_points)
    return total_surface

def select_random_vehicle(list_of_indexes=list(range(0, 35))):
    global vehicle
    global world
    global bp_lib
    global vehicle_bp
    global all_vehicle_bp
    global test_rectangle_flag
    if test_rectangle_flag:
        test_rectangle_flag = False
        for actor in world.get_actors().filter('*box02*'):
            actor.destroy()
        place_cylinder_and_car()
    if not vehicle.destroy():
        for actor in world.get_actors().filter('*vehicle*'): actor.destroy()
    time.sleep(0.1)
    all_vehicle_bp = bp_lib.filter('*vehicle*')
    if type(list_of_indexes) is int:
        rand_index = list_of_indexes
    elif type(list_of_indexes) is list:
        rand_index = list_of_indexes[random.randint(0, len(list_of_indexes)-1)]
    # for some reason, entries 33 and 35 do not exist and cause crashes
    while rand_index == 33 or rand_index == 35 or rand_index > 38:
        rand_index = list_of_indexes[random.randint(0, len(list_of_indexes)-1)]
    vehicle = world.try_spawn_actor(all_vehicle_bp[rand_index], carla.Transform(carla.Location(x=0, y=0, z=0.3)))
    if vehicle is None:
        vehicle = world.spawn_actor(all_vehicle_bp[rand_index], carla.Transform(carla.Location(x=0, y=0, z=0.3)))
    return

""" debug methods: """

def create_color_cylinder_surface(sensor_coords):
    placeSensor(np.array([[0.0, 0.0, 2.0]]),  place_sensor_outside_of_vehicle=False)
    measure_points = np.asarray(
        open3d.geometry.PointCloud(open3d.utility.Vector3dVector(fetchLiDARMeasurement())).voxel_down_sample(
            voxel_size=.005).points)
    cylinder_points = cartesian_to_cylindrical_coords(measure_points)
    # debugVisualizer.debugVisualizer(cylinder_points, cylindrical=True, downsample=False)
    tlo_surface = calculate_tlo_with_occlusion(coords=sensor_coords, cylinder_shape=(720, 40), front_and_back_scaling=False)
    colors = np.zeros(shape=measure_points.shape)
    for point in range(cylinder_points.shape[0]):
        if 0 <= cylinder_points[point, 2] <= 4 and cylinder_points[point, 0] > 9.0:
            colors[point] = [0.0, tlo_surface[int(tlo_surface.shape[0]*cylinder_points[point, 1]/(2*math.pi)), int(tlo_surface.shape[1]*(cylinder_points[point, 2]/4.0))], 0.0]
    return measure_points, colors

def find_tlo_of_car_mesh(testBox=True, const_coords=np.zeros(shape=(0, 3))):
    setup_carla_environment(no_render=True, macos=True)
    place_cylinder_and_car()
    color_map = np.array(cm.get_cmap('turbo').colors)
    color_range = np.linspace(0.0, 1.0, color_map.shape[0])
    if testBox:
        points = testBoxMesh(5000, True, True)
    else:
        points = findCarMesh(5000, True, True)
    debugVisualizer.debugVisualizer(points)
    colors = np.zeros(shape=points.shape)
    print("range: ", points.shape[0])
    max_value = 0.0
    for point in range(points.shape[0]):
        start_time = time.thread_time()
        sensor_coord = correct_input_points_to_point_on_vehicle(np.copy(points[point]))
        intensity = calculate_tlo_with_occlusion(np.append(const_coords, sensor_coord, axis=0), cylinder_shape=(720, 20), front_and_back_scaling=False).sum()/720/20
        if intensity > max_value:
            max_value = intensity
        print("point: ", point, " |return : ", intensity, " |time: ", time.thread_time() - start_time)
        colors[point] = np.array([np.interp(intensity, color_range, color_map[:, 0]),
                                  np.interp(intensity, color_range, color_map[:, 1]),
                                  np.interp(intensity, color_range, color_map[:, 2])])
    if const_coords.shape[0] > 0:
        sensor_colors = np.zeros(shape=const_coords.shape)
        sensor_colors[:] = [0.0, 0.0, 1.0]
        points = np.append(points, const_coords, axis=0)
        colors = np.append(colors, sensor_colors, axis=0)
    print(max_value)
    debugVisualizer.debugVisualizer(points, color_list=colors, colors=True, downsample=False)

def find_car_mesh_q_function(testBox=False, load_model_file='', const_coords=np.zeros(shape=(0, 3))):
    from TensorflowInterface import create_critic
    import tensorflow as tf
    critic_network = create_critic((6000, 3), (1, 3), load_model_file + '/critic')
    color_map = np.array(cm.get_cmap('turbo').colors)
    color_range = np.linspace(0.0, 1.0, color_map.shape[0])
    if testBox:
        points = testBoxMesh(6000, True, False)
    else:
        points = findCarMesh(6000, True, False)
    # debugVisualizer.debugVisualizer(points)
    print("range: ", points.shape[0])

    vehicle_points_batch = np.stack([points]*points.shape[0])
    const_coords_batch = np.stack([const_coords]*points.shape[0])
    sensor_points_batch = np.copy(points).reshape((points.shape[0], -1, 3))
    sensor_points_batch = np.concatenate((const_coords_batch, sensor_points_batch), axis=1)

    intensity = critic_network.predict([vehicle_points_batch, sensor_points_batch])

    def color_func(_inten=np.array):
        return np.array([np.interp(_inten, color_range, color_map[:, 0]),
                         np.interp(_inten, color_range, color_map[:, 1]),
                         np.interp(_inten, color_range, color_map[:, 2])])

    colors = np.apply_along_axis(color_func, 1, intensity).reshape((-1, 3))
    debugVisualizer.debugVisualizer(points, color_list=colors, colors=True, downsample=False)
    # np.savez("vehicle_points_batch_1", vehicle_points_batch)
    np.savez("sensor_points_batch_1", sensor_points_batch)

def show_color_lidar_measurement(testBox=False, coords=np.array([[0.0, 0.0, 2.0], [0.4, 0.0, 2.0], [0.4*math.sin(7*math.pi/6), 0.4*math.cos(7*math.pi/6), 2.0], [0.4*math.sin(11*math.pi/6), 0.4*math.cos(11*math.pi/6), 2.0]])):
    color_map = np.array(cm.get_cmap('viridis').colors)
    if testBox:
        points = testBoxMesh(5000, True, True)
    else:
        points = findCarMesh(5000, True, True)
    # debugVisualizer.debugVisualizer(points)
    colors = np.zeros(shape=points.shape)
    colored_surface, colors = create_color_cylinder_surface(coords)
    combined_points = np.vstack((points, colored_surface, coords))
    sensor_colors = np.zeros(shape=coords.shape)
    sensor_colors[0] = [0.0, 0.0, 1.0]
    sensor_colors[1:] = [0.0, 0.0, 1.0]
    combined_colors = np.vstack((np.full(shape=points.shape, fill_value=0.7), colors, sensor_colors))
    color_cloud = open3d.geometry.PointCloud(open3d.utility.Vector3dVector(combined_points))
    color_cloud.colors = open3d.utility.Vector3dVector(combined_colors)
    debugVisualizer.debugVisualizer(color_cloud, colors=False, downsample=False)

def closeEnvironment(macos=False):
    global world
    global carlaClientApp
    for actor in world.get_actors().filter('*vehicle*'):
        actor.destroy()
    for actor in world.get_actors().filter('*sensor*'):
        actor.stop()
        actor.destroy()
    if macos:
        os.system("killall CarlaUE4-Mac-Shipping")

if __name__ == '__main__':
    try:
        coords = np.array([
            [0.0, 0.0, 1.9],
            [-1.5, 0.0, 1.9]
        ])
        """
        setup_carla_environment(macos=True)
        place_cylinder_and_car()
        findCarMesh(6000, True, False)
        #testBoxMesh(setNumber=True)
        find_car_mesh_q_function(testBox=False, load_model_file='model_1_rand')
        show_color_lidar_measurement(testBox=False, coords=coords)
        print(calculate_tlo_with_occlusion(coords=coords, cylinder_shape=(720, 40), front_and_back_scaling=False).sum()/720/40)
        """
        # find_tlo_of_car_mesh(False, const_coords=coords)  # """
    finally:
        closeEnvironment(macos=True)


from collections import defaultdict

from django.db import IntegrityError, transaction
from django.db.models import Count
from django.core.management.base import BaseCommand
from bot.models import Tag

class Command(BaseCommand):
    help = 'remove stop word duplicates'

    def add_arguments(self, parser):
        parser.add_argument('--dry_run', dest='dry_run', action='store_true')

    def handle(self, *args, **options):

        duplicates = Tag.objects.values('text').annotate(Count('id')).order_by().filter(id__count__gt=1)
        duplicate_qs = Tag.objects.filter(text__in=[d['text'] for d in duplicates])

        self.stdout.write(self.style.SUCCESS('Found {} duplicates"'.format(duplicate_qs.count())))

        counter_map = defaultdict(list)
        for stop_word in duplicate_qs:
            counter_map[stop_word.text].append(stop_word)

        if options.get('dry_run'):
            self.stdout.write(self.style.SUCCESS('Duplicates..."'))
            for text, duplicates in counter_map.items():
                self.stdout.write(self.style.SUCCESS(
                    'For text {} there are duplicates {}"'.format(text, [s.id for s in duplicates]))
                )

        duplicates_removed, quotes_moved, quotes_deleted = 0, 0, 0

        for text, duplicates in counter_map.items():

            original = duplicates[0]

            for duplicate in duplicates[1:]:
                quotes = duplicate.quotes.all()

                with transaction.atomic():
                    for quote in quotes:

                        self.stdout.write(self.style.SUCCESS('adding quote "{}, to keyword {}"'.format(quote.text,
                                                                                original.id)))
                        if not options.get('dry_run'):
                            try:
                                quote.stop_word = original
                                quote.save()
                                quotes_moved += 1
                            except IntegrityError:
                                self.stdout.write(self.style.WARNING('Removing duplicated quote "%s"'.format(
                                    quote.text)
                                ))
                                quote.delete()
                                quotes_deleted += 1

                    if not options.get('dry_run'):
                        duplicate.delete()
                        duplicates_removed += 1
                    else:
                        self.stdout.write(self.style.WARNING('Duplicated keyword {} would be removed"'.format(
                            duplicate.text)
                        ))
                        duplicates_removed += 1

        self.stdout.write(self.style.SUCCESS('Duplicates removed {}, quotes moved {}, quotes deleted {}' .format(
            duplicates_removed, quotes_moved, quotes_deleted
        )))


#!/usr/local/bin/python3
import os
import sys
import time
import glob

from selenium import webdriver
# available since 2.4.0
from selenium.webdriver.support.ui import WebDriverWait
# available since 2.26.0
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException

from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler

class DownloadCompleteEvent(FileSystemEventHandler):
    _destPath = None
    _complete = False
    DOWNLOAD_TIMEOUT_SEC = 5.0

    def __init__(self, target):
        observer = Observer()
        observer.schedule(self, path = target)
        observer.start()

    def on_moved(self, event):
        print(event.dest_path + ' downloaded')
        self._complete = True
        self._destPath = event.dest_path

    def GetDestPath(self):
        return self._destpath

    def WaitComplete(self):
        elapsed = 0.0
        while not self._complete and self.DOWNLOAD_TIMEOUT_SEC > elapsed:
            time.sleep(0.2)
            elapsed += 0.2

        if elapsed >= self.DOWNLOAD_TIMEOUT_SEC:
            return False
        else:
            return True

class Chrome:

    _chrome = None

    def __init__(self):
        options = webdriver.ChromeOptions()
        options.add_argument('headless')
        options.add_argument('window-size=1920x1080')
        options.add_argument('blink-settings=imagesEnabled=false')
        options.add_argument('disable-popup-blocking')

        self._chrome = webdriver.Chrome(
            'chromedriver',
            chrome_options=options,
            service_args=['--verbose', '--log-path=./chromedriver.log']
        )

    def MkDir(self, path):
        try:
            os.makedirs(path)
        except OSError:
            #print('{} : already exists'.format(path))
            return False
        return True

    def GetPath(self):
        return None

    def CreateDir(self):
        path = self.GetPath()
        try:
            os.makedirs(path)
        except OSError:
            if not glob.glob(path + '/*.torrent'):
                print('no torrent file!!')
                return True
            else:
                return False
        return True

    def SetDownloadDir(self, path):
        #add missing support for chrome "send_command"  to selenium webdriver

        self._chrome.command_executor._commands["send_command"] = ("POST", '/session/$sessionId/chromium/send_command')
        params = {
                'cmd': 'Page.setDownloadBehavior',
                'params': {'behavior': 'allow', 'downloadPath': path}
        }
        self._chrome.execute("send_command", params)

    def WaitElementLocate(self, by, locate):
        return WebDriverWait(self._chrome, 10).until(EC.presence_of_element_located((by, locate)))

    def WaitElementClickable(self, by, locate):
        return  WebDriverWait(self._chrome, 10).until(EC.element_to_be_clickable((by, locate)))

    def Exit(self, msg = ''):
        print(msg)
        self._chrome.quit()
        sys.exit(0)