from collections import OrderedDict
import math

import numpy as np
import pandas as pd
import scipy

from mobmess import utils
from mobmess.dummy_module import DummyModule

try:
    import matplotlib.pyplot as plt
except ImportError:
    plt = DummyModule("matplotlib")

try:
    import seaborn as sns
except ImportError:
    sns = DummyModule("seaborn")

def histplot(*args,
             xlabel=None, ylabel=None, title=None, show=None,
             **kwargs):
    """
    Wrapper around seaborn.histplot. Adds extra convenience parameters
    """

    ax = sns.histplot(*args, **kwargs)
    ax.set(xlabel=xlabel, ylabel=ylabel, title=title)

    if show:
        plt.show()

    return ax

def linkage(data, metric='precompute', method='ward', fast=False, **kwargs):
    """Does the same thing with scipy.cluster.hierarchy.linkage, but it
    handles my desired default parameters, and easily handles the
    usage of a precomputed distance metric.

    fast :

        If True, use fastcluster package's implementation of linkage(). Otherwise, use scipy's (default).

        Note: not recommended to use fastcluster unless necessary. My
        empirical tests show that the fastcluster package is about 2x
        faster with method='ward' and precomputed distances for
        ~16K-by-16K.  However, in a direct comparison of Z created by
        scipy, it is not exactly the same -- although discrepancies
        may just be degenerate permutations of the
        leaves. Additionally, I'm not sure if fastcluster is computing
        optimal leaf ordering (this is done by default in scipy, which
        may also exlain why it's slower)
    """

    if metric=='precompute':
        data = scipy.spatial.distance.squareform(data, checks=False)
    # else:
    #     raise Exception()

    if fast:
        import fastcluster
        linkage = fastcluster.linkage
    else:
        linkage = scipy.cluster.hierarchy.linkage

    # print('metric', metric)
    # print(data)
    Z = linkage(data, metric=metric, method=method, **kwargs)

    return Z

def plot_colors(colors_df, ncols=None):
    """Visualizes the mapping of values to their colors as created by utils.make_category_colors()"""

    if ncols is None:
        ncols = 3

    # Remove unused categorical. These will attempted to be colored (which is wasteful of color space) and plotted (which results in an error)
    colors_df = utils.unused_cat(colors_df)

    # Map values in the dataframe to colors
    # - dataframe with colors
    columns = colors_df.columns
#    print(columns)
    colors_df = pd.concat([colors_df,
                           make_category_colors(None, colors_df, fmt='rgb').rename(columns=lambda x : f'{x}_color')],
                           axis=1)
#    display(colors_df)

    fig, axes = subplots(None, ncols, 3, 1, n=colors_df.shape[1], flatten=True)
    for ax, c in zip(axes, columns):
        data = colors_df[[c, f'{c}_color']].drop_duplicates().copy()
        if len(data)<=1:
            continue

        data['height'] = 1
        # display(data)
        # display(data.dtypes)
        # print(c)
        sns.barplot(x=c, y='height', data=data, ax=ax, palette=data.set_index(c)[f'{c}_color'].to_dict())
        ax.tick_params(labelleft=False, left=False, labelrotation=-45, direction='in')
        ax.set(xlabel=None, ylabel=None)
        for x in ['left', 'bottom', 'top', 'right']:
            ax.spines[x].set_visible(False)
#        ax.set_title(c)
        ax.set_ylabel(c)
    for ax in axes[len(columns) : ]:
        ax.remove()

def make_category_colors(names, assignments, palette_name='husl', fmt='hex', na_color=None):
    """Given a categorization of a set of objects (`names`), assign a
    color to each object by their category. Multiple types of
    categorizations are given as a dataframe

    assignments : dataframe of objects-by-categories

    names : the subset of object to be colored

    na_color : the color used for objects with no category (value of nan in `assignments`)

    """

    import seaborn as sns

    if isinstance(assignments, pd.Series):
        assignments = assignments.to_frame()

    if names is None:
        assignments = assignments.astype('category')
    else:
        assignments = assignments.loc[names].astype('category')

    colors = pd.DataFrame(index=assignments.index)
    n_classes_dict = OrderedDict()
    for c in assignments.columns:
        assignments[c] = assignments[c].cat.remove_unused_categories()
        n_classes = assignments[c].cat.categories.size
        n_classes_dict[c] = n_classes
        palette = sns.color_palette(palette_name, n_classes)

        if fmt=='hex':
            palette = palette.as_hex()
            if na_color is None:
                na_color = '#ffffff'
        elif fmt=='rgb':
            if na_color is None:
                na_color = (1.,1.,1.)

        colors[c] = [palette[code] if (code != -1) else na_color \
                        for code in assignments[c].cat.codes]

    return colors

def subplots(nrows, ncols, width, height, total_height=None, total_width=None,
             flatten=False, width_ratios=None, height_ratios=None, tight_layout=True, n=None, downsize=False, figsize=None, gridspec_kw=None, **kwargs):
    """Convenient wrapper around plt.subplots(). Calculates figsize
    based on (width, height) of each axes"""

    if gridspec_kw is None:
        gridspec_kw = {}

    if width_ratios is not None:
        assert 'width_ratios' not in gridspec_kw
        gridspec_kw['width_ratios'] = width_ratios
    if height_ratios is not None:
        assert 'height_ratios' not in gridspec_kw
        gridspec_kw['height_ratios'] = height_ratios

    if n is not None:
        if ncols is None:
            # Infer number of columns
            ncols = math.ceil(n / nrows)
        else:
            # Infer number of rows
            nrows = math.ceil(n / ncols)

            if downsize and (nrows==1):
                # Decrease number of columns becaues `n` is small
                ncols = min(n, ncols * nrows)

    if figsize is None:
        if total_width is None:
            total_width = ncols * width
        if total_height is None:
            total_height = nrows * height
        figsize = (total_width, total_height)

    fig, axes = plt.subplots(nrows=nrows, ncols=ncols, figsize=figsize,
                             tight_layout=tight_layout, squeeze=False, gridspec_kw=gridspec_kw,
                             **kwargs)
    if flatten:
        axes = axes.flatten()

    return fig, axes

def clustermap(data,
               Z=None,
               method=None,
               figsize=None,
               metric=None,
               colors_ratio=None,
               dendrogram_ratio=None,
               is_affinity=True,
               row_colors=None,
               col_colors=None,
               number_labels=False,
               show_cluster=None,
               show_row_cluster=None,
               show_col_cluster=None,
               verbose_colors=False,
               optimal_ordering=None,
               center=None, cmap=None,
               square=None,
               row_Z=None, col_Z=None,
               verbose=True,
               show=False,
               co_cluster=None, n_clusters=None, random_state=None,
               row_cluster=None, col_cluster=None,
               xticklabels=True,
               yticklabels=True,
               cbar_pos=None,
               convert_row_colors=None,
               **kwargs):
    """
    **** DEPRECATED : See clustermap() *************

    Plots a heatmap. Acts like sns.clustermap, but uses my desired
    default paramters, and does basic formatting I typically want

    Notes on runtime:

    ## Seems like biggest time is to plot pixels
    #
    ## Using distances as-is
    # -- 4m 36s, all plasmids, 16827-by-16827
         suggested figsize=(30,30)
    # -- 2m 20s, 12K-by-12K, ward, figsize=20-by-20, row/col_cluster default or False (this setting doesn't matter).
         But you want to let row/col_cluster=True to reorder the leaves
         suggested figsize=(20,20)
    # -- 34s, 6000-by-6000, ward
    # -- 16s, 6000-by-6000, complete
    ## From scratch, default metric
    # -- 24s for 3000-by-3000
    # -- 3m 12s for 6000-by-6000
    """

    import seaborn as sns

    from scipy.cluster import hierarchy

    if optimal_ordering is None:
        optimal_ordering = False

    if square is None:
        square = False

    if metric is None:
        if square:
            metric = 'precompute'
        else:
            metric = 'euclidean'

    if figsize is None:
        if data.shape[0] > 15000:
            figsize = (32,30)
        elif data.shape[0] > 6000:
            figsize = (22,20)
        elif data.shape[0] > 3000:
            figsize = (17, 15)
        elif data.shape[0] > 100:
            figsize = (11,10)
        elif data.shape[0] > 40:
            figsize = (8.75, 8)
        else:
#            figsize = (5.5, 5)
            figsize = (7.5, 7)

    def rearrange_by_Z(data, data_arr, names, Z, row=True):
        """Rearranges the data rows or cols according to a linkage Z"""

        leaves = hierarchy.leaves_list(Z)
        if row:
            # This linkage represents rows
            data, data_arr, names = data.iloc[leaves,:], data_arr[leaves,:], names[leaves]
        else:
            # This linkage represents cols
            data, data_arr, names = data.iloc[:, leaves], data_arr[:,leaves], names[leaves]

        return data, data_arr, names

    if isinstance(data, pd.DataFrame):
        # Test if dataframe contains sparse datatypes. If so, then convert to dense
        if any('Sparse' in x.name for x in data.dtypes.values):
            data = data.sparse.to_dense()

        rownames, colnames = data.index.values, data.columns.values

        if square and not np.all(rownames==colnames):
            assert Z is None

            # Sync rownames and colnames to make square
            data = data.copy()
            union = data.index.union(data.columns)
            data = data.reindex(union).reindex(columns=union).fillna(0)
            rownames, colnames = data.index.values, data.columns.values

        data_arr = data.values
    else:
        rownames, colnames = np.arange(data.shape[0]), np.arange(data.shape[1])
        data_arr = data
        data = pd.DataFrame(data)

    if verbose:
        print('Plotting shape:', data.shape)

    if square:
        assert data.shape[0]==data.shape[1]
        if Z is not None:
            assert (row_Z is not None) and (col_Z is not None)
            row_Z, col_Z = Z, Z

    if is_affinity:
        # The specified matrix is affinities/similarities (rather than
        # distances). So invert it, before giving it to linkage()
        data_linkage = 1 - data_arr
    else:
        data_linkage = data_arr

    if show_cluster:
        assert (show_row_cluster is None) and (show_col_cluster is None)
        show_row_cluster, show_col_cluster = True, True

    to_rearrange_rows, to_rearrange_cols = False, False
    if co_cluster:
        ### Do bi-clustering using sklearn's SpectralCoclustering() method
        if verbose:
            print('Calculating co-clustering')

        import sklearn.cluster
        if random_state is None:
            random_state = 0
        if n_clusters is None:
            n_clusters = 20
        model = sklearn.cluster.SpectralCoclustering(n_clusters=n_clusters, random_state=random_state)
        # print('wacka', np.isinf(data_arr).sum())
        # print('wacka', data_arr.dtype)

        # Normalize values
        data_arr = sklearn.preprocessing.StandardScaler().fit_transform(data_arr)

        data_arr = data_arr.astype(np.float64) + 1e-10 # Add a small number to avoid floating point errors (don't know why this happens)
#        return data_arr
        model.fit(data_arr)
        data = data.iloc[np.argsort(model.row_labels_), :].iloc[:, np.argsort(model.column_labels_)]
        data_arr = data_arr[np.argsort(model.row_labels_), :][:, np.argsort(model.column_labels_)]

        assert (row_Z is None) and (col_Z is None) and (not show_row_cluster) and (not show_col_cluster)
        show_row_cluster, show_col_cluster = False, False
    else:
        ### Cluster rows and cols separately

        # If either row_Z or col_Z is set, then make sure that
        # row/col_cluster is set to the default None or is not False
        if (row_Z is None) and (row_cluster is None):
            # Default is to cluster rows
            row_cluster = True
        elif row_Z is not None:
            if row_cluster is None:
                row_cluster = True
            else:
                assert row_cluster is not False, "If row_Z is specified, then row_cluster cannot be False"
        if (col_Z is None) and (col_cluster is None):
            # Default is to cluster cols
            col_cluster = True
        elif col_Z is not None:
            if col_cluster is None:
                col_cluster = True
            else:
                assert col_cluster is not False, "If col_Z is specified, then col_cluster cannot be False"

        # print(row_cluster, col_cluster)
        # print(show_row_cluster, show_col_cluster)

        if method is None:
            method = 'ward'

        if row_cluster:
            # Make sure a row linkage is calculated
            if row_Z is None:
                if verbose:
                    utils.tprint('Calculating row linkage with metric={}, method={}, optimal_ordering={}'.format(metric, method, optimal_ordering))
                row_Z = linkage(data_linkage, metric=metric, method=method, optimal_ordering=optimal_ordering)

            if show_row_cluster is None:
                # If row_cluster is True, then default is to show them
                show_row_cluster = True

            if show_row_cluster:
                to_rearrange_rows = True
            else:
                ## If not showing the row dendrogram, then need to internally reorder the rows
                data, data_arr, rownames = rearrange_by_Z(data, data_arr, rownames, row_Z, row=True)
        else:
            assert not show_row_cluster, "Cannot show a clustering when one isn't being calculated"
            show_row_cluster = False

        if col_cluster:
            # Make sure a col linkage is calculated
            if col_Z is None:
                if verbose:
                    utils.tprint('Calculating col linkage with metric={}, method={}, optimal_ordering={}'.format(metric, method, optimal_ordering))
                col_Z = linkage(data_linkage.T, metric=metric, method=method, optimal_ordering=optimal_ordering)

            if show_col_cluster is None:
                # If col_cluster is True, then default is to show them
                show_col_cluster = True

            if show_col_cluster:
                to_rearrange_cols = True
            else:
                ## If not showing the col dendrogram, then need to internally reorder the cols
                data, data_arr, colnames = rearrange_by_Z(data, data_arr, colnames, col_Z, row=False)
                to_rearrange_cols = False
        else:
            assert not show_col_cluster, "Cannot show a clustering when one isn't being calculated"
            show_col_cluster = False

    # Calculate colors for heatmap
    if row_colors is not None:
        row_colors = row_colors.loc[row_colors.index.intersection(data.index)]
        row_colors = utils.unused_cat(row_colors)

        if square:
            assert col_colors is None, "Cannot specify `row_colors` and `col_colors` simultaneously when square==True"
            col_colors = row_colors

        if (convert_row_colors is None) or convert_row_colors:
            plot_colors(row_colors)
            row_colors = make_category_colors(None, row_colors, fmt='rgb')

    if col_colors is not None:
        empty_col_colors = col_colors.reindex(colnames).isna().all()
        if verbose_colors:
            print('These col_colors are all NA across rows: {}'.format(empty_col_colors.index[empty_col_colors]))
            display(col_colors.loc[colnames,~empty_col_colors])

        col_colors = col_colors.loc[col_colors.index.intersection(data.columns)]
        col_colors = utils.unused_cat(col_colors)

#        display(col_colors)
        plot_colors(col_colors)
        # print(colnames)
        # display(col_colors)
        # display(empty_col_colors)
        col_colors = make_category_colors(colnames, col_colors.loc[:,~empty_col_colors], fmt='rgb')

    if colors_ratio is None:
        ncol_colors = 1 if (col_colors is None) else col_colors.shape[1]
        nrow_colors = 1 if (row_colors is None) else row_colors.shape[1]
        colors_ratio = (1/figsize[0] / nrow_colors, 1/figsize[1] / ncol_colors)
        #print('colors_ratio:', colors_ratio)

    if dendrogram_ratio is None:
        dendrogram_ratio = (min(0.1, 2 / figsize[0]), min(0.1, 1 / figsize[1]))

    if cbar_pos=='None':
        cbar_pos = None # A workaround to disable the cbar by setting it to 'None' (the string), rather than None
    elif cbar_pos is None:
        # Set the colorbar to a thin vertical bar
        height = min(0.8, 10/figsize[1])
#        cbar_pos=(0, 0.8 - height, min(2/figsize[0], 0.025), height)
        cbar_pos=(0.02, 0.02, min(2/figsize[0], 0.025), height)
        # print('figsize:', figsize)
        # print('cbar_pos:', cbar_pos)

    if verbose:
        utils.tprint('Figure size:', figsize)
        utils.tprint('colors_ratio:', colors_ratio)
        utils.tprint('dendrogram_ratio:', dendrogram_ratio)
        utils.tprint('cbar_pos:', cbar_pos)
        utils.tprint('row/col_cluster:', row_cluster, col_cluster)
        utils.tprint('show_row/col_cluster:', show_row_cluster, show_col_cluster)
        utils.tprint('row/col_Z is set:', row_Z is not None, col_Z is not None)
        utils.tprint('Plotting')

    fig = sns.clustermap(data,
                         row_cluster=show_row_cluster,
                         col_cluster=show_col_cluster,
                         row_linkage=row_Z,
                         col_linkage=col_Z,
                         figsize=figsize,
                         colors_ratio=colors_ratio,
                         dendrogram_ratio=dendrogram_ratio,
                         row_colors=row_colors,
                         col_colors=col_colors,
                         center=center,
                         cmap=cmap,
                         cbar_pos=cbar_pos,
                         **kwargs
    )
    fig.ax_heatmap.tick_params(axis='x', which='both', labelbottom=xticklabels)
    fig.ax_heatmap.tick_params(axis='y', which='both', labelbottom=yticklabels)

    # # Add a tick to every row
    # fig.ax_heatmap.set_yticks(np.arange(min(fig.ax_heatmap.get_ylim()) + 0.5, max(fig.ax_heatmap.get_ylim()) + 0.5, 1))

    if show:
        if verbose:
            utils.tprint('Showing Figure')
        plt.show()
    else:
        if verbose:
            utils.tprint('Done')

    if to_rearrange_rows:
        if verbose:
            utils.tprint('Rearranging rows')
        data, _, _ = rearrange_by_Z(data, data_arr, rownames, row_Z, row=True)
    if to_rearrange_cols:
        if verbose:
            utils.tprint('Rearranging cols')
        data, _, _ = rearrange_by_Z(data, data_arr, colnames, col_Z, row=False)

    return data, fig, row_Z, col_Z


import itertools

import matplotlib.pyplot as plt
import pytorch_lightning as pl
import torch
import wandb
from fish_audio_preprocess.utils.loudness_norm import loudness_norm
from mmengine.optim import OPTIMIZERS
from pytorch_lightning.loggers import TensorBoardLogger, WandbLogger
from torch.nn import functional as F

from fish_diffusion.modules.vocoders.nsf_hifigan.models import (
    AttrDict,
    MultiPeriodDiscriminator,
    MultiScaleDiscriminator,
    discriminator_loss,
    feature_loss,
    generator_loss,
)
from fish_diffusion.schedulers import LR_SCHEUDLERS
from fish_diffusion.utils.audio import dynamic_range_compression, get_mel_transform
from fish_diffusion.utils.viz import plot_mel

from .core import HiFiSinger

class HiFiSingerV1Lightning(pl.LightningModule):
    def __init__(self, config):
        super().__init__()

        self.h = AttrDict(config.model.encoder)
        self.config = config

        self.generator = HiFiSinger(config.model)
        self.mpd = MultiPeriodDiscriminator(self.h["discriminator_periods"])
        self.msd = MultiScaleDiscriminator()

        # This is for validation only
        self.mel_transform = get_mel_transform(
            sample_rate=config.sampling_rate,
            n_fft=self.h.n_fft,
            hop_length=self.h.hop_size,
            win_length=self.h.win_size,
            f_min=self.h.fmin,
            f_max=self.h.fmax,
            n_mels=self.h.num_mels,
        )

        # The below are for training
        self.multi_scale_mels = [
            get_mel_transform(
                sample_rate=config.sampling_rate,
                n_fft=n_fft,
                hop_length=hop_length,
                win_length=win_length,
                # Shouldn't ise fmin and fmax here
                # Otherwise high frequency will be cut off
                f_min=0,
                f_max=config.sampling_rate // 2,
                n_mels=self.h.num_mels,
            )
            for (n_fft, hop_length, win_length) in self.h.multi_scale_mels
        ]

        self.automatic_optimization = False

    def configure_optimizers(self):
        optim_g = OPTIMIZERS.build(
            {
                "params": self.generator.parameters(),
                **self.config.optimizer,
            }
        )
        optim_d = OPTIMIZERS.build(
            {
                "params": itertools.chain(self.msd.parameters(), self.mpd.parameters()),
                **self.config.optimizer,
            }
        )

        scheduler_g = LR_SCHEUDLERS.build(
            {
                "optimizer": optim_g,
                **self.config.scheduler,
            }
        )
        scheduler_d = LR_SCHEUDLERS.build(
            {
                "optimizer": optim_d,
                **self.config.scheduler,
            }
        )

        return [optim_g, optim_d], [scheduler_g, scheduler_d]

    def training_step(self, batch, batch_idx):
        optim_g, optim_d = self.optimizers()

        y = batch["audio"].float()

        y_g_hat = self.generator(
            speakers=batch["speaker"],
            contents=batch["contents"],
            contents_lens=batch["contents_lens"],
            contents_max_len=batch["contents_max_len"],
            pitches=batch["pitches"],
            pitch_shift=batch.get("key_shift", None),
            phones2mel=batch.get("phones2mel", None),
            energy=batch.get("energy", None),
        )

        optim_d.zero_grad()

        # MPD
        y_df_hat_r, y_df_hat_g, _, _ = self.mpd(y, y_g_hat.detach())
        loss_disc_f, _, _ = discriminator_loss(y_df_hat_r, y_df_hat_g)

        # MSD
        y_ds_hat_r, y_ds_hat_g, _, _ = self.msd(y, y_g_hat.detach())
        loss_disc_s, _, _ = discriminator_loss(y_ds_hat_r, y_ds_hat_g)

        loss_disc_all = loss_disc_s + loss_disc_f

        self.manual_backward(loss_disc_all)
        optim_d.step()

        self.log(
            f"train_loss_disc",
            loss_disc_all,
            on_step=True,
            on_epoch=True,
            prog_bar=True,
            logger=True,
            sync_dist=True,
            batch_size=batch["contents"].shape[0],
        )

        # Generator
        optim_g.zero_grad()

        # We referenced STFT and Mel-Spectrogram loss from SingGAN
        # L1 STFT Loss

        loss_stft = 0
        for n_fft, hop_length, win_length in self.h.multi_scale_stfts:
            y_stft = torch.stft(
                y.squeeze(1), n_fft, hop_length, win_length, return_complex=True
            )
            y_g_hat_stft = torch.stft(
                y_g_hat.squeeze(1), n_fft, hop_length, win_length, return_complex=True
            )
            y_stft = torch.view_as_real(y_stft)
            y_g_hat_stft = torch.view_as_real(y_g_hat_stft)

            loss_stft += F.l1_loss(y_stft, y_g_hat_stft)

        loss_stft /= len(self.h.multi_scale_stfts)

        # L1 Mel-Spectrogram Loss
        loss_mel = 0
        for mel_transform in self.multi_scale_mels:
            y_mel = self.get_mels(y, mel_transform)
            y_g_hat_mel = self.get_mels(y_g_hat, mel_transform)
            loss_mel += F.l1_loss(y_mel, y_g_hat_mel)

        loss_mel /= len(self.multi_scale_mels)

        loss_aux = 0.5 * loss_stft + loss_mel

        # Discriminator Loss
        y_df_hat_r, y_df_hat_g, fmap_f_r, fmap_f_g = self.mpd(y, y_g_hat)
        y_ds_hat_r, y_ds_hat_g, fmap_s_r, fmap_s_g = self.msd(y, y_g_hat)
        loss_fm_f = feature_loss(fmap_f_r, fmap_f_g)
        loss_fm_s = feature_loss(fmap_s_r, fmap_s_g)
        loss_gen_f, _ = generator_loss(y_df_hat_g)
        loss_gen_s, _ = generator_loss(y_ds_hat_g)
        loss_gen_all = loss_gen_s + loss_gen_f + loss_fm_s + loss_fm_f + loss_aux * 45

        self.manual_backward(loss_gen_all)
        optim_g.step()

        self.log(
            f"train_loss_gen",
            loss_gen_all,
            on_step=True,
            on_epoch=True,
            prog_bar=True,
            logger=True,
            sync_dist=True,
            batch_size=batch["contents"].shape[0],
        )

        self.log(
            f"train_loss_aux",
            loss_aux,
            on_step=True,
            on_epoch=False,
            prog_bar=False,
            logger=True,
            sync_dist=True,
            batch_size=batch["contents"].shape[0],
        )

        # Manual LR Scheduler
        scheduler_g, scheduler_d = self.lr_schedulers()

        if self.trainer.is_last_batch:
            scheduler_g.step()
            scheduler_d.step()

    def get_mels(self, audios, transform=None):
        if transform is None:
            transform = self.mel_transform

        transform = transform.to(audios.device)
        x = transform(audios.squeeze(1))

        return dynamic_range_compression(x)

    def validation_step(self, batch, batch_idx):
        y_g_hat = self.generator(
            speakers=batch["speaker"],
            contents=batch["contents"],
            contents_lens=batch["contents_lens"],
            contents_max_len=batch["contents_max_len"],
            pitches=batch["pitches"],
            pitch_shift=batch.get("key_shift", None),
            phones2mel=batch.get("phones2mel", None),
            energy=batch.get("energy", None),
        )

        mels = self.get_mels(batch["audio"])
        y_g_hat_mel = self.get_mels(y_g_hat)[:, :, : mels.shape[2]]

        # L1 Mel-Spectrogram Loss
        mel_lens = batch["contents_lens"]

        # create mask
        mask = (
            torch.arange(mels.shape[2], device=mels.device)[None, :] < mel_lens[:, None]
        )
        mask = mask[:, None].float()

        loss_mel = F.l1_loss(mels * mask, y_g_hat_mel * mask)
        self.log(
            "valid_loss",
            loss_mel,
            on_step=True,
            on_epoch=True,
            prog_bar=True,
            logger=True,
            sync_dist=True,
            batch_size=batch["contents"].shape[0],
        )

        for idx, (mel, gen_mel, audio, gen_audio, mel_len, audio_len) in enumerate(
            zip(
                mels.cpu().numpy(),
                y_g_hat_mel.cpu().numpy(),
                batch["audio"].cpu().type(torch.float32).numpy(),
                y_g_hat.type(torch.float32).cpu().numpy(),
                batch["contents_lens"].cpu().numpy(),
                batch["audio_lens"].cpu().numpy(),
            )
        ):
            image_mels = plot_mel(
                [
                    gen_mel[:, :mel_len],
                    mel[:, :mel_len],
                ],
                ["Sampled Spectrogram", "Ground-Truth Spectrogram"],
            )

            wav_gt = loudness_norm(audio[0, :audio_len], 44100, block_size=0.1)
            wav_prediction = loudness_norm(
                gen_audio[0, :audio_len], 44100, block_size=0.1
            )

            if isinstance(self.logger, WandbLogger):
                self.logger.experiment.log(
                    {
                        f"reconstruction_mel": wandb.Image(image_mels, caption="mels"),
                        f"wavs": [
                            wandb.Audio(
                                wav_gt,
                                sample_rate=self.config.sampling_rate,
                                caption=f"gt",
                            ),
                            wandb.Audio(
                                wav_prediction,
                                sample_rate=self.config.sampling_rate,
                                caption=f"prediction",
                            ),
                        ],
                    },
                )

            if isinstance(self.logger, TensorBoardLogger):
                self.logger.experiment.add_figure(
                    f"sample-{idx}/mels",
                    image_mels,
                    global_step=self.global_step,
                )
                self.logger.experiment.add_audio(
                    f"sample-{idx}/wavs/gt",
                    wav_gt,
                    self.global_step,
                    sample_rate=self.config.sampling_rate,
                )
                self.logger.experiment.add_audio(
                    f"sample-{idx}/wavs/prediction",
                    wav_prediction,
                    self.global_step,
                    sample_rate=self.config.sampling_rate,
                )

            plt.close(image_mels)


from copy import deepcopy
from random import choice
from typing import Dict, List, Tuple

class Headers:
    def __init__(self, options: Dict[str, str] | None = None) -> None:
        self.options = options or {}

        self.chrome_version: Tuple[str, str] | None = None

        # sec-ch-ua
        self.sec_ch_ua_conf: Dict | None = None

        self.sec_ch_ua: str | None = None
        self.sec_ch_ua_full_version: str | None = None
        self.sec_ch_ua_full_version_list: str | None = None
        self.sec_ch_ua_platform: str | None = None
        self.sec_ch_ua_platform_version: str | None = None
        self.sec_ch_ua_arch: str | None = None
        self.sec_ch_ua_bitness: str | None = None
        self.sec_ch_ua_mobile: str | None = None
        self.sec_ch_ua_model: str | None = None

        # user-agent
        self.user_agent: str | None = None

    @property
    def headers(self) -> Dict[str, str]:
        return self.options

class ChromeHeadersBuilder:
    CHROME_VERSIONS: Dict[str, List[str]] = {
        "100": [
            "100.0.4896.127",  # ~  04-2022
            "100.0.4896.143",  # ~  04-2022
        ],
        "101": [
            "101.0.4951.41",  # ~   04-2022
            "101.0.4951.67",  # ~   05-2022
        ],
        "102": [
            "102.0.5005.61",  # ~   05-2022
            "102.0.5005.62",  # ~   05-2022
            "102.0.5005.63",  # ~   05-2022
            "102.0.5005.134",  # ~  06-2022
            "102.0.5005.167",  # ~  07-2022
        ],
        "103": [
            "103.0.5060.53",  # ~   06-2022
            "103.0.5060.66",  # ~   06-2022
            "103.0.5060.134",  # ~  07-2022
        ],
        "104": [
            "104.0.5112.101",  # ~  08-2022
            "104.0.5112.111",  # ~  08-2022
        ],
        "105": [
            "105.0.5195.52",  # ~   08-2022
        ],
        "106": [
            "106.0.5249.61",  # ~   09-2022
            "106.0.5249.91",  # ~   09-2022
            "106.0.5249.165",  # ~  10-2022
            "106.0.5249.168",  # ~  10-2022
            "106.0.5249.181",  # ~  11-2022
            "106.0.5249.199",  # ~  11-2022
        ],
        "107": [
            "107.0.5304.62",  # ~   10-2022
            "107.0.5304.87",  # ~   10-2022
            "107.0.5304.110",  # ~  11-2022
            "107.0.5304.121",  # ~  11-2022
        ],
        "108": [
            "108.0.5359.71",  # ~   11-2022
            "108.0.5359.94",  # ~   12-2022
            "108.0.5359.98",  # ~   12-2022
            "108.0.5359.124",  # ~  12-2022
            "108.0.5359.179",  # ~  01-2023
            "108.0.5359.215",  # ~  01-2023
        ],
        "109": [
            "109.0.5414.74",  # ~   01-2023
            "109.0.5414.119",  # ~  01-2023
        ],
        "110": [
            "110.0.5481.77",  # ~   02-2023
            "110.0.5481.208",  # ~  03-2023
        ],
        "111": [
            "111.0.5563.110",  # ~  03-2023
            "111.0.5563.147",  # ~  03-2023
        ],
        "112": [
            "112.0.5615.49",  # ~   04-2023
            "112.0.5615.86",  # ~   04-2023
            "112.0.5615.87",  # ~   04-2023
            "112.0.5615.121",  # ~  04-2023
            "112.0.5615.137",  # ~  04-2023
            "112.0.5615.138",  # ~  04-2023
        ],
        "113": [
            "113.0.5672.126",  # ~  05-2023
        ],
        "114": [
            "114.0.5735.90",  # ~   05-2023
            "114.0.5735.133",  # ~  06-2023
            "114.0.5735.198",  # ~  06-2023
        ],
    }

    # sec-ch-ua
    __SEC_CH_UA_BASE_POS: str = "{}, {}, {}"
    __SEC_CH_UA_ITEMS: Dict[int, str] = {
        1: '"Chromium";v="{v}"',
        2: '"Google Chrome";v="{v}"',
        3: '"Not{spec_1}A{spec_2}Brand";v="{v}"',
    }
    __SEC_CH_UA_PLATFORM: List[str] = ["Linux", "Windows"]
    __SEC_CH_UA_PLATFORM_VERSION: Dict[str, List[str]] = {  # OS versions numbers
        "Linux": [
            "6.4.0",
            "6.3.0",
            "6.2.0",
            "6.1.0",
            "6.0.0",
            "5.19.0",
            "5.18.0",
            "5.17.0",
            "5.16.0",
            "5.15.0",
        ],
        "Windows": ["13.0.0", "10.0.0"],
    }
    __SEC_CH_UA_ARCH: List[str] = ["x86"]
    __SEC_CH_UA_BITNESS: List[str] = ["64", "32"]
    __SEC_CH_UA_MODEL: List[str] = ['""']
    __SEC_CH_UA_MOBILE: List[str] = [
        "?0",
        # '?1',
    ]

    # user-agent
    __USER_AGENT_STRING: str = (
        "Mozilla/5.0 ({system_information}) {platform} ({platform_details}) {extensions}"
    )
    __USER_AGENT_SYSTEM_INFORMATION: Dict[str, Dict[str, str] | Dict[str, str]] = {
        "Linux": {"64": "X11; Linux x86_64", "32": "X11; Linux x86_64"},
        "Windows": {
            "64": "Windows NT 10.0; Win64; x64",
            "32": "Windows NT 10.0",
        },
    }
    __USER_AGENT_PLATFORM: Tuple[str, str] = ("AppleWebKit", "537.36")
    __USER_AGENT_PLATFORM_DETAILS: str = "KHTML, like Gecko"
    __USER_AGENT_EXTENSIONS: str = "Chrome/{chrome_v}.0.0.0 Safari/{safari_v}"

    def __random_chrome_version(self) -> Tuple[str, str]:
        ver_num = choice(list(self.CHROME_VERSIONS.keys()))
        ver_val = choice(self.CHROME_VERSIONS.get(ver_num))

        return ver_num, ver_val

    def __build_extensions(self, item: Headers) -> str:
        return self.__USER_AGENT_EXTENSIONS.format(
            chrome_v=item.chrome_version[0],
            safari_v=self.__USER_AGENT_PLATFORM[1],
        )

    @classmethod
    def __random_pick(cls, array: List, remove: bool = True) -> str:
        item = choice(array)

        if remove:
            array.remove(item)

        return item

    @classmethod
    def __not_a_brand__random(cls) -> Tuple[str, str, str, str]:
        # 8 = './'
        # 99 = '_ ' ':-' ' :' ' ;'
        # 24 = '-.' '=?'

        spec_symbols = {
            "8": [
                "./",
            ],
            "24": [
                "-.",
                "=?",
            ],
            "99": [
                "_ ",
                ":-",
                " :",
                " ;",
                "/)",  # 115
            ],
        }

        not_a_brand_v = ["99", "24", "8"]

        v = choice(not_a_brand_v)
        v_full = f"{v}.0.0.0"
        spec_1, spec_2 = choice(spec_symbols.get(v, [0]))

        return v, v_full, spec_1, spec_2

    def __sec_ch_ua_conf__random(self) -> Dict:
        keys = list(self.__SEC_CH_UA_ITEMS.keys())

        return {
            "pos": (
                self.__random_pick(keys),
                self.__random_pick(keys),
                self.__random_pick(keys),
            ),
            "not_a_brand": self.__not_a_brand__random(),
        }

    @classmethod
    def __sec_ch_ua_full_version__random(cls, item: Headers) -> str:
        return f"{item.chrome_version[1]}"

    def __positioning(self, position: int, item: Headers) -> Tuple[str, str, str]:
        sec_ch_ua_items = deepcopy(self.__SEC_CH_UA_ITEMS)

        sec_ch_ua_items[1] = sec_ch_ua_items.get(1).format(
            v=item.chrome_version[position],
        )
        sec_ch_ua_items[2] = sec_ch_ua_items.get(2).format(
            v=item.chrome_version[position],
        )
        sec_ch_ua_items[3] = sec_ch_ua_items.get(3).format(
            v=item.sec_ch_ua_conf["not_a_brand"][position],
            spec_1=item.sec_ch_ua_conf["not_a_brand"][2],
            spec_2=item.sec_ch_ua_conf["not_a_brand"][3],
        )

        pos_1 = sec_ch_ua_items[item.sec_ch_ua_conf["pos"][0]]
        pos_2 = sec_ch_ua_items[item.sec_ch_ua_conf["pos"][1]]
        pos_3 = sec_ch_ua_items[item.sec_ch_ua_conf["pos"][2]]

        return pos_1, pos_2, pos_3

    def __sec_ch_ua__random(self, item: Headers) -> str:
        return self.__SEC_CH_UA_BASE_POS.format(*self.__positioning(0, item))

    def __sec_ch_ua_full_version_list__random(self, item: Headers) -> str:
        return self.__SEC_CH_UA_BASE_POS.format(*self.__positioning(1, item))

    def __sec_ch_ua_platform__random(self) -> str:
        return choice(self.__SEC_CH_UA_PLATFORM)

    def __sec_ch_ua_platform_version__random(self, item: Headers) -> str:
        return choice(self.__SEC_CH_UA_PLATFORM_VERSION[item.sec_ch_ua_platform])

    def __sec_ch_ua_arch__random(self) -> str:
        return choice(self.__SEC_CH_UA_ARCH)

    def __sec_ch_ua_bitness_random(self) -> str:
        return choice(self.__SEC_CH_UA_BITNESS)

    def __sec_ch_ua_model_random(self) -> str:
        return choice(self.__SEC_CH_UA_MODEL)

    def __sec_ch_ua_mobile__random(self) -> str:
        return choice(self.__SEC_CH_UA_MOBILE)

    def __user_agent__random(self, item: Headers) -> str:
        platform = f"{self.__USER_AGENT_PLATFORM[0]}/{self.__USER_AGENT_PLATFORM[1]}"

        extensions = self.__build_extensions(item)
        system_information = self.__USER_AGENT_SYSTEM_INFORMATION[
            item.sec_ch_ua_platform
        ][item.sec_ch_ua_bitness]

        return self.__USER_AGENT_STRING.format(
            system_information=system_information,
            platform=platform,
            platform_details=self.__USER_AGENT_PLATFORM_DETAILS,
            extensions=extensions,
        )

    @classmethod
    def add__sec_ch_ua(cls, item: Headers) -> None:
        """
        Set sec-ch-ua headers:
            - sec-ch-ua
            - sec-ch-ua-full-version
            - sec-ch-ua-full-version-list
            - sec-ch-ua-platform
            - sec-ch-ua-arch
            - sec-ch-ua-bitness
            - sec-ch-ua-mobile
            - sec-ch-ua-model
        """
        item.options["sec-ch-ua"] = item.sec_ch_ua
        item.options["sec-ch-ua-full-version"] = item.sec_ch_ua_full_version
        item.options["sec-ch-ua-full-version-list"] = item.sec_ch_ua_full_version_list
        item.options["sec-ch-ua-platform"] = item.sec_ch_ua_platform
        item.options["sec-ch-ua-platform-version"] = item.sec_ch_ua_full_version_list
        item.options["sec-ch-ua-arch"] = item.sec_ch_ua_arch
        item.options["sec-ch-ua-bitness"] = item.sec_ch_ua_bitness
        item.options["sec-ch-ua-mobile"] = item.sec_ch_ua_mobile
        item.options["sec-ch-ua-model"] = item.sec_ch_ua_model

    @classmethod
    def add__user_agent(cls, item: Headers) -> None:
        """
        Set user-agent header
        """
        item.options["user-agent"] = item.user_agent

    def randomize_headers(self, headers: Headers) -> Dict[str, str]:
        return self.__randomize(headers).headers

    def __randomize(self, item: Headers) -> Headers:
        # Chrome version
        item.chrome_version = self.__random_chrome_version()

        # sec-ch-ua
        item.sec_ch_ua_conf = self.__sec_ch_ua_conf__random()

        item.sec_ch_ua = self.__sec_ch_ua__random(item)
        item.sec_ch_ua_full_version = self.__sec_ch_ua_full_version__random(item)
        item.sec_ch_ua_full_version_list = self.__sec_ch_ua_full_version_list__random(
            item,
        )
        item.sec_ch_ua_platform = self.__sec_ch_ua_platform__random()
        item.sec_ch_ua_platform_version = self.__sec_ch_ua_platform_version__random(item)
        item.sec_ch_ua_mobile = self.__sec_ch_ua_mobile__random()
        item.sec_ch_ua_arch = self.__sec_ch_ua_arch__random()
        item.sec_ch_ua_bitness = self.__sec_ch_ua_bitness_random()
        item.sec_ch_ua_model = self.__sec_ch_ua_model_random()

        self.add__sec_ch_ua(item)

        # user-agent
        item.user_agent = self.__user_agent__random(item)

        self.add__user_agent(item)

        return item


import streamlit as st
import pandas as pd
import scanpy
from SinCel_funcs import *

st.set_page_config(layout="wide")
#Navigation bar
#st.markdown('<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-TX8t27EcRE3e/ihU7zmQxVncDAy5uIKz4rEkgIXeMed4M0jlfIDPvg6uqKI2xXr2" crossorigin="anonymous">', unsafe_allow_html=True)
#
#st.markdown("""
#
#  ScRNA-pipl
#  <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
#
#  </button>
#

#

#

#        Home (current)
#

#

#

#
#""", unsafe_allow_html=True)

print('Running scRNA Pipeline APP')

TITLE = st.container()
#MARKDOWN = st.container()
#FILE_UPLOADER = st.container()

##st.markdown(
##    '<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@4.5.3/dist/css/bootstrap.min.css" integrity="sha384-TX8t27EcRE3e/ihU7zmQxVncDAy5uIKz4rEkgIXeMed4M0jlfIDPvg6uqKI2xXr2" crossorigin="anonymous">',
##    unsafe_allow_html=True,
##)
##
##query_params = st.experimental_get_query_params()
##tabs = ["Home", "About", "Contact"]
##if "tab" in query_params:
##    active_tab = query_params["tab"][0]
##else:
##    active_tab = "Home"
##
##if active_tab not in tabs:
##    st.experimental_set_query_params(tab="Home")
##    active_tab = "Home"
##
##li_items = "".join(
##    f"""
##

##        {t}
##

##    """
##    for t in tabs
##)
##tabs_html = f"""
##

##    {li_items}
##

##"""
##st.markdown(tabs_html, unsafe_allow_html=True)
##st.markdown("
", unsafe_allow_html=True)

with TITLE:
    st.title('Single Cell Analysis Tool Kit')
    st.markdown('
Catalogue of tools for analysing single-cell sequencing data to identifying biomarkers with added exploratory data analysis tools and visualization tools.This tool also supports between the cell types and within the cell conditions
', \
                unsafe_allow_html = True)

def file_uploader():
    col1, col2, col3 = st.columns([3, 1, 1])
    with open('apps/STYLE.css') as f:
        st.markdown(f'', unsafe_allow_html=True)
    with col1:
        uploaded_file = st.file_uploader("Choose .h5ad file")
    if uploaded_file is not None:
        dataf = scanpy.read_h5ad(uploaded_file)
        ob, va = dataf.shape
        all_celltypes, all_samples = dataf.obs['cell_type'].nunique(), dataf.obs['patient_id'].nunique()
        with col2:
            st.markdown(f'''>
Cells

{ob}
''',  unsafe_allow_html=True)
        with col2:
            st.markdown(f'''>
Genes

{va}''',  unsafe_allow_html=True)
        with col3:
            st.markdown(f'''>
Cell Types

{all_celltypes}
''',  unsafe_allow_html=True)
        with col3:
            st.markdown(f'''>
Total samples

{all_samples}
''',  unsafe_allow_html=True)
        return dataf, True
    else:
        st.markdown('
Upload your file
', \
            unsafe_allow_html = True)
        ob, va, all_celltypes, all_samples = 0,0,0,0
        with col2:
            st.markdown(f'''>
Cells

{ob}
''',  unsafe_allow_html=True)
        with col2:
            st.markdown(f'''>
Genes

{va}''',  unsafe_allow_html=True)
        with col3:
            st.markdown(f'''>
Cell Types

{all_celltypes}
''',  unsafe_allow_html=True)
        with col3:
            st.markdown(f'''>
Total samples

{all_samples}
''',  unsafe_allow_html=True)
        return False,False

menu = st.sidebar.radio(
    "",
    ("Summary", "Quality Control", "Vizualization", "Differential Gene Expression"),
)

fileup_container = st.container()

with open('apps/STYLE.css') as f:
    st.markdown(f'', unsafe_allow_html=True)

adata, upfile = file_uploader()
if upfile:
    if menu == 'Summary':
        meta_info(adata)
    elif menu == 'Quality Control':
        qc_info(adata)
    elif menu == 'Vizualization':
        viz_info(adata)
    elif menu == 'Differential Gene Expression':
        deg_info(adata)

st.sidebar.markdown('---')
st.sidebar.write('contact raghukiran812@gmail.com for suggestions')

#9603986566
#31dfa4


# -*- coding: utf-8 -*-
"""
Created on Thu Jan 12 15:09:23 2023

@author: ramz
"""

from PyQt5 import QtWidgets, QtGui, QtCore
# from PyQt5.QtWidgets import QMainWindow, QApplication
from PyQt5.QtCore import QRunnable, QObject
import traceback, sys
import flash_script_sdk
import flash_script_ssh
import subprocess
import re
from concurrent.futures import ThreadPoolExecutor
import shutil

class WorkerSignals(QObject):
    finished = QtCore.pyqtSignal()
    error = QtCore.pyqtSignal(tuple)
    result = QtCore.pyqtSignal(object)
    wrote = QtCore.pyqtSignal(str)

class FlashSDKWorker(QRunnable):
    def __init__(self, *args, **kwargs):
        super(FlashSDKWorker, self).__init__()
        self.args = args
        self.popen = args[0]
        self.kwargs = kwargs
        self.signals = WorkerSignals()

    def run(self):
        try:
            for i in iter(self.popen.stdout.readline,""):
                if(i != '\n'):
                    self.signals.wrote.emit(i)

            return_code = self.popen.wait()
            if return_code:
                raise subprocess.CalledProcessError(return_code, 'flash')

        except:
            traceback.print_exc()
            exectype,value = sys.exc_info()[:2]
            self.signals.error.emit((exectype,value,traceback.format_exc()))
        finally:
            self.popen.stdout.close()
            self.popen.kill()
            self.signals.finished.emit()

class FlashSSHWorker(QRunnable):
    def __init__(self, *args, **kwargs):
        super(FlashSSHWorker, self).__init__()
        self.args = args
        self.image_path = args[0]
        self.ip_address_server = args[1]
        self.kwargs = kwargs
        self.signals = WorkerSignals()

    def run(self):
        try:
            with ThreadPoolExecutor(max_workers=5) as executor:
                flash_hilo = executor.submit(flash_script_ssh.flash_ssh,self.image_path,self.ip_address_server)
                stdout,stderr,error = flash_hilo.result()
                self.signals.result.emit((stdout,stderr,error))
        except:
            traceback.print_exc()
            exectype,value = sys.exc_info()[:2]
            self.signals.error.emit((exectype,value,traceback.format_exc()))
        finally:
            self.signals.finished.emit()

class FlashWindow(QtWidgets.QWidget):

    def __init__(self,server_host):
        super().__init__()
        self.ip_address_server = server_host
        self.setWindowTitle("Program Flash Memory")
        self.setWindowIcon(QtGui.QIcon('resources/icons/Flash.png'))
        self.setGeometry(650, 200, 700, 500)
        self.setFixedSize(700, 500)

        self.thread_pool_flash = QtCore.QThreadPool.globalInstance()

        self.flash_main_layout = QtWidgets.QVBoxLayout(self)

        self.flash_file_layout = QtWidgets.QHBoxLayout()
        spacerItem1 = QtWidgets.QSpacerItem(20, 40, QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Fixed)
        self.flash_file_layout.addItem(spacerItem1)

        self.flash_label = QtWidgets.QLabel()
        self.flash_file_layout.addWidget(self.flash_label)

        self.flash_line = QtWidgets.QLineEdit()
        self.flash_line.setDragEnabled(True)
        self.flash_file_layout.addWidget(self.flash_line)

        spacerItem2 = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Minimum)
        self.flash_file_layout.addItem(spacerItem2)

        self.browse_button = QtWidgets.QPushButton()
        self.flash_file_layout.addWidget(self.browse_button)

        self.flash_main_layout.addLayout(self.flash_file_layout)

        spacerItem3 = QtWidgets.QSpacerItem(20, 40, QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Fixed)
        self.flash_main_layout.addItem(spacerItem3)

        self.select_mode_layout = QtWidgets.QHBoxLayout()

        spacerItem14 = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Minimum)
        self.select_mode_layout.addItem(spacerItem14)
        self.mode_box = QtWidgets.QComboBox()

        self.mode_box.addItem("")
        self.mode_box.addItem("")
        self.mode_box.addItem("")

        self.select_mode_layout.addWidget(self.mode_box)

        self.select_mode_layout.addItem(spacerItem14)
        self.flash_main_layout.addLayout(self.select_mode_layout)
        self.flash_main_layout.addItem(spacerItem3)

        self.buttons_layout = QtWidgets.QHBoxLayout()

        spacerItem4 = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Minimum)
        self.buttons_layout.addItem(spacerItem4)

        self.cancel_flash_button = QtWidgets.QPushButton()
        self.buttons_layout.addWidget(self.cancel_flash_button)

        spacerItem5 = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Minimum)
        self.buttons_layout.addItem(spacerItem5)

        self.program_flash_button = QtWidgets.QPushButton()
        self.buttons_layout.addWidget(self.program_flash_button)

        spacerItem6 = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Minimum)
        self.buttons_layout.addItem(spacerItem6)

        self.flash_main_layout.addLayout(self.buttons_layout)

        spacerItem7 = QtWidgets.QSpacerItem(20, 20, QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Fixed)
        self.flash_main_layout.addItem(spacerItem7)

        self.timer_label_layout = QtWidgets.QHBoxLayout()

        spacerItem8 = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed)
        self.timer_label_layout.addItem(spacerItem8)

        self.timer_name_label = QtWidgets.QLabel()
        self.timer_name_label.setStyleSheet('font-size: 16px;font-weight:normal;border:0px;')
        self.timer_label_layout.addWidget(self.timer_name_label)

        self.timer_value_label = QtWidgets.QLabel()
        self.timer_value_label.setStyleSheet('font-size: 16px;font-weight:normal;border:0px;')
        self.timer_label_layout.addWidget(self.timer_value_label)
        self.flash_main_layout.addLayout(self.timer_label_layout)

        self.timer_label_layout.addItem(spacerItem8)

        # spacerItem7 = QtWidgets.QSpacerItem(20, 40, QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Fixed)
        self.flash_main_layout.addItem(spacerItem7)

        self.flash_log_text = QtWidgets.QTextBrowser()
        self.flash_log_text.setReadOnly(True)
        self.flash_main_layout.addWidget(self.flash_log_text)

        self.flash_settings = QtCore.QSettings('atp_settings')

        if self.flash_settings.contains('image_path'):
            self.image_path = self.flash_settings.value('image_path')
            self.flash_line.setText(self.image_path[0])
        else:
            self.image_path = ''

        self.retranslateUi()

    def retranslateUi(self):
         self.flash_label.setText("Image File")
         self.browse_button.setText("Browse")
         self.cancel_flash_button.setText("Cancel")
         self.program_flash_button.setText("Program")
         self.mode_box.setItemText(0, 'JTAG Mode - Secure Boot')
         self.mode_box.setItemText(1, 'JTAG Mode - SDK')
         self.mode_box.setItemText(2, 'SSH Mode')
         self.timer_name_label.setText("Timer: ")
         self.timer_value_label.setText("0.0")

         self.browse_button.clicked.connect(self.browse_image_file)
         self.cancel_flash_button.clicked.connect(self.cancel_flash)
         self.program_flash_button.clicked.connect(self.program_flash)

         self.timer_count = 0
         self.timer_minut = 0
         self.timer_flag = False

         self.timer = QtCore.QTimer(self)
         self.timer.timeout.connect(self.showTime)
         self.timer.start(100)

    # Overwrite Close
    def closeEvent(self,a0):
        self.flash_log_text.clear()

    def browse_image_file(self):
        aux = QtWidgets.QFileDialog.getOpenFileName(self,filter='*.BIN')
        if aux[0] != "":
            self.image_path = aux
            self.flash_line.setText(self.image_path[0])
            self.flash_settings.setValue('image_path', aux)

    def cancel_flash(self):
        self.program_flash_button.blockSignals(False)
        self.flash_log_text.clear()
        try:
            self.popen.kill()
        except:
            pass
        finally:
            self.close()

    def program_flash(self):

        self.timer_flag = True
        self.timer_value_label.repaint()

        if self.image_path[0] != '':
            aux = self.image_path[0].split("/")
            if re.match(r"^.+(BOOT\.BIN)$",aux[-1]):
                aux2 = [ e+"/" for e in aux[:-1]]
                aux2.append("BOOT.BIN")
                target = ''.join(aux2)

                shutil.copyfile(self.image_path[0], target)
                self.image_path = list(self.image_path)
                self.image_path[0] = target

            mode = self.mode_box.currentIndex()
            self.program_flash_button.setText('Flashing...')
            self.flash_log_text.clear()
            self.program_flash_button.repaint()
            self.program_flash_button.blockSignals(True)
            self.cancel_flash_button.blockSignals(True)

            try :
                with ThreadPoolExecutor(max_workers=5) as executor:
                    if mode == 0:
                        flash_hilo = executor.submit(flash_script_sdk.flash_sdk,self.image_path[0],'0x40000')

                        self.popen = flash_hilo.result()

                        self.flash_worker = FlashSDKWorker(self.popen)
                        self.thread_pool_flash.start(self.flash_worker)
                        self.flash_worker.signals.wrote.connect(self.print_log)
                        self.flash_worker.signals.finished.connect(self.end_flash)

                    if mode == 1:
                        flash_hilo = executor.submit(flash_script_sdk.flash_sdk,self.image_path[0],'0x0')

                        self.popen = flash_hilo.result()

                        self.flash_worker = FlashSDKWorker(self.popen)
                        self.thread_pool_flash.start(self.flash_worker)
                        self.flash_worker.signals.wrote.connect(self.print_log)
                        self.flash_worker.signals.finished.connect(self.end_flash)

                    elif mode == 2:
                        self.flash_log_text.setPlainText("Flashing as SSH Mode, please wait.\n")
                        self.flash_worker = FlashSSHWorker(self.image_path[0],self.ip_address_server)
                        self.thread_pool_flash.start(self.flash_worker)
                        self.flash_worker.signals.result.connect(self.resultSSH)
                        self.flash_worker.signals.finished.connect(self.end_flash)

            except Exception as e:
                exception_error = QtWidgets.QMessageBox()
                exception_error.critical(self,"Error",e.__str__())

    def print_log(self,s):
        self.flash_log_text.append(s)

    def resultSSH(self,s):
        if not s[2]:
            ok = False

            for data in s[0]:
                ok = True
                self.flash_log_text.append(data)

            #err (With dd command the output is in stderr. It's ok)
            # if err is empty -> error
            if s[1] == []:
                ok = False
            else:
                for data in s[1]:
                    ok = True
                    self.flash_log_text.append(data)

            if ok:
                self.flash_log_text.append("\nFlash operation Successful")

        else:
            self.flash_log_text.append(str(s[1]))

    def end_flash(self):
        self.stopTime()
        self.program_flash_button.setText('Program')
        self.program_flash_button.repaint()
        self.program_flash_button.blockSignals(False)
        self.cancel_flash_button.blockSignals(False)

    def error_window(self,s):
        msg = s[1] + '\n' + s[2]
        exception_error = QtWidgets.QMessageBox()
        exception_error.critical(self,s[0],msg)

    def showTime(self):
        if self.timer_flag:
            self.timer_count+= 1

        if self.timer_count ==600:
            self.timer_count = 0
            self.timer_minut += 1

        text = str(self.timer_minut) + ':' + str(self.timer_count / 10)
        self.timer_value_label.setText(text)

    def stopTime(self):
        self.timer_flag = False
        self.timer_count = 0
        self.timer_minut = 0


from datetime import datetime

now = datetime.now()

with open('deneme.txt', 'w', encoding='utf8') as f:
    print(f.name)
    f.write('Merhaba Dünya\n')
    f.write(f'Bugün {now}.\n')
    f.write("""

    Bu bir paragraf

""")

dizi = [
    '\n',
    '\n',
    '    <title>İlk Dosya</title>\n',
    '\n',
    '\n'
]

for i in range(10):
    dizi.append(f'
{i+1}. Satır
\n')

dizi.append('\n')
dizi.append('\n')

with open('deneme.html', 'w', encoding='utf8') as f:
    f.writelines(dizi)


#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
ZetCode PyQt4 tutorial

In this example, we create three toggle buttons.
They will control the background color of a
QtGui.QFrame.

author: Jan Bodnar
website: zetcode.com
last edited: September 2011
"""

import sys
import serial
from PyQt4 import QtGui, QtCore

class CommandInterface:
    def open(self, aport='/dev/ttyAMA0', abaudrate=115200) :
        self.sp = serial.Serial(
            port=aport,
            baudrate=abaudrate,     # baudrate
            bytesize=8,             # number of databits
            parity=serial.PARITY_EVEN,
            stopbits=1,
            xonxoff=0,              # enable software flow control
            rtscts=0,               # disable RTS/CTS flow control
            timeout=5               # set a timeout value, None for waiting forever
        )

    def write(self, cmd):
        self.sp.write(chr(cmd))

class Example(QtGui.QWidget):

    def __init__(self):
        super(Example, self).__init__()

        self.initUI()

    def initUI(self):

        conf = {
                'port': '/dev/ttyAMA0',
                'baud': 115200,
                'address': 0x08000000,
                'erase': 0,
                'write': 0,
                'verify': 0,
                'read': 0,
            }

        self.col = QtGui.QColor(0, 0, 0)
        self.uart = CommandInterface()
        self.uart.open(conf['port'], conf['baud'])

        RedLabel=QtGui.QLabel("Red",self)
        RedLabel.setGeometry(60, 20, 30, 10)

        BlueLabel=QtGui.QLabel("Blue",self)
        BlueLabel.setGeometry(55, 60, 40, 10)

        GreenLabel=QtGui.QLabel("Green",self)
        GreenLabel.setGeometry(50, 100, 50, 10)

        Redsld = QtGui.QSlider(QtCore.Qt.Horizontal, self)
        Redsld.setFocusPolicy(QtCore.Qt.NoFocus)
        Redsld.setGeometry(100, 20, 260, 30)
        Redsld.valueChanged[int].connect(self.RedchangeValue)

        Bluesld = QtGui.QSlider(QtCore.Qt.Horizontal, self)
        Bluesld.setFocusPolicy(QtCore.Qt.NoFocus)
        Bluesld.setGeometry(100, 60, 260, 30)
        Bluesld.valueChanged[int].connect(self.BluechangeValue)

        Greensld = QtGui.QSlider(QtCore.Qt.Horizontal, self)
        Greensld.setFocusPolicy(QtCore.Qt.NoFocus)
        Greensld.setGeometry(100, 100, 260, 30)
        Greensld.valueChanged[int].connect(self.GreenchangeValue)

        Redsld.setRange(0,255)
        Bluesld.setRange(0,255)
        Greensld.setRange(0,255)

        Redsld.setObjectName("Red")
        Bluesld.setObjectName("Blue")
        Greensld.setObjectName("Green")

        GreenLabel=QtGui.QLabel("Color show below:",self)
        GreenLabel.setGeometry(100, 180, 200, 10)

        self.square = QtGui.QFrame(self)
        self.square.setGeometry(100, 200, 250, 250)
        self.square.setStyleSheet("QWidget { background-color: %s }" %
            self.col.name())

        self.setGeometry(500, 500, 500, 500)
        self.setWindowTitle('color set')
        self.show()

    def RedchangeValue(self, value):
        if value >255:
            value =0
        print("Red value = %(value)d " % {'value': value})
        self.col.setRed(value)
        self.uart.write(165)
        self.uart.write(64)
        self.uart.write(5)
        self.uart.write(1)
        self.uart.write(1)
        self.uart.write(value)
        self.uart.write(90)
        self.square.setStyleSheet("QFrame { background-color: %s }" %
            self.col.name())

    def BluechangeValue(self, value):
        if value >255:
            value =0
        print("Blue value = %(value)d " % {'value': value})
        self.col.setBlue(value)
        self.uart.write(165)
        self.uart.write(64)
        self.uart.write(5)
        self.uart.write(3)
        self.uart.write(1)
        self.uart.write(value)
        self.uart.write(90)
        self.square.setStyleSheet("QFrame { background-color: %s }" %
            self.col.name())

    def GreenchangeValue(self, value):
        if value >255:
            value =0
        print("Green value = %(value)d " % {'value': value})
        self.col.setGreen(value)
        self.uart.write(165)
        self.uart.write(64)
        self.uart.write(5)
        self.uart.write(2)
        self.uart.write(1)
        self.uart.write(value)
        self.uart.write(90)
        self.square.setStyleSheet("QFrame { background-color: %s }" %
            self.col.name())

def main():

    app = QtGui.QApplication(sys.argv)

    ex = Example()
    sys.exit(app.exec_())

if __name__ == '__main__':
    main()


from aiogram import Router

from inclusive_dance_bot.bot.dialogs.admins import (
    feedbacks,
    mailings,
    main_menu,
    manage_admins,
    submenu,
    url,
)

dialog_router = Router(name="admin_router")
dialog_router.include_routers(
    feedbacks.router,
    main_menu.dialog,
    submenu.router,
    url.router,
    manage_admins.router,
    mailings.router,
)


import logging
from typing import Optional

from core import db
from player import PlayerApp
from ui.components.tabs_layout import Tab

from .models import Playlist
from .views import PlaylistsComponent, PlaylistTracksComponent

logger = logging.getLogger("plugins.playlist")

class PlaylistController:
    def __init__(self):
        self.app: PlayerApp = PlayerApp.get_instance()
        self.playlist_container = self.app.ui.top_layout

        self.playlists_view: Optional[PlaylistsComponent] = None

        self.app.injector.provide(PlaylistController, lambda: self)

    def toggle_playlists(self):
        if self.playlists_view:
            self.app.window.blur_active_component()
            self.playlists_view.detach()
            self.playlists_view = None
            if self.subscription:
                self.subscription.dispose()
                self.subscription = None
        else:
            self.playlists_view = PlaylistsComponent(size=40)
            self.playlist_container.insert(0, self.playlists_view)
            self.app.window.focus(self.playlists_view)

            self.subscription = self.playlists_view.selected_item.subscribe(
                self.open_playlist
            )

    def open_playlist(self, playlist_name: str):
        playlist = (
            db.get_session()
            .query(Playlist)
            .filter(Playlist.name == playlist_name)
            .one()
        )

        if not playlist:
            logger.error(f"Unable to open playlist {playlist_name}")
            return

        tabs_layout = self.app.ui.tabs_layout
        tab = tabs_layout.get_tab(playlist.name)
        if tab:
            self.remove_current_tab_from_stack()
            tabs_layout.switch_to_tab(tab)
            self.app.window.focus(tab.component)
        else:
            tracks_view = PlaylistTracksComponent(playlist)
            self.remove_current_tab_from_stack()
            self.app.ui.tabs_layout.add_tab(
                Tab(title=playlist.name, component=tracks_view)
            )
            self.app.window.focus(tracks_view)

    def remove_current_tab_from_stack(self):
        component = self.app.ui.tabs_layout.displayed_tab.component
        if component in self.app.window.active_component_stack:
            self.app.window.active_component_stack.remove(component)


#!/usr/bin/env python
# encoding: utf-8

import argparse
import json
import os
from os import path
import numpy as np
import scipy as sp
import scipy.sparse as smat
import ctypes
from ctypes import *
from sklearn.preprocessing import normalize as sk_normalize

from xbert.rf_util import PyMatrix, fillprototype, load_dynamic_library

class RandomProject(object):

    """Encode and decode a label into a K-way D-dimensional code.

    feat_mat: L by P matrix
    L: number of label
    P: label feature dimension
    K: range of the code = {0, 1, 2, ..., K-1}
    D: length of the codes for each label (number of hashing functions)
    """

    def __init__(self, feat_mat, kdim, depth, algo, seed):
        self.feat_mat = feat_mat
        self.code_delim = " | "
        self.K, self.D = kdim, depth
        self.L, self.P = feat_mat.shape
        self.algo = algo
        self.random_matrix = np.random.randn(self.P, self.D)

    def get_codes(self):
        if self.algo == 2:  # ordinal
            Z_quant = self.ordinal_quantization(self.feat_mat)
        elif self.algo == 3:  # uniform
            Z_quant = self.uniform_quantization(self.feat_mat)
        else:
            raise NotImplementedError("unknown algo {}".format(self.algo))
        self.hash_func = np.array([self.K ** d for d in range(0, self.D)])
        return Z_quant.dot(self.hash_func)

    # Z: L by D projected label embedding
    # Z_quant: L by D quantized projected label embeddings
    # quantize it by ordinal ranking
    def ordinal_quantization(self, label_embedding):
        Z = label_embedding.dot(self.random_matrix)
        Z_argsort = np.argsort(Z, axis=0)
        bin_size = math.ceil(self.L * 1.0 / self.K)
        Z_quant = []
        for d in range(self.D):
            rank = np.zeros(self.L, dtype=np.int64)
            rank[Z_argsort[:, d]] = np.arange(self.L)
            quantized_rows = (rank // bin_size).tolist()
            Z_quant.append(quantized_rows)
        Z_quant = np.array(Z_quant).T
        return Z_quant

    # quantize it by min/max of each cols into K bins
    def uniform_quantization(self, label_embedding):
        Z = label_embedding.dot(self.random_matrix)
        Z_quant = []
        for d in range(self.D):
            bins = np.linspace(min(Z[:, d]), max(Z[:, d]), self.K)
            quantized_rows = np.digitize(Z[:, d], bins) - 1  # bits = {0, 1, ..., K-1}
            Z_quant.append(quantized_rows)
        Z_quant = np.array(Z_quant).T
        return Z_quant

    def prepare_coding(self, Z_quant):
        # L dimensional array,
        # each entry is the hash code (row idx)
        self.hash_code_arr = Z_quant.dot(self.hash_func)
        rows, cols, vals = [], [], []
        for l in range(self.L):
            rows.append(self.hash_code_arr[l])
            cols.append(l)
            vals.append(1)

        m = self.K ** self.D
        n = self.L
        M = sp.csr_matrix((vals, (rows, cols)), shape=(m, n))
        self.code2label_mat = M

        code2label_set = {}
        for code in np.nonzero(M.indptr[1:] - M.indptr[:-1])[0]:
            code2label_set[code] = set(M.indices[M.indptr[code] : M.indptr[code + 1]])
        self.code2label_set = code2label_set

class corelib(object):
    def __init__(self, dirname, soname, forced_rebuild=False):
        self.clib_float32 = load_dynamic_library(dirname, soname + "_float32", forced_rebuild=forced_rebuild)
        self.clib_float64 = load_dynamic_library(dirname, soname + "_float64", forced_rebuild=forced_rebuild)
        arg_list = [
            POINTER(PyMatrix),
            c_uint32,
            c_uint32,
            c_int32,
            c_uint32,
            c_int32,
            POINTER(c_uint32),
        ]
        fillprototype(self.clib_float32.get_codes, None, arg_list)
        fillprototype(self.clib_float64.get_codes, None, arg_list)

    def get_codes(self, py_feat_mat, depth, algo, seed, codes, verbose=0, max_iter=10, threads=-1):
        clib = self.clib_float32
        if py_feat_mat.dtype == sp.float64:
            clib = self.clib_float64
            if verbose != 0:
                print("perform float64 computation")
        else:
            clib = self.clib_float32
            if verbose != 0:
                print("perform float32 computation")
        clib.get_codes(
            byref(py_feat_mat), depth, algo, seed, max_iter, threads, codes.ctypes.data_as(POINTER(c_uint32)),
        )

forced_rebuild = False
corelib_path = path.join(path.dirname(path.abspath(__file__)), "corelib/")
soname = "rf_linear"
clib = corelib(corelib_path, soname, forced_rebuild)

# SEmatic-aware Code
class SeC(object):
    def __init__(self, kdim, depth, algo, seed, codes):
        assert kdim == 2
        self.kdim = kdim
        self.depth = depth
        self.algo = algo
        self.seed = seed
        self.codes = codes
        self.indptr = sp.cumsum(sp.bincount(codes + 1, minlength=(self.nr_codes + 1)), dtype=sp.uint64)
        self.indices = sp.argsort(codes * sp.float64(self.nr_elements) + sp.arange(self.nr_elements))

    @property
    def nr_elements(self):
        return len(self.codes)

    @property
    def nr_codes(self):
        return 1 << self.depth

    def __len__(self):
        return len(self.codes)

    def get_code_for_element(self, eid):
        assert 0 <= eid and eid < self.nr_elements
        return self.codes[eid]

    def get_elements_with_code(self, code):
        assert 0 <= code and code < self.nr_codes
        begin, end = self.indptr[code], self.indptr[code + 1]
        return self.indices[begin:end]

    def get_csc_matrix(self):
        return smat.csc_matrix(
            (sp.ones_like(self.indices, dtype=sp.float64), self.indices, self.indptr), shape=(self.nr_elements, self.nr_codes),
        )

    def print(self):
        print("nr_codes: {}".format(self.nr_codes))
        print("nr_elements: {}".format(self.nr_elements))
        print("algo: {}".format(Indexer.algos[self.algo]))
        for nid in range(self.nr_codes):
            labels = " ".join(map(str, self.get_elements_with_code(nid)))
            print("code({nid}): {labels}".format(nid=nid, labels=labels))

class Indexer(object):
    KMEANS = 0
    KDTREE = 1  # KDTREE with Roound-Robin feature splits
    ORDINAL = 2  # random projection with ordinal quantization
    UNIFORM = 3  # random projection with uniform quantization
    BALANCED_ORDINAL = 4  # random projection with balaced ordinal quantization
    SKMEANS = 5  # Spherical KMEANS
    KDTREE_CYCLIC = 11  # KDTREE with cyclic feature splits( 0,...,0, 1,...,1, 2,...,2)
    PURE_RANDOM = 12  # Random assign cluster for each element

    algos = {v: k for k, v in vars().items() if isinstance(v, int)}

    def __init__(self, feat_mat):
        self.py_feat_mat = PyMatrix.init_from(feat_mat)

    @property
    def feat_mat(self):
        return self.py_feat_mat.buf

    @property
    def nr_labels(self):
        return self.feat_mat.shape[0]

    @staticmethod
    def load_indexed_code(code_path, label_feat):
        C = None
        mapping = {
            "none": Indexer.SKMEANS,
            "skmeans": Indexer.SKMEANS,
            "kmeans": Indexer.KMEANS,
            "kdtree": Indexer.KDTREE,
            "random": Indexer.PURE_RANDOM,
            "ordinal": Indexer.BALANCED_ORDINAL,
            "uniform": Indexer.UNIFORM,
        }
        if code_path is None:
            code_path = "none"

        if code_path.lower() in mapping:
            if label_feat is not None:
                algo = mapping[code_path.lower()]
                if algo == Indexer.SKMEANS:
                    label_feat = sk_normalize(label_feat, axis=1, norm="l2", copy=False)
                indexer = Indexer(label_feat)
                code = indexer.gen(
                    kdim=2,
                    depth=indexer.estimate_depth_with_cluster_size(100),
                    algo=algo,
                    seed=0,
                    max_iter=20,
                    threads=1,
                )
                C = code.get_csc_matrix()
        else:
            if code_path.endswith(".npz") and path.exists(code_path):
                C = smat.load_npz(code_path)
            elif path.isdir(code_path) and path.exists(path.join(code_path, "code.npz")):
                C = smat.load_npz(path.join(code_path, "code.npz"))
            else:
                assert False, f"'{code_path}' does not exist. Valid ones {mapping.keys()}"
        return C

    def estimate_depth_with_nr_clusters(self, nr_clusters):
        depth = int(sp.log2(nr_clusters))
        return depth

    def estimate_depth_with_cluster_size(self, cluster_size):
        return self.estimate_depth_with_nr_clusters(self.nr_labels // cluster_size + 1)

    def ordinal_gen(self, kdim, depth, seed):
        sp.random.seed(seed)
        random_matrix = sp.randn(self.feat_mat.shape[1], depth)
        X = self.feat_mat.dot(random_matrix)
        m = self.feat_mat.shape[0] // kdim + [1, 0][self.feat_mat.shape[0] % kdim != 0]
        X = sp.argsort(sp.argsort(X, axis=0), axis=0) // m
        print(X)
        codes = sp.array((X * (kdim ** sp.arange(depth)).reshape(1, -1)).sum(axis=1), dtype=sp.uint32)
        return codes

    def balaced_ordinal_gen(self, kdim, depth, seed, threads=1):
        assert int(2 ** sp.log2(kdim)) == kdim
        sp.random.seed(seed)
        random_matrix = sp.randn(self.feat_mat.shape[1], depth)
        X = PyMatrix(self.feat_mat.dot(random_matrix))
        codes = sp.zeros(X.rows, dtype=sp.uint32)
        new_depth = depth * int(sp.log2(kdim))
        clib.get_codes(X, new_depth, Indexer.KDTREE_CYCLIC, seed, codes, threads=threads)
        return codes

    def gen(self, kdim, depth, algo, seed, max_iter=10, threads=1):
        assert algo in [
            Indexer.KMEANS,
            Indexer.KDTREE,
            Indexer.ORDINAL,
            Indexer.UNIFORM,
            Indexer.BALANCED_ORDINAL,
            Indexer.KDTREE_CYCLIC,
            Indexer.SKMEANS,
            Indexer.PURE_RANDOM,
        ]
        if algo in [
            Indexer.KMEANS,
            Indexer.KDTREE,
            Indexer.KDTREE_CYCLIC,
            Indexer.SKMEANS,
        ]:
            feat_mat = self.py_feat_mat
            codes = sp.zeros(feat_mat.rows, dtype=sp.uint32)
            clib.get_codes(feat_mat, depth, algo, seed, codes, max_iter=max_iter, threads=threads)
        elif algo in [Indexer.ORDINAL, Indexer.UNIFORM]:
            rp_clf = RandomProject(self.feat_mat, kdim, depth, algo, seed)
            codes = rp_clf.get_codes()
        elif algo in [Indexer.BALANCED_ORDINAL]:
            assert int(2 ** sp.log2(kdim)) == kdim
            codes = self.balaced_ordinal_gen(kdim, depth, seed, threads=threads)
        elif algo in [Indexer.PURE_RANDOM]:
            feat_mat = self.py_feat_mat
            codes = sp.zeros(feat_mat.rows, dtype=sp.uint32)
            cluster_size = kdim ** depth
            for idx in range(feat_mat.rows):
                codes[idx] = np.random.randint(0, cluster_size)
        else:
            raise NotImplementedError("unknown algo {}".format(algo))
        return SeC(kdim, depth, algo, seed, codes)

def run_test(data_folder="./datasets/Eurlex-4K"):
    import xbert.rf_linear as rf_linear

    data = rf_linear.Data.load(data_folder, label_emb=None)
    L = smat.load_npz(data_folder + "/L.pifa.npz")
    code = Indexer(L).gen(kdim=2, depth=6, algo=0, seed=5, max_iter=20, threads=1)
    code.print()
    code = Indexer(L).gen(kdim=2, depth=6, algo=5, seed=5, max_iter=20, threads=1)
    code.print()

def load_feature_matrix(src, dtype=sp.float32):
    if src.endswith(".npz"):
        return smat.load_npz(src).tocsr().astype(dtype)
    elif src.endswith(".npy"):
        return smat.csr_matrix(sp.ascontiguousarray(sp.load(src), dtype=dtype))
    else:
        raise ValueError("src must end with .npz or .npy")

def main(args):
    # set hyper-parameters
    input_feat_path = args.input_feat_path
    kdim = args.kdim
    algo = args.algo
    seed = args.seed
    max_iter = args.max_iter
    threads = args.threads
    output_code_dir = args.output_code_dir

    # load label feature matrix (nr_labels * nr_features)
    if path.exists(input_feat_path):
        feat_mat = load_feature_matrix(input_feat_path)
    else:
        raise ValueError("label embedding path does not exist {}".format(input_feat_path))

    if not path.exists(output_code_dir):
        os.makedirs(output_code_dir, exist_ok=True)

    # Indexing algorithm
    # C: nr_labels x nr_codes, stored in csr sparse matrix
    indexer = Indexer(feat_mat)
    if algo == indexer.SKMEANS:
        feat_mat = sk_normalize(feat_mat, axis=1, norm="l2", copy=False)
    depth = indexer.estimate_depth_with_cluster_size(100)+1
    code = indexer.gen(kdim=kdim, depth=depth, algo=algo, seed=seed, max_iter=max_iter, threads=threads)
    C = code.get_csc_matrix()
    print("INDEXER DEPTH", depth, "C", C.shape)

    # save code and args
    output_code_path = path.join(output_code_dir, "code.npz")
    smat.save_npz("{}".format(output_code_path), C, compressed=False)
    output_config_path = path.join(output_code_dir, "config.json")
    with open(output_config_path, "w") as fout:
        fout.write(json.dumps(vars(args), indent=True))

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    ## Required parameters
    parser.add_argument(
        "-i",
        "-L",
        "--input-feat-path",
        type=str,
        required=True,
        default="./datasets/Eurlex-4K/L.pifa.npz",
        help="path to the npz file of input label feature matrix (nr_labels * nr_features, CSR)",
    )
    parser.add_argument(
        "-o",
        "-c",
        "--output-code-dir",
        type=str,
        required=True,
        default="./save_models/Eurlex-4K/indexer/code.npz",
        help="path to the output npz file of indexing codes (nr_labels * nr_codes, CSR)",
    )
    # optional
    parser.add_argument("--algo", type=int, default=5, help="0 for KMEANS 5 for SKMEANS (default 5)")
    parser.add_argument("--seed", type=int, default=0, help="random seed (default 0)")
    parser.add_argument("--kdim", type=int, default=2)
    parser.add_argument("--threads", type=int, default=1)
    parser.add_argument("--max-iter", type=int, default=20)

    args = parser.parse_args()
    print(args)
    main(args)


import discord, asyncio
from discord.ext import commands

class Misc(commands.Cog):
    def __init__(self, client):
        self.client = client

    @commands.command()
    async def magisk(self, ctx, links="asdf"):
        if links == "asdf":
            whatismagisk = discord.Embed(title = "__Magisk: The magic mask of Android__", color = 0x301c24, url = "https://github.com/topjohnwu/Magisk")

            whatismagisk.add_field(name = "What is Magisk?", value = "In simple terms, Magisk is a way to root your phone systemlessly and install *modules* on it. It also allows you to bypass Safetynet checks while you're rooted.", inline = False)
            whatismagisk.add_field(name = "Should I use it compared to SuperSU and others?", value = "YES! SuperSU is quite outdated nowadays and others range from not-well-reputable to downright spyware.", inline = False)
            whatismagisk.add_field(name = "How do I install it on my phone?", value = "Make sure you have bootloader unlocked, then check out the installation guide for more info\n", inline = False)
            whatismagisk.set_thumbnail(url = "https://cdn.discordapp.com/attachments/867824004072210452/868167290628210739/download.jpg")
            await ctx.send(embed = whatismagisk)
        else:
            embed = discord.Embed(title = "__Magisk related links:__", color = 0x301c24)

            embed.add_field(name = "Download Link", value = "https://github.com/topjohnwu/Magisk/releases", inline = False)
            embed.add_field(name = "Official Installation Guide", value = "https://topjohnwu.github.io/Magisk/install.html", inline = False)
            embed.add_field(name = "Magisk Documentation (Users and Devs)", value = "https://topjohnwu.github.io/Magisk/", inline = False)
            await ctx.send(embed = embed)

    @commands.command()
    async def remind(self, ctx, time, *msg):
        """Reminds you of something after a certain amount of time"""
        uid = ctx.author.id
        await ctx.send("Reminding you in {} seconds".format(time))
        await asyncio.sleep(int(time))
        await ctx.send(f'<@{uid}> Reminder: {" ".join(msg)}')

    @commands.command()
    async def ping(self, ctx):
        await ctx.send(f"Pong! {round(self.client.latency*1000)}ms")

def setup(client):
    client.add_cog(Misc(client))


from numbers import Number
from typing import Iterable, Tuple
from PIL import Image
import psd_tools

from constants import Title_Image_Zip
from lib.image_processor import ImageProcessor

def flatten_layers(psd):
    """
    Flatten layers for psd file
    """
    for layer in psd.descendants():
        if layer.is_group():
            flatten_layers(layer)
        else:
            yield layer

def bulk_resize_images(
    replacement_images,
    replacement_layer_map
):
    """
    Resize the replacement image to fit within the bounds of the layer
    """

    for title, replacement_image in replacement_images:
        layer_to_replace = replacement_layer_map[title]

        resized_image = ImageProcessor.resize_image(
            replacement_image,
            (layer_to_replace.width, layer_to_replace.height),
            keep_aspect_ratio='Logo' in title
        )
        print(title, resized_image.width, resized_image.height)
        yield Title_Image_Zip(title, resized_image)

def bulk_replicate_text(text_layers, psd_size, font_type_map, text_value_map):
    for layer in text_layers:
        text = text_value_map.get(layer.name)
        replicated_image = ImageProcessor.replicate_text_image(layer, text, psd_size, font_type_map)
        yield Title_Image_Zip(layer.name, replicated_image)

def bulk_layer_composites(
    layers,
    replacement_images,
    filesize
):
    replace_image_map = {title: image for title, image in replacement_images}
    for layer in layers:
        print('Replicate text')
        print(layer.name, layer.kind)
        print(layer.bbox)
        print('---------------')
        layer_image = Image.new(mode='RGBA', size=filesize, color=(0, 0, 0, 0))
        layer_data = layer.composite()
        replacement_image = replace_image_map.get(layer.name)
        layer_image.paste(
            replacement_image if replacement_image else layer_data,
            box=layer.bbox[:2],
            mask=None if replacement_image else layer_data
        )
        yield layer_image


from math import fabs
import torch
import torch.nn as nn
import torchvision

def dconv_bn_relu(in_dim, out_dim):
    """_summary_

    Args:
        in_dim (int): input channel
        out_dim (int): output channel

    Returns:
        nn.Model: Sequential model ConvTranspos2d -> BN -> Relu()

    """
    return nn.Sequential(
        nn.ConvTranspose2d(in_dim, out_dim, kernel_size = 5, stride = 2,
                                   padding=2, output_padding=1, bias=False),
        nn.BatchNorm2d(out_dim),
        nn.ReLU())

class Generator(torch.nn.Module):
    def __init__(self, channels,z_dim = 100):
        super().__init__()
        k = 5 # 4 5
        p = 2 # 1 2
        op = 1 # 0 1
        self.z_dim=z_dim
        self.main_module = nn.Sequential(
            # Z latent vector 100
            nn.ConvTranspose2d(in_channels=z_dim, out_channels=1024, kernel_size=5, stride=1, padding=2),
            nn.BatchNorm2d(num_features=1024),
            nn.ReLU(True),

            # State (1024x4x4)
            nn.ConvTranspose2d(in_channels=1024, out_channels=512, kernel_size=k, stride=2, padding=p,output_padding=op),
            nn.BatchNorm2d(num_features=512),
            nn.ReLU(True),

            # State (512x8x8)
            nn.ConvTranspose2d(in_channels=512, out_channels=256, kernel_size=k, stride=2, padding=p,output_padding=op),
            nn.BatchNorm2d(num_features=256),
            nn.ReLU(True),

            # State (256x16x16)
            nn.ConvTranspose2d(in_channels=256, out_channels=channels, kernel_size=k, stride=2, padding=p, output_padding=op))

            # output of main module --> Image (Cx32x32)

        self.output = nn.Tanh()

    def forward(self, x):
        x = self.main_module(x)
        return self.output(x)

class Discriminator(torch.nn.Module):
    def __init__(self, channels=3,z_dim = 100):
        """
        Args:
            channels (int): channel of the input image
        Return:
            output (tensor): B 1 H/8 W/8 (128,256)->(16,32)
        """
        super().__init__()
        # Filters [256, 512, 1024]
        self.z_dim = z_dim #(Cx64x64)
        k = 5 # 4 5
        p = 2 # 1 2
        op = 1 # 0 1
        # Output_dim = 1
        self.main_module = nn.Sequential(
            # Omitting batch normalization in critic because our new penalized training objective (WGAN with gradient penalty) is no longer valid
            # in this setting, since we penalize the norm of the critic's gradient with respect to each input independently and not the enitre batch.
            # There is not good & fast implementation of layer normalization --> using per instance normalization nn.InstanceNorm2d()
            # Image (Cx32x32)
            nn.Conv2d(in_channels=channels, out_channels=256, kernel_size=k, stride=2, padding=p),
            nn.InstanceNorm2d(256, affine=True),
            nn.LeakyReLU(0.2, inplace=True),

            # State (256x16x16)
            nn.Conv2d(in_channels=256, out_channels=512, kernel_size=k, stride=2, padding=p),
            nn.InstanceNorm2d(512, affine=True),
            nn.LeakyReLU(0.2, inplace=True),

            # State (512x8x8)
            nn.Conv2d(in_channels=512, out_channels=1024, kernel_size=k, stride=2, padding=p),
            nn.InstanceNorm2d(1024, affine=True),
            nn.LeakyReLU(0.2, inplace=True))
            # output of main module --> State (1024x4x4)

        self.output = nn.Sequential(
            # The output of D is no longer a probability, we do not apply sigmoid at the output of D.
            nn.Conv2d(in_channels=1024, out_channels=1, kernel_size=k, stride=1, padding=p))

    def forward(self, x):
        x = self.main_module(x)
        return self.output(x)

    def feature_extraction(self, x):
        # Use discriminator for feature extraction then flatten to vector of 16384
        x = self.main_module(x)
        return x.vie

if __name__ == "__main__":
    x = torch.randn(2,100,16,32) # 2x100
    G = Generator(3) # output channels
    fake = G(x) # 2,3, 32 ,32
    print(fake.size())
    D = Discriminator(3)
    r_logit = D(fake)
    print(r_logit.size())


from matplotlib.ft2font import BOLD
from src.utils import (
    get_model_dir,
    ham10000_metadata,
    short_to_full_name_dict,
    full_name_to_short_dict,
    bening_or_malignant_dict,
    get_image_path,
    calculate_metrics,
)
from PIL import Image
from src.config import only_lesion_id, resnet_mixup_id
from sklearn.metrics import confusion_matrix
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import numpy as np
import scipy.stats as stats
from fastai.interpret import ClassificationInterpretation

def build_segmented_prediction_strength():
    print("Building segmented prediction strength...")

    # Read the validation results
    predictions = pd.read_csv(f"{get_model_dir(only_lesion_id)}/predictions.csv")
    only_lesion_predictions = pd.read_csv(
        f"{get_model_dir(only_lesion_id)}/only_lesions_predictions.csv"
    )

    # Merge with the metadata
    predictions = predictions.merge(ham10000_metadata, on="image_id", how="left")
    only_lesion_predictions = only_lesion_predictions.merge(
        ham10000_metadata, on="image_id", how="left"
    )

    predictions["real_class"] = predictions.dx.map(lambda x: short_to_full_name_dict[x])
    only_lesion_predictions["real_class"] = only_lesion_predictions.dx.map(
        lambda x: short_to_full_name_dict[x]
    )

    predictions["correct"] = predictions.classification == predictions.real_class
    only_lesion_predictions["correct"] = (
        only_lesion_predictions.classification == only_lesion_predictions.real_class
    )

    # Make a confusion matrix
    confusion_matrix = (
        predictions.groupby(["real_class", "classification"]).size().unstack()
    )
    only_lesion_confusion_matrix = (
        only_lesion_predictions.groupby(["real_class", "classification"])
        .size()
        .unstack()
    )

    # Normalize over the x axis
    confusion_matrix_normalized = confusion_matrix.div(
        confusion_matrix.sum(axis=1), axis=0
    )
    only_lesion_confusion_matrix_normalized = only_lesion_confusion_matrix.div(
        only_lesion_confusion_matrix.sum(axis=1), axis=0
    )

    confusion_matrix_normalized.columns = confusion_matrix_normalized.columns.map(
        lambda x: full_name_to_short_dict[x]
    )
    only_lesion_confusion_matrix_normalized.columns = (
        only_lesion_confusion_matrix_normalized.columns.map(
            lambda x: full_name_to_short_dict[x]
        )
    )

    confusion_matrix_normalized.index = confusion_matrix_normalized.index.map(
        lambda x: full_name_to_short_dict[x]
    )
    only_lesion_confusion_matrix_normalized.index = (
        only_lesion_confusion_matrix_normalized.index.map(
            lambda x: full_name_to_short_dict[x]
        )
    )

    # Plot the confusion matrix using seaborn
    plt.figure(figsize=(8, 8))
    p = sns.heatmap(
        confusion_matrix_normalized, annot=True, cmap="Blues", annot_kws={"size": 10}
    )
    p.set_xlabel("Predicted label")
    p.set_ylabel("True label")
    plt.savefig("only_lesion_on_original_confusion_matrix.png")

    # Plot the confusion matrix using seaborn
    plt.figure(figsize=(8, 8))
    p = sns.heatmap(
        only_lesion_confusion_matrix_normalized,
        annot=True,
        cmap="Blues",
        annot_kws={"size": 10},
    )
    p.set_xlabel("Predicted label")
    p.set_ylabel("True label")
    plt.savefig("only_lesion_on_segmented_confusion_matrix.png")

    # Calculate precision, recall and f1 score on each of the two datasets

    # With recall we think malignant recall
    # With precision we calculate both malignant precision and general class precision
    segmented_on_normal_metrics = calculate_metrics(only_lesion_id, "normal")
    segmented_on_only_lesion_metrics = calculate_metrics(only_lesion_id, "only_lesions")
    normal_on_normal_metrics = calculate_metrics(resnet_mixup_id, "normal")
    normal_on_segmented_metrics = calculate_metrics(resnet_mixup_id, "only_lesions")

    full_image_label = "Full images"
    segmented_image_label = "Segmented"
    index = pd.MultiIndex.from_tuples(
        [
            (full_image_label, full_image_label),
            (full_image_label, segmented_image_label),
            (segmented_image_label, full_image_label),
            (segmented_image_label, segmented_image_label),
        ],
        names=["Training set", "Evaluation set"],
    )

    score_table = pd.DataFrame(
        {
            "Multi-class precision": [
                normal_on_normal_metrics["mc-precision"],
                normal_on_segmented_metrics["mc-precision"],
                segmented_on_normal_metrics["mc-precision"],
                segmented_on_only_lesion_metrics["mc-precision"],
            ],
            "Multi-class F1 score": [
                normal_on_normal_metrics["mc-f1"],
                normal_on_segmented_metrics["mc-f1"],
                segmented_on_normal_metrics["mc-f1"],
                segmented_on_only_lesion_metrics["mc-f1"],
            ],
            "Binary precision": [
                normal_on_normal_metrics["b-precision"],
                normal_on_segmented_metrics["b-precision"],
                segmented_on_normal_metrics["b-precision"],
                segmented_on_only_lesion_metrics["b-precision"],
            ],
            "Malignant F1 score": [
                normal_on_normal_metrics["b-f1"],
                normal_on_segmented_metrics["b-f1"],
                segmented_on_normal_metrics["b-f1"],
                segmented_on_only_lesion_metrics["b-f1"],
            ],
            "Malignant recall": [
                normal_on_normal_metrics["b-recall"],
                normal_on_segmented_metrics["b-recall"],
                segmented_on_normal_metrics["b-recall"],
                segmented_on_only_lesion_metrics["b-recall"],
            ],
        },
        index=index,
    )

    score_table.transpose().to_latex(
        "score_table.tex",
        bold_rows=True,
        multicolumn=True,
        # column_format="l|rr|rr|",
    )

    # Make a table containing just the metrics from the original model
    metrics_table = pd.DataFrame(
        {
            "Multi-class precision": [
                normal_on_normal_metrics["mc-precision"],
            ],
            "Multi-class F1 score": [
                normal_on_normal_metrics["mc-f1"],
            ],
            "Binary precision": [
                normal_on_normal_metrics["b-precision"],
            ],
            "Malignant F1 score": [
                normal_on_normal_metrics["b-f1"],
            ],
            "Malignant recall": [
                normal_on_normal_metrics["b-recall"],
            ],
        },
        index=pd.Index(
            [
                "Model metrics"
            ],
            name="",
        ),
    ).transpose()

    metrics_table.to_latex("metrics_table.tex", bold_rows=True)


import sqlite3 as db
from datetime import datetime

class DB():
    def __init__(self, filename):
        self._filename = filename
        conn = db.connect(self._filename)
        cursor = conn.cursor()

        cursor.execute('''
            CREATE TABLE IF NOT EXISTS games (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                player_name TEXT,
                date INTEGER,
                turns_played INTEGER,
                used_cheats NUMERIC
            )'''
        )
        conn.commit()

        conn.close()

    @property
    def filename(self):
        return self._filename

    @filename.setter
    def filename(self, value):
        self._filename = value

    def saveGame(self, player, date, turns_played, has_cheated):
        conn = db.connect(self._filename)
        cursor = conn.cursor()
        cursor.execute("INSERT INTO games (player_name, date, turns_played, used_cheats) VALUES (:name, :date, :turns, :cheated)", {'name': player.name.lower(), 'date': date, 'turns': turns_played, 'cheated': has_cheated})
        conn.commit()
        conn.close()

    def getGame(self, game_id):
        conn = db.connect(self._filename)
        cursor = conn.cursor()
        cursor.execute("SELECT * FROM games WHERE games.id == :id", {'id': game_id})
        game = cursor.fetchone()[0]
        conn.commit()
        conn.close()
        return game

    def getStatisticsByName(self, player_name):
        stats = {}

        conn = db.connect(self._filename)
        cursor = conn.cursor()

        cursor.execute("SELECT COUNT(*) FROM games WHERE player_name == :name", {'name': player_name.lower()})
        stats['games_played'] = cursor.fetchone()[0]
        cursor.execute("SELECT date, turns_played, used_cheats FROM games WHERE player_name == :name ORDER BY date", {'name': player_name.lower()})
        # stats['turns_per_game'] = cursor.fetchall()
        results = []
        for row in cursor:
            results.append({
                'date': datetime.utcfromtimestamp(row[0]).strftime('%Y-%m-%d %H:%M:%S'),
                'turns_played': row[1],
                'used_cheats': row[2]
                })
        stats['turns_per_game'] = results
        conn.commit()
        conn.close()
        return stats