# General
import os

# Tools/utils
import itertools
import multiprocessing
from tqdm.notebook import tqdm
from tqdm import tqdm as tqdm_cli
from functools import reduce  # for aggregate functions
from itertools import chain  # for aggregate functions

# Data management
import math
import numpy as np
import pandas as pd
import networkx as nx
import igraph as ig
import leidenalg as la
from community import community_louvain

# Visualization
import matplotlib.pyplot as plt
import seaborn as sns
import pygraphviz as pgv
import colorcet as cc
from matplotlib.colors import ListedColormap
from wordcloud import WordCloud, STOPWORDS
from termcolor import colored  # colored text output

from sklearn.preprocessing import MinMaxScaler

stopwords = STOPWORDS.union({
    'regulation', 'activity', 'positive', 'negative', 'catabolic', 'process', 'protein', 'complex',
    'binding', 'response', 'gene', 'genes', 'encoding', 'defining', 'GeneID', 'regulated',
})

def get_tf_targ_ctx(df):
    tf_target_dict = {'TF': [], 'target': [], 'importance': []}
    tf_target_info = (
        df.droplevel(axis=0, level=1).droplevel(axis=1, level=0)['TargetGenes']
          .map(set)  # transform each list into set
          .groupby('TF').agg(lambda x: reduce(lambda a, b: a.union(b), x))  # combine all targets per TF
    )
    for tf, target_info in tf_target_info.iteritems():
        tf_target_dict['TF'] += [tf for target_name, score in target_info]
        tf_target_dict['target'] += [target_name for target_name, score in target_info]
        tf_target_dict['importance'] += [score for target_name, score in target_info]
    return pd.DataFrame(tf_target_dict)

def netgraph_community_layout(G, node_to_community, community_scale=1., node_scale=2., seed=42):
    """
    Compute the node positions for a modular graph.
    """

    # assert that there multiple communities in the graph; otherwise abort
    communities = set(node_to_community.values())
    if len(communities) < 2:
        warnings.warn("Graph contains a single community. Unable to compute a community layout. Computing spring layout instead.")
        return nx.spring_layout(G, weight='importance', **kwargs)

    community_size = _get_community_sizes(node_to_community)
    community_centroids = _get_community_positions(G, node_to_community, community_scale, seed=seed)
    relative_node_positions = _get_node_positions(G, node_to_community, node_scale, seed=seed)

    # combine positions
    node_positions = dict()
    for node, community in node_to_community.items():
        xy = community_centroids[node]
        delta = relative_node_positions[node] * community_size[community]
        node_positions[node] = xy + delta

    return node_positions

def _get_community_sizes(node_to_community):
    """
    Compute the area of the canvas reserved for each community.
    """

    def _invert_dict(mydict):
        """Invert a dictionary such that values map to keys."""
        inverse = dict()
        for key, value in mydict.items():
            inverse.setdefault(value, set()).add(key)
        return inverse

    scale = (1, 1)

    total_nodes = len(node_to_community)
    max_radius = np.linalg.norm(scale) / 2
    scalar = max_radius / total_nodes
    community_to_nodes = _invert_dict(node_to_community)
    community_size = {community : len(nodes) * scalar for community, nodes in community_to_nodes.items()}

    return community_size

def _get_community_positions(G, node_to_community, community_scale, seed, simple=True):
    """
    Compute a centroid position for each community.
    """

    # create a weighted graph, in which each node corresponds to a community,
    # and each edge weight to the number of edges between communities
    between_community_edges = _find_between_community_edges(G, node_to_community)

    communities = set(node_to_community.values())
    hypergraph = nx.DiGraph()
    hypergraph.add_nodes_from(communities)

    if not simple:
        for (ci, cj), edges in between_community_edges.items():
            hypergraph.add_edge(ci, cj, weight=len(edges))

    # find layout for communities
    pos_communities = nx.spring_layout(hypergraph, scale=community_scale, seed=seed)

    # set node positions to position of community
    pos = dict()
    for node, community in node_to_community.items():
        pos[node] = pos_communities[community]

    return pos

def _find_between_community_edges(G, node_to_community, fixed_community=None):
    """Convert the graph into a weighted network of communities."""
    edges = dict()

    for (ni, nj) in G.edges():
        ci = node_to_community[ni]
        cj = node_to_community[nj]

        if fixed_community is not None:
            if fixed_community != ci and fixed_community != cj:
                continue

        if ci != cj:
            try:
                edges[(ci, cj)] += [(ni, nj)]
            except KeyError:
                edges[(ci, cj)] = [(ni, nj)]

    return edges

def _get_node_positions(G, node_to_community, node_scale, seed):
    """
    Positions nodes within communities.
    """
    communities = dict()
    for node, community in node_to_community.items():
        try:
            communities[community] += [node]
        except KeyError:
            communities[community] = [node]

    pos = dict()
    for ci, nodes in communities.items():
        subgraph = G.subgraph(nodes)
        pos_subgraph = nx.spring_layout(subgraph, weight='importance', scale=node_scale, seed=seed)
        pos.update(pos_subgraph)

    return pos

def squeeze_graph(G, partition, approximate_size=4000):
    """
    Squeeze graph by picking only top nodes (according to number of connections) in each partition. This
    step is needed to speed up the networkx visualization and show only the general POV on the graph.
    """

    #### STEP 1 - filtering nodes

    # Getting the number of partitions
    num_partitions = len(set(partition.values()))

    # Getting partition parameters
    partition_sizes = {i: len([1 for node, k in partition.items() if k == i]) for i in range(num_partitions)}
    min_partition_size = min(partition_sizes.values())

    # Normalizing partition size: divide each partition size by the minimal partition size
    normalized_partition_size = {i: (size // min_partition_size) for i, size in partition_sizes.items()}

    # Getting scale factor - to get approximately size of the graph close to approximate_size
    scale_factor = math.ceil(approximate_size / sum(normalized_partition_size.values()))
    squeezed_partition = {i: (size * scale_factor) for i, size in normalized_partition_size.items()}

    top_nodes = []
    for i, num_nodes in squeezed_partition.items():
        # Getting partition graph
        partition_i = G.subgraph([node for node, k in partition.items() if k == i])

        # Finding inter-community edges
        intercommunity_edges = _find_between_community_edges(G, partition, i)

        # Calculating node importance according to number of inter-community edges
        node_importance = {}
        for (part_1, part_2), edges in intercommunity_edges.items():
            for node_1, node_2 in edges:
                curr_node = node_1 if part_1 == i else node_2
                if curr_node in node_importance:
                    node_importance[curr_node] += 1
                else:
                    node_importance[curr_node] = 1

        # Getting top nodes in the partition according to maximum number of inter-community edge (node_importance)
        top_nodes += list(dict(sorted(node_importance.items(), key=lambda x: x[1], reverse=True)[:squeezed_partition[i]]).keys())

    filtered_partition = {node: i for node, i in partition.items() if node in top_nodes}
    filtered_G = G.subgraph(top_nodes)

    #### STEP 2 - filtering edges

    # Setting up the size of the squeezed graph (number of edges)
    keep_num_edges = 20000
    edges_to_keep = \
        list(
            dict(
                sorted(
                    {
                        (st, end): data['importance'] for st, end, data in filtered_G.edges(data=True)
                    }.items(), key=lambda x: x[1], reverse=True)[:keep_num_edges]
            ).keys()
    )
    squeezed_G = filtered_G.edge_subgraph(edges_to_keep)
    squeezed_partition = {node: i for node, i in filtered_partition.items() if node in squeezed_G.nodes()}

    return squeezed_G, squeezed_partition

def get_elipsis_mask():
        h, w = 600, 800
        center = (int(w/2), int(h/2))
        radius_x = w // 2
        radius_y = h // 2

        Y, X = np.ogrid[:h, :w]
        mask = ((X - center[0])**2/radius_x**2 + (Y - center[1])**2/radius_y**2 >= 1)*255

        return mask

def plot_cloud(G, partition, squeezed_pos, ax, anno_db, filter_genes=True,
               limit_anno_until=50, display_func=False, if_betweenness=True,
               k=3000):
    """
    Plot word cloud that indicates the function(s) of each gene cluster.
    """

    # Loading the gene functional annotation
    gene_func = load_gene_func_db(anno_db, reload=False, as_series=True)

    # Reversing partition dict -> {group_1: [gene_1, gene_2, ...], group_2: [gene_3, gene_4, ...], ...}
    partition_genes_ = {}
    for gene, i in partition.items():
        if i not in partition_genes_.keys():
            partition_genes_[i] = [gene]
        else:
            partition_genes_[i] += [gene]

    # If display gene function in the word clouds
    if display_func:

        # Whether to filter the genes on which we compute the word cloud (most important genes)
        if filter_genes:
            compute_centrality = nx.betweenness_centrality if if_betweenness else nx.closeness_centrality
            distance_metric = {'weight': 'distance'} if if_betweenness else {'distance': 'distance'}
            partition_genes = {}
            t = tqdm(partition_genes_.items())
            for i, genes in t:
                t.set_description(f'Processing cluster {i}, size={G.subgraph(genes).order()}')
                top_len = min(limit_anno_until, len(genes))
                top_gene_scores = dict(
                    sorted(
                        compute_centrality(
                            G.subgraph(genes), k=min(G.subgraph(genes).order(), k), **distance_metric
                        ).items(),
                        key=lambda x: x[1], reverse=True
                    )[:top_len]
                )
                # Renormalizing centrality scores between 1 and 100, and rounding them to use later when
                # displaying wordclouds (higher score - higher "frequency" or word size)
                norm_top_gene_scores = dict(
                    zip(
                        top_gene_scores.keys(), list(map(lambda x: int(x), scale(list(top_gene_scores.values()), 1, 100)))
                    )
                )
                partition_genes[i] = norm_top_gene_scores
            print('Filtered genes for generating the function word cloud..')
        else:
            partition_genes = {{gene_: 1 for gene_ in gene_list} for i, gene_list in partition_genes_.items()}

        # Computing functional annotation for each cluster as a concatenated list of annotations
        # Each annotation is weighted by its duplication gene_score times (e.g. a gene has score = 2 ->
        # the functional annotation is duplicated and have bigger font in WordCloud)
        partition_funcs = {
            i: ' '.join(
                chain.from_iterable([
                   gene_func[gene_func.index == gene].to_list()*gene_score
                        for gene, gene_score in gene_score_list.items()
            ])) for i, gene_score_list in partition_genes.items()
        }

        # Generating word counts from aggregated gene annotation texts -> obtaining main (most frequent) function tokens
        word_counts = {i: WordCloud(max_words=30, min_font_size=15, stopwords=stopwords).process_text(text) for i, text in partition_funcs.items()}
        word_counts = {
            i: (freqs if freqs else {'no found function': 1}) for i, freqs in word_counts.items()
        }  # dealing with no word case
        wordclouds = {
            i: WordCloud(
                max_words=30, min_font_size=15, stopwords=stopwords, background_color='white', mask=get_elipsis_mask()
            ).generate_from_frequencies(freqs) for i, freqs in word_counts.items()
        }

    # Display main genes in decreasing order of importance (top `top_len` genes)
    else:

        compute_centrality = nx.betweenness_centrality if if_betweenness else nx.closeness_centrality
        distance_metric = {'weight': 'distance'} if if_betweenness else {'distance': 'distance'}
        partition_genes = {}
        t = tqdm(partition_genes_.items())
        for i, genes in t:
            t.set_description(f'Processing cluster {i}, size={G.subgraph(genes).order()}')
            top_len = min(limit_anno_until, len(genes))
            top_gene_scores = dict(
                sorted(
                    compute_centrality(
                        G.subgraph(genes), k=min(G.subgraph(genes).order(), k), **distance_metric
                    ).items(),
                    key=lambda x: x[1], reverse=True
                )[:top_len]
            )
            # Renormalizing centrality scores between 1 and 100, and rounding them to use later when
            # displaying wordclouds (higher score - higher "frequency" or word size)
            norm_top_gene_scores = dict(
                zip(
                    top_gene_scores.keys(), list(map(lambda x: int(x), scale(list(top_gene_scores.values()), 1, 100)))
                )
            )
            partition_genes[i] = norm_top_gene_scores
        print('Obtained top genes for generating the gene word cloud..')

        wordclouds = {
            i: WordCloud(
                max_words=30, min_font_size=15, background_color='white', mask=get_elipsis_mask()
            ).generate_from_frequencies(gene_score_dict) for i, gene_score_dict in partition_genes.items()
        }

    # Plotting
    partition_coords = {}
    for gene, coords in squeezed_pos.items():
        if partition[gene] not in partition_coords:
            partition_coords[partition[gene]] = [coords]
        else:
            partition_coords[partition[gene]] += [coords]
    for i, coords in partition_coords.items():
        x, y = zip(*coords)
        min_x, max_x = min(x), max(x)
        min_y, max_y = min(y), max(y)
        ax.imshow(wordclouds[i], interpolation='bilinear', extent=[min_x, max_x, min_y, max_y])

    return ax

def process_communities(data, pat=None, algo='leiden', filter_quantile=0.95, if_betweenness=True,
                        limit_anno_until=50, k=5000, save_top_intercommunity_links_until=20,
                        other_functions_until=20, save_top_new_found_cluster_links=20, seed=42):
    """
    Process graph by finding its communities, annotate its communities, and save everything into .tsv format.
    """

    from joblib import Parallel, delayed

    def highlight_TFs(word, font_size, position, orientation, font_path, random_state):
        TF_color = (255, 0, 0)  # red
        if word in lambert_TF_names or word in dorothea_TF_names:
            return TF_color
        else:
            r, g, b, alpha = plt.get_cmap('viridis')(font_size / 120)
            return (int(r * 255), int(g * 255), int(b * 255))

    print('\nPerforming community analysis..\n\n')

    # Setting pathways to files
    _PROJ_PATH = '/gpfs/projects/bsc08/bsc08890'
    _FMETA = os.path.join(_PROJ_PATH, 'data/GSE145926_RAW/metadata.tsv')
    _DATA_HOME = os.path.join(_PROJ_PATH, 'res/covid_19')

    # Loading sample meta data, reordering patients
    full_meta = pd.read_csv(_FMETA, sep='\t', index_col=0)

    # Prepare everything to save the figs and dataframe
    if data == 'all_data':
        data = 'raw_data'
    elif 'raw_data_' not in data:
        data = f'raw_data_{data}'
    else:
        pass

    if pat is None or pat == 'all_data':

        # Cell-type aggregated data
        data_folder = 'all_data' if data == 'raw_data' else data.replace('raw_data_', '')

        figs_as = os.path.join(_DATA_HOME, 'cell_types', data_folder, 'figs', 'grnboost2', f'raw_data')

        data_to = os.path.join(_DATA_HOME, 'cell_types', data_folder, 'data', 'grnboost2', f'{algo}_communities')
        data_as = os.path.join(data_to, f'raw_data_communities_info.pickle')

    elif pat in ['C', 'M', 'S']:

        # Patient-type aggregated data
        data_folder = 'all_data' if data == 'raw_data' else data.replace('raw_data_', '')

        figs_as = os.path.join(_DATA_HOME, 'cell_types', data_folder, 'figs', 'grnboost2',
                               f'raw_data_{pat}_type')

        data_to = os.path.join(_DATA_HOME, 'cell_types', data_folder, 'data', 'grnboost2', f'{algo}_communities')
        data_as = os.path.join(data_to, f'raw_data_{pat}_type_communities_info.pickle')

    else:

        # Loading patient-specific data
        figs_as = os.path.join(_DATA_HOME, pat, 'figs', 'grnboost2', f'{data}')

        data_to = os.path.join(_DATA_HOME, pat, 'data', 'grnboost2', f'{algo}_communities')
        data_as = os.path.join(data_to, f'{data}_communities_info.pickle')

    os.makedirs(data_to, exist_ok=True)
    os.makedirs(os.path.dirname(figs_as), exist_ok=True)

    # Loading lists of TFs from Lambert 2018 and DoRothEA, in the latter case we will keep only confident regulons
    lambert_TF_names = pd.read_csv(os.path.join(_PROJ_PATH, 'data/TF_lists/lambert2018.txt'), header=None)[0].to_list()
    dorothea_TF_names = list(
        pd.read_csv(os.path.join(_PROJ_PATH, 'data/TF_lists/dorothea_regulons.tsv'), sep='\t') \
            .loc[lambda x: x['confidence'].isin(['A', 'B', 'C'])]['tf'].unique()
    )

    # Loading the graph
    G = get_nx_graph(data=data, data_type='all', pat=pat, get_filtered=filter_quantile)
    print(f"Loaded the graph: {colored('pat', 'green')}='{colored(pat, 'red')}', "
          f"{colored('data', 'green')}='{colored(data, 'red')}', "
          f"{colored('data_type', 'green')}='{colored('all', 'red')}'\n")

    ###### FINDING COMMUNITIES IN THE GRAPH #######

    print('Finding communities in the graph..')

    if algo == 'louvain':
        partition = community_louvain.best_partition(G.to_undirected(), weight='importance', random_state=seed)
    elif algo == 'leiden':
        G_igraph = ig.Graph.from_networkx(G.to_undirected())
        la_partition = la.find_partition(G_igraph, la.ModularityVertexPartition, weights='importance', seed=seed)
        partition = {G_igraph.vs[node]['_nx_name']: i for i, cluster_nodes in enumerate(la_partition) for node in cluster_nodes}
    else:
        raise NotImplementedError

    num_partitions = len(set(partition.values()))
    print(f'Number of partitions using {algo} algorithm: {colored(num_partitions, "cyan")}\n')

    ###### FINDING HIGH-CENTRALITY GENES IN THE WHOLE GRAPH

    print('Finding high-centrality genes in the whole graph..')

    num_workers = max(multiprocessing.cpu_count() // 2, 1)
    whole_G_central_genes = dict(
        sorted(betweenness_centrality_parallel(G, processes=num_workers).items(), key=lambda x: x[1], reverse=True)[:limit_anno_until]
    )
    print(f'Computed the {"betweenness" if if_betweenness else "closeness"} centrality for all genes in the graph\n')

    ###### FINDING HIGH-CENTRALITY GENES AND CORRESPONDING FUNCTIONS IN EACH COMMUNITY USING GO ANNOTATION ######

    print('Finding high-centrality genes/functions in each cluster..')

    # Loading the gene functional annotation
    anno_db_tags = ['GO', 'KEGG', 'immunological', 'hallmark']
    gene_func_dbs = {tag: load_gene_func_db(tag, as_series=True) for tag in anno_db_tags}

    # Reversing partition dict -> {group_1: [gene_1, gene_2, ...], group_2: [gene_3, gene_4, ...], ...}
    partition_genes_ = {}
    for gene, i in partition.items():
        if i not in partition_genes_.keys():
            partition_genes_[i] = [gene]
        else:
            partition_genes_[i] += [gene]

    # Whether to filter the genes on which we compute the word cloud (most important genes)
    compute_centrality = nx.betweenness_centrality if if_betweenness else nx.closeness_centrality
    distance_metric = {'weight': 'distance'} if if_betweenness else {'distance': 'distance'}
    all_partition_genes = {}
    norm_partition_genes = {}
    t = tqdm_cli(partition_genes_.items(), ascii=True)
    for i, genes in t:
        t.set_description(f'Processing cluster {i}, size={G.subgraph(genes).order()}')
        gene_scores = dict(
            sorted(
                compute_centrality(
                    G.subgraph(genes), k=min(G.subgraph(genes).order(), k), normalized=True, **distance_metric
                ).items(),
                key=lambda x: x[1], reverse=True
            )
        )
        all_partition_genes[i] = gene_scores
        central_gene_scores = {gene: gene_scores[gene] for k, gene in enumerate(gene_scores.keys()) if k < limit_anno_until}

        # Renormalizing centrality scores between 1 and 100, and rounding them to use later when
        # displaying wordclouds (higher score - higher "frequency" or word size)
        norm_partition_genes[i] = dict(
            zip(
                central_gene_scores.keys(),
                list(map(lambda x: int(x), scale(list(central_gene_scores.values()), 1, 100)))
            )
        )
    print('Computed centrality scores for each gene in each community\n')

    print('Finding functional annotations for each cluster..')

    # Computing functional annotation for each cluster as a concatenated list of annotations
    # Each annotation is weighted by its duplication gene_score times (e.g. a gene has score = 2 ->
    # the functional annotation is duplicated and have bigger font in WordCloud)
    # We also do it for different functional annotations like GO, KEGG, Hallmark, etc..
    partition_funcs = {
        tag:
            {
                i: ' '.join(
                    chain.from_iterable([
                       gene_func[gene_func.index == gene].to_list()*gene_score
                            for gene, gene_score in gene_score_list.items()
                ])) for i, gene_score_list in norm_partition_genes.items()
            } for tag, gene_func in gene_func_dbs.items()
    }

    print('Computed functional annotations for each cluster\n')

    ###### PLOTTING GENE AND FUNC COMMUNITY CLOUDS ######

    print('Plotting clusters..')

    # Getting positions of squeezed graph - we do not plot every gene on the figure
    squeezed_G, squeezed_partition = squeeze_graph(G, partition)
    print('Computed a squeezed graph representation..')

    squeezed_pos = netgraph_community_layout(squeezed_G, squeezed_partition, seed=seed)  # nx.nx_agraph.pygraphviz_layout(G.to_undirected(), prog="sfdp")  # nx.nx.spring_layout(G, seed=seed, k=0.2, iterations=20)
    partition_coords = {}
    for gene, coords in squeezed_pos.items():
        if partition[gene] not in partition_coords:
            partition_coords[partition[gene]] = [coords]
        else:
            partition_coords[partition[gene]] += [coords]
    print('Computed node positions of the squeezed graph representation..')

    cmap = ListedColormap(sns.color_palette(cc.glasbey_bw, n_colors=num_partitions).as_hex())

    for plot_type in ['genes'] + list(map(lambda x: f"func_{x}", anno_db_tags)):

        if plot_type.startswith('func'):
            # Getting current functional annotation
            curr_partition_funcs = partition_funcs[plot_type[plot_type.find('_') + 1:]]

        f, ax = plt.subplots(figsize=(20, 35))

        if plot_type == 'genes':
            wordclouds = {
                i: WordCloud(
                    max_words=30, min_font_size=15, background_color='white', mask=get_elipsis_mask()
                ).generate_from_frequencies(gene_score_dict).recolor(color_func=highlight_TFs)
                    for i, gene_score_dict in norm_partition_genes.items()
            }
        else:
            word_counts = {
                i: WordCloud(max_words=30, min_font_size=15, stopwords=stopwords).process_text(text) for i, text in curr_partition_funcs.items()
            }
            word_counts = {
                i: (freqs if freqs else {'no found function': 1}) for i, freqs in word_counts.items()
            }  # dealing with no word case
            wordclouds = {
                i: WordCloud(
                    max_words=30, min_font_size=15, stopwords=stopwords, background_color='white', mask=get_elipsis_mask()
                ).generate_from_frequencies(freqs) for i, freqs in word_counts.items()
            }

        # Plotting clouds
        for i, coords in partition_coords.items():
            x, y = zip(*coords)
            min_x, max_x = min(x), max(x)
            min_y, max_y = min(y), max(y)
            ax.imshow(wordclouds[i], interpolation='bilinear', extent=[min_x, max_x, min_y, max_y])
        print(f'Finished plotting {plot_type} word cloud..')

        nx.draw(squeezed_G, squeezed_pos, ax=ax, arrowstyle="->", arrowsize=20,
                connectionstyle=f'arc3, rad = 0.25', edge_color='gray', width=0.4,
                node_color='k', node_size=50, alpha=0.02)
        nx.draw_networkx_nodes(squeezed_G, squeezed_pos, ax=ax, node_size=100,
                               nodelist=list(squeezed_partition.keys()),
                               node_color=list(squeezed_partition.values()),
                               cmap=cmap, alpha=0.005)
        print(f'Finished plotting {plot_type} nodes..')

        ax.set_title(f'Found communities ({pat}, "all", {data}), '
                     f'annotation - {plot_type}',
                     fontsize=30)
        plt.axis('off')

        plt.savefig(f'{figs_as}_{plot_type}.png', bbox_inches='tight', dpi=400)

    print('Finished plotting..\n')

    ###### SAVING DATAFRAME CONTAINING INFORMATION ABOUT EACH COMMUNITY ######

    def compute_community_info(i):
        """
        Parallel saving of the dataframe.
        """

        # Getting information for each community
        genes = list(all_partition_genes[i].keys())
        community_subgraph = G.subgraph(genes)

        communities_i = pd.Series(dtype='object')

        # Setting tqdm logs
        # t.set_description(f'Saving info about {i} cluster, size={community_subgraph.order()}')

        # Getting information about cluster genes
        central_genes_and_scores = {
            gene: all_partition_genes[i][gene] for k, gene in enumerate(genes) if k < limit_anno_until
        }

        non_lambert_TFs = [
            f'{gene} (rank={k})' for k, gene in enumerate(central_genes_and_scores.keys(), start=1) if gene not in lambert_TF_names
        ]
        non_dorothea_TFs = [
            f'{gene} (rank={k})' for k, gene in enumerate(central_genes_and_scores.keys(), start=1) if gene not in dorothea_TF_names
        ]

        # Filling dataframe with the information
        communities_i['num_nodes'] = community_subgraph.number_of_nodes()
        communities_i['num_edges'] = community_subgraph.number_of_edges()
        communities_i['all_sorted_genes'] = '; '.join(
            f'{gene} (score={score})' for gene, score in all_partition_genes[i].items()
        )
        communities_i['sorted_central_genes_scores'] = '; '.join(
            f'{gene} (score={score:.2f})' for gene, score in central_genes_and_scores.items()
        )
        communities_i['non_lambert_2018_TF_central_genes'] = '; '.join(non_lambert_TFs)
        communities_i['non_dorothea_TF_central_genes'] = '; '.join(non_dorothea_TFs)
        communities_i['whole_G_central_genes_scores'] = '; '.join(
            f'{gene} (score={score:.2f})' for gene, score in whole_G_central_genes.items()
        )

        # Filling information about newly found gene-gene links (based on absence in KEGG and Hallmark)
        top_cluster_links = set()

        iter_i = 0

        for st, end, edge_info in sorted(community_subgraph.edges(data=True),
                                         key=lambda t: t[2]['importance'],
                                         reverse=True):

            # If the current (reverse directed) link was not encountered previously..
            if (end, st) not in [(uniq_st, uniq_end) for uniq_st, uniq_end, _ in top_cluster_links]:
                top_cluster_links.add((st, end, edge_info['importance']))
                iter_i += 1
            if iter_i == save_top_new_found_cluster_links:
                break

        for anno_tag in ['KEGG', 'hallmark']:

            curr_db = load_gene_func_db(anno_tag)
            tmp_list = []

            # if `st` gene and `end` gene have non-overlapping annotations..
            for st, end, imp in top_cluster_links:
                st_anno_IDs = set(curr_db[curr_db.index == st]['ID'])
                end_anno_IDs = set(curr_db[curr_db.index == end]['ID'])
                if len(st_anno_IDs.intersection(end_anno_IDs)) == 0 and \
                        (len(st_anno_IDs) != 0 or len(end_anno_IDs) != 0):
                    tmp_list.append(f"{st} ({' & '.join(st_anno_IDs)}) <-> {end} ({' & '.join(end_anno_IDs)})")

            communities_i[f'new_gene_gene_links_{anno_tag}'] = '; '.join(tmp_list)

        # Filling information about cluster functions
        for tag, gene_func in gene_func_dbs.items():

            curr_partition_funcs = partition_funcs[tag]

            # Filling main functions - non duplicates at the top
            main_functions = list(dict.fromkeys([  # dropping duplicates, but preserving order
                func for gene in central_genes_and_scores.keys()
                    for func in gene_func[gene_func.index == gene].to_list()
            ]))
            gene_with_main_functions = [
                ','.join(
                    gene_func[gene_func == func].loc[lambda x: x.index.isin(genes)].index.to_list()
                ) for func in main_functions
            ]
            main_functions = [
                f'>>> {func} <<<: {gene}' for gene, func in zip(gene_with_main_functions, main_functions)
            ]
            communities_i[f'main_functions_{tag}'] = '; '.join(main_functions)  # saving..

            # Saving functions corresponding to each gene
            central_functions_per_gene = [
                f">>> {gene} <<<: {' & '.join(gene_func[gene_func.index == gene].to_list())}" for gene in central_genes_and_scores.keys()
            ]
            communities_i[f'sorted_central_functions_{tag}'] = '; '.join(central_functions_per_gene)  # saving..

            # Saving most frequent function words
            freq_words = WordCloud(
                max_words=30, min_font_size=15, stopwords=stopwords
            ).process_text(curr_partition_funcs[i])
            freq_words = dict(
                sorted(freq_words.items(), key=lambda x: x[1], reverse=True)
            ) if freq_words else {'no found function': 1}  # dealing with no word case
            communities_i[f'most_frequent_function_words_{tag}'] = '; '.join(freq_words.keys())  # saving

            # Saving other functions present in this cluster
            other_functions = list(dict.fromkeys([  # dropping duplicates, but preserving order
                func for gene in genes if gene not in central_genes_and_scores.keys()
                    for func in gene_func[gene_func.index == gene].to_list() if func not in main_functions
            ]))[:other_functions_until]
            genes_with_other_functions = [
                ','.join(
                    gene_func[gene_func == func].loc[lambda x: x.index.isin(genes)].index.to_list()
                ) for func in other_functions
            ]
            other_functions = [
                f'>>> {func} <<<: {gene}' for gene, func in zip(genes_with_other_functions, other_functions)
            ]
            communities_i[f'other_functions_{tag}'] = '; '.join(other_functions)  # saving

        # Filling information about top inter-community links
        # t_sub = tqdm(range(num_partitions), ascii=True, leave=False)
        for k in range(num_partitions):  # t_sub:
            # t_sub.set_description(f'Extracting top inter-community links with {k}')

            if i != k:
                genes_in_k = list(all_partition_genes[k].keys())

                # Getting the subgraph that contains central genes in community_i and all genes in comunity_k
                G_central_i_k = G.subgraph(list(central_genes_and_scores.keys()) + genes_in_k)
                # Getting the subgraph that contains all genes from community_i and community_k
                G_i_k = G.subgraph(genes + genes_in_k)

                # Creating two helper sets that allow us to keep only unique links
                links_central_i_k = set()
                links_i_k = set()

                iter_i = 0

                # Getting out top links from the second subgraph
                for st, end, edge_info in sorted(G_central_i_k.edges(data=True),
                                                 key=lambda t: t[2]['importance'],
                                                 reverse=True):
                    # If the current (reverse directed) link was not encountered previously..
                    if (end, st) not in [(uniq_st, uniq_end) for uniq_st, uniq_end, _ in links_central_i_k] and \
                            ((st in genes and end not in genes) or (end in genes and st in genes)):
                        links_central_i_k.add((st, end, edge_info['importance']))
                        iter_i += 1
                    if iter_i == save_top_intercommunity_links_until:
                        break

                iter_i = 0

                # Getting out top links from the second subgraph
                for st, end, edge_info in sorted(G_i_k.edges(data=True),
                                                 key=lambda t: t[2]['importance'],
                                                 reverse=True):
                    # If the current (reverse directed) link was not encountered previously..
                    if (end, st) not in [(uniq_st, uniq_end) for uniq_st, uniq_end, _ in links_i_k] and \
                            ((st in genes and end not in genes) or (end in genes and st in genes)):
                        links_i_k.add((st, end, edge_info['importance']))
                        iter_i += 1
                    if iter_i == save_top_intercommunity_links_until:
                        break

                # Adding top links to the dataframe
                communities_i[f'top_links_scores_central_genes<->community_{k}'] = \
                    '; '.join(f'{st} <-> {end} (score={score:.2f})' for st, end, score in links_central_i_k)
                communities_i[f'top_links_scores_with_community_{k}'] = \
                    '; '.join([f'{st} <-> {end} (score={score:.2f})' for st, end, score in links_i_k])

        return communities_i

    print('Saving info dataframe..')

    t = tqdm_cli(range(num_partitions), ascii=True)

    # Getting dataframe
    result = Parallel(n_jobs=num_workers)(delayed(compute_community_info)(i) for i in t)
    communities_df = pd.concat(result, axis=1).T.reindex(
        columns=[
            'num_nodes', 'num_edges',
            'main_functions_GO', 'main_functions_KEGG', 'main_functions_immunological', 'main_functions_hallmark',
            'non_lambert_2018_TF_central_genes', 'non_dorothea_TF_central_genes',
            'new_gene_gene_links_KEGG', 'new_gene_gene_links_hallmark',
            'whole_G_central_genes_scores',
            'other_functions_GO', 'other_functions_KEGG', 'other_functions_immunological', 'other_functions_hallmark',
            'sorted_central_genes_scores',
            'sorted_central_functions_GO', 'sorted_central_functions_KEGG', 'sorted_central_functions_immunological', 'sorted_central_functions_hallmark',
            'most_frequent_function_words_GO', 'most_frequent_function_words_KEGG', 'most_frequent_function_words_immunological', 'most_frequent_function_words_hallmark',
            'all_sorted_genes'] +
            [f'top_links_scores_central_genes<->community_{i}' for i in range(num_partitions)] +
            [f'top_links_scores_with_community_{i}' for i in range(num_partitions)
            ]
    )

    # Saving dataframe
    communities_df.to_pickle(data_as)
    print(f"Saved the data to {data_as}!\n")

def run_enrichr(data, is_communities=False, is_positive_markers=True, group_types = 'all', on_targets=False, choose_fixed_tf=None,
                data_type='all', top_n=50, algo='leiden', enrichr_library='MSigDB_Hallmark_2020'):
    """
    Run enrichment analysis with Enrichr.
    """

    import json
    import requests
    import sys
    import io

    out_folder = 'community_ana' if is_communities else 'cohort_ana'

    if is_communities == True:

        print('Running EnrichR on communities..')

        algo = 'leiden'
        _DATA_HOME = '/gpfs/projects/bsc08/bsc08890/res/covid_19'

        if data_type == 'all':
            community_data = pd.read_pickle(os.path.join(
                _DATA_HOME, 'cell_types', data, 'data', 'grnboost2', f'{algo}_communities',
                f'raw_data_communities_info.pickle'
            ))
        else:
            community_data = pd.read_pickle(os.path.join(
                _DATA_HOME, 'cell_types', data, 'data', 'grnboost2', f'{algo}_communities',
                f'raw_data_{data_type}_type_communities_info.pickle'
            ))

        df = pd.concat([
            pd.DataFrame({
                'cluster': f'cluster_{i}',
                'gene': [el[: el.find(' ')] for el in vals.split('; ')][:top_n]
            }) for i, vals in community_data['all_sorted_genes'].iteritems()
        ], axis=0).reset_index(drop=True)

    else:

        if on_targets:

            print('Running EnrichR on targets between 3 group types..')

            types = ['C', 'M', 'S']

            df = pd.concat([
                pd.read_csv(
                    f'/gpfs/home/bsc08/bsc08890/tmp/cohort_ana/tmp_enrichr_{data}_{t}_{choose_fixed_tf}_target_list.tsv',
                    header=None, names=['gene']
                ).assign(cluster=t) for t in types
            ], axis=0)

        else:

            if group_types == 'all':
                print('Running EnrichR on TFs between 3 group types..')
                df = pd.read_csv(f'/gpfs/home/bsc08/bsc08890/tmp/tf_markers_df_{data}.tsv', sep='\t')
            else:
                print('Running EnrichR on 2 group types..')
                if group_types == 'M_S':
                    group_types = 'S_M'
                if group_types == 'C_M':
                    group_types = 'M_C'
                if group_types == 'C_S':
                    group_types = 'S_C'
                df_1 = pd.read_csv(f'/gpfs/home/bsc08/bsc08890/tmp/tf_markers_df_{group_types}_{data}.tsv', sep='\t')
                df_1['gene'] = df_1.index
                df_2 = df_1.copy()
                df_2['avg_log2FC'] = - df_2['avg_log2FC']
                df_1['cluster'], df_2['cluster'] = group_types.split('_')

                df = pd.concat([df_1, df_2], axis=0)

            if is_positive_markers:
                df = df[(df['p_val_adj'] < 0.05) & (df['avg_log2FC'] > 1)]
            else:
                df = df[(df['p_val_adj'] < 0.05) & (df['avg_log2FC'] < -1)]

    cluster_dfs = {}
    for cl in df['cluster'].unique():

        print(f'Processing {cl}..')

        ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/addList'
        genes_str = '\n'.join(df[df['cluster'] == cl]['gene'])
        description = f"{data}_{data_type}_{cl}"

        if is_communities == True:
            filename = f'tmp/{out_folder}/tmp_enrichr_{data}_{data_type}_{cl}.tsv'
        elif on_targets:
            filename = f'tmp/{out_folder}/tmp_enrichr_{data}_{data_type}_{choose_fixed_tf}_target_{cl}.tsv'
        elif group_types == 'all':
            filename = f'tmp/{out_folder}/tmp_enrichr_{data}_{data_type}_{cl}.tsv'
        else:
            filename = f'tmp/{out_folder}/tmp_enrichr_{data}_2_groups_{cl}.tsv'

        payload = {
          'list': (None, genes_str),
          'description': (None, description)
        }
        response = requests.post(ENRICHR_URL, files=payload)

        if not response.ok:
            raise Exception('Error analyzing gene list')

        job_id = json.loads(response.text)

        ################################################################################
        # Get enrichment results
        #
        ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/export'
        query_string = '?userListId=%s&filename=%s&backgroundType=%s'
        user_list_id = str(job_id['userListId'])
        gene_set_library = str(enrichr_library)
        url = ENRICHR_URL + query_string % (user_list_id, filename, gene_set_library)

        response = requests.get(url, stream=True)

        print('        Enrichr API : Downloading file of enrichment results: Job Id:', job_id)
        with open(filename, 'wb') as f:
            for chunk in response.iter_content(chunk_size=1024):
                if chunk:
                    f.write(chunk)

        print(f'        Saved to {filename}')

        cluster_dfs[cl] = pd.read_csv(filename, sep='\t')

    return cluster_dfs

def betweenness_centrality_parallel(G, processes=None):
    """Parallel betweenness centrality  function"""
    from multiprocessing import Pool

    def chunks(l, n):
        """Divide a list of nodes `l` in `n` chunks"""
        l_c = iter(l)
        while 1:
            x = tuple(itertools.islice(l_c, n))
            if not x:
                return
            yield x

    p = Pool(processes=processes)
    node_divisor = len(p._pool) * 4
    node_chunks = list(chunks(G.nodes(), int(G.order() / node_divisor)))
    num_chunks = len(node_chunks)
    bt_sc = p.starmap(
        nx.betweenness_centrality_subset,
        zip(
            [G] * num_chunks,
            node_chunks,
            [list(G)] * num_chunks,
            [True] * num_chunks,
            ['distance'] * num_chunks
        ),
    )

    # Reduce the partial solutions
    bt_c = bt_sc[0]
    for bt in bt_sc[1:]:
        for n in bt:
            bt_c[n] += bt[n]
    return bt_c


#!/usr/bin/python3

from time import sleep
from datetime import date, datetime
from pynput.keyboard import Key, Controller
from logging.handlers import RotatingFileHandler
import sys, signal, argparse, logging, platform, subprocess

# ----------------------------------Configuration--------------------------------
VOLUME = "0.3"
BREAK_NUM = 1
WORK_DURATION = 900
BREAK_DURATION = 120

MAC = False
LINUX = False
WINDOWS = False

LINUX_PATH = ""
MAC_PATH = "/Users/mutnawaz/Desktop/Muteeb/Code/timer/"
WINDOWS_PATH = "C:\\Users\\Muteeb\\Desktop\\RV Major Project\\Personal\\timer\\"

# ---------------------------------end of Configuration---------------------------

log = None

def __init_logger():
    global log
    if log is not None:
        log.debug("logger already initialized.")
        return None

    try:
        "log format <data/time:level:filename:line:function:message>"
        log_formatter = logging.Formatter("%(levelname)5.5s  %(filename)5s#%(lineno)3s  %(message)s")

        "Refer the log file path"
        PATH = get_path()
        log_file = PATH + "timer.log"

        "Max size of the log file is 2MB, it rotate if size exceeds"
        handler = RotatingFileHandler(
            log_file,
            mode="a",
            maxBytes=(2 * 1024 * 1024),
            backupCount=4,
            encoding=None,
            delay=0,
        )

        "appy the log format and level"
        handler.setFormatter(log_formatter)
        handler.setLevel(logging.DEBUG)
        log = logging.getLogger("timer.log")
        log.setLevel(logging.DEBUG)

        "apply the settings to the log"
        log.addHandler(handler)
        log.debug("Start logging the times")
        return handler

    except Exception as e:
        log.error("Failed to create logger: %s", str(e))

def exit_handler(sig, frame):
    print("\nGood bye. Have a nice day!\n")
    greet()
    sys.exit(0)

def greet():
    try:
        print(subprocess.check_output("motivate", shell=True, stderr=subprocess.DEVNULL).decode())
    except:
        print("\n******************************************************")
        print("*                                                    *")
        print("*                                                    *")
        print("*   You can do it! Sending lots of energy to you :)  *")
        print("*                                                    *")
        print("*                                                    *")
        print("******************************************************")

def get_time():
    now = datetime.now()
    time = now.strftime("%H:%M:%S")
    return time

def play_sound(sound_file):
    if MAC:
        subprocess.check_output("afplay --volume " + VOLUME + " {}".format(sound_file), shell=True)
    elif LINUX:
        subprocess.check_output("aplay -q {}&".format(sound_file), shell=True)
    else:
        winsound.PlaySound(sound_file, winsound.SND_ASYNC)

def get_path():
    if MAC:
        return MAC_PATH
    elif LINUX:
        return LINUX_PATH
    else:
        return WINDOWS_PATH

def display_sleep():
    if MAC:
        # subprocess.check_output("pmset displaysleepnow", shell=True)  # Put system to sleep.
        subprocess.check_output("open -a ScreenSaverEngine", shell=True)

def wakeup():
    if MAC:
        # subprocess.check_output("pmset relative wake 1", shell=True)  # Wakeup the system.
        # log.debug("Waking up.")
        keyboard = Controller()
        key = Key.esc

        keyboard.press(key)
        keyboard.release(key)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("-s", "--slient", action="store_true", help="Run in silent mode.")
    args = vars(parser.parse_args())

    if platform.system() == "linux" or platform.system() == "linux2":
        LINUX = True
    elif platform.system() == "darwin" or platform.system() == "Darwin":
        MAC = True
    elif platform.system() == "win32" or platform.system() == "Windows":
        WINDOWS = True
        if not args["slient"]:
            try:
                import winsound
            except Exception as e:
                print("Sound is not supported in windows. Reason: {0}".format(e))
                args["slient"] = True

    __init_logger()
    PATH = get_path()
    signal.signal(signal.SIGINT, exit_handler)
    greet()

    if args["slient"]:
        print("Running in slient mode...")

    log.info("Today's date: {0}".format(date.today()))
    if not args["slient"]:
        play_sound(PATH + "start_timer.wav")

    while True:

        log.info("Work number  {0}, start time  {1}".format(BREAK_NUM, get_time()))
        sleep(WORK_DURATION)
        log.info("Work number  {0}, end time    {1}".format(BREAK_NUM, get_time()))
        if not args["slient"]:
            play_sound(PATH + "take_break.wav")

        display_sleep()

        log.info("Break number {0}, start time  {1}".format(BREAK_NUM, get_time()))
        sleep(BREAK_DURATION)
        log.info("Break number {0}, end time    {1}".format(BREAK_NUM, get_time()))
        if not args["slient"]:
            play_sound(PATH + "two_mins_up.wav")

        wakeup()
        BREAK_NUM += 1


import sys
sys.path.insert(0, '../../class')

import os
import time
import nnet
import cubelattice as cl
import multiprocessing
from functools import partial
from scipy.io import loadmat
import numpy as np
import argparse

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description='Verification Settings')
    parser.add_argument('--property', type=str, default='1')
    parser.add_argument('--n1', type=int, default=2)
    parser.add_argument('--n2', type=int, default=3)
    parser.add_argument('--compute_unsafety', action='store_true')
    args = parser.parse_args()

    i = args.n1
    j = args.n2

    def verification(afv):
        safe = True
        return safe

    print("neural_network_"+str(i)+str(j))
    nn_path = "nets/neural_network_information_"+str(i)+str(j)+".mat"
    filemat = loadmat(nn_path)
    if not os.path.isdir('logs'):
        os.mkdir('logs')

    W = filemat['W'][0]
    b = filemat['b'][0]

    lb = [-0.1,-0.1,-0.1]
    ub = [0.1,0.1,0.1]

    nnet0 = nnet.nnetwork(W, b)
    nnet0.verification = verification
    initial_input = cl.cubelattice(lb, ub).to_lattice()
    cpus = multiprocessing.cpu_count()
    pool = multiprocessing.Pool(cpus)

    nnet0.start_time = time.time()
    nnet0.filename = "logs/output_info"+str(i)+str(j)+'.txt'
    outputSets = []
    nputSets0 = nnet0.singleLayerOutput(initial_input, 0)
    pool.map(partial(nnet0.layerOutput, m=1), nputSets0)
    pool.close()
    elapsed_time = time.time() - nnet0.start_time

    print('time elapsed: %f seconds \n' % elapsed_time)
    print('result: safe\n')
    filex = open(nnet0.filename, 'w')
    filex.write('time elapsed: %f seconds \n' % elapsed_time)
    filex.write('result: safe\n')
    filex.close()


#compare parameter between abc-smc
import seaborn as sns
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sys
from scipy import stats
from matplotlib.colors import LogNorm, Normalize
from scipy.signal import argrelextrema

filename=["ACDC_X2","ACDC_Y2","ACDC_Z2"]#,"ACDC_all"]
#filename=['ACDC_X2']
filename=['ACDC_X2','ACDC_X21ind']
n=['final']
#n=['1','2','3','4','5','6','7','8','9','10','11','12','final']#'13','14','15','final']
#n=['1','2','3','4','5','6','7','8','9','10','11','12','13','final']#,'12','13','14','final']#,'15']#,'final']

path='C:/Users/Administrator/Desktop/Modeling/AC-DC/'
path='/users/ibarbier/AC-DC/'
sys.path.insert(0, path + filename[0])

import model_equation as meq

parlist=meq.parlist

namelist=[]
for i,par in enumerate(parlist):
    namelist.append(parlist[i]['name'])

par0 = {
    'K_ARAX':-3.5,#0.01,
    'n_ARAX':2,
    'K_XY':-2.5,
    'n_XY':2,
    'K_XZ':-1.55,#-1.25
    'n_XZ':2,
    'beta_X':1,
    'alpha_X':0,
    'delta_X':1,

    'K_ARAY':-3.5,
    'n_ARAY':2,
    'K_YZ':-3.5,
    'n_YZ':2,
    'beta_Y':1,
    'alpha_Y':0,
    'delta_Y':1,

    'K_ZX':-2.5,
    'n_ZX':2,
    'beta_Z':1,
    'alpha_Z':0,
    'delta_Z':1,

    'beta/alpha_X':2,
    'beta/alpha_Y':2,
    'beta/alpha_Z':2

}

def pars_to_dict(pars,parlist):
### This function is not necessary, but it makes the code a bit easier to read,
### it transforms an array of pars e.g. p[0],p[1],p[2] into a
### named dictionary e.g. p['k0'],p['B'],p['n'],p['x0']
### so it is easier to follow the parameters in the code
    dict_pars = {}
    for ipar,par in enumerate(parlist):
        dict_pars[par['name']] = pars[ipar]
    return dict_pars

def load(number= n,filename=filename,parlist=parlist):
    namelist=[]
    for i,par in enumerate(parlist):
      namelist.append(parlist[i]['name'])
    number=str(number)
    filepath = path+filename+'/smc/pars_' + number + '.out'
    dist_path =  path+filename+'/smc/distances_' + number + '.out'
    raw_output= np.loadtxt(filepath)
    dist_output= np.loadtxt(dist_path)
    df = pd.DataFrame(raw_output, columns = namelist)
    df['dist']=dist_output
    df=df.sort_values('dist',ascending=False)
    distlist= sorted(df['dist'])
    p=[]
    for dist in distlist:
        p_0=df[df['dist']==dist]
        p0=[]
        for n in namelist:
          p0.append(p_0[n].tolist()[0])
        p0=pars_to_dict(p0,parlist)
        p.append(p0)
    return p, df

def get_stats(filename,namelist):
    stats_df = pd.DataFrame( columns = ['par','file','mean','sd','mode'])
    parl = np.append(namelist,'dist')
   # for fi,fnm in enumerate(filename):
    fnm=filename[0]
    p,df= load(n[0],fnm,parlist)
    mean=np.mean(df).tolist()
    sd=np.std(df).tolist()
    mode=stats.mode(df)[0][0]
    new_row={'par':parl,'file':[fnm]*len(parl),'mean':mean,'sd':sd,'mode':mode}
    df2=pd.DataFrame(new_row)
    stats_df =stats_df.append(df2)
    return stats_df

def bar_plot(filename,namelist, t="mean"):
    stats_df=get_stats(filename,namelist)
    # set width of bars
    barWidth = 0.20
    # Set position of bar on X axis
    r1 = np.arange(len(parl))

    #mean
    if t=="mean":
        for i,nm in enumerate(filename):
            v=stats_df[stats_df['method']==nm]
            plt.bar((r1+barWidth*i),v['mean'],yerr=v['sd'], capsize=2,width=barWidth, label=nm)

        plt.xlabel('par', fontweight='bold')
        plt.xticks([r + barWidth for r in range(len(parl))], parl)
        plt.legend()
        plt.show()

    #mode
    if t == "mode":
        for i,nm in enumerate(filename):
            v=stats_df[stats_df['method']==nm]
            plt.bar((r1+barWidth*i),v['mode'],width=barWidth, label=nm)

        plt.xlabel('par', fontweight='bold')
        plt.xticks([r + barWidth for r in range(len(parl))], parl)
        plt.legend()
        plt.show()

def plot_compare(n,filename,namelist):

    parl = np.append(namelist,'dist')
    index=1
    size=round(np.sqrt(len(parl)))
    for i,name in enumerate(parl):
        plt.subplot(size,size,index)
        plt.tight_layout()
        for fi,fnm in enumerate(filename):
              p,df= load(n,fnm,namelist1)
              sns.kdeplot(df[name],bw_adjust=.8,label=fnm)
        #plt.ylim(0,1)
        if i < (len(parl)-2):
            plt.xlim((parlist[i]['lower_limit'],parlist[i]['upper_limit']))
        index=index+1
        if index==5:
            plt.legend(bbox_to_anchor=(1.05, 1))

    #sns.kdeplot(df['K_XZ'])
    plt.savefig(str(filename)+str(n)+"_compareplot.pdf", bbox_inches='tight')
    plt.show()

#plot_compare(n[0],filename,namelist)

def plot_alltime(filename,namelist):
    parl = np.append(namelist,'dist')
    index=1
    for i,name in enumerate(parl):
        plt.subplot(4,4,index)
        plt.tight_layout()
        for ni,nmbr in enumerate(n):
            p,df= load(nmbr,filename[0],parlist)
            sns.kdeplot(df[name],bw_adjust=.8,label=nmbr)
        #plt.ylim(0,1)
        if i < (len(parl)-2):
            plt.xlim((parlist[i]['lower_limit'],parlist[i]['upper_limit']))
        index=index+1
        #if index==5:
    plt.legend(bbox_to_anchor=(1.05, 1))
    plt.show()

#plot_alltime(['ACDC_X2'],namelist)

def plotdistpar(filename,namelist):
    index=1
    for ni,nb in enumerate(n):
        p,df= load(nb,filename[0],parlist)
        for i,name in enumerate(namelist):
            plt.subplot(len(n),len(namelist),index)
           # plt.tight_layout()
            plt.scatter(df['dist'],df[name],s=1)
            mean=np.mean(df[name]).tolist()
            mode=stats.mode(df[name])[0][0]

            plt.plot([0,40],[mean,mean],'r',label="mean")
            plt.plot([0,40],[mode,mode],'g',label="meode")

            plt.ylim((parlist[i]['lower_limit'],parlist[i]['upper_limit']))
            plt.ylabel(name)
            index=index+1
    plt.legend(bbox_to_anchor=(1.05, 1))
    plt.show()

'''
ARA=np.logspace(-4.5,-2.,10,base=10)
p,df= load(n[0],filename[0],parlist)

stdf=get_stats(filename,namelist)
pmean=pars_to_dict(stdf['mean'])
pmode=pars_to_dict(stdf['mode'])

for i,p in enumerate([p[0],pmean,pmode,p[999]]):

    X,Y,Z=meq.model(ARA,p)
    df_X=pd.DataFrame(X,columns=ARA)
    df_Y=pd.DataFrame(Y,columns=ARA)
    df_Z=pd.DataFrame(Z,columns=ARA)

    plt.subplot(4,3,(1+3*i))
    sns.heatmap(df_X, cmap="Reds")
    plt.subplot(4,3,(2+3*i))
    sns.heatmap(df_Y, cmap ='Blues')
    plt.subplot(4,3,(3+3*i))
    sns.heatmap(df_Z, cmap ='Greens')

plt.show()

X,Y,Z=meq.model(ARA,pmode)
plt.plot(X[:,0],label="DCoff")
plt.plot(X[:,3],label="AC1")
plt.plot(X[:,6],label="AC2")
plt.plot(X[:,9],label="DCon")
plt.plot([200,200],[0,1000],'--')

plt.legend(bbox_to_anchor=(1.05, 1))
plt.tight_layout()
plt.show()
'''

#####1indvs2ind
def plotdesnity1vs2():
    p2,df2= load('final','ACDC_X2',parlist)
    parlist1=parlist.copy()
    del parlist1[7:9]
    p1,df1= load('final','ACDC_X21ind',parlist1)

    namelist=[]
    for i,par in enumerate(parlist1):
        namelist.append(par['name'])

    parl = np.append(namelist,'dist')
    index=1
    for i,name in enumerate(parl):
        plt.subplot(4,4,index)
        plt.tight_layout()

        sns.kdeplot(df1[name],bw_adjust=.8,label='X_1ind')
        sns.kdeplot(df2[name],bw_adjust=.8,label='X_2ind')
            #plt.ylim(0,1)
        if i < (len(parl)-2):
            plt.xlim((parlist1[i]['lower_limit'],parlist1[i]['upper_limit']))
        index=index+1
        if index==5:
            plt.legend(bbox_to_anchor=(1.05, 1))

        #sns.kdeplot(df['K_XZ'])
    plt.savefig("1vs2ind"+str(n[0])+"_compareplot.pdf", bbox_inches='tight')
    #plt.show()

plotdesnity1vs2()

def ind1vs2indmeanandmode():
    p2,df2= load('final','ACDC_X',parlist)
    df2=df2.drop(columns=['K_ARAY', 'n_ARAY'])
    mean_df2=np.mean(df2)
    sd_df2=np.std(df2)
    mode_df2=stats.mode(df2)[0][0]
    parlist1=parlist.copy()
    del parlist1[7:9]
    p1,df1= load('12','ACDC_1ind',parlist1)
    mean_df1=np.mean(df1)
    sd_df1=np.std(df1)
    mode_df1=stats.mode(df1)[0][0]

    namelist=[]
    for i,par in enumerate(parlist1):
        namelist.append(par['name'])
    parl = np.append(namelist,'dist')
    # set width of bars
    barWidth = 0.30
    # Set position of bar on X axis
    r1 = np.arange(len(parl))
    plt.bar((r1+barWidth*0),mean_df1,yerr=sd_df1, capsize=2,width=barWidth, label="1ind")
    plt.bar((r1+barWidth*1),mean_df2,yerr=sd_df2, capsize=2,width=barWidth, label="2ind")
    plt.xlabel('par', fontweight='bold')
    plt.xticks([r + barWidth for r in range(len(parl))], parl)
    plt.legend()
    plt.show()
    plt.bar((r1+barWidth*0),mode_df1,width=barWidth, label="1ind")
    plt.bar((r1+barWidth*1),mode_df2,width=barWidth, label="2ind")
    plt.xlabel('par', fontweight='bold')
    plt.xticks([r + barWidth for r in range(len(parl))], parl)
    plt.legend()
    plt.show()

def calculateSS(ARA,parUsed):
    #sort ss according to their stabilitz
    #create stability list of shape : arabinose x steady x X,Y,Z
    unstable=np.zeros((len(ARA),3,3))
    stable=np.zeros((len(ARA),3,3))
    oscillation=np.zeros((len(ARA),3,3))
    unstable[:]=np.nan
    stable[:]=np.nan
    oscillation[:]=np.nan

    for ai,a in enumerate(ARA):
        ss=meq.findss(a,parUsed)
        if len(ss) > 3:
            print("error: more than 3 steadystates")
        else:
            d = b = c=0 # can replace a,b,c by si, but allow to have osccilation on the same level
            for si,s in enumerate(ss):
                e=meq.stability(a,parUsed,[s])[0][0]
                if all(e<0):
                        stable[ai][d]=s
                        d+=1
                if any(e>0):
                    pos=e[e>0]
                    if len(pos)==2:
                        if pos[0]-pos[1] == 0:
                            oscillation[ai][b]=s
                            b+=1
                        else:
                            unstable[ai][c]=s
                            c+=1
                    else:
                        unstable[ai][c]=s
                        c+=1
    return unstable,stable,oscillation

#chose parameter
def bifurcation(parUsed=None):
    p,df= load('final','ACDC_X2',parlist)
    #parUsed=par0
    if parUsed == None:
        parUsed=p[0]
    ARA=np.logspace(-4.5,-2.,20,base=10)
    ss=meq.findss(ARA[0],parUsed)[0]
    #print(ss)
    init=[ss[0],ss[1],ss[2]]
    X,Y,Z=meq.model(ARA,parUsed,totaltime=100,init=init)
    df_X=pd.DataFrame(X[500:],columns=ARA)
    sns.heatmap(df_X, cmap="Reds", norm=LogNorm())
    plt.show()

    xss,yss,zss = calculateSScurve(ARA,parUsed)

    maxX=[]
    minX=[]
    maxY=[]
    minY=[]
    maxZ=[]
    minZ=[]
   # X,Y,Z=meq.model(ARA,parUsed,totaltime=400)
    delta=10e-5
    for i in np.arange(0,len(ARA)):
        min_x=[np.nan,np.nan,np.nan]
        max_x=[np.nan,np.nan,np.nan]
        ss=meq.findss(ARA[i],parUsed)
        for si,s in enumerate(ss):
            init=[s[0]+delta,s[1]+delta,s[2]+delta]
            X,Y,Z=meq.model(ARA,parUsed,totaltime=100,init=init)
           # print(max(X[200:,i]))

            max_x[si]=max(X[200:,i])
            min_x[si]=min(X[200:,i])

        maxX.append(max_x)
        minX.append(min_x)

       # minX.append(min(X[200:,i]))
        maxY.append(max(Y[200:,i]))
        minY.append(min(Y[200:,i]))
        maxZ.append(max(Z[200:,i]))
        minZ.append(min(Z[200:,i]))
    plt.subplot(3,1,1)
    plt.plot(ARA,xss,'--o')
    plt.plot(ARA,maxX,'-b')
    plt.plot(ARA,minX,'-g')
    #plt.fill_between(ARA,maxX,minX,alpha=0.2,facecolor='red')
    plt.yscale("log")
    plt.xscale("log")
    plt.subplot(3,1,2)
    plt.plot(ARA,yss,'--b')
   # plt.plot(ARA,maxY,'-b')
   # plt.plot(ARA,minY,'-b')
   # plt.fill_between(ARA,maxY,minY,alpha=0.2,facecolor='blue')
    plt.yscale("log")
    plt.xscale("log")
    plt.subplot(3,1,3)
    plt.plot(ARA,zss,'--g')
  #  plt.plot(ARA,maxZ,'-g')
  #  plt.plot(ARA,minZ,'-g')
  #  plt.fill_between(ARA,maxZ,minZ,alpha=0.2,facecolor='green')
    plt.yscale("log")
    plt.xscale("log")
    plt.show()

def getlimitcycle(ARA,ssl,par,tt=500):
    M=np.ones((len(ARA),3,3))*np.nan
    m=np.ones((len(ARA),3,3))*np.nan
    delta=10e-5
    transient=500
    for ai,a in enumerate(ARA):
            ss=ssl[ai]
            for si,s in enumerate(ss):
                if any(np.isnan(s)) == False:
                    init=[s[0]+delta,s[1]+delta,s[2]+delta]
                    X,Y,Z=meq.model([a],par,totaltime=tt,init=init)
                    M[ai,si,0]=max(X[transient:])
                    M[ai,si,1]=max(Y[transient:])
                    M[ai,si,2]=max(Z[transient:])
                    m[ai,si,0]=min(X[transient:])
                    m[ai,si,1]=min(Y[transient:])
                    m[ai,si,2]=min(Z[transient:])

                    max_list=argrelextrema(X[transient:], np.greater)
                    maxValues=X[transient:][max_list]
                    min_list=argrelextrema(X[transient:], np.less)
                    minValues=X[transient:][min_list]

                    maximaStability = abs(maxValues[-2]-minValues[-2])-(maxValues[-3]-minValues[-3])
                    if maximaStability > 0.01:
                        print("limit cycle not achieved for ARA["+str(ai)+"]:" + str(a) + " at st.s:"+ str(s))

    return M,m

def bifurcation_plot(n,filename):
    p,df= load(n,filename,parlist)
    ARA=np.logspace(-4.5,-2.,200,base=10)
    un,st,osc=calculateSS(ARA,p[1])
    M,m=getlimitcycle(ARA,osc,p[1],tt=500)
    for i,col in enumerate(['r','b','g']):
        plt.subplot(3,1,i+1)
        plt.plot(ARA,un[:,:,i],'--'+col)
        plt.plot(ARA,st[:,:,i],'-'+col)
        plt.plot(ARA,osc[:,:,i],'--'+col)
        plt.fill_between(ARA,M[:,0,i],m[:,0,i],alpha=0.2,facecolor=col)
        plt.fill_between(ARA,M[:,1,i],m[:,1,i],alpha=0.2,facecolor=col)
        plt.fill_between(ARA,M[:,2,i],m[:,2,i],alpha=0.2,facecolor=col)
        plt.yscale("log")
        plt.xscale("log")
    plt.show()

#bifurcation(p[1])


from pygame import *
from random import randrange
from math import *

from Pong.GameStats import GameStats
from Pong.Player.Goal import Goal
from Pong.Player.PlayerRacket import PlayerRacket

class Ball:

    MAX_SPEED_Y = 12
    SPEED_X = 6
    COLOR = (int(255), int(255), int(255))
    RADIUS: int = 10
    WIN_SCORE = 10

    def __init__(self, players):
        self.velocity = (Ball.SPEED_X, randrange(-Ball.MAX_SPEED_Y, Ball.MAX_SPEED_Y))
        self.pos = (int(GameStats.width/2), int(GameStats.height/2))
        self.players = players

    def update_move(self):
        # if there is collision
        self.pos = (self.velocity[0] + self.pos[0], self.pos[1] + self.velocity[1])
        self.collision_update()
        if self.pos[0] < -5 or self.pos[0] > 640:
            self.pos = (320, 320)
            self.velocity = (Ball.SPEED_X, randrange(-Ball.MAX_SPEED_Y, Ball.MAX_SPEED_Y))
        elif self.pos[1] < 0 or self.pos[1] > GameStats.height:
            self.velocity = (self.velocity[0], -self.velocity[1])

    def draw(self, surface):
        self.update_move()
        draw.circle(surface, Ball.COLOR, self.pos, Ball.RADIUS)

    def collision_update(self):
        col_pos = (0, 0)
        col_body = None
        collision = False

        for p in [self.players[0].racket, self.players[1].racket, self.players[0].goal, self.players[1].goal]:
            for point in ((self.pos[0] + Ball.RADIUS*cos(theta*0.01), self.pos[1] + Ball.RADIUS*sin(theta*0.01))
                          for theta in range(0, int(pi*2*100))):
                if p[0] < point[0] < p[0] + p[2] and \
                        p[1] < point[1] < p[1] + p[3]:
                    col_pos = point
                    col_body = p
                    collision = True
                    break
            if collision:
                break
        if collision:
            if type(col_body) is PlayerRacket:
                self.velocity = (-self.velocity[0], int((col_pos[1] - col_body[1] -
                                                        col_body[3]/2)/col_body[3]*Ball.MAX_SPEED_Y*2))
            elif type(col_body) is Goal:
                if self.players[0].goal == col_body:
                    if self.players[0].score() == self.WIN_SCORE:
                        self.players[0].reset()
                        self.players[1].reset()

                if self.players[1].goal == col_body:
                    if self.players[1].score() == self.WIN_SCORE:
                        self.players[0].reset()
                        self.players[1].reset()
                self.pos = (GameStats.width//2, GameStats.height//2)
                self.velocity = ((Ball.SPEED_X * ((-1) ** randrange(2))), randrange(-Ball.MAX_SPEED_Y, Ball.MAX_SPEED_Y))

    def __getitem__(self, key):
        return self.pos[key]


'''----------------------------------------------------------------------------

engine.py

----------------------------------------------------------------------------'''

from engine.ssc.image_ini import *
import numpy as np
#import sunpy.instr.aia

def standard_multitype_ini(observations):
    '''Standard initialization for different kind of observation. The
    initialization contains ratiation, limb darkening correction, Bz
    estimation and limb out region remove.

    Parameter
    ---------
        observations - Sunpy map object, it can contain multiple images.

    Return
    ------
        observations - Sunpy map object with modified data'''

    # Create a new list for the initialized observations
    initialized_observations = []

    for obs in observations:

        if obs.detector == 'HMI':
            # Replace np.nan-s with zero for rotating
            obs._data = np.nan_to_num(obs.data)

            # Rotate the observations
            obs = obs.rotate()

            # Limb darkening correction, only HIM white lighe image
            if obs.measurement == 'continuum':
                obs = dark_limb.limb_darkening_correct(obs, limb_cut=0.99)

            # Longitudinal magnetic field to Bz estimation
            if obs.measurement == 'magnetogram':
                obs = blbz.LOS2Bz(obs)

            # Cut the limb and replace outlimb region with np.nan
            obs = cut.solar_limb(obs)

        #if obs.detector == 'AIA':
            # Processes a level 1 AIAMap into a level 1.5 AIAMap
            #obs = sunpy.instr.aia.aiaprep(obs)

        # Append the new maps
        initialized_observations.append(obs)

    # Delete raw observations
    del observations

    return initialized_observations


# -*- coding: utf-8 -*-
"""
Created on Fri Oct 28 12:54:38 2016

@author: Kylin
"""
import math
import quyu
import erfenbijin
import pylab as pl

a     = 200
Rx    = 10
Ry    = 20
V0    = 100
theta = math.pi/5
dt    = 0.1
Vx    = V0*math.cos(theta)
Vy    = V0*math.sin(theta)
R_x   = []
V_x   = []
i     = 0
while 1 :
    Rx=Rx+Vx*dt
    Ry=Ry+Vy*dt
    if Ry*(Ry-Vy*dt)<0:
        k=(Ry-0)/(0-(Ry-Vy*dt))
        x0=((1+k)*Rx-k*Vx*dt)/(1+k)
        R_x+=[x0]
        V_x+=[Vx]
    if quyu.inZhengfangxing(Rx,Ry,a)==1\
    or quyu.inZhengfangxing(Rx,Ry,a)==0:
        continue
    if quyu.inZhengfangxing(Rx,Ry,a)==-1:
        x1=Rx-Vx*dt
        y1=Ry-Vy*dt
        x2=Rx
        y2=Ry
        t=erfenbijin.Zhengfangxing_erFenbijin(x1,y1,x2,y2,0,a)
        Rx=t[0]
        Ry=t[1]
        #continue
    if quyu.inZhengfangxing(Rx,Ry,a)==0:
        if (Rx== a or Rx==-a) and Ry>-a and Ry<a:
            Vx=-Vx
            Vy= Vy
        if (Ry== a or Ry==-a) and Rx>-a and Rx<a:
            Vx= Vx
            Vy=-Vy
        if (Rx== a and Ry== a)or(Rx==-a and Ry==a)or(Rx==a and Ry==-a)or(Rx==-a and Ry==-a):
            Vx=-Vx
            Vy=-Vy
    i+=1
    print(i)
    if i>10000:
        break
pl.plot(R_x, V_x,"o",label="Vx-Rx")
pl.title(u"正方形".encode("gb2312"))
pl.xlabel('Rx')
pl.ylabel('Vx')
pl.legend()
pl.show()


"""
文件名: Code/Chapter05/C01_ConfigManage/E02_Config.py
创建时间: 2023/2/26 3:47 下午
作 者: @空字符
公众号: @月来客栈
知 乎: @月来客栈 https://www.zhihu.com/people/the_lastest
"""

import os

class ModelConfig(object):
    def __init__(self,
                 train_file_path=os.path.join('data', 'train.txt'),
                 val_file_path=os.path.join('data', 'val.txt'),
                 test_file_path=os.path.join('data', 'test.txt'),
                 split_sep='_!_',
                 is_sample_shuffle=True,
                 batch_size=16,
                 learning_rate=3.5e-5,
                 max_sen_len=None,
                 num_labels=3,
                 epochs=5):
        self.train_file_path = train_file_path
        self.val_file_path = val_file_path
        self.test_file_path = test_file_path
        self.split_sep = split_sep
        self.is_sample_shuffle = is_sample_shuffle
        self.batch_size = batch_size
        self.learning_rate = learning_rate
        self.max_sen_len = max_sen_len
        self.num_labels = num_labels
        self.epochs = epochs

#
def train(config):
    dataset = get_dataset(config)
    model = get_mode(config)

if __name__ == '__main__':
    config = ModelConfig(epochs=10)
    print(f"epochs = {config.epochs}")
    # train(config)


import const
import sys, os
import string
import random

QUESTION_TOOL='What are the tools used in the attack?'
QUESTION_GROUP='Who is the attack group?'

INPUT_FILE='input/sample_attack_report_raw.txt'
TRAIN_RATE=0.8
VUL_RATE=0.1
LABEL_TRAIN='train'
LABEL_VAL='dev'
LABEL_TEST='test'

SENTENSE_DELIMETER=". "
WORD_DELIMETER=" "
LAVEL_GROUP='B-AT'
LAVEL_I_GROUP='I-AT'
LAVEL_TOOL='B-TL'
LAVEL_I_TOOL='I-TL'
LAVEL_SEC='B-SC'
LAVEL_I_SEC='I-SC'
LAVEL_COM='B-CM'
LAVEL_I_COM='I-CM'
LAVEL_OTHER='O'
DATASET_DELIMETER="\t"
TRAIN_FILE='train.txt'
VAL_FILE='dev.txt'
TEST_FILE='test.txt'
MAX_WORD_NUM=200
MAX_WORD=1000
NUM_SENTENSE_PER_ROW=100
LONG_SENTENSE='long.txt'
O_RATE=1
EXCLUSIVE_LIST=['at']
LEN_RANDOM=10
alldataset={}

def get_tools():
    tools=[]
    with open(const.OUT_TOOL_FILE, 'r') as file:
        for row in file:
            tool = row.replace(const.NEWLINE, "")
            #tool = tool.lower()
            tools.append(tool)
        return tools

def get_groups():
    groups=[]
    with open(const.OUT_GROUP_FILE, 'r') as file:
        for row in file:
            group = row.replace(const.NEWLINE, "")
            #group=group.lower()
            groups.append(group)
        return groups

def get_sectors():
    sectors=[]
    with open(const.OUT_SECTOR_FILE, 'r') as file:
        for row in file:
            sector = row.replace(const.NEWLINE, "")
            #sector=sector.lower()
            sectors.append(sector)
        return sectors

def get_companies():
    companies=[]
    with open(const.OUT_COMPANY_FILE, 'r') as file:
        for row in file:
            company = row.replace(const.NEWLINE, "")
            #company=company.lower()
            companies.append(company)
        return companies

def random_str(word):
    dat = string.digits + string.ascii_lowercase + string.ascii_uppercase
    return ''.join([random.choice(dat) for i in range(len(word))]).lower()

def get_random_TOOL(start,end):
    index=random.randint(start,end)
    tool=tools[index]
    name=tool.split(" ")[0]
    return name

def get_random_TA(start,end):
    index=random.randint(start,end)
    ta_name=groups[index]
    name = ta_name.split(" ")[0]
    return name

def create_dataset(mode,num_dataset, start_a, end_a, start_t, end_t):

    cnt=0

    data=[]
    data_O=[]
    data_tag = []

    if mode == LABEL_TRAIN:
        data=lines[:num_train-1]
    elif mode==LABEL_VAL:
        data=lines[num_train:num_train+num_val]

    else:
        data = lines[num_train+num_val:]

    for row in data:
        print("cnt: "+str(cnt))
        if cnt>num_dataset:
            print("Exceed "+str(num_data))
            return

        sentenses = row.split(SENTENSE_DELIMETER)
        #print(str(len(sentenses)))
        for sentense in sentenses:
            words= sentense.split(WORD_DELIMETER)
            if len(words) >=MAX_WORD_NUM:
                # with open(LONG_SENTENSE, "a", encoding='utf8') as out_sentense:
                #     out_sentense.write(sentense + const.NEWLINE)
                continue

            len_word=0
            for word in words:
                len_word=len_word+len(word)
            if len_word >= MAX_WORD:
                continue

            prev=''
            prev_org=''
            dataset=[]
            index=0
            for word in words:
                lavel = LAVEL_OTHER
                word=word.strip()
                tmp_word = word

                # groups
                if tmp_word in groups:
                    lavel=LAVEL_GROUP

                elif prev+WORD_DELIMETER+tmp_word in groups:
                    lavel = LAVEL_I_GROUP
                    prev_org = get_random_TA(start_a, end_a)
                    dataset[index-1]=prev_org + DATASET_DELIMETER + LAVEL_GROUP + const.NEWLINE

                # tools

                elif tmp_word in tools and tmp_word.lower() not in EXCLUSIVE_LIST:
                    lavel=LAVEL_TOOL

                elif prev + WORD_DELIMETER + tmp_word in tools:
                    lavel = LAVEL_I_TOOL
                    prev_org = get_random_TOOL(start_t,end_t)
                    dataset[index - 1] = prev_org + DATASET_DELIMETER + LAVEL_TOOL + const.NEWLINE

                # # sectors
                # elif tmp_word in sectors:
                #     lavel = LAVEL_SEC
                #
                # elif prev + WORD_DELIMETER + tmp_word in sectors:
                #     lavel = LAVEL_I_SEC
                #     dataset[index - 1] = prev_org + DATASET_DELIMETER + LAVEL_SEC + const.NEWLINE
                #
                # # companies
                # elif tmp_word in companies:
                #     lavel = LAVEL_COM
                #
                # elif prev + WORD_DELIMETER + tmp_word in companies:
                #     lavel = LAVEL_I_COM
                #     dataset[index - 1] = prev_org + DATASET_DELIMETER + LAVEL_COM + const.NEWLINE

                if lavel ==LAVEL_GROUP or lavel==LAVEL_I_GROUP:
                    word=get_random_TA(start_a, end_a)
                    word=word

                elif lavel ==LAVEL_TOOL or lavel==LAVEL_I_TOOL:
                    word=get_random_TOOL(start_t,end_t)
                    word = word

                dataset.append(word + DATASET_DELIMETER + lavel + const.NEWLINE)
                prev=tmp_word
                prev_org=word
                index=index+1

            num_data=0
            for item in dataset:
                label=item.split(DATASET_DELIMETER)[1].strip()
                if label!=LAVEL_OTHER:
                    num_data=num_data+1

            if num_data == 0:
                data_O.append(dataset)

            else:
                data_tag.append(dataset)

        cnt = cnt + 1

    O_num = len(data_O)
    max_O_num = int(O_num* O_RATE)

    alldataset[mode]=data_tag+data_O[:max_O_num]

    return(mode)

with open(INPUT_FILE, 'r') as file:
    lines = file.readlines()

context=len(lines)
print("total context:" +str(context))

if len(sys.argv)>1:
    context  = int(sys.argv[1])

num_train=round(context*TRAIN_RATE)
num_val=round(context*VUL_RATE)
num_test=context-num_train-num_val

print("num_train:" +str(num_train))
print("num_val:" +str(num_val))
print("num_test:" +str(num_test))

tools=get_tools()
groups=get_groups()
# sectors=get_sectors()
# companies=get_companies()

train_ta_end=round(len(groups)*TRAIN_RATE)
dev_ta_end=train_ta_end+round(len(groups)*VUL_RATE)
test_ta_end=len(groups)-1

train_tl_end=round(len(tools)*TRAIN_RATE)
dev_tl_end=train_tl_end+round(len(tools)*VUL_RATE)
test_tl_end=len(tools)-1

if os.path.exists(TRAIN_FILE):
    os.remove(TRAIN_FILE)

if os.path.exists(VAL_FILE):
    os.remove(VAL_FILE)

if os.path.exists(TEST_FILE):
    os.remove(TEST_FILE)

if os.path.exists(LONG_SENTENSE):
    os.remove(LONG_SENTENSE)

create_dataset(LABEL_TRAIN, num_train,0,train_ta_end,0,train_tl_end)
create_dataset(LABEL_VAL, num_val,train_ta_end+1,dev_ta_end,train_tl_end+1,dev_tl_end)
create_dataset(LABEL_TEST, num_test,dev_ta_end+1,test_ta_end,dev_tl_end+1,test_tl_end)

with open(LABEL_TRAIN + '.txt', "a", encoding='utf8') as out:
    for dataset in alldataset[LABEL_TRAIN]:
        out.writelines(dataset)
        out.write('.' + DATASET_DELIMETER + LAVEL_OTHER + const.NEWLINE + const.NEWLINE)

with open(LABEL_VAL + '.txt', "a", encoding='utf8') as out:
    for dataset in alldataset[LABEL_VAL]:
        out.writelines(dataset)
        out.write('.' + DATASET_DELIMETER + LAVEL_OTHER + const.NEWLINE + const.NEWLINE)

with open(LABEL_TEST + '.txt', "a", encoding='utf8') as out:
    for dataset in alldataset[LABEL_TEST]:
        out.writelines(dataset)
        out.write('.' + DATASET_DELIMETER + LAVEL_OTHER + const.NEWLINE + const.NEWLINE)


from random import randint
y=int(randint(1,10))
for i in range(3):
    x = int(input("猜数字:\n"))
    if x >y:
        print("大了")
    elif x<y:
        print("小了")
    else:
        print("猜对了")
        break
print("Game over!")


#El bloque else justo después de for / while se ejecuta solo cuando el ciclo
#NO termina con una declaración de interrupción.

"""for i in range(1,5):
    print (i)
else :
    print("Sin descanso/ se ejecuta porque no hay break")"""

# Program to check if an array consists
# of even number

"""def evenumbers (lista) :
    for l in lista :
        if l % 2 == 0 :
            print ("Hay numeros pares en la lista")
            break
    else :
        print ("No hay numeros pares en la lista")

print ("Lista 1 :" )
evenumbers([5,3,6])

print ("Lista 2 :" )
evenumbers([1,3,5])"""

count = 4
while count < 1 :
    count += 1
    print (count)

else:
    print ("no break")


'''
Problem Statement
Your company has a big conference coming up and needs to book conference rooms in a convention center. To help the company save budget, we want to book as few conference rooms as possible given a list of meeting schedules that contains only the starting and ending time of each meeting. Write a program that helps figure out the minumum number of conference rooms needed.

Example:

[(2,7)] -> Output: 1
[(0,30),(5,10),(15,20) (21 22) (21 28) ] -> Explanation: Room1: (0,30) Room2: (5,10),(15,20) -> Output: 2
(0, 30), (0, 10), (5,15), (11, 20), (17, 25), (21,30)

examples
(0,30),
(5,10), (15,20), (21 22)
(21 28)

(0, 30), (0, 10), (5,15), (11, 20), (17, 25), (21,30)
assumptions
approaches
1)
(0,30),
(5,22),
(21 28)

2)
(0, 30), (0, 10), (5,15), (11, 20), (17, 25), (21,30)
0         1         2        3        4         5
0 30
count: 1

create a res array
for any new interval, look in res for a place where int has no intersection. this space defines a room!

tradeoffs
this appears to be the only way
'''
from typing import List, Tuple
def roomcount(times: List[Tuple[int, int]]) -> int:
    '''
        s1------e1
                s2-------e2
    '''
    def intersects(start1, end1, start2, end2):
        return min(end1, end2) > max(start1, start2)

    def no_intersects(lis):
        for int_ in lis:
            if intersects(*int_, start, end): # return true if they touch?
                return False
        return True

    rooms = []
    for start, end in times:
        for lis in rooms:
            if no_intersects(lis):
                lis.append((start, end))
                break
        else:
            rooms.append([(start, end)])
    return len(rooms)

ints = [(2,7)] # -> Output: 1
print(roomcount(ints))
ints = [(0,30),(5,10),(15,20), (21, 22), (21, 28) ] #3
print(roomcount(ints))
ints = [(0,30),(5,10),(15,20),(21, 22), (22, 28) ] #2
print(roomcount(ints))


from typing import Dict, List, Optional, Tuple, Union

import numpy as np
from rl_nav import constants
from rl_nav.environments import wrapper

try:
    import cv2
    import matplotlib
    from matplotlib import cm
    from matplotlib import pyplot as plt
    from mpl_toolkits.axes_grid1 import make_axes_locatable
except ModuleNotFoundError:
    raise AssertionError(
        "To use visualisation wrapper, further package requirements "
        "need to be satisfied. Please consult README."
    )

class VisualisationEnv(wrapper.Wrapper):

    COLORMAP = cm.get_cmap("plasma")
    NORMALISE = False

    def __init__(self, env):
        super().__init__(env=env)

    def render(
        self,
        save_path: Optional[str] = None,
        dpi: Optional[int] = 60,
        format: str = "state",
    ) -> None:
        """Method to render environment.

        Args:
            save_path: optional path to which to save image.
            dpi: optional pixel.
            format: state of environment to render.
        """
        if format == constants.STATE:
            assert (
                self._env.active
            ), "To render map with state, environment must be active."
            "call reset_environment() to reset environment and make it active."
            "Else render stationary environment skeleton using format='stationary'"
        if save_path:
            fig = plt.figure()
            plt.imshow(
                self._env._env_skeleton(
                    rewards=format,
                    agent=format,
                ),
                origin="lower",
            )
            fig.savefig(save_path, dpi=dpi)
        else:
            plt.imshow(
                self._env._env_skeleton(
                    rewards=format,
                    agent=format,
                ),
                origin="lower",
            )

    def visualise_episode_history(
        self, save_path: str, history: Union[str, List[np.ndarray]] = "train"
    ) -> None:
        """Produce video of episode history.

        Args:
            save_path: name of file to be saved.
            history: "train", "test" to plot train or test history,
            else provide an independent history.
        """
        if isinstance(history, str):
            if history == constants.TRAIN:
                history = self._env.train_episode_history
            elif history == constants.TEST:
                history = self._env.test_episode_history
            elif history == constants.TRAIN_PARTIAL:
                history = self._env.train_episode_partial_history
            elif history == constants.TEST_PARTIAL:
                history = self._env.test_episode_partial_history

        SCALING = 20
        FPS = 30

        map_shape = history[0].shape
        frameSize = (SCALING * map_shape[1], SCALING * map_shape[0])

        out = cv2.VideoWriter(
            filename=save_path,
            fourcc=cv2.VideoWriter_fourcc("m", "p", "4", "v"),
            fps=FPS,
            frameSize=frameSize,
        )

        for frame in history:
            bgr_frame = frame[..., ::-1].copy()
            flipped_frame = np.flip(bgr_frame, 0)
            scaled_up_frame = np.kron(flipped_frame, np.ones((SCALING, SCALING, 1)))
            out.write((scaled_up_frame * 255).astype(np.uint8))

        out.release()

    def _plot_normalised_heatmap_over_env(
        self, heatmap: Dict[Tuple[int, int], float], save_name: str
    ):
        split_save_name = save_name.split(".pdf")[0]
        save_name = f"{split_save_name}_normalised.pdf"
        environment_map = self._env._env_skeleton(
            rewards=None,
            agent=None,
        )

        all_values = list(heatmap.values())
        current_max_value = np.max(all_values)
        current_min_value = np.min(all_values)

        for position, value in heatmap.items():
            # remove alpha from rgba in colormap return
            # normalise value for color mapping
            environment_map[position[::-1]] = self.COLORMAP(
                (value - current_min_value) / (current_max_value - current_min_value)
            )[:-1]

        fig = plt.figure()
        plt.imshow(environment_map, origin="lower", cmap=self.COLORMAP)
        plt.colorbar()
        fig.savefig(save_name, dpi=60)
        plt.close()

    def _plot_unnormalised_heatmap_over_env(
        self, heatmap: Dict[Tuple[int, int], float], save_name: str
    ):
        environment_map = self._env._env_skeleton(
            rewards=None,
            agent=None,
        )

        for position, value in heatmap.items():
            # remove alpha from rgba in colormap return
            environment_map[position[::-1]] = self.COLORMAP(value)[:-1]

        fig = plt.figure()
        plt.imshow(environment_map, origin="lower", cmap=self.COLORMAP)
        plt.colorbar()
        fig.savefig(save_name, dpi=60)
        plt.close()

    def plot_heatmap_over_env(
        self,
        heatmap: Dict[Tuple[int, int], float],
        save_name: str,
    ) -> None:
        """plot quantities over top of environmen (e.g. value function)

        Args:
            heatmap: data to plot; dictionary of states (keys) and quantities (values).
            fig: figure on which to plot.
            ax: axis on which to plot.
            save_name: path to which to save plot.
        """
        self._plot_unnormalised_heatmap_over_env(heatmap=heatmap, save_name=save_name)
        self._plot_normalised_heatmap_over_env(heatmap=heatmap, save_name=save_name)

    def plot_numbered_values_over_env(
        self, values: Dict[Tuple[int], np.ndarray], save_name: str
    ):
        fig = plt.figure()
        environment_map = self._env._env_skeleton(
            rewards=None,
            agent=None,
        )
        plt.imshow(environment_map, origin="lower", cmap=self.COLORMAP)
        all_states = list(values.keys())
        for state, action_values in values.items():
            for i, action_value in enumerate(action_values):
                if all_states[i] != state:
                    xytext = np.array(state) + 0.2 * (
                        np.array(all_states[i]) - np.array(state)
                    )
                    plt.annotate(
                        f"{i}: {round(action_value, 2)}",
                        xy=state,
                        xytext=xytext,
                        arrowprops={
                            "headlength": 2,
                            "headwidth": 2,
                            "width": 0.5,
                            "linewidth": 0.1,
                        },
                        color="y",
                        size=5,
                    )
                else:
                    plt.annotate(
                        i,
                        xy=state,
                        color="g",
                        size=5,
                    )
        fig.savefig(save_name, dpi=60)
        plt.close()


__author__ = "Vanessa Sochat, Alec Scott"
__copyright__ = "Copyright 2021-2023, Vanessa Sochat and Alec Scott"
__license__ = "Apache-2.0"

import json
import os
import re
import shlex
import subprocess

import pakages.builders.spack.cache as spack_cache
import pakages.client
import pakages.oras
import pakages.utils
from pakages.logger import logger

class SpackClient(pakages.client.PakagesClient):
    """
    Pakages has a main controller for interacting with pakages.
    """

    def parse_package_request(self, packages):
        """
        Parse the packages and repo (if any) from it.
        This is shared between install and build
        """
        # By defualt, assume not adding a repository
        repo = None

        if not isinstance(packages, list):
            packages = shlex.split(packages)

        # Case 1: we have an install directed at the present working directory
        if packages and packages[0] == ".":
            repo = os.getcwd()
            packages.pop(0)

        # If we have a path (akin to the first)
        if packages and os.path.exists(packages[0]):
            repo = packages.pop(0)

        # OR if we have a github URI TODO, can clone here
        if packages and re.search("(http|https)://github.com", packages[0]):
            repo = packages.pop(0)

        # If we don't have packages and we have a repo, derive from PWD
        if repo and not packages:
            for path in pakages.utils.recursive_find(repo, "package.py"):
                packages.append(os.path.basename(os.path.dirname(path)))

        # Finally, add the repository
        if repo:
            self.add_repository(repo)

        return packages

    def list_installed(self):
        """
        List installed packages
        """
        command = ["spack", "find"]
        for line in pakages.utils.stream_command(command):
            print(line.strip("\n"))

        command = ["spack", "find", "--json"]
        result = pakages.utils.run_command(command)
        return json.loads(result["message"])

    def build(self, packages, cache_dir=None, key=None, **kwargs):
        """
        Build a package into a cache
        """
        packages = self.parse_packages(packages)

        # Prepare a cache directory
        cache = spack_cache.BuildCache(
            spec_name=packages,
            cache_dir=cache_dir or self.settings.cache_dir,
            username=self.settings.username,
            email=self.settings.email,
            settings=self.settings,
        )

        # Install all packages
        self._install(packages)
        cache.create(packages, key=key)

        # Push function is on cache, if desired
        return cache

    def parse_packages(self, packages):
        """
        Helper function to ensure we return consistent names.
        """
        packages = self.parse_package_request(packages)
        if isinstance(packages, list):
            packages = packages[0]
        if " " in packages:
            logger.exit("We currently only support one package for build.")
        logger.info(f"Preparing package {packages}")
        return packages

    def add_repository(self, path):
        """
        Add a repository.

        Given a path that exists, add the repository to the
        underlying spack. If you need to add a GitHub uri, create a
        pakages.repo.PakRepo first.
        """
        try:
            command = ["spack", "repo", "add", path]
            for line in pakages.utils.stream_command(command):
                logger.info(line.strip("\n"))
        except subprocess.CalledProcessError as e:
            if "Repository is already registered" in e.output:
                pass
            else:
                raise e

    def download_cache(self, target, download_dir=None):
        """
        Download a target to a cache download directory
        """
        download_dir = download_dir or pakages.utils.get_tmpdir()
        reg = pakages.oras.get_oras_client()

        # This will error if not successful, result is a list of files
        reg.pull(target=target, outdir=download_dir)
        return download_dir

    def install(self, packages, **kwargs):
        """
        Install one or more packages.
        """
        packages = self.parse_packages(packages)
        use_cache = kwargs.get("use_cache", False)
        if use_cache:
            cache_dir = self.download_cache(use_cache)
            cache = spack_cache.BuildCache(
                packages, cache_dir=cache_dir, settings=self.settings
            )

            # Cache is named after target, this is a filesystem mirror
            cache.add_as_mirror(re.sub("(-|:|/)", "-", use_cache))

        # Prepare install command with or without cache
        command = ["spack", "install"]
        if use_cache:
            command.append("--use-cache")
        if isinstance(packages, list):
            command.append(" ".join(packages))
        else:
            command.append(packages)

        # Install packages using system spack - we aren't responsible for this working
        for line in pakages.utils.stream_command(command):
            logger.info(line.strip("\n"))

    def _install(self, packages):
        """
        Install one or more packages.

        This eventually needs to take into account using the GitHub packages bulid cache
        """
        # Install packages using system spack - we aren't responsible for this working
        for line in pakages.utils.stream_command(["spack", "install", packages]):
            logger.info(line.strip("\n"))

    def uninstall(self, packages):
        """
        Uninstall a spack package
        """
        for line in pakages.utils.stream_command(["spack", "uninstall", packages]):
            logger.info(line.strip("\n"))


import sys
import oneflow as flow
import oneflow.typing as tp
import argparse
import numpy as np
import os
import shutil
import json
from typing import Tuple

from textcnn import TextCNN

sys.path.append("../..")
from text_classification.utils import pad_sequences, load_imdb_data

parser = argparse.ArgumentParser()
parser.add_argument('--ksize_list', type=str, default='2,3,4,5')
parser.add_argument('--n_filters', type=int, default=100)
parser.add_argument('--emb_dim', type=int, default=100)
parser.add_argument('--dropout', type=float, default=0.5)
parser.add_argument('--lr', type=float, default=1e-4)
parser.add_argument('--sequence_length', type=int, default=150)
parser.add_argument('--batch_size', type=int, default=32)
parser.add_argument('--model_load_dir', type=str, default='')
parser.add_argument('--model_save_every_n_iter', type=int, default=1000)
parser.add_argument('--n_steps', type=int, default=10000)
parser.add_argument('--n_epochs', type=int, default=15)
parser.add_argument('--model_save_dir', type=str, default='./best_model')

args = parser.parse_args()
assert ',' in args.ksize_list
args.ksize_list = [int(n) for n in args.ksize_list.split(',')]
args.emb_num = 50000
args.n_classes = 2

model = TextCNN(
    args.emb_num, args.emb_dim,
    ksize_list=args.ksize_list,
    n_filters_list=[args.n_filters] * len(args.ksize_list),
    n_classes=args.n_classes, dropout=args.dropout)

def get_train_config():
    config = flow.function_config()
    config.default_data_type(flow.float)
    return config

def get_eval_config():
    config = flow.function_config()
    config.default_data_type(flow.float)
    return config

@flow.global_function('train', get_train_config())
def train_job(text: tp.Numpy.Placeholder((args.batch_size, args.sequence_length), dtype=flow.int32),
              label: tp.Numpy.Placeholder((args.batch_size,), dtype=flow.int32)
              ) -> tp.Numpy:
    with flow.scope.placement("gpu", "0:0"):
        logits = model.get_logits(text, is_train=True)
        loss = flow.nn.sparse_softmax_cross_entropy_with_logits(label, logits, name="softmax_loss")
    lr_scheduler = flow.optimizer.PiecewiseConstantScheduler([], [args.lr])
    flow.optimizer.Adam(lr_scheduler).minimize(loss)
    return loss

@flow.global_function('predict', get_eval_config())
def eval_job(text: tp.Numpy.Placeholder((args.batch_size, args.sequence_length), dtype=flow.int32),
             label: tp.Numpy.Placeholder((args.batch_size,), dtype=flow.int32)
             ) -> Tuple[tp.Numpy, tp.Numpy]:
    with flow.scope.placement("gpu", "0:0"):
        logits = model.get_logits(text, is_train=False)
        loss = flow.nn.sparse_softmax_cross_entropy_with_logits(label, logits, name="softmax_loss")

    return label, logits

def suffle_batch(data, label, batch_size):
    permu = np.random.permutation(len(data))
    data, label = data[permu], label[permu]

    batch_n = len(data) // batch_size

    x_batch = np.array([data[i * batch_size:i * batch_size + batch_size] for i in range(batch_n)], dtype=np.int32)
    y_batch = np.array([label[i * batch_size:i * batch_size + batch_size] for i in range(batch_n)], dtype=np.int32)

    return x_batch, y_batch

def acc(labels, logits, g):
    predictions = np.argmax(logits, 1)
    right_count = np.sum(predictions == labels)
    g["total"] += labels.shape[0]
    g["correct"] += right_count

def train(checkpoint):
    path = '../imdb'
    (train_data, train_labels), (test_data, test_labels) = load_imdb_data(path)

    with open(os.path.join(path, 'word_index.json')) as f:
        word_index = json.load(f)
    word_index = {k: (v + 2) for k, v in word_index.items()}
    word_index["<PAD>"] = 0
    word_index["<UNK>"] = 1

    train_data = pad_sequences(train_data, value=word_index["<PAD>"], padding='post', maxlen=args.sequence_length)
    test_data = pad_sequences(test_data, value=word_index["<PAD>"], padding='post', maxlen=args.sequence_length)

    best_accuracy = 0.0
    best_epoch = 0
    for epoch in range(1, args.n_epochs + 1):
        print("[Epoch:{}]".format(epoch))
        data, label = suffle_batch(train_data, train_labels, args.batch_size)
        for i, (texts, labels) in enumerate(zip(data, label)):
            loss = train_job(texts, labels).mean()
            if i % 20 == 0:
                print(loss)

        data, label = suffle_batch(test_data, test_labels, args.batch_size)
        g = {"correct": 0, "total": 0}
        for i, (texts, labels) in enumerate(zip(data, label)):
            labels, logits = eval_job(texts, labels)
            acc(labels, logits, g)

        accuracy = g["correct"] * 100 / g["total"]
        print("[Epoch:{0:d} ] accuracy: {1:.1f}%".format(epoch, accuracy))
        if accuracy > best_accuracy:
            best_accuracy = accuracy
            best_epoch = epoch
            if not os.path.exists(args.model_save_dir):
                os.mkdir(args.model_save_dir)
            else:
                shutil.rmtree(args.model_save_dir)
                assert not os.path.exists(args.model_save_dir)
                os.mkdir(args.model_save_dir)
            print("Epoch:{} save best model.".format(best_epoch))
            checkpoint.save(args.model_save_dir)

    print("Epoch:{} get best accuracy:{}".format(best_epoch, best_accuracy))

if __name__ == '__main__':
    checkpoint = flow.train.CheckPoint()
    checkpoint.init()
    train(checkpoint)