"""Scraper for gif, html, jpeg, tif, pdf and wav files using JHove."""
from __future__ import unicode_literals

try:
    import lxml.etree
except ImportError:
    pass

from file_scraper.base import BaseScraper
from file_scraper.shell import Shell
from file_scraper.jhove.jhove_model import (JHoveAiffMeta, JHoveDngMeta,
                                            JHoveEpubMeta, JHoveGifMeta,
                                            JHoveHtmlMeta, JHoveJpegMeta,
                                            JHovePdfMeta, JHoveTiffMeta,
                                            JHoveUtf8Meta, JHoveWavMeta,
                                            get_field)

class JHoveScraperBase(BaseScraper):
    """Scraping methods for all specific JHove scrapers."""

    _supported_metadata = []
    _jhove_module = None
    _only_wellformed = True

    def __init__(self, filename, mimetype, version=None, params=None):
        """
        Initialize JHove base scarper.

        :filename: File path
        :mimetype: Predefined mimetype
        :version: Predefined file format version
        :params: Extra parameters needed for the scraper
        """
        self._report = None  # JHove report
        super(JHoveScraperBase, self).__init__(
            filename=filename, mimetype=mimetype, version=version,
            params=params)

    def scrape_file(self):
        """Run JHove command and store XML output to self.report."""
        exec_cmd = ["jhove", "-h", "XML", "-m",
                    self._jhove_module, self.filename]
        shell = Shell(exec_cmd)

        if shell.returncode != 0:
            self._errors.append("JHove returned invalid return code: %s\n%s" %
                                (shell.returncode, shell.stderr))
        self._report = lxml.etree.fromstring(shell.stdout_raw)

        status = get_field(self._report, "status")
        self._messages.append(status)
        if "Well-Formed and valid" not in status:
            self._errors.append("Validator returned error.")
            self._errors.append(shell.stdout)
            self._errors.append(shell.stderr)

        self.streams = list(self.iterate_models(
            well_formed=self.well_formed, report=self._report))

        self._check_supported(allow_unav_version=True,
                              allow_unap_version=True)

class JHoveGifScraper(JHoveScraperBase):
    """Variables for scraping gif files."""

    _jhove_module = "GIF-hul"
    _supported_metadata = [JHoveGifMeta]

class JHoveHtmlScraper(JHoveScraperBase):
    """Variables for scraping html files."""

    _jhove_module = "HTML-hul"
    _supported_metadata = [JHoveHtmlMeta]

    def scrape_file(self):
        """
        Scrape the file.

        Check the character encoding declaration additionally. JHove HTML
        module seems to check only the declaration or metadata elements from
        the (X)HTML file, and these are optional in practice. If these are
        missing, then we just need to rely on other scraper tools.
        """
        super(JHoveHtmlScraper, self).scrape_file()

        # self.streams is empty if MIME type is not supported.
        # We run _check_supported() in super() where this case
        # is handled by giving an error message.
        if not self.streams:
            return
        if not self._params.get("charset", None):
            self._errors.append("Character encoding not defined.")
            return
        encoding = self.streams[0].charset()
        if encoding is not None and \
                encoding.upper() != self._params["charset"]:
            self._errors.append(
                "Found encoding declaration %s from the file %s, but %s "
                "was expected." % (encoding, self.filename,
                                   self._params["charset"]))

class JHoveJpegScraper(JHoveScraperBase):
    """Variables for scraping jpeg files."""

    _jhove_module = "JPEG-hul"
    _supported_metadata = [JHoveJpegMeta]

class JHoveTiffScraper(JHoveScraperBase):
    """Variables for scraping tiff files."""

    _jhove_module = "TIFF-hul"
    _supported_metadata = [JHoveTiffMeta]

class JHovePdfScraper(JHoveScraperBase):
    """PDF scraper using JHove."""

    _jhove_module = "PDF-hul"
    _supported_metadata = [JHovePdfMeta]

    def scrape_file(self):
        """
        Scrape the file, and check and log PDF root version.

        JHove does not support PDF 1.7, but it can detect it. Thus if PDF 1.7
        is detected, any possible errors JHove might detect will be
        disregarded. Regardless of what version is detected, the PDF root
        version is logged in messages, which is useful for PDF-A file scraping.
        """

        super(JHovePdfScraper, self).scrape_file()

        if self.streams:
            mimetype = self.streams[0].mimetype()
            version = self.streams[0].version()

            if mimetype == "application/pdf":
                if version == "1.7":
                    self._errors = []
                    self._messages = ["JHove does not support PDF 1.7: "
                                      "All errors and messages ignored."]

                self._messages.append("PDF root version is {}".format(version))

class JHoveWavScraper(JHoveScraperBase):
    """Variables for scraping wav files."""

    _jhove_module = "WAVE-hul"
    _supported_metadata = [JHoveWavMeta]

    def scrape_file(self):
        """
        Scrape file.
        Add extra error message, if RF64 profile used.
        """
        super(JHoveWavScraper, self).scrape_file()
        if "RF64" in get_field(self._report, "profile"):
            self._errors.append("RF64 is not a supported format")

class JHoveUtf8Scraper(JHoveScraperBase):
    """
    Scraper for checking wether a text file is a valid UTF-8 file.

    We don't want to run this for all files, but just for UTF-8 text files
    separately. This must be run after actual scraping, since we have to know
    the charset of the file.
    """
    _jhove_module = "UTF8-hul"
    _supported_metadata = [JHoveUtf8Meta]

    def _check_supported(self, allow_unav_mime=False,
                         allow_unav_version=False,
                         allow_unap_version=False):
        """Do nothing: we dont care about the mimetype or version."""

    def iterate_models(self, **kwargs):
        """
        Iterate Scraper models and create streams.

        We need to override this since _supported attribute is empty and
        the scraper is run differenty.
        """
        yield self._supported_metadata[0](**kwargs)

class JHoveEpubScraper(JHoveScraperBase):
    """Variables for scraping EPUB files."""

    _jhove_module = "EPUB-ptc"
    _supported_metadata = [JHoveEpubMeta]

class JHoveDngScraper(JHoveScraperBase):
    """Variables for scraping dng files."""

    _jhove_module = "TIFF-hul"
    _supported_metadata = [JHoveDngMeta]

class JHoveAiffScraper(JHoveScraperBase):
    """Variables for scraping AIFF files."""

    _jhove_module = "AIFF-hul"
    _supported_metadata = [JHoveAiffMeta]


__author__ = 'minhtule'

from parameter import *
import urllib2
import logging
import utility

def append_parameter(parameters, parameter_creator_func, value, is_required=False):
    if is_required or value is not None:
        param = parameter_creator_func(value)
        parameters.append(param)

class HTTPRequest(object):
    """

    """

    GOOGLE_ANALYTICS_HOST = "http://www.google-analytics.com/collect"

    def __init__(self, tracker, hit_type, other_parameters):
        self.__tracker = tracker
        self.__params = []
        self.__params.append(ProtocolVersion())  # Protocol Version
        self.__params.append(TrackingID(tracker.tracking_id))  # Tracking ID
        self.__params.append(ClientID(tracker.client_id))  # Client ID
        self.__params.append(HitType(hit_type))  # Hit type
        self.__params.append(CacheBuster())  # Cache Buster, random number to clear cache
        append_parameter(self.__params, UserLanguage, tracker.original_request_language)
        self.__params.extend(other_parameters)

    def add_custom_dimension(self, index, value):
        self.__params.append(CustomDimension(CustomDimension.key_for_index(index), value))

    def add_custom_metric(self, index, value):
        self.__params.append(CustomMetric(CustomMetric.key_for_index(index), value))

    def send(self):
        request = urllib2.Request(self.GOOGLE_ANALYTICS_HOST,
                                  origin_req_host=self.__tracker.original_request_ip)
        request.add_header("User-Agent", self.__tracker.original_request_user_agent)

        data_payload = self.__build_data_payload()
        request.add_data(data_payload)

        result = urllib2.urlopen(request)

        logging.debug(LOGGING_PREFIX + data_payload)

        if result.code == 200:
            return True
        else:
            logging.error(LOGGING_PREFIX + "Request to GA server failed with code " + result.code)
            return False

    def __build_data_payload(self):
        return "&".join([param.url_format() for param in self.__params])

class PageTrackingRequest(HTTPRequest):
    PAGE_TRACKING_HIT_TYPE = "pageview"

    def __init__(self, tracker, document_hostname=None, document_path=None, document_title=None):
        other_params = []
        append_parameter(other_params, DocumentHostName, document_hostname, is_required=False)
        append_parameter(other_params, DocumentPath, document_path, is_required=False)
        append_parameter(other_params, DocumentTitle, document_title, is_required=False)
        super(PageTrackingRequest, self).__init__(tracker, self.PAGE_TRACKING_HIT_TYPE, other_params)

class EventTrackingRequest(HTTPRequest):
    EVENT_TRACKING_HIT_TYPE = "event"

    def __init__(self, tracker, category, action, label=None, value=None):
        other_params = []
        append_parameter(other_params, EventCategory, category, is_required=True)
        append_parameter(other_params, EventAction, action, is_required=True)
        append_parameter(other_params, EventLabel, label, is_required=False)
        append_parameter(other_params, EventValue, value, is_required=False)

        super(EventTrackingRequest, self).__init__(tracker, self.EVENT_TRACKING_HIT_TYPE, other_params)

class TransactionTrackingRequest(HTTPRequest):
    TRANSACTION_TRACKING_HIT_TYPE = 'transaction'

    def __init__(self, tracker, transaction_id, transaction_affiliation=None, transaction_revenue=None, transaction_shipping=None, transaction_tax=None, currency_code=None):
        other_params = []

        append_parameter(other_params, TransactionID, transaction_id, is_required=True)
        append_parameter(other_params, TransactionAffiliation, transaction_affiliation, is_required=False)
        append_parameter(other_params, TransactionRevenue, transaction_revenue, is_required=False)
        append_parameter(other_params, TransactionShipping, transaction_shipping, is_required=False)
        append_parameter(other_params, TransactionTax, transaction_tax, is_required=False)
        append_parameter(other_params, CurrencyCode, currency_code, is_required=False)

        super(TransactionTrackingRequest, self).__init__(tracker, self.TRANSACTION_TRACKING_HIT_TYPE, other_params)

class ItemTrackingRequest(HTTPRequest):
    ITEM_TRACKING_HIT_TYPE = 'item'

    def __init__(self, tracker, transaction_id, item_name, item_price=None, item_quantity=None, item_code=None, item_category=None, currency_code=None):
        other_params = []

        append_parameter(other_params, TransactionID, transaction_id, is_required=True)
        append_parameter(other_params, ItemName, item_name, is_required=True)
        append_parameter(other_params, ItemPrice, item_price, is_required=False)
        append_parameter(other_params, ItemQuantity, item_quantity, is_required=False)
        append_parameter(other_params, ItemCode, item_code, is_required=False)
        append_parameter(other_params, ItemCategory, item_category, is_required=False)
        append_parameter(other_params, CurrencyCode, currency_code, is_required=False)

        super(ItemTrackingRequest, self).__init__(tracker, self.ITEM_TRACKING_HIT_TYPE, other_params)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Dec  3 09:04:55 2021

@author: clement
"""

# =============================================================================
## Library dependancies
# =============================================================================

import pandas as pd
import numpy as np
import seaborn as sns
import networkx as nx
import os
import matplotlib.pyplot as plt
import mpl_toolkits.mplot3d.axes3d as p3

from mpl_toolkits.mplot3d import proj3d
from matplotlib.patches import FancyArrowPatch
from pathlib import Path
from utilitaries import toEdgeList, dfListConnex, includeSJ
from clusteringCoefficient import unweightedCC, weightedCC

# =============================================================================
## Graphical functions
# =============================================================================

###############################################################################

class Arrow3D(FancyArrowPatch):
    def __init__(self, xs, ys, zs, *args, **kwargs):
        FancyArrowPatch.__init__(self, (0,0), (0,0), *args, **kwargs)
        self._verts3d = xs, ys, zs

    def draw(self, renderer):
        xs3d, ys3d, zs3d = self._verts3d
        xs, ys, zs = proj3d.proj_transform(xs3d, ys3d, zs3d, renderer.M)
        self.set_positions((xs[0],ys[0]),(xs[1],ys[1]))
        FancyArrowPatch.draw(self, renderer)

###############################################################################

def network3D(dir_in,name, step):
    """
    A retester
    """

    df_lst = dfListConnex(dir_in,name)
    k = step

    fig = plt.figure(k)
    ax = p3.Axes3D(fig)
    ax.set_axis_off()
    sc = ax.scatter(df_lst[k]['x'],df_lst[k]['z'],df_lst[k]['y'],
                    s = df_lst[k]['weight']*10,
                    c = df_lst[k]['small_junction'],
                    cmap = 'bwr_r',
                    alpha=0.8)

    for cluster in df_lst[k]['junction']:
        # check if the cluster is sufficiently heavy
        # if not df_lst[k].loc[df_lst[k]['cluster'] == cluster, 'small_cluster'].item():
            L = df_lst[k].loc[df_lst[k]['junction'] == cluster, 'neighbors_list'].to_list()[0]
            for neighbour in L:
                # check if the neighbour is sufficiently heavy
                if not df_lst[k].loc[df_lst[k]['junction'] == neighbour,'small_junction'].item():
                    df_cluster = df_lst[k].loc[df_lst[k]['junction'] == cluster].copy()
                    df_neighbour = df_lst[k].loc[df_lst[k]['junction'] == neighbour].copy()
                    # return df_cluster, df_neighbour
                    arw = Arrow3D([df_cluster['x'].item(),df_neighbour['x'].item()],
                              [df_cluster['z'].item(),df_neighbour['z'].item()],
                              [df_cluster['y'].item(),df_neighbour['y'].item()],
                              color="grey",
                              lw = 1, mutation_scale=0.5,alpha=0.95)
                    ax.add_artist(arw)
                else:
                    pass
        # else:
        #     pass
    # L = [[5,0,0,'black'],[0,5,0,'green'],[0,0,5,'orange']]
    # df = pd.DataFrame(L,columns=['x','y','z','c'])
    # sc = ax.scatter(df['x'],df['y'],df['z'],
    #                 c = df['c'],
    #                 alpha=1)
    ax.view_init(0,0)

    return sc

def graphMass(dir_in, lst, loop = True, SJ = False):
    """
    Return a seaborn plot showing the mass of the junction with or without the
    loop counted and with or without the small junctions.

    Parameters:
    -----------
    dir_in: string
        Input directory, where the script will search the directory named name.
    lst: list of strings
        List of subdirectories in the input directory in which the files lie.
    loop: boolean, default True
        Indicate if you want the loop counted or not.
    SJ: boolean, default False
        Indicate if you want the small junctions counted or not.

    Returns:
    --------
    figure: seaborn violinplot
        The plot showing the distribution of the mass of the junctions in the
        network.
    """

    L = []
    for elem in lst:
        data_list = dfListConnex(dir_in, elem)
        for j in range(len(data_list)):
            aff = data_list[j].iloc[0]['affinity']
            M = []
            df = includeSJ(data_list[j], SJ)
            for k in range(len(df)):
                if loop:
                    M += [[aff, df.iloc[k]['mass']]]
                else:
                    M += [[aff, df.iloc[k]['degreeMG']]]
            L += M

    df = pd.DataFrame(L,columns=['aff','mass'])
    T = list(set(df['aff'].to_list()))
    if (5.5 in T) or (6.5 in T):
        for elem in [0.5*k for k in range(2,14)]:
            if elem not in T:
                df =  df.append({'aff': elem,
                                 'mass': -1},
                                ignore_index=True)
    df['affinity'] = df['aff'].apply(lambda x: round(1 - (x/25),2))
    figure = sns.violinplot(x="affinity",y='mass',data=df)

    for tick in figure.xaxis.get_major_ticks():
        tick.label.set_fontsize(20)

    for tick in figure.yaxis.get_major_ticks():
        tick.label.set_fontsize(20)

    plt.xlabel('Affinity', size=28, family='helvetica')
    if loop:
        plt.ylabel('Junction mass with loops included', size=28, family='helvetica')
    else:
        plt.ylabel('Junction mass without loops', size=28, family='helvetica')
    figure.set(ylim=(0, None))
    return figure

def graphDegree(dir_in, lst, multi = True, SJ = False):
    """
    Return a seaborn plot showing the degree or multoidegree of the junction
    with or without the loop counted and with or without the small junctions.

    Parameters:
    -----------
    dir_in: string
        Input directory, where the script will search the directory named name.
    lst: list of strings
        List of subdirectories in the input directory in which the files lie.
    multi: boolean, default True
        Indicate if you want the degree or the multidegree.
    SJ: boolean, default False
        Indicate if you want the small junctions counted or not. The degree
        (or multidegree) of a small junction is always 0.

    Returns:
    --------
    figure: seaborn violinplot
        The plot showing the distribution of the degree or multidegree of the
        junctions in the network.
    """

    L = []
    for elem in lst:
        data_list = dfListConnex(dir_in, elem)
        for j in range(len(data_list)):
            aff = data_list[j].iloc[0]['affinity']
            M = []
            df = includeSJ(data_list[j], SJ)
            for k in range(len(df)):
                if multi:
                    M += [[aff, df.iloc[k]['degreeMG']]]
                else:
                    M += [[aff, df.iloc[k]['degreeG']]]
            L += M

    df = pd.DataFrame(L,columns=['aff','degree'])
    T = list(set(df['aff'].to_list()))
    if (5.5 in T) or (6.5 in T):
        for elem in [0.5*k for k in range(2,14)]:
            if elem not in T:
                df =  df.append({'aff': elem,
                                 'degree': -1},
                                ignore_index=True)
    df['affinity'] = df['aff'].apply(lambda x: round(1 - (x/25),2))

    figure = sns.violinplot(x="affinity",y='degree',data=df)

    for tick in figure.xaxis.get_major_ticks():
        tick.label.set_fontsize(20)

    for tick in figure.yaxis.get_major_ticks():
        tick.label.set_fontsize(20)

    plt.xlabel('Affinity', size=28, family='helvetica')
    if multi:
        plt.ylabel('Multidegree', size=28, family='helvetica')
    else:
        plt.ylabel('Degree', size=28, family='helvetica')
    figure.set(ylim=(0, None))
    return figure

def graphClustering(dir_in, lst, unweighted = True, SJ = False):
    """
    Return a seaborn plot showing the mass of the junction with or without the
    loop counted and with or without the small junctions.

    Parameters:
    -----------
    dir_in: string
        Input directory, where the script will search the directory named name.
    lst: list of strings
        List of subdirectories in the input directory in which the files lie.
    unweighted: boolean, default True
        Indicate if you want the weighted or unweighted degree.
    SJ: boolean, default False
        Indicate if you want the small junctions counted or not.

    Returns:
    --------
    figure: seaborn violinplot
        The plot showing the distribution of the average clustering coefficient
        of the network.
    """

    df_clustering = pd.DataFrame()
    df_random = pd.DataFrame()

    affinityList = []
    typeList = []

    affinityRandomList = []
    CCList = []
    CCRandomList = []
    runList = []
    runRandomList = []
    if unweighted:
        for name in lst:
            df_lst = dfListConnex(dir_in, name)
            n = len(df_lst)
            try:
                affinityList += [df_lst[0].at[0,'affinity']]*n
            except KeyError:
                print(name)
                break
            typeList += [str(df_lst[0].at[0,'type'])]*n
            affinityRandomList += ['small world']*n
            #label = df_lst[0].at[0,'label']
            for k in range(len(df_lst)):
                df_edges = toEdgeList(includeSJ(df_lst[k], SJ))
                G, avgUWCC = unweightedCC(df_edges)

                # Get the parameters
                n = len(df_lst[k])
                L = list(G.degree())
                M = [x[1] for x in L]
                m = sum(M)/2

                # Create a small-world graph and exploit
                k = int(np.ceil((m//n) + 1))
                K = nx.connected_watts_strogatz_graph(n, k, 0.95, tries=100, seed=None)

                avgSW = nx.average_clustering(K)

                CCList += [avgUWCC]
                CCRandomList += [avgSW]
                runList += [df_lst[k].at[0,'label']]
                runRandomList += ['small world']
    else:
        for name in lst:
            df_lst = dfListConnex(dir_in, name)
            n = len(df_lst)
            try:
                affinityList += [df_lst[0].at[0,'affinity']]*n
            except KeyError:
                print(name)
                break
            typeList += [str(df_lst[0].at[0,'type'])]*n
            # affinityRandomList += ['small world']*n
            for k in range(len(df_lst)):
                df_edges = toEdgeList(includeSJ(df_lst[k], SJ))
                G, avgWCC = weightedCC(df_edges)

                # # Get the parameters
                # n = len(df_lst[k])
                # L = list(G.degree())
                # M = [x[1] for x in L]
                # m = sum(M)/2

                # # Create a small-world graph and exploit
                # k = int(np.ceil((m//n) + 1))
                # K = nx.connected_watts_strogatz_graph(n, k, 0.95, tries=100, seed=None)

                # degSW = nx.degree(K)
                # avgSW = 0
                # for key in K.nodes():
                #     avgSW += degSW[key]
                # avgSW /= len(degSW)

                CCList += [avgWCC]
                # CCRandomList += [avgSW]
                runList += [name]
                # runRandomList += ['small world']

    df_random['type'] = affinityRandomList
    df_random['affinity'] = affinityRandomList
    df_random['run'] = runRandomList
    df_random['CC'] = CCRandomList

    # newRunList = []
    # for elem in runList:
    #     if elem == '6E48/14001':
    #         newRunList += ['A']
    #     elif elem == '6E48/14002':
    #         newRunList += ['B']
    #     elif elem == '6E48/14003':
    #         newRunList += ['C']
    #     elif elem == '6E48/14004':
    #         newRunList += ['D']
    #     elif elem == '7E48/14001':
    #         newRunList += ['A two off']
    #     elif elem == '7E48/14002':
    #         newRunList += ['B two off']
    #     elif elem == '7E48/14003':
    #         newRunList += ['C two off']
    #     elif elem == '7E48/14004':
    #         newRunList += ['D two off']

    df_clustering['type'] = typeList
    df_clustering['aff'] = affinityList
    df_clustering['run'] = runList
    df_clustering['CC'] = CCList

    T = list(set(df_clustering['aff'].to_list()))
    if (5.5 in T) or (6.5 in T):
        for elem in [0.5*k for k in range(2,14)]:
            if elem not in T:
                df_clustering =  df_clustering.append({'aff': elem,
                                                        'CC': -1},
                                                      ignore_index=True)
    df_clustering['affinity'] = df_clustering['aff'].apply(lambda x: round(1 - (float(x)/25),2))

    #df_clustering = df_clustering.sort_values(by=['run'])

    df_random = df_random.append(df_clustering,ignore_index=True)
    sns.set_style({'axes.grid': True})
    figure = sns.violinplot(x = 'run', y = 'CC',
                     scale = "width",
                     data = df_clustering
                     )

    # Set the graphical properties
    for tick in figure.xaxis.get_major_ticks():
        tick.label.set_fontsize(20)
    for tick in figure.yaxis.get_major_ticks():
        tick.label.set_fontsize(20)

    # Labels for the polymer type and concentration graphs
    # ax.set_xticks(range(len(df_random['run'].unique()))) # <--- set the ticks first
    # ax.set_xticklabels(['small world',
    #                      '2B48 - 0.0008', '0.001', '0.002', '0.004', '0.006', '0.008',
    #                      '4B16 - 0.0005', '0.001', '0.002', '0.004',
    #                      '5B8 - 0.0004', '0.0006', '0.0008', '0.001',
    #                      '6B6 - 0.0004', '0.0006', '0.0008', '0.001'
    #                      ])
    plt.xticks(rotation=19, horizontalalignment='right')

    plt.xlabel('Polymer Type and concentration', size=28, family='helvetica')

    if unweighted:
        plt.ylabel('Average clustering coefficient', size=28, family='helvetica')
        figure.set(ylim=(0,1))
    else:
        plt.ylabel('Average weighted clustering coefficient', size=28, family='helvetica')
        figure.set(ylim=(0,None))

def graphClusteringComparison(dir_in, lst, SJ = False):
    """
    Return a seaborn plot showing the mass of the junction with or without the
    loop counted and with or without the small junctions.

    Parameters:
    -----------
    dir_in: string
        Input directory, where the script will search the directory named name.
    lst: list of strings
        List of subdirectories in the input directory in which the files lie.
    unweighted: boolean, default True
        Indicate if you want the weighted or unweighted degree.
    SJ: boolean, default False
        Indicate if you want the small junctions counted or not.

    Returns:
    --------
    figure: seaborn violinplot
        The plot showing the distribution of the average clustering coefficient
        of the network.
    """

    df_clusteringCC = pd.DataFrame()
    df_random = pd.DataFrame()

    CCaffinityList = []
    CCtypeList = []

    affinityRandomList = []
    CCList = []
    CCRandomList = []
    CCrunList = []
    runRandomList = []

    for name in lst:
        df_lst = dfListConnex(dir_in, name)
        n = len(df_lst)
        try:
            CCaffinityList += [df_lst[0].at[0,'affinity']]*n
        except KeyError:
            print(name)
            break
        CCtypeList += [str(df_lst[0].at[0,'type'])]*n
        affinityRandomList += ['small world']*n
        for k in range(len(df_lst)):
            df_edges = toEdgeList(includeSJ(df_lst[k], SJ))
            G, avgUWCC = unweightedCC(df_edges)

            # Get the parameters
            n = len(df_lst[k])
            L = list(G.degree())
            M = [x[1] for x in L]
            m = sum(M)/2

            # Create a small-world graph and exploit
            k = int(np.ceil((m//n) + 1))
            K = nx.connected_watts_strogatz_graph(n, k, 0.95, tries=100, seed=None)

            avgSW = nx.average_clustering(K)

            CCList += [avgUWCC]
            CCRandomList += [avgSW]
            CCrunList += [name]
            runRandomList += ['small world']

    df_random['type'] = affinityRandomList
    df_random['affinity'] = affinityRandomList
    df_random['run'] = runRandomList
    df_random['CC'] = CCRandomList

    df_clusteringCC['type'] = CCtypeList
    df_clusteringCC['aff'] = CCaffinityList
    df_clusteringCC['run'] = CCrunList
    df_clusteringCC['CC'] = CCList

    T = list(set(df_clusteringCC['aff'].to_list()))
    if (5.5 in T) or (6.5 in T):
        for elem in [0.5*k for k in range(2,14)]:
            if elem not in T:
                df_clusteringCC =  df_clusteringCC.append({'aff': elem,
                                                        'CC': -1},
                                                      ignore_index=True)
    df_clusteringCC['affinity'] = df_clusteringCC['aff'].apply(lambda x: round(1 - (float(x)/25),2))

    df_clusteringCC = df_clusteringCC.sort_values(by=['type'])

    df_random = df_random.append(df_clusteringCC,ignore_index=True)

    f, (subfigure1, subfigure2) = plt.subplots(1, 2, gridspec_kw={'width_ratios': [5, 4]})
    sns.set_style({'axes.grid': True})
    plt.rcParams["axes.titlesize"] = 25
    sns.violinplot(x = 'type', y = 'CC',
                     scale = "width",
                     data = df_random,
                     ax = subfigure1
                     )

    # Set the graphical properties
    for tick in subfigure1.xaxis.get_major_ticks():
        tick.label.set_fontsize(20)
    for tick in subfigure1.yaxis.get_major_ticks():
        tick.label.set_fontsize(20)

    # WCC
    df_clusteringWCC = pd.DataFrame()

    WCCaffinityList = []
    WCCtypeList = []

    affinityRandomList = []
    WCCList = []
    WCCrunList = []

    for name in lst:
        df_lst = dfListConnex(dir_in, name)
        n = len(df_lst)
        try:
            WCCaffinityList += [df_lst[0].at[0,'affinity']]*n
        except KeyError:
            print(name)
            break
        WCCtypeList += [str(df_lst[0].at[0,'type'])]*n
        for k in range(len(df_lst)):
            df_edges = toEdgeList(includeSJ(df_lst[k], SJ))
            G, avgWCC = weightedCC(df_edges)

            WCCList += [avgWCC]
            WCCrunList += [name]

    df_clusteringWCC['type'] = WCCtypeList
    df_clusteringWCC['aff'] = WCCaffinityList
    df_clusteringWCC['run'] = WCCrunList
    df_clusteringWCC['CC'] = WCCList

    T = list(set(df_clusteringWCC['aff'].to_list()))
    if (5.5 in T) or (6.5 in T):
        for elem in [0.5*k for k in range(2,14)]:
            if elem not in T:
                df_clusteringWCC =  df_clusteringWCC.append({'aff': elem,
                                                        'CC': -1},
                                                      ignore_index=True)
    df_clusteringWCC['affinity'] = df_clusteringWCC['aff'].apply(lambda x: round(1 - (float(x)/25),2))

    df_clusteringWCC = df_clusteringWCC.sort_values(by=['type'])

    sns.set_style({'axes.grid': True})
    sns.violinplot(x = 'type', y = 'CC',
                     scale = "width",
                     data = df_clusteringWCC,
                     ax = subfigure2
                     )

    # Set the graphical properties
    for tick in subfigure2.xaxis.get_major_ticks():
        tick.label.set_fontsize(20)
    for tick in subfigure2.yaxis.get_major_ticks():
        tick.label.set_fontsize(20)

    # Labels for the polymer type and concentration graphs
    # ax.set_xticks(range(len(df_random['run'].unique()))) # <--- set the ticks first
    # ax.set_xticklabels(['small world',
    #                      '2B48 - 0.0008', '0.001', '0.002', '0.004', '0.006', '0.008',
    #                      '4B16 - 0.0005', '0.001', '0.002', '0.004',
    #                      '5B8 - 0.0004', '0.0006', '0.0008', '0.001',
    #                      '6B6 - 0.0004', '0.0006', '0.0008', '0.001'
    #                      ])
    # plt.xticks(rotation=19, horizontalalignment='right')

    subfigure1.set_xlabel('Polymer Type', size=28, family='helvetica')
    subfigure1.set_ylabel('Average clustering coefficient', size=28, family='helvetica')
    subfigure1.set(ylim=(0,1))

    subfigure2.set_xlabel('Polymer Type', size=28, family='helvetica')
    subfigure2.set_ylabel('Average weighted clustering coefficient', size=28, family='helvetica')
    subfigure2.set(ylim=(0,None))

    return f.tight_layout()

def graphFluidity(dir_in, filename):
    """
    Return a seaborn plot showing the mass of the junction with or without the
    loop counted and with or without the small junctions.

    Parameters:
    -----------
    dir_in: string
        Input directory, where the script will search the file filename.
    filename: string
        Name of the file that will de converted in a dataframe and where we
        already have saved the fluidity (it's a time consuming 'function thats
        why it's already done).

    Returns:
    --------
    figure: seaborn lineplot
        The plot showing the evolution of the average fluidity of the network.
    """

    dir_path = Path(os.getcwd())
    file_path = dir_path.parent.joinpath(dir_in, filename)
    df = pd.read_pickle(file_path)

    figure = sns.lineplot(x = 'step', y = 'fluidity',
                     hue = "affinity",
                     data = df
                     )

    return figure

# =============================================================================
## List to test
# =============================================================================

P = ['1011', '2414', '2450', '622', '2214', '2806']
Q = ['2B/6091', '2B/6092', '2B/6093', '2B/6094', '2B/6095', '2B/6096',
     '4C/4025', '4C/4026', '4C/4027', '4C/4028',
     '5D/11001', '5D/11002', '5D/11003', '5D/11004',
     '6E/14001', '6E/14002', '6E/14003', '6E/14004']
R = ['2B/6091',
     '4C/4025',
     '5D/11003',
     '6E/14003']
S = ['6E48/14001','7E48/14001',
     '6E48/14002','7E48/14002',
     '6E48/14003','7E48/14003',
     '6E48/14004','7E48/14004']
T = ['4/6e48/14001', '4/6e48/14002', '4/6e48/14003', '4/6e48/14004',
     '4/6f48/15001', '4/6f48/15002', '4/6f48/15003', '4/6f48/15004',
     #'65/6d48/13001r6', '65/6d48/13002r6', '65/6d48/13003r6', '65/6d48/13004r6',
     '65/6e48/14001', '65/6e48/14002', '65/6e48/14003', '65/6e48/14004',
     '65/6d48/13001', '65/6d48/13002', '65/6d48/13003', '65/6d48/13004',
     '65/6f48/15001', '65/6f48/15002', '65/6f48/15003', '65/6f48/15004'
     ]

# =============================================================================
## Graphs generations
# =============================================================================

graphClustering('neo_junctions',T[:8])


# -*- coding: utf-8 -*-
"""
Created on Mon Dec 17 21:21:43 2018

@author: Mikko Impiö
"""
import pandas as pd
import numpy as np
from utils import find_similar

if __name__ == "__main__":

    # IMAGE CLASSIFICATION LABELS
    labelpath = 'C:\\koodia\\huawei\\food-101\\food-101\\meta\\labels.txt'
    classpath = 'C:\\koodia\\huawei\\food-101\\food-101\\meta\\classes.txt'
    label_names = pd.read_csv(labelpath, header=None)
    label_names = label_names.values.flatten().tolist()

    class_names = pd.read_csv(classpath, header=None)
    class_names = class_names.values.flatten().tolist()

    # RECIPE NAMES
    recipepath = "C:\\koodia\\huawei\\Yummly28K\\data_records_27638.txt"
    recipes = pd.read_csv(recipepath, header=None)
    orig_recipes = pd.read_csv(recipepath, header=None, sep='\t')

    recipes = recipes.replace("\d|\t", "",regex=True)
    recipes = recipes.replace("-|_", " ",regex=True)

    rem_words = ["My recipes",
                 "My Recipes",
                 "Myrecipes",
                 "MyRecipes",

                 "Allrecipes",
                 "AllRecipes",
                 "All recipes",
                 "All Recipes",

                 "Martha Stewart",
                 "Epicurious"]

    for word in rem_words:
        recipes = recipes.replace(word, "", regex=True)

    ## FIND SIMILAR
    from difflib import SequenceMatcher

    def diff(a, b):
        return SequenceMatcher(None, a, b).ratio()

    recipelist = recipes.values.flatten().tolist()

#    diffs = {}
#    for label in label_names:
#        diffs[label] = []
#        for i in range(len(recipelist)):
#            dif = diff(recipelist[i], label)
#            diffs[label].append(dif)
#            print("{}: {}".format(label, i))

#    np.save('diffs.npy',diffs)

#    similars = {}
#    sim_indices = {}
#    for key in diffs.keys():
#        top_5 = np.asarray(diffs[key]).argsort(axis=0)[-10::][::-1]
#        recipe = np.asarray(recipelist)[top_5]
#        similars[key] = recipe
#        sim_indices[key] = top_5

    diffs, similars, sim_indices = find_similar(label_names, recipelist)
    diffs = np.load('diffs.npy').item()

    # MAP THE CORRESPONDING RECIPE TO EACH LABEL
    food_mappings = {}
    # GO THROUGH FIRST ONES IN PROPOSED SIMILARS LIST
    for i,name in enumerate(label_names):
        food_index = sim_indices[name][0]
        food = recipelist[food_index]
        food_mappings[name] = [food, food_index]

    #  GO THROUGH ALTERNATIVE NAMES FROM THE SIMILARS LIST
    alts = pd.read_csv('C:\\Users\\Mikko Impiö\\Google Drive\\koodia\\huawei\\CO2_classifier\\recipe_alt_place_indices.txt', sep=",")
    for i, name in enumerate(alts.name.values):
        ind = alts.ind.values[i]
        food_index = sim_indices[name][ind]
        food = recipelist[food_index]
        food_mappings[name] = [food, food_index]

    # GO THROUGH MANUALLY SELECTED RECIPES
    alts = pd.read_csv('C:\\Users\\Mikko Impiö\\Google Drive\\koodia\\huawei\\CO2_classifier\\recipe_notfound_indices.txt', sep="\t")
    for i, name in enumerate(alts.name.values):
        food_index = alts.ind.values[i]
        if food_index != -1:
            food = recipelist[food_index]
            food_mappings[name] = [food, food_index]
        else:
            food_mappings[name] = ['nan', -1]

    np.save('food_mappings.npy',food_mappings)


import numpy

import matplotlib.pyplot as plt

def display_data(x_array, example_width=None, figsize=(10, 10)):
    """
    Displays 2D data in a nice grid.

    Parameters
    ----------
    x_array : array_like
        The input data of size (m x n) where m is the number of examples and n is the number of
        features.

    example_width : int, optional
        THe width of each 2-D image in pixels. If not provided, the image is assumed to be square,
        and the width is the floor of the square root of total number of pixels.

    figsize : tuple, optional
        A 2-element tuple indicating the width and height of figure in inches.
    """
    # Compute rows, cols
    if x_array.ndim == 2:
        num_examples, num_features = x_array.shape
    elif x_array.ndim == 1:
        num_features = x_array.size
        num_examples = 1
        x_array = x_array[None]  # Promote to a 2 dimensional array
    else:
        raise IndexError('Input x_array should be 1 or 2 dimensional.')

    example_width = example_width or int(numpy.round(numpy.sqrt(num_features)))
    example_height = int(num_features / example_width)

    # Compute number of items to display
    display_rows = int(numpy.floor(numpy.sqrt(num_examples)))
    display_cols = int(numpy.ceil(num_examples / display_rows))

    fig, ax_array = plt.subplots(display_rows, display_cols, figsize=figsize)
    fig.subplots_adjust(wspace=0.025, hspace=0.025)

    ax_array = [ax_array] if num_examples == 1 else ax_array.ravel()

    for i, ax in enumerate(ax_array):
        ax.imshow(x_array[i].reshape(example_height, example_width, order='F'), cmap='gray')
        ax.axis('off')
    fig.show()

    return 0


import json
import os
import sys
import boto3
import random
import urllib.request
# AWS lambda expects all the code to be in /var/task/ directory. for this project dir is below (shared/python has custom code)
# sh-4.2# ls -la /var/task/
# total 40
# drwxr-xr-x 1 root root 4096 Aug 19 23:47 .
# drwxr-xr-x 1 root root 4096 Aug 19 23:46 ..
# drwxrwxr-x 2 root root 4096 Aug 19 23:12 handle_portfolio_update
# -rw-r--r-- 1 root root    0 Aug 19 23:12 __init__.py
# drwxrwxr-x 2 root root 4096 Aug 19 23:12 schedule_listener
# drwxr-xr-x 1 root root 4096 Aug 19 23:47 shared
# drwxrwxr-x 2 root root 4096 Aug 19 23:12 system_event_listener
# drwxrwxr-x 2 root root 4096 Aug 19 23:12 test_custom_layer
if (True):
    try:
        sys.path.append('/var/task/shared/python')
        import aws_quant_infra as aq_i
        print(f"import aws_quant_infra is done!")
        print(aq_i.__file__)
        import aws_quant_risk as aq_r
        print(f"import aws_quant_risk is done!")
        print(aq_r.__file__)
    except Exception as exc:
        print(f'error importing:{exc}')
else:
    pass

REGION = os.environ['AWS_REGION']
print(f'aws region: {REGION}')

def boto3_test():
    print('Starting tests to check various boto3 calls')
    env, app = aqi_test()
    aqr_test(env, app)

def aqi_test():
    print('Testing aws_quant_infra.get_app_config_from_paramstore')
    env_config = aq_i.get_app_config_from_paramstore(REGION)
    print(env_config)

    print('Testing appconfig call')
    env = 'dev'
    app_config_details = json.loads(env_config.get('Mvp-PortfolioMonitoring-AppConfigDetails'))
    app = app_config_details.get('Application')
    config = app_config_details.get('Configuration')
    version = app_config_details.get(env)
    print(f'App: {app} Config: {config} Version: {version}')
    app_config = (boto3.client('appconfig'). \
                      get_configuration(Application=app
                                        ,Environment=env
                                        ,Configuration=config
                                        , ClientId=f'{random.randint(-sys.maxsize,sys.maxsize)}'
                                        ,ClientConfigurationVersion=version
                                        ).get('Content').read())
    # app_config = urllib.request.urlopen(
    #     f'http://localhost:2772/applications/{app}/environments/{env}/configurations/{config}'
    # ).read()

    app_config=json.loads(app_config.decode('utf-8'))
    print(app_config)

    print('Testing aws_quant_infra.get_secret')
    secret_name = app_config.get('secrets')[0]
    # secret = f'MvpPortfolioMonitoring-{secret_name}'
    secret = f'{secret_name}'
    secret_data = aq_i.get_secret(secret, REGION)
    print(secret_data)
    try:
        print('Testing format of response')
        print(secret_data[secret])
    except:
        print('get_secret response in wrong format!!!')

    return env_config, app_config

def aqr_test(env_config, app_config):
    print('Testing Batch call')
    batch_client = boto3.client('batch')
    batch_jobs_resp = batch_client.list_jobs(jobQueue='MvpPortfolioMonitoring_q_ec2')
    print(batch_jobs_resp)

    print('Testing aws_quant_risk.PortfolioTrackerFactory')
    PTF = aq_r.PortfolioTrackerFactory(env_config, app_config, 'Mvp-')
    print(str(PTF))

    print('Testing aq_r.PTF.get_all_portfolios (Dynamodb call)')
    PTF.load_all_portfolios()
    print(PTF.all_portfolios)

    print('Testing Events calls')
    for rule_name in [ 'trading_EOD', 'trading_SOD']:
        print(f'Checking call for rule name: {rule_name}')
        print(boto3.client('events').list_targets_by_rule(Rule=rule_name))

    print('Testing timestream call')
    timestream_db = env_config.get('Mvp-PortfolioMonitoring-TimestreamDb')
    timestream_table = 'iex_realtime_data'
    print('Checking for query endpoints')
    query = f'SELECT * FROM {timestream_db}.{timestream_table} LIMIT 10'
    print(boto3.client('timestream-query').query(
        QueryString=query,
    ))
    print('Checking for write endpoints')
    print(boto3.client('timestream-write').describe_database(
        DatabaseName=timestream_db
    ))

def lambda_handler(event, context):
    # TODO implement
    dirs = ['/var/task/', '/var/task/shared', '/var/task/shared/python']
    for one_dir in dirs:
        try:
            print(f"dir list {one_dir}:", os.listdir(one_dir),"\n")
        except:
            print(f"NO dir: {one_dir}","\n")
    print("PYTHONPATH:", os.environ.get('PYTHONPATH'),"\n")
    print("PATH:", os.environ.get('PATH'),"\n")
    print("LD_LIBRARY_PATH:", os.environ.get('LD_LIBRARY_PATH'),"\n")
    print("sys.path:", sys.path,"\n")
    print("incoming event:", event,"\n")

    boto3_test()

    return {
            'statusCode': 200,
            'body': json.dumps('Hello from Lambda!')
        }


import codecs
from setuptools import setup, find_packages

entry_points = {
    "z3c.autoinclude.plugin": [
        'target = nti.app',
    ],
}

def _read(fname):
    with codecs.open(fname, encoding='utf-8') as f:
        return f.read()

setup(
    name='nti.app.messaging',
    version=_read('version.txt').strip(),
    author='Jason Madden',
    author_email='jason@nextthought.com',
    description="NTI Application Messaging",
    long_description=(_read('README.rst') + '\n\n' + _read('CHANGES.rst')),
    license='Apache',
    keywords='pyramid messaging',
    classifiers=[
        'Intended Audience :: Developers',
        'Natural Language :: English',
        'Operating System :: OS Independent',
        'License :: OSI Approved :: Apache Software License',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: Implementation :: CPython',
    ],
    zip_safe=True,
    packages=find_packages('src'),
    package_dir={'': 'src'},
    include_package_data=True,
    namespace_packages=['nti', 'nti.app'],
    install_requires=[
        'setuptools',
        'nti.messaging',
    ],
    entry_points=entry_points,
)


"""
    导出所有资源到文件中  用于导入到ocean_ctf 平台
"""
import os

import yaml

def find_files(directory, filename):
    results = []
    for root, dirs, files in os.walk(directory):
        for file in files:
            if file == filename:
                results.append(os.path.join(root, file))
    return results

def main():
    current_directory = os.getcwd()
    if os.path.exists("vulnerability.yaml"):
        # 备份文件
        os.rename("vulnerability.yaml", "_vulnerability.yaml")
    vuln_files = find_files(current_directory, "vulnerability.yaml")
    datas = []
    for vuln_file in vuln_files:
        with open(vuln_file, "r") as f:
            try:
                yaml_data = yaml.safe_load(f)
            except yaml.YAMLError as e:
                print("Error while parsing YAML file:", e)
                continue
        if yaml_data.get("build"):
            print("building {}".format(yaml_data['image']))
            work_dir = os.path.dirname(vuln_file)
            context = yaml_data.get("context", ".")
            dockerfile = yaml_data.get("dockerfile", "Dockerfile")
            os.system(f"cd {work_dir} docker build {context} -t {yaml_data['image']} -f {dockerfile}")
        datas.append(yaml_data)
    print("Export Vulnerability Count {}".format(len(datas)))
    with open("vulnerability.yaml", 'w') as file:
        try:
            yaml.dump(datas, file, default_flow_style=False)
        except yaml.YAMLError as e:
            print("Error while writing YAML file:", e)

if __name__ == "__main__":
    main()


from gtts import gTTS
from playsound import playsound
from PIL import Image, ImageGrab
import pytesseract
import sys
import os

imageFileName = "image.png"
try:
    im = ImageGrab.grabclipboard()
    im.save(imageFileName,'PNG')
except AttributeError:
    sys.exit("no image in the clipboard!")

text = pytesseract.image_to_string(Image.open(imageFileName) )
filename = "sample.mp3"

lang='en'
tts = gTTS(text=text, lang=lang)
try:
    tts.save(filename)
except PermissionError:
    print('Permission denied! Remove the sample.mp3 file')
playsound(filename)
os.remove(filename)


"""
Logging setup.
"""
import logging

from istub.constants import LOGGER_NAME

def setup_logging(level: int = 0) -> logging.Logger:
    """
    Get Logger instance.

    Arguments:
        level -- Logging level

    Returns:
        Created Logger.
    """
    logger = logging.getLogger(LOGGER_NAME)
    stream_handler = logging.StreamHandler()
    formatter = logging.Formatter("%(levelname)s %(message)s", datefmt="%H:%M:%S")
    stream_handler.setFormatter(formatter)
    stream_handler.setLevel(level)
    logger.addHandler(stream_handler)
    logger.setLevel(level)
    return logger


import sqlite3
import json
import os
import sys
from addons import utils
from addons.checks import checks
from discord.ext import commands
from discord import utils as discord

class Service(commands.Cog):
    """
    Service commands (owner only)
    """

    #Construct
    def __init__(self, bot):
        self.bot = bot
        print('Addon "{}" loaded'.format(self.__class__.__name__))

    #send message
    #async def send(self, ctx, msg):
    #    await self.ctx.send(msg)

    #commands
    @commands.command()
    @checks.is_access_allowed(required_level=9000)
    async def load(self, ctx, name : str):
        extension = "addons.{}".format(name)
        try:
            self.bot.load_extension(extension)
            print("{} loaded".format(extension))
            await ctx.message.channel.send("{} loaded".format(extension))
        except Exception as e:
            print('{} failed to load.\n{}: {}'.format(extension, type(e).__name__, e))
            await ctx.message.channel.send('{} failed to load.\n{}: {}'.format(extension, type(e).__name__, e))

    @commands.command()
    @checks.is_access_allowed(required_level=9000)
    async def unload(self, ctx, name: str):
        extension = "addons.{}".format(name)
        self.bot.unload_extension(extension)
        print("{} unloaded".format(extension))
        await ctx.message.channel.send("{} unloaded".format(extension))

    @commands.command()
    @checks.is_access_allowed(required_level=9000)
    async def reload(self, ctx):
        """Reload addons, db and config (owner only)"""

        #Reload configuration file so we can apply some settings on the fly
        #might be useful
        with open('config.json') as data:
            self.bot.config = json.load(data)

        #TESTING: potentially unsafe and might lead to data corruption.
        #commit all changes and close connection before proceed
        self.bot.db.commit()
        self.bot.db.close()
        #Reinitialize connection to database
        self.bot.db = sqlite3.connect('main.db')

        #reload extensions
        for extension in self.bot.config['extensions']:
            try:
                self.bot.unload_extension(extension['name'])
                self.bot.load_extension(extension['name'])
            except Exception as e:
                print('{} failed to load.\n{}: {}'.format(extension['name'], type(e).__name__, e))
                await ctx.message.channel.send('{} failed to load.\n{}: {}'.format(extension['name'], type(e).__name__, e))
        await ctx.message.channel.send("Reload complete!")

    @commands.command()
    @checks.is_access_allowed(required_level=9000)
    async def shutdown(self, ctx):
        """This kills the bot"""

        await ctx.message.channel.send("bye fren :(")

        sys.exit(0)

    @commands.group()
    async def roles(self, ctx):
        """Roles management"""
        if ctx.invoked_subcommand is None:
            msg = "Have you ever tried `apu help roles`? I suggest you to do it now, fren"
            await ctx.message.channel.send(msg)

    @roles.command(name="list")
    @checks.is_access_allowed(required_level=2)
    async def roles_list(self, ctx):
        """ - Returns roles list for this server (Level 2)"""
        cursor = self.bot.db.cursor()

        if not await utils.db_check(self.bot, ctx.message, cursor, "roles"):
            return

        msg = "```List of roles:\n"
        cursor.execute("SELECT * FROM roles WHERE serverid=?", (ctx.message.guild.id,))
        data = cursor.fetchall()
        cursor.close()
        if data:
            for row in data:
                msg += "ID: {} | Role name: {} | Level: {}\n".format(row[1], row[2], row[3])
        else:
            msg += "Empty"
        msg += "```"
        await  ctx.message.channel.send(msg)

    @roles.command(name="add")
    @checks.is_access_allowed(required_level=3)
    async def roles_add(self, ctx, id : int, level : int):
        """ - Add role"""
        cursor = self.bot.db.cursor()

        if not await utils.db_check(self.bot, ctx.message, cursor, "roles"):
            return

        role = discord.get(ctx.message.guild.roles, id=id)

        if role is None:
            print("Role wasn't found")
            await ctx.message.channel.send("Role wasn't found, fren")
            return

        db = self.bot.db

        record = (role.id, role.name.lower(), level, ctx.message.guild.id)
        query = 'INSERT INTO roles(role_id, role, level, serverid) VALUES (?,?,?,?)'

        try:
            db.execute(query, record)
            db.commit()
            #Add role
            self.bot.access_roles[ctx.message.guild.id].update({id : level})
            await ctx.message.channel.send("Your record has been successfully added, fren")
        except sqlite3.Error as e:
            await ctx.message.channel.send("Failed to add new record, fren :(\n"
                                           "```{}```".format(str(e)))

    @roles.command(name="rm")
    @checks.is_access_allowed(required_level=3)
    async def roles_remove(self, ctx, id : int):
        """ - Remove role"""
        cursor = self.bot.db.cursor()

        if not await utils.db_check(self.bot, ctx, cursor, "roles"):
            return

        role = discord.get(ctx.message.guild.roles, id=id)

        if role is None:
            print("Role wasn't found")
            await ctx.message.channel.send("Role wasn't found, fren")
            return

        db = self.bot.db

        record = (role.name.lower(), ctx.message.guild.id)
        query = 'DELETE FROM roles WHERE role=? AND serverid=?'
        if db.execute(query, record).rowcount == 0:
            await ctx.message.channel.send("Failed to remove this record, fren")
        else:
            db.commit()
            #Remove role from storage
            self.bot.access_roles[ctx.message.guild.id].pop(role.id)
            await ctx.message.channel.send("This record has been successfully removed, fren")

    @commands.group()
    async def db(self, ctx):
        """Database management (owner only)"""
        if ctx.invoked_subcommand is None:
            msg = "Have you ever tried `apu help db`? I suggest you to do it now, fren"
            await ctx.message.channel.send(msg)

    @db.command(name="init")
    @checks.is_access_allowed(required_level=9000)
    async def db_init(self, ctx):
        """Initializes db (required on first start)"""
        db = self.bot.db

        try:
            roles = [
                ('commander', 3, 87159376091033600), ('moderator', 2, 87159376091033600)
            ]
            db.executemany('INSERT INTO roles(role, level, serverid) VALUES (?,?,?)', roles)

            db.commit()

            #put roles in storage
            #role[0] - role name
            #role[1] - access level
            for role in roles:
                self.bot.access_roles[ctx.message.guild.id].update({role[0] : role[1]})

            await ctx.message.channel.send("Database has been initialized, fren!")
        except sqlite3.Error as e:
            await ctx.message.channel.send("Failed to initialize db, fren :(")

    @db.command(name="add")
    @checks.is_access_allowed(required_level=9000)
    async def db_add(self, ctx, table : str, name : str, content : str):
        """ - Add record"""
        db = self.bot.db
        record = ('{}'.format(name), '{}'.format(content))
        query = 'INSERT INTO {} VALUES (?,?)'.format(table)

        try:
            db.execute(query, record)
            db.commit()
            await ctx.message.channel.send("Your record has been successfully added, fren")
        except sqlite3.Error:
            await ctx.message.channel.send("Failed to add new record, fren :(")

    @db.command(name="edit")
    @checks.is_access_allowed(required_level=9000)
    async def db_edit(self, ctx, table : str, name : str, column : str, value : str):
        """ - Edit record"""
        db = self.bot.db
        record = (name,)
        query = 'UPDATE {} SET {} = "{}" WHERE name=?'.format(table, column, value)
        if db.execute(query, record).rowcount == 0:
            await ctx.message.channel.send("This record wasn't found, fren")
        else:
            db.commit()
            await ctx.message.channel.send("This record has been successfully edited, fren")

    @db.command(name="rm")
    @checks.is_access_allowed(required_level=9000)
    async def db_remove(self, ctx, table : str, name : str):
        """ - Remove record"""

        db = self.bot.db
        record = (name,)
        query = 'DELETE FROM {} WHERE name=?'.format(table)
        if db.execute(query, record).rowcount == 0:
            await ctx.message.channel.send("Failed to remove this record, fren")
        else:
            db.commit()
            await ctx.message.channel.send("This record has been successfully removed, fren")

def setup(bot):
    bot.add_cog(Service(bot))


'''
문) bmi.csv 데이터셋을 이용하여 다음과 같이 softmax classifier 모델을 생성하시오.
   조건1> bmi.csv 데이터셋
       -> x변수 : height, weight 칼럼
       -> y변수 : label(3개 범주) 칼럼
    조건2> 딥러닝 최적화 알고리즘 : Adam
    조건3> learning rage : 0.001 or 0.005 선택(분류정확도 높은것)
    조건4> 반복학습, step 단위로 loss : <출력결과> 참고
    조건5> 분류정확도 출력
    조건6> 예측치와 정답 15개 출력

  <출력 결과>
step = 500 , loss = 0.44498476
step = 1000 , loss = 0.34861678
step = 1500 , loss = 0.28995454
step = 2000 , loss = 0.24887484
step = 2500 , loss = 0.2177721
step = 3000 , loss = 0.19313334
step = 3500 , loss = 0.17303815
step = 4000 , loss = 0.15629826
step = 4500 , loss = 0.1421249
step = 5000 , loss = 0.12996733
========================================
accuracy = 0.9769
========================================
y_pred :  [0 0 1 1 1 1 0 2 0 2 1 2 1 0 2]
y_true :  [0 0 1 1 1 1 0 2 0 2 1 2 1 0 2]
========================================
'''

import tensorflow as tf # ver1.x
from sklearn.preprocessing import minmax_scale # x data 정규화(0~1)
from sklearn.metrics import accuracy_score
import numpy as np
import pandas as pd

bmi = pd.read_csv('C:\\ITWILL\\5_Tensoflow\\data/bmi.csv')
print(bmi.info())

# 칼럼 추출
col = list(bmi.columns)
print(col)

# x,y 변수 추출
x_data = bmi[col[:2]] # x변수

# x_data 정규화
x_data = minmax_scale(x_data)

# label one hot encoding
label_map = {"thin": [1,0,0], "normal": [0,1,0], "fat": [0,0,1]}
bmi["label"] = bmi["label"].apply(lambda x : np.array(label_map[x]))

y_data = list(bmi["label"]) # 중첩list : [[1,0,0], [1,0,0]]

# numpy 객체 변환
x_data = np.array(x_data)
y_data = np.array(y_data)

################# X,Y 데이터 전처리 완료 #####################

# 1. X,Y변수 정의 : 공급형 변수
X = tf.constant(x_data, tf.float32)# [? 2]
Y = tf.constant(y_data, tf.float32)# [? 3]

# 2. w,b 변수 정의
w = tf.Variable(tf.random.normal([2, 3])) # [입력수, 출력수]
b = tf.Variable(tf.zeros([3]))  # [출력수]

# 3. 회귀방정식
def linear_model(X) : # train, test
    y_pred = tf.matmul(X, w) + b  # 행렬곱 : [None,3]*[3,1]=[None,1]
    return y_pred

# 4. softmax 활성함수 적용
def soft_fn(X):
    y_pred = linear_model(X)
    soft = tf.nn.softmax(y_pred)
    return soft

# 5. 손실 함수 정의 : 손실계산식 수정
def loss_fn() : #  인수 없음
    soft = soft_fn(X) # 훈련셋 -> 예측치 : 회귀방정식
    loss = -tf.reduce_mean(Y*tf.math.log(soft)+(1-Y)*tf.math.log(1-soft))
    return loss

# 6. 최적화 객체
opt = tf.optimizers.Adam(learning_rate=0.005)

# 7. 반복학습
for step in range(5000):
    opt.minimize(loss=loss_fn, var_list=[w,b])

    # 10 배수 단위 출력
    if (step+1) % 500 == 0:
        print('step=',(step+1),",loss val =", loss_fn().numpy())

# 8. 최적화된 model 검증
y_pred =soft_fn(X).numpy()
print(y_pred)
'''

'''

y_pred.sum(axis = 1)
# array([0.99999994, 0.9999999 , 1.        , 0.99999994, 1.        , 1.        ], dtype=float32)

# index 반환 : 확률 -> 10진수
y_pred = tf.argmax(y_pred, axis = 1)
#print(y_pred.numpy())

# 정답: one hot encoding -> 10진수
y_true = tf.argmax(Y, axis = 1)
#print(y_true.numpy())

acc = accuracy_score(y_true,y_pred)
print('accuracy=',acc)
# accuracy= 1.0

print('y_pred :' ,y_pred[0:14].numpy())
print('y_true :',y_true[0:14].numpy())


import sys
from collections import deque
sys.stdin = open('input.txt')
input = sys.stdin.readline

n = int(input())
for _ in range(n):
    nn = int(input())
    nl = int(input())
    route = [[] for _ in range(nn+1)]
    for _ in range(nl):
        a, b = map(int, input().split())
        route[a].append(b)
        route[b].append(a)

    queue = deque([1])
    visit = [0] * (nn+1)
    visit[1] = 1
    flag = True

    #dfs
    while queue and flag:
        node = queue.popleft()
        count = 0
        for i in route[node]:
            if visit[i] == 1:
                count += 1
                if count > 1:
                    flag = False
                    break
            else:
                queue.append(i)
                visit[i] = 1
    print("tree" if flag and sum(visit) == nn else "graph")


import unittest
from processor import persist
import psycopg2
import json

class PersistDataTest(unittest.TestCase):
    def test_read_from_pg(self):
        dbObject = persist.PersistData("postgres")
        courses = dbObject.read_from_pg("futureschema.futurex_course_catalog")
        print(len(courses[0]))
        self.assertEqual(len(courses[0]),5)

    def test_read_from_pg_2(self):
        dbObject = persist.PersistData("postgres")

        with self.assertRaises(psycopg2.errors.UndefinedTable):
            dbObject.read_from_pg("futureschema.table_invalid")

    def test_write_from_json_to_pg(self):
        dbObject = persist.PersistData("postgres")

        with self.assertRaises(KeyError):
            dbObject.write_from_json_to_pg("futureschema.futurex_course_catalog",
                                           {})

    def test_write_from_json_to_pg_2(self):
        dbObject = persist.PersistData("postgres")
        course_json = json.dumps({"course_name": "New course 20", "author_name": "futurex","course_section": {"section1": ' \
                      '"Value", "section2": "Value3"},"creation_date": "2021-03-20"} )

        dbObject.write_from_json_to_pg("futureschema.futurex_course_catalog",
                                           course_json)

if __name__ == '__main__':
    unittest.main()


import torch
from torch.nn import Module
import abc
from typing import Dict, Optional, Callable
from torch.optim.optimizer import Optimizer, required
import math

def exponential_learning_rate_decay(base_lr, decay_rate, decay_steps, staircase):
    def call(round_i):
        if staircase:
            return base_lr * (decay_rate ** (round_i // decay_steps))
        else:
            return base_lr * (decay_rate ** (round_i / decay_steps))
    return call

def inverse_linear_decay_learning_rate(base_lr, decay_rate, decay_steps, staircase):
    def call(round_i):
        if staircase:
            return base_lr / (1.0 + decay_rate * (round_i // decay_steps))
        else:
            return base_lr / (1.0 + decay_rate * (round_i / decay_steps))
    return call

def inverse_sqrt_decay_learning_rate(base_lr, decay_rate, decay_steps, staircase):
    def call(round_i):
        if staircase:
            return base_lr / math.sqrt(1.0 + decay_rate * (round_i // decay_steps))
        else:
            return base_lr / math.sqrt(1.0 + decay_rate * (round_i / decay_steps))
    return call

def warmup_learning_rate(base_lr: float, warmup_steps: int, fn: Callable[[int], float]) -> Callable[[int], float]:
    if warmup_steps is None or warmup_steps <= 0:
        def call(round_i) -> float:
            return fn(round_i)
        return call
    else:
        def warmup_and_decay_fn(round_num) -> float:
            warmedup_value = base_lr * (round_num + 1) / warmup_steps
            if round_num < warmup_steps:
                return warmedup_value
            else:
                return fn(round_num - warmup_steps)
        return warmup_and_decay_fn

class AdaptiveOptimizer(Optimizer):
    """
    基于论文 Adaptive Federated Optimization实现的服务端的优化器
    """

    def __init__(self, global_model: Module,
                 lr=required,
                 lr_decay_policy: str='constant',
                 decay_rate=None,
                 decay_steps=None,
                 staircase=False,
                 warmup_steps=None,
                 more_defaults: Optional[Dict]=None,
                 weight_decay=0,
                 partial_weight_decay=False):
        """
        创建对象
        :param global_model: 模型
        :param lr:
        :param momentum:
        :param dampening:
        :param weight_decay:
        :param nesterov:
        """
        if lr_decay_policy == 'constant':
            scheduler = warmup_learning_rate(lr, warmup_steps=warmup_steps, fn=lambda _: lr)
        elif lr_decay_policy == 'exp_decay':
            scheduler = warmup_learning_rate(lr, warmup_steps, exponential_learning_rate_decay(lr, decay_rate=decay_rate, decay_steps=decay_steps, staircase=staircase))
        elif lr_decay_policy == 'inv_lin':
            scheduler = warmup_learning_rate(lr, warmup_steps, inverse_linear_decay_learning_rate(lr, decay_rate=decay_rate, decay_steps=decay_steps, staircase=staircase))
        elif lr_decay_policy == 'inv_sqrt':
            scheduler = warmup_learning_rate(lr, warmup_steps, inverse_sqrt_decay_learning_rate(lr, decay_rate=decay_rate, decay_steps=decay_steps, staircase=staircase))
        else:
            raise ValueError(
                'Unrecognized schedule type {!s}'.format(lr_decay_policy))
        self.scheduler = scheduler
        if lr is not required and lr < 0.0:
            raise ValueError("Invalid learning rate: {}".format(lr))
        if weight_decay < 0.0:
            raise ValueError("Invalid weight_decay value: {}".format(weight_decay))

        defaults = dict(lr=lr, weight_decay=weight_decay)
        # 获取模型参数的相关信息, 记录了当前参数的名称, 方便从计算后的 weight diff 中找到对应的参数名
        param_names = []
        params = []
        weight_decay_mask = []
        for k, v in global_model.named_parameters():
            param_names.append(k)
            params.append(v)
            # 检查相关的参数是否需要 wd
            if getattr(v, 'enable_weight_decay', False):
                weight_decay_mask.append(True)
            else:
                if partial_weight_decay:
                    weight_decay_mask.append(False)
                else:
                    weight_decay_mask.append(True)
        defaults.update(param_names=param_names, weight_decay_mask=weight_decay_mask)
        if more_defaults is not None:
            defaults.update(more_defaults)
        super(AdaptiveOptimizer, self).__init__(params, defaults)

    def step_lr_scheduler(self, round_i):
        lr = self.scheduler(round_i)
        for p in self.param_groups:
            p['lr'] = lr

    def get_current_lr(self):
        return self.param_groups[0]['lr']

    @torch.no_grad()
    def step(self, closure=None):
        raise NotImplemented('Must implement the optimizer steps')

    @abc.abstractmethod
    def step_pseudo_grads(self, pseudo_grads: Dict[str, torch.Tensor]):
        """
        :param pseudo_grads: 加权聚合后的模型的参数(在联邦学习中包括了buffer和param)
        :return:
        """
        pass

if __name__ == '__main__':
    scheduler = warmup_learning_rate(1.0, 10, exponential_learning_rate_decay(1.0, decay_rate=0.9,
                                                                                decay_steps=10,
                                                                                staircase=False))
    lr0 = scheduler(0)
    lr1 = scheduler(9)
    lr2 = scheduler(20)
    print(lr0, lr1, lr2)


import curses, os
from copy import deepcopy

from .tile   import Tile
from .world  import World
from .enemy  import Enemy
from .camera import Camera
from .player import Player
from .queue  import queue
from .item   import Item
from . import bindings
from . import views
from . import colors

class Game(object):
    """ Main game class. Call run with curses.wrapper to start. """

    def __init__(self):
        # Get all items
        items_file = os.path.join(os.path.dirname(__file__), 'data/items.json')
        self.items = Item.get_all_json(open(items_file))

        # Set the queue screen position
        queue.set_position(3, 20)
        queue.lines = 7

        """ World setup. """
        # Current floor in the game, 0 is the first floor, the index for self.worlds
        self.world_index = 0

        # Dict containing each world in the game
        self.worlds = {}

        # Setup the first world
        self.worlds[0] = World.Dungeon_World(width=1000, height=1000,
                room_x = 500, room_y = 500)

        # Make first world always light
        self.worlds[0].dark = False

        self.camera = Camera()

        # Setup the player
        self.player = Player()
        #self.player.random_floor_tile(self.world)
        self.player.x = self.world.width//2 + 2
        self.player.y = self.world.height//2 + 2

        self.player.equip(deepcopy(self.items['steel longsword']))
        self.player.add_item(deepcopy(self.items['worn axe']))
        self.player.add_item(deepcopy(self.items['apple']))

        knives = deepcopy(self.items['throwing knives'])
        knives.quantity = 5
        self.player.add_item(knives)

        self.world.add_entity(self.player)

        self.camera.center_on(self.player, self.world)

        # Make sure player is visible
        if not self.camera.is_visible(self.player):
            raise Exception()

        # Equip all enemies with knives
        for entity in self.world.entities:
            if entity.tag == 'enemy':
                entity.equip(deepcopy(self.items['rusty knife']))

    def run(self, window):
        """ Run main curses game with curses window. """
        self.window = window
        self.window.keypad(1)
        curses.curs_set(0)

        if curses.has_colors():
            colors.start_colors()

        while True:
            if self.update() == "quit":
                return

    def update(self):
        self.draw()

        key = self.window.getkey()

        # Get function depending on key pressed
        fn = bindings.key_functions.get(key)
        # Only execute if function exists for that key
        if fn is not None:
            res = fn(self)
            if res == 'quit':
                return 'quit'

    def draw(self):
        """ Draw the game world and UI elements. """
        self.window.clear()
        # Draw heads up display
        views.hud(self)
        # Refresh the messages on screen
        queue.draw(self.window)
        # Draw the map
        self.camera.draw(self.window, self.world, point=(self.player.x, self.player.y))

    @property
    def world(self):
        """ Get the current game world at world_index.

        Return None of no world at self.world_index.
        """
        return self.worlds.get(self.world_index)

    @world.setter
    def world(self, value):
        """ Set the current world at world_index. """
        self.worlds[self.world_index] = value

def main():
    game = Game()
    curses.wrapper(game.run)

if __name__ == '__main__':
    main()


from json import loads
from urllib.request import urlopen
from re import (
    compile as re_compile,
    search as re_search
)
from hashlib import sha256

def http_get_dict(url: str) -> dict:
    return loads(urlopen(url, timeout=10).read())

def get_latest_alpine() -> str:
    data = http_get_dict(
        "https://hub.docker.com/v2/repositories/library/alpine/tags/?page_size=25&page=1"
    )

    latest_digest = None
    for image in data.get("results"):
        if image.get("name") == "latest":
            latest_digest = image.get("digest")
            break

    if latest_digest is None:
        exit("Faild to get latest alpine version")

    version = None
    for image in data.get("results"):
        if len(image.get("name").split(".")) == 3:
            if image.get("digest") == latest_digest:
                version = image.get("name")
                break

    if version is None:
        exit("Faild to get latest alpine version")

    return version

def get_latest_pip() -> dict:
    data = http_get_dict("https://pypi.org/pypi/pip/json")

    version = data.get("info").get("version")

    for package in data.get("releases").get(version):
        if package.get("packagetype") == "bdist_wheel":
            return {
                "version": version,
                "url": package.get("url"),
                "sha256": package.get("digests").get("sha256"),
                "filename": package.get("filename")
            }

    exit("Faild to get latest pip version")

def get_latest_pypy() -> dict:
    data = http_get_dict(
        "https://foss.heptapod.net/api/v4/projects/76/repository/tags")

    tag_list = []
    for tag in data:
        tag_list.append(tag.get("name"))

    tags = "\n".join(tag_list)
    version_pattern = re_compile(
        r"release-pypy\d\.\d-v(?P<version>\d*\.\d*\.\d*)"
    )

    result = re_search(version_pattern, tags)
    groups = result.groupdict()

    if groups.get("version") is None:
        exit("Faild to get latest pypy version")

    return groups.get("version")

def get_pypy_archive_sha256(python_version: str, pypy_version: str):
    return sha256(
        urlopen(
            f"https://foss.heptapod.net/pypy/pypy/-/archive/release-pypy{python_version}-v{pypy_version}/pypy-release-pypy{python_version}-v{pypy_version}.tar.bz2",
            timeout=25
        ).read()
    ).hexdigest()

def main() -> None:
    latest_pip = get_latest_pip()
    latest_pypy = get_latest_pypy()
    print(f"""
ALPINE_VERSION = "{get_latest_alpine()}"

pypy_version = "{latest_pypy}"
pypy2_7_sha256 = "{get_pypy_archive_sha256("2.7", latest_pypy)}"
pypy3_8_sha256 = "{get_pypy_archive_sha256("3.8", "7.3.11")}"
pypy3_9_sha256 = "{get_pypy_archive_sha256("3.9", latest_pypy)}"
pypy3_10_sha256 = "{get_pypy_archive_sha256("3.10", latest_pypy)}"

python2_pip_version = "20.3.4"
python2_pip_sha256 = "217ae5161a0e08c0fb873858806e3478c9775caffce5168b50ec885e358c199d"
python2_pip_whl_name = "pip-20.3.4-py2.py3-none-any.whl"
python2_pip_downlaod_url = "https://files.pythonhosted.org/packages/27/79/8a850fe3496446ff0d584327ae44e7500daf6764ca1a382d2d02789accf7/pip-20.3.4-py2.py3-none-any.whl"

python3_pip_version = "{latest_pip.get("version")}"
python3_pip_sha256 = "{latest_pip.get("sha256")}"
python3_pip_whl_name = "{latest_pip.get("filename")}"
python3_pip_downlaod_url = "{latest_pip.get("url")}"

pycparser_version = "2.21"
pycparser_whl_name = "pycparser-2.21-py2.py3-none-any.whl"
pycparser_sha256 = "8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"
pycparser_downlaod_url = "https://files.pythonhosted.org/packages/62/d5/5f610ebe421e85889f2e55e33b7f9a6795bd982198517d912eb1c76e1a53/pycparser-2.21-py2.py3-none-any.whl"
""")

if __name__ == "__main__":
    main()


from collections import deque
import numpy as np

class Memory():

    def __init__(self, MaxSize = 10000):
        self.memory = deque(maxlen = MaxSize)

    def append(self, elements):
        self.memory.append(elements)

    def sample(self, batch_size = 32):

        indices = np.random.choice(len(self.memory), batch_size, replace = False)
        states, actions, rewards, dones, next_states = zip(*[self.memory[idx] for idx in indices])
        return np.array(states),\
               np.array(actions),\
               np.array(rewards),\
               np.array(dones),\
               np.array(next_states)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import os, sys
import numpy as np
import scipy.misc
import time
import argparse

import image_utils
import tfutils_reader
from tfutils_reader import TfutilsReader
from instance_stats import analysis

def sample_bbox(orig_shape):
    bbox = tf.constant([0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4])
    sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box(
            orig_shape,
            bounding_boxes=bbox,
            min_object_covered=0.1,
            aspect_ratio_range=(3. / 4, 4. / 3.),
            area_range=(0.2, 1.0),
            max_attempts=100,
            use_image_if_no_bounding_boxes=True)
    bbox_begin, bbox_size, bbox = sample_distorted_bounding_box
    return bbox[0][0]

def crop_and_resize_img(img, batch_size):
    bboxes = [sample_bbox(img.shape) for i in range(batch_size)]
    bboxes = tf.stack(bboxes, axis=0)
    img_batch = tf.stack([img for _ in range(batch_size)])
    print(img_batch.get_shape().as_list())
    print(bboxes.get_shape().as_list())
    return tf.image.crop_and_resize(
        img_batch,
        boxes=bboxes,
        box_ind=tf.constant(range(batch_size)),
        crop_size=tf.constant([224,224]))

def compute_gradients(img_idx, coord):
    outpath = ('/mnt/fs3/azhai/derivatives/epoch100/'
               'idx%i/d%i.npy' % (args.img_idx, coord))
    img = image_utils.get_training_img_by_index(img_idx)
    img = img.as_tensor()
    print(img.shape)

    sess = tf.Session()
    input_var = tf.get_variable(
        "input_image", initializer=tf.zeros(img.shape),
        dtype=tf.float32)
    sess.run(tf.assign(input_var, img))

    batch_size = 64
    cropped = crop_and_resize_img(input_var, batch_size)
    print(cropped.shape, cropped.dtype)

    tr = TfutilsReader('instance_task', 'control', 'full',
                       port=27009, cache_dir='/data/azhai/tmp')
    epoch_num = 100
    model = tfutils_reader.load_model(
        sess, tr, 10009 * epoch_num,
        input_handle=cropped
    )
    model_output = model['embedding']
    model_input = model['input_handle']

    N = 16
    total_grad = np.zeros_like(img, dtype=np.float32)

    last_time = time.time()
    for i in range(N):
        tot = tf.reduce_sum(model_output, axis=0)
        grad = sess.run(tf.gradients(tot[coord], input_var))
        grad = grad[0]
        assert grad.shape == total_grad.shape
        total_grad += grad

        print('Processed trial', i, 'elapsed =', time.time() - last_time)
        last_time = time.time()

    total_grad /= (N * batch_size)
    np.save(open(outpath, 'w'), total_grad / N)

def save_tensor_to_img(tensor, outpath):
    scipy.misc.imsave(outpath, np.absolute(tensor))

def do_pca(img_idx, num_components=1):
    shape = None
    derivs = []
    for i in range(128):
        row = np.load('/mnt/fs3/azhai/derivatives/epoch100/idx%i/d%i.npy'
                      % (img_idx, i))
        shape = row.shape
        row = np.ndarray.flatten(row)
        derivs.append(row)
        print('loaded row', i)
    derivs = np.stack(derivs)

    weights, _, topvecs = analysis.pca_analysis(derivs)
    print(topvecs[0].shape)

    for i in range(num_components):
        vec = topvecs[i].reshape(shape)
        outpath = ('/mnt/fs3/azhai/derivatives/epoch100/idx%i/grad%i.jpg'
                   % (img_idx, i))
        save_tensor_to_img(vec, outpath)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description="Calculate averaged gradients over random croppings.")
    parser.add_argument('--gradient_pca', action='store_true')
    parser.add_argument('--num_components', type=int)
    parser.add_argument('--gpu', type=str)
    parser.add_argument('--coord', type=int)
    parser.add_argument('--img_idx', type=int)
    args = parser.parse_args()

    if args.gradient_pca:
        do_pca(args.img_idx, num_components=args.num_components)
    else:
        os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
        compute_gradients(args.img_idx, args.coord)


from fake_useragent import UserAgent
import requests

ua = UserAgent()
print(ua.chrome)
header = {'User-Agent':str(ua.chrome)}
print(header)
url = "https://www.hybrid-analysis.com/recent-submissions?filter=file&sort=^timestamp"
htmlContent = requests.get(url, headers=header)
print(htmlContent)


from window import Window
from windowstringreader import WindowStringReader
from partitionpane import PartitionPane
from readmultext import ReadMulText
from confirmwindow import ConfirmWindow
from actionresult import ActionResult
from device import Device
from installer import BIOSSIZE,ESPSIZE

class CustomPartition(object):
    def __init__(self, maxy, maxx, install_config):
        self.maxx = maxx
        self.maxy = maxy
        self.win_width = maxx - 4
        self.win_height = maxy - 4
        self.install_config = install_config
        self.path_checker = []

        self.win_starty = (self.maxy - self.win_height) // 2
        self.win_startx = (self.maxx - self.win_width) // 2

        self.text_starty = self.win_starty + 4
        self.text_height = self.win_height - 6
        self.text_width = self.win_width - 6
        self.cp_config = {}
        self.cp_config['partitionsnumber'] = 0
        self.devices = None
        self.has_slash = False
        self.has_remain = False
        self.has_empty = False

        self.disk_size = []
        self.disk_to_index = {}

        self.window = Window(self.win_height, self.win_width, self.maxy, self.maxx,
                             'Welcome to the Photon installer', False, can_go_next=False)
        Device.refresh_devices()

    def initialize_devices(self):
        self.devices = Device.refresh_devices_bytes()

        # Subtract BIOS&ESP SIZE from the disk_size since this much is hardcoded for bios
        # and efi partition in installer.py
        for index, device in enumerate(self.devices):
            self.disk_size.append((device.path, int(device.size) / 1048576 - (BIOSSIZE + ESPSIZE + 2)))
            self.disk_to_index[device.path] = index

    def display(self):
        self.initialize_devices()

        if 'autopartition' in self.install_config and self.install_config['autopartition'] == True:
            return ActionResult(True, None)

        self.device_index = self.disk_to_index[self.install_config['disk']]

        self.disk_buttom_items = []
        self.disk_buttom_items.append(('<Next>', self.next))
        self.disk_buttom_items.append(('<Create New>', self.create_function))
        self.disk_buttom_items.append(('<Delete All>', self.delete_function))
        self.disk_buttom_items.append(('<Go Back>', self.go_back))

        self.text_items = []
        self.text_items.append(('Disk', 20))
        self.text_items.append(('Size', 5))
        self.text_items.append(('Type', 5))
        self.text_items.append(('Mountpoint', 20))
        self.table_space = 5

        title = 'Current partitions:\n'
        self.window.addstr(0, (self.win_width - len(title)) // 2, title)

        info = ("Unpartitioned space: " +
                str(self.disk_size[self.device_index][1])+
                " MB, Total size: "+
                str(int(self.devices[self.device_index].size)/ 1048576) + " MB")

        self.partition_pane = PartitionPane(self.text_starty, self.maxx, self.text_width,
                                  self.text_height, self.disk_buttom_items,
                                  config=self.cp_config,
                                  text_items=self.text_items, table_space=self.table_space,
                                  info=info,
                                  size_left=str(self.disk_size[self.device_index][1]))

        self.window.set_action_panel(self.partition_pane)

        return self.window.do_action()

    def validate_partition(self, pstr):
        if not pstr:
            return ActionResult(False, None)
        sizedata = pstr[0]
        mtdata = pstr[2]
        typedata = pstr[1]
        devicedata = self.devices[self.device_index].path

        #no empty fields unless swap
        if (typedata == 'swap' and
                (len(mtdata) != 0 or len(typedata) == 0 or len(devicedata) == 0)):
            return False, "invalid swap data "

        if (typedata != 'swap' and
                (len(sizedata) == 0 or
                 len(mtdata) == 0 or
                 len(typedata) == 0 or
                 len(devicedata) == 0)):
            if not self.has_empty and mtdata and typedata and devicedata:
                self.has_empty = True
            else:
                return False, "Input cannot be empty"

        if typedata not in ['swap', 'ext3', 'ext4', 'xfs', 'btrfs']:
            return False, "Invalid type"

        if len(mtdata) != 0 and mtdata[0] != '/':
            return False, "Invalid path"

        if mtdata in self.path_checker:
            return False, "Path already existed"
        #validate disk: must be one of the existing disks
        i = self.device_index

        #valid size: must not exceed memory limit
        curr_size = self.disk_size[i][1]
        if len(sizedata) != 0:
            try:
                int(sizedata)
            except ValueError:
                return False, "invalid device size"

            if int(curr_size) - int(sizedata) < 0:
                return False, "invalid device size"
            #if valid, update the size and return true
            new_size = (self.disk_size[i][0], int(curr_size)- int(sizedata))
            self.disk_size[i] = new_size

        if mtdata == "/":
            self.has_slash = True

        self.path_checker.append(mtdata)
        return True, None

    def create_function(self):
        self.window.hide_window()

        self.cp_config['partition_disk'] = self.devices[self.device_index].path
        self.partition_items = []
        self.partition_items.append(('Size in MB: ' +
                                     str(self.disk_size[self.device_index][1]) +
                                     ' available'))
        self.partition_items.append(('Type: (ext3, ext4, xfs, btrfs, swap)'))
        self.partition_items.append(('Mountpoint:'))
        self.create_window = ReadMulText(
            self.maxy, self.maxx, 0,
            self.cp_config,
            str(self.cp_config['partitionsnumber']) + 'partition_info',
            self.partition_items,
            None,
            None,
            None,
            self.validate_partition,   #validation function of the input
            None,
            True,
            )
        result = self.create_window.do_action()
        if result.success:
            self.cp_config['partitionsnumber'] = self.cp_config['partitionsnumber'] + 1

        #parse the input in install config
        return self.display()

    def delete_function(self):
        self.delete()
        return self.display()

    def go_back(self):
        self.delete()
        self.window.hide_window()
        self.partition_pane.hide()
        return ActionResult(False, {'goBack':True})

    def next(self):
        if self.cp_config['partitionsnumber'] == 0:
            window_height = 9
            window_width = 40
            window_starty = (self.maxy-window_height) // 2 + 5
            confirm_window = ConfirmWindow(window_height, window_width, self.maxy,
                                           self.maxx, window_starty,
                                           'Partition information cannot be empty',
                                           info=True)
            confirm_window.do_action()
            return self.display()
        #must have /
        if not self.has_slash:
            window_height = 9
            window_width = 40
            window_starty = (self.maxy - window_height) // 2 + 5
            confirm_window = ConfirmWindow(window_height, window_width, self.maxy,
                                           self.maxx, window_starty, 'Missing /',
                                           info=True)
            confirm_window.do_action()
            return self.display()

        self.window.hide_window()
        self.partition_pane.hide()

        partitions = []
        for i in range(int(self.cp_config['partitionsnumber'])):
            if len(self.cp_config[str(i)+'partition_info'+str(0)]) == 0:
                sizedata = 0
            else:
                sizedata = int(self.cp_config[str(i) + 'partition_info' + str(0)])
            mtdata = self.cp_config[str(i) + 'partition_info' + str(2)]
            typedata = self.cp_config[str(i) + 'partition_info'+str(1)]

            partitions = partitions + [{"mountpoint": mtdata,
                                        "size": sizedata,
                                        "filesystem": typedata},]
        self.install_config['partitions'] = partitions

        return ActionResult(True, {'goNext':True})

    def delete(self):
        for i in range(int(self.cp_config['partitionsnumber'])):
            self.cp_config[str(i)+'partition_info'+str(0)] = ''
            self.cp_config[str(i)+'partition_info'+str(1)] = ''
            self.cp_config[str(i)+'partition_info'+str(2)] = ''
            self.cp_config[str(i)+'partition_info'+str(3)] = ''
        del self.disk_size[:]
        for index, device in enumerate(self.devices):
            self.disk_size.append((device.path, int(device.size) / 1048576 - (BIOSSIZE + ESPSIZE + 2)))
        del self.path_checker[:]
        self.has_slash = False
        self.has_remain = False
        self.has_empty = False
        self.cp_config['partitionsnumber'] = 0


'''
Created on 9. juli 2012

@author: Joakim
'''
import cgi
#import logging

def smartEscape(s):
    s = cgi.escape(s, quote=True)
    #logging.error(s)
    s = s.replace("<br>", '
') # newline
    s = s.replace("<b>", '').replace("</b>", '') # blod
    s = s.replace("<h2>", '
').replace("</h2>", '
') # h2
    #logging.error(s)
    return s


from tensorflow import keras
from tensorflow.keras import layers
from utils.metrics import get_disparate_impact
from models.flip_gradient_tf import GradReverse

class FRTrain:
    def __init__(self, input_sizes, output_sizes, weights):
        self.input_sizes = input_sizes
        self.output_sizes = output_sizes
        self.latent_dim = 64
        self.weights = weights
        parts = self._build_parts()
        self.encoder = parts[0]
        self.model = self._compose_parts(parts)

    def _build_parts(self):
        enc_inp = keras.Input(shape=(self.input_sizes[0],), name='img_input')
        protected_input = keras.Input(shape=(self.input_sizes[1]))
        full_enc_inp = layers.Concatenate()([enc_inp, protected_input])
        feature_enc = layers.Dense(self.latent_dim, activation="relu")(full_enc_inp)
        encoder = keras.Model(inputs=[enc_inp, protected_input], outputs=feature_enc, name='encoder')

        pred_inp = keras.Input(shape=(self.latent_dim,), name='pre_input')
        y = layers.Dense(self.output_sizes[0], activation='softmax')(pred_inp)
        pred = keras.Model(inputs=pred_inp, outputs=y, name='predictor')

        adv_fair_inp = keras.Input(shape=(self.output_sizes[0],), name='adv_input')  # Pred
        # a = layers.Dense(64, activation='relu')(adv_fair_inp)
        # a = layers.Dense(self.output_sizes[1], activation='softmax')(a)
        x = layers.Dense(128, activation="relu")(adv_fair_inp)
        x = layers.BatchNormalization()(x)
        x = layers.Dense(128, activation="relu")(x)
        x = layers.BatchNormalization()(x)
        a = layers.Dense(self.output_sizes[1], activation='softmax')(x)
        fair_adv = keras.Model(inputs=adv_fair_inp, outputs=a, name='fairAdv')

        adv_robust_inp = keras.Input(shape=(self.input_sizes[0] + sum(self.output_sizes[:2]),))
        x = layers.LeakyReLU(0.2)(adv_robust_inp)
        a2 = layers.Dense(self.output_sizes[2], activation='softmax')(x)
        adv_robust = keras.Model(inputs=adv_robust_inp, outputs=a2, name="robustAdv")

        return encoder, pred, fair_adv, adv_robust

    def _compose_parts(self, parts):
        encoder, pred, f_adv, r_adv = parts
        x = keras.Input(shape=(self.input_sizes[0],))
        s = keras.Input(shape=(self.input_sizes[1]))
        z = encoder([x, s])
        y = pred(z)

        flipped_y = GradReverse()(y)
        fair = f_adv(flipped_y)

        r_inp = layers.Concatenate()([flipped_y, s, x])
        robust = r_adv(r_inp)
        overall = keras.Model(inputs=[x, s], outputs=[y, fair, robust])
        overall.compile(optimizer='adam',
                        loss=['categorical_crossentropy', 'categorical_crossentropy', 'categorical_crossentropy'],
                        loss_weights=self.weights,
                        metrics=['accuracy'])
        return overall

    def train(self, training_data, num_epochs):
        inps, outs = training_data
        self.model.fit(inps, outs, batch_size=16, epochs=num_epochs, verbose=0, validation_split=0.0)

    def evaluate(self, x, y_one_hot, ys):
        eval_results = self.model.evaluate(x, y_one_hot)
        print("Eval results", eval_results)
        predictions = self.model.predict(x)
        di, dd, s_acc = get_disparate_impact(predictions[0], predictions[0], ys)
        return eval_results[-3], di, dd, s_acc


"""IAM Database queries that are not specific to either the Actions, ARNs, or Condition Keys tables."""
import logging
import functools
from policy_sentry.shared.iam_data import iam_definition, get_service_prefix_data

logger = logging.getLogger(__name__)

@functools.lru_cache(maxsize=1024)
def get_all_service_prefixes():
    """
    Gets all the AWS service prefixes from the actions table.

    If the action table does NOT have specific IAM actions (and therefore only supports * actions),
    then it will not be included in the response.

    Returns:
        List: A list of all AWS service prefixes present in the table.
    """
    # results = [d["prefix"] for d in iam_definition]
    results = list(set(iam_definition.keys()))
    results.sort()
    return results

@functools.lru_cache(maxsize=1024)
def get_all_actions(lowercase=False):
    """
    Gets a huge list of all IAM actions. This is used as part of the policyuniverse approach to minimizing
    IAM Policies to meet AWS-mandated character limits on policies.

    :param lowercase: Set to true to have the list of actions be in all lowercase strings.
    :return: A list of all actions present in the database.
    """
    all_actions = set()

    all_service_prefixes = get_all_service_prefixes()
    for service_prefix in all_service_prefixes:
        service_prefix_data = get_service_prefix_data(service_prefix)
        for action_name in service_prefix_data["privileges"]:
            if lowercase:
                all_actions.add(
                    f"{service_prefix}:{action_name.lower()}"
                )
            else:
                all_actions.add(
                    f"{service_prefix}:{action_name}"
                )

    # results = list(set(results))
    # results.sort()
    return all_actions


import datetime
from dataclasses import asdict, dataclass

from sqlalchemy import delete, select, text
from sqlalchemy.ext.asyncio import AsyncSession

from air_bot.adapters.repo import orm
from air_bot.domain import model
from air_bot.domain.ports.repository import TicketRepo

@dataclass(kw_only=True)
class TicketDB:
    id: int | None = None
    direction_id: int
    price: float
    departure_at: datetime.datetime
    duration_to: int
    return_at: datetime.datetime | None = None
    duration_back: int | None = None
    link: str

    @staticmethod
    def from_ticket(ticket: model.Ticket, direction_id: int):
        duration_to = ticket.duration_to.seconds // 60
        if ticket.duration_back:
            duration_back = ticket.duration_back.seconds // 60
        else:
            duration_back = None
        return TicketDB(
            direction_id=direction_id,
            price=ticket.price,
            departure_at=ticket.departure_at,
            duration_to=duration_to,
            return_at=ticket.return_at,
            duration_back=duration_back,
            link=ticket.link,
        )

class SqlAlchemyTicketRepo(TicketRepo):
    def __init__(self, session: AsyncSession):
        super().__init__()
        self.session = session

    async def add(self, tickets: list[model.Ticket], direction_id: int):
        for ticket in tickets:
            ticket_db = TicketDB.from_ticket(ticket, direction_id)
            stmt = text(
                "INSERT INTO tickets (direction_id, price, departure_at, duration_to, return_at, "
                "duration_back, link) VALUES (:direction_id, :price, :departure_at, :duration_to, "
                ":return_at, :duration_back, :link)"
            )
            ticket_db_dict = asdict(ticket_db)
            del ticket_db_dict["id"]
            stmt = stmt.bindparams(**ticket_db_dict)
            await self.session.execute(stmt)

    async def get_direction_tickets(
        self, direction_id: int, limit: int | None = None
    ) -> list[model.Ticket]:
        stmt = (
            select(TicketDB)
            .filter_by(direction_id=direction_id)
            .order_by(orm.tickets_table.c.price)
        )
        if limit:
            stmt = stmt.limit(limit)
        result = await self.session.execute(stmt)
        tickets = []
        for row in result:
            ticket_db: TicketDB = row[0]
            duration_to = datetime.timedelta(minutes=ticket_db.duration_to)
            duration_back = None
            if ticket_db.duration_back:
                duration_back = datetime.timedelta(minutes=ticket_db.duration_back)
            ticket = model.Ticket(
                price=ticket_db.price,
                departure_at=ticket_db.departure_at,
                duration_to=duration_to,
                return_at=ticket_db.return_at,
                duration_back=duration_back,
                link=ticket_db.link,
            )
            tickets.append(ticket)
        return tickets

    async def remove_for_direction(self, direction_id: int):
        stmt = delete(TicketDB).where(orm.tickets_table.c.direction_id == direction_id)
        await self.session.execute(stmt)