"""Interpolation objects."""

from collections.abc import Hashable

import numpy as np
import pandas as pd
from scipy import interpolate

from bgc_data_processing.core.storers import Storer
from bgc_data_processing.verbose import with_verbose

class Interpolator:
    """Interpolate slices with common index from a reference dataframe.

    Parameters
    ----------
    base : pd.DataFrame
        DataFrame to use a base data for the interpolation.
    x_column_name : str
        name of the column to use as x for the interpolation.
    kind : str, optional
        Type of interpolation, to pass to scipy.interpolate.interp1d.
        , by default "linear"
    """

    def __init__(
        self,
        base: Storer,
        x_column_name: str,
        y_columns_name: list[str],
        kind: str = "linear",
    ) -> None:
        self._storer = base
        self._columns_order = base.data.columns
        self._x = base.variables.get(x_column_name).label
        self._ys = [base.variables.get(name).label for name in y_columns_name]
        self.kind = kind

    @with_verbose(
        trigger_threshold=2,
        message="Interpolation depth outbound -> keeping max depth value.",
    )
    def _handle_outbound_max(
        self,
        ref_slice: pd.DataFrame | pd.Series,
        obs_depth: float,
        name: Hashable | None,
    ) -> pd.Series:
        """Deal with 'interpolation' when observation depth is outbound.

        Parameters
        ----------
        ref_slice : pd.DataFrame | pd.Series
            Slice of the reference Dataframe to use.
        obs_depth : float
            Depth of the observation.
        name : Hashable | None
            Name for the ouput series (index of the slice).

        Returns
        -------
        pd.Series
            Result for outbound observation.
        """
        if isinstance(ref_slice, pd.Series):
            max_series = ref_slice
        else:
            max_series: pd.Series = ref_slice.iloc[ref_slice[self._x].argmax()]
        max_series[self._x] = obs_depth
        max_series.name = name
        return max_series[self._columns_order]

    @with_verbose(
        trigger_threshold=2,
        message="Interpolation depth outbound -> keeping min depth value.",
    )
    def _handle_outbound_min(
        self,
        ref_slice: pd.DataFrame | pd.Series,
        obs_depth: float,
        name: Hashable | None,
    ) -> pd.Series:
        """Deal with 'interpolation' when observation depth is outbound.

        Parameters
        ----------
        ref_slice : pd.DataFrame | pd.Series
            Slice of the reference Dataframe to use.
        obs_depth : float
            Depth of the observation.
        name : Hashable | None
            Name for the ouput series (index of the slice).

        Returns
        -------
        pd.Series
            Result for outbound observation.
        """
        if isinstance(ref_slice, pd.Series):
            min_series = ref_slice
        else:
            min_series: pd.Series = ref_slice.iloc[ref_slice[self._x].argmin()]
        min_series[self._x] = obs_depth
        min_series.name = name
        return min_series[self._columns_order]

    def _get_columns_to_interp(self, dataframe: pd.DataFrame) -> pd.DataFrame:
        """Return columns to interpolate (non constant columns).

        Parameters
        ----------
        dataframe : pd.DataFrame
            Dataframe to get the columns from.

        Returns
        -------
        pd.DataFrame
            Dataframe slice (on columns).
        """
        return dataframe.loc[:, dataframe.columns.isin(self._ys)]

    def _get_constant_columns(self, dataframe: pd.DataFrame) -> pd.DataFrame:
        """Return columns not to interpolate (single valued columns).

        Parameters
        ----------
        dataframe : pd.DataFrame
            Dataframe to get the columns from.

        Returns
        -------
        pd.DataFrame
            Dataframe slice (on columns).
        """
        return dataframe.loc[:, ~dataframe.columns.isin(self._ys)]

    @with_verbose(
        trigger_threshold=2,
        message="NaN depth encountered -> setting values to NaN.",
    )
    def _handle_nan_depth(
        self,
        ref_slice: pd.DataFrame,
        obs_depth: float,
        name: Hashable | None,
    ) -> pd.Series:
        """Deal with 'interpolation' when observation depth is nan.

        Parameters
        ----------
        ref_slice : pd.DataFrame
            Slice of the reference Dataframe to use.
        obs_depth : float
            Depth of the observation.
        name : Hashable | None
            Name for the ouput series (index of the slice).

        Returns
        -------
        pd.Series
            NaN series.
        """
        to_interp = self._get_columns_to_interp(ref_slice)
        constant_columns = self._get_constant_columns(ref_slice)
        constant_values = constant_columns.iloc[0, :]
        non_constant_values = pd.Series(
            np.nan,
            index=to_interp.columns,
        )
        result = pd.concat([constant_values, non_constant_values])
        result.name = name
        result[self._x] = obs_depth
        return result[self._columns_order]

    @with_verbose(
        trigger_threshold=2,
        message="Valid interpolation depth -> interpolation.",
    )
    def _interpolate(
        self,
        ref_slice: pd.DataFrame,
        obs_depth: float,
        name: Hashable | None,
    ):
        """Interpolate data.

        Parameters
        ----------
        ref_slice : pd.DataFrame
            Slice of the reference Dataframe to use.
        obs_depth : float
            Depth of the observation.
        name : Hashable | None
            Name for the ouput series (index of the slice).

        Returns
        -------
        pd.Series
            Interpolated series.
        """
        non_constant = self._get_columns_to_interp(ref_slice)
        constant_columns = self._get_constant_columns(ref_slice)
        constant_values = constant_columns.iloc[0, :]
        interpolation = interpolate.interp1d(
            x=ref_slice[self._x],
            y=non_constant,
            axis=0,
            fill_value=np.nan,
            kind=self.kind,
        )
        non_constant_values = pd.Series(
            interpolation(obs_depth),
            index=non_constant.columns,
            name=name,
        )
        concatenated = pd.concat([constant_values, non_constant_values])
        concatenated[self._x] = obs_depth
        return concatenated[self._columns_order]

    @with_verbose(trigger_threshold=1, message="Interpolating row.")
    def interpolate(
        self,
        row: pd.Series,
    ) -> pd.Series:
        """Interpolate a self.reference slice with same index as row.

        This method is mostly meant to be applied using pd.DataFrame.apply method.

        Parameters
        ----------
        row : pd.Series
            Row to use for interpolation.

        Returns
        -------
        pd.Series
            Interpolated series with same depth as row.
        """
        ref_slice: pd.DataFrame = self._storer.data.loc[row.name, :]
        obs_depth = row[self._x]
        ref_depths = ref_slice[self._x]
        if np.isnan(obs_depth):
            return self._handle_nan_depth(ref_slice, obs_depth, row.name)
        if obs_depth > ref_depths.max():
            return self._handle_outbound_max(ref_slice, obs_depth, row.name)
        if obs_depth < ref_depths.min():
            return self._handle_outbound_min(ref_slice, obs_depth, row.name)
        return self._interpolate(ref_slice, obs_depth, row.name)

    @with_verbose(
        trigger_threshold=0,
        message="Interpolating Data to match observations' depth values.",
    )
    def interpolate_storer(
        self,
        observations_storer: Storer,
    ) -> Storer:
        """Interpolate over all rows a Store's dataframe.

        Parameters
        ----------
        observations_storer : Storer
            Storer to interpolate on each row.

        Returns
        -------
        Storer
            Storer with interpolated rows to match observations_storer's x values.
        """
        obs_data = observations_storer.data
        matching_index = obs_data.loc[self._storer.data.index.unique(), :]
        interpolated_df = matching_index.apply(self.interpolate, axis=1)
        return Storer(
            data=interpolated_df,
            category=self._storer.category,
            providers=self._storer.providers,
            variables=self._storer.variables,
        )


import cv2  # Not actually necessary if you just want to create an image.
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

height = 10
width = 10
count = 0
interval = 16

for R in range(0, 256 , interval):
    for G in range(0, 256 , interval):
        for B in range(0, 256 , interval):
            #image = np.zeros((height, width, 3), np.uint8)
            '''another way to fill image'''
            #image[:] = (255,0,0)

            image = np.full((height ,width ,3),(R,G,B),dtype=np.uint8)
            '''uint8 range is 0 to 255'''

            #print(image)
            #print(image.shape)
            image_name = str(count)+".png"
            print(image_name)

            if(count%100==0):
                plt.imshow(image)
                plt.title(image_name)
                plt.show(block=False)
                plt.axis('off')
                plt.pause(0.000001)
                plt.clf()

            cv2.imwrite(str(count)+'.png', image)
            count=count+1


import os
import pipes
from subprocess import Popen, PIPE, STDOUT

import grp

import pwd
from jupyterhub.auth import LocalAuthenticator
from oauthenticator import GoogleOAuthenticator
from traitlets import Unicode

class CustomLocalAuthenticator(LocalAuthenticator):
    environment_config_file = Unicode(
        '',
        help="""Path to an environment configuration file that can be used
        to create environments for each user created in the system.
        """).tag(config=True)

    def add_system_user(self, user):
        """
        Create a new local UNIX user on the system.
        Create default environment for user from the specified environment file
        """
        super().add_system_user(user)

        self.create_default_conda_environment(user)

    def create_default_conda_environment(self, user):

        """
        Create environment for user from the specified environment file
        Append environment to txt file so it appears in jupyter as a kernel
        option
        """

        if self.environment_config_file:

            name = user.name

            cmd = ['su', '-', name, "-c",
                   "/opt/anaconda/bin/conda-env create -f {0} -v "
                   "--force".format(self.environment_config_file, name)
            ]

            self.log.info("Creating user environment from config: %s",
                          ' '.join(map(
                              pipes.quote, cmd)))

            p = Popen(cmd, stdout=PIPE, stderr=STDOUT)

            stdout, stderr = p.communicate()

            if p.returncode:
                err = p.stdout.read().decode('utf8', 'replace')
                raise RuntimeError(
                    "Failed to create user default environment "
                    "{0}: {1}".format(name, err)
                )

            location = '/data/{0}/.conda/envs/local\n'.format(name)

            with open("/data/{0}/.conda/environments.txt".format(name), "a+") \
                    as file:

                for line in file:
                    if line == location:
                        break
                else:  # not found, we are at the eof
                    file.write(location)  # append missing data

            uid = pwd.getpwnam(name).pw_uid
            gid = grp.getgrnam(name).gr_gid

            os.chown("/data/{0}/.conda/environments.txt".format(name), uid, gid)

class CustomGoogleOAuthenticator(
    CustomLocalAuthenticator, GoogleOAuthenticator):
    """A version that mixes in local system user creation"""
    pass


import os
import sys
import tempfile
import threading
from unittest.mock import MagicMock

import pytest

from rtbot_plugin_manager.plugin_manager import PluginManager, PluginLoadingError
from rtbot_plugin_manager.tests.utils import OpenableNamedTemporaryFile, SysPath

class PluginBase(MagicMock):
    def __init__(self):
        super(PluginBase, self).__init__()
        self.bar = 'bar'

@pytest.fixture
def pm():
    yield PluginManager()

@pytest.fixture
def pm_with_class_selector():
    yield PluginManager(class_selector="PluginBase")

PLUGIN = """
class SimplePlugin:
    def foo(self):
        return 'simple plugin'
"""

PLUGIN_DIFFERENT_IMPLEMENTATION = """
class SimplePlugin:
    def foo(self):
        return 'different implementation'
"""

PLUGIN_WITH_CONSTRUCTOR_EXCEPTION = """
class SimplePlugin:
    def __init__(self):
        raise RuntimeException('Example plugin constructor exception')
"""

PLUGIN_WITH_CONSTRUCTOR_DESTRUCTOR = """
import threading
class SimplePlugin:
    def __init__(self):
        threadlocals = threading.local()
        threadlocals.constructor_called = True
    def __del__(self):   # DESIGN: the __del__ special method is problematic (https://docs.python.org/3.6/reference/datamodel.html#object.__del__), should we declare our own instead, e.g. on_destroy()
        threadlocals = threading.local()
        threadlocals.destructor_called = True
"""

PLUGIN_WITH_REFERENCE = """
import temprefmodule
class SimplePlugin: pass
"""

ANOTHER_PLUGIN = """
class AnotherPlugin: pass
"""

PLUGIN_WITH_ASYNC_METHOD = """
class SimplePlugin(PluginBase):
    async def foo(self, val):
        self.bar(val)
"""

MULTIPLE_PLUGINS_MODULE = """
from rtbot_plugin_manager.tests.test_plugin_manager import PluginBase   # FIXME: is this the right import? And is that module available in the test environment?
class NotAPlugin:
    def __init__(self):
        raise Exception('I should not be instantiated!')
class PluginWithBaseClass(PluginBase):
    def foo(self):
        return 'plugin with base class'
class AlsoNotAPlugin:
    def __init__(self):
        raise Exception('I should not be instantiated!')
"""

PLUGIN_WITH_BASE_CLASS = """
from rtbot_plugin_manager.tests.test_plugin_manager import PluginBase   # FIXME: is this the right import? And is that module available in the test environment?
class PluginWithBaseClass(PluginBase):
    pass
"""

PLUGIN_WITH_SYNTAX_ERROR = """
class PluginWithSyntaxError:
    -
"""

def _assert_single_plugin(pm):
    assert len(pm.repository) == 1
    return list(pm.repository.values())[0]

def test_load_plugin(pm):
    # load plugin and assert its ID
    plugin_id = pm.load_plugin_from_string(PLUGIN)
    assert plugin_id

    # assert that there's exactly one plugin in the registry now
    plugin = _assert_single_plugin(pm)

    # check that the right class was loaded, instantiated and can be invoked
    assert plugin.__class__.__name__ == 'SimplePlugin'
    assert plugin.foo() == 'simple plugin'

def test_load_plugin_from_file(pm):
    # create temporary file to load plugin from
    with OpenableNamedTemporaryFile(mode='w+', suffix='.py', content=PLUGIN) as filename:
        # load plugin and assert its ID
        plugin_id = pm.load_plugin_from_file(filename)
        assert plugin_id

        # assert that there's exactly one plugin in the registry now
        plugin = _assert_single_plugin(pm)

        # check that the right class was loaded, instantiated and can be invoked
        assert plugin.__class__.__name__ == 'SimplePlugin'
        assert plugin.foo() == 'simple plugin'

def test_plugin_loading_by_class_selector(pm_with_class_selector):
    plugin_id = pm_with_class_selector.load_plugin_from_string(MULTIPLE_PLUGINS_MODULE)
    assert plugin_id

    plugin = pm_with_class_selector.repository[plugin_id]

    assert plugin.__class__.__name__ == 'PluginWithBaseClass'
    assert plugin.foo() == 'plugin with base class'

def test_load_and_unload_plugin(pm):
    # load plugin
    plugin_id = pm.load_plugin_from_string(PLUGIN)
    plugin = _assert_single_plugin(pm)
    assert plugin.foo() == 'simple plugin'

    # unload plugin
    assert pm.unload_plugin(plugin_id)

    # check that the plugin repository is empty
    assert len(pm.repository) == 0

def test_load_and_unload_plugin_with_references(pm):
    # create a temp directory, which we'll pretend is an import path later
    with tempfile.TemporaryDirectory() as d:
        # create a helper module, which the plugin will load into the interpreter for us
        with open(os.path.join(d, 'temprefmodule.py'), 'w') as f:
            f.write('print("temprefmodule loaded")')

        # temporarily add our temporary 'library' directory to the PATH
        with SysPath(d):
            # load and unload plugin
            plugin_id = pm.load_plugin_from_string(PLUGIN_WITH_REFERENCE)
            assert pm.unload_plugin(plugin_id)
        # at this point, we're expecting the plugin to be unloaded, *including* its references!

    # first check that the module is no longer in sys.modules (that's the first step, but doesn't conclude that it
    # isn't loaded anymore)
    assert 'temprefmodule' not in sys.modules

    # Here comes the real test: now try to load the module directly: if it's still known to the interpreter ('cached'),
    # it will just load it anyway. If not, it will try to find a file and fail.
    with pytest.raises(ModuleNotFoundError):
        # if this does *not* raise a ModuleNotFoundError, then we know it wasn't properly unloaded
        import temprefmodule

def test_plugins_have_unique_id(pm):
    # load and unload one plugin
    first_id = pm.load_plugin_from_string(PLUGIN)
    assert first_id
    assert pm.unload_plugin(first_id)

    # now load another plugin (not the same plugin with a different implementation)
    second_id = pm.load_plugin_from_string(ANOTHER_PLUGIN)
    assert second_id

    # assert that these two plugins were assigned unique IDs
    assert first_id != second_id

def test_plugin_id_is_independent_from_implementation(pm):
    # load and unload a plugin
    first_id = pm.load_plugin_from_string(PLUGIN)
    assert first_id
    assert pm.unload_plugin(first_id)

    # now load the same plugin with a different implementation
    second_id = pm.load_plugin_from_string(PLUGIN_DIFFERENT_IMPLEMENTATION)
    assert second_id

    # assert that these are considered the same plugin and were assigned the same ID
    assert first_id == second_id

def test_plugin_can_be_reloaded(pm):
    # load plugin and assert its ID
    plugin_id = pm.load_plugin_from_string(PLUGIN)
    assert plugin_id

    # assert that there's exactly one plugin in the registry now
    plugin = _assert_single_plugin(pm)

    # check that the right class was loaded, instantiated and can be invoked
    assert plugin.foo() == 'simple plugin'

    # now load a plugin with the same identification, and expect this to trigger a reload
    assert pm.load_plugin_from_string(PLUGIN_DIFFERENT_IMPLEMENTATION)
    plugin = _assert_single_plugin(pm)
    assert plugin.foo() == 'different implementation'

def test_loading_plugin_calls_hook(pm):
    # TODO: anyone got a better idea than threadlocals (which seems rather unstable for a test case) to check whether these were invoked?

    # reset some thread-local variables
    threadlocals = threading.local()
    threadlocals.constructor_called = False
    threadlocals.destructor_called = False

    # load plugin
    plugin_id = pm.load_plugin_from_string(PLUGIN_WITH_CONSTRUCTOR_DESTRUCTOR)
    assert plugin_id

    # check that the constructor was called
    assert threadlocals.constructor_called

    # unload plugin
    pm.unload_plugin(plugin_id)
    assert threadlocals.destructor_called

def test_fire_plugin_events(pm):
    # load a plugin, which derives from MagicMock
    plugin_id = pm.load_plugin_from_string(PLUGIN_WITH_BASE_CLASS)

    # ask the plugin manager to invoke .foo('bar') on all plugins
    pm.fire_event(lambda plugin: plugin.foo('bar'))

    # check via the MagicMock that foo() has indeed been called once with argument 'bar'
    plugin = pm.repository[plugin_id]
    plugin.foo.assert_called_once_with('bar')

def test_fire_plugin_events_async(pm):
    # load a plugin, which derives from MagicMock
    plugin_id = pm.load_plugin_from_string(PLUGIN_WITH_ASYNC_METHOD)

    # ask the plugin manager to invoke .foo('bar') on all plugins
    async def call(plugin):
        await plugin.foo('bar')
    pm.fire_event_async(call)

    # check via the MagicMock that foo() has indeed been called once with argument 'bar'
    plugin = pm.repository[plugin_id]
    plugin.bar.assert_called_once_with('bar')

def test_load_syntactically_invalid_plugin(pm):
    with pytest.raises(PluginLoadingError) as exc:
        pm.load_plugin_from_string(PLUGIN_WITH_SYNTAX_ERROR)
    assert isinstance(exc.__cause__, SyntaxError)

    # make sure the plugin doesn't appear loaded
    assert 'PluginWithSyntaxError' not in pm.repository
    assert len(pm.repository) == 0

def test_load_plugin_with_error_during_initialization(pm):
    with pytest.raises(PluginLoadingError) as exc:
        pm.load_plugin_from_string(PLUGIN_WITH_CONSTRUCTOR_EXCEPTION)
    assert isinstance(exc.__cause__, RuntimeError)

    # make sure the plugin doesn't appear loaded
    assert 'SimplePlugin' not in pm.repository
    assert len(pm.repository) == 0

def test_reloading_plugin_unloads_original_instance(pm):
    # reset some thread-local variables
    threadlocals = threading.local()
    threadlocals.constructor_called = False
    threadlocals.destructor_called = False

    # load plugin
    assert pm.load_plugin_from_string(PLUGIN_WITH_CONSTRUCTOR_DESTRUCTOR)
    assert threadlocals.constructor_called

    # reloading plugin with a different implementation should unload the original instance
    pm.load_plugin_from_string(PLUGIN)
    assert threadlocals.destructor_called

def test_reload_plugin_with_newly_introduced_syntax_error(pm):
    # load a plugin
    plugin_id = pm.load_plugin_from_string(PLUGIN)
    assert plugin_id

    # now effectively reload the plugin after introducing a syntax error
    with pytest.raises(PluginLoadingError):
        pm.load_plugin_from_string(PLUGIN_WITH_CONSTRUCTOR_EXCEPTION)

    # this should result in the original plugin being unloaded
    assert 'SimplePlugin' not in pm.repository
    assert len(pm.repository) == 0


from tornado.web import RequestHandler
from tornado import gen
from server import ChangePwdServer
from sql import mysql
class ChangePwdHandler(RequestHandler):
    @gen.coroutine
    def post(self):
        stuid=self.get_argument('stuid','none')
        localcookie,oldpwd=self.get_argument('cookie','none'),self.get_argument('oldpwd','none')
        newpwd,newpwd1=self.get_argument('newpwd','none'),self.get_argument('newpwd1','none')
        if localcookie=='none' or oldpwd=='none' or stuid=='none' or newpwd=='none' or newpwd1=='none':
            self.finish("failure")
        res=ChangePwdServer.change_pwd_server(localcookie,oldpwd,newpwd,newpwd1)
        result=yield res
        if result==None:
            self.finish("reqfailure")
        if result.find('success:true')!=-1:
            # change db data
            try:
                mysql.Sqlutils().updateSelective('userlist',stuid,newpwd)
            except Exception:
                self.finish('success')
            self.finish('success')
        else:
            self.finish('failure')


import tensorflow as tf
import numpy as np
import cv2
from numpy import genfromtxt
from keras.layers import Conv2D, ZeroPadding2D, Activation, Input
from keras.models import Model
from keras.layers.normalization import BatchNormalization
from keras.layers.pooling import MaxPooling2D, AveragePooling2D
import h5py
_FLOATX = 'float32'

def shape(x):
    return x.get_shape()

def square(x):
    return tf.square(x)

def concatenate(tensors, axis=-1):
    if axis < 0:
        axis = axis % len(tensors[0].get_shape())
    return tf.concat(axis, tensors)

def LRN2D(x):
    return tf.nn.lrn(x, alpha=1e-4, beta=0.75)

def conv2d_bn(x,
              layer=None,
              cv1_out=None,
              cv1_filter=(1, 1),
              cv1_strides=(1, 1),
              cv2_out=None,
              cv2_filter=(3, 3),
              cv2_strides=(1, 1),
              padding=None):

    num = '' if cv2_out == None else '1'
    tensor = Conv2D(cv1_out, cv1_filter, strides=cv1_strides, data_format='channels_first', name=layer+'_conv'+num)(x)
    tensor = BatchNormalization(axis=1, epsilon=0.00001, name=layer+'_bn'+num)(tensor)
    tensor = Activation('relu')(tensor)
    if padding == None:
        return tensor
    tensor = ZeroPadding2D(padding=padding, data_format='channels_first')(tensor)
    if cv2_out == None:
        return tensor
    tensor = Conv2D(cv2_out, cv2_filter, strides=cv2_strides, data_format='channels_first', name=layer+'_conv'+'2')(tensor)
    tensor = BatchNormalization(axis=1, epsilon=0.00001, name=layer+'_bn'+'2')(tensor)
    tensor = Activation('relu')(tensor)
    return tensor

def triplet_loss(y_true, y_pred, alpha = 0.2):
    anchor, positive, negative = y_pred[0], y_pred[1], y_pred[2]
    pos_dist = tf.reduce_sum(tf.square(tf.subtract(anchor,positive)),axis=-1)
    neg_dist = tf.reduce_sum(tf.square(tf.subtract(anchor,negative)),axis=-1)
    basic_loss = tf.add(tf.subtract(pos_dist,neg_dist),alpha)
    loss = tf.reduce_sum(tf.maximum(basic_loss,0))
    return loss


#!/usr/bin/env python3
import itertools
from abc import ABC

import utils
from year_2017.day_18.part_a import RegisterRValueInstruction, \
    RvalueRValueInstruction, Mul
from year_2017.day_18.part_b import ProgramExtended, InstructionExtended

class Challenge(utils.BaseChallenge):
    def solve(self, _input, debug=False):
        """
        >>> Challenge().default_solve()
        3969
        """
        return ProgramExtendedTwice.from_program_text(_input)\
            .step_and_count_instructions().get(Mul, 0)

class ProgramExtendedTwice(ProgramExtended):
    def step_and_count_instructions(self, registers=None, count=None):
        if count is None:
            steps = itertools.count()
        else:
            steps = range(count)
        if registers is None:
            registers = self.registers_class()
        instruction_counts = {}
        for _ in steps:
            if self.has_finished(registers):
                break
            instruction_type = \
                type(self.instructions[registers.instruction_pointer])
            instruction_counts.setdefault(instruction_type, 0)
            instruction_counts[instruction_type] += 1
            finished = self.apply(registers)
            if finished:
                break

        return instruction_counts

class InstructionExtendedTwice(InstructionExtended, ABC):
    instruction_classes = dict(InstructionExtended.instruction_classes)

ProgramExtendedTwice.instruction_class = InstructionExtendedTwice

@InstructionExtendedTwice.register
class Sub(InstructionExtendedTwice, RegisterRValueInstruction):
    name = 'sub'

    regex = RegisterRValueInstruction.make_regex(name)

    def apply(self, registers):
        registers[self.register] -= registers.resolve(self.rvalue)

@InstructionExtendedTwice.register
class Jgz(RvalueRValueInstruction):
    name = 'jnz'

    regex = RvalueRValueInstruction.make_regex(name)

    def apply(self, registers):
        if registers.resolve(self.rvalue_a) != 0:
            next_instruction_pointer = \
                registers.instruction_pointer + registers.resolve(self.rvalue_b)
        else:
            next_instruction_pointer = None

        return next_instruction_pointer

Challenge.main()
challenge = Challenge()


from flask import Flask, render_template, url_for, send_from_directory, request
from domain.application import Application
from domain.model import Model
from util import parse_json, pictures, texts
import os
from os import listdir
from os.path import isfile, join
from werkzeug.utils import secure_filename

import random

from execute_models import run_all_models

def random_picture_input(appNames:list, app_name:str, data:dict, applications:dict, CONFIG:dict):
    """
    Function that handles random pictures

    appNames (List): Menu generated.
    app_name (str): The name of app used by the user
    data (dict): values that the user uses in the form
    applications (dict): Application list
    CONFIG (dict): configuration of the server
    """

    # Check if the application exist
    if app_name in applications:
        picture_name = ""
        # find the key in the dictionary that contains the file of the picture
        if "model" in data:
            if data['model'] in applications[app_name].models:
                picture_name = "randomPicture" + data['model']
            else:
                # Model doesn't exist
                return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "The model doesn't exist")
        else:
            picture_name = "randomPicture"

        if picture_name in data:
            models_list = []
            # Retrieve selected model list
            if 'model' in data:
                models_list.append(data['model'])
            else:
                for i in data:
                    if i.startswith("checkbox"):
                        mod = i[8:]
                        if mod in applications[app_name].models:
                            models_list.append(mod)
            # No models selected
            if len(models_list) == 0:
                return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "You didn't select a model!. Try again")
            # Retrieve allowed files
            allowed_files = retrieve_allowed_files(models_list, app_name, applications)

            picture_in_system = False
            for f in listdir(CONFIG["random_pictures_path"] + "/"):
                if isfile(join(CONFIG["random_pictures_path"] + "/", f)) and f == data[picture_name]:
                    picture_in_system = True

            if picture_in_system:
                # check if correct file extension
                if "." + data[picture_name].split('.')[1] not in allowed_files:
                    return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "File extension not valid. Try again")

                # Execute the models
                output = run_all_models(models_list, app_name, applications, CONFIG["random_pictures_path"] + "/" + data[picture_name])

                return render_template("outputImage.html", menu = appNames, title = applications[app_name].name, allowed_file = allowed_files, models = output, app = applications[app_name].name, picture = CONFIG["random_pictures_path"] + "/" + data[picture_name])
            else:
                return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "The picture doesn't exist in our system")
        else:
            return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "Error while trying to retrieve the random picture")
    return render_template("wrongOutput.html", menu = appNames, title = "Error", info= "Application name not found")

def random_text_input(appNames:list, app_name:str, data:dict, applications:dict, CONFIG:dict):
    """
    Function that handles random texts

    appNames (List): Menu generated.
    app_name (str): The name of app used by the user
    data (dict): values that the user uses in the form
    applications (dict): Application list
    CONFIG (dict): configuration of the server
    """

    # Check if the application exist
    if app_name in applications:
        text_name = ""
        # find the key in the dictionary that contains the file of the text
        if "model" in data:
            if data['model'] in applications[app_name].models:
                text_name = "randomText" + data['model']
            else:
                # Model doesn't exist
                return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "The model doesn't exist")
        else:
            text_name = "randomText"

        if text_name in data:
            models_list = []
            # Retrieve selected model list
            if 'model' in data:
                models_list.append(data['model'])
            else:
                for i in data:
                    if i.startswith("checkbox"):
                        mod = i[8:]
                        if mod in applications[app_name].models:
                            models_list.append(mod)
            # No models selected
            if len(models_list) == 0:
                return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "You didn't select a model!. Try again")
            # Retrieve allowed files
            allowed_files = retrieve_allowed_files(models_list, app_name, applications)

            text_in_system = False
            for f in listdir(CONFIG["random_texts_path"] + "/"):
                if isfile(join(CONFIG["random_texts_path"] + "/", f)) and f == data[text_name]:
                    text_in_system = True

            if text_in_system:
                # check if correct file extension
                if "." + data[text_name].split('.')[1] not in allowed_files:
                    return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "File extension not valid. Try again")

                output_text = texts.read_text_file(CONFIG["random_texts_path"] + "/" + data[text_name])
                # Execute the models
                output = run_all_models(models_list, app_name, applications, output_text)
                return render_template("outputText.html", menu = appNames, title = applications[app_name].name, allowed_file = allowed_files, models = output, app = applications[app_name].name, text = output_text)
            else:
                return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "The text doesn't exist in our system")
        else:
            return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "Error while trying to retrieve the random text")
    return render_template("wrongOutput.html", menu = appNames, title = "Error", info= "Application name not found")

def custom_input(appNames:list, app_name:str, data:dict, applications:dict, CONFIG:dict):
    """
    Function that handles custom input

    appNames (List): Menu generated.
    app_name (str): The name of app used by the user
    data (dict): values that the user uses in the form
    applications (dict): Application list
    CONFIG (dict): configuration of the server
    """
    # Check if the application exist
    if app_name in applications:
        input_location_key = ""
        # find the key in the dictionary that contains the file of the text
        if "model" in data:
            if data['model'] in applications[app_name].models:
                input_location_key = "custom-input" + data['model']
            else:
                # Model doesn't exist
                return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "The model doesn't exist")
        else:
            input_location_key = "custom-input"

        if input_location_key in data:
            input_content = data[input_location_key]

            models_list = []
            # Retrieve selected model list
            if 'model' in data:
                models_list.append(data['model'])
            else:
                for i in data:
                    if i.startswith("checkbox"):
                        mod = i[8:]
                        if mod in applications[app_name].models:
                            models_list.append(mod)
            # No models selected
            if len(models_list) == 0:
                return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "You didn't select a model!. Try again")

            if applications[app_name].app_type == "text":
                # Execute the models
                output = run_all_models(models_list, app_name, applications, input_content)
                return render_template("outputText.html", menu = appNames, title = applications[app_name].name, allowed_file = ["Custom text input"], models = output, app = applications[app_name].name, text = input_content)
            else:
                return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "Application type not valid")
        else:
            return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "Error while trying to retrieve your text")
    return render_template("wrongOutput.html", menu = appNames, title = "Error", info= "Application name not found")

def user_file_input(appNames:list, app_name:str, data:dict, applications:dict, CONFIG:dict):

    """
    Function that handles the pictures uploaded by a user

    appNames (List): Menu generated.
    app_name (str): The name of app used by the user
    data (dict): values that the user uses in the form
    applications (dict): Application list
    CONFIG (dict): configuration of the server
    """
    # Case when user uploads a picture
    if 'userFile' not in request.files:
        return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, test = app_name, info= "The file couldn't be upload to the server. Try again")

    # Search for the uploaded file
    file = request.files['userFile']

    # Error if nothing is sent
    if file.filename == '':
        return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, test = app_name, info= "You didn't select a file. Try again")

    # Search for the name of the application
    if app_name in applications:

        models_list = []
        # Retrieve selected model list
        if 'model' in data:
            if data['model'] in applications[app_name].models:
                models_list.append(data['model'])
            else:
                # Model doesn't exist
                return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "The model doesn't exist")
        else:
            for i in data:
                if i.startswith("checkbox"):
                    mod = i[8:]
                    if mod in applications[app_name].models:
                        models_list.append(mod)
        # No models selected
        if len(models_list) == 0:
            return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "You didn't select a model!. Try again")

        # Retrieve allowed files
        allowed_files = retrieve_allowed_files(models_list, app_name, applications)
        extension = "." + file.filename.split(".")[1]
        # Save the file
        if file and extension in allowed_files:
            filename = secure_filename(file.filename)
            file.save(os.path.join(CONFIG["upload_folder"], filename))
            new_name = generate_random_names() + extension
            os.rename(os.path.join(CONFIG["upload_folder"], filename), os.path.join(CONFIG["upload_folder"], new_name))

            if applications[app_name].app_type == "image":
                # Execute the models
                output = run_all_models(models_list, app_name, applications, CONFIG["upload_folder"] + "/" + new_name)
                return render_template("outputImage.html", menu = appNames, title = applications[app_name].name, allowed_file = allowed_files, models = output, app = applications[app_name].name, picture = CONFIG["upload_folder"] + "/" + new_name)
            elif applications[app_name].app_type == "text":
                # Execute the models
                output_text = texts.read_text_file(CONFIG["upload_folder"] + "/" + new_name)
                output = run_all_models(models_list, app_name, applications, output_text)
                return render_template("outputText.html", menu = appNames, title = applications[app_name].name, allowed_file = allowed_files, models = output, app = applications[app_name].name, text = output_text)
            else:
                return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "Application type not valid")
        else:
            return render_template("wrongOutput.html", menu = appNames, title = applications[app_name].name, info= "File extension not valid. Try again")

    return render_template("wrongOutput.html", menu = appNames, title = "Error", info= "Application name not found")

def retrieve_allowed_files(models_name_list:list, app_name:str, applications: dict):

    """
    Function that handles the allowed files of given models name. In case of more than one models the intersection of all of them is made.

    app_name (str): The name of app used by the user
    models_name_list (list): names of the models
    applications (dict): Application list
    """

    allowed_files = set()
    for i in models_name_list:
        if len(allowed_files) == 0:
            allowed_files = set(applications[app_name].models[i].file_format)
        else:
            allowed_files = allowed_files.intersection(set(applications[app_name].models[i].file_format))
    return allowed_files

def get_menu_items(selected_item:str, applications: dict):

    """
    This function returns a list with the values of the menu. For each of the we add the short name, the name and a boolean to specify
    if the item needs to be highlighted

    selected_item (str): Element to be highlighted
    applications (dict): list of applications

    """
    # Home value
    menu_items = [["", "Home", False]]

    # Highlight if it is the selected item
    if selected_item == "Home":
        menu_items[0][2] = True

    # For each application
    for i in applications.keys():
        menu_items.append([i, applications[i].name, False])
        # Highlight if it is the selected item
        if i == selected_item:
            menu_items[-1][2] = True
    return menu_items

def get_apps_overview(applications: dict):

    """
    This function retrieves an overview of the applications (application name and description)

    applications (dict): list of applications
    """
    # If no applications return an empty list
    if len(applications.keys()) == 0:
        return []

    # Add the information of each application.
    overview, i = [[]], 0
    for key in applications.keys():
        overview[i].append((applications[key].name, key, applications[key].description))

        # Every two applications we create a new list inside (this if for aesthetics of the webpage).
        if len(overview[i]) == 2:
            i += 1
            overview.append([])

    return overview

def get_models_by_app(app_name: str, applications: dict):

    """
    This function retrieves information (model name, model description and allowed file formats) and restriction of each model.

    app_name (str): name of the application
    applications (dict): list of applications

    """
    models = []
    restriction = []

    # Iterate through all the models
    for key in applications[app_name].models.keys():
        # get the model and append the information
        model = applications[app_name].models[key]
        models.append([model.short_name, model.description, ", ".join(model.file_format), False, model.name])

        # Restrictions taking into account all the models (used for comparing models)
        if len(restriction) == 0:
            restriction = set(model.file_format)
        else:
            restriction = restriction.intersection(set(model.file_format))

    # The first model needs to be to True in order to activate the first "tab" of the HTML code
    if len(models) > 0:
        models[0][3] = True

    return models, ", ".join(list(restriction))

def generate_random_names(length = 20, base = 64):
    """
    This function generates a random name of a given length

    length (int): length of the output
    base (int): base to be used in order to generate the random name

    """
    characters = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ+-"
    # Upper and lower bounds control
    if base > 64:
        base = 64
    elif base < 2:
        base = 0
    output_name = []

    # Generate de name
    for _ in range(length):
        output_name.append(characters[random.randint(0, base-1)])

    return "".join(output_name)


import glob
import os

from numpy import genfromtxt
from sqlalchemy import Column, Integer, String
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker

Base = declarative_base()

class counters(Base):
    __tablename__ = 'counters'

    id = Column(Integer, primary_key=True)
    hero = Column(String(250), nullable=False)
    counter = Column(String(250), nullable=False)

# Create the database
engine = create_engine('sqlite:///database.db')
Base.metadata.create_all(engine)

# create the session
session = sessionmaker()
session.configure(bind=engine)
s = session()

def Load_Data(file_name):
    data = genfromtxt(file_name, delimiter=',', skip_header=1, dtype=None, encoding=None)
    return data.tolist()

try:
    current_path = os.getcwd()
    # Get list of files in folder
    files = glob.glob(os.path.join(current_path, "*.csv"))

    # Loop through all files in list
    for file_path in files:
        file_name = file_path.split('/')[-1]
        data = Load_Data(file_name)

        for i in data:
            record = counters(**{
                'hero': i[0],
                'counter': i[1]
            })
            s.add(record)
    s.commit()

except:

    s.rollback()  # Rollback the changes on error
finally:
    s.close()  # Close the connection


from django.urls import path
from . import views

app_name = 'blog'

urlpatterns = [
    path('', views.Home, name='home'),
    path('post/', views.BlogFormView, name='post'),
    path('blog-details/<slug:slug>', views.BlogDetails, name='blog_details'),
    path('blog-category/<int:pk>', views.CategoryBlogs, name='blog-category'),
]


__author__ = 'mpetyx'

from collections import OrderedDict

from Parser import Parser
from pyapi.entities import APIRoot, APIResource, APIMethod
from pyapi.libraries.pyhydra import Hypermedia

class HydraParser(Parser):
    def parse(self, location):

        hm = Hypermedia()
        hm.open(location)
        # if hm.open(location):
        # print hm.doc.apidoc
        # for klassi in hm.doc.classes:
        # print "######################"
        #         print hm.doc.classes[klassi].dump()
        # for operation in  hm.doc.classes[klassi].operations:
        #     print

        # first_line, c = c.split('\n', 1)
        # raml_version = _validate_raml_header(first_line)

        # context = ParseContext(yaml.load(c), location)

        root = APIRoot(raml_version=str(0.8))
        # root.title = context.get_string_property('title', True)
        #
        # root.baseUri = context.get_string_property('baseUri')
        # root.version = context.get('version')
        # root.mediaType = context.get_string_property('mediaType')

        # root.documentation = context.get_property_with_schema('documentation', APIRoot.documentation)
        # root.traits = parse_traits(context, APIRoot.traits.field_name, root.mediaType)
        # root.resourceTypes = parse_resource_type(context)

        resources = OrderedDict()
        for klass in hm.doc.classes:
            resource = APIResource()
            resource.displayName = "yolo"
            resource.description = "example of the api"
            # Parse methods

            methods = OrderedDict()

            for operation in hm.doc.classes[klass].operations:
                method = APIMethod(notNull=True)
                method.description = str(operation.subj)
                methods[str(operation.method)] = method

            if len(methods):
                resource.methods = methods

            resources[str(klass)] = resource  #parse_resource(context, property_name, root, root.mediaType)

        if resources > 0:
            root.resources = resources

        #
        # api = APIRoot()
        # g = Graph().parse(location, format='json-ld')

        return root


import uuid as uuid_pkg
from typing import TYPE_CHECKING

from sqlmodel import Field, Relationship

from app.internals.model import BaseModelDb
from app.internals.user.model import UserBudgetLink, UserDb

if TYPE_CHECKING:
    from app.internals.user.model import UserDb

class BudgetDb(BaseModelDb, table=True):
    __tablename__ = "budget"

    id: uuid_pkg.UUID = Field(
        default_factory=uuid_pkg.uuid4,
        primary_key=True,
        index=True,
        nullable=False,
    )
    name: str
    description: str
    users: list["UserDb"] = Relationship(
        back_populates="budgets", link_model=UserBudgetLink
    )


# ===== LIBRARIES ======
# Basic libraries to handle data
import numpy as np
import pandas as pd

# To load data to SQL database
from sqlalchemy import create_engine

# To access configuration variables
import sys; sys.path.append('.')
from pathlib import Path
from config.core import DATASET_DIR, config

# ===== FUNCTIONS ======
def load_data(messages_file_name:str, categories_file_name:str):
    """Loads and merge the messages & categories datasets

    Arguments:
        messages_file_name: name file of message data
        categories_file_name: name file of categories data
    Returns:
        data: pandas dataframe
    """
    # Loads datasets
    messages = pd.read_csv(Path(f'{DATASET_DIR/messages_file_name}'))
    categories = pd.read_csv(Path(f'{DATASET_DIR/categories_file_name}'))

    # Merges the two datasets
    data = messages.merge(categories, on='id')

    return data

def clean_data(data):
    """Loads data, splits categories into multiple columns,
    ensures targets are binary, and then removes duplicates

    Arguments:
        data: pandas dataframe
    Returns:
        data: pandas dataframe
    """
    # ===== Transforming "Categories" ======
    # Splitting categories (one col) into n columns
    cats = data.categories.str.split(';', expand=True)
    cols = cats.iloc[0,:].apply(lambda x:x[:-2])
    cats.columns = cols
    for column in cats:
        cats[column] = pd.to_numeric(cats[column].str[-1])

    # Concat categories with the rest of the data
    data = pd.concat([data.drop(['categories'], axis=1), cats], axis=1)

    # The "related" category includes 3 unique values. To binarize
    # this class, I'll assume "2" as "1" to not lose those observations
    data['related'] = data['related'].replace({2:1})

    # ===== Removing duplicates ======
    # Removing unnecesary feature
    data = data.drop_duplicates()

    return data

def save_data(data):
    """Saves cleaned data into an SQLite database following
    configuration parameters for path file

    Arguments:
        data: pandas dataframe
    Returns:
        None
    """
    save_file_name = config.app_config.database_file_name
    table_name = config.app_config.table_name

    engine = create_engine(f'sqlite:///{DATASET_DIR}/{save_file_name}')
    data.to_sql(f'{table_name}', engine, index=False, if_exists='replace')

def etl_process():
    """Contains the 3 steps of the ETL pipeline

    Arguments:
        None
    Returns:
        None
    """
    data = load_data(messages_file_name = config.app_config.messages_file_name,
                   categories_file_name = config.app_config.categories_file_name)
    print('Data extraction complete')

    data = clean_data(data)
    print('Data transformed correctly')

    save_data(data)
    print('Loaded clean data into SQLite database')

if __name__ == '__main__':
    # run data pipeline
    etl_process()


# -*- coding: utf-8 -*-
import numpy
from numpy import loadtxt
import scipy
import pylab
import time
import copy
import os
import re
import sys
import matplotlib.pyplot as plt
from matplotlib import rc
from scipy import stats
from itertools import combinations
from ast import literal_eval
from matplotlib.colors import ListedColormap, NoNorm
from pylab import rcParams

# the axes attributes need to be set before the call to subplot
plt.rc('text', usetex=True)
plt.rc('font', family='serif', weight='bold', size=18)
plt.rc('xtick.major', size=5, pad=7)
plt.rc('ytick.major', size=5, pad=7)
rcParams['text.latex.preamble'] = [r'\usepackage{sfmath} \boldmath']
plt.rc('xtick', labelsize=18)
plt.rc('ytick', labelsize=18)

#filename = "_feats_shuffled_across_all_nouns.dat"
#filename = "_feats_shuffled_within_clusters.dat"
#filename = "_feats_shuffled_within_clusters_offblock_mean.dat"
filename = "_feats_shuffled_within_clusters_offblock_mean_inblock_mean.dat"
#filename = ".dat"

D = loadtxt("D%s"%filename)
N_words = 60
power = 1
#triplets

delta1 = []
delta2 = []

i_else = numpy.arange(5,11+1)

D=numpy.delete(D, numpy.s_[5:11+1], 0)
D=numpy.delete(D, numpy.s_[5:11+1], 1)

N_words = numpy.shape(D)[0]
for item in list(combinations(range(N_words),3)):
	#print(item)
	d1 = D[item[0],item[1]]#numpy.fabs()
	d2 = D[item[1],item[2]]#numpy.fabs()
	d3 = D[item[2],item[0]]#numpy.fabs()

	#print(d1)
	#print(d2)
	#print(d3)
	dmin = (numpy.sort([d1,d2,d3]))[0]
	dint = (numpy.sort([d1,d2,d3]))[1]
	dmax = (numpy.sort([d1,d2,d3]))[2]
	#print("dmin=",dmin)
	#print("dint=",dint)
	#print("dmax=",dmax)
	#print(10*"--")

	if (dmin > 0 and dint > 0 and dmax > 0):
		delta1 = numpy.append(delta1, dmin/dmax)
		delta2 = numpy.append(delta2, dint/dmax)

#print("delta1=",delta1)
#print("delta2=",delta2)

# ------------------------------------- compute UM content
lamda = []

for i in range(numpy.size(delta1)):
	if( (numpy.log(delta1[i]) + numpy.log(delta2[i])) != 0 ):
		x = (numpy.log(delta1[i]) - numpy.log(delta2[i]))/(numpy.log(delta1[i]) + numpy.log(delta2[i]))
		lamda = numpy.append(lamda,x)
print(lamda)
print("umcontent=", numpy.mean(lamda))

# ------------------------------------- PLOT

fig = plt.figure(figsize=(3.5, 5))
plt.plot(numpy.linspace(0.0,1.1,11),numpy.linspace(0.0,1.1,11), alpha=1, color='white', lw=3, zorder=2)
plt.plot(numpy.linspace(0.0,1.1,11),1-numpy.linspace(0.0,1.1,11), alpha=1, color='white', lw=2, zorder=2)
#plt.axvline(1.0, alpha=1, color='black', zorder=3)
plt.scatter(delta2, delta1, marker='o',s=0.1, c='black', zorder=3)
#####
#plt.hexbin(numpy.append(numpy.linspace(0.0,0.8),delta2), numpy.append(numpy.linspace(0.0,0.8), delta1), gridsize=50, bins='log', cmap='jet', vmin=0, vmax=2)
#plt.colorbar()
#####
plt.xlabel('$\delta_2 (d_{med}/d_{max})$')
plt.ylabel('$\delta_1 (d_{min}/d_{max})$')
plt.xlim(0.5,1.1)
plt.ylim(0.,1.1)

print(list(combinations(range(8),2)))

#plt.savefig('um_content%s_dpow%.1f.pdf'%(filename,power), bbox_inches='tight')
#plt.savefig('um_content%s_dpow%.1f.png'%(filename,power), bbox_inches='tight')


#!/usr/bin/env python

import os
import datetime
from pprint import pprint
import json
import logging

from icinga2api.client import Client as Icinga2Client

icinga2api = Icinga2Client(config_file='icinga2-api.ini')

CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))

logger = logging.getLogger('autoack')

def get_downtime(filters, filter_vars, object_type='Host'):
    """Lookup downtime by filters. Returns active downtime
       with the farthest end time"""
    assert object_type in ['Service', 'Host']
    downtimes = icinga2api.objects.list(
        'Downtime',
        filters=filters, filter_vars=filter_vars)

    expires_at = 0
    the_downtime = None
    for d in downtimes:
        # skip service downtimes when lookup up for host ones
        if object_type == 'Host' and 'service_name' in d['attrs']:
            continue
        # skip inactive and floating downtimes
        if not d['attrs']['active']:
            continue
        if not d['attrs']['fixed']:
            continue
        if expires_at < d['attrs']['end_time']:
            expires_at = d['attrs']['end_time']
            the_downtime = d

    return the_downtime

def acknowledge_on_downtime(the_downtime, object_type, filters, filter_vars):
    expires_at = the_downtime['attrs']['end_time']
    downtime_id = int(the_downtime['attrs']['legacy_id'])
    logger.info("Found downtime {downtime_id}, doing ACK.".format(
        downtime_id=downtime_id))
    comment = 'This is auto acknowledge. {object_type} is in downtime #{downtime_id}'.format(
        downtime_id=downtime_id, object_type=object_type,
    )
    icinga2api.actions.acknowledge_problem(
        object_type=object_type,
        filters=filters,
        filter_vars=filter_vars,
        author='AutoAck bot',
        comment=comment,
        sticky=False,
        notify=False,
        expiry=expires_at,
    )

def acknowledge_unconditionally(object_type, filters, filter_vars):
    logger.info("Doing autoack")
    comment = 'This is auto acknowledge. {object_type} has autoack attribute set'.format(
        object_type=object_type,
    )
    icinga2api.actions.acknowledge_problem(
        object_type=object_type,
        filters=filters,
        filter_vars=filter_vars,
        author='AutoAck bot',
        comment=comment,
        sticky=False,
        notify=False
    )

def has_autoack(object_type, object_name):
    data = icinga2api.objects.get(object_type=object_type, name=object_name)
    try:
        autoack = data['attrs']['vars']['autoack']
    except KeyError:
        return False
    if autoack:
        return True
    return False

def handle_stream():
    logger.info("Starting")
    for event in icinga2api.events.subscribe(
            types=['StateChange', 'DowntimeStarted', 'DowntimeTriggered'],
            queue='auto-ack'):

        if type(event) is bytes:
            event = str(event, encoding='utf-8')
        event = json.loads(event)
        pprint(event)

        host_name = event['host']
        try:
            service_name = event['service']
        except KeyError:
            service_name = None
        logger.info("Got change state event {event}".format(event=event))
        if event['state'] == 0:  # the service is up now
            continue

        if not service_name:
            object_type = 'Host'
            object_name = host_name
            filters = 'match(host.name,filter_host)'
            filter_vars = dict(filter_host=host_name)

        else:
            object_type = 'Service'
            object_name = '!'.join([host_name, service_name])
            filters = 'match(host.name,filter_host) && match(service.name,filter_service)',
            filter_vars = dict(filter_host=host_name,
                               filter_service=service_name)
        #
        the_downtime = get_downtime(object_type=object_type,
                                    filters=filters, filter_vars=filter_vars)

        if the_downtime:
            acknowledge_on_downtime(the_downtime=the_downtime,
                                    object_type=object_type,
                                    filters=filters, filter_vars=filter_vars)
            continue

        autoack_enabled = has_autoack(object_type=object_type,
                                      object_name=object_name)

        if autoack_enabled:
            acknowledge_unconditionally(object_type=object_type,
                                        filters=filters,
                                        filter_vars=filter_vars)
            continue

        if not the_downtime:
            logger.info("No active downtimes found")
            continue

if __name__ == '__main__':
    # logging.basicConfig(level=logging.DEBUG)
    logging.basicConfig(level=logging.INFO)
    handle_stream()


import torch
import torch.nn as nn
from dl_toolbox.lightning_modules import BaseModule
import dl_toolbox.augmentations as aug
from dl_toolbox.utils import TorchOneHot
from dl_toolbox.callbacks import log_batch_images

class CE(BaseModule):

    def __init__(self,
                 network,
                 weights,
                 mixup=0.,
                 ignore_index=-1,
                 *args,
                 **kwargs):

        super().__init__(*args, **kwargs)

        net_cls = self.net_factory.create(network)
        self.network = net_cls(*args, **kwargs)
        self.num_classes = self.network.out_channels
        out_dim = self.network.out_dim
        self.weights = list(weights) if len(weights)>0 else [1]*self.num_classes
        self.ignore_index = ignore_index
        self.loss = nn.CrossEntropyLoss(
            ignore_index=self.ignore_index,
            weight=torch.Tensor(self.weights)
            #weight=torch.Tensor(self.weights).reshape(1,-1,*out_dim)
        )
        self.onehot = TorchOneHot(
            range(self.num_classes)
        )
        self.mixup = aug.Mixup(alpha=mixup) if mixup > 0. else None
        #self.save_hyperparameters('network', 'weights', 'mixup', 'ignore_index')
        self.save_hyperparameters()

    @classmethod
    def add_model_specific_args(cls, parent_parser):

        parser = super().add_model_specific_args(parent_parser)
        parser.add_argument("--ignore_index", type=int, default=-1)
        parser.add_argument("--network", type=str)
        parser.add_argument("--weights", type=float, nargs="+", default=())
        parser.add_argument("--mixup", type=float, default=0.)

        return parser

    def forward(self, x):

        return self.network(x)

    @classmethod
    def _compute_probas(cls, logits):

        return logits.softmax(dim=1)

    @classmethod
    def _compute_conf_preds(cls, probas):

        return torch.max(probas, dim=1)

    def training_step(self, batch, batch_idx):

        batch = batch["sup"]
        inputs = batch['image']
        labels = batch['mask']
        if self.mixup:
            labels = self.onehot(labels).float()
            inputs, labels = self.mixup(inputs, labels)
            #batch['image'] = inputs
        logits = self.network(inputs)
        loss = self.loss(logits, labels)
        self.log('Train_sup_CE', loss)
        batch['logits'] = logits.detach()

        if self.trainer.current_epoch % 10 == 0 and batch_idx == 0:
            log_batch_images(
                batch,
                self.trainer,
                prefix='Train'
            )

        return {'batch': batch, "loss": loss}

    def validation_step(self, batch, batch_idx):

        outs = super().validation_step(batch, batch_idx)
        labels = batch['mask']
        logits = outs['logits']
        loss = self.loss(logits, labels)
        self.log('Val_CE', loss)

        return outs


from datetime import datetime

from telegram.ext import Updater, CommandHandler
from CameraModules.PiCam import PiCam
from Config import config

class TelegramBot:
    def startTelegramBot(self):
        updater = Updater(token=config["telegramApiKey"])
        dispatcher = updater.dispatcher

        try:
            start_CommandHandler = CommandHandler('start', self.__startHandler)
            capture_CommandHandler = CommandHandler('capture', self.__captureHandler)
            stop_CommandHandler = CommandHandler('stop', self.__stopHandler)

            dispatcher.add_handler(start_CommandHandler)
            dispatcher.add_handler(capture_CommandHandler)
            dispatcher.add_handler(stop_CommandHandler)

            updater.start_polling(0.5)
            print("Telegram bot started!")

        except Exception as e:
            print(e)

    def __startHandler(self, bot, update):
        if (update.message.chat_id == int(config['userId'])):
            try:
                time = datetime.now()
                print("Bot online: "+ str(time))
                bot.send_message(chat_id=update.message.chat_id, text="Bot online: "+ str(time))
            except Exception as e:
                print(e)
        else:
            bot.send_message(chat_id=update.message.chat_id, text="Unauthorised")
            print("unauthorised message from " + update.message.chat_id)

    def __captureHandler(self, bot, update):
        if update.message.chat_id == int(config['userId']):
            try:
                bot.send_message(chat_id=update.message.chat_id, text="Capturing...")
                camera = PiCam()
                camera.Snap()
                #subprocess.call("raspistill -o image.jpg")
                bot.sendPhoto(chat_id=update.message.chat_id, photo=open("/home/pi/image.jpg", 'rb'))

            except Exception as e:
                bot.send_message(chat_id=update.message.chat_id, text="Failed to capture")
                print(e)

    def __stopHandler(self, bot, update):
        if update.message.chat_id == int(config['userId']):
            bot.send_message(chat_id=update.message.chat_id, text="Exit terminal")


from torchvision.transforms import transforms
from gaussian_blur import GaussianBlur
from torchvision import transforms, datasets
from get_dataset_util import ViewGen

class GetTransformedDataset:
    @staticmethod
    def get_simclr_transform(size, s=1):
        color_jitter = transforms.ColorJitter(
            0.8 * s, 0.8 * s, 0.8 * s, 0.2 * s)
        data_transforms = transforms.Compose([transforms.RandomResizedCrop(size=size),
                                              transforms.RandomHorizontalFlip(),
                                              transforms.RandomApply(
                                                  [color_jitter], p=0.8),
                                              transforms.RandomGrayscale(
                                                  p=0.2),
                                              GaussianBlur(
                                                  kernel_size=int(0.1 * size)),
                                              transforms.ToTensor()])
        return data_transforms

    def get_cifar10_train(self, n_views):
        return datasets.CIFAR10('.', train=True,
                                transform=ViewGen(
                                    self.get_simclr_transform(
                                        32),
                                    n_views),
                                download=True)

    def get_cifar10_test(self, n_views):
        return datasets.CIFAR10('.', train=False,
                                transform=transforms.ToTensor(),
                                download=True)

    def get_cifar100_test(self, n_views):
        return datasets.CIFAR100('.', train=False,
                                 transform=transforms.ToTensor(),
                                 download=True)


# Ref: ATmega16U4/ATmega32U4 8-bit Microcontroller with 16/32K bytes of ISP Flash and USB Controller datasheet
# Accession: G00058

import math
import logging
from amaranth import *

from ....interface.spi_controller import SPIControllerSubtarget, SPIControllerInterface
from .... import *
from .. import *

class ProgramAVRSPISubtarget(Elaboratable):
    def __init__(self, controller, reset_t, dut_reset):
        self.controller = controller
        self.reset_t = reset_t
        self.dut_reset = dut_reset

    def elaborate(self, platform):
        m = Module()

        m.submodules.controller = self.controller

        m.d.comb += [
            self.controller.bus.oe.eq(self.dut_reset),
            self.reset_t.oe.eq(1),
            self.reset_t.o.eq(~self.dut_reset)
        ]

        return m

class ProgramAVRSPIInterface(ProgramAVRInterface):
    def __init__(self, interface, logger, addr_dut_reset):
        self.lower   = interface
        self._logger = logger
        self._level  = logging.DEBUG if self._logger.name == __name__ else logging.TRACE
        self._addr_dut_reset = addr_dut_reset
        self._extended_addr  = None
        self.erase_time      = None

    def _log(self, message, *args):
        self._logger.log(self._level, "AVR SPI: " + message, *args)

    async def _command(self, byte1, byte2, byte3, byte4):
        command = [byte1, byte2, byte3, byte4]
        self._log("command %s", "{:08b} {:08b} {:08b} {:08b}".format(*command))
        result = await self.lower.transfer(command)
        self._log("result  %s", "{:08b} {:08b} {:08b} {:08b}".format(*result))
        return result

    async def programming_enable(self):
        self._log("programming enable")

        await self.lower.lower.device.write_register(self._addr_dut_reset, 1)
        await self.lower.delay_ms(20)

        _, _, echo, _ = await self._command(0b1010_1100, 0b0101_0011, 0, 0)
        if echo == 0b0101_0011:
            self._log("synchronization ok")
        else:
            raise ProgramAVRError("device not present or not synchronized")

    async def programming_disable(self):
        self._log("programming disable")
        await self.lower.synchronize()
        await self.lower.lower.device.write_register(self._addr_dut_reset, 0)
        await self.lower.delay_ms(20)

    async def _is_busy(self):
        if self.erase_time is not None:
            self._log("wait for completion")
            await self.lower.delay_ms(self.erase_time)
            return False
        self._log("poll ready/busy flag")
        _, _, _, busy = await self._command(0b1111_0000, 0b0000_0000, 0, 0)
        return bool(busy & 1)

    async def read_signature(self):
        self._log("read signature")
        signature = []
        for address in range(3):
            _, _, _, sig_byte = await self._command(0b0011_0000, 0b0000_0000, address & 0b11, 0)
            signature.append(sig_byte)
        return tuple(signature)

    async def read_fuse(self, address):
        self._log("read fuse address %#04x", address)
        a0, a1 = {
            0: (0b0000, 0b0000),
            1: (0b1000, 0b1000),
            2: (0b0000, 0b1000),
        }[address]
        _, _, _, data = await self._command(
            0b0101_0000 | a0,
            0b0000_0000 | a1,
            0,  0)
        return data

    async def write_fuse(self, address, data):
        self._log("write fuse address %#04x data %02x", address, data)
        a = {
            0: 0b0000,
            1: 0b1000,
            2: 0b0100,
        }[address]
        await self._command(
            0b1010_1100,
            0b1010_0000 | a,
            0,
            data)
        while await self._is_busy(): pass

    async def read_lock_bits(self):
        self._log("read lock bits")
        _, _, _, data = await self._command(0b0101_1000, 0b0000_0000, 0,  0)
        return data

    async def write_lock_bits(self, data):
        self._log("write lock bits data %02x", data)
        await self._command(
            0b1010_1100,
            0b1110_0000,
            0,
            0b1100_0000 | data)
        while await self._is_busy(): pass

    async def read_calibration(self, address):
        self._log("read calibration address %#04x", address)
        _, _, _, data = await self._command(0b0011_1000, 0b0000_0000, address, 0)
        return data

    async def load_extended_address_byte(self, address):
        extended_addr = (address >> 17) & 0xff
        if self._extended_addr != extended_addr:
            self._log("load extended address %#02x", extended_addr)
            await self._command(0b0100_1101, 0, extended_addr, 0)
            self._extended_addr = extended_addr

    async def read_program_memory(self, address):
        await self.load_extended_address_byte(address)
        self._log("read program memory address %#06x", address)
        _, _, _, data = await self._command(
            0b0010_0000 | (address & 1) << 3,
            (address >> 9) & 0xff,
            (address >> 1) & 0xff,
            0)
        return data

    async def load_program_memory_page(self, address, data):
        self._log("load program memory address %#06x data %02x", address, data)
        await self._command(
            0b0100_0000 | (address & 1) << 3,
            (address >> 9) & 0xff,
            (address >> 1) & 0xff,
            data)

    async def write_program_memory_page(self, address):
        await self.load_extended_address_byte(address)
        self._log("write program memory page at %#06x", address)
        await self._command(
            0b0100_1100,
            (address >> 9) & 0xff,
            (address >> 1) & 0xff,
            0)
        while await self._is_busy(): pass

    async def read_eeprom(self, address):
        self._log("read EEPROM address %#06x", address)
        _, _, _, data = await self._command(
            0b1010_0000,
            (address >> 8) & 0xff,
            (address >> 0) & 0xff,
            0)
        return data

    async def load_eeprom_page(self, address, data):
        self._log("load EEPROM address %#06x data %02x", address, data)
        await self._command(
            0b1100_0001,
            (address >> 8) & 0xff,
            (address >> 0) & 0xff,
            data)

    async def write_eeprom_page(self, address):
        self._log("write EEPROM page at %#06x", address)
        await self._command(
            0b1100_0010,
            (address >> 8) & 0xff,
            (address >> 0) & 0xff,
            0)
        while await self._is_busy(): pass

    async def chip_erase(self):
        self._log("chip erase")
        await self._command(0b1010_1100, 0b1000_0000, 0, 0)
        while await self._is_busy(): pass

class ProgramAVRSPIApplet(ProgramAVRApplet):
    logger = logging.getLogger(__name__)
    help = f"{ProgramAVRApplet.help} via SPI"
    description = """
    Identify, program, and verify Microchip AVR microcontrollers using low-voltage serial (SPI)
    programming.

    While programming is disabled, the programming interface is tristated, so the applet can be
    used for in-circuit programming even if the device uses SPI itself.

    The standard AVR ICSP connector layout is as follows:

    ::
        CIPO @ * VCC
         SCK * * COPI
        RST# * * GND
    """

    __pins = ("reset", "sck", "cipo", "copi")

    @classmethod
    def add_build_arguments(cls, parser, access):
        super().add_build_arguments(parser, access)

        access.add_pin_argument(parser, "reset", default=True)
        access.add_pin_argument(parser, "sck",   default=True)
        access.add_pin_argument(parser, "cipo",  default=True)
        access.add_pin_argument(parser, "copi",  default=True)

        parser.add_argument(
            "-f", "--frequency", metavar="FREQ", type=int, default=100,
            help="set SPI frequency to FREQ kHz (default: %(default)s)")

    def build(self, target, args):
        self.mux_interface = iface = target.multiplexer.claim_interface(self, args)
        controller = SPIControllerSubtarget(
            pads=iface.get_pads(args, pins=self.__pins),
            out_fifo=iface.get_out_fifo(),
            in_fifo=iface.get_in_fifo(auto_flush=False),
            period_cyc=math.ceil(target.sys_clk_freq / (args.frequency * 1000)),
            delay_cyc=math.ceil(target.sys_clk_freq / 1e6),
            sck_idle=0,
            sck_edge="rising",
            cs_active=0,
        )

        dut_reset, self.__addr_dut_reset = target.registers.add_rw(1)
        subtarget = ProgramAVRSPISubtarget(controller, iface.pads.reset_t, dut_reset)

        iface.add_subtarget(subtarget)

    async def run_lower(self, cls, device, args):
        iface = await device.demultiplexer.claim_interface(self, self.mux_interface, args)
        return SPIControllerInterface(iface, self.logger)

    async def run(self, device, args):
        spi_iface = await self.run_lower(ProgramAVRSPIApplet, device, args)
        avr_iface = ProgramAVRSPIInterface(spi_iface, self.logger, self.__addr_dut_reset)
        return avr_iface

    @classmethod
    def tests(cls):
        from . import test
        return test.ProgramAVRSPIAppletTestCase


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import six

import os
import re
import codecs
import zipfile
import gzip
try:
  import lzma
except ImportError:
  try:
    from backports import lzma
  except ImportError:
    import warnings
    warnings.warn('Install backports.lzma for xz support')
from collections import Counter
import  tensorflow as tf
import numpy as np
#------------------------------------------------------
class Flag():

    # =============================================================
    def __init__(self, placeholder_shape=[None], config=None, isTrain=True, isDev=False):
        """"""

        self.placeholder = tf.placeholder(tf.int64, placeholder_shape, name=self.classname)

        self._config = config
        self.id2flag = {}
        self.index2flag = {}
        if isTrain:
            if self.fix:
                self.n_sent = self.set_flag_train_fixlabeldata()
            else:
                self.n_sent = self.set_flag_train()
                if self.portion > 1:
                    self.split()
        # else:
        #     self.n_sent = self.set_flag_dev()
        elif isDev:
            self.n_sent = self.set_flag_dev()
        else:
            self.n_sent = self.set_flag_test()
        return

    # =============================================================

    # =============================================================
    def set_placeholders(self, indices, feed_dict={}):
        """"""
        flag = self.indices2flag(indices)
        feed_dict[self.placeholder] = flag
        return feed_dict

    # =============================================================

    # =============================================================
    def indices2flag(self, indices):
        """"""

        return [self.index2flag[index] for index in indices]

    # =============================================================

    # =============================================================
    def set_flag_train_fixlabeldata(self):
        """"""
        num = 0

        if self.train_conllu_file[0].endswith('.zip'):
            open_func = zipfile.Zipfile
            kwargs = {}
        elif self.train_conllu_file[0].endswith('.gz'):
            open_func = gzip.open
            kwargs = {}
        elif self.train_conllu_file[0].endswith('.xz'):
            open_func = lzma.open
            kwargs = {'errors': 'ignore'}
        else:
            open_func = codecs.open
            kwargs = {'errors': 'ignore'}

        with open_func(self.train_conllu_file[0], 'rb') as f:
            reader = codecs.getreader('utf-8')(f, **kwargs)
            for line in reader:
                line = line.strip()
                if line and line.startswith('#'):
                    # if re.match('[0-9]+[-.][0-9]+', line):
                    if num < self.labeled_num:
                        self.id2flag[line] = 1
                        self.index2flag[num] = 1
                    else:
                        self.id2flag[line] = 0
                        self.index2flag[num] = 0
                    num += 1
        self.dump()
        return num
    # =============================================================

    # =============================================================
    def set_flag_train(self):
        """"""
        num = 0

        if self.train_conllu_file[0].endswith('.zip'):
            open_func = zipfile.Zipfile
            kwargs = {}
        elif self.train_conllu_file[0].endswith('.gz'):
            open_func = gzip.open
            kwargs = {}
        elif self.train_conllu_file[0].endswith('.xz'):
            open_func = lzma.open
            kwargs = {'errors': 'ignore'}
        else:
            open_func = codecs.open
            kwargs = {'errors': 'ignore'}

        with open_func(self.train_conllu_file[0], 'rb') as f:
            reader = codecs.getreader('utf-8')(f, **kwargs)
            for line in reader:
                line = line.strip()
                if line and line.startswith('#'):
                    # if re.match('[0-9]+[-.][0-9]+', line):
                    if num%self.portion == 0:
                        self.id2flag[line] = 1
                        self.index2flag[num] = 1
                    else:
                        self.id2flag[line] = 0
                        self.index2flag[num] = 0
                    num += 1
        self.dump()
        return num
    # =============================================================

    # =============================================================
    def set_flag_dev(self):
        """"""
        num = 0

        if self.dev_conllu_file[0].endswith('.zip'):
            open_func = zipfile.Zipfile
            kwargs = {}
        elif self.dev_conllu_file[0].endswith('.gz'):
            open_func = gzip.open
            kwargs = {}
        elif self.dev_conllu_file[0].endswith('.xz'):
            open_func = lzma.open
            kwargs = {'errors': 'ignore'}
        else:
            open_func = codecs.open
            kwargs = {'errors': 'ignore'}

        with open_func(self.dev_conllu_file[0], 'rb') as f:
            reader = codecs.getreader('utf-8')(f, **kwargs)
            for line in reader:
                line = line.strip()
                if line and line.startswith('#'):
                    # if re.match('[0-9]+[-.][0-9]+', line):
                    self.id2flag[line] = 0
                    self.index2flag[num] = 0
                    num += 1
        return num
    # =============================================================

    #------------------test---------------------------------------
    # =============================================================
    def set_flag_test(self):
        """"""
        num = 0

        if self.dev_conllu_file[0].endswith('.zip'):
            open_func = zipfile.Zipfile
            kwargs = {}
        elif self.dev_conllu_file[0].endswith('.gz'):
            open_func = gzip.open
            kwargs = {}
        elif self.dev_conllu_file[0].endswith('.xz'):
            open_func = lzma.open
            kwargs = {'errors': 'ignore'}
        else:
            open_func = codecs.open
            kwargs = {'errors': 'ignore'}

        with open_func(self.test_conllu_file[0], 'rb') as f:
            reader = codecs.getreader('utf-8')(f, **kwargs)
            for line in reader:
                line = line.strip()
                if line and line.startswith('#'):
                    # if re.match('[0-9]+[-.][0-9]+', line):
                    self.id2flag[line] = 0
                    self.index2flag[num] = 0
                    num += 1
        return num
    # =============================================================
    #--------------------test---------------------------------------

    # =============================================================
    def dump(self):
        """"""
        with codecs.open(self.flag_savename, 'w', encoding='utf-8', errors='ignore') as f:
            for id in self.id2flag:
                f.write(u'{}\t{}\n'.format(id, self.id2flag[id]))
        return
    # =============================================================

    # =============================================================
    def split(self):

        if self.train_conllu_file[0].endswith('.zip'):
            open_func = zipfile.Zipfile
            kwargs = {}
        elif self.train_conllu_file[0].endswith('.gz'):
            open_func = gzip.open
            kwargs = {}
        elif self.train_conllu_file[0].endswith('.xz'):
            open_func = lzma.open
            kwargs = {'errors': 'ignore'}
        else:
            open_func = codecs.open
            kwargs = {'errors': 'ignore'}

        su_ids = [ids for ids, flag in self.id2flag.items() if flag == 1]

        su_file = self.train_conllu_file[0] + '.portion%d.su'%(self.portion)
        w = codecs.open(su_file, 'w', encoding='utf-8', errors='ignore')

        buff = []

        with open_func(self.train_conllu_file[0], 'rb') as f:
            reader = codecs.getreader('utf-8')(f, **kwargs)
            for line in reader:
                buff.append(line)
                if line.startswith('#'):
                    current = line
                if line == '\n':
                    if current.strip() in su_ids:
                        for contents in buff:
                            w.write(contents)
                    buff = []
        w.close()

        return
    # =============================================================

    @property
    def save_dir(self):
        return self._config.getstr(self, 'save_dir')
    @property
    def train_conllu_file(self):
        return self._config.getfiles(self, 'train_conllus')

    @property
    def dev_conllu_file(self):
        return self._config.getfiles(self, 'dev_conllus')

    @property
    def test_conllu_file(self):
        return self._config.getfiles(self, 'test_conllus')

    @property
    def flag_savename(self):
        return os.path.join(self.save_dir, 'flag' + '.lst')

    @property
    def classname(self):
        return self.__class__.__name__

    @property
    def portion(self):
        return self._config.getint(self, 'portion')

    @property
    def fix(self):
        return self._config.getboolean(self, 'fix_label_data')

    @property
    def labeled_num(self):
        return self._config.getint(self, 'labeled_num')


from fastapi import APIRouter, Depends, HTTPException,Query
from typing import List,Optional
from fastapi.responses import RedirectResponse,JSONResponse
from app.Core.Schemas.Book import BookBase,BookTitleQuery
from sqlalchemy.orm import Session
from app.database.db_connect import get_db
from app.Core.Models.Book import Book
from app.database.CRUD.Books import create_book,get_book_by_book_id,update_book,get_all_books,get_book_by_date,get_books_by_authors
from app.Routes import BookRoute
from app.Services.LoadData import GetBooks
from app.Services.CompareBooks import CompareBooks,CompareArraysBooks
from app.Services.GenerateBookSchema import GenerateBookSchema

router = APIRouter(
    prefix="/books",
    tags=["Books"],
    responses={404: {"description": "Not found"}},
)

@router.get("/",response_model=List[BookBase])
def getAll(db:Session = Depends(get_db),
            published_date:Optional[List[int]] = Query(None),
            sort:Optional[str] = "ASC",
            authors:Optional[List[str]] = Query(None),idBook:Optional[str] = None):
    books = []
    if published_date:
        books.extend(get_book_by_date(db,published_date))
    if authors:
        arr_temp = []
        for i in authors:
            arr_temp.extend(get_books_by_authors(db,i))
        if books:
            new_arr = CompareArraysBooks(arr_temp,books)
            books = new_arr
        else:
            books = arr_temp

    if idBook and books == None:
        books.append(get_book_by_book_id(db,idBook))

    if not books:
        books = get_all_books(db)

    if sort == "ASC":
        books.sort(key=lambda x: x.published_date)
    else:
        books.sort(key=lambda x: x.published_date,reverse=True)

    return books

@router.post("/")
def root(query:BookTitleQuery,db:Session = Depends(get_db)):
    books = GetBooks(query.q)
    BookSchemasList = GenerateBookSchema(books["items"])
    for book_compare in BookSchemasList:
        book = get_book_by_book_id(db,book_compare.book_id)
        if book == None:
            create_book(db,book_compare)
            continue
        if CompareBooks(book_compare,book):
            update_book(db,book_compare)
    url = router.url_path_for("getAll")
    return JSONResponse(content={"books":url})


# -*- coding:utf-8 -*-

import os

from steem.comment import SteemComment
from steem.settings import settings, STEEM_HOST
from data.reader import SteemReader
from utils.logging.logger import logger
from blog.message import get_message

BLOG_CONTENT_FOLDER = "./source/_posts"

class BlogBuilder(SteemReader):

    def __init__(self, account=None, tag=None, days=None):
        SteemReader.__init__(self, account=account, tag=tag, days=days)

        # create blog folder
        if self.account:
            self.blog_folder = os.path.join(BLOG_CONTENT_FOLDER, "account", self.account)
        elif self.tag:
            self.blog_folder = os.path.join(BLOG_CONTENT_FOLDER, "tag", self.tag)
        if not os.path.exists(self.blog_folder):
            os.makedirs(self.blog_folder)

    def get_name(self):
        name = "blog"
        target = self.account or self.tag
        return "{}-{}-{}".format(name, target, self._get_time_str())

    def is_qualified(self, post):
        return True

    def _get_content_folder(self):
        return self.blog_folder

    def _write_content(self, post):
        folder = self._get_content_folder()
        c = SteemComment(comment=post)

        # retrieve necessary data from steem
        title = post.title.replace('"', '')
        body = post.body
        date_str = post.json()["created"]
        date = date_str.replace('T', ' ')
        tags = "\n".join(["- {}".format(tag) for tag in c.get_tags()])
        category = c.get_tags()[0]
        thumbnail = c.get_pic_url() or ''
        url = c.get_url()

        # build content with template
        template = get_message("blog")
        content = template.format(title=title, date=date, tags=tags, category=category, thumbnail=thumbnail, body=body, url=url)

        # write into MD files
        filename = os.path.join(folder, "{}_{}.md".format(date_str.split('T')[0], post["permlink"]))
        with open(filename, "w", encoding="utf-8") as f:
            f.write(content)

        logger.info("Download post [{}] into file {}".format(title, filename))

    def download(self):
        if len(self.posts) == 0:
            self.get_latest_posts()
        if len(self.posts) > 0:
            for post in self.posts:
                self._write_content(post)


from functools import wraps

from tonguetwister.lib.stream import ByteBlockIO

registry = {}

class PropertyReaderRegistry(type):
    def __init__(cls, name, bases, attrs):
        super().__init__(name, bases, attrs)

        for key, val in attrs.items():
            prop_id = getattr(val, '_register', None)
            if prop_id is not None:
                if name not in registry:
                    registry[name] = {}
                registry[name][prop_id] = val.__name__

def property_reader(index):
    def wrap(func):
        func._register = index

        @wraps(func)
        def decorated_function(*args, **kwargs):
            return func(*args, **kwargs)

        return decorated_function

    return wrap

class PropertyReader(metaclass=PropertyReaderRegistry):
    unknown_prop_prefix = 'unknown_property_'

    def __init__(self):
        self._key_map = {}
        if self.__class__.__name__ in registry:
            # Clone so that dynamically added keys doesn't register on the class itself
            self._key_map = {k: v for k, v in registry[self.__class__.__name__].items()}

    def read(self, prop_id: int, stream: ByteBlockIO, length: int) -> dict:
        _bytes = stream.read_bytes(length)
        property_stream = ByteBlockIO(_bytes, endianess=stream.endianess)

        if prop_id in self._key_map:
            method_name = self._key_map[prop_id]
            method = getattr(self, method_name)
        elif 'default' in self._key_map:
            method_name = f'{self.unknown_prop_prefix}{prop_id}'
            method = getattr(self, self._key_map['default'])
        else:
            method_name = f'{self.unknown_prop_prefix}{prop_id}'
            method = lambda s: s.read_bytes()

        if length > 0:
            return_value = method(property_stream)

            if isinstance(return_value, tuple):
                result = return_value[0]
                split = return_value[1]
            else:
                result = return_value
                split = True
        else:
            result = bytes()
            split = False

        if not property_stream.is_depleted():
            print(property_stream.get_processed_bytes_string())
            print(property_stream.get_unprocessed_bytes_array())
            raise RuntimeError(f'Property reader did not read correct number of bytes for property id {prop_id}')

        if isinstance(result, dict) and split:
            return result
        else:
            return {method_name: result}

    def register(self, index, method_name, method):
        self._key_map[index] = method_name
        setattr(self, method_name, method)