import sys
import argparse
import math

def calculate_nominal_interest(interest):
    nominal_interest = interest / (12 * 100)
    return nominal_interest

def calculate_months(principal, monthly_payment, interest):
    nominal_interest = calculate_nominal_interest(interest)
    num_months = math.ceil(math.log(monthly_payment / (monthly_payment - nominal_interest * principal), 1 + nominal_interest))
    return num_months

def calculate_annuity_payment(principal, interest, num_months):
    nominal_interest = calculate_nominal_interest(interest)
    payment = math.ceil(principal * (nominal_interest * (1 + nominal_interest) ** num_months) / ((1 + nominal_interest) ** num_months - 1))
    return payment

def calculate_differentiated_payment(principal, interest, num_months, current_period):
    nominal_interest = calculate_nominal_interest(interest)
    payment = math.ceil((principal/num_months) + nominal_interest * (principal - ((principal * (current_period - 1))/ num_months)))
    return payment

def calculate_principal(interest, num_months, monthly_payment):
    nominal_interest = calculate_nominal_interest(interest)
    principal = math.floor(monthly_payment / ((nominal_interest * (1 + nominal_interest) ** num_months) / ((1 + nominal_interest) ** num_months - 1)))
    return principal

def calculate_total_payment(payment_type, principal, interest, periods):
    if payment_type == 'diff':
        total_payment = differentiated_payment(principal, interest, periods, verbose=False)
    elif payment_type == 'annuity':
        total_payment = payment * periods
    return total_payment

def print_months(num_months):
    years = math.floor(num_months / 12)
    months = num_months % (years * 12)
    if years == 1:
        yrs = 'year'
    else:
        yrs = 'years'
    if months == 1:
        mon = 'month'
    else:
        mon = 'months'
    if years >= 1 and months == 0:
        print(f'You need {years} {yrs} to repay this credit!')
    elif years == 0 and months >= 1:
        print(f'You need {months} {mon} to repay this credit!')
    else:
        print(f'You need {years} {yrs} and {months} {mon} to repay this credit!')

def differentiated_payment(principal, interest, periods, verbose=True):
    current_period = 0
    total_payment = 0
    while current_period < periods:
        current_period += 1
        payment = calculate_differentiated_payment(principal, interest, periods, current_period)
        total_payment += payment
        if verbose:
            print(f'Month {current_period}: payment is {payment}')
    return total_payment

def error(value_1, value_2, value_3, value_4, value_5):
    if value_1 or value_2 or value_3 or value_4 or value_5:
        print('Incorrect parameters')
        sys.exit()

# returns True to trigger error if --interest is None
def check_interest(interest):
    if interest == None:
        return True

# returns True to trigger error if --type is not 'annuity' or 'diff'
def check_type(payment_type):
    if payment_type == 'annuity' or payment_type == 'diff':
        return False
    else:
        return True

# returns True to trigger error if # parameters with a value is == 4
def check_parameters(params):
    counter = 0
    for param in params:
        if param is None:
            pass
        else:
            counter += 1
    if counter == 4:
        return False
    else:
        return True

# returns True if param is less than 0
def check_for_neg(param):
    if param is None:
        return False
    elif int(param) < 0:
        return True
    else:
        return False

# returns True to trigger error if param in params list returns a result in check_for_neg
def check_negs(params):
    for param in params[1:]:
        result = check_for_neg(param)
        if result:
            return True
    return False

# returns True to trigger error if payment_type is 'diff' and payment is not None
def check_diff_payment(payment_type, payment):
    if payment_type == 'diff' and payment != None:
        return True
    else:
        return False

parser = argparse.ArgumentParser()

# required argument
parser.add_argument('--type', type=str, required=True)

#optional arguments
parser.add_argument('--principal', type=float, default=None, help='the value of the credit principal')
parser.add_argument('--payment', type=float, default=None, help='the value of the monthly payment')
parser.add_argument('--periods', type=float, default=None, help='the number of periods to repay the credit')
parser.add_argument('--interest', type=float, default=None, help='the percentage interest on the credit')

args = parser.parse_args()

payment_type = args.type
principal = args.principal
payment = args.payment
periods = args.periods
interest = args.interest

params = [payment_type, principal, payment, periods, interest]

# check for errors
error(check_interest(interest),
      check_type(payment_type),
      check_parameters(params),
      check_negs(params),
      check_diff_payment(payment_type, payment))

# do the math
if payment_type == 'annuity':
    if principal is None:
        principal = calculate_principal(interest, periods, payment)
        print(f'Your credit principal = {principal}!')
    elif periods is None:
        periods = calculate_months(principal, payment, interest)
        print_months(periods)
    else:
        payment = calculate_annuity_payment(principal, interest, periods)
        print(f'Your annuity payment = {payment}!')
elif payment_type == 'diff':
    if principal is None:
        principal = calculate_principal(interest, periods, payment)
        print(f'Your credit principal = {principal}!')
    elif periods is None:
        periods = calculate_months(principal, payment, interest)
        print_months(periods)
    else:
        differentiated_payment(principal, interest, periods)
total = calculate_total_payment(payment_type, principal, interest, periods)
print(f'Overpayment = {int(total - principal)}')


# Assuming Python 3
import csv
from io import StringIO
import re
from munkres import Munkres

class QualtricsKBFCSVImporter(object):

    def __init__(self, filename=None, csv_content_string=None, legacy_format=False):
        # Initial inputs
        self.filename = filename
        self.csv_content_string = csv_content_string
        self.legacy_format = legacy_format
        #
        self.raw_data = list()
        self.raw_fieldnames = list()
        self.data = dict()
        self.organization_rankings = list()
        self.respondents = list()
        self.organizations = list()
        # Variables used in the initial processing of the CSV data, not meant to be used after
        self.__header_row_index = None
        self.__data_start_row_index = None
        self.__organization_name_pattern = None
        self.__organizations_by_column_index = dict()
        # This is the alternate value provided in case there is no response in the file, e.g. ''. This will be
        # parsed into an integer. This should represent a value larger than any of the actual values in the data.
        self.__NONRESPONSE_VALUE_TEXT = '99'
        self.__first_name_column_index = None
        self.__last_name_column_index = None

        if self.legacy_format:
            self.__header_row_index = 1
            self.__data_start_row_index = 2
            # Created regex using https://pythex.org/
            self.__organization_name_pattern = re.compile(r'-(?P<organization>.*)(?=-Rank)')
        else:
            self.__header_row_index = 1
            self.__data_start_row_index = 3
            # Created regex using https://pythex.org/
            self.__organization_name_pattern = re.compile(r'(?<=- Ranks - Preference - )(?P<organization>.*)(?= - Rank)')

        if self.filename:
            with open(self.filename, 'r') as f:
                self.raw_data = list(csv.reader(f))
        elif self.csv_content_string:
            self.raw_data = list(csv.reader(StringIO(self.csv_content_string)))

        # Extract column indices for the fields we care about, and extract organization names from the column headings they're embedded in
        self.raw_fieldnames = self.raw_data[self.__header_row_index]
        for column_index, field in enumerate(self.raw_fieldnames):
            if re.search(self.__organization_name_pattern, field):
                organization_name = re.search(self.__organization_name_pattern, field).groupdict()['organization']
                self.__organizations_by_column_index[organization_name] = column_index
                self.organizations.append(organization_name)
            elif 'First Name' in field:
                self.__first_name_column_index = column_index
            elif 'Last Name' in field:
                self.__last_name_column_index = column_index
        # Extract the data we care about, keyed by the full name of the respondent and then by the organization name
        respondent_name_from_row = lambda row: row[self.__last_name_column_index] + ', ' + row[self.__first_name_column_index]
        for row in self.raw_data[self.__data_start_row_index:]:
            respondent_name = respondent_name_from_row(row)
            self.respondents.append(respondent_name)
            self.data[respondent_name] = dict()
            for organization_name, column_index in self.__organizations_by_column_index.items():
                            value = row[column_index] or self.__NONRESPONSE_VALUE_TEXT
                            if value == '0':
                                value = self.__NONRESPONSE_VALUE_TEXT
                            self.data[respondent_name][organization_name] = int(value)
        self.respondents = sorted(self.respondents)
        self.organizations = sorted(self.organizations)
        # Build the numeric 2-d array, with the rows corresponding to the respondents and the columns
        # corresponding to the organizations, with each numeric value representing the priority that
        # respondent gave to that organization (max value if the respondent did not specifically
        # rank it).
        for respondent in self.respondents:
            organization_rankings_row = list()
            for organization in self.organizations:
                organization_rankings_row.append(self.data[respondent][organization])
            self.organization_rankings.append(organization_rankings_row)

    def BestOrganizationFit(self):
        m = Munkres()
        indexes = m.compute(self.organization_rankings)
        bestfit_array = list()
        for row, column in indexes:
            respondent = self.respondents[row]
            organization = self.organizations[column]
            ranking = self.organization_rankings[row][column]
            bestfit_array.append({'respondent': respondent, 'organization': organization, 'ranking': ranking})
        return bestfit_array


from __future__ import annotations

from pathlib import Path
import typing as t

import yaml

from dcdb.types import *

if t.TYPE_CHECKING:
    from .types import Card, Character
    from .engine import Engine, PathLike

class YMLua(t.NamedTuple):

    front_matter: t.Dict[str, t.Any]
    script: str

    @classmethod
    def parse(cls, path: PathLike) -> YMLua:
        with open(path) as f:
            docs = f.read().split('---\n')
        front_matter = docs[1]
        # Add blank lines so script errors report the correct line number
        front_matter_length = len(front_matter.split('\n')) + 1
        script = '\n' * front_matter_length + docs[2]
        front_matter = yaml.safe_load(front_matter)
        return cls(front_matter, script)

class EntityLoader:

    extension: str = '.ymlua'

    engine: Engine
    script_dir: Path

    def __init__(self, engine: Engine, script_dir: PathLike) -> None:
        self.engine = engine
        self.script_dir = Path(script_dir)

    def script_path(self, id: str) -> PathLike:
        return self.script_dir.joinpath(*id.split('.')).with_suffix(self.extension)

    def build_card(self, id: str) -> Card:
        self.engine.logger.debug('Building card %s', id)
        path = self.script_path(id)
        front_matter, script = YMLua.parse(path)
        name = front_matter['name']
        type_name = front_matter['type'].upper()
        try:
            type = CardType[type_name]
        except KeyError as e:
            raise ValueError(f'Invalid card type: {type_name}') from e
        color_name = front_matter['color'].upper()
        try:
            color = Color[color_name]
        except KeyError as e:
            raise ValueError(f'Invalid color: {color_name}') from e
        cost = int(front_matter['cost'])
        vp = int(front_matter['vp'])
        card = Card(self.engine, id, name, type, color, cost, vp)
        initialize = self.engine.interpreter.initializer(script, str(path))
        initialize(card)
        self.engine.logger.debug('Finished building card %s', id)
        return card

    def build_character(self, id: str) -> Character:
        self.engine.logger.debug('Building character %s', id)
        path = self.script_path(id)
        front_matter, script = YMLua.parse(path)
        name = front_matter['name']
        type_name = front_matter['type'].upper()
        try:
            type = CharacterType[type_name]
        except KeyError as e:
            raise ValueError(f'Invalid character type: {type_name}') from e
        character = Character(self.engine, id, name, type)
        initialize = self.engine.interpreter.initializer(script, str(path))
        initialize(character)
        self.engine.logger.debug('Finished building character %s', id)
        return character

class SetLoader:

    set: t.Mapping
    engine: Engine
    entity_loader: EntityLoader

    def __init__(self, engine: Engine, set_path: PathLike, script_dir: PathLike) -> None:
        with open(set_path) as f:
            self.set = yaml.safe_load(f)
        self.engine = engine
        self.entity_loader = EntityLoader(self.engine, script_dir)

    def build(self, key: str) -> t.List[Card]:
        cards = []
        for card_id, count in self.set[key].items():
            for _ in range(count):
                cards.append(self.entity_loader.build_card(card_id))
        return cards

    def build_starters(self) -> t.Iterable[Card]:
        return self.build('starters')

    def build_main_deck(self) -> t.Iterable[Card]:
        return self.build('main_deck')

    def build_kicks(self) -> t.Iterable[Card]:
        return self.build('kicks')

    def build_weaknesses(self) -> t.Iterable[Card]:
        return self.build('weaknesses')

    def build_sv_top(self) -> t.Iterable[Card]:
        cards = []
        try:
            for card_id, count in self.set['super_villains']['top'].items():
                for _ in range(count):
                    cards.append(self.entity_loader.build_card(card_id))
        except KeyError:
            pass
        return cards

    def build_sv_middle(self) -> t.Iterable[Card]:
        cards = []
        try:
            for card_id, count in self.set['super_villains']['middle'].items():
                for _ in range(count):
                    cards.append(self.entity_loader.build_card(card_id))
        except KeyError:
            pass
        return cards

    def build_character(self, id: str) -> Character:
        return self.entity_loader.build_character(id)


#############################################################
#                                                           #
#   Functionalities to transfer data from Mechanical        #
#   to Excel                                                #
# --------------------------------------------------------- #
#                                                           #
#   Programmer:         Josef Behal                         #
#   Company:            SVS FEM --> www.svsfem.cz           #
#   Ansys version:      2020 R1                             #
#                                                           #
# --------------------------------------------------------- #
#                                                           #
#   Changes:                                                #
#       25.03.2020 - Initial version                        #
#                                                           #
#############################################################

#
#   IMPORTS
#
import clr
clr.AddReference("Microsoft.Office.Interop.Excel")
import Microsoft.Office.Interop.Excel as ExcelNS
import units as UnitsLibrary

#
#   MECHANICAL
#
class MechanicalActions(object):
    '''
    Reading Mechanical Tabular data
    '''

    @staticmethod
    def GetMechData(sheetName = "ANSYS_Export"):
        sheetData = {}
        objData = {}
        aos = Tree.ActiveObjects

        # Get tabular data
        for ao in aos:
            ao.Activate()
            pane  = ExtAPI.UserInterface.GetPane(MechanicalPanelEnum.TabularData)
            table = pane.ControlUnknown
            colls = []
            for mCol in range(2, table.ColumnsCount+1):
                isActive = table.Cell(1,mCol).CheckStateChecked
                col      = [isActive] + [table.Cell(mRow,mCol).Text for mRow in range(1, table.RowsCount+1)]
                colls   += [col]

            objData[ao.ObjectId] = colls
            del(pane)

            type  = str(ao.GetType()).Split(".")[-1]
            uLab = ""
            try:    uLab = ao.MinimumTotal.Unit
            except: pass
            try:    uLab = ao.Minimum.Unit
            except: pass

            units = UnitsLibrary.Ansys.Core.Units.UnitsManager.GetQuantityNamesForUnit(uLab)
            for unit in units:
                if type.Contains(unit): break
            unit = "Displacement" if unit.ToLower() == "length" else unit
            aoUnit = "{} [{}]".format(unit,uLab)

            try:    sheetData[sheetName] += [[ao.Name, aoUnit, objData[ao.ObjectId]]]
            except: sheetData[sheetName]  = [[ao.Name, aoUnit, objData[ao.ObjectId]]]

        return sheetData

#
#   EXCEL
#
class ExcelActions(object):
    '''
    MS Excel operations
    '''
    worksheets = []
    @staticmethod
    def CreateNewWB(maximaze = True, visible = True, scrUpdate = True):
            """
            wb = ExcelActions.CreateNewWB(maximaze = True, visible = True, scrUpdate = True) --> excel workbook
            """
            excel = ExcelNS.ApplicationClass()
            wb    = excel.Workbooks.Add()

            if maximaze:
                window             = excel.Windows(1)
                window.WindowState = ExcelNS.XlWindowState.xlMaximized
            excel.Visible        = visible
            excel.ScreenUpdating = scrUpdate

            ExcelActions.currentWB = wb
            ExcelActions.currentEX = excel
            return wb

    @staticmethod
    def CreateSheet(wb = None, name = "New Sheet"):
        if wb == None: wb = ExcelActions.CreateNewWB()
        ws      = wb.Worksheets.Add()
        ws.Name = name
        ExcelActions.worksheets += [ws]
        return ws

    class Table(object):
        def __init__(self,ws):
            self.ws       = ws

        def CreateTable(self,
                        title      = "",
                        units      = "",
                        pivot      = [],
                        columns    = [],
                        actives    = [],
                        initCell   = [3,2]):
            """
            tab = Table()
            tab.AddData(title = [["","",""]], pivot = [["a","b","c"]], columns = [[1,2,3],[4,5,6],[7,8]])
            """
            self.units = units
            self.name  = title

            titleRng = self.AddDataSeries(dataLists = [[""] for i in range(len(pivot))],   initCell = [initCell[0]-2,initCell[1]])
            self.Format(titleRng, merge = True, bgColor = ExcelNS.XlRgbColor.rgbBlack , hAlign   = "xlCenter", vAlign   = "xlCenter", fntColor = 53758, fntSize  = 13, fntBold  = True,  wrapText = True)
            titleRng.Value2 = title
            self.titleRng   = titleRng

            self.pivotNames = pivot
            pivotRng = self.AddDataSeries(dataLists = [[title + ": " + p] for p in pivot],   initCell = [initCell[0]-1,initCell[1]], resizeDelim = ":")
            self.Format(pivotRng, bgColor = 53758 , hAlign   = "xlCenter", vAlign   = "xlCenter", fntBold  = True,  wrapText = True)
            self.pivotRng = pivotRng

            dataRng  = self.AddDataSeries(dataLists = columns, initCell = initCell)
            self.Format(dataRng, bgColor = ExcelNS.XlRgbColor.rgbWhite , hAlign   = "xlCenter", vAlign   = "xlCenter")
            self.dataRng = dataRng
            self.actives = actives

            return self

        def Format(self,
                   obj,
                   excelNS  = "ExcelNS",
                   merge    = False,
                   bgColor  = None, # 53758 (svsyellow), ExcelNS.XlRgbColor.rgbBlack, ...
                   hAlign   = None, # "xlCenter", "xlBottom", ...
                   vAlign   = None, # "xlCenter", "xlBottom", ...
                   fntColor = None, # 53758 (svsyellow), ExcelNS.XlRgbColor.rgbBlack, ...
                   fntSize  = None, # 13
                   fntBold  = False,
                   wrapText = False):

            if merge: obj.Merge()
            if bgColor  != None: obj.Interior.Color      = bgColor
            if hAlign   != None: obj.HorizontalAlignment = eval("{}.Constants.{}".format(excelNS,hAlign))
            if vAlign   != None: obj.VerticalAlignment   = eval("{}.Constants.{}".format(excelNS,vAlign))
            if fntColor != None: obj.Font.Color          = fntColor
            if fntSize  != None: obj.Font.Size           = fntSize
            obj.Font.Bold           = fntBold
            obj.WrapText            = wrapText

        def AddDataSeries(self, dataLists, initCell, resizeDelim = None):
            """
            dataList = [["aaaaaaaaaaaaaaaaaaaaaaa","b","c"]]
            initCell = [1,1]
            table    = Table()
            t = table.AddDataSeries(dataList, initCell)
            """

            rowInd, colInd = initCell
            for mCol, data in enumerate(dataLists):
                colS     = self.ws.Cells(rowInd, colInd)
                colE     = self.ws.Cells(rowInd + len(data), colInd)
                colRange = self.ws.Range(colS,colE)

                colVals  = colRange.Value2
                for rCol, val in enumerate(data): colVals.SetValue(val, rCol+1, 1)

                if resizeDelim:
                    column = self.ws.Columns(colInd)
                    width = column.ColumnWidth

                    valLen = max([len(word) for values in data for word in values.Split(resizeDelim)]) + 3
                    if width < valLen: column.ColumnWidth = valLen

                colRange.Value2 = colVals
                colInd += 1

            maxCol = max([len(data) for data in dataLists])
            dataS  = self.ws.Cells(initCell[0],initCell[1])
            dataE  = self.ws.Cells(rowInd + maxCol -1, colInd - 1)

            dataRange = self.ws.Range(dataS, dataE)

            return dataRange

#
# UTILS
#
def Msg(msg):ExtAPI.Log.WriteMessage(msg)

def ExportData():
    '''
    main function for data export
    '''
    sheetData = MechanicalActions.GetMechData()
    wb = ExcelActions.CreateNewWB()
    for sheetName, sheetTabs in sorted(sheetData.iteritems()):
        ws = ExcelActions.CreateSheet(wb, name = sheetName)

        tabs = []
        iRow, iCol = 5 , 2
        for name, shObjUnit, tabData in sheetTabs:

            pivot   = [col[1] for col in tabData]
            actives = [col[0] for col in tabData]
            columns = [col[2:] for col in tabData]

            tab = ExcelActions.Table(ws)
            tab.CreateTable(title = name, pivot = pivot, columns = columns, actives = actives, initCell = [iRow, iCol], units = shObjUnit)
            iCol += (len(pivot)+1)
            tabs += [tab]

#
#   CALLS
#
ExportData()


def get_django_application():
    if "django_application" not in globals():
        #django 处理wsgi的函数
        import django
        if django.get_version().startswith('1.3'):
            import django.core.handlers.wsgi
            django_application = django.core.handlers.wsgi.WSGIHandler()
        else:
            # 在模块级别保存wsgi application
            from django.core.wsgi import get_wsgi_application
            django_application = get_wsgi_application()
        globals()['django_application'] = django_application
    else:
        django_application = globals()['django_application']
    return django_application

class PatchDjango(object):
    def __init__(self, server, engine):
        self._server = server
        self._engine = engine
        self._init = False

    def __monkey_patch_django(self):
        from django.core import urlresolvers
        from .server import appengine_scheduler
        import functools

        def get_engine_callback(callback):
            #resolve会递归查找，没必要返回多次。
            if '__is_engine_callback' in callback.__dict__:
                return callback

            @functools.wraps(callback)
            def __engine_callback(*args, **kwargs):
                return appengine_scheduler(self._engine, callback, args, kwargs)

            setattr(__engine_callback, '__is_engine_callback', True)
            return __engine_callback

        # 覆盖RegexURLResolver，使得执行handler的时候使用engine的调度器
        class PatchRegexURLResolver(urlresolvers.RegexURLResolver):
            def resolve(self, path):
                resolver_match = super(PatchRegexURLResolver, self).resolve(path)
                callback = resolver_match.func

                # 重新设置callback函数
                resolver_match.func = get_engine_callback(callback)
                return resolver_match

        urlresolvers.RegexURLResolver = PatchRegexURLResolver

        # patch staitc file handler
        from django.contrib.staticfiles import views
        serve = views.serve

        views.serve = get_engine_callback(serve)

    def patch_django_Server(self):
        from .config import settings
        from .. import helper
        from .util import str_startswith_str_list

        def match_failure(environ, start_response):
            PATH_INFO = environ.get("PATH_INFO")
            if settings.DJANGO_URLS and not str_startswith_str_list(PATH_INFO, settings.DJANGO_URLS):
                return helper.not_found(start_response)
            else:
                try:
                    if not self._init:
                        # 动态载入django设置
                        # 使django的callback 和 static file handler支持appengine调度器
                        self.__monkey_patch_django()
                        self._init = True
                    django_application = get_django_application()

                    ret = django_application(environ=environ, start_response=start_response)
                except ImportError:
                    # 载入django 框架出错
                    ret = None
                # 处理结果
                if ret is None:
                    return helper.server_error(start_response)
                return ret

        self._server.match_failure = match_failure

def patch_django(server, engine):
    # 当访问到django的时候才载入django引擎，节省资源
    PatchDjango(server, engine).patch_django_Server()

#=======================================================================================================================

{'urlresolvers': 'django.core.urlresolvers', 'appengine_scheduler': 'server.appengine_scheduler', 'functools': 'functools', 'views': 'django.contrib.staticfiles.views', 'settings': 'config.settings', 'str_startswith_str_list': 'util.str_startswith_str_list'}


from flask import Flask, request, redirect, render_template
from flask_sqlalchemy import SQLAlchemy
from datetime import datetime
import cgi

app = Flask(__name__)
app.config['DEBUG'] = True      # displays runtime errors in the browser, too
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://build-a-blog:blogpassword@localhost:8889/build-a-blog'
app.config['SQLALCHEMY_ECHO'] = True

db = SQLAlchemy(app)

class BlogPost(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    title = db.Column(db.String(120))
    post_txt = db.Column(db.String(1000))
    post_datetime = db.Column(db.TIMESTAMP,default=datetime.utcnow)
    def __init__(self, title, post_txt):
        self.title = title
        self.post_txt = post_txt

def isvalidpost(title,newpost):
    title_error = ''
    blog_error = ''
    if not title:
        title_error = 'Please fill in the title'
    if not newpost:
        blog_error = 'Please fill in the body'
    return (title_error, blog_error)

@app.route('/newpost', methods=['POST', 'GET'])
def newpost():
    error = request.args.get("error")
    if request.method == 'POST':
        title = request.form['title']
        newpost = request.form['newpost']
        title_error, blog_error = isvalidpost(title,newpost)
        if (title_error or blog_error):
            return render_template('newpost.html', page_title='Add a Blog Entry', title_error=title_error, blog_error=blog_error, title=title, newpost=newpost)
        new_blog_post = BlogPost(title, newpost)
        db.session.add(new_blog_post)
        db.session.commit()
        return redirect('/blog?id={}'.format(new_blog_post.id))
    return render_template('newpost.html', page_title='Add a Blog Entry', error=error)

@app.route("/blog")
def blog():
    id = request.args.get("id")
    if id:
        the_blogs = BlogPost.query.filter_by(id=id).first()
        if the_blogs:
            title=the_blogs.title
            return render_template('1blog.html', page_title=title, post=the_blogs)
        return redirect('/') # there was no db return with the requested id.
    the_blogs = BlogPost.query.order_by('post_datetime DESC').all()
    return render_template('blog.html', page_title='Build a Blog', posts=the_blogs)

@app.route("/")
def index():
    return redirect('/blog')

if __name__ == "__main__":
    app.run()


import json
import logging
import unittest

from mock import Mock, patch

from jarr.bootstrap import conf
from jarr.controllers import ArticleController, FeedController
from jarr.crawler.crawlers.classic import ClassicCrawler
from jarr.crawler.main import clusterizer, process_feed
from jarr.crawler.requests_utils import (response_calculated_etag_match,
                                         response_etag_match)
from jarr.lib.enums import FeedType
from jarr.lib.const import UNIX_START
from jarr.lib.utils import digest
from jarr.models.feed import Feed
from tests.base import JarrFlaskCommon

logger = logging.getLogger('jarr')
BASE_COUNT = 36

def crawler():
    user_ids = set()
    for feed in FeedController().list_fetchable(limit=1):
        process_feed.apply(args=[feed.id])
        user_ids.add(feed.user_id)
    for user_id in user_ids:
        clusterizer.apply(args=[user_id])

class CrawlerTest(JarrFlaskCommon):
    feed_path = 'tests/fixtures/example.feed.atom'

    def setUp(self):
        super().setUp()
        with open(self.feed_path) as fd:
            self._content = fd.read()
        self._is_secure_served \
            = patch('jarr.lib.url_cleaners.is_secure_served')
        self._p_req = patch('jarr.crawler.crawlers.abstract.jarr_get')
        self._p_con = patch('jarr.crawler.crawlers.classic.FeedBuilderControl'
                            'ler.construct_from_xml_feed_content')
        self.is_secure_served = self._is_secure_served.start()
        self.jarr_req = self._p_req.start()
        self.jarr_con = self._p_con.start()

        self.is_secure_served.return_value = True
        self.resp_status_code = 200
        self.resp_headers = {}
        self.resp_raise = None

        def _api_req(url, **kwargs):
            if url.startswith('feed') and len(url) == 6:
                resp = Mock(status_code=self.resp_status_code,
                            headers=self.resp_headers, history=[],
                            content=self._content, text=self._content)
                resp.raise_for_status.return_value = self.resp_raise
                resp.json = lambda: json.loads(self._content)
                return resp

            url = url.split(conf.api_root)[1].strip('/')
            kwargs.pop('allow_redirects', None)
            kwargs.pop('params', None)
            kwargs.pop('json', None)
            if 'auth' in kwargs:
                kwargs['user'] = kwargs['auth'][0]
                del kwargs['auth']
            response = self._api('get', url, **kwargs)
            response.raise_for_status = Mock()
            return response

        self.jarr_con.return_value = {}
        self.jarr_req.side_effect = _api_req

    def tearDown(self):
        self._is_secure_served.stop()
        self._p_req.stop()
        self._p_con.stop()
        super().tearDown()

    @staticmethod
    def _reset_feeds_freshness(**kwargs):
        if 'expires' not in kwargs:
            kwargs['expires'] = UNIX_START
        if 'last_retrieved' not in kwargs:
            kwargs['last_retrieved'] = UNIX_START
        if 'etag' not in kwargs:
            kwargs['etag'] = ''
        if 'last_modified' not in kwargs:
            kwargs['last_modified'] = ''
        FeedController().update({}, kwargs)

    def test_http_crawler_add_articles(self):
        self.assertEqual(BASE_COUNT, ArticleController().read().count())

        crawler()
        articles = list(ArticleController().read())
        new_count = len(articles)
        self.assertNotEqual(BASE_COUNT, new_count)
        self.assertTrue(BASE_COUNT < new_count)

        for art in articles:
            self.assertFalse('srcset=' in art.content)
            self.assertFalse('src="/' in art.content)

        self.resp_status_code = 304
        crawler()
        self.assertEqual(new_count, ArticleController().read().count())

    def test_no_add_on_304(self):
        self.resp_status_code = 304
        self.assertEqual(BASE_COUNT, ArticleController().read().count())
        crawler()
        self.assertEqual(BASE_COUNT, ArticleController().read().count())

    def test_no_add_feed_skip(self):
        self.resp_status_code = 304
        self.assertEqual(BASE_COUNT, ArticleController().read().count())
        crawler()
        FeedController().update({}, {'filters': [{"type": "tag contains",
                                                  "action on": "match",
                                                  "pattern": "pattern5",
                                                  "action": "skipped"},
                                                 {"type": "simple match",
                                                  "action on": "match",
                                                  "pattern": "pattern5",
                                                  "action": "mark as read"},
                                                 {"type": "regex",
                                                  "action on": "match",
                                                  "pattern": "pattern5",
                                                  "action": "skipped"}]})

        crawler()
        self.assertEqual(BASE_COUNT, ArticleController().read().count())

    def test_matching_etag(self):
        self._reset_feeds_freshness(etag='fake etag')
        self.resp_headers = {'etag': 'fake etag'}
        self.assertEqual(BASE_COUNT, ArticleController().read().count())

        crawler()

        self.assertEqual(BASE_COUNT, ArticleController().read().count())
        self._reset_feeds_freshness(etag='jarr/"%s"' % digest(self._content))
        self.resp_headers = {'etag': 'jarr/"%s"' % digest(self._content)}

        crawler()
        self.assertEqual(BASE_COUNT, ArticleController().read().count())

        self._reset_feeds_freshness(etag='jarr/fake etag')
        self.resp_headers = {'etag': '########################'}

        crawler()
        self.assertNotEqual(BASE_COUNT, ArticleController().read().count())

class JsonCrawlerTest(CrawlerTest):
    feed_path = 'tests/fixtures/feed.json'

    def setUp(self):
        super().setUp()
        self.resp_headers = {'content-type': 'application/json'}
        FeedController().update({}, {'feed_type': 'json'})

class CrawlerMethodsTest(unittest.TestCase):

    def setUp(self):
        super().setUp()
        self.feed = Feed(user_id=1, id=1, title='title',
                         description='description', etag='', error_count=5,
                         feed_type=FeedType.classic, link='link')
        self.resp = Mock(text='text', headers={}, status_code=304, history=[])

    def test_etag_matching_w_constructed_etag(self):
        self.feed.etag = 'jarr/"%s"' % digest('text')
        self.assertFalse(response_etag_match(self.feed, self.resp))
        self.assertTrue(response_calculated_etag_match(self.feed, self.resp))

    def test_etag_no_matching_wo_etag(self):
        self.feed.etag = ''
        self.assertFalse(response_etag_match(self.feed, self.resp))
        self.assertFalse(response_calculated_etag_match(self.feed, self.resp))

    def test_etag_matching(self):
        self.resp.headers['etag'] = self.feed.etag = 'etag'
        self.assertTrue(response_etag_match(self.feed, self.resp))
        self.assertFalse(response_calculated_etag_match(self.feed, self.resp))

    @patch('jarr.crawler.main.FeedController.update')
    def test_set_feed_error_w_error(self, fctrl_update):
        original_error_count = self.feed.error_count
        ClassicCrawler(self.feed).set_feed_error(Exception('an error'))

        fctrl_update.assert_called_once()
        filters, data = fctrl_update.mock_calls[0][1]
        self.assertEqual(filters['id'], self.feed.id)
        self.assertEqual(original_error_count + 1, data['error_count'])
        self.assertEqual('an error', data['last_error'])

    @patch('jarr.crawler.main.FeedController.update')
    def test_set_feed_error_w_parsed(self, fctrl_update):
        original_error_count = self.feed.error_count
        ClassicCrawler(self.feed).set_feed_error(
                parsed_feed={'bozo_exception': 'an error'})

        fctrl_update.assert_called_once()
        filters, data = fctrl_update.mock_calls[0][1]
        self.assertEqual(filters['id'], self.feed.id)
        self.assertEqual(original_error_count + 1, data['error_count'])
        self.assertEqual('an error', data['last_error'])

    @patch('jarr.crawler.main.FeedController.update')
    def test_clean_feed(self, fctrl_update):
        ClassicCrawler(self.feed).clean_feed(self.resp)
        fctrl_update.assert_called_once()
        filters, data = fctrl_update.mock_calls[0][1]

        self.assertEqual(filters['id'], self.feed.id)
        self.assertTrue('link' not in data)
        self.assertTrue('title' not in data)
        self.assertTrue('description' not in data)
        self.assertTrue('site_link' not in data)
        self.assertTrue('icon_url' not in data)

    @patch('jarr.crawler.main.FeedController.update')
    def test_clean_feed_update_link(self, fctrl_update):
        self.resp.history.append(Mock(status_code=301))
        self.resp.url = 'new_link'
        ClassicCrawler(self.feed).clean_feed(self.resp)

        fctrl_update.assert_called_once()
        filters, data = fctrl_update.mock_calls[0][1]
        self.assertEqual(filters['id'], self.feed.id)
        self.assertEqual('new_link', data['link'])
        self.assertTrue('title' not in data)
        self.assertTrue('description' not in data)
        self.assertTrue('site_link' not in data)
        self.assertTrue('icon_url' not in data)


#ADD thư viện cần sử dụng
import numpy as np
import cv2
#Tìm đường viền
img = cv2.imread('D:\\Python_XLA\\phathiendoituong\\imgs.png')
imgGry = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)          #Chuyển ảnh xám thành ảnh grayscale
#img1 = img.copy()
hsv_frame = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)        #Chuyển ảnh xám thành ảnh grayscale
ret,thrash = cv2.threshold(imgGry, 240, 255, cv2.CHAIN_APPROX_NONE)     #Nhị phân hóa bức ảnh bằng cách đặt ngưỡng, với giá trị của ngưỡng là 240
contours, hierachy = cv2.findContours(thrash, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)     #Tìm contour
#Bắt đầu vòng lặp tìm contour từng hình
for contour in contours:
    M = cv2.moments(contour)        # Tính toán trung tâm của đường viền
    cX = int(M['m10']/M['m00'])
    cY = int(M['m01']/M['m00'])

    approx = cv2.approxPolyDP(contour, 0.01 * cv2.arcLength(contour, True), True)       #Tính xấp xỉ đường viền để xác định hình
    cv2.drawContours(img, [approx], 0, (0, 0, 0), 5)  #Vẽ lại ảnh contour vào ảnh gốc
    x = approx.ravel()[0]
    y = approx.ravel()[1] - 5
    if len(approx) == 3:            #Hình tam giác có 3 đỉnh
        cv2.putText(img, "Tam Giac", (x, y), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 0))
    elif len(approx) == 4:      #Hình chữ nhật, hình vuông có 4 đỉnh
        x, y, w, h = cv2.boundingRect(approx)           #Tính toán giới hạn của đường bao và tỉ lệ khung hình để xác đinh hình vuông hay hình chữ nhật
        AspectRatio = float(w) / h      #Tính tỉ lệ khung hình
        if AspectRatio >= 0.95 and AspectRatio < 1.05:   #Hình vuông tỉ lệ khung hình gần bằng 1, ngược lại là hình chữ nhật
            cv2.putText(img, "Hinh Vuong", (x, y-10), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 0))     #Viết một chuỗi kỹ tự lên bức hình
        else:
            cv2.putText(img, "Hinh Chu Nhat", (x, y), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 0))
    elif len(approx) == 5:      #Hình ngũ giác có 5 đỉnh
        cv2.putText(img, "Ngu Giac", (x, y), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 0))
    elif len(approx) == 10:        #Hình ngôi sao 10 đỉnh
        cv2.putText(img, "Ngoi Sao", (x, y), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 0))       #Viết một chuỗi kỹ tự lên bức hình
    elif len(approx) == 7:      #Hình mũi tên có 7 đỉnh
        cv2.putText(img, "Mui Ten", (x-90, y-60), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 0))
    elif len(approx) == 11:   #Hình tia sét có 11 đỉnh
        cv2.putText(img, "Tia Set", (x, y-30), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 0))
    elif len(approx) > 11 and len(approx) < 13 :    #Hình tròn
        cv2.putText(img, "Hinh Tron", (x, y), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 0))
    else:    #Hình trái tim
        cv2.putText(img, "Heart", (x, y), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 0))

    pixel_center = hsv_frame[cY, cX]       #Tọa độ trung tâm
    hue_value = pixel_center[0]     #Giá trị tông màu HUE

    color = 'undefined'
    if hue_value <10:
        color = "DO"
    elif hue_value <22:
        color = "CAM"
    elif hue_value <33:
        color = "VANG"
    elif hue_value < 80:
        color = "LUC"
    elif hue_value <100:
        color = "XANH DA TROI"
    elif hue_value <150:
        color = "LAM"
    elif hue_value <170:
        color = "TIM"
    elif hue_value <171:
        color = "HONG"
    else:
        color = "DO"

    #pixel_center_bgr = img[cY, cX]
    #b, g, r = int(pixel_center_bgr[0]), int(pixel_center_bgr[1]), int(pixel_center_bgr[2])
    cv2.putText(img, color, (cX+10, cY+10), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 0), 1)        #Viết một chuỗi ký tự name color
    cv2.circle(img, (cX,cY),3,(255,255,255),-1)     #Vẽ đường viền
cv2.imshow('image',img)
cv2.waitKey(0)


import sys
import os
from dotenv import load_dotenv
import importlib.util

class Config:

    def __init__(self):
        self._data = {}
        self.load_env()
        self.load_configs()

    @staticmethod
    def load_env():
        try:
            load_dotenv('.env')  # Cargamos el archivo env
        except Exception as e:
            print("Error: No fue posible cargar el archivo .env")
            print(e)
            sys.exit(1)

    def load_configs(self):
        path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), os.getenv("DIR_CONFIGURATIONS"))
        files = os.listdir(path)
        for file in files:
            # Ignorar archivos que no son configuraciones
            if not file.endswith(".py"):
                continue
            # Importar la configuracion
            module_name = file[:-3]  # Quitar la extensión .py
            module_path = os.path.join(path, file)
            spec = importlib.util.spec_from_file_location(module_name, module_path)
            module = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(module)
            variables = module.__dict__
            for variable_name, variable_value in variables.items():
                if not variable_name.startswith("__") and not variable_name == "Tools":  # Ignorar variables especiales
                    self._data[variable_name] = variable_value

    def get(self,config,prop):
        try:
         return self._data[config][prop]
        except (Exception,):
            return None


import shutil
from typing import Callable, Optional

from wandb.sdk.lib import filenames
from wandb.sdk.lib.filenames import WANDB_DIRS

from models.adapted_op import AdaptedLeafCNN1, AdaptedLeNet, AdaptedLeafCNN3
from pFedGate.gate_aggregator import pFedGateAggregator
from pFedGate.gated_learner import GatedLearner
from pFedGate.gated_client import pFedGateClient
from models.gating_layers import GatingLayer
from models.nn_nets import *
from datasets import *
from learners.learner import *
from learners.learners_ensemble import *
from client import *
from aggregator import *

from .optim import *
from .metrics import *
from .constants import *
from .decentralized import *

from torch.utils.data import DataLoader
import tempfile
import tarfile

from tqdm import tqdm

import math

from .sparse_factor_schedule import SparsityLinearScheduler, SparsityMultiStepScheduler, \
    SparsityReduceLROnPlateauScheduler

def get_data_dir(experiment_name):
    """
    returns a string representing the path where to find the datafile corresponding to the experiment

    :param experiment_name: name of the experiment
    :return: str

    """
    if "." in experiment_name:
        name, sub_name = experiment_name.split(".")
        data_dir = os.path.join("data", name, sub_name, "all_data")
    else:
        data_dir = os.path.join("data", experiment_name, "all_data")

    return data_dir

def get_sparse_factor_scheduler(scheduler_name, n_rounds, s_target):
    if scheduler_name == "constant":
        return None
    elif scheduler_name == "linear":
        # similar ref <<To prune, or not to prune: exploring the efficacy of pruning for model compression>>
        prune_begin_round = 0.2 * n_rounds
        total_decay_rounds = 0.5 * n_rounds
        return SparsityLinearScheduler(prune_begin_round=prune_begin_round, total_rounds=total_decay_rounds,
                                       s_target=s_target, s_begin=1)
    elif scheduler_name == "multi_step":
        decay_times = 4
        decay_each_time = (1 - s_target) / decay_times
        total_decay_rounds = n_rounds * 0.8
        decay_round_delta = total_decay_rounds / decay_times
        s_list = [1] + [1 - decay_each_time * (i + 1) for i in range(decay_times)] + [s_target]
        round_list = [0] + [int(decay_round_delta * (i + 1)) for i in range(decay_times)] + [int(n_rounds)]
        return SparsityMultiStepScheduler(change_round_list=round_list, s_list=s_list)
    elif scheduler_name == "reduce_on_plateau":
        decay_times = 4
        decay_factor_bound = math.pow(s_target, 1 / decay_times)
        return SparsityReduceLROnPlateauScheduler(decay_factor=decay_factor_bound, init_s=1, min_sparse_factor=s_target,
                                                  mode="max", patience=10)
    else:
        raise NotImplementedError(f"Unsupported sparse_factor_scheduler: {scheduler_name}")

def get_learner(
        name,
        device,
        optimizer_name,
        scheduler_name,
        initial_lr,
        mu,
        n_rounds,
        seed,
        input_dim=None,
        output_dim=None,
        gated_learner=False,
        alpha=0,
        beta=0,
        sparse_factor=1,
        block_wise_prune=False,
        fine_grained_block_split=1,
        sparse_factor_scheduler="constant",
        track_running_stats=1,
        shared_model=None,
        args_=None
):
    """
    constructs the learner corresponding to an experiment for a given seed

    :param name: name of the experiment to be used; possible are
                 {`synthetic`, `cifar10`, `emnist`, `shakespeare`}
    :param device: used device; possible `cpu` and `cuda`
    :param optimizer_name: passed as argument to utils.optim.get_optimizer
    :param scheduler_name: passed as argument to utils.optim.get_lr_scheduler
    :param initial_lr: initial value of the learning rate
    :param mu: proximal term weight, only used when `optimizer_name=="prox_sgd"`
    :param input_dim: input dimension, only used for synthetic dataset
    :param output_dim: output_dimension; only used for synthetic dataset
    :param n_rounds: number of training rounds, only used if `scheduler_name == multi_step`, default is None;
    :param seed:
    :return: Learner

    """
    torch.manual_seed(seed)

    criterion, is_binary_classification, metric, model = get_model(device, input_dim, name, output_dim, shared_model,
                                                                   model_type=args_.model_type)

    if isinstance(initial_lr, list):
        lr_model, lr_gating = initial_lr
    else:
        lr_model = lr_gating = initial_lr
    optimizer = \
        get_optimizer(
            optimizer_name=optimizer_name,
            model=model,
            lr_initial=lr_model,
            mu=mu
        )
    lr_scheduler = \
        get_lr_scheduler(
            optimizer=optimizer,
            scheduler_name=scheduler_name,
            n_rounds=n_rounds,
            gated_learner=gated_learner,
        )
    sparse_factor_scheduler = get_sparse_factor_scheduler(
        scheduler_name=sparse_factor_scheduler, n_rounds=n_rounds, s_target=sparse_factor)

    if not gated_learner:
        if name == "shakespeare":
            return LanguageModelingLearner(
                model=model,
                criterion=criterion,
                metric=metric,
                device=device,
                optimizer=optimizer,
                lr_scheduler=lr_scheduler,
                is_binary_classification=is_binary_classification
            )
        else:
            return Learner(
                model=model,
                criterion=criterion,
                metric=metric,
                device=device,
                optimizer=optimizer,
                lr_scheduler=lr_scheduler,
                is_binary_classification=is_binary_classification
            )
    else:
        # set up learned gating layer and its optimizer, lr_scheduler
        gating_layer = GatingLayer(model_to_mask=model, device=device, dataset_name=name,
                                   fine_grained_block_split=fine_grained_block_split,
                                   seperate_trans=args_.seperate_trans)
        opt_for_gating = \
            get_optimizer(
                optimizer_name=optimizer_name,
                model=gating_layer,
                lr_initial=lr_gating,
                mu=mu
            )
        lr_scheduler_for_gating = \
            get_lr_scheduler(
                optimizer=opt_for_gating,
                scheduler_name=scheduler_name,
                n_rounds=n_rounds,
                gated_learner=gated_learner
            )

        return GatedLearner(
            base_model=model,
            criterion=criterion,
            metric=metric,
            device=device,
            optimizer=optimizer,
            lr_scheduler=lr_scheduler,
            is_binary_classification=is_binary_classification,
            gating_layer=gating_layer,
            opt_for_gating=opt_for_gating,
            lr_scheduler_for_gating=lr_scheduler_for_gating,
            alpha=alpha,
            beta=beta,
            sparse_factor=sparse_factor,
            block_wise_prune=block_wise_prune,
            sparse_factor_scheduler=sparse_factor_scheduler,
            bn_running_stats=track_running_stats,
            args_=args_
        )

def get_model(device, input_dim, name, output_dim, shared_model=None, model_type=None):
    model = shared_model
    if name == "synthetic":
        if output_dim == 2:
            criterion = nn.BCEWithLogitsLoss(reduction="none").to(device)
            metric = binary_accuracy
            if not shared_model:
                model = LinearLayer(input_dim, 1).to(device)
            is_binary_classification = True
        else:
            criterion = nn.CrossEntropyLoss(reduction="none").to(device)
            metric = accuracy
            if not shared_model:
                model = LinearLayer(input_dim, output_dim).to(device)
            is_binary_classification = False
    elif name == "cifar10":
        criterion = nn.CrossEntropyLoss(reduction="none").to(device)
        metric = accuracy
        if not shared_model:
            # model = get_mobilenet(n_classes=10).to(device)
            # model = get_resnet18(n_classes=10).to(device)
            model = AdaptedLeNet(num_classes=10).to(device)
        is_binary_classification = False
    elif name == "cifar100":
        criterion = nn.CrossEntropyLoss(reduction="none").to(device)
        metric = accuracy
        if not shared_model:
            if model_type == "lenet5":
                model = AdaptedLeNet(num_classes=100, n_kernels=32, in_channels=3, fc_factor=1).to(device)
            elif model_type == "lenet5_wide":
                model = AdaptedLeNet(num_classes=100, n_kernels=32 * 4, in_channels=3, fc_factor=4).to(device)
            elif model_type == "lenet5_fc_wide":
                model = AdaptedLeNet(num_classes=100, n_kernels=32, in_channels=3, fc_factor=16, fc_factor2=6).to(
                    device)
            elif model_type == "cnn_leaf":
                model = AdaptedLeafCNN3(num_classes=100).to(device)
            elif model_type == "cnn_leaf_wide":
                model = AdaptedLeafCNN3(num_classes=100, n_kernels=32 * 2, in_channels=3, fc_factor=2).to(device)
            else:
                model = AdaptedLeNet(num_classes=100, n_kernels=32, in_channels=3, fc_factor=1).to(device)
        is_binary_classification = False
    elif name == "emnist" or name == "femnist":
        criterion = nn.CrossEntropyLoss(reduction="none").to(device)
        metric = accuracy
        if not shared_model:
            model = AdaptedLeafCNN1(num_classes=62).to(device)
        is_binary_classification = False
    else:
        raise NotImplementedError
    return criterion, is_binary_classification, metric, model

def get_learners_ensemble(
        n_learners,
        name,
        device,
        optimizer_name,
        scheduler_name,
        initial_lr,
        mu,
        n_rounds,
        seed,
        input_dim=None,
        output_dim=None,
        gated_learner=False,
        alpha=0,
        beta=0,
        sparse_factor=1,
        block_wise_prune=False,
        fine_grained_block_split=1,
        sparse_factor_scheduler="constant",
        track_running_stats=1,
        shared_model=None,
        args_=None
):
    """
    constructs the learner corresponding to an experiment for a given seed

    :param n_learners: number of learners in the ensemble
    :param name: name of the experiment to be used; possible are
                 {`synthetic`, `cifar10`, `emnist`, `shakespeare`}
    :param device: used device; possible `cpu` and `cuda`
    :param optimizer_name: passed as argument to utils.optim.get_optimizer
    :param scheduler_name: passed as argument to utils.optim.get_lr_scheduler
    :param initial_lr: initial value of the learning rate
    :param mu: proximal term weight, only used when `optimizer_name=="prox_sgd"`
    :param input_dim: input dimension, only used for synthetic dataset
    :param output_dim: output_dimension; only used for synthetic dataset
    :param n_rounds: number of training rounds, only used if `scheduler_name == multi_step`, default is None;
    :param seed:
    :return: LearnersEnsemble

    """
    learners = [
        get_learner(
            name=name,
            device=device,
            optimizer_name=optimizer_name,
            scheduler_name=scheduler_name,
            initial_lr=initial_lr,
            input_dim=input_dim,
            output_dim=output_dim,
            n_rounds=n_rounds,
            seed=seed + learner_id,
            mu=mu,
            gated_learner=gated_learner,
            alpha=alpha,
            beta=beta,
            sparse_factor=sparse_factor,
            block_wise_prune=block_wise_prune,
            fine_grained_block_split=fine_grained_block_split,
            sparse_factor_scheduler=sparse_factor_scheduler,
            track_running_stats=track_running_stats,
            shared_model=shared_model,
            args_=args_
        ) for learner_id in range(n_learners)
    ]

    learners_weights = torch.ones(n_learners) / n_learners
    if name == "shakespeare":
        return LanguageModelingLearnersEnsemble(learners=learners, learners_weights=learners_weights)
    else:
        return LearnersEnsemble(learners=learners, learners_weights=learners_weights)

def get_loaders(type_, root_path, batch_size, is_validation):
    """
    constructs lists of `torch.utils.DataLoader` object from the given files in `root_path`;
     corresponding to `train_iterator`, `val_iterator` and `test_iterator`;
     `val_iterator` iterates on the same dataset as `train_iterator`, the difference is only in drop_last

    :param type_: type of the dataset;
    :param root_path: path to the data folder
    :param batch_size:
    :param is_validation: (bool) if `True` validation part is used as test
    :return:
        train_iterator, val_iterator, test_iterator
        (List[torch.utils.DataLoader], List[torch.utils.DataLoader], List[torch.utils.DataLoader])

    """
    if type_ == "cifar10":
        inputs, targets = get_cifar10()
    elif type_ == "cifar100":
        inputs, targets = get_cifar100()
    elif type_ == "emnist":
        inputs, targets = get_emnist()
    else:
        inputs, targets = None, None

    train_iterators, val_iterators, test_iterators = {}, {}, {}

    # IO-intensive init since data loader indices are stored in many small (pickle) files,
    # set PARALLEL_INIT as true to speed up the initialization via multi-processing
    PARALLEL_INIT = True

    if not PARALLEL_INIT:
        for _, task_dir in enumerate(tqdm(os.listdir(root_path))):
            test_iterator, train_iterator, val_iterator, task_id = init_iterators_for_one_client(batch_size, inputs,
                                                                                is_validation,
                                                                                root_path,
                                                                                targets, task_dir,
                                                                                type_)

            train_iterators[task_id] = train_iterator
            val_iterators[task_id] = val_iterator
            test_iterators[task_id] = test_iterator
    else:
        from concurrent.futures import ThreadPoolExecutor
        torch.multiprocessing.set_sharing_strategy('file_system')
        import resource
        rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
        assert 1048576 <= rlimit[1]
        resource.setrlimit(resource.RLIMIT_NOFILE, (1048576, rlimit[1]))

        def loader_init_callback_multi_process(ret):
            test_iterator, train_iterator, val_iterator, task_id = ret
            train_iterators[task_id] = train_iterator
            val_iterators[task_id] = val_iterator
            test_iterators[task_id] = test_iterator

        def loader_init_callback_multi_thread(ret):
            test_iterator, train_iterator, val_iterator, task_id = ret.result()
            train_iterators[task_id] = train_iterator
            val_iterators[task_id] = val_iterator
            test_iterators[task_id] = test_iterator

        with ThreadPoolExecutor() as pool:
            # with ProcessPoolExecutor() as pool:
            for _, task_dir in enumerate(tqdm(os.listdir(root_path))):
                future = pool.submit(init_iterators_for_one_client, batch_size, inputs,
                                     is_validation, root_path,
                                     targets, task_dir, type_)
                future.add_done_callback(loader_init_callback_multi_thread)

    return train_iterators, val_iterators, test_iterators

def init_iterators_for_one_client(batch_size, inputs, is_validation, root_path, targets, task_dir, type_):
    task_data_path = os.path.join(root_path, task_dir)
    task_id = task_dir.split("_")[-1]
    logging.info(f"BEGIN: init iterators for task {task_id}")
    train_iterator = \
        get_loader(
            type_=type_,
            path=os.path.join(task_data_path, f"train{EXTENSIONS[type_]}"),
            batch_size=batch_size,
            inputs=inputs,
            targets=targets,
            train=True
        )
    if os.path.exists(os.path.join(task_data_path, f"val{EXTENSIONS[type_]}")):
        val_iterator = \
            get_loader(
                type_=type_,
                path=os.path.join(task_data_path, f"val{EXTENSIONS[type_]}"),
                batch_size=batch_size,
                inputs=inputs,
                targets=targets,
                train=False
            )
    else:
        val_iterator = None
    if is_validation:
        test_set = "val"
    else:
        test_set = "test"
    test_iterator = \
        get_loader(
            type_=type_,
            path=os.path.join(task_data_path, f"{test_set}{EXTENSIONS[type_]}"),
            batch_size=batch_size,
            inputs=inputs,
            targets=targets,
            train=False
        )
    logging.info(f"END: init iterators for task {task_id}")
    return test_iterator, train_iterator, val_iterator, task_id

def get_loader(type_, path, batch_size, train, inputs=None, targets=None):
    """
    constructs a torch.utils.DataLoader object from the given path

    :param type_: type of the dataset; possible are `tabular`, `images` and `text`
    :param path: path to the data file
    :param batch_size:
    :param train: flag indicating if train loader or test loader
    :param inputs: tensor storing the input data; only used with `cifar10`, `cifar100` and `emnist`; default is None
    :param targets: tensor storing the labels; only used with `cifar10`, `cifar100` and `emnist`; default is None
    :return: torch.utils.DataLoader

    """
    if type_ == "tabular":
        dataset = TabularDataset(path)
    elif type_ == "cifar10":
        dataset = SubCIFAR10(path, cifar10_data=inputs, cifar10_targets=targets)
    elif type_ == "cifar100":
        dataset = SubCIFAR100(path, cifar100_data=inputs, cifar100_targets=targets)
    elif type_ == "emnist":
        dataset = SubEMNIST(path, emnist_data=inputs, emnist_targets=targets)
    elif type_ == "femnist":
        dataset = SubFEMNIST(path)
    elif type_ == "shakespeare":
        dataset = CharacterDataset(path, chunk_len=SHAKESPEARE_CONFIG["chunk_len"])
    else:
        raise NotImplementedError(f"{type_} not recognized type; possible are {list(LOADER_TYPE.keys())}")

    if len(dataset) == 0:
        return

    # drop last batch, because of BatchNorm layer used in mobilenet_v2
    drop_last = ((type_ == "cifar100") or (type_ == "cifar10")) and (len(dataset) > batch_size) and train

    g = torch.Generator()
    g.manual_seed(1234)

    return DataLoader(dataset, batch_size=batch_size, shuffle=train, drop_last=drop_last, generator=g)

def log_codes(
        wandb_run,
        outdir,
        log_to_wandb=False,  # Error 411
        root: str = ".",
        name: str = None,
        include_fn: Callable[[str], bool] = lambda path: path.endswith(".py"),
        exclude_fn: Callable[[str], bool] = filenames.exclude_wandb_fn,
) -> Optional[wandb.Artifact]:
    """Saves the current state of your code to a W&B artifact.

    By default it walks the current directory and logs all files that end with `.py`.

    Arguments:
        root (str, optional): The relative (to `os.getcwd()`) or absolute path to
            recursively find code from.
        name (str, optional): The name of our code artifact. By default we'll name
            the artifact `source-$RUN_ID`. There may be scenarios where you want
            many runs to share the same artifact. Specifying name allows you to achieve that.
        include_fn (callable, optional): A callable that accepts a file path and
            returns True when it should be included and False otherwise. This
            defaults to: `lambda path: path.endswith(".py")`
        exclude_fn (callable, optional): A callable that accepts a file path and
            returns `True` when it should be excluded and `False` otherwise. This
            defaults to: `lambda path: False`

    Examples:
        Basic usage
        ```python
        run.log_code()
        ```

        Advanced usage
        ```python
        run.log_code("../", include_fn=lambda path: path.endswith(".py") or path.endswith(".ipynb"))
        ```

    Returns:
        An `Artifact` object if code was logged
    """
    name = name or "{}-{}".format("source", wandb_run.id)
    if log_to_wandb:
        art = wandb.Artifact(name, "code")
    files_added = False
    temp = tempfile.NamedTemporaryFile('wb', suffix='.tar.gz', delete=False)
    tar = tarfile.open(fileobj=temp, mode='w:gz')
    if root is not None:
        root = os.path.abspath(root)
        for file_path in filtered_dir(root, include_fn, exclude_fn):
            files_added = True
            save_name = os.path.relpath(file_path, root)
            if log_to_wandb:
                art.add_file(file_path, name=save_name)
            tar.add(file_path, arcname=save_name)
    tar.close()
    shutil.copyfile(temp.name, os.path.join(outdir, name + '-code.tar.gz'))

    # Add any manually staged files such is ipynb notebooks
    for dirpath, _, files in os.walk(wandb_run._settings._tmp_code_dir):
        for fname in files:
            file_path = os.path.join(dirpath, fname)
            save_name = os.path.relpath(file_path, wandb_run._settings._tmp_code_dir)
            files_added = True
            if log_to_wandb:
                art.add_file(file_path, name=save_name)
    if not files_added:
        return None
    if log_to_wandb:
        wandb_run.log_artifact(art)
    return

def filtered_dir(
        root: str, include_fn: Callable[[str], bool], exclude_fn: Callable[[str], bool]
):
    """Simple generator to walk a directory"""
    for dirpath, dirs, files in os.walk(root, topdown=True):
        dirs[:] = [d for d in dirs if not exclude_fn(d)]
        for fname in files:
            file_path = os.path.join(dirpath, fname)
            if include_fn(file_path) and not exclude_fn(file_path):
                yield file_path

def exclude_wandb_fn(path: str) -> bool:
    return any(os.sep + wandb_dir + os.sep in path for wandb_dir in WANDB_DIRS)

def get_client(
        client_type,
        learners_ensemble,
        q,
        train_iterator,
        val_iterator,
        test_iterator,
        logger,
        local_steps,
        tune_locally,
        node_id=0,
        bi_level_opt=1
):
    """

    :param client_type:
    :param learners_ensemble:
    :param q: fairness hyper-parameter, ony used for FFL client
    :param train_iterator:
    :param val_iterator:
    :param test_iterator:
    :param logger:
    :param local_steps:
    :param tune_locally

    :return:

    """
    if client_type == "mixture":
        return MixtureClient(
            learners_ensemble=learners_ensemble,
            train_iterator=train_iterator,
            val_iterator=val_iterator,
            test_iterator=test_iterator,
            logger=logger,
            local_steps=local_steps,
            node_id=node_id,
            tune_locally=tune_locally
        )
    elif client_type == "AFL":
        return AgnosticFLClient(
            learners_ensemble=learners_ensemble,
            train_iterator=train_iterator,
            val_iterator=val_iterator,
            test_iterator=test_iterator,
            logger=logger,
            local_steps=local_steps,
            tune_locally=tune_locally
        )
    elif client_type == "FFL":
        return FFLClient(
            learners_ensemble=learners_ensemble,
            train_iterator=train_iterator,
            val_iterator=val_iterator,
            test_iterator=test_iterator,
            logger=logger,
            local_steps=local_steps,
            tune_locally=tune_locally,
            q=q
        )
    elif client_type == "pFedGate":
        return pFedGateClient(
            learners_ensemble=learners_ensemble,
            train_iterator=train_iterator,
            val_iterator=val_iterator,
            test_iterator=test_iterator,
            logger=logger,
            local_steps=local_steps,
            tune_locally=tune_locally,
            node_id=node_id,
            bi_level_opt=bi_level_opt
        )
    else:
        return Client(
            learners_ensemble=learners_ensemble,
            train_iterator=train_iterator,
            val_iterator=val_iterator,
            test_iterator=test_iterator,
            logger=logger,
            local_steps=local_steps,
            tune_locally=tune_locally
        )

def get_aggregator(
        aggregator_type,
        clients,
        global_learners_ensemble,
        lr,
        lr_lambda,
        mu,
        communication_probability,
        q,
        sampling_rate,
        log_freq,
        global_train_logger,
        global_test_logger,
        test_clients,
        verbose,
        seed=None,
        aggregate_sampled_clients=0,
        online_aggregate=0,
        outdir=""
):
    """
    `personalized` corresponds to pFedMe

    :param aggregator_type:
    :param clients:
    :param global_learners_ensemble:
    :param lr: oly used with FLL aggregator
    :param lr_lambda: only used with Agnostic aggregator
    :param mu: penalization term, only used with L2SGD
    :param communication_probability: communication probability, only used with L2SGD
    :param q: fairness hyper-parameter, ony used for FFL client
    :param sampling_rate:
    :param log_freq:
    :param global_train_logger:
    :param global_test_logger:
    :param test_clients
    :param verbose: level of verbosity
    :param seed: default is None
    :return:

    """
    seed = (seed if (seed is not None and seed >= 0) else int(time.time()))
    if aggregator_type == "no_communication":
        return NoCommunicationAggregator(
            clients=clients,
            global_learners_ensemble=global_learners_ensemble,
            log_freq=log_freq,
            global_train_logger=global_train_logger,
            global_test_logger=global_test_logger,
            test_clients=test_clients,
            sampling_rate=sampling_rate,
            verbose=verbose,
            seed=seed
        )
    elif aggregator_type == "centralized":
        return CentralizedAggregator(
            clients=clients,
            global_learners_ensemble=global_learners_ensemble,
            log_freq=log_freq,
            global_train_logger=global_train_logger,
            global_test_logger=global_test_logger,
            test_clients=test_clients,
            sampling_rate=sampling_rate,
            verbose=verbose,
            seed=seed
        )
    elif aggregator_type == "personalized":
        return PersonalizedAggregator(
            clients=clients,
            global_learners_ensemble=global_learners_ensemble,
            log_freq=log_freq,
            global_train_logger=global_train_logger,
            global_test_logger=global_test_logger,
            test_clients=test_clients,
            sampling_rate=sampling_rate,
            verbose=verbose,
            seed=seed
        )
    elif aggregator_type == "clustered":
        return ClusteredAggregator(
            clients=clients,
            global_learners_ensemble=global_learners_ensemble,
            log_freq=log_freq,
            test_clients=test_clients,
            global_train_logger=global_train_logger,
            global_test_logger=global_test_logger,
            sampling_rate=sampling_rate,
            verbose=verbose,
            seed=seed
        )
    elif aggregator_type == "L2SGD":
        return LoopLessLocalSGDAggregator(
            clients=clients,
            global_learners_ensemble=global_learners_ensemble,
            log_freq=log_freq,
            global_train_logger=global_train_logger,
            global_test_logger=global_test_logger,
            test_clients=test_clients,
            communication_probability=communication_probability,
            penalty_parameter=mu,
            sampling_rate=sampling_rate,
            verbose=verbose,
            seed=seed
        )
    elif aggregator_type == "AFL":
        return AgnosticAggregator(
            clients=clients,
            global_learners_ensemble=global_learners_ensemble,
            log_freq=log_freq,
            test_clients=test_clients,
            lr_lambda=lr_lambda,
            global_train_logger=global_train_logger,
            global_test_logger=global_test_logger,
            sampling_rate=sampling_rate,
            verbose=verbose,
            seed=seed
        )
    elif aggregator_type == "FFL":
        return FFLAggregator(
            clients=clients,
            global_learners_ensemble=global_learners_ensemble,
            log_freq=log_freq,
            test_clients=test_clients,
            lr=lr,
            q=q,
            global_train_logger=global_train_logger,
            global_test_logger=global_test_logger,
            sampling_rate=sampling_rate,
            verbose=verbose,
            seed=seed
        )
    elif aggregator_type == "decentralized":
        n_clients = len(clients)
        mixing_matrix = get_mixing_matrix(n=n_clients, p=0.5, seed=seed)

        return DecentralizedAggregator(
            clients=clients,
            global_learners_ensemble=global_learners_ensemble,
            mixing_matrix=mixing_matrix,
            log_freq=log_freq,
            test_clients=test_clients,
            global_train_logger=global_train_logger,
            global_test_logger=global_test_logger,
            sampling_rate=sampling_rate,
            verbose=verbose,
            seed=seed
        )
    elif aggregator_type == "pFedGate":
        return pFedGateAggregator(
            clients=clients,
            global_learners_ensemble=global_learners_ensemble,
            log_freq=log_freq,
            global_train_logger=global_train_logger,
            global_test_logger=global_test_logger,
            test_clients=test_clients,
            sampling_rate=sampling_rate,
            verbose=verbose,
            seed=seed,
            aggregate_sampled_clients=aggregate_sampled_clients,
            online_aggregate=online_aggregate,
            outdir=outdir
        )
    else:
        raise NotImplementedError(
            "{aggregator_type} is not a possible aggregator type."
            " Available are: `no_communication`, `centralized`,"
            " `personalized`, `clustered`, `fednova`, `AFL`,"
            " `FFL` and `decentralized`."
        )

class RunningStats(object):

    def __init__(self):
        self.n = 0
        self.old_m = 0
        self.new_m = 0
        self.old_s = 0
        self.new_s = 0

    def reset(self):
        self.n = 0
        self.old_m = 0
        self.new_m = 0
        self.old_s = 0
        self.new_s = 0

    def clear(self):
        self.n = 0

    def push(self, x):
        self.n += 1

        if self.n == 1:
            self.old_m = self.new_m = x
            self.old_s = 0
        else:
            self.new_m = self.old_m + (x - self.old_m) / self.n
            self.new_s = self.old_s + (x - self.old_m) * (x - self.new_m)

            self.old_m = self.new_m
            self.old_s = self.new_s

    def mean(self):
        return self.new_m if self.n else 0.0

    def variance(self):
        return self.new_s / (self.n - 1) if self.n > 1 else 0.0

    def standard_deviation(self):
        if isinstance(self.new_s, torch.Tensor):
            return torch.sqrt(self.variance())
        else:
            return math.sqrt(self.variance())


# coding=utf-8
import os
from collections import OrderedDict
from qgis.PyQt.QtCore import QSettings
from qgis.PyQt.QtSql import QSqlQuery
from qgis.core import QgsMessageLog, Qgis

from veriso.base.utils.utils import tr, dynamic_import
from veriso.base.utils.exceptions import VerisoError
from veriso.base.utils.utils import (get_modules_dir, yaml_load_file,
                                     get_subdirs, get_default_db)

def get_layers_from_topic(topic, module_name):
    """Converts the layer information into a dictionary from
    a topic list.

    Adds the geometry column name if there are more than
    one geometry column in a layer.

    Returns: Dictionary with all the necessary data.
    """
    dd = {}
    for table in topic["tables"]:
        dd[table] = dd.get(table, 0) + 1

    i = 0
    layers = []
    for table in topic["tables"]:
        my_layer = {
            "type": "postgres", "featuretype": table,
            "key": topic["primary_keys"][i],
            "geom": topic["geometry_columns"][i],
            "group": tr(topic["topic"], context=module_name),
            "title": tr(topic["class_names"][i], context=module_name)}
        i += 1
        # If there is more than one geometry column in the table
        # the name of the geometry columns is written in brackets
        # following the name of the table.
        if dd[table] > 1:
            my_layer["title"] = my_layer["title"] + " (" + tr(
                my_layer["geom"], context=module_name) + ")"

        layers.append(my_layer)

    return layers

def get_topics_tables(module_name):
    """Requests the topics and tables from the topic_tables database table.
    This table was created in the postprocessing step.

    Returns:
      False: If something went wrong when trying to get the list from the
      database. Otherwise a python dictionary.
    """
    try:
        db = get_default_db()

        # I think libpg cannot deal with arrays from postgresql. So we
        # return a comma sperated string.
        # Everything is ordered alphanumerical. Not sure if we would know
        #  enough to sort by interlis model ordering?!
        db_schema = QSettings("CatAIS", "VeriSO").value("project/dbschema")

        sql = ("SELECT topic,"
               "array_to_string(array_agg(sql_name ORDER BY sql_name),',') "
               "as tables, "
               "array_to_string(array_agg(coalesce(f_geometry_column,'') "
               "ORDER BY sql_name),',') as geometry_columns, "
               "array_to_string(array_agg(class_name ORDER BY sql_name), "
               "',') as class_names, "
               "array_to_string(array_agg(primary_key ORDER BY sql_name),"
               "',') as primary_keys "
               "FROM " + db_schema + ".t_topics_tables GROUP BY topic "
                                     "ORDER BY topic;")

        query = db.exec_(sql)

        if not query.isActive():
            message = "Error while reading from database."
            QgsMessageLog.logMessage(tr(message), module_name,
                                     Qgis.Critical)
            QgsMessageLog.logMessage(
                str(QSqlQuery.lastError(query).text()),
                module_name,
                Qgis.Critical)
            return

        topics = []
        record = query.record()
        while query.next():
            topic = {"topic": str(query.value(record.indexOf("topic")))}

            tables = []
            for table in str(query.value(record.indexOf("tables"))).split(
                    ","):
                tables.append(table)
            topic["tables"] = tables

            geometry_columns = []
            for geometry_column in str(
                    query.value(record.indexOf("geometry_columns"))).split(
                    ","):
                geometry_columns.append(geometry_column)
            topic["geometry_columns"] = geometry_columns

            class_names = []
            for class_name in str(
                    query.value(record.indexOf("class_names"))).split(","):
                class_names.append(class_name)
            topic["class_names"] = class_names

            primary_keys = []
            for primary_key in str(
                    query.value(record.indexOf("primary_keys"))).split(","):
                primary_keys.append(primary_key)
            topic["primary_keys"] = primary_keys

            topics.append(topic)

        db.close()
        del db

        return topics

    except Exception as e:
        message = "Something went wrong catching the topics tables list " \
                  "from the database."
        QgsMessageLog.logMessage(tr(message), module_name,
                                 Qgis.Critical)
        QgsMessageLog.logMessage(str(e), module_name,
                                 Qgis.Critical)
        return

def get_baselayers(module_name):
    """Reads all baselayer definitions from a yaml file.

    Returns
      A list of dictionaries with all baselayer definitions. Otherwise
      False.
    """
    filename = os.path.join(
        get_modules_dir(),
        module_name,
        "baselayer",
        "baselayer.yml")
    try:
        baselayers = yaml_load_file(filename)
        return baselayers
    except Exception as e:
        QgsMessageLog.logMessage(str(e), module_name,
                                 Qgis.Critical)
        return

def get_check_topics(module_name):
    """Get all check topics (aka groups).

    Different languages are support. See the yaml file how to deal with it.

    Returns:
      A ordered dictionary with the topic name and corresponding check
      files (python). False if something went wrong.
    """

    topics_dir = os.path.join(
        get_modules_dir(), module_name, 'checks')

    checks = []
    for topic_dir in get_subdirs(topics_dir):
        topic_file = os.path.join(topics_dir, topic_dir, 'topic.yml')
        if os.path.isfile(topic_file):
            try:
                topic = yaml_load_file(topic_file)
                topic['topic_dir'] = topic_dir
                checks.append(topic)
            except VerisoError:
                raise
    try:
        locale = QSettings().value('locale/userLocale')[0:2]
    except TypeError:
        locale = 'de'
    try:
        topics = OrderedDict()
        checks = sorted(checks,
                        key=lambda k: k['topic'][locale])
        for check in checks:
            topic = check["topic"]
            topic_dir = check["topic_dir"]

            # Check, if yaml file is multilingual.
            try:
                if topic in topics:
                    continue
                topics[topic] = check
                # TODO control this whe using checks_from_files
                # yaml is *not* multilingual.
            except:
                # yaml is multilingual.
                # If the language set in QGIS is not available in the
                # yaml file, the first language will be chosen

                # dinamically get the checks based on the available files
                checks_from_files = get_checks_from_files(
                    module_name, topic_dir)
                try:
                    my_topic = topic[locale]
                    my_check = OrderedDict()
                    my_check["topic"] = my_topic
                    # my_check["checks"] = check["checks"]
                    my_check["checks"] = checks_from_files
                    my_check["topic_dir"] = topic_dir
                    topics[my_topic] = my_check
                    # language found

                except:
                    # language *not* found
                    my_check = OrderedDict()
                    my_check["topic"] = list(topic.values())[0]
                    # my_check["checks"] = check["checks"]
                    my_check["checks"] = checks_from_files
                    my_check["topic_dir"] = topic_dir
                    topics[my_check["topic"]] = my_check
        return topics
    except Exception as e:
        print(str(e))
        raise VerisoError(str(e), e, tag=module_name)

def get_checks_from_files(module_name, topic_dir, modules_dir=None):
    """

    :param module_name:
    :param topic_dir:
    :param modules_dir:
    :return:
    """

    if modules_dir is None:
        modules_dir = get_modules_dir()
    path = os.path.join(modules_dir, module_name, 'checks', topic_dir)
    package = 'veriso.modules.%s.checks.%s' % (module_name, topic_dir)
    checks = []

    files = os.listdir(path)
    # alpha-numeric sorting to assure the same order in the menus
    # see http://stackoverflow.com/a/2669523/1193450
    # files = sorted(files, key=lambda item: (
    #    int(item.partition(' ')[0]) if item[0].isdigit() else float('inf'),
    #    item))
    try:
        locale = QSettings().value('locale/userLocale')[0:2]
    except TypeError:
        locale = 'de'

    for f in files:
        check = {}
        if f.endswith(".py") and f != '__init__.py':
            filename = f[:-3]
            module = '%s.%s' % (package, filename)
            try:
                module = dynamic_import(module)
                check['id'] = filename
                check['file'] = filename
                check['name'] = module.ComplexCheck.get_name()
                if check['name'] is not None:
                    # name is defined in the class
                    check['shortcut'] = module.ComplexCheck.get_shortcut()
                else:
                    # let's see if there is a yaml file as our last resort
                    yaml_path = os.path.join(path, filename + '.yml')
                    check['name'], check['shortcut'] = get_info_from_yaml(
                        yaml_path)

                # if we made it to here, it's all good
                checks.append(check)
            except:
                continue
        elif f.endswith('_separator'):
            check['name'] = 'separator'
            # checks.append(check)
    checks = sorted(checks, key=lambda k: k["name"][locale])
    return checks

def get_info_from_yaml(path):
    try:
        data = yaml_load_file(path)
        name = data['name']
    except Exception as e:
        raise VerisoError('error parsing %s' % path, e)
    try:
        shortcut = data['shortcut']
    except KeyError:
        shortcut = ''

    return name, shortcut


import json

from bs4 import BeautifulSoup

def find_elements(bs_content, structure):

    circuit = bs_content.find(structure)
    elements = [tag.name for tag in circuit.find_all() if "list" in tag.name]
    total_components = {}
    for element in elements:
        components = circuit.find(element)
        components = circuit.find_all(element.replace("_list",""))
        total_components[element.replace("_list","")] = {}
        if components is not None:
            for component in components:
                tags = [(tag.name,tag.text) for tag in component.find_all()]
                for i in range(len(tags)):
                    tag_name = tags[i][0]
                    tag_value = tags[i][1]
                    if i == 0:
                        total_components[element.replace("_list","")][tag_value] = {}
                    else:
                        total_components[element.replace("_list","")][tags[0][1]][tag_name] = tag_value

    with open(structure + ".json", "w") as file:
        json.dump(total_components, file, indent=2)

def main(archieve):

    with open(archieve, "r") as file:
        content = file.readlines()
        content = "".join(content)

        bs_content = BeautifulSoup(content,"lxml")

    find_elements(bs_content,"powerflow")
    find_elements(bs_content,"assert")
    find_elements(bs_content,"tape")

archieve = "ejemplo.xml"
main(archieve)


# -*- coding: utf-8 -*-
# April 2020
# Author: Dimbi

from zato.server.service import Service
from contextlib import closing
import datetime
import json

class DateTimeEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime.datetime) or isinstance(obj, datetime.date):
            encoded_object = obj.strftime("%Y%m%d%H%M%S%z")
        else:
            encoded_object = json.JSONEncoder.default(self, obj)
        return encoded_object

class SqlToRedisServiceFinal(Service):
    name = 'Sql.To.Redis.Service.Final'

    class SimpleIO:
        input_required = ('query', 'out_name', 'output_dict_name', 'output_dict_key')
        input_optional = ('operation_name', 'table_name')

    def handle(self):
        query = self.request.input.query
        outName = self.request.input.out_name
        tableName = self.request.input.table_name
        outputDictName = self.request.input.output_dict_name
        outputDictKey = self.request.input.output_dict_key
        operationName = self.request.input.operation_name

        log_msg_head = """{} -- Stockage des donnees de la table '{}' dans Redis \n""".format(operationName, tableName)
        log_warn_head = """{} -- Erreur lors du stockage des donnees de la table '{}' dans Redis\n""".format(operationName, tableName)
        log_msg = log_warn = ""
        try:
            # Etape 1
            etape = "Etape 1"
            operation = """Selection dans la table '{}'""".format(tableName)
            inputVariable = {"query": query,
                             "out_name": outName,
                             "operation": "select"}
            response = self.invoke("Request.Sql.Service.Final", inputVariable)
            result = response.get("response").get("result")
            log_msg += """{} -- {}: terminee \n""".format(etape, operation)
            # Etape 2
            etape = "Etape 2"
            operation = """Stockage dans Redis""".format(tableName)
            self.kvdb.conn.delete(outputDictName)
            for i in range(len(result)):
                row = json.dumps(result[i], cls=DateTimeEncoder)
                dictKey = result[i].get(outputDictKey)
                self.kvdb.conn.hset(outputDictName, dictKey, row)
            log_msg += """{} -- {}: termine \n""".format(etape, operation)
            self.logger.info(log_msg_head + log_msg)
        except Exception as es:
            log_warn += log_msg
            log_warn += """{} -- {} -- Erreur : {} \n""".format(etape, operation, es)
            log_warn += """ ================== Erreur \n"""
            self.logger.warn(log_warn_head + log_warn)


import datetime
from flask import Flask, render_template, request, redirect, url_for
from flask_login import LoginManager, login_user, login_required, logout_user, current_user
from werkzeug.security import check_password_hash

# looks not good, better to have DataBase class with all those methods.
from database_handlers import get_assignment_from_base, get_task_from_base, get_tasks_from_base, \
    delete_task_from_base, add_task, stop_channel, stop_task, start_new_loop, \
    get_user_by_login
from config import SECRET_KEY, host, port
from utils2 import string_to_list
from user_login import UserLogin

app = Flask(__name__)
app.config['SECRET_KEY'] = SECRET_KEY
login_manager = LoginManager(app)
process = None
app.permanent_session_lifetime = datetime.timedelta(days=1)

@login_manager.unauthorized_handler
def unauthorized():
    return redirect('/login')

@login_manager.user_loader
def load_user(user_id):
    print('load_user')
    return UserLogin().from_db(user_id)

@app.route('/login', methods=["POST", "GET"])
def login():
    if request.method == "POST":
        user = get_user_by_login(request.form['username'])
        if user and check_password_hash(user.password, request.form['password']):
            user_login = UserLogin().create(user)
            login_user(user_login)
            return redirect(url_for('start_page'))
        else:
            return "Mistake"
    return render_template('login.html')

@app.route('/')
@login_required
def start_page():
    if not current_user.is_authenticated:
        return redirect(url_for('login'))
    tasks = get_assignment_from_base()
    return render_template('main.html', tasks=tasks)

@app.route('/task_list<int:channel>')
@login_required
def task_list(channel):
    if not current_user.is_authenticated:
        return redirect(url_for('login'))
    if channel:
        title = "task"
    else:
        title = "tasks"
    if channel:
        task = get_task_from_base(channel)
        h1 = f"{channel} channel task list:"
    else:
        task = get_tasks_from_base()
        h1 = f"Full task list:"

    time = datetime.datetime.now()
    context = {"title": title,
               "task": task,
               "h1": h1,
               "time": time}

    return render_template('task_list.html', context=context)

@app.route('/add_task', methods=['GET', 'POST'])
@login_required
def task_form():
    if request.method == "POST":
        try:
            channel = int(request.form['channel'])
            working_time = int(request.form['working-time'])
            pause_time = int(request.form['pause-time'])
            subchannels = str(request.form['subchannels'])
            assert isinstance(channel, int)
            assert isinstance(working_time, int)
            assert isinstance(pause_time, int)
            assert isinstance(subchannels, str)
            delete_task_from_base(channel)
            subchannels = string_to_list(subchannels)
            add_task(*subchannels, main_channel=channel, working_time=working_time, pause_time=pause_time)
            return redirect(url_for('task_list', channel=channel))
        except Exception as ER:
            return render_template('error.html', error_message=f"Ошибка {ER} попробуйте ещё", task_form=url_for('task_form'))
    else:
        return render_template('add_task.html')

@app.route('/kill_process<int:channel>')
@login_required
def kill_process(channel):
    if not current_user.is_authenticated:
        return redirect(url_for('login'))
    if channel:
        delete_task_from_base(channel)
        stop_channel(channel)
        return redirect(url_for('start_page'))
    else:
        return redirect(url_for('start_page'))

@app.route('/stop_process<int:channel>')
@login_required
def stop_process(channel):
    if not current_user.is_authenticated:
        return redirect(url_for('login'))
    if channel:
        stop_task(channel)
        stop_channel(channel)
        return redirect(url_for('start_page'))
    else:
        return redirect(url_for('start_page'))

@app.route('/start_process<int:channel>')
@login_required
def start_process(channel):
    if not current_user.is_authenticated:
        return redirect(url_for('login'))
    if channel:
        start_new_loop(channel)
    return redirect(url_for('start_page'))

@app.route('/logout')
def logout():
    logout_user()
    return redirect(url_for('login'))

if __name__ == "__main__":
    app.run(host=host, port=port,debug=True)


from typing import Optional, Any
from hyperiontf.logging import getLogger, Logger

from hyperiontf.typing import (
    ContentSwitchingException,
    StaleElementReferenceException,
    DEFAULT_CONTENT,
)
from hyperiontf.typing import AnyContentContainer
from .iframe import IFrame

class ContentManager:
    """
    The ContentManager class manages context switching for WebPage and Iframe instances.

    It allows switching the context to different iframes within the page.
    """

    def __init__(
        self,
        owner: AnyContentContainer,
        logger: Logger = getLogger("PageObject"),
        current_content: Optional[str] = DEFAULT_CONTENT,
    ):
        """
        Initialize the ContentManager with the provided owner, logger, and current_content.

        :param owner: The WebPage or WebView instance that owns the context.
        :param logger: The logger to use for logging context switching information.
        :param current_content: The initial current context of the manager, defaults to 'default'.
        """
        self.owner = owner
        self.current_context = current_content
        self._logger = logger

    @property
    def automation_adapter(self):
        """
        Get the automation_adapter from the owner, if available.

        :return: The automation_adapter if available, otherwise None.
        """
        if hasattr(self.owner, "automation_adapter"):
            return self.owner.automation_adapter
        return None

    def set_context(self, iframe: Any):
        """
        Set the context to the provided iframe.

        :param iframe: The IFrame instance to switch the context to.
        :raises ContextSwitchingException: If context switching fails.
        """
        # The 'element_adapter' is used to resolve element references and can trigger API calls to find elements or
        # recover from errors. To ensure the correct logging order, we first retrieve the 'element_adapter' before
        # posting a content switch message. This way, the log chronology remains accurate.
        adapter = iframe.element_adapter
        self._logger.debug(
            f"[{self.owner.__full_name__}] Switching content to '{iframe.__full_name__}' iframe."
        )
        self.automation_adapter.switch_to_iframe(adapter)
        self.current_context = iframe.__full_name__

    def set_default_context(self):
        """
        Set the context to the default page context.

        This method switches back to the default context of the page or WebView.
        """
        self._logger.debug(
            f"[{self.owner.__full_name__}] Switching content to default - page content."
        )
        self.automation_adapter.switch_to_default_content()
        self.current_context = DEFAULT_CONTENT

    def resolve_content(self, target: Any, force: bool = False):
        """
        Resolve the context to the target IFrame or default context.

        :param target: The target IFrame to resolve the context to.
        :param force: If True, forces the context to switch even if the current context is already the target.
        """
        if self.current_context == target.__full_name__ and not force:
            return

        if not isinstance(target, IFrame):
            if self.current_context == DEFAULT_CONTENT:
                return
            # If the target is not an IFrame, then traverse back to the page or WebView where the context is default.
            return self.set_default_context()

        try:
            self.set_context(target)
        except ContentSwitchingException as e:
            self._logger.debug(
                f"[{self.owner.__full_name__}] Exception Intercepted!\n{e.__class__.__name__}: {str(e)}"
            )
            # If an exception occurred during context switching, attempt to resolve the parent context first,
            # and then retry setting the target context.
            parent_holder = target.document_holder
            if parent_holder != target:
                parent_holder.__resolve__()
            self.set_context(target)
        except StaleElementReferenceException as e:
            self._logger.debug(
                f"[{self.owner.__full_name__}] Exception Intercepted!\n{e.__class__.__name__}: {str(e)}"
            )
            target.find_itself()
            target.__resolve__()

    # Define an alias for resolve_context
    resolve = resolve_content