import numpy as np
from sklearn.dummy import DummyRegressor
import sys
import os

SEP = '.' * 100
LINE = '=' * 100

def testDummyRegressor(case, desc, const, X, y):
    '''
    [Issue #22478]
    Some parameters passed into DummyRegressor are erroneously converted
    into Numpy formart after the fit() method is invoked.
    Source: https://github.com/scikit-learn/scikit-learn/issues/22478
    '''
    print(LINE)
    print(case, ":", desc)
    print(LINE)

    print("Creating a DummyRegressor...")
    regressor = DummyRegressor(constant=const, strategy="constant")
    params = regressor.get_params()
    print("Before fit() Method Invocation:")
    print(" " * 4, "params =", params)
    print(" " * 4, "X =", X)
    print(" " * 4, "y =", y)
    print(SEP)

    regressor.fit(X=X, y=y)
    result = regressor.get_params()
    print("After fit() Method Invocation:")

    print("    Expected:", params)
    print("    Actual:  ", result)
    print(SEP)

    print("Test Results:")

    try:
        assert type(result['constant']) == type(const)
        assert type(result['constant']) != np.ndarray
        print("    [PASS] Test 1: constant is of the expected type.")
    except:
        print("    [FAIL] Test 1: constant should be", type(const), "not", type(result['constant']))

    try:
        assert result == params
        assert result['constant'] == const
        print("    [PASS] Test 2: constant holds the expected value.")
    except:
        print("    [FAIL] Test 2: incorrect value of constant received.")

    print(LINE, '\n' * 2)

def testAll(cases):
    os.system('cls' if os.name == 'nt' else 'clear')
    for case in cases.keys():
        testCase = cases[case]
        desc = testCase['desc']
        const = testCase['args'][0]
        X = testCase['args'][1]
        y = testCase['args'][2]
        testDummyRegressor(case, desc, const, X, y)

testCases = {
    'Test Set 1: 0 Float Constant' : {
        'desc': 'strategy = constant; constant = 0.0',
        'args': [0.0, [[1,1], [2,2], [3,3]], [[0], [0], [0]]]
        },
    'Test Set 2: 0 Integer Constant' : {
        'desc': 'strategy = constant; constant = 0',
        'args': [0, [[1,1], [2,2], [3,3]], [[0], [0], [0]]]
        },
    'Test Set 3: Postive Integer Constant' : {
        'desc': 'strategy = constant; constant = 2',
        'args': [2, [[1,1], [2,2], [3,3]], [[0], [0], [0]]]
        },
    'Test Set 4: Negative Integer Constant' : {
        'desc': 'strategy = constant; constant = -2',
        'args': [-2, [[1,1], [2,2], [3,3]], [[0], [0], [0]]]
        },
    'Test Set 5: Negative Float Constant Part 1' : {
        'desc': 'strategy = constant; constant = -3.14',
        'args': [-3.14, [[1,1], [2,2], [3,3]], [[0], [0], [0]]]
        },
    'Test Set 6: Negative Float Constant Part 2' : {
        'desc': 'strategy = constant; constant = -3.14',
        'args': [-3.14, [[1,2], [2,1], [3,1]], [[3], [1], [4]]]
        },
    'Test Set 7: Large Positive Float Constant' : {
        'desc': 'strategy = constant; constant = sys.float_info.max',
        'args': [sys.float_info.max, [[1,2], [2,1], [3,1]], [[3], [1], [4]]]
        },
    'Test Set 8: Large Negative Float Constant' : {
        'desc': 'strategy = constant; constant = -sys.float_info.max',
        'args': [-sys.float_info.max, [[1,2], [2,1], [3,1]], [[3], [1], [4]]]
        },
    'Test Set 9: Small Positive Float Constant' : {
        'desc': 'strategy = constant; constant = sys.float_info.min',
        'args': [sys.float_info.min, [[1,2], [2,1], [3,1]], [[3], [1], [4]]]
        },
    'Test Set 10: Small Negative Float Constant' : {
        'desc': 'strategy = constant; constant = -sys.float_info.min',
        'args': [-sys.float_info.min, [[1,2], [2,1], [3,1]], [[3], [1], [4]]]
        }
}

testAll(testCases)


import sys
import os
import pandas_datareader.data as web
from datetime import date, timedelta
from currency_converter import CurrencyConverter

sys.path.append(os.path.realpath(os.path.dirname(__file__))+"/src")
from objects.BalanceItemObj import BalanceItem

class Stock(object):
    """This object contains stock informations"""
    def __init__(self, name, isin, amount, currency):
        self.name = name
        self.isin = isin
        self.amount = amount
        self.currency = currency

    def get_balance(self):
        """Returns the balance values for the different accounts that exit for this bank
        :returns: list of BalanceItems

        """
        today =  date.today()
        yesterday = date.today() - timedelta(5)
        stock_data = web.DataReader(self.isin, 'morningstar', yesterday, today)
        if self.currency == "USD":
            c = CurrencyConverter()
            stock_value_usd = stock_data.as_matrix()[len(stock_data.as_matrix()) -1 ][0]
            stock_value_eur = c.convert(stock_value_usd, 'USD', 'EUR')
        else:
            stock_value_eur = stock_data.as_matrix()[len(stock_data.as_matrix()) -1 ][0]
        return(BalanceItem(self.name, stock_value_eur * self.amount))


#!/usr/bin/env python
# general_plotter for quick plotting

import argparse
# parse first since matplotlib takes a long time to load
parser = argparse.ArgumentParser()
parser.add_argument('column', nargs='+')
parser.add_argument('--string-data', '-s', nargs='+')
parser.add_argument('--xaxis', '-a', nargs='+')
parser.add_argument('--xlabel', '-x', type=str)
parser.add_argument('--ylabel', '-y', type=str)
parser.add_argument('--xtics-column', 't')
#parser.add_argument('--title', type=str)
arg = parser.parse_args()

import matplotlib
gui_envs = ['GTKAgg','TKAgg','Qt4Agg','WXAgg']
for gui in gui_envs:
    try:
        matplotlib.use(gui,warn=False, force=True)
        from matplotlib import pyplot as plt
        break
    except:
        continue

# list of fonts
#flist = matplotlib.font_manager.get_fontconfig_fonts()
#for fname in flist:
#    print(fname)
#    if not '/Library/Fonts/NISC18030.ttf' in fname:
#        matplotlib.font_manager.FontProperties(fname=fname).get_name()
import numpy as np
font = {'family': 'sans-serif',
        #'sans-serif': '/System/Library/Fonts/Helvetica.ttc'}
        'sans-serif': '/Library/Fonts/Skia.ttf'}
#font = fm.FontProperties(fname='/System/Library/Fonts/Helvetica.ttc')
#plt.rcParams['font.family'] = 'cursive'
plt.rc('font', **font)

def plot_single(data_file, xc):
    # load
    if arg.string_data:
        #str_data = np.loadtxt(data_file, usecols=(0, 1), dtype=str)
        str_data = np.loadtxt(data_file, usecols=arg.string_data, dtype=str)
    num_data = np.loadtxt(data_file, usecols=(2,3,4,5,6))

    if str_data:
        str_data = np.array([ x for x in sorted(str_data,
            key=lambda x: x[XC_COLUMN]) if x[XC_COLUMN] == xc ])
    num_data = np.array([ x  for _,x in sorted(zip(str_data[:, XC_COLUMN], num_data),
        key=lambda x: x[0]) if _ == xc ])

    if arg.xaxis:
        t = num_data
    else:
        t = np.arange(0, np.shape(num_data)[0])
    for column in arg.column:
        plt.plot(t, num_data[:, column])

    if str_data and arg.xtics_column:
        plt.xtics(t, str_data[:, arg.xtics_column])
    #plt.xticks(x, str_data[:, POS_COLUMN], **font)
    #plt.title(xc, **font)
    #plt.xticks(x, str_data[:, POS_COLUMN], fontproperties=font)
    #plt.title(xc, fontproperties=font)
    #plt.xticks(x, str_data[:, POS_COLUMN])
    #plt.title(xc)
    #plt.plot(x, num_data[:, E_COLUMN])

if __name__ == '__main__':
    #plt.subplot(2,1,1)
    plot_single(arg.files[0], 'pbe')
    #plt.subplot(2,1,2)
    #plot_single(arg.files[0], 'pz')
    plt.show()


import wave, struct, math, subprocess, platform
import matplotlib.pyplot as plot

# print params of a .wav file
def show_info(fname):
    obj = wave.open(fname)
    print( "Number of channels",obj.getnchannels())
    print ( "Sample width",obj.getsampwidth())
    print ( "Frame rate.",obj.getframerate())
    print ("Number of frames",obj.getnframes())
    print ( "parameters:",obj.getparams())
    obj.close()

def nframes(fname):
    obj = wave.open(fname)
    return obj.getnframes()

# draw a graph of the first N frames of a .wav file
def graph_wav(fname, nframes):
    obj = wave.open(fname)
    x = obj.readframes(nframes)
    obj.close()
    #print(len(x))
    y = struct.unpack("%dh"%(nframes*2), x)
    y1 = [0]*nframes
    y2 = [0]*nframes
    for i in range(nframes):
        y1[i] = y[i*2]
        y2[i] = y[i*2+1]
    plot.rcParams['figure.figsize'] = (12,8)
    plot.plot(range(nframes), y1, label='L')
    plot.plot(range(nframes), y2, label='R')
    plot.show()

# graph a position array
def graph_pos(pos_array):
    nframes = len(pos_array)
    plot.rcParams['figure.figsize'] = (12,8)
    plot.plot(range(nframes), pos_array)
    plot.show()

# read .wav file as array of floats
def read_wav(fname):
    f = wave.open(fname)
    nsamples = f.getnframes()*2
    samples = [0.]*nsamples
    buf_frames = 10000
    ns = 0
    while True:
        x = f.readframes(buf_frames)
        nbytes = len(x)
        if nbytes == 0: break
        buf_nsamples = nbytes//2
        s = struct.unpack("%dh"%(buf_nsamples), x)
        for i in range(buf_nsamples):
            a = float(s[i])
            samples[ns] = a
            ns += 1
    print('read',nsamples//2,'frames from', fname)
    return samples

# write array of floats as .wav file
def write_wav(fname, samples):
    f = wave.open(fname, 'wb')
    f.setnchannels(2)
    f.setsampwidth(2)
    f.setframerate(44100)
    buf_nsamples = 10000
    nsamples = len(samples)
    nwritten = 0
    while nwritten < nsamples:
        ns = buf_nsamples
        if nwritten + ns > nsamples:
            ns = nsamples-nwritten
        isamples = [0]*ns
        for i in range(ns):
            isamples[i] = int(samples[nwritten+i])
        x = struct.pack("%dh"%(ns), *isamples)
        f.writeframes(x)
        nwritten += ns
    print('wrote',nsamples//2,'frames to',fname)

# create a zero signal of given length
def zero_signal_t(dur, framerate):
    nsamples = math.ceil(2*dur*framerate)
    return [0]*nsamples

def zero_signal_ns(nsamples):
    return [0]*nsamples

# scale signal
def scale(samples, gain):
    for i in range(len(samples)):
        samples[i] *= gain

# add the input signal "isig" to the output signal "osig",
# panning it according to the array pos: 0..1
# "ang" is the separation (0..1) between the input chans.
# if 0, the channels are summed (mono)
def pan_signal(isig, framerate, ang, pos_array, osig):
    inframes = len(isig)//2
    pnframes = len(pos_array)
    print('pan_signal; input %d frames, pos_array %d frames'%(inframes, pnframes))
    for i in range(inframes):
        a = isig[i*2]
        b = isig[i*2+1]
        if i < pnframes:
            p = pos_array[i]
        else:
            p = pos_array[pnframes-1]
        if ang == 0:
            avg = (a+b)/2
            pp = p*math.pi/2
            osig[i*2] += avg*math.cos(pp)
            osig[i*2+1] += avg*math.sin(pp)
        else:
            p0 = max(p-ang, 0)*math.pi/2
            p1 = min(p+ang, 1)*math.pi/2
            osig[i*2] += a*math.cos(p0)
            osig[i*2+1] += a*math.sin(p0)
            osig[i*2] += b*math.cos(p1)
            osig[i*2+1] += b*math.sin(p1)

def play_wav(fname):
    s = platform.system()
    if s == 'Windows':
        prog = "C:\Program Files (x86)\Windows Media Player/wmplayer.exe"
        cmd = '"%s" %%cd%%/%s'%(prog, fname)
            # wmplayer requires full path
    else:
        raise Exception("OS not supported")

    print(cmd)
    subprocess.call(cmd, shell=True)

play_wav("data/pan_test.wav")


import math
from scipy import stats
import numpy as np

def AIBN(Oop, Sr):
    R = 8.31434
    t = 80
    T = t + 273.15
    theta = 4.575 * T / 1000
    beta = 3.436 * math.exp(-1200 * 4.19 /(R * T))
    HIn = 0.005
    k2 = 10**(6.6 - 9.8/theta)
    k6 = 10**(9.2 - 6.00/theta)
    ki = 10**(15 - 30.45/theta)
    RH = 7.17

    data1 = []
    with open(Oop, 'r') as f:
        dat = f.readlines()
        for line in dat:
            spl = str.split(line, '\t')
            data1.append(spl)

    data2 = []
    with open(Sr, 'r') as f:
        dat = f.readlines()
        for line in dat:
            spl = str.split(line, '\t')
            data2.append(spl)

    n = len(data1)
    W0 = []
    AIBN = []
    W = []
    Wi = []
    Wok0 = []
    Wok = []
    tau = []
    x = []
    y = []
    for i in range(n):
        W0.append(float(data1[i][1]) / (60 * 22.4))
        W.append(float(data2[i][1]) / (60 * 22.4))
        AIBN.append(float(data2[i][0]))
        Wi.append(2 * beta * ki * AIBN[i])
        Wok0.append(W0[i] + Wi[i] * (1 - beta) / (2 * beta))
        Wok.append(W[i] + Wi[i] * (1 - beta) / (2 * beta))
        tau.append(float(data2[i][2]) * 60)
        x.append(HIn / Wi[i])
        y.append(tau[i])

    a, b, r_value, p_value, std_err = stats.linregress(x,y)
    f = a
    yrs = []
    for i in range(n):
        yrs.append(a * x[i] + b)

    Z = n * sum(map(lambda xx: xx * xx, x)) - sum(x) * sum(x)
    _s = 0
    for i in range(n):
        _s += (y[i] - yrs[i]) ** 2
    Sy = math.sqrt(_s / (n - 1))
    Sa_f = math.sqrt(n * Sy * Sy / Z)
    eps = abs(100 * Sa_f / a)

    x = []
    y = []
    for i in range(n):
        x.append(math.log(Wi[i]))
        y.append(math.log(Wok[i]))

    a, b, r_value, p_value, std_err = stats.linregress(x,y)
    n_AIBN = a
    yrs = []
    for i in range(n):
        yrs.append(a * x[i] + b)

    Z = n * sum(map(lambda xx: xx * xx, x)) - sum(x) * sum(x)
    _s = 0
    for i in range(n):
        _s += (y[i] - yrs[i]) ** 2
    Sy = math.sqrt(_s / (n - 1))
    Sa_n_AIBN = math.sqrt(n * Sy * Sy / Z)
    eps = abs(100 * Sa_n_AIBN / a)

    Ft = []
    for i in range(n):
        Ft.append((Wok0[i] / Wok[i]) * abs(1 - ((Wok[i] ** 2) / (Wok0[i] ** 2))))

    x = []
    y = []
    for i in range(n):
        x.append(math.log(Wi[i]))
        y.append(math.log(Ft[i]))

    a, b, r_value, p_value, std_err = stats.linregress(x,y)
    nf_AIBN = a
    yrs = []
    for i in range(n):
        yrs.append(a * x[i] + b)

    Z = n * sum(map(lambda xx: xx * xx, x)) - sum(x) * sum(x)
    _s = 0
    for i in range(n):
        _s += (y[i] - yrs[i]) ** 2
    Sy = math.sqrt(_s / (n - 1))
    Sa_nf_AIBN = math.sqrt(n * Sy * Sy / Z)
    eps = abs(100 * Sa_nf_AIBN / a)

    x = []
    y = []
    for i in range(n):
        x.append(RH * math.sqrt(Wi[i]))
        y.append(Wok0[i])

    a, b, r_value, p_value, std_err = stats.linregress(x,y)
    k2k6 = a
    yrs = []
    for i in range(n):
        yrs.append(a * x[i] + b)

    Z = n * sum(map(lambda xx: xx * xx, x)) - sum(x) * sum(x)
    _s = 0
    for i in range(n):
        _s += (y[i] - yrs[i]) ** 2
    Sy = math.sqrt(_s / (n - 1))
    Sa_k2k6 = math.sqrt(n * Sy * Sy / Z)
    eps = abs(100 * Sa_k2k6 / a)

    x = []
    y = []

    for i in range(n):
        x.append(RH * Wi[i] / (HIn * f))
        y.append(Wok[i])

    a, b, r_value, p_value, std_err = stats.linregress(x,y)
    k2k7 = a
    yrs = []
    for i in range(n):
        yrs.append(a * x[i] + b)

    k7 = k2 / k2k7

    Z = n * sum(map(lambda xx: xx * xx, x)) - sum(x) * sum(x)
    _s = 0
    for i in range(n):
        _s += (y[i] - yrs[i]) ** 2
    Sy = math.sqrt(_s / (n - 1))
    Sa_k2k7 = math.sqrt(n * Sy * Sy / Z)
    eps = abs(100 * Sa_k2k7 / a)
    Sa_k7=k7 * eps / 100
    fk7 = f * k7
    Sa_fk7 = fk7 * math.sqrt(math.pow(Sa_f/f, 2) + math.pow(Sa_k7/k7, 2))

    return AIBN, W, Wok, tau, Wi, Ft, f, n_AIBN, nf_AIBN, fk7, k2k6, k2k7, k7, Sa_f, Sa_n_AIBN, Sa_nf_AIBN, Sa_k2k6, Sa_k2k7, Sa_k7, Sa_fk7


import discord
from data import settings
from discord.ext import commands
from datetime import datetime
from modules.storage import info, cargos

class Info(commands.Cog, name='Informativo', description='''Comandos de informações gerais sobre o nosso servidor.'''):

    def __init__(self, client):
        self.client = client

    @commands.command(aliases=['servidor'], help='Exibe informações sobre o servidor do Discord.', usage=f'{settings.bot_prefix}server')
    @commands.guild_only()
    async def server(self, ctx):
        embed=discord.Embed(color=info.welcome_colour, description='Informações sobre este servidor:')
        embed.set_author(name=ctx.guild.name, url=settings.url, icon_url=ctx.guild.icon.url)
        embed.set_footer(text=f'Informações requisitadas por {ctx.author}. | Data: {datetime.now().strftime("às %H:%M:%S em %d/%m/%Y")}', icon_url=ctx.author.avatar.url)
        if ctx.guild.banner:
            embed.set_thumbnail(ctx.guild.banner.url)
        embed.add_field(name='Dono do servidor:', value=ctx.guild.owner, inline=False)
        embed.add_field(name='Data de criação do servidor:', value=ctx.guild.created_at.__format__("às %H:%M:%S em %d/%m/%Y"), inline=False)
        embed.add_field(name='Contagem de membros:', value=ctx.guild.member_count, inline=False)
        embed.add_field(name='Região do servidor:', value=ctx.guild.region, inline=False)
        embed.add_field(name='Canal de regras:', value=ctx.guild.rules_channel.mention, inline=False)
        embed.add_field(name='Canal de avisos públicos:', value=ctx.guild.public_updates_channel.mention, inline=False)
        embed.add_field(name='Contagem de boosts:', value=len(ctx.guild.premium_subscribers), inline=False)
        embed.add_field(name='Cargo intrínseco ao bot:', value=ctx.guild.self_role.mention, inline=False)
        #embed.add_field(name=':', value=ctx.guild., inline=False)
        #embed.add_field(name=':', value=ctx.guild., inline=False)
        await ctx.send(embed=embed)

    @commands.command(aliases=['pfp', 'foto'], help='Envia a foto de perfil de um usuário.', usage=f'{settings.bot_prefix}avatar *@membro*')
    @commands.guild_only()
    @commands.has_any_role(cargos.member_role_id)
    async def avatar(self, ctx, member:discord.Member=None):
        if not member:
            member = ctx.author
        user = await self.client.fetch_user(member.id)
        if user.accent_color:
            embed=discord.Embed(color=user.accent_color.value, description=f'Comando de avatar requisitado por {ctx.author.mention}')
        else:
            embed=discord.Embed(color=info.welcome_colour, description=f'Comando de avatar requisitado por {ctx.author.mention}')
        embed.set_author(name=f'Avatar de {user}', url=ctx.author.avatar.url)
        embed.set_image(url=user.avatar.url)
        embed.set_footer(text=ctx.guild.name, icon_url=ctx.guild.icon.url)
        await ctx.reply(embed=embed)

    @commands.command(aliases=['perfil', 'conta', 'sobre'], help='Exibe informações gerais sobre o membro mencionado.', usage=f'{settings.bot_prefix}')
    @commands.guild_only()
    @commands.has_any_role(cargos.member_role_id)
    async def profile(self, ctx, member:discord.Member=None):
        if not member:
            member = ctx.author
        user = await self.client.fetch_user(member.id)
        if user.accent_color:
            embed=discord.Embed(color=user.accent_color.value, description=f'Comando requisitado por {ctx.author.mention}:')
        else:
            embed=discord.Embed(color=info.rules_colour, description='Comando de informação de usuários:')
        embed.set_author(name=f'Dados da conta do Discord de {user}', url=settings.url, icon_url=member.avatar.url)
        embed.set_thumbnail(url=user.avatar.url)
        if user.banner:
            embed.set_image(url=user.banner.url)
        embed.set_footer(text=ctx.guild.name, icon_url=ctx.author.avatar.url)
        embed.add_field(name='Nome e discriminador:', value=user, inline=False)
        embed.add_field(name='ID da conta:', value=user.id, inline=False)
        embed.add_field(name='Data de criação da conta:', value=user.created_at.strftime("às %H:%M:%S em %d/%m/%Y"), inline=False)
        embed.add_field(name='Data de entrada neste servidor:', value=member.joined_at.strftime("às %H:%M:%S em %d/%m/%Y"), inline=False)
        isbot = 'Não'
        if user.bot:
            isbot = 'Sim'
        embed.add_field(name='É um bot?', value=isbot)
        embed.add_field(name='Cargos:', value=', '.join(map(str,member.roles)), inline=False)
        embed.add_field(name='Cargo mais alto na hierarquia:', value=member.top_role.mention)
        if member.premium_since:
            embed.add_field(name='Dá boost no servidor desde:', value=member.premium_since.strftime("às %H:%M:%S em %d/%m/%Y"))
        await ctx.send(embed=embed)

def setup(client):
    client.add_cog(Info(client))


#!/usr/bin/env python

from __future__ import print_function
import numpy as np
from astropy.io import fits

def loadspec(specfile, i):

    # log_10 of starting wavelength in Angstroms
    lam0 = 10**fits.open(specfile)[2].data[i]

    # flux density
    flambda = fits.open(specfile)[0].data[i]

    # inverse variance
    ivar = fits.open(specfile)[1].data[i]

    # Definition of the wavelength array: 1e-4 spacing in log_10(lambda)
    lam = lam0*10**(1e-4*np.arange(len(ivar)))

    return [lam, flambda, ivar]

def savespec(lam, flambda, ivar, outfile=None):

    specarr = np.zeros((len(lam), 3))
    specarr[:, 0] = lam
    specarr[:, 1] = flambda
    specarr[:, 2] = ivar

    if outfile is None:
        return specarr
    else:
        np.savetxt(outfile, specarr, fmt="%.5g")
        return

if __name__ == "__main__":

    specfile = 'skyfibers_nativelam.fits'

    # Save the 101st spectrum (index 100 from 0) as an example
    i = 100
    lam, flambda, ivar = loadspec(specfile, i)
    savespec(lam, flambda, ivar, 'spec_%06d.dat' % (i))

    # flambda is the excess sky spectrum-it should be nearly zero
    # apart from measurement errors.  Quantify this with chi squared:

    chisq_dof = np.sum(flambda**2*ivar)/len(lam)
    print('chi2 per degree of freedom of the sky spectrum: %.2f' % chisq_dof)


"""Methods for plotting binned data per-month."""
import calendar  # pylint: disable=E0401

import matplotlib.pyplot as plt
import matplotlib.ticker as mticker
import pandas as pd

from ..bhom.analytics import bhom_analytics
from ..helpers import validate_timeseries
from .utilities import contrasting_color

@bhom_analytics()
def monthly_histogram_proportion(
    series: pd.Series,
    bins: list[float],
    ax: plt.Axes = None,
    labels: list[str] = None,
    show_year_in_label: bool = False,
    show_labels: bool = False,
    show_legend: bool = False,
    **kwargs,
) -> plt.Axes:
    """Create a monthly histogram of a pandas Series.

    Args:
        series (pd.Series):
            The pandas Series to plot. Must have a datetime index.
        bins (list[float]):
            The bins to use for the histogram.
        ax (plt.Axes, optional):
            An optional plt.Axes object to populate. Defaults to None, which creates a new plt.Axes object.
        labels (list[str], optional):
            The labels to use for the histogram. Defaults to None, which uses the bin edges.
        show_year_in_label (bool, optional):
            Whether to show the year in the x-axis label. Defaults to False.
        show_labels (bool, optional):
            Whether to show the labels on the bars. Defaults to False.
        show_legend (bool, optional):
            Whether to show the legend. Defaults to False.
        **kwargs:
            Additional keyword arguments to pass to plt.bar.

    Returns:
        plt.Axes:
            The populated plt.Axes object.
    """

    validate_timeseries(series)

    if ax is None:
        ax = plt.gca()

    t = pd.cut(series, bins=bins, labels=labels)
    t = t.groupby([t.index.year, t.index.month, t], observed=True).count().unstack().T
    t = t / t.sum()

    # adjust column labels
    if show_year_in_label:
        t.columns = [
            f"{year}\n{calendar.month_abbr[month]}" for year, month in t.columns.values
        ]
    else:
        t.columns = [f"{calendar.month_abbr[month]}" for _, month in t.columns.values]

    t.T.plot.bar(
        ax=ax,
        stacked=True,
        legend=False,
        width=1,
        **kwargs,
    )
    ax.set_xlim(-0.5, len(t.columns) - 0.5)
    ax.set_ylim(0, 1)
    plt.setp(ax.get_xticklabels(), ha="center", rotation=0)
    for spine in ["top", "right", "left", "bottom"]:
        ax.spines[spine].set_visible(False)
    ax.yaxis.set_major_formatter(mticker.PercentFormatter(1))

    if show_legend:
        ax.legend(
            bbox_to_anchor=(1, 1),
            loc="upper left",
            borderaxespad=0.0,
            frameon=False,
        )

    if show_labels:
        for i, c in enumerate(ax.containers):
            label_colors = [contrasting_color(i.get_facecolor()) for i in c.patches]
            labels = [
                f"{v.get_height():0.1%}" if v.get_height() > 0.1 else "" for v in c
            ]
            ax.bar_label(
                c,
                labels=labels,
                label_type="center",
                color=label_colors[i],
                fontsize="x-small",
            )

    return ax


"""
"Adaptive Binning" algorithm by Nguyen and O'Connor
(Algorithm 1)  https://arxiv.org/pdf/1508.05154.pdf

Author: Katie Keith
2020-09-13
"""
import numpy as np
from collections import defaultdict, Counter

def sort_pairs(pred_label_pairs):
    """
    Input
        pred_label_pairs: list of tuples

    Sorts by first element of the tuple, returns in assending order
    """
    return sorted(pred_label_pairs, key=lambda x: x[0])

def assign_bin_labels(sorted_pred_label_pairs, beta):
    """
    b_k = floor((k-1)/beta)+ 1
    """

    #k = pair label
    pair_label2bin_label = {}
    for pair_label, _ in enumerate(sorted_pred_label_pairs):
        k = pair_label + 1 #their k starts at 1
        bin_label = np.floor((k-1)/beta)+ 1
        pair_label2bin_label[pair_label] = int(bin_label)

    return pair_label2bin_label

def define_bins(pair_label2bin_label, beta):
    bin2pairs = defaultdict(list)
    for pair_label, bin_label in pair_label2bin_label.items():
        bin2pairs[bin_label].append(pair_label)

    #from algorithm: If the last bin has size less than β, merge it with the second-to-last bin (if one exists).
    bin2pairs = dict(bin2pairs)
    last_bin = len(bin2pairs)
    last_bin_pairs = bin2pairs[last_bin]
    len_last_bin = len(last_bin_pairs)
    if bin2pairs.get(last_bin-1) != None:
        if len_last_bin < beta:
            bin2pairs[last_bin-1] = bin2pairs[last_bin-1] + last_bin_pairs
            del bin2pairs[last_bin]
    return bin2pairs

def calc_per_bin_probs(bin2pairs, sorted_pred_label_pairs):
    q_i_hat_list = []
    p_i_hat_list = []

    for bbin, pairs in bin2pairs.items():
        p_list = []
        q_list = []
        for pair_indx in pairs:
            q, p = sorted_pred_label_pairs[pair_indx]
            q_list.append(q)
            p_list.append(p)

        #calculate hat_{q_i}
        q_hat = np.mean(np.array(q_list))
        q_i_hat_list.append(q_hat)

        #calculate \hat_{p_i}
        p_hat = np.mean(np.array(p_list))
        p_i_hat_list.append(p_hat)
    return q_i_hat_list, p_i_hat_list

def calc_calib_error(q_i_hat_list, p_i_hat_list, bin2pairs, N):
    assert len(q_i_hat_list) == len(p_i_hat_list) == len(bin2pairs)

    ssum = 0
    for i, (q_i, p_i) in enumerate(zip(q_i_hat_list, p_i_hat_list)):
        len_b_i = len(bin2pairs[i+1])
        squared_term = (q_i - p_i)**2
        ssum+= len_b_i * squared_term

    calib_error = np.sqrt(1/N*ssum)
    return calib_error

def graph_calibration_curve(q_i_hat_list, p_i_hat_list):
	import matplotlib.pyplot as plt
	fig, ax = plt.subplots()
	plt.plot(p_i_hat_list, q_i_hat_list)
	plt.scatter(p_i_hat_list, q_i_hat_list)
	plt.xlim(0, 1)
	plt.ylim(0, 1)
	plt.xlabel('Prediction Strength')
	plt.ylabel('Empirical Frequency')
	plt.title("Calibration curve")

	#plot y=x line
	lims = [
	    np.min([ax.get_xlim(), ax.get_ylim()]),  # min of both axes
	    np.max([ax.get_xlim(), ax.get_ylim()]),  # max of both axes
	]
	ax.plot(lims, lims, 'k-', alpha=0.75, zorder=0)
	ax.set_aspect('equal')

	plt.show()
	return plt

def create_pred_label_pairs(q_list, y_list):
    """
    - Use the model to predict given the docids

    Returns:
     [(q_i, y_i), ...]
         - where q_i = probability out of logreg
         - y_i = "true" label averaged over multiple annotators
    """
    pred_label_pairs = [(q, y) for q, y in zip(q_list, y_list)]
    return pred_label_pairs

def mean_calib_error_over_bins(q_list, y_list, calib_bin_range):
	pred_label_pairs = create_pred_label_pairs(q_list, y_list)

	all_calib_error = []
	print("num_bins", "beta", "calib_error")
	print("==="*10)
	for num_bins in calib_bin_range: #TODO, automate this list
	    beta = int(np.round(len(y_list)/num_bins))
	    calib_error = adaptive_binning_algo(list(pred_label_pairs), beta, output_q_y_est=False)
	    print(num_bins, beta, calib_error)
	    all_calib_error.append(calib_error)
	all_calib_error = np.array(all_calib_error)
	print()
	mn = np.mean(all_calib_error)
	print('mean calib error=', mn)
	print('std calib error=', np.std(all_calib_error))
	return mn

def adaptive_binning_algo(pred_label_pairs, beta, output_q_y_est=True):
    """
    Inputs:
    	pred_label_pairs: (list of tuples)

    		Example:
    			[(q_1, y_1), (q_2, y_2), ... (q_n, y_n)]
    		Where q is the predicted probability and y is the label

    	beta : (int) target bin size (i.e. number of documents in a bin)

    Output:
    	q_i_hat_list : (list of floats) each entry is the average predicted probabilities per bin
    		length = number of bins

    	p_i_hat_list : (list of floats) each entry is the average empirical probabilites per bin
    		length = number of bins

    	calib_error : (float) The calibration error (root mean squared error per bin)
    	len_B_i     : (list of floats) each entry is the number of samples in a bin
    		length = number of bins

    """
    N = len(pred_label_pairs)

    #Step 1: Sort pairs by prediction values qk in ascending order
    sorted_pred_label_pairs = sort_pairs(pred_label_pairs)

    #Step 2: For each, asign bin label
    pair_label2bin_label = assign_bin_labels(sorted_pred_label_pairs, beta)

    #Step 3: Define bins
    bin2pairs = define_bins(pair_label2bin_label, beta)

    #Step 4: Calculate the empirical and predicted probabilities per bin
    q_i_hat_list, p_i_hat_list = calc_per_bin_probs(bin2pairs, sorted_pred_label_pairs)

    #Step 5: Calculate calibration error
    calib_error = calc_calib_error(q_i_hat_list, p_i_hat_list, bin2pairs, N)
    len_B_i = [len(bin2pairs[i+1]) for i in range(len(q_i_hat_list))]

    if output_q_y_est:
    	return q_i_hat_list, p_i_hat_list, len_B_i, calib_error
    else:
    	return calib_error, len_B_i

def calib_error_score(y_true, y_pred, NUM_BINS=10, output_q_y_est=False):
	"""
	Scorer for sklearn
		See https://scikit-learn.org/stable/modules/model_evaluation.html#scoring
	"""
	pred_label_pairs = create_pred_label_pairs(y_pred, y_true)
	beta = np.round(len(y_true)/NUM_BINS)
	return adaptive_binning_algo(pred_label_pairs, beta, output_q_y_est)

def draw_samples_for_caliberation(NUM_BINS, NUM_SAMPLES, p_i_hat_list, q_i_hat_list, sigma_hat_list):
    '''Estimate caliberation error with 95% confidence interval'''
    CalibErr = []
    for s in range(NUM_SAMPLES):
        s_i_hat_list = []
        for j in range(NUM_BINS):
            p_s = np.random.normal(p_i_hat_list[j], np.sqrt(sigma_hat_list[j]))
            p_s = min(1, max(0, p_s))
            s_i_hat_list.append(p_s)
        calib_error, len_B_i = calib_error_score(s_i_hat_list, q_i_hat_list, NUM_BINS=NUM_BINS, output_q_y_est=False)
        CalibErr.append(calib_error)
    return CalibErr


from telegram import (
    InlineKeyboardButton,
    InlineKeyboardMarkup,
    ReplyKeyboardMarkup,
    ParseMode
)

from db import get_menu, get_dish

from glob import glob

FIRST, SECOND, THIRD, FOURTH = range(4)

MENU = ['hot_dishes', 'soup', 'pizza', 'drinks']

HOT_DISHES, SOUP, PIZZA, DRINKS = (
    'Горячие блюда',
    'Супы',
    'Пицца',
    'Напитки'
)

BACK, ADD_TO_SHOPPING_CART, REMOVE_FROM_SHOPPING_CART, MY_SHOPPING_CART, MAKE_ORDER = (
    'Назад',
    'Добавить в корзину',
    'Удалить',
    'Моя корзина',
    'Оформить заказ'
    )

INCREASE, DECREASE = (
    '▲',
    '▼'
)

KEYBOARD_NAVIGATION = [
    # [InlineKeyboardButton("Показать еще", callback_data=str(PIZZA))],
    [InlineKeyboardButton("Моя корзина", callback_data=str(MY_SHOPPING_CART))],
    # [InlineKeyboardButton("Оформить заказ", callback_data=str(DRINKS))],
    [InlineKeyboardButton("Назад", callback_data=str(BACK))]
]

KEYBOARD_MY_SHOPPING_CART = [
    [
        InlineKeyboardButton("Оформить заказ", callback_data=str(MAKE_ORDER)),
        InlineKeyboardButton("Назад", callback_data=str(BACK)),
    ]
]

KEYBOARD_MENU = [
    [
        InlineKeyboardButton("Горячие блюда", callback_data=str(HOT_DISHES)),
        InlineKeyboardButton("Супы", callback_data=str(SOUP))
    ],
    [
        InlineKeyboardButton("Пицца", callback_data=str(PIZZA)),
        InlineKeyboardButton("Напитки", callback_data=str(DRINKS))
    ],
    [InlineKeyboardButton("Моя корзина", callback_data=str(MY_SHOPPING_CART))]
]

KEYBOARD_SHOPPING_CART = [
    [
        InlineKeyboardButton("Добавить в корзину", callback_data=str(ADD_TO_SHOPPING_CART)),
    ]
]

KEYBOARD_SET_PORTION = [
    [
        InlineKeyboardButton('▼', callback_data=str(DECREASE)),
        InlineKeyboardButton(str(1), callback_data=str(1)), #Надо вставить переменную для указания количества добавленных порций.
        InlineKeyboardButton('▲', callback_data=str(INCREASE))
    ],
    [
        InlineKeyboardButton('Удалить', callback_data=str(REMOVE_FROM_SHOPPING_CART))
    ]
]

KEYBOARD_REMOVE_FROM_SHOPPING_CART = [
    [
        InlineKeyboardButton('Удалить', callback_data=str(REMOVE_FROM_SHOPPING_CART))
    ]
]

def order_keyboard():
    return ReplyKeyboardMarkup([
        ['Заказать', 'Отменить']
    ])

def main_keyboard():
    return ReplyKeyboardMarkup([
        ["Меню", "Заказать"]
    ])

def save_message_id(context, message_id):
        context.user_data['message_ids'].append(message_id['message_id'])

def delete_message_one(context, chat_id, message_id):
    context.bot.deleteMessage(
            chat_id=chat_id,
            message_id=message_id
        )
    context.user_data['message_ids'].remove(message_id)

def delete_message_all(context, chat_id):
    for message_id in context.user_data['message_ids']:
        context.bot.deleteMessage(
            chat_id=chat_id,
            message_id=message_id
        )
    context.user_data['message_ids'].clear()

def send_message(context, chat_id, menu, menu_type):
    for item in menu:
        message_id = context.bot.send_photo(
            chat_id=chat_id,
            photo=open(f"menu_imgs/{menu_type}/{item['id']}.png", 'rb'),
            caption=f"""
            Название: {item['name']}
Цена: {item['price']}
Описание: {item['description']}
ID: {menu_type}.{item['id']}
            """,
            reply_markup=InlineKeyboardMarkup(KEYBOARD_SHOPPING_CART),
            parse_mode = ParseMode.HTML
        )
        save_message_id(context, message_id)
    message_id = context.bot.send_message(
        chat_id=chat_id,
        text="Для оформления заказа выбирите интересующее блюдо и перейдите в корзину.",
        reply_markup=InlineKeyboardMarkup(KEYBOARD_NAVIGATION)
    )
    save_message_id(context, message_id)

def create_menu_list(context, menu_type):
    if not context.user_data.get(menu_type):
        context.user_data[menu_type] = []

def edit_message_reply_markup(context, dish_count, message_id, chat_id):
    context.bot.edit_message_reply_markup(
        message_id=message_id,
        chat_id=chat_id,
        reply_markup = InlineKeyboardMarkup(
            [[
                InlineKeyboardButton('▼', callback_data=str(DECREASE)),
                InlineKeyboardButton(str(dish_count), callback_data=str(dish_count)), #Надо вставить переменную для указания количества добавленных порций.
                InlineKeyboardButton('▲', callback_data=str(INCREASE))
            ],
            [
                InlineKeyboardButton('Удалить', callback_data=str(REMOVE_FROM_SHOPPING_CART))
            ]]
        )
    )

def show_my_shopping_cart(context, chat_id):
    total_price = 0
    for key, value in context.user_data.items():
        if key in MENU:
            for dish_dict in value:
                dish = get_dish(key, next(iter(dish_dict.keys())))
                dish_position_price = dish['price']*int(next(iter(dish_dict.values())))
                total_price += dish_position_price
                message_id = context.bot.send_photo(
                    chat_id=chat_id,
                    photo=open(f"menu_imgs/{key}/{dish['id']}.png", 'rb'),
                    caption=f"Блюдо: {dish['name']}\nЦена: {dish['price']}\nКоличество порций: {next(iter(dish_dict.values()))}\nСтоимость: {dish_position_price}\nID: {key}.{dish['id']}",
                    reply_markup=InlineKeyboardMarkup(KEYBOARD_REMOVE_FROM_SHOPPING_CART),
                    parse_mode = ParseMode.HTML
                )
                save_message_id(context, message_id)
    message_id = context.bot.send_message(
        chat_id=chat_id,
        text=f"К оплате: {total_price}",
        reply_markup=InlineKeyboardMarkup(KEYBOARD_MY_SHOPPING_CART)
    )
    save_message_id(context, message_id)
    context.user_data['total_price'] = total_price

def refresh_total_price(context, chat_id, message):
    message_id = context.user_data['message_ids'][-1]
    dish_price = message['caption'].split('\n')[3].split()[-1]
    total_price = context.user_data['total_price']
    refreshed_price = total_price - int(dish_price)
    context.user_data['total_price'] = refreshed_price
    context.bot.edit_message_text(
        message_id=message_id,
        chat_id=chat_id,
        text=f'К оплате: {refreshed_price}',
        reply_markup=InlineKeyboardMarkup(KEYBOARD_MY_SHOPPING_CART)
    )

def delete_dish(context, chat_id, message_id, message=0, flag=0):
    if flag == 'menu':
        message = context.bot.edit_message_reply_markup(
            message_id=message_id,
            chat_id=chat_id,
            reply_markup=InlineKeyboardMarkup(KEYBOARD_SHOPPING_CART)
        )
    else:
        delete_message_one(context, chat_id,  message_id)
        refresh_total_price(context, chat_id, message)
    print(message['caption'])
    dish_type = message['caption'].split()[-1].split('.')[0]
    dish_id = message['caption'].split()[-1].split('.')[-1]
    for dish in context.user_data[dish_type]:
        if dish.get(dish_id, False):
            context.user_data[dish_type].remove(dish)
            break
    print(context.user_data)


from django.shortcuts import render

# Create your views here.
from django.http import HttpResponse

#  밑줄 그을 텍스트
def index1(request) :
    return HttpResponse("Hello...")

#
문단

def index2(request) :
    return HttpResponse("
index2 함수 호출</>")

#  볼드체
def home(request) :
    return HttpResponse("Home")

def main(request) :
    return HttpResponse("Main")

# 데이터 입력
from .models import Curriculum

def insert(request) :
    msg = ""
    # 1) linux 입력
    Curriculum.objects.create(name='linux')
    msg += '1) linux 입력 성공
'
    # 2) python 입력
    c = Curriculum(name='python')
    c.save()
    msg += '2) python 입력 성공
'
    # 3) html/css/js 입력
    Curriculum(name='html/css/js').save()
    msg += '3) html/css/js 입력 성공
'
    # 4) django 입력
    Curriculum(name='django')
    msg += '4) django 입력 성공
'
    return HttpResponse(msg)

# 전체 데이터 조회
def show(request) :
    data = Curriculum.objects.all()

    msg = ''
    for dt in data :
        msg += dt.name + '
'

    return HttpResponse(msg)

# 한건 조회
def oneshow(request) :
    onedata = Curriculum.objects.get(pk=4)
    return HttpResponse(onedata.name)

def show2(request) :
    return render(
        request,
        # templates 폴더 내 경로만 작성
        "firstapp/show2.html",
        {}
    )

def show3(request) :
    data = Curriculum.objects.all()

    return render(
        request,
        # templates 폴더 내 경로만 작성
        "firstapp/show3.html",
        {'data' : data}
    )

def show4(request) :
    data = Curriculum.objects.all()

    return render(
        request,
        # templates 폴더 내 경로만 작성
        "firstapp/show4.html",
        {'data' : data}
    )

# 수정하기
def update(request) :
    data = Curriculum.objects.get(pk=1)
    data.name = 'linux_update'
    data.save()
    return HttpResponse('수정 성공')

# 삭제하기
def delete(request) :
    data1 = Curriculum.objects.get(pk=1)
    data1.delete()
    data2 = Curriculum.objects.get(pk=2)
    data2.delete()
    return HttpResponse('삭제 성공')


from numpy import random
import matplotlib.pyplot as plt

N=10 # 100, 1000
K=[i+1 for i in range(N)] # Total number of arms
R = {}  # Reward corresponding to each arm
for i in range(N):
    R[i+1]=1
prob = {}  # Probability corresponding to each arm
equi_space=1/N
prob_start=round(1/N-1/(N*10),6)
for i in range(N):
    prob[i + 1] = prob_start
    prob_start = round(prob_start + equi_space, 6)
T = 10000
eplison = 1
count = dict(zip(list(range(1, N+1)), [0] * N))  # Record the times of each arm
u={}
for i in range(N):
    u[i+1]=R[i+1]*prob[i+1]
u=sorted(u.items(), key=lambda d:d[1], reverse = True)
best_k=u[0][0]
best_u=u[0][1]

Q = dict(zip(list(range(1, N+1)), [0] * N))
r = 0
c=0.5
d_list=[]
u_list=[]
for i in range(T):
    if random.random() < eplison:
        k = random.choice(K)
    else:
        k = max(Q, key=Q.get)
    v = random.choice([R[k], 0], p=[prob[k], 1 - prob[k]])
    r += v*prob[k]
    Q[k] = (Q[k] * count[k] + v*prob[k]) / (count[k] + 1)
    temp_u=round(best_u*(i+1)-r,2)
    u_list.append(temp_u)
    count[k] = count[k] + 1
    d=best_u-prob[k]*v
    if d!=0:
        d_list.append(d)
    d = min(d_list)
    eplison=min(1,c*(N)/((i+1)*(d**2)))

print('eplision: ',eplison)
print("The total reward is {}".format(r))
plt.plot(list(range(T)),u_list)
plt.xlabel('turn')
plt.ylabel('regret')
print('Number of times each slot machine is selected: ',count)
print('The best slot machine to choose: ',best_k)
plt.show()


#!/usr/bin/python3
# -*- coding: utf-8 -*-
"""
https://pytorch.org/tutorials/beginner/translation_transformer.html
"""
import argparse
from dataclasses import asdict, dataclass, field, fields
import json
import math
import os
from pathlib import Path
import pickle
import platform
import random
from typing import Any, Callable, Dict, Iterable, List, Union

import numpy as np
import torch
import torch.nn as nn
from torch.nn import Transformer
from torch.nn.utils.rnn import pad_sequence
from torch.optim.lr_scheduler import StepLR
from torch.utils.data.dataloader import DataLoader
from torchtext.data.utils import get_tokenizer
from torchtext.datasets import multi30k, Multi30k
from torchtext.vocab import build_vocab_from_iterator, Vocab
from torchtext._torchtext import Vocab as VocabPybind
from tqdm import tqdm

from toolbox.torch.utils.data.vocabulary import Vocabulary

def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--multi30k_train_url",
        default="https://raw.githubusercontent.com/neychev/small_DL_repo/master/datasets/Multi30k/training.tar.gz",
        type=str
    )
    parser.add_argument(
        "--multi30k_valid_url",
        default="https://raw.githubusercontent.com/neychev/small_DL_repo/master/datasets/Multi30k/validation.tar.gz",
        type=str
    )
    parser.add_argument("--src_language", default="de", type=str)
    parser.add_argument("--tgt_language", default="en", type=str)

    parser.add_argument("--src_tokenizer_name", default="spacy", type=str)
    parser.add_argument("--src_tokenizer_language", default="de_core_news_sm", type=str)
    parser.add_argument("--tgt_tokenizer_name", default="spacy", type=str)
    parser.add_argument("--tgt_tokenizer_language", default="en_core_web_sm", type=str)

    parser.add_argument("--vocabulary_dir", default="./vocabulary", type=str)

    parser.add_argument("--learning_rate", default=2e-5, type=float)
    parser.add_argument("--num_train_epochs", default=200, type=int)
    parser.add_argument("--batch_size", default=128, type=int)
    parser.add_argument("--keep_most_recent_by_count", default=10, type=int)
    parser.add_argument("--patience", default=-1, type=int)
    parser.add_argument("--serialization_dir", default="serialization_dir", type=str)

    parser.add_argument("--lr_scheduler_step_size", default=500000, type=int)
    parser.add_argument("--lr_scheduler_gamma", default=1.0, type=float)

    parser.add_argument("--seed", default=3407, type=str, help="https://arxiv.org/abs/2109.08203")
    parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu", type=str)
    parser.add_argument("--device_id", default=0, type=int)

    parser.add_argument("--rank", default=0, type=int)
    parser.add_argument("--world_size", default=1, type=int)
    parser.add_argument("--ddp_backend", default="gloo", type=str)
    parser.add_argument("--ddp_timeout", default=1800, type=int)

    args = parser.parse_args()
    return args

class PositionalEncoding(nn.Module):
    def __init__(self,
                 emb_size: int,
                 dropout: float,
                 maxlen: int = 5000):
        super(PositionalEncoding, self).__init__()
        den = torch.exp(- torch.arange(0, emb_size, 2) * math.log(10000) / emb_size)
        pos = torch.arange(0, maxlen).reshape(maxlen, 1)
        pos_embedding = torch.zeros((maxlen, emb_size))
        pos_embedding[:, 0::2] = torch.sin(pos * den)
        pos_embedding[:, 1::2] = torch.cos(pos * den)
        pos_embedding = pos_embedding.unsqueeze(-2)

        self.dropout = nn.Dropout(dropout)
        self.register_buffer('pos_embedding', pos_embedding)

    def forward(self, token_embedding: torch.Tensor):
        return self.dropout(token_embedding + self.pos_embedding[:token_embedding.size(0), :])

class TokenEmbedding(nn.Module):
    def __init__(self, vocab_size: int, emb_size):
        super(TokenEmbedding, self).__init__()
        self.embedding = nn.Embedding(vocab_size, emb_size)
        self.emb_size = emb_size

    def forward(self, tokens: torch.Tensor):
        return self.embedding(tokens.long()) * math.sqrt(self.emb_size)

class Seq2SeqTransformer(nn.Module):
    def __init__(self,
                 num_encoder_layers: int,
                 num_decoder_layers: int,
                 emb_size: int,
                 num_head: int,
                 src_vocab_size: int,
                 tgt_vocab_size: int,
                 dim_feedforward: int = 512,
                 dropout: float = 0.1):
        super(Seq2SeqTransformer, self).__init__()
        self.transformer = Transformer(d_model=emb_size,
                                       nhead=num_head,
                                       num_encoder_layers=num_encoder_layers,
                                       num_decoder_layers=num_decoder_layers,
                                       dim_feedforward=dim_feedforward,
                                       dropout=dropout)
        self.generator = nn.Linear(emb_size, tgt_vocab_size)
        self.src_tok_emb = TokenEmbedding(src_vocab_size, emb_size)
        self.tgt_tok_emb = TokenEmbedding(tgt_vocab_size, emb_size)
        self.positional_encoding = PositionalEncoding(
            emb_size, dropout=dropout)

    def forward(self,
                src: torch.Tensor,
                trg: torch.Tensor,
                src_mask: torch.Tensor,
                tgt_mask: torch.Tensor,
                src_padding_mask: torch.Tensor,
                tgt_padding_mask: torch.Tensor,
                memory_key_padding_mask: torch.Tensor):
        """
        :param src: shape=[src_seq_len, batch_size].
        :param trg: shape=[trg_seq_len, batch_size].
        :param src_mask:
        :param tgt_mask:
        :param src_padding_mask:
        :param tgt_padding_mask:
        :param memory_key_padding_mask:
        :return: logits, shape=[batch_size, tgt_vocab_size, tgt_out_seq_len]
        """
        src_emb = self.positional_encoding(self.src_tok_emb(src))
        tgt_emb = self.positional_encoding(self.tgt_tok_emb(trg))
        outs = self.transformer(
            src_emb, tgt_emb, src_mask, tgt_mask, None,
            src_padding_mask, tgt_padding_mask, memory_key_padding_mask
        )
        logits: torch.Tensor = self.generator(outs)
        return logits

    def encode(self, src: torch.Tensor, src_mask: torch.Tensor) -> torch.Tensor:
        memory = self.transformer.encoder(
            self.positional_encoding(
                self.src_tok_emb(src)
            ),
            src_mask
        )
        return memory

    def decode(self, tgt: torch.Tensor, memory: torch.Tensor, tgt_mask: torch.Tensor) -> torch.Tensor:
        tgt_out = self.transformer.decoder(
            self.positional_encoding(
                self.tgt_tok_emb(tgt)
            ),
            memory,
            tgt_mask
        )
        return tgt_out

class CollateFunction(object):
    def __init__(self,
                 src_tokenizer,
                 tgt_tokenizer,
                 src_vocab: Vocab,
                 tgt_vocab: Vocab,
                 bos_idx: int,
                 eos_idx: int,
                 pad_idx: int,
                 ):
        self.src_tokenizer = src_tokenizer
        self.tgt_tokenizer = tgt_tokenizer
        self.src_vocab = src_vocab
        self.tgt_vocab = tgt_vocab
        self.bos_idx = bos_idx
        self.eos_idx = eos_idx
        self.pad_idx = pad_idx

    @staticmethod
    def make_mask(
        src: torch.Tensor,
        tgt_input: torch.Tensor,
        pad_idx: int,
    ):
        src_seq_len = src.shape[0]
        tgt_seq_len = tgt_input.shape[0]

        mask = torch.triu(torch.ones(size=(tgt_seq_len, tgt_seq_len)) == 1).transpose(0, 1)
        tgt_mask = mask.float().masked_fill(mask == 0, float("-inf")).masked_fill(mask == 1, float(0.0))

        src_mask = torch.zeros((src_seq_len, src_seq_len)).type(torch.bool)

        src_padding_mask = (src == pad_idx).transpose(0, 1)
        tgt_padding_mask = (tgt_input == pad_idx).transpose(0, 1)
        return src_mask, tgt_mask, src_padding_mask, tgt_padding_mask

    def __call__(self, batch: List[tuple]):
        src_batch, tgt_batch = [], []
        for src_sample, tgt_sample in batch:
            src_tokens = self.src_tokenizer(src_sample)
            src_token_ids = self.src_vocab(src_tokens)
            src_input_ids = torch.cat(
                tensors=(
                    torch.tensor([self.bos_idx]),
                    torch.tensor(src_token_ids),
                    torch.tensor([self.eos_idx])
                )
            )

            tgt_tokens = self.tgt_tokenizer(tgt_sample)
            tgt_token_ids = self.tgt_vocab(tgt_tokens)
            tgt_input_ids = torch.cat(
                tensors=(
                    torch.tensor([self.bos_idx]),
                    torch.tensor(tgt_token_ids),
                    torch.tensor([self.eos_idx])
                )
            )

            src_batch.append(src_input_ids)
            tgt_batch.append(tgt_input_ids)

        # [seq_len, batch_size]
        src_batch = pad_sequence(src_batch, padding_value=self.pad_idx)
        tgt_batch = pad_sequence(tgt_batch, padding_value=self.pad_idx)

        tgt_input = tgt_batch[:-1, :]
        tgt_out = tgt_batch[1:, :]

        # mask
        # src_mask.shape = [src_seq_len, batch_size]
        src_mask, tgt_mask, src_padding_mask, tgt_padding_mask = self.make_mask(
            src_batch, tgt_input, pad_idx=self.pad_idx
        )

        inputs = {
            "src": src_batch,
            "trg": tgt_input,
            "src_mask": src_mask,
            "tgt_mask": tgt_mask,
            "src_padding_mask": src_padding_mask,
            "tgt_padding_mask": tgt_padding_mask,
            "memory_key_padding_mask": src_padding_mask,
        }
        return inputs, tgt_out

@dataclass
class TrainingArguments(object):
    serialization_dir: str = field(
        metadata={"help": "The output directory where the model predictions and checkpoints will be written."},
    )
    num_train_epochs: int = field(default=3, metadata={"help": "Total number of training epochs to perform."})
    seed: int = field(default=42, metadata={"help": "Random seed that will be set at the beginning of training."})
    keep_most_recent_by_count: int = field(default=10, metadata={"help": "how many checkpoint to keep."})

@dataclass
class TrainerState(object):
    epoch_idx: int = 0
    global_step: int = 0

    best_epoch: int = -1
    best_validation_loss: float = float("inf")

    training_loss: float = float("inf")
    validation_loss: float = float("inf")

class Trainer(object):
    def __init__(self,
                 model: nn.Module,
                 args: TrainingArguments,
                 train_dataloader: DataLoader,
                 valid_dataloader: DataLoader,
                 loss_fn: Callable,
                 optimizer: torch.optim.Optimizer,
                 lr_scheduler: torch.optim.lr_scheduler.LRScheduler,
                 device=None
                 ):
        self.model = model
        self.args = args
        self.train_dataloader = train_dataloader
        self.valid_dataloader = valid_dataloader

        self.loss_fn = loss_fn
        self.optimizer = optimizer
        self.lr_scheduler = lr_scheduler

        # save checkpoint
        self.model_state_filename_list = list()
        self.training_state_filename_list = list()

        self.serialization_dir = Path(args.serialization_dir)
        self.serialization_dir.mkdir(exist_ok=True)

        self.device = device

        # global params
        self.state: TrainerState = None

    def _prepare_input(self, data: Union[torch.Tensor, Any]) -> Union[torch.Tensor, Any]:
        if isinstance(data, dict):
            return dict({k: self._prepare_input(v) for k, v in data.items()})
        elif isinstance(data, list):
            return list(self._prepare_input(v) for v in data)
        elif isinstance(data, tuple):
            return tuple(self._prepare_input(v) for v in data)
        elif isinstance(data, torch.Tensor):
            return data.to(self.device)
        return data

    def _prepare_inputs(self, inputs: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
        inputs = self._prepare_input(inputs)
        return inputs

    def train(self):
        self.model.to(self.device)

        self.state = TrainerState()

        for epoch_idx in range(self.args.num_train_epochs):
            self.state.epoch_idx = epoch_idx

            self.training_epoch(epoch_idx)
            self.evaluation_epoch(epoch_idx)
            self._save_checkpoint()
            self._save_metrics()

        return

    def training_epoch(self, epoch_idx: int):
        # train
        training_total_loss: float = 0.0
        training_total_steps: int = 0

        self.model.train()
        progress_bar = tqdm(self.train_dataloader, desc='Epoch={} (train)'.format(epoch_idx), leave=True)
        for step, batch in enumerate(progress_bar):
            inputs, targets = batch
            loss, _ = self.training_step(inputs, targets)

            training_total_loss += loss.item()
            training_total_steps += 1

            # progress_bar
            progress_bar_postfix = {
                "loss": training_total_loss / training_total_steps,
            }
            progress_bar.set_postfix(**progress_bar_postfix)

        training_loss: float = training_total_loss / training_total_steps
        self.state.training_loss = training_loss

    def training_step(self,
                      inputs: Dict[str, torch.Tensor],
                      targets: torch.Tensor
                      ) -> torch.Tensor:
        inputs = self._prepare_inputs(inputs)
        targets = targets.to(self.device)

        logits = self.model.forward(**inputs)

        loss = self.loss_fn(logits.reshape(-1, logits.shape[-1]), targets.reshape(-1))

        loss.backward()

        self.optimizer.step()
        self.optimizer.zero_grad()
        self.lr_scheduler.step()

        return loss, logits

    def evaluation_epoch(self, epoch_idx: int):
        # validation
        validation_total_loss: float = 0.0
        validation_total_steps: int = 0

        self.model.eval()
        progress_bar = tqdm(self.valid_dataloader, desc='Epoch={} (valid)'.format(epoch_idx), leave=True)
        for step, batch in enumerate(progress_bar):
            self.model.train()
            inputs, targets = batch
            loss, _ = self.prediction_step(inputs, targets)

            validation_total_loss += loss.item()
            validation_total_steps += 1

            # progress_bar
            progress_bar_postfix = {
                "loss": validation_total_loss / validation_total_steps,
            }
            progress_bar.set_postfix(**progress_bar_postfix)

        validation_loss: float = validation_total_loss / validation_total_steps
        self.state.validation_loss = validation_loss

    def prediction_step(self,
                        inputs: Dict[str, torch.Tensor],
                        targets: torch.Tensor
                        ) -> torch.Tensor:
        inputs = self._prepare_inputs(inputs)
        targets = targets.to(self.device)

        with torch.no_grad():
            logits = self.model.forward(**inputs)

            loss = self.loss_fn(logits.reshape(-1, logits.shape[-1]), targets.reshape(-1))

        return loss, logits

    def _save_checkpoint(self):
        # keep most recent by count
        if len(self.model_state_filename_list) >= self.args.keep_most_recent_by_count > 0:
            model_state_filename_ = self.model_state_filename_list.pop(0)
            os.remove(model_state_filename_)
            training_state_filename_ = self.training_state_filename_list.pop(0)
            os.remove(training_state_filename_)
        model_state_filename = self.serialization_dir / "model_state_{}.th".format(self.state.epoch_idx)
        training_state_filename = self.serialization_dir / "training_state_{}.th".format(self.state.epoch_idx)
        self.model_state_filename_list.append(model_state_filename.as_posix())
        with open(model_state_filename.as_posix(), "wb") as f:
            torch.save(self.model.state_dict(), f)
        self.training_state_filename_list.append(training_state_filename.as_posix())
        with open(training_state_filename.as_posix(), "wb") as f:
            torch.save(self.optimizer.state_dict(), f)

    def _save_metrics(self):
        # metrics epoch
        metrics = {
            "best_epoch": self.state.best_epoch,
            "epoch": self.state.epoch_idx,

            "training_loss": self.state.training_loss,
            "validation_loss": self.state.validation_loss,

            "best_validation_loss": self.state.best_validation_loss,
        }
        metrics_epoch_filename = self.serialization_dir / "metrics_epoch_{}.json".format(self.state.epoch_idx)
        with open(metrics_epoch_filename, "w", encoding="utf-8") as f:
            json.dump(metrics, f, indent=4, ensure_ascii=False)

def main():
    args = get_args()

    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)

    device = torch.device("{}:{}".format(args.device, args.device_id))
    n_gpu = torch.cuda.device_count()

    vocabulary = Vocabulary.from_files(args.vocabulary_dir)

    unk_idx = 0
    pad_idx = 1
    bos_idx = 2
    eos_idx = 3
    special_symbols = ["<unk>", "<pad>", "<bos>", "<eos>"]

    multi30k.URL["train"] = args.multi30k_train_url
    multi30k.URL["valid"] = args.multi30k_valid_url

    src_token_to_index = vocabulary.get_token_to_index_vocabulary(namespace="src_tokens")
    src_token_to_index = list(sorted(src_token_to_index.items(), key=lambda x: x[1], reverse=False))
    src_tokens = [k for k, v in src_token_to_index]
    src_vocab = Vocab(VocabPybind(src_tokens, None))
    src_vocab.set_default_index(unk_idx)

    tgt_token_to_index = vocabulary.get_token_to_index_vocabulary(namespace="tgt_tokens")
    tgt_token_to_index = list(sorted(tgt_token_to_index.items(), key=lambda x: x[1], reverse=False))
    tgt_tokens = [k for k, v in tgt_token_to_index]
    tgt_vocab = Vocab(VocabPybind(tgt_tokens, None))
    tgt_vocab.set_default_index(unk_idx)

    src_tokenizer = get_tokenizer(args.src_tokenizer_name, language=args.src_tokenizer_language)
    tgt_tokenizer = get_tokenizer(args.tgt_tokenizer_name, language=args.tgt_tokenizer_language)

    collate_fn = CollateFunction(
        src_tokenizer=src_tokenizer,
        tgt_tokenizer=tgt_tokenizer,
        src_vocab=src_vocab,
        tgt_vocab=tgt_vocab,
        bos_idx=bos_idx,
        eos_idx=eos_idx,
        pad_idx=pad_idx,
    )

    train_dataset = Multi30k(split="train", language_pair=(args.src_language, args.tgt_language))
    train_dataloader = DataLoader(
        train_dataset,
        batch_size=args.batch_size,
        shuffle=False,
        num_workers=0 if platform.system() == "Windows" else os.cpu_count(),

        collate_fn=collate_fn,
        pin_memory=False,
        drop_last=False
    )
    valid_dataset = Multi30k(split="valid", language_pair=(args.src_language, args.tgt_language))
    valid_dataloader = DataLoader(valid_dataset, batch_size=args.batch_size, collate_fn=collate_fn)

    transformer = Seq2SeqTransformer(
        num_encoder_layers=3,
        num_decoder_layers=3,
        emb_size=512,
        num_head=8,
        src_vocab_size=len(src_vocab),
        tgt_vocab_size=len(tgt_vocab),
        dim_feedforward=512,
    )

    for p in transformer.parameters():
        if p.dim() > 1:
            nn.init.xavier_uniform_(p)

    loss_fn = torch.nn.CrossEntropyLoss(ignore_index=pad_idx)
    optimizer = torch.optim.Adam(
        transformer.parameters(),
        lr=args.learning_rate, betas=(0.9, 0.98), eps=1e-9
    )
    lr_scheduler = StepLR(
        optimizer=optimizer,
        step_size=args.lr_scheduler_step_size,
        gamma=args.lr_scheduler_gamma
    )

    training_args = TrainingArguments(
        serialization_dir=args.serialization_dir,
        num_train_epochs=args.num_train_epochs,
        seed=args.seed,
        keep_most_recent_by_count=args.keep_most_recent_by_count,
    )
    trainer = Trainer(
        model=transformer,
        args=training_args,
        train_dataloader=train_dataloader,
        valid_dataloader=valid_dataloader,
        loss_fn=loss_fn,
        optimizer=optimizer,
        lr_scheduler=lr_scheduler,
        device=device,
    )
    trainer.train()

    return

if __name__ == '__main__':
    main()


import logging
import multiprocessing
import queue
import threading

import fiona
from shapely.geometry import LineString

from swiss_TML_api.name_finding.helper_index.flurnamen_index import FlurnamenIndex
from swiss_TML_api.name_finding.helper_index.street_index import StreetIndex
from swiss_TML_api.name_finding.index_builder.index_builder import IndexBuilder
from swiss_TML_api.name_finding.swiss_name import SwissName

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

class ForestBorders(IndexBuilder):
    """
    This IndexBuilder inserts all points where a path or road enters the forest.
    """

    def load(self):

        if self.reduced:
            self.insert_forest_intersections(
                ['swissTLM3D_TLM_BODENBEDECKUNG.shp'])
        else:
            self.insert_forest_intersections(
                ['swissTLM3D_TLM_BODENBEDECKUNG_west.shp', 'swissTLM3D_TLM_BODENBEDECKUNG_ost.shp'])

    def calc_intersection(self, path_index, obj):
        geo = obj["geometry"]['coordinates'][0]
        obj_type = obj['properties']['OBJEKTART']

        intersections = set()

        if obj_type in ['Wald'] and obj["geometry"]['type'] == 'Polygon':

            geo = list(map(lambda p: p[:-1], geo))  # remove elevation information

            polygon = LineString(geo)
            paths = path_index.intersection(coordinates=polygon.bounds, objects='raw')

            for path in paths:

                path = LineString(path['geo'])
                inters = polygon.intersection(path)

                if inters.geom_type == 'Point':
                    intersections.add((int(inters.x), int(inters.y)))

                elif inters.geom_type == 'MultiPoint':
                    for pkt in inters.geoms:
                        intersections.add((int(pkt.x), int(pkt.y)))

                elif inters.geom_type == 'LineString':
                    for pkt in inters.coords:
                        intersections.add((int(pkt[0]), int(pkt[1])))

        return intersections

    def insert_forest_intersections(self, shp_files):

        print(self.base_path)
        # Check if path index exists, if not it will be build
        street_index_builder = StreetIndex(self.base_path + 'swissTLM3D_TLM_STRASSE.shp')
        path_index = street_index_builder.get_street_index()
        path_index.close()  # as we use multiprocessing we have to reload the index for each thread

        flurname_index_builder = FlurnamenIndex(self.base_path + 'swissTLM3D_TLM_FLURNAME.shp')
        flurname_index = flurname_index_builder.get_flurname_index()

        pois = set()

        for shp_file in shp_files:
            pois = pois.union(self.process_shp_file(shp_file))

            logger.debug("\t -> insert {} results to index".format(len(pois)))
            for poi in pois:
                flurname: SwissName = next(flurname_index.nearest(coordinates=poi, num_results=1, objects='raw'))
                name = "Waldrand bei {}".format(flurname.name)
                self.index.insert(id=0,
                                  coordinates=poi,
                                  obj=SwissName(name=name, object_type="Waldrand", x=poi[0], y=poi[1], h=0))

            logger.debug("\t -> insertion completed")

    def process_shp_file(self, shp_file):

        # Calculate intersections with forest
        with fiona.open(self.base_path + shp_file) as src:

            q = queue.Queue()
            intersections = set()
            lock = threading.Lock()

            def worker(res_set, set_lock):

                street_index_builder = StreetIndex(self.base_path + 'swissTLM3D_TLM_STRASSE.shp')
                path_index = street_index_builder.get_street_index()

                while True:
                    batch = q.get()

                    batch_set = []
                    for o in batch:
                        batch_set.append(self.calc_intersection(path_index, o))

                    # merge results
                    set_lock.acquire()
                    res_set.update(*batch_set)  # we need update here and not union!
                    set_lock.release()

                    logger.debug("\t -> a worker has completed a batch")

                    q.task_done()

            batch_size = 20_000  # the workers process the elements in batches, was recognised empirically as faster
            logger.debug("\t -> populate queue with batches of size {}".format(batch_size))

            # TODO: Fix: Collection slicing is deprecated and will be disabled in a future version.
            for obj in [src[i:i + batch_size] for i in range(0, len(src), batch_size)]:
                q.put(obj)

            # Split work over multiple workers (n = number of CPU kernels)
            n_workers = multiprocessing.cpu_count()
            logger.debug("\t -> starting {} workers".format(n_workers))
            for i in range(n_workers):
                threading.Thread(target=worker, args=(intersections, lock), daemon=True).start()

            logger.debug("\t -> {} workers got started".format(n_workers))
            q.join()
            logger.debug("\t -> parallel work completed")

            return intersections


import xml.etree.ElementTree as ET

COMPARE_KEY_X = 1
COMPARE_KEY_Y = 2
COMPARE_KEY_WIDTH = 3
COMPARE_KEY_HEIGHT = 4
COMPARE_KEY_ROW = 5
COMPARE_KEY_COLUMN = 6

class HtmlTree:
    def __init__(self):
        self.root = ET.Element('div')
        self.root.set('class', 'container_main')
        self.root.set('name', 'main')
        self.tree = ET.ElementTree(self.root)
        self.current_node = self.root
    def add_node(self, node, parent=None):
        if parent:
            parent.append(node)
        else:
            self.root.append(node)

    def create_node(self,node_name, parent=None):
        self.current_node = ET.SubElement(parent,str(node_name))
        return self.current_node

    def add_attribute(self,node,**kwargs):

        for k,v in kwargs.items():
            node.set(k,v)

    def add_text(self,node,text):
        node.text = str(text)

html = HtmlTree()

class plugins:
    def __init__(self, left,top,width,height):
        self.x = left
        self.y = top
        self.width = width
        self.height = height
        self.option  = {COMPARE_KEY_X: self.compare_x,
                        COMPARE_KEY_Y: self.compare_y,
                        COMPARE_KEY_WIDTH: self.compare_width,
                        COMPARE_KEY_HEIGHT: self.compare_height,
                        COMPARE_KEY_ROW: self.compare_row,
                        COMPARE_KEY_COLUMN: self.compare_column
                        }

    def is_less(self,obj):
        if(self.x < obj.x and self.y < obj.y):
            return True
        else:
            return False
    def print_node(self):
        print(("( "+ str(self.x) + " - "+ str(self.y) +", "+ str(self.width) +", "+ str(self.height) + ")"))

    def node_str(self):
        return str("( "+ str(self.x) + " - "+ str(self.y) +", "+ str(self.width) +", "+ str(self.height) + ")")

    def compare(self,obj,key=COMPARE_KEY_X):
        return self.option[key](obj)

    def compare_x(self,obj):
        if self.x > obj.x:
            return 1
        elif obj.x > self.x:
            return -1
        else:
            return 0

    def compare_y(self,obj):
        if self.y > obj.y:
            return 1
        elif obj.y > self.y:
            return -1
        else:
            return 0

    def compare_width(self,obj):
        if self.width > obj.width:
            return 1
        elif obj.width > self.width:
            return -1
        else:
            return 0

    def compare_height(self,obj):
        if self.height > obj.height:
            return 1
        elif obj.height > self.height:
            return -1
        else:
            return 0

    def compare_row(self,obj):
        if self.x > obj.x:
            if self.y > obj.y:
                return 1
            elif self.y < obj.y:
                return -1
            else:
                return 1
        elif self.x < obj.x:
            if self.y < obj.y:
                return -1
            elif self.y > obj.y:
                return 1
            else:
                return -1
        else:
            if self.y < obj.y:
                return -1
            elif self.y > obj.y:
                return 1
            else:
                return 0

    def compare_column(self,obj):
        if self.y > obj.y:
            if self.x > obj.x:
                return 1
            elif self.x < obj.x:
                return -1
            else:
                return 1
        elif self.y < obj.y:
            if self.x < obj.x:
                return -1
            elif self.x > obj.x:
                return 1
            else:
                return -1
        else:
            if self.x < obj.x:
                return -1
            elif self.x > obj.x:
                return 1
            else:
                return 0

def insert_sort(obj_list,key):
    for i in range(len(obj_list)):
        for j in range(i,0,-1):
            if obj_list[j].compare(obj_list[j-1],key) <= 0:
                tmp = obj_list[j-1]
                obj_list[j-1] = obj_list[j]
                obj_list[j] = tmp
            else:
                break

    return obj_list

def create_row(obj_list,root):
    """
    1. Pop first object from the list.
    2. Set MAX_HEIGHT to obj.y + obj.height
    3. for all other objects do following:
        a. if obj.y  < MAX_HEIGHT the add it to temp list and pop it from original list.
        b. if obj.y + obj.height > MAX_HEIGHT then change it to obj.y + obj.height.
    4. Open the ROW div.
    5. Call the create_column(temp_list).
    6. close the ROW div
    """
    if len(obj_list) == 0:
        return

    #sort the list accordingly
    obj_list = insert_sort(obj_list,COMPARE_KEY_ROW)

#     print "create_row called with list"
#
#     for elem in obj_list:
#         elem.print_node()

    # Pop first element and set max HEIGHT
    obj = obj_list.pop(0)
    MAX_HEIGHT = obj.y + obj.height
    row_list = [obj]

    # create the row list to be passed to the column function

    i = 0
    n = len(obj_list)
    while i < n:
        element = obj_list[i]
        if element.y < MAX_HEIGHT:
            row_list.append(element)
            if element.y + element.height > MAX_HEIGHT:
                MAX_HEIGHT = element.y + element.height
            del obj_list[i]
            n = n - 1
        else:
            i = i + 1

    tmp = html.create_node('div',root)
    kwargs = {"name":'row',"class":'ROW'}
    html.add_attribute(tmp,**kwargs)
#     print("
")
    create_column(row_list,tmp)
#     print("
 !< ROW !>")

    if len(obj_list) == 0:
        return
    create_row(obj_list,root)

def create_column(obj_list,root):
    """
    1. Pop first object from the list.
    2. Set MAX_WIDTH to obj.x + obj.width
    3. for all other objects do following:
        a. if obj.x  < MAX_WIDTH the add it to temp list and pop it from original list.
        b. if obj.x + obj.width > MAX_WIDTH then change it to obj.x + obj.width.
    4. Open the COLUMN div.
    5. Call the create_row(temp_list).
    6. close the COLUMN div
    """
    if len(obj_list) == 0:
        return
    #sort the list accordingly
    obj_list = insert_sort(obj_list,COMPARE_KEY_COLUMN)

#     print "create_column called with list"
#
#     for elem in obj_list:
#         elem.print_node()

    # Pop first element and set max HEIGHT
    obj = obj_list.pop(0)
    MAX_WIDTH = obj.x + obj.width
    column_list = [obj]

    if len(obj_list) == 0:
        tmp = html.create_node('div',root)
        kwargs = {"name":'column',"class":'COLUMN'}
        html.add_attribute(tmp,**kwargs)
        html.add_text(tmp,obj.node_str())

#         print("
")
#         obj.print_node()
#         print("
 !< COLUMN !>")
        return

    # create the column list to be passed to the row function
    i = 0
    n = len(obj_list)
    while i < n:
        element = obj_list[i]
        if element.x < MAX_WIDTH:
            column_list.append(element)
            if element.x + element.width > MAX_WIDTH:
                MAX_WIDTH = element.x + element.width
            del obj_list[i]
            n = n - 1
        else:
            i = i + 1

    # pass to create row only if it contains more than 1 items
    tmp = html.create_node('div',root)
    kwargs = {"name":'column',"class":'COLUMN'}
    html.add_attribute(tmp,**kwargs)
#     print("
")
    if len(column_list) > 1:
        create_row(column_list,tmp)
    else:
        html.add_text(tmp,obj.node_str())
#         column_list[0].print_node()
#     print("
 !< COLUMN !>")
    create_column(obj_list,root)

def create_array(object_list):
#     obj_list = [plugins(0,0,10,15),plugins(12,17,5,10),plugins(20,30,8,12),plugins(0,45,10,15),plugins(12,45,10,15),plugins(24,45,5,10)]
#     obj_list = [plugins(0,0,10,15),plugins(12,0,5,10),plugins(20,0,8,12)]

#     for elem in  obj_list:
#         elem.print_node()
#     print "after sorting the array in row"
#     obj_list = insert_sort(obj_list, COMPARE_KEY_ROW)
#     for elem in  obj_list:
#         elem.print_node()
#
#     print "after sorting the array in column"
#     obj_list = insert_sort(obj_list, COMPARE_KEY_COLUMN)
#     for elem in  obj_list:
#         elem.print_node()

    create_row(object_list,html.root)

    print("\n\n\n Printing the Tree \n\n\n")

    ET.dump(html.root)

if __name__ == "__main__":
    create_array()


import connexion
from openapi_server.annotator.annotate import Annotator
from openapi_server.models.error import Error  # noqa: E501
from openapi_server.models.text_person_name_annotation_request import TextPersonNameAnnotationRequest  # noqa: E501
from openapi_server.models.text_person_name_annotation import TextPersonNameAnnotation  # noqa: E501
from openapi_server.models.text_person_name_annotation_response import TextPersonNameAnnotationResponse  # noqa: E501

annotator: Annotator = Annotator.load(
    'openapi_server/annotator/cached/annotator_model.joblib')

def create_text_person_name_annotations():  # noqa: E501
    """Annotate person names in a clinical note

    Return the person name annotations found in a clinical note # noqa: E501

    :rtype: TextPersonNameAnnotationResponse
    """
    res = None
    status = None
    if connexion.request.is_json:
        try:
            annotation_request = TextPersonNameAnnotationRequest.from_dict(connexion.request.get_json())  # noqa: E501
            note = annotation_request.note  # noqa: E501
            annotation_set, = annotator.annotate([note.text])
            annotations = [TextPersonNameAnnotation(
                start=annotation['start'],
                length=annotation['end'] - annotation['start'],
                text=annotation['text'],
                confidence=95.0
            ) for annotation in annotation_set]

            res = TextPersonNameAnnotationResponse(annotations)
            status = 200
        except Exception as error:
            status = 500
            res = Error("Internal error", status, str(error))
    return res, status


import json

from odoo_test_helper import FakeModelLoader

from .common import LocoCommonCase, mock_site_api

class TestSiteSearchEngineExportBase(LocoCommonCase):

    maxDiff = None

    @classmethod
    def setUpClass(cls):
        super(TestSiteSearchEngineExportBase, cls).setUpClass()
        cls._setup_search_engine()
        cls.routes = [
            [
                "*",
                {
                    "name": "category",
                    "template_handle": "category",
                    "index": "categories",
                },
            ],
            [
                "*",
                {
                    "name": "product",
                    "template_handle": "product",
                    "index": "products",
                },
            ],
        ]
        cls.indexes = [
            {"name": "categories", "index": "fake_se_shopinvader_category"},
            {"name": "products", "index": "fake_se_shopinvader_variant"},
        ]
        cls.expected_settings = {
            "indices": json.dumps(cls.indexes),
            "routes": json.dumps(cls.routes),
        }

    @classmethod
    def _setup_search_engine(cls):
        raise NotImplementedError()

    def setUp(self):
        super(TestSiteSearchEngineExportBase, self).setUp()
        self.metafields = {self.search_engine_name: {}}
        # simplified version of site data
        self.site = {
            "name": "My site",
            "handle": "shopinvader",
            "_id": "space_id",
            "metafields": json.dumps(self.metafields),
        }

    def _update_site_metafields(self, key, values):
        self.metafields[key] = values
        self.site["metafields"] = json.dumps(self.metafields)

    def _extract_metafields(self, metafields):
        metafields = json.loads(metafields)
        self.assertIn("_store", metafields)
        return metafields

    def _test_sync(self, expected, reset=False):
        with mock_site_api(self.base_url, self.site) as mock:
            if reset:
                self.backend.reset_site_settings()
            else:
                self.backend.synchronize_metadata()
            metafields = self._extract_metafields(
                mock.request_history[0].json()["site"]["metafields"]
            )
            self.assertDictEqual(metafields[self.search_engine_name], expected)

class TestSiteSearchEngineExport(TestSiteSearchEngineExportBase):
    @classmethod
    def _setup_search_engine(cls):
        # Load fake models ->/
        cls.loader = FakeModelLoader(cls.env, cls.__module__)
        cls.loader.backup_registry()
        from odoo.addons.connector_search_engine.tests.models import SeBackendFake

        cls.loader.update_registry((SeBackendFake,))
        # ->/ Load fake models
        cls.se_backend = (
            cls.env[SeBackendFake._name].create({"name": "Fake SE"}).se_backend_id
        )
        cls.search_engine_name = cls.se_backend.search_engine_name
        cls.backend.se_backend_id = cls.se_backend
        cls.env["se.index"].create(
            {
                "backend_id": cls.backend.se_backend_id.id,
                "name": "index-product",
                "lang_id": cls.env.ref("base.lang_en").id,
                "model_id": cls.env.ref("shopinvader.model_shopinvader_variant").id,
                "exporter_id": cls.env.ref("shopinvader.ir_exp_shopinvader_variant").id,
            }
        )
        cls.env["se.index"].create(
            {
                "backend_id": cls.backend.se_backend_id.id,
                "name": "index-category",
                "lang_id": cls.env.ref("base.lang_en").id,
                "model_id": cls.env.ref("shopinvader.model_shopinvader_category").id,
                "exporter_id": cls.env.ref(
                    "shopinvader.ir_exp_shopinvader_category"
                ).id,
            }
        )

    @classmethod
    def tearDownClass(cls):
        cls.loader.restore_registry()
        super(TestSiteSearchEngineExport, cls).tearDownClass()

    def test_search_engine_synchronize_01(self):
        """
        Data:
            * SE data not filled into locomotive
        Test case:
            * synchronize metadata
        Expected result:
            * SE data is filled
        :return:
        """
        # wipe current settings
        self._update_site_metafields(self.search_engine_name, {})
        # run sync -> we get the expected default ones
        self._test_sync(self.expected_settings)

    def test_search_engine_synchronize_02(self):
        """
        Data:
            * SE data filled into locomotive
        Test case:
            * synchronize metadata
        Expected result:
            * SE data not updated into locomotive
        :return:
        """
        old_values = {"indices": "foo", "routes": "baz"}
        # force this values in settings
        self._update_site_metafields(self.search_engine_name, old_values)
        # run sync w/no reset -> they are preserved
        self._test_sync(old_values)

    def test_search_engine_synchronize_03(self):
        """
        Data:
            * SE data filled into locomotive
        Test case:
            * force reset settings
        Expected result:
            * SE data updated into locomotive
        :return:
        """
        old_values = {"indices": "foo", "routes": "baz"}
        # force this values in settings
        self._update_site_metafields(self.search_engine_name, old_values)
        # run sync w/ reset -> they are computed again
        self._test_sync(self.expected_settings, reset=True)


from django.urls import path
from .import views

app_name = "cases"

urlpatterns = [
    # DEFINE URL FOR HOME PAGE
    path('', views.HomeView.as_view(), name="home"),
    # DEFINE URL FOR CLIENTS
    path('clients/', views.ClientListView.as_view(), name="client_list"),
    path('clients/create/', views.ClientCreateView.as_view(), name="client_create"),
    path('clients/<int:pk>/update/',
         views.ClientUpdateView.as_view(), name="client_update"),
    path('clients/<int:pk>/delete/',
         views.ClientDeleteView.as_view(), name="client_delete"),

    # DEFINE URL FOR LAWYERS
    path('lawyers/', views.LawyerListView.as_view(), name="lawyer_list"),
    path('lawyers/create/', views.LawyerCreateView.as_view(), name="lawyer_create"),
    path('lawyers/<int:pk>/update/',
         views.LawyerUpdateView.as_view(), name="lawyer_update"),
    path('lawyers/<int:pk>/delete/',
         views.LawyerDeleteView.as_view(), name="lawyer_delete"),

    # DEFINE URL FOR CASES
    path('cases/', views.CaseListView.as_view(), name="case_list"),
    path('cases/create/', views.CaseCreateView.as_view(), name="case_create"),
    path('cases/<int:pk>/update/',
         views.CaseUpdateView.as_view(), name="case_update"),
    path('cases/<int:pk>/delete/',
         views.CaseDeleteView.as_view(), name="case_delete"),

]


import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from torch.autograd import Variable
import torch
import torch.nn as nn
import torchvision.datasets as dsets
import torchvision.transforms as transforms
from torch.autograd import Variable
from PIL import Image

# same function as ConcatTable container in Torch7.
class ConcatTable(nn.Module):
    def __init__(self, layer1, layer2):
        super(ConcatTable, self).__init__()
        self.layer1 = layer1
        self.layer2 = layer2

    def forward(self, x):
        y = [self.layer1(x), self.layer2(x)]
        return y

class Flatten(nn.Module):
    def __init__(self):
        super(Flatten, self).__init__()

    def forward(self, x):
        return x.view(x.size(0), -1)

class fadein_layer(nn.Module):
    def __init__(self, config):
        super(fadein_layer, self).__init__()
        self.alpha = 0.0

    def update_alpha(self, delta):
        self.alpha = self.alpha + delta
        self.alpha = max(0, min(self.alpha, 1.0))

    # input : [x_low, x_high] from ConcatTable()
    def forward(self, x):
        return torch.add(x[0].mul(1.0 - self.alpha), x[1].mul(self.alpha))

class minibatch_std_concat_layer(nn.Module):
    def __init__(self):
        super(minibatch_std_concat_layer, self).__init__()

    def forward(self, x):
        std = x.clone().std(1, keepdim=True)
        return torch.cat((x, std), 1)

class pixelwise_norm_layer(nn.Module):
    def __init__(self):
        super(pixelwise_norm_layer, self).__init__()
        self.elipson = 1e-8

    def forward(self, x):
        nf = x.size(1)
        t = x.clone()
        t.data.fill_(0)
        norm = torch.sqrt(x.pow(2).sum(1, keepdim=True).expand_as(x).div(nf).add(self.elipson))
        return torch.addcdiv(t, 1, x, norm)

# for equaliaeed-learning rate.
class equalized_conv2d(nn.Module):
    def __init__(self, c_in, c_out, k_size, stride, pad, initializer='kaiming', bias=False):
        super(equalized_conv2d, self).__init__()
        self.conv = nn.Conv2d(c_in, c_out, k_size, stride, pad, bias=False)
        if initializer == 'kaiming':
            torch.nn.init.kaiming_normal(self.conv.weight)
        elif initializer == 'xavier':
            torch.nn.init.xavier_normal(self.conv.weight)

        conv_w = self.conv.weight.data.clone()
        self.bias = torch.nn.Parameter(torch.FloatTensor(c_out).fill_(0))
        self.scale = np.sqrt(conv_w.pow(2).mean())
        inv_w = conv_w.clone().fill_(self.scale)
        t = inv_w.clone().fill_(0)
        self.conv.weight.data = torch.addcdiv(t, 1, self.conv.weight.data, inv_w)  # adjust weights dynamically.

    def forward(self, x):
        x = self.conv(x.mul(self.scale))
        return x + self.bias.view(1, -1, 1, 1).expand_as(x)

class equalized_deconv2d(nn.Module):
    def __init__(self, c_in, c_out, k_size, stride, pad, initializer='kaiming'):
        super(equalized_deconv2d, self).__init__()
        self.deconv = nn.ConvTranspose2d(c_in, c_out, k_size, stride, pad, bias=False)
        if initializer == 'kaiming':
            torch.nn.init.kaiming_normal(self.deconv.weight)
        elif initializer == 'xavier':
            torch.nn.init.xavier_normal(self.deconv.weight)

        deconv_w = self.deconv.weight.data.clone()
        self.scale = np.sqrt(deconv_w.pow(2).mean())
        self.bias = torch.nn.Parameter(torch.FloatTensor(c_out).fill_(0))
        inv_w = deconv_w.clone().fill_(self.scale)
        t = inv_w.clone().fill_(0)
        self.deconv.weight.data = torch.addcdiv(t, 1, self.deconv.weight.data, inv_w)  # adjust weights dynamically.

    def forward(self, x):
        x = self.deconv(x.mul(self.scale))
        return x + self.bias.view(1, -1, 1, 1).expand_as(x)

class equalized_linear(nn.Module):
    def __init__(self, c_in, c_out, initializer='kaiming'):
        super(equalized_linear, self).__init__()
        self.linear = nn.Linear(c_in, c_out, bias=False)
        if initializer == 'kaiming':
            torch.nn.init.kaiming_normal(self.linear.weight)
        elif initializer == 'xavier':
            torch.nn.init.xavier_normal(self.linear.weight)

        linear_w = self.linear.weight.data.clone()
        self.bias = torch.nn.Parameter(torch.FloatTensor(c_out).fill_(0))
        self.scale = np.sqrt(linear_w.pow(2).mean())
        inv_w = linear_w.clone().fill_(self.scale)
        t = inv_w.clone().fill_(0)
        self.linear.weight.data = torch.addcdiv(t, 1, self.linear.weight.data, inv_w)  # adjust weights dynamically.

    def forward(self, x):
        x = self.linear(x.mul(self.scale))
        return x + self.bias.view(1, -1).expand_as(x)


from appdirs import AppDirs
from yaml import safe_load, safe_dump
from os import mkdir
from os.path import dirname, isdir
from platform import system
from PySide6.QtCore import QTime
from PySide6.QtGui import QIcon
from PySide6.QtWidgets import QApplication

app = QApplication()
path = dirname(__file__)

CONFIG_FILE = "AwakeGuardian"
CONFIG_FILE_EXT = ".conf"

D = "DEFAULT"
LHT = "last_hold_time"
P = "profile"
R = "remind"
N = "nag"
TTRM = "time_to_remind_m"
TTRS = "time_to_remind_s"
TTNM = "time_to_nag_m"
TTNS = "time_to_nag_s"
PM = "power_management"
PMA = "power_management_action"
TTEM = "time_to_event_m"
TTES = "time_to_event_s"
IVR = "increase_volume_remind"
IVN = "increase_volume_nag"
TR = "time_range_active"  # FUTURE: this should be just "time_range" in v2.x, kept for backward compatibility
TRF = "time_range_from"
TRT = "time_range_to"
GS = "geometry_settings"
GH = "geometry_hold"

def defaults(key):
    d = {
        LHT: 15,
        P: 0,
        R: 1,
        N: 1,
        TTRM: 10,
        TTRS: 0,
        TTNM: 15,
        TTNS: 0,
        PM: 0,
        PMA: 1,
        TTEM: 20,
        TTES: 0,
        IVR: 0,
        IVN: 1,
        TR: 1,
        TRF: "20:00:00",
        TRT: "08:00:00",
        GH: None,
        GS: None,
    }
    return key, d[key]

class Icon:
    beep = QIcon(f"{path}/images/beep.png")
    clock = QIcon(f"{path}/images/clock.png")
    exit = QIcon(f"{path}/images/exit.png")
    eyes = QIcon(f"{path}/images/eyes.png")
    inactive = QIcon(f"{path}/images/inactive.png")
    settings = QIcon(f"{path}/images/settings.png")
    shout = QIcon(f"{path}/images/shout.png")

class Audio:
    coin = f"{path}/audio/coin.mp3"
    wilhelm = f"{path}/audio/wilhelm.mp3"

class Config:
    def __init__(self):
        self.config_file = self.setup_config_file()
        self.load_config()

    def setup_config_file(self):
        dirs = AppDirs(CONFIG_FILE)
        config_dir = dirname(dirs.user_config_dir)
        if not isdir(config_dir):
            try:
                mkdir(config_dir)
            except Exception as e:
                raise e
        return f"{dirs.user_config_dir}{CONFIG_FILE_EXT}"

    def set_hold_time(self, minutes):
        self.last_hold_time = minutes
        self.save_config()

    def set_time_to_remind(self, minutes, seconds):
        if minutes is not None:
            self.t_to_remind_m = minutes
        if seconds is not None:
            self.t_to_remind_s = seconds
        self.save_config()

    def set_time_to_nag(self, minutes, seconds):
        if minutes is not None:
            self.t_to_nag_m = minutes
        if seconds is not None:
            self.t_to_nag_s = seconds
        self.save_config()

    def set_time_to_event(self, minutes, seconds):
        if minutes is not None:
            self.t_to_event_m = minutes
        if seconds is not None:
            self.t_to_event_s = seconds
        self.save_config()

    def set_remind(self, value):
        self.remind = int(value)
        self.save_config()

    def set_nag(self, value):
        self.nag = int(value)
        self.save_config()

    def set_power_management(self, value):
        self.power_management = int(value)
        self.save_config()

    def set_power_management_action(self, value):
        self.power_management_action = int(value)
        self.save_config()

    def set_inc_volume_remind(self, value):
        self.inc_volume_remind = value
        self.save_config()

    def set_inc_volume_nag(self, value):
        self.inc_volume_nag = value
        self.save_config()

    def set_time_range(self, value):
        self.t_range = int(value)
        self.save_config()

    def set_time_range_from(self, time):
        self.t_range_f = time
        self.save_config()

    def set_time_range_to(self, time):
        self.t_range_t = time
        self.save_config()

    def set_geometry_settings(self, geometry):
        x = geometry.x()
        y = geometry.y()
        w = geometry.width()
        h = geometry.height()
        self.geometry_settings = f"{x} {y} {w} {h}"
        self.save_config()

    def set_geometry_hold(self, geometry):
        x = geometry.x()
        y = geometry.y()
        w = geometry.width()
        h = geometry.height()
        self.geometry_hold = f"{x} {y} {w} {h}"
        self.save_config()

    def save_config(self):
        settings = {
            P: self.profile,
            R: self.remind,
            N: self.nag,
            TTRM: self.t_to_remind_m,
            TTRS: self.t_to_remind_s,
            TTNM: self.t_to_nag_m,
            TTNS: self.t_to_nag_s,
            IVR: self.inc_volume_remind,
            IVN: self.inc_volume_nag,
            PM: self.power_management,
            PMA: self.power_management_action,
            TTEM: self.t_to_event_m,
            TTES: self.t_to_event_s,
            LHT: self.last_hold_time,
            TR: self.t_range,
            TRF: self.t_range_f.toString(),
            TRT: self.t_range_t.toString(),
            GH: "" if self.geometry_hold is None else self.geometry_hold,
            GS: "" if self.geometry_settings is None else self.geometry_settings,
        }
        with open(self.config_file, "w") as conf_file:
            conf_file.write(safe_dump(settings))

    def clear_config(self):
        open(self.config_file, "w")

    def load_config(self):
        cfg = {}
        try:
            loaded = safe_load(open(self.config_file))
            if loaded:
                cfg = loaded
        except:
            pass

        self.profile = int(cfg.get(*defaults(P)))
        self.remind = int(cfg.get(*defaults(R)))
        self.nag = int(cfg.get(*defaults(N)))

        self.last_hold_time = int(cfg.get(*defaults(LHT)))
        self.t_to_remind_m = int(cfg.get(*defaults(TTRM)))
        self.t_to_remind_s = int(cfg.get(*defaults(TTRS)))
        self.t_to_nag_m = int(cfg.get(*defaults(TTNM)))
        self.t_to_nag_s = int(cfg.get(*defaults(TTNS)))

        self.inc_volume_remind = int(cfg.get(*defaults(IVR)))
        self.inc_volume_nag = int(cfg.get(*defaults(IVN)))
        self.volume = None

        self.power_management = int(cfg.get(*defaults(PM)))
        self.power_management_action = int(cfg.get(*defaults(PMA)))
        self.t_to_event_m = int(cfg.get(*defaults(TTEM)))
        self.t_to_event_s = int(cfg.get(*defaults(TTES)))

        self.t_range = int(cfg.get(*defaults(TR)))
        self.t_range_f = QTime.fromString(cfg.get(*defaults(TRF)))
        self.t_range_t = QTime.fromString(cfg.get(*defaults(TRT)))

        self.geometry_hold = cfg.get(*defaults(GH))
        self.geometry_settings = cfg.get(*defaults(GS))

    def restore_defaults(self):
        self.clear_config()
        self.load_config()
        self.save_config()


# Status: Complete and working
# Description: Exports the following mentioned DynamoDB tables into csv in 'exports' directory
# How to: python3 export-dynamodb.py
# Note: Just update the TABLES array before running the script
import csv
import boto3
from datetime import datetime
import pytz

TABLES = [
        'timbre_eval',
        'users',
        'timbre_survey',
        'dontknow',
        'less',
        'more',
        'notapplicable',
        'users-2'
        ]

def export_table(dynamodb, TABLE_NAME, ts):
    print('Exporting table: ' + TABLE_NAME)

    table = dynamodb.Table(TABLE_NAME)
    response = table.scan()
    items = response['Items']

    print('\tDownloaded data for table: ' + TABLE_NAME + ' (' + str(len(items)) + ')')

    keys = set()
    for item in items: keys.update(item.keys())

    with open('exports-dynamo/' + ts + '-' + TABLE_NAME + '.csv', 'w', newline='')  as output_file:
        dict_writer = csv.DictWriter(output_file, list(keys))
        dict_writer.writeheader()
        dict_writer.writerows(items)

if __name__ == '__main__':
    dynamodb = boto3.resource('dynamodb')

    timezone = pytz.timezone('Asia/Tokyo')
    now = timezone.localize(datetime.now())
    timestamp = now.strftime("%Y%m%d%H%M%S")
    print('Timestamp for the export: ' + timestamp)

    for table in TABLES:
        export_table(dynamodb, table, timestamp)