# # -*- coding: utf-8 -*-

import datetime
from datetime import datetime
import pytz
from odoo import models,fields
from datetime import datetime, timedelta
from time import mktime

class as_invoices_emit_excel(models.AbstractModel):
    _name = 'report.l10n_cl_tichile_reports.invoices_emit_report_xls.xlsx'
    _inherit = 'report.report_xlsx.abstract'

    def generate_xlsx_report(self, workbook, data, lines):

        consulta = ("""
            SELECT
            to_char(((ai.fecha_boliviana AT TIME ZONE 'UTC' AT TIME ZONE 'BOT')::timestamp),'DD/MM/YYYY') AS fecha_nota,
                    """)
        moduleObj = self.env['ir.module.module'].sudo().search([("name","=","as_sales"),("state","=","installed")])
        if data['form']['as_tipo']=='ventas':
            if moduleObj:
                consulta += "so.as_numeracion_interna AS numero_nota,"
            else:
                consulta += "so.name AS numero_nota,"
        else:
            consulta += "ai.name AS numero_nota,"
        if data['form']['as_tipo']=='ventas':
            consulta += ("""
                ai.id,
                ai.as_nit AS nit_cliente,
                ai.as_razon_social AS nombre_cliente,
                ewqc.numero_autorizacion AS numero_autorizacion,
                ai.control_code AS codigo_control,
            CASE
                WHEN rc.NAME = 'BOB'
                OR rc.symbol = 'Bs.' THEN
                ROUND( ai.amount_total, 4 ) ELSE ai.amount_total
                END AS importe_neto_venta,
                ROUND( ai.amount_total, 4 ) AS importe_neto_factura,
                ai.invoice_number AS numero_factura,
                to_char((( ai.fecha_boliviana AT TIME ZONE'UTC' AT TIME ZONE'BOT' ) :: TIMESTAMP ), 'DD/MM/YYYY' ) AS fecha_factura,
                asesor.NAME AS nombre_asesor,
                ROUND( ai.amount_total, 4 ) AS impote_bruto_fact,
                ROUND( ai.amount_total, 4 ) AS total_descuento_fact,
                ai.fecha_boliviana AS fecha_factura2,
                fp.name AS tipo_venta
            FROM
                account_invoice ai
                JOIN sale_order so ON so.NAME = ai.origin
                JOIN res_partner AS cliente ON cliente.ID = so.partner_id
                JOIN res_users AS usuario ON usuario.ID = so.user_id
                JOIN res_partner AS asesor ON asesor.ID = usuario.partner_id
                JOIN qr_code AS ewqc ON ewqc.ID = ai.qr_code_id
                JOIN product_pricelist pp ON pp.ID = so.pricelist_id
                JOIN res_currency rc ON rc.ID = pp.currency_id
                JOIN as_metodo_pago_ventas fp ON fp.id = so.as_forma_pago_id
            WHERE
                ai.TYPE = 'out_invoice'
                AND ai.STATE NOT IN ( 'draft', 'cancel' )
                AND ai.order_status = 'valida'
                AND ai.invoice_number != 0
                AND ai.type = 'out_invoice'
            """)
        elif data['form']['as_tipo']=='pos':
            consulta += ("""
                ai.id,
                ai.as_nit AS nit_cliente,
                ai.as_razon_social AS nombre_cliente,
                ewqc.numero_autorizacion AS numero_autorizacion,
                ai.control_code AS codigo_control,
            CASE
                WHEN rc.NAME = 'BOB'
                OR rc.symbol = 'Bs.' THEN
                ROUND( ai.amount_total, 4 ) ELSE ai.amount_total
                END AS importe_neto_venta,
                ROUND( ai.amount_total, 4 ) AS importe_neto_factura,
                ai.invoice_number AS numero_factura,
                to_char((( ai.fecha_boliviana AT TIME ZONE'UTC' AT TIME ZONE'BOT' ) :: TIMESTAMP ), 'DD/MM/YYYY' ) AS fecha_factura,
                asesor.NAME AS nombre_asesor,
                ROUND( ai.amount_total, 4 ) AS impote_bruto_fact,
                ROUND( ai.amount_total, 4 ) AS total_descuento_fact,
                ai.fecha_boliviana AS fecha_factura2,
                'N/A'
            FROM
                account_invoice ai
                JOIN pos_order so ON so.NAME = ai.origin
                JOIN res_partner AS cliente ON cliente.ID = so.partner_id
                JOIN res_users AS usuario ON usuario.ID = so.user_id
                JOIN res_partner AS asesor ON asesor.ID = usuario.partner_id
                JOIN qr_code AS ewqc ON ewqc.ID = ai.qr_code_id
                JOIN product_pricelist pp ON pp.ID = so.pricelist_id
                JOIN res_currency rc ON rc.ID = pp.currency_id
            WHERE
                ai.TYPE = 'out_invoice'
                AND ai.STATE NOT IN ( 'draft', 'cancel' )
                AND ai.order_status = 'valida'
                AND ai.invoice_number != 0
                AND ai.type = 'out_invoice'
            """)
        else:
            consulta += ("""
                ai.id,
                ai.as_nit AS nit_cliente,
                ai.as_razon_social AS nombre_cliente,
                ewqc.numero_autorizacion AS numero_autorizacion,
                ai.control_code AS codigo_control,
            CASE
                WHEN rc.NAME = 'BOB'
                OR rc.symbol = 'Bs.' THEN
                ROUND( ai.amount_total, 4 ) ELSE ai.amount_total
                END AS importe_neto_venta,
                ROUND( ai.amount_total, 4 ) AS importe_neto_factura,
                ai.invoice_number AS numero_factura,
                to_char((( ai.fecha_boliviana AT TIME ZONE'UTC' AT TIME ZONE'BOT' ) :: TIMESTAMP ), 'DD/MM/YYYY' ) AS fecha_factura,
                asesor.NAME AS nombre_asesor,
                ROUND( ai.amount_total, 4 ) AS impote_bruto_fact,
                ROUND( ai.amount_total, 4 ) AS total_descuento_fact,
                ai.fecha_boliviana AS fecha_factura2,
                'N/A'
            FROM
                account_invoice ai
                JOIN res_partner AS cliente ON cliente.ID = ai.partner_id
                JOIN res_users AS usuario ON usuario.ID = ai.user_id
                JOIN res_partner AS asesor ON asesor.ID = usuario.partner_id
                JOIN qr_code AS ewqc ON ewqc.ID = ai.qr_code_id
                JOIN res_currency rc ON rc.ID = ai.currency_id
            WHERE
                ai.TYPE = 'out_invoice'
                AND ai.STATE NOT IN ( 'draft', 'cancel' )
                AND ai.order_status = 'valida'
                AND ai.invoice_number != 0
                AND ai.type = 'out_invoice'
            """)

        if data['form']['asesor']:
            consulta += " AND usuario.id = " + str(data['form']['asesor'])
        if data['form']['nombre_cliente']:
            consulta += " AND cliente.id = " + str(data['form']['nombre_cliente'])
        if data['form']['fecha_final']:
            consulta +=  " AND ai.fecha_boliviana BETWEEN '" + str(data['form']['fecha_inicial']) + "' AND '" + str(data['form']['fecha_final'])+ "'"

        #Definiciones generales del archivo, formatos, titulos, hojas de trabajo
        sheet = workbook.add_worksheet('Detalle de Movimientos')
        titulo1 = workbook.add_format({'font_size': 16,'font_name': 'Lucida Sans', 'align': 'center', 'bold':True,'bg_color': '#ffffff'})
        titulo2 = workbook.add_format({'font_size': 10, 'align': 'center', 'bold':True })
        titulo3 = workbook.add_format({'font_size': 9, 'align': 'left', 'bold':True })
        titulo3_number = workbook.add_format({'font_size': 10, 'align': 'right', 'bold':True, 'num_format': '#,##0.00' })
        titulo4 = workbook.add_format({'font_size': 11, 'font_name': 'Lucida Sans','align': 'center','bg_color': '#ffffff', 'bold':True })
        titulo10 = workbook.add_format({'font_size': 10, 'align': 'right','bg_color': '#ffffff', 'bold':True })
        titulo5 = workbook.add_format({'font_size': 10, 'align': 'center', 'right': False, 'bold':False })
        titulo9 = workbook.add_format({'font_size': 10, 'align': 'right', 'right': False, 'bold':False })
        titulo6 = workbook.add_format({'font_size': 10, 'align': 'center', 'right': False, 'bold':False, 'color': 'red'})
        titulo12 = workbook.add_format({'font_size': 10, 'align': 'right', 'right': False, 'bold':False, 'color': 'red'})
        titulo7 = workbook.add_format({'font_size': 10, 'align': 'left', 'right': False, 'bold':False})
        titulo8 = workbook.add_format({'font_size': 10, 'align': 'right', 'right': False, 'bold':False})

        number_left = workbook.add_format({'font_size': 9, 'align': 'left', 'num_format': '#,##0.00','bg_color': '#ffffff'})
        number_right = workbook.add_format({'font_size': 9, 'align': 'right', 'num_format': '#,##0.00','bg_color': '#ffffff'})
        number_right_bold = workbook.add_format({'font_size': 9, 'align': 'right', 'num_format': '#,##0.00', 'bold':True,'bg_color': '#ffffff'})
        number_right_col = workbook.add_format({'font_size': 9, 'align': 'right', 'num_format': '#,##0.00','bg_color': 'silver'})
        number_center = workbook.add_format({'font_size': 9, 'align': 'center', 'num_format': '#,##0.00','bg_color': '#ffffff'})
        number_right_col.set_locked(False)

        letter1 = workbook.add_format({'font_size': 9, 'align': 'left','bg_color': '#ffffff'})
        letter2 = workbook.add_format({'font_size': 9, 'align': 'left', 'bold':True,'bg_color': '#ffffff'})
        letter3 = workbook.add_format({'font_size': 9, 'align': 'right','font_size': 11,'font_name': 'Lucida Sans','bg_color': '#ffffff'})
        letter4 = workbook.add_format({'font_size': 9, 'align': 'left', 'bold': True,'bg_color': '#ffffff'})
        letter4C = workbook.add_format({'font_size': 9, 'align': 'left', 'bold': True,'color':'#000000','font_name': 'Lucida Sans','bg_color': '#ffffff' })
        letter4F = workbook.add_format({'font_size': 9, 'align': 'left', 'bold': True,'color': '#000000','bg_color': '#ffffff','font_name': 'Lucida Sans',})
        letter4G = workbook.add_format({'font_size': 9, 'align': 'left', 'bold': True,'color': '#000000','bg_color': '#f0f0f0','font_name': 'Lucida Sans',})
        letter4G2 = workbook.add_format({'font_size': 9, 'align': 'left', 'bold': True,'color': '#000000','bg_color': '#FFFAF0','font_name': 'Lucida Sans',})
        letter4S = workbook.add_format({'font_size': 9, 'align': 'left', 'bold': True,'bg_color': '#ffffff'})
        letter4S_right = workbook.add_format({'font_size': 9, 'align': 'right', 'bold': True,'bg_color': '#ffffff','num_format': '#,##0.00',})
        letter41S = workbook.add_format({'font_size': 9, 'align': 'left','bg_color': '#ffffff'})
        letter41Si = workbook.add_format({'font_size': 9, 'align': 'right','bg_color': '#ffffff'})
        letter41Sr = workbook.add_format({'font_size': 9, 'align': 'left','color': 'red','bg_color': '#ffffff'})
        letter_locked = letter3
        letter_locked.set_locked(False)

        # Aqui definimos en los anchos de columna
        sheet.set_column('A:A',15, letter1)
        sheet.set_column('B:B',15, letter1)
        sheet.set_column('C:C',15, letter1)
        sheet.set_column('D:D',18, letter1)
        sheet.set_column('E:E',15, letter1)
        sheet.set_column('F:F',15, letter1)
        sheet.set_column('G:G',15, letter3)
        sheet.set_column('H:H',15, letter3)
        sheet.set_column('I:I',15, letter3)
        sheet.set_column('J:J',15, number_center)
        sheet.set_column('K:K',18, letter1)
        sheet.set_column('L:L',18, letter1)
        sheet.set_column('M:M',18, letter1)

        # Titulos, subtitulos, filtros y campos del reporte
        sheet.merge_range('A1:M1', 'FACTURAS EMITIDAS', titulo1)
        sheet.set_row(0, 40)
        fecha = (datetime.now() - timedelta(hours=4)).strftime('%d/%m/%Y %H:%M:%S')
        fecha_inicial = datetime.strptime(data['form']['fecha_inicial'], '%Y-%m-%d').strftime('%d/%m/%Y')
        fecha_final = datetime.strptime(data['form']['fecha_final'], '%Y-%m-%d').strftime('%d/%m/%Y')
        sheet.write(1, 0, 'Rango de Fechas: ', letter4)
        sheet.merge_range('B2:D2', fecha_inicial +' - '+ fecha_final)
        sheet.merge_range('H3:I3', 'Fecha impresion: ', letter5)
        sheet.merge_range('J3:M3', fecha)
        sheet.merge_range('H4:I4', 'Usuario: ', letter5)
        sheet.merge_range('J4:M4', self.env.user.name)

        sheet.write(4, 0,'FECHA', letter4)
        sheet.write(4, 1,'NUMERACION INTERNA', letter4)
        sheet.write(4, 2,'NIT CLIENTE', letter4)
        sheet.write(4, 3,'RAZON SOCIAL', letter4)
        sheet.write(4, 4,'No AUTORIZACION', letter4)
        sheet.write(4, 5, 'CODIGO DE CONTROL', letter4)
        sheet.write(4, 6, 'IMPORTE BRUTO DE FACTURA AL CREDITO BS.', letter4)
        sheet.write(4, 7, 'DESCUENTO CREDITO BS.', letter4)
        sheet.write(4, 8, 'IMPORTE NETO DE FACTURA CREDITO BS.', letter4)
        sheet.write(4, 9, 'No FACTURA', letter4)
        sheet.write(4, 10, 'FECHA FACTURA', letter4)
        sheet.write(4, 11, 'COMERCIAL', letter4)
        sheet.write(4, 12, 'FORMA DE PAGO', letter4)
        sheet.freeze_panes(5, 0)
        sheet.set_row(4, 40)
        self.env.cr.execute(consulta)
        invoices = [j for j in self.env.cr.fetchall()]
        filas=5
        cols=0
        total_bruto=0
        total_descuento=0
        total_neto=0
        cols=0
        for invoice in invoices:
            sheet.write(filas, 0, invoice[0]) #fecha
            sheet.write(filas, 1, invoice[1]) #numeracion interna
            sheet.write(filas, 2, invoice[3]) #nit cliente
            sheet.write(filas, 3, invoice[4]) #razon social
            sheet.write(filas, 4, invoice[5]) #numero autorizacion
            sheet.write(filas, 5, invoice[6]) #codigo control
            sheet.write(filas, 6, round(self.amount_bruto(invoice[2],),6)) #importe bruto
            total_bruto+= self.amount_bruto(invoice[2])
            sheet.write(filas, 7, round(self.amount_discount(invoice[2],),6)) #descuento
            total_descuento += self.amount_discount(invoice[2])
            sheet.write(filas, 8, invoice[8],) #importe neto
            total_neto +=invoice[8]
            sheet.write(filas, 9, invoice[9]) #numero de factura
            sheet.write(filas, 10, invoice[10]) #fecha factura
            sheet.write(filas, 11, invoice[11]) #comercial
            sheet.write(filas, 12, invoice[15]) #forma de pago
            filas+=1
        coltotal= 'A'+str(filas+1)+':'+'F'+str(filas+1)
        sheet.merge_range(coltotal, 'TOTALES',letter5) #fecha
        sheet.write(filas, 6, total_bruto,letter5) #importe neto
        sheet.write(filas, 7, total_descuento,letter5) #importe neto
        sheet.write(filas, 8, total_neto,letter5) #importe neto

    def amount_bruto(self,invoice_id):
        invoices_line = self.env['account.invoice.line'].sudo().search([('invoice_id', '=', invoice_id)])
        monto=0.00
        for line in invoices_line:
            monto += (line.price_unit * line.quantity)
        return monto

    def amount_discount(self,invoice_id):
        invoices_line = self.env['account.invoice.line'].sudo().search([('invoice_id', '=', invoice_id)])
        monto=0.00
        monto_discount=0.00
        for line in invoices_line:
            if line.discount > 0.00:
                monto = (line.price_unit * line.quantity)
                monto_discount += (monto*line.discount)/100
        return monto_discount


from django.urls import reverse
from rest_framework import status

from api.tests.test_setup import TestSetUp
from api.models import List

class TestLists(TestSetUp):
    def test_create_list(self):
        self.user_data = {'title': 'my_list', 'owner': self.super_user.id}
        response = self.client.post(reverse('api:list-lists'), data=self.user_data)
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
        self.assertEqual(self.user_data['title'], str(List.objects.filter(owner=self.super_user).latest('title')))

    def test_get_all_lists(self):
        response = self.client.get(reverse('api:list-lists'))
        self.assertEqual((len(response.data)), self.list_count)
        self.assertEqual(response.status_code, status.HTTP_200_OK)

    def test_get_single_list(self):
        self.obj = List.objects.get(pk=self.list_pk)
        response = self.client.get(reverse('api:list-single-list', kwargs={'pk': self.obj.pk}))
        self.assertEqual(response.data['id'], self.obj.pk)
        self.assertEqual(response.status_code, status.HTTP_200_OK)

    def test_update_single_list(self):
        self.obj = List.objects.get(pk=self.list_pk)
        new_title = 'Updated Content'
        response = self.client.put(reverse('api:list-single-list', kwargs={'pk': self.obj.pk}), data={
            'title': new_title,
            'owner': self.super_user.id})
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(response.data['title'], new_title)

    def test_delete_single_list(self):
        self.obj = List.objects.get(pk=self.list_pk)
        response = self.client.delete(reverse('api:list-single-list', kwargs={'pk': self.obj.pk}))
        self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)

    def test_if_test_user_can_get_all_lists(self):
        self.client.force_login(self.test_user)
        lists = List.objects.all().count()
        response = self.client.get(reverse('api:list-lists'))
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertNotEqual(lists, len(response.data))


import os
import time
from itertools import chain
from scipy.stats import pearsonr
import numpy as np
import torch as pt
import torch.nn as nn
from torch import optim
from Layers.GGR import ESGGR

class MTAweGNN(nn.Module):

    def __init__(self, layer_list, eta_rng=None, kappa_rng=(2, 5), tau_rng=(0.5, 1.5), ker='lor', datasets=['LC50DM','LC50','IGC50','LD50'],
                 model_num='', opt='AMSGrad', betas=(0.9, 0.999)):

        # Allow methods from nn.Module
        super(MTAweGNN, self).__init__()

        # Initialize the geometric graph representation layer and the input normalization layer
        self.GGR = ESGGR(eta_rng=eta_rng, kappa_rng=kappa_rng, ker=ker, tau_rng=tau_rng)
        self.scaling = nn.BatchNorm1d(self.GGR.num_feat)

        # Add the hidden layers
        self.layers = nn.ModuleList(layer_list)

        # Find the number of features of the last hidden layer
        i = 1
        while True:
            print(i)
            try:
                num_neurons_last_layer = layer_list[-i].out_features
                break
            except:
                i += 1

        # Add the output layers
        self.out = nn.ModuleList([nn.Linear(num_neurons_last_layer, 1) for _ in datasets])

        # Initialize the chosen optimizer
        if opt == 'Adam':
            self.optimizer = optim.Adam(self.parameters(), 0, betas=betas)
        if opt == 'AMSGrad':
            self.optimizer = optim.Adam(self.parameters(), 0, betas=betas, amsgrad=True)
        if opt == 'RMSprop':
            self.optimizer = optim.RMSprop(self.parameters(), 0, momentum=betas[0], centered=False)
        if opt == 'SGD':
            self.optimizer = optim.SGD(self.parameters(), 0, momentum=betas[0], nesterov=False)
        if opt == 'Adagrad':
            self.optimizer = optim.Adagrad(self.parameters(), 0)

        # Record intial options for saving
        self.kappa_rng = kappa_rng
        self.eta_rng = eta_rng
        self.tau_rng = tau_rng
        self.ker = ker
        self.opt = opt
        self.betas = betas
        self.datasets = datasets
        self.model_num = model_num

        # Initialize modes
        self.save_mode = False
        self.stats_mode = False

        # Other attributes for saving
        self.save_sched = [0]
        self.save_directory = os.getcwd()
        self.model_stats = None

        # Keep a running tally of the amount of time it takes to complete an epoch
        self.time_list = []

    def forward(self, data_list):
        # Record a staggered number of samples in each data set for later
        n = [0]
        for i in range(len(data_list)):
            n.append(n[-1] + len(data_list[i][0]))

        # Combine the distances and sparse indices from each data set separately
        dist = list(chain(*[data[0] for data in data_list]))
        spar_ind = np.concatenate([data[1] for data in data_list], axis=0)

        # Combine into one piece of data for input
        X = (dist, spar_ind)

        # Run through the input layer
        X = self.GGR(X)
        X = self.scaling(X)

        # Run through the hidden layers
        for layer in self.layers:
            X = layer(X)

        # Split the samples up, then run through the separate outputs
        hidden_out = [X[n[i]:n[i+1], :] for i in range(len(data_list))]
        return tuple(self.out[i](hidden_out[i]) for i in range(len(data_list)))

    def test(self, x_list):
        self.eval()
        y_list = self(x_list)
        self.train()
        return y_list

    def display_stats(self, y_train_list, y_train_hat_list, y_test_list, y_test_hat_list, epoch):

        # Display the relevant statistics
        if len(self.time_list) != 0:
            print('Epoch %d completed in %.2f sec. Average completion time is %.2f sec'%(epoch, self.time_list[-1], sum(self.time_list)/(epoch)))
        for i, data_name in enumerate(self.datasets):
            print(data_name)
            print('-----')
            print('Train MAE:', pt.sum(pt.abs(y_train_hat_list[i] - y_train_list[i]) / y_train_list[i].shape[0]).item())
            print('Train RMSE:', pt.sqrt(pt.sum((y_train_hat_list[i] - y_train_list[i])**2) / y_train_list[i].shape[0]).item())
            print('Train pearson:', pearsonr(y_train_hat_list[i].detach().cpu().view(-1), y_train_list[i].cpu().detach().view(-1))[0])
            print('')
            print('Test MAE:', pt.sum(pt.abs(y_test_hat_list[i] - y_test_list[i]) / y_test_list[i].shape[0]).item())
            print('Test RMSE:', pt.sqrt(pt.sum((y_test_hat_list[i] - y_test_list[i])**2) / y_test_list[i].shape[0]).item())
            print('Test R2 score:', pearsonr(y_test_hat_list[i].cpu().detach().view(-1), y_test_list[i].cpu().detach().view(-1))[0]**2)
            print('----------------------------------')

    def save(self, y_train_hat_list, y_test_hat_list, epoch):

        # Get strings representing the hidden layers, betas, and learning schedule
        lay_str = '-'.join([str(w.shape[0]) for w in self.parameters() if len(w.shape) == 2])[:-2*len(self.datasets)]
        lr_sched_str = str(self.lr_sched[0])+'-'+str(self.lr_sched[1])
        betas_str = str(self.betas[0])+'-'+str(self.betas[1])

        # Figure out tau or eta initialization
        if self.tau_rng != None:
            param_str = f'_tau{self.tau_rng[0]}-{self.tau_rng[1]}'
        else:
            param_str = f'_eta{self.eta_rng[0]}-{self.eta_rng[1]}'

        # Write the base filename
        stats_filename = ('MT_'+'_'.join(self.datasets)+'_epochs'+str(self.total_epochs)+'_batch'+str(self.num_batches)+
                          '_lay'+lay_str+'_lr'+str(self.lr)+'_alpha'+str(self.alpha)+param_str+'_kappa'+str(self.kappa_rng[0])+
                          '-'+str(self.kappa_rng[1])+'_ker'+self.ker+'_opt'+self.opt+'_lrsch'+lr_sched_str+'_betas'+
                          betas_str+'_model'+self.model_num)

        # Initialize recording of the statistics for the first time
        if epoch == 0:
            self.model_stats = {'saved_etas':pt.zeros(self.GGR.num_grps, self.total_epochs+1),
                                'saved_kappas':pt.zeros(self.GGR.num_grps, self.total_epochs+1),
                                'model_state':{}}
            for i, data_name in enumerate(self.datasets):
                self.model_stats[f'{data_name}_train_predictions'] = pt.zeros(len(y_train_hat_list[i]), self.total_epochs+1)
                self.model_stats[f'{data_name}_test_predictions'] = pt.zeros(len(y_test_hat_list[i]), self.total_epochs+1)
        # Or change the existing statistics
        else:
            self.model_stats['saved_etas'][:, epoch] = self.GGR.eta.detach().cpu()
            self.model_stats['saved_kappas'][:, epoch] = self.GGR.kappa.detach().cpu()
            for i, data_name in enumerate(self.datasets):
                self.model_stats[f'{data_name}_train_predictions'][:, epoch] = y_train_hat_list[i].view(-1).detach().cpu()
                self.model_stats[f'{data_name}_test_predictions'][:, epoch] = y_test_hat_list[i].view(-1).detach().cpu()

        # Record statistics dictionary at special times (default: beginning and end)
        if epoch in self.save_sched:
            stats_dict = {'model_state_dict':self.state_dict(),
                          'optimizer_state_dict':self.optimizer.state_dict(),
                          'save_schedule':self.save_sched,
                          'scheduler':self.scheduler,
                          'time_list':self.time_list}

            self.model_stats['model_state'][epoch] = stats_dict

        # Save the model statitics dictionary into the specified save directory (default: current directory)
        pt.save(self.model_stats, f'{self.save_directory}/{stats_filename}')

    def fit(self, D_train_list, y_train_list, D_test_list, y_test_list, lr=0.001, total_epochs=2000, num_batches=20, alpha=0, start=1,
            lr_sched=None):

        # Set the variables for potential saving access
        self.lr = lr
        self.total_epochs = total_epochs
        self.num_batches = num_batches
        self.alpha = alpha
        self.lr_sched = lr_sched
        self.save_sched.append(total_epochs)

        # Set the loss function
        loss_function = nn.MSELoss()

        # Sets the parameter learning rates
        for param_group in self.optimizer.param_groups:
            param_group['lr'] = lr
            param_group['initial_lr'] = lr

        # Set the learning rate scheduler
        if lr_sched != None:
            self.scheduler = optim.lr_scheduler.StepLR(self.optimizer, lr_sched[0], lr_sched[1])
        else:
            self.scheduler = None

        # Gather the distance data, sparse indices, number of samples, and batch_sizes for each data set separately
        D_list = [pair[0] for pair in D_train_list]
        spar_ind_list = [pair[1] for pair in D_train_list]
        num_samp_list = [len(D) for D in D_list]
        batch_size_list = [num_samp//num_batches for num_samp in num_samp_list]

        # Preliminarily save or display stats
        if start == 1 and (self.save_mode == True or self.stats_mode == True):

            # Run through full train and test data
            y_train_hat_list = self.test(D_train_list)
            y_test_hat_list = self.test(D_test_list)

            # Save and/or display stats
            if self.save_mode == True:
                self.save(y_train_hat_list, y_test_hat_list, epoch=0)
            if self.stats_mode == True:
                self.display_stats(y_train_hat_list, y_train_list, y_test_hat_list, y_test_list, epoch=0)

        # Begin the fitting at the given starting epoch
        for epoch in range(start, total_epochs+1):

            # Start the timer for the epoch
            start_time = time.time()

            print('epoch', epoch)
            print('----------------------------------')

            # Mix the indices
            idx_list = [np.random.choice(num_samp, num_samp, replace=False) for num_samp in num_samp_list]

            for param_group in self.optimizer.param_groups: # sets the parameter learning rates
                print(param_group['lr'])

            print('epoch', epoch)
            print('----------------------------------')

            # Go through each mini-batch
            for batch_num in range(1, num_batches+1):

                print('batch', batch_num)

                # Get the batch indices for each data set
                batch_idx_list = []
                for i, batch_size in enumerate(batch_size_list):

                    start = (batch_num - 1) * batch_size  # Get the starting index
                    if batch_num == num_batches:  # Go to the end of the list for the last batch
                        end = num_samp_list[i]
                    else:
                        end = batch_num * batch_size
                    batch_idx_list.append(list(idx_list[i][start:end]))

                # Get the batches for each data set
                batch_list = []
                for i in range(len(D_list)):
                    batch_list.append(([D_list[i][ind] for ind in batch_idx_list[i]], spar_ind_list[i][batch_idx_list[i]]))

                # Run batches through to get all the outputs
                y_hats = self(batch_list)

                # Get the average of the losses
                loss = sum([loss_function(y_hats[i], y_train_list[i][batch_idx_list[i]]) for i in range(len(D_list))]) / len(D_list)

                # Regularization
                for w in self.parameters():
                    if len(w.shape) == 2:
                        loss += alpha*pt.sum(w**2)

                print('train loss:', loss.item())
                print('--------------')

                # Back propagation and reset gradient
                loss.backward()
                self.optimizer.step()
                self.optimizer.zero_grad()

            # Update the scheduler if there is one
            if lr_sched != None:
                self.scheduler.step()

            # Add time to list
            self.time_list.append(time.time()-start_time)

            # Preliminarily save or display stats
            if self.save_mode == True or self.stats_mode == True:

                # Run through full train and test data
                y_train_hat_list = self.test(D_train_list)
                y_test_hat_list = self.test(D_test_list)

                # Save and/or display stats
                if self.save_mode == True:
                    self.save(y_train_hat_list, y_test_hat_list, epoch=epoch)
                if self.stats_mode == True:
                    self.display_stats(y_train_hat_list, y_train_list, y_test_hat_list, y_test_list, epoch=epoch)


from te import tvm
from te.platform.cce_params import VEC
from te.platform.cce_params import AIC
from te.platform.cce_params import ASCEND_910
from te.platform.cce_params import ASCEND_910AIC
from te.platform.cce_params import HI3796CV300ESAIC
from te.platform.cce_params import HI3796CV300CSAIC
from te.platform.cce_params import ASCEND_610
from te.platform.cce_params import ASCEND_620
from te.platform.cce_conf import api_check_support
from .tik_expr import Expr
from ..api.tik_scalar import Scalar
from .. import debug
from .tik_util import type_convert
from ..api.tik_ir_builder import TikIRBuilder
from .tik_params import MAX_ATOMIC_ADD_MODE, ATOMIC_ADD_MODE_SHIFT_POS,\
    MAX_SMALL_CHANNEL_MODE, SMALL_CHANNEL_ENABLE_SHIFT_POS, FP2INT_SHIFT_POS, \
    MAX_SYSTEM_CACHE_MODE, SYSTEM_CACHE_MODE_SHIFT_POS, MAX_FP2INT_MODE, \
    MAX_TWO_BITS_VALUE, MAX_ONE_BIT_VALUE, ATOMIC_ADD_MASK, \
    SMALL_CHANNEL_MASK, SYSTEM_CACHE_MASK, FP2INT_MASK, ONE_IR, \
    INSTR_DTYPE_SUPPORT_STATEMENT
from .tik_check_util import TikCheckUtil
from .tik_source_info import source_info_decorator
from ..common.tik_get_soc_name import get_soc_name
from ..common.tik_get_soc_name import get_soc_core_type
_MAX_OVERFLOW_STATUS = 2

class TikSysControlApi(TikIRBuilder):
    """
    Proposal Operation Api
    """
    def __init__(self):
        super(TikSysControlApi, self).__init__()

    def _mov_ctrl_spr_to_scalar(self):
        """mov all bits of CTRL register to scalar

        Parameters
        ----------
        Returns
        -------
        ctrl : CTRL register
        """
        # disable it because calls subclass method
        ctrl = self.Scalar_(dtype="uint64")  # pylint: disable=E1101
        with self.new_scope():
            self.emit(
                tvm.call_extern(ctrl.dtype, "reg_set", ctrl.get(),
                                tvm.call_extern(ctrl.dtype, "get_ctrl")),
                ONE_IR)
        return ctrl

    @source_info_decorator()
    @debug.get_ctrl_bits(ATOMIC_ADD_MODE_SHIFT_POS,
                         ATOMIC_ADD_MODE_SHIFT_POS + 2)
    def mov_atomic_add_to_scalar(self, scalar):
        """mov the atomic_add bits of CTRL register to scalar

        Parameters
        ----------
        scalar : destination scalar operation

        Returns
        -------
        scalar : scalar with the atomic_add bits
        """
        TikCheckUtil.check_equality(
            get_soc_name(), ASCEND_910,
            "this api doesn't support version: %s" % get_soc_name())
        TikCheckUtil.check_type_match(
            scalar, Scalar, "input scalar should be Scalar")
        TikCheckUtil.check_equality(
            scalar.dtype, "uint64", "scalar must be uint64")
        ctrl = self._mov_ctrl_spr_to_scalar()
        ctrl.set_as(ctrl & (MAX_TWO_BITS_VALUE << ATOMIC_ADD_MODE_SHIFT_POS))
        ctrl.set_as(ctrl >> ATOMIC_ADD_MODE_SHIFT_POS)
        scalar.set_as(ctrl)
        return scalar

    @source_info_decorator()
    @debug.get_ctrl_bits(SMALL_CHANNEL_ENABLE_SHIFT_POS)
    def mov_small_channel_to_scalar(self, scalar):
        """mov the small_channel bits of CTRL register to scalar

        Parameters
        ----------
        scalar : destination scalar operation

        Returns
        -------
        scalar : scalar with the small_channel bits
        """
        TikCheckUtil.check_equality(
            get_soc_name(), ASCEND_910,
            "this api doesn't support version: %s" % get_soc_name())
        TikCheckUtil.check_type_match(
            scalar, Scalar, "input scalar should be Scalar")
        TikCheckUtil.check_equality(
            scalar.dtype, "uint64", "scalar must be uint64")
        ctrl = self._mov_ctrl_spr_to_scalar()
        ctrl.set_as(ctrl & (MAX_ONE_BIT_VALUE <<
                            SMALL_CHANNEL_ENABLE_SHIFT_POS))
        ctrl.set_as(ctrl >> SMALL_CHANNEL_ENABLE_SHIFT_POS)
        scalar.set_as(ctrl)
        return scalar

    @source_info_decorator()
    @debug.get_ctrl_bits(SYSTEM_CACHE_MODE_SHIFT_POS,
                         SYSTEM_CACHE_MODE_SHIFT_POS + 2)
    def mov_system_cache_to_scalar(self, scalar):
        """mov the system_cache bits of CTRL register to scalar

        Parameters
        ----------
        scalar : destination scalar operation

        Returns
        -------
        scalar : scalar with the system_cache bits
        """
        TikCheckUtil.check_equality(
            get_soc_name(), ASCEND_910,
            "this api doesn't support version: %s" % get_soc_name())
        TikCheckUtil.check_type_match(
            scalar, Scalar, "input scalar should be Scalar")
        TikCheckUtil.check_equality(
            scalar.dtype, "uint64", "scalar must be uint64")
        ctrl = self._mov_ctrl_spr_to_scalar()
        ctrl.set_as(ctrl & (MAX_TWO_BITS_VALUE <<
                            SYSTEM_CACHE_MODE_SHIFT_POS))
        ctrl.set_as(ctrl >> SYSTEM_CACHE_MODE_SHIFT_POS)
        scalar.set_as(ctrl)
        return scalar

    @source_info_decorator()
    @debug.get_ctrl_bits(FP2INT_SHIFT_POS)
    def mov_fp2int_mode_to_scalar(self, scalar):
        """mov the fp2int_mode bits of CTRL register to scalar

        Parameters
        ----------
        scalar : destination scalar operation

        Returns
        -------
        scalar : scalar with the fp2int_mode bits
        """
        TikCheckUtil.check_equality(
            get_soc_name(), ASCEND_910,
            "this api doesn't support version: %s" % get_soc_name())
        TikCheckUtil.check_type_match(
            scalar, Scalar, "input scalar should be Scalar")
        TikCheckUtil.check_equality(
            scalar.dtype, "uint64", "scalar must be uint64")
        ctrl = self._mov_ctrl_spr_to_scalar()
        ctrl.set_as(ctrl & (MAX_ONE_BIT_VALUE << FP2INT_SHIFT_POS))
        ctrl.set_as(ctrl >> FP2INT_SHIFT_POS)
        scalar.set_as(ctrl)
        return scalar

    @source_info_decorator()
    @debug.set_ctrl_bits(ATOMIC_ADD_MODE_SHIFT_POS,
                         ATOMIC_ADD_MODE_SHIFT_POS + 2)
    def set_atomic_add(self, mode):
        """set mode to the atomic_add bits of CTRL register

        Parameters
        ----------
        mode : atomic add mode, 0 - disable, 1 - float32, 2 - float16

        Returns
        -------
        None
        """
        arch_version_mode = {ASCEND_910: (0, 1),
                             ASCEND_610: (0, 1, 2, 3),
                             ASCEND_620: (0, 1, 2, 3)}
        TikCheckUtil.check_type_match(
            mode, (int, Scalar), "mode should be int or Scalar")
        TikCheckUtil.check_equality(api_check_support("tik." +
                                                      "set_atomic_add",
                                                      ""), True,
                                    INSTR_DTYPE_SUPPORT_STATEMENT.
                                    format("", "set_atomic_add"))
        if isinstance(mode, int):
            TikCheckUtil.check_in_range(
                mode, arch_version_mode[get_soc_name()],
                "mode should be in the range of %s, input mode: %s" % (
                    arch_version_mode[get_soc_name()], mode))
        if isinstance(mode, Scalar):
            TikCheckUtil.check_equality(
                mode.dtype, "uint64",
                "scalar_mode should be a scalar of uint64")
        ctrl = self._mov_ctrl_spr_to_scalar()
        ctrl.set_as(ctrl & ATOMIC_ADD_MASK)
        ctrl.set_as(ctrl | (mode << ATOMIC_ADD_MODE_SHIFT_POS))

        with self.new_scope():
            self.emit(tvm.call_extern("uint64", "set_ctrl", ctrl.get()),
                      ONE_IR)

    @source_info_decorator()
    @debug.set_ctrl_bits(SMALL_CHANNEL_ENABLE_SHIFT_POS)
    def set_small_channel(self, enable):
        """set enable mode to the small_channel bit of CTRL register

        Parameters
        ----------
        enable : small_channel mode, 0 - disable, 1 - enable

        Returns
        -------
        None
        """
        TikCheckUtil.check_type_match(
            enable, (int, Scalar), "enable should be int or Scalar")
        TikCheckUtil.check_equality(
            get_soc_name(), ASCEND_910,
            "this api doesn't support version: %s" % get_soc_name())
        if isinstance(enable, int):
            TikCheckUtil.check_in_range(enable, range(MAX_SMALL_CHANNEL_MODE),
                                        "enable should be 0 or 1")
        if isinstance(enable, Scalar):
            TikCheckUtil.check_equality(
                enable.dtype, "uint64",
                "scalar_enable should be a scalar of uint64")
        ctrl = self._mov_ctrl_spr_to_scalar()
        ctrl.set_as(ctrl & SMALL_CHANNEL_MASK)
        ctrl.set_as(ctrl | (enable << SMALL_CHANNEL_ENABLE_SHIFT_POS))
        with self.new_scope():
            self.emit(tvm.call_extern("uint64", "set_ctrl", ctrl.get()),
                      ONE_IR)

    @source_info_decorator()
    @debug.set_ctrl_bits(SYSTEM_CACHE_MODE_SHIFT_POS,
                         SYSTEM_CACHE_MODE_SHIFT_POS + 2)
    def set_system_cache(self, mode):
        """set mode to the system_cache bits of CTRL register

        Parameters
        ----------
        mode : system_cache mode, 0 - Normalread, 1 - ReadLast,
               2 - ReadInvalid, 3 - NotNeedWriteBack

        Returns
        -------
        None
        """
        TikCheckUtil.check_equality(
            get_soc_name(), ASCEND_910,
            "this api doesn't support version: %s" % get_soc_name())
        TikCheckUtil.check_type_match(
            mode, (int, Scalar), "mode should be int or Scalar")
        if isinstance(mode, int):
            TikCheckUtil.check_in_range(
                mode, range(MAX_SYSTEM_CACHE_MODE),
                "mode should be in the range of [0, 3]")
        if isinstance(mode, Scalar):
            TikCheckUtil.check_equality(
                mode.dtype, "uint64",
                "scalar_mode should be a scalar of uint64")
        ctrl = self._mov_ctrl_spr_to_scalar()
        ctrl.set_as(ctrl & SYSTEM_CACHE_MASK)
        ctrl.set_as(ctrl | (mode << SYSTEM_CACHE_MODE_SHIFT_POS))
        with self.new_scope():
            self.emit(tvm.call_extern("uint64", "set_ctrl", ctrl.get()),
                      ONE_IR)

    @source_info_decorator()
    @debug.set_ctrl_bits(FP2INT_SHIFT_POS)
    def set_fp2int_mode(self, mode):
        """set mode to the fp2int_mode bit of CTRL register

        Parameters
        ----------
        mode : fp2int_mode mode, 0 - Saturated, 1 - Truncted

        Returns
        -------
        None
        """
        TikCheckUtil.check_equality(
            get_soc_name(), ASCEND_910,
            "this api doesn't support version: %s" % get_soc_name())
        TikCheckUtil.check_type_match(
            mode, (int, Scalar), "mode should be int or Scalar")
        if isinstance(mode, int):
            TikCheckUtil.check_in_range(
                mode, range(MAX_FP2INT_MODE), "mode should be 0 or 1")
        if isinstance(mode, Scalar):
            TikCheckUtil.check_equality(
                mode.dtype, "uint64",
                "scalar_mode should be a scalar of uint64")
        ctrl = self._mov_ctrl_spr_to_scalar()
        ctrl.set_as(ctrl & FP2INT_MASK)
        ctrl.set_as(ctrl | (mode << FP2INT_SHIFT_POS))
        with self.new_scope():
            self.emit(tvm.call_extern("uint64", "set_ctrl", ctrl.get()),
                      ONE_IR)

    @source_info_decorator()
    def instr_preload(self, icache_base_addr, instr_num):
        """
            instr_num must be multiple of 128/4=32
        """
        # disable it because pylint can't recognize symbols from back-end
        arch_version_str = get_soc_name() + get_soc_core_type()
        TikCheckUtil.check_in_range(
            arch_version_str, [ASCEND_910AIC, AIC, VEC, HI3796CV300ESAIC,
                               HI3796CV300CSAIC],
            "%s doesn't support instr_preload." % (arch_version_str))
        TikCheckUtil.check_type_match(
            instr_num, (int, Scalar, Expr),
            "instr_num should be int, Scalar or Expr")
        from .tik_params import INSTR_UNIT, CACHE_LINE_SIZE
        if isinstance(instr_num, int):
            TikCheckUtil.check_equality(instr_num % INSTR_UNIT, 0,
                                        "instr_num must be multiple of 32.")
        with self.new_scope():
            with self.context.freeze():  # pylint: disable=E1101
                icache_addr = self.Scalar_("uint64")  # pylint: disable=E1101
                with self.for_range_(0, instr_num // INSTR_UNIT) as i:
                    icache_addr.set_as(icache_base_addr + i*CACHE_LINE_SIZE)
                    self.emit(
                        tvm.call_extern("uint64", "preload",
                                        icache_addr.get()), ONE_IR)

    @source_info_decorator()
    def get_overflow_status(self, status):
        """get overflow status stored in scalar buffer

        Parameters
        ----------
        status : destination scalar operation

        Returns
        -------
        None
        """
        TikCheckUtil.check_equality(
            get_soc_name(), ASCEND_910,
            "this api doesn't support version: %s" % get_soc_name())
        TikCheckUtil.check_type_match(
            status, Scalar, "variable should be Scalar, invalid type "
                            "for status: {}".format(type(status)))
        TikCheckUtil.check_equality(
            status.dtype, "uint64",
            "variable should be a Scalar of uint64, invalid "
            "dtype for status: {}".format(status.dtype))
        with self.new_scope():
            # overflow status is stored on scalar buffer, the address is 0x40000
            self.emit(
                tvm.call_extern("uint64_t", "reg_set",
                                status.get(),
                                tvm.call_pure_intrin(
                                    "uint64", "tvm_cce_string_print",
                                    "*(uint64_t *) 0x40000")),
                ONE_IR)

    @source_info_decorator()
    def set_overflow_status(self, value):
        """set overflow status to scalar buffer

        Parameters
        ----------
        value : overflow status

        Returns
        -------
        None
        """
        TikCheckUtil.check_equality(
            get_soc_name(), ASCEND_910,
            "this api doesn't support version: %s" % get_soc_name())
        TikCheckUtil.check_type_match(
            value, int, "value should be Int, "
                        "invalid type: {}".format(type(value)))
        TikCheckUtil.check_in_range(
            value, range(_MAX_OVERFLOW_STATUS),
            "value should be 0 or 1, invalid value: {}".format(value))
        with self.new_scope():
            self.emit(
                tvm.call_extern("uint64_t", "set_overflow",
                                type_convert(Expr(value, dtype="uint64"))),
                ONE_IR)


import requests
import pandas as pd
from flask import Flask
import financialanalysis as fa
import atexit
from apscheduler.schedulers.background import BackgroundScheduler
from sklearn.model_selection import train_test_split
import pickle

app =Flask(__name__)
def train():
    url="https://edonations.000webhostapp.com/api-last-day-donation.php"
    headers = {'Content-Type': 'application/json' }
    response = requests.get(url=url,headers=headers)
    df = pd.read_json(response.text)
    if df.empty:
        pass
    else:
        df.drop([u'name',u'donationId',u'name'],inplace=True,axis=1)
        X = df["date"].to_list() # convert Series to list
        X = fa.datetimeToFloatyear(X) # for example, 2020-07-01 becomes 2020.49589041
        # [2020.0054794520547, 2020.0082191780823, 2020.01643835616, ...]
        df["date"] = X
        # separate the independent and target variable
        train_X = df.drop(columns=['quantity'])
        train_Y = df['quantity']
        # randomly split the data
        train_x, test_x, train_y, test_y = train_test_split(train_X, train_Y,test_size=0.25,random_state=0)
        # create an object of the LinearRegression Model
        model1 = pickle.load(open('lassoModel.pkl', 'rb'))
        model2 = pickle.load(open('linearModel.pkl', 'rb'))
        model3 = pickle.load(open('ridgeModel.pkl', 'rb'))
        # fit the model with the training data
        model1.fit(train_x, train_y)
        model2.fit(train_x, train_y)
        model3.fit(train_x, train_y)
        #save traning
        pickle.dump(model1,open('lassoModel.pkl', 'rb'))
        pickle.dump(model2, open('linearModel.pkl', 'rb'))
        pickle.dump(model3,open('ridgeModel.pkl', 'rb'))

if __name__ == '__main__':
    scheduler = BackgroundScheduler({'apscheduler.timezone': 'UTC'}, daemon=True)
    scheduler.add_job(func=train, trigger='interval',hour = 24 )
    scheduler.start()
#Exit on app exit
    atexit.register(lambda: scheduler.shutdown())
    app.run(debug = False)


import json
import os

class ConfigException(Exception):
    pass

class Config:
    """Configuration class, load from filename, file or python dictionary. Environment variable aware."""

    def __init__(self, filename=None, file=None, config=None):
        if config is not None:
            self.config = config
        elif file is not None:
            self.config = json.load(file)
        elif filename is not None:
            with open(filename) as f:
                self.config = json.load(f)
        else:
            raise ConfigException("No configuration specified")

    def __getitem__(self, key):
        config = self.config
        for subkey in key.split('.'):
            config = config[subkey]
        if type(config) is str and config.startswith("ENV{") and config.endswith("}"):
            config = os.environ[config[len("ENV{"):-1]]
        return config


import torch.nn as nn
from torchtext import data
import copy
import layers as layers

#class Embedder(nn.Module):
#    def __init__(self, vocab_size, d_model):
#        super().__init__()
#        self.vocab_size = vocab_size
#        self.d_model = d_model
#
#        self.embed = nn.Embedding(vocab_size, d_model)
#
#    def forward(self, x):
#        return self.embed(x)

class EncoderLayer(nn.Module):
    def __init__(self, d_model, heads, dropout=0.1):
        """An layer of the encoder. Contain a self-attention accepting padding mask
        Args:
            d_model: the inner dimension size of the layer
            heads: number of heads used in the attention
            dropout: applied dropout value during training
            """
        super().__init__()
        self.norm_1 = layers.Norm(d_model)
        self.norm_2 = layers.Norm(d_model)
        self.attn = layers.MultiHeadAttention(heads, d_model, dropout=dropout)
        self.ff = layers.FeedForward(d_model, dropout=dropout)
        self.dropout_1 = nn.Dropout(dropout)
        self.dropout_2 = nn.Dropout(dropout)

    def forward(self, x, src_mask):
        """Run the encoding layer
        Args:
            x: the input (either embedding values or previous layer output), should be in shape [batch_size, src_len, d_model]
            src_mask: the padding mask, should be [batch_size, 1, src_len]
        Return:
            an output that have the same shape as input, [batch_size, src_len, d_model]
            the attention used [batch_size, src_len, src_len]
        """
        x2 = self.norm_1(x)
        # Self attention only
        x_sa, sa = self.attn(x2, x2, x2, src_mask)
        x = x + self.dropout_1(x_sa)
        x2 = self.norm_2(x)
        x = x + self.dropout_2(self.ff(x2))
        return x, sa

class DecoderLayer(nn.Module):
    def __init__(self, d_model, heads, dropout=0.1):
        """An layer of the decoder. Contain a self-attention that accept no-peeking mask and a normal attention tha t accept padding mask
        Args:
            d_model: the inner dimension size of the layer
            heads: number of heads used in the attention
            dropout: applied dropout value during training
            """
        super().__init__()
        self.norm_1 = layers.Norm(d_model)
        self.norm_2 = layers.Norm(d_model)
        self.norm_3 = layers.Norm(d_model)

        self.dropout_1 = nn.Dropout(dropout)
        self.dropout_2 = nn.Dropout(dropout)
        self.dropout_3 = nn.Dropout(dropout)

        self.attn_1 = layers.MultiHeadAttention(heads, d_model, dropout=dropout)
        self.attn_2 = layers.MultiHeadAttention(heads, d_model, dropout=dropout)
        self.ff = layers.FeedForward(d_model, dropout=dropout)

    def forward(self, x, memory, src_mask, trg_mask):
        """Run the decoding layer
        Args:
            x: the input (either embedding values or previous layer output), should be in shape [batch_size, tgt_len, d_model]
            memory: the outputs of the encoding section, used for normal attention. [batch_size, src_len, d_model]
            src_mask: the padding mask for the memory, [batch_size, 1, src_len]
            tgt_mask: the no-peeking mask for the decoder, [batch_size, tgt_len, tgt_len]
        Return:
            an output that have the same shape as input, [batch_size, tgt_len, d_model]
            the self-attention and normal attention received [batch_size, head, tgt_len, tgt_len] & [batch_size, head, tgt_len, src_len]
        """
        x2 = self.norm_1(x)
        # Self-attention
        x_sa, sa = self.attn_1(x2, x2, x2, trg_mask)
        x = x + self.dropout_1(x_sa)
        x2 = self.norm_2(x)
        # Normal multi-head attention
        x_na, na = self.attn_2(x2, memory, memory, src_mask)
        x = x + self.dropout_2(x_na)
        x2 = self.norm_3(x)
        x = x + self.dropout_3(self.ff(x2))
        return x, (sa, na)

def get_clones(module, N, keep_module=True):
    if(keep_module and N >= 1):
        # create N-1 copies in addition to the original
        return nn.ModuleList([module] + [copy.deepcopy(module) for i in range(N-1)])
    else:
        # create N new copy
        return nn.ModuleList([copy.deepcopy(module) for i in range(N)])

class Encoder(nn.Module):
    """A wrapper that embed, positional encode, and self-attention encode the inputs.
    Args:
        vocab_size: the size of the vocab. Used for embedding
        d_model: the inner dim of the module
        N: number of layers used
        heads: number of heads used in the attention
        dropout: applied dropout value during training
        max_seq_length: the maximum length value used for this encoder. Needed for PositionalEncoder, due to caching
    """
    def __init__(self, vocab_size, d_model, N, heads, dropout, max_seq_length=200):
        super().__init__()
        self.N = N
        self.embed = nn.Embedding(vocab_size, d_model)
        self.pe = layers.PositionalEncoder(d_model, dropout=dropout, max_seq_length=max_seq_length)
        self.layers = get_clones(EncoderLayer(d_model, heads, dropout), N)
        self.norm = layers.Norm(d_model)

        self._max_seq_length = max_seq_length

    def forward(self, src, src_mask, output_attention=False, seq_length_check=False):
        """Accepts a batch of indexed tokens, return the encoded values.
        Args:
            src: int Tensor of [batch_size, src_len]
            src_mask: the padding mask, [batch_size, 1, src_len]
            output_attention: if set, output a list containing used attention
            seq_length_check: if set, automatically trim the input if it goes past the expected sequence length.
        Returns:
            the encoded values [batch_size, src_len, d_model]
            if available, list of N (self-attention) calculated. They are in form of [batch_size, heads, src_len, src_len]
        """
        if(seq_length_check and src.shape[1] > self._max_seq_length):
            src = src[:, :self._max_seq_length]
            src_mask = src_mask[:, :, :self._max_seq_length]
        x = self.embed(src)
        x = self.pe(x)
        attentions = [None] * self.N
        for i in range(self.N):
            x, attn = self.layers[i](x, src_mask)
            attentions[i] = attn
        x = self.norm(x)
        return x if(not output_attention) else (x, attentions)

class Decoder(nn.Module):
    """A wrapper that receive the encoder outputs, run through the decoder process for a determined input
    Args:
        vocab_size: the size of the vocab. Used for embedding
        d_model: the inner dim of the module
        N: number of layers used
        heads: number of heads used in the attention
        dropout: applied dropout value during training
        max_seq_length: the maximum length value used for this encoder. Needed for PositionalEncoder, due to caching
    """
    def __init__(self, vocab_size, d_model, N, heads, dropout, max_seq_length=200):
        super().__init__()
        self.N = N
        self.embed = nn.Embedding(vocab_size, d_model)
        self.pe = layers.PositionalEncoder(d_model, dropout=dropout, max_seq_length=max_seq_length)
        self.layers = get_clones(DecoderLayer(d_model, heads, dropout), N)
        self.norm = layers.Norm(d_model)

        self._max_seq_length = max_seq_length

    def forward(self, trg, memory, src_mask, trg_mask, output_attention=False):
        """Accepts a batch of indexed tokens and the encoding outputs, return the decoded values.
        Args:
            trg: input Tensor of [batch_size, trg_len]
            memory: output of Encoder [batch_size, src_len, d_model]
            src_mask: the padding mask, [batch_size, 1, src_len]
            trg_mask: the no-peeking mask, [batch_size, tgt_len, tgt_len]
            output_attention: if set, output a list containing used attention
        Returns:
            the decoded values [batch_size, tgt_len, d_model]
            if available, list of N (self-attention, attention) calculated. They are in form of [batch_size, heads, tgt_len, tgt/src_len]
        """
        x = self.embed(trg)
        x = self.pe(x)

        attentions = [None] * self.N
        for i in range(self.N):
            x, attn = self.layers[i](x, memory, src_mask, trg_mask)
            attentions[i] = attn
        x = self.norm(x)
        return x if(not output_attention) else (x, attentions)

class Config:
    """Deprecated"""
    def __init__(self):
        self.opt = {
            'train_src_data':'/workspace/khoai23/opennmt/data/iwslt_en_vi/train.en',
            'train_trg_data':'/workspace/khoai23/opennmt/data/iwslt_en_vi/train.vi',
            'valid_src_data':'/workspace/khoai23/opennmt/data/iwslt_en_vi/tst2013.en',
            'valid_trg_data':'/workspace/khoai23/opennmt/data/iwslt_en_vi/tst2013.vi',
            'src_lang':'en', # useless atm
            'trg_lang':'en',#'vi_spacy_model', # useless atm
            'max_strlen':160,
            'batchsize':1500,
            'device':'cuda',
            'd_model': 512,
            'n_layers': 6,
            'heads': 8,
            'dropout': 0.1,
            'lr':0.0001,
            'epochs':30,
            'printevery': 200,
            'k':5,
        }


"""
This is a module used for doing grid searches using python
"""
import math
import sys
import numpy as np
from sklearn import preprocessing
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
import mlflow
import mlflow.sklearn
from pyspark.sql import SparkSession
from hyperopt import fmin, tpe, hp, SparkTrials, STATUS_OK
from sklearn.model_selection import cross_val_score

def main():
    """
    This is the entry point for the application
    :return:
    """
    spark = SparkSession \
        .builder \
        .appName("MLops_search_python") \
        .getOrCreate()
    uid = "testing123" if sys.argv[1] == "" else sys.argv[1]
    spark.conf.set("spark.sql.execution.arrow.enabled", True)
    print(f"reading delta table: dbfs:/datalake/stocks_{uid}/data")
    try:
        df = spark.read.format("delta").load(f"dbfs:/datalake/stocks_{uid}/data")
    except Exception as e:
        print(f"There was an error loading the delta stock table, : error:{e}")
    pdf = df.select("*").toPandas()
    df_2 = pdf.loc[:, ["AdjClose", "Volume"]]
    df_2["High_Low_Pert"] = (pdf["High"] - pdf["Low"]) / pdf["Close"] * 100.0
    df_2["Pert_change"] = (pdf["Close"] - pdf["Open"]) / pdf["Open"] * 100.0
    df_2.fillna(value=-99999, inplace=True)
    forecast_out = int(math.ceil(0.01 * len(df_2)))
    forecast_col = "AdjClose"
    df_2['label'] = df_2[forecast_col].shift(-forecast_out)
    X = np.array(df_2.drop(['label'], 1))
    X = preprocessing.scale(X)
    X = X[:-forecast_out]
    y = np.array(df_2['label'])
    y = y[:-forecast_out]
    X_train, _, y_train, _ = train_test_split(X, y, test_size=0.33, random_state=42)
    print("creating MLflow project")
    mlflow.set_experiment(f"/Users/bclipp770@yandex.com/datalake/stocks/experiments/cluster_{uid}")
    print("building our model")
    algo = tpe.suggest

    def objective(hypers):
        regr = RandomForestRegressor(max_depth=hypers["max_depth"],
                                     max_features=hypers["max_features"],
                                     min_samples_leaf=hypers["min_samples_leaf"],
                                     min_samples_split=hypers["min_samples_split"],
                                     n_estimators=hypers["n_estimators"]
                                     )
        accuracy = cross_val_score(regr, X_train, y_train).mean()
        return {'loss': -accuracy, 'status': STATUS_OK}

    search_space = {
        'max_depth': hp.choice('max_depth', range(1, 110)),
        'max_features': hp.choice('max_features', np.arange(0.1, 1.0, 0.1)),
        "min_samples_leaf": hp.choice('min_samples_leaf', range(3, 5)),
        "min_samples_split": hp.choice("min_samples_split", range(8, 12)),
        'n_estimators': hp.choice('n_estimators', range(100, 500))}

    spark_trials = SparkTrials()

    with mlflow.start_run():
        argmin = fmin(
            fn=objective,
            space=search_space,
            algo=algo,
            max_evals=16,
            trials=spark_trials)

    print("Best value found: ", argmin)

if __name__ == "__main__":
    main()


import torch
from omegaconf import DictConfig
from torch import nn

from modules.graph_encoder import GraphEncoder, GraphEncoderSequential
from utils.torch.models.mlp import MLP

class GlobalCritic(nn.Module):
    def __init__(self,
                 graph_size: int,
                 n_actions: int,
                 n_agents: int,
                 input_dim: int,
                 node_dim: int,
                 config: DictConfig):
        super().__init__()
        self.n_actions = n_actions
        self.n_agents = n_agents

        self.embed_dim = config.embed_dim
        self.node_dim = node_dim  # number of features per node
        self.other_dim = input_dim - graph_size
        self.graph_size = graph_size  # number of nodes * node_dim
        self.number_of_nodes = int(self.graph_size / self.node_dim)
        graph_encoder_layers = [GraphEncoder(self.embed_dim, config) for _ in range(config.number_graph_encoder_layers)]
        self.graph_encoder = GraphEncoderSequential(graph_encoder_layers, self.node_dim, self.embed_dim)

        self.q_dim = self.embed_dim + self.other_dim  # graph dim + other input
        self.q_net = MLP(input_size=self.q_dim, output_size=self.n_actions, **config.q_layer)

    def forward(self, x):
        # split in graph specific part and agent specific part
        batch_shape = x.shape[:-1]
        agent_specific = x[:, :, :, self.graph_size:]
        nodes = x[:, :, :, :self.graph_size].reshape(-1, self.number_of_nodes, self.node_dim)
        _, graph_embedding = self.graph_encoder.forward(nodes)

        agent_specific = agent_specific.reshape(-1, self.other_dim)
        output = self.q_net(torch.cat([graph_embedding, agent_specific], dim=-1))
        output = output.view(*batch_shape, -1)

        return output


"""Plotting the 9 KNN accuracy plots"""
from utils.plots import load_data_from_csv_wrapper, add_subplot_axes
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import numpy as np
import os
import json

plt.rcParams['interactive'] = False
fig = plt.figure(figsize=(10.0, 8.0))
subpos = [0.55, 0.3, 0.5, 0.4]

# wrn, cifar10
root_dir = '/data/gilad/logs/metrics/wrn/cifar10/log_1405_230818_metrics_w_confidence-SUPERSEED=23081800'
json_file = os.path.join(root_dir, 'data_for_figures', 'data.json')
with open(json_file) as f:
    data = json.load(f)
ax2 = fig.add_subplot(321)
ax2.plot(data['test']['regular']['dnn_score']['steps'], [v * 100 for v in data['test']['regular']['dnn_score']['values']], 'k')
ax2.plot(data['test']['regular']['knn_score']['steps'], [v * 100 for v in data['test']['regular']['knn_score']['values']], 'r')
ax2.plot(data['test']['regular']['svm_score']['steps'], [v * 100 for v in data['test']['regular']['svm_score']['values']], 'b')
ax2.plot(data['test']['regular']['lr_score']['steps'] , [v * 100 for v in data['test']['regular']['lr_score']['values']] , 'g')

ax2.set_ylim(bottom=0, top=110)
ax2.set_title('CIFAR-10', fontdict={'fontsize': 15})
# ax2.text(-15000, 52, 'Wide ResNet 28-10', va='center', rotation='vertical', fontdict={'fontsize': 15})
ax2.yaxis.grid()
ax2.set_ylabel('accuracy (%)', labelpad=0.5, fontdict={'fontsize': 15})
ax2.get_xaxis().set_visible(False)
ax2.tick_params(labelsize=13)

# ax2.add_patch(patches.Polygon(xy=np.array([[40000, 99], [27513.4, 65.6], [50000, 65.6], [50000, 32.2], [50000, 90]]), closed=True, color='silver'))
# ax2.add_patch(patches.Rectangle(xy=(40000, 90), width=10000, height=10, facecolor='moccasin'))

# wrn, cifar100
root_dir = '/data/gilad/logs/metrics/wrn/cifar100/log_1405_230818_metrics_w_confidence-SUPERSEED=23081800'
json_file = os.path.join(root_dir, 'data_for_figures', 'data.json')
with open(json_file) as f:
    data = json.load(f)
ax3 = fig.add_subplot(322)
ax3.plot(data['test']['regular']['dnn_score']['steps'], [v * 100 for v in data['test']['regular']['dnn_score']['values']], 'k')
ax3.plot(data['test']['regular']['knn_score']['steps'], [v * 100 for v in data['test']['regular']['knn_score']['values']], 'r')
ax3.plot(data['test']['regular']['svm_score']['steps'], [v * 100 for v in data['test']['regular']['svm_score']['values']], 'b')
ax3.plot(data['test']['regular']['lr_score']['steps'] , [v * 100 for v in data['test']['regular']['lr_score']['values']] , 'g')

ax3.set_ylim(bottom=0, top=85)
ax3.set_title('CIFAR-100', fontdict={'fontsize': 15})
ax3.yaxis.grid()
ax3.get_xaxis().set_visible(False)
ax3.tick_params(labelsize=13)
# ax3.add_patch(patches.Polygon(xy=np.array([[40000, 83.1], [27700, 50.75], [50000, 50.75], [50000, 25], [50000, 75]]), closed=True, color='silver'))
# ax3.add_patch(patches.Rectangle(xy=(40000, 75), width=10000, height=8, facecolor='moccasin'))

# lenet, cifar10
root_dir = '/data/gilad/logs/metrics/lenet/cifar10/log_1319_120918_metrics-SUPERSEED=12091800'
json_file = os.path.join(root_dir, 'data_for_figures', 'data.json')
with open(json_file) as f:
    data = json.load(f)
ax5 = fig.add_subplot(323)
ax5.plot(data['test']['regular']['dnn_score']['steps'], [v * 100 for v in data['test']['regular']['dnn_score']['values']], 'k')
ax5.plot(data['test']['regular']['knn_score']['steps'], [v * 100 for v in data['test']['regular']['knn_score']['values']], 'r')
ax5.plot(data['test']['regular']['svm_score']['steps'], [v * 100 for v in data['test']['regular']['svm_score']['values']], 'b')
ax5.plot(data['test']['regular']['lr_score']['steps'] , [v * 100 for v in data['test']['regular']['lr_score']['values']] , 'g')

ax5.set_ylim(bottom=0, top=110)
# ax5.text(-12000, 52, 'LeNet', va='center', rotation='vertical', fontdict={'fontsize': 13})
ax5.yaxis.grid()
ax5.set_ylabel('accuracy (%)', labelpad=0.5, fontdict={'fontsize': 15})
ax5.get_xaxis().set_visible(False)
ax5.tick_params(labelsize=13)
# ax5.add_patch(patches.Polygon(xy=np.array([[40000, 87.8], [27800, 66.5], [50000, 66.5], [50000, 33.1], [50000, 78.7]]), closed=True, color='silver'))
# ax5.add_patch(patches.Rectangle(xy=(40000, 78), width=10000, height=10, facecolor='moccasin'))

# lenet, cifar100
root_dir = '/data/gilad/logs/metrics/lenet/cifar100/log_1319_120918_metrics-SUPERSEED=12091800'
json_file = os.path.join(root_dir, 'data_for_figures', 'data.json')
with open(json_file) as f:
    data = json.load(f)
ax6 = fig.add_subplot(324)
ax6.plot(data['test']['regular']['dnn_score']['steps'], [v * 100 for v in data['test']['regular']['dnn_score']['values']], 'k')
ax6.plot(data['test']['regular']['knn_score']['steps'], [v * 100 for v in data['test']['regular']['knn_score']['values']], 'r')
ax6.plot(data['test']['regular']['svm_score']['steps'], [v * 100 for v in data['test']['regular']['svm_score']['values']], 'b')
ax6.plot(data['test']['regular']['lr_score']['steps'] , [v * 100 for v in data['test']['regular']['lr_score']['values']] , 'g')

ax6.set_ylim(bottom=0, top=60)
ax6.yaxis.grid()
ax6.get_xaxis().set_visible(False)
ax6.tick_params(labelsize=13)
# ax6.add_patch(patches.Polygon(xy=np.array([[40000, 56.2], [27800, 35.6], [50000, 35.6], [50000, 18], [50000, 50]]), closed=True, color='silver'))
# ax6.add_patch(patches.Rectangle(xy=(40000, 50), width=10000, height=6, facecolor='moccasin'))

# fc2net, cifar10
root_dir = '/data/gilad/logs/metrics/fc2net/cifar10/log_1025_150918_metrics-SUPERSEED=15091800'
json_file = os.path.join(root_dir, 'data_for_figures', 'data.json')
with open(json_file) as f:
    data = json.load(f)
ax8 = fig.add_subplot(325)
ax8.plot(data['test']['regular']['dnn_score']['steps'], [v * 100 for v in data['test']['regular']['dnn_score']['values']], 'k')
ax8.plot(data['test']['regular']['knn_score']['steps'], [v * 100 for v in data['test']['regular']['knn_score']['values']], 'r')
ax8.plot(data['test']['regular']['svm_score']['steps'], [v * 100 for v in data['test']['regular']['svm_score']['values']], 'b')
ax8.plot(data['test']['regular']['lr_score']['steps'] , [v * 100 for v in data['test']['regular']['lr_score']['values']] , 'g')

ax8.set_ylim(bottom=0, top=65)
# ax8.text(-12000, 30, 'MLP-640', va='center', rotation='vertical', fontdict={'fontsize': 13})
ax8.set_ylabel('accuracy (%)', labelpad=6, fontdict={'fontsize': 15})
ax8.yaxis.grid()
ax8.tick_params(labelsize=13)

# ax8.add_patch(patches.Polygon(xy=np.array([[40000, 87.8], [27800, 66.5], [50000, 66.5], [50000, 33.1], [50000, 78.7]]), closed=True, color='silver'))
# ax8.add_patch(patches.Rectangle(xy=(40000, 78), width=10000, height=10, facecolor='moccasin'))

# fc2net, cifar100
root_dir = '/data/gilad/logs/metrics/fc2net/cifar100/log_1025_150918_metrics-SUPERSEED=15091800'
json_file = os.path.join(root_dir, 'data_for_figures', 'data.json')
with open(json_file) as f:
    data = json.load(f)
ax9 = fig.add_subplot(326)
ax9.plot(data['test']['regular']['dnn_score']['steps'], [v * 100 for v in data['test']['regular']['dnn_score']['values']], 'k')
ax9.plot(data['test']['regular']['knn_score']['steps'], [v * 100 for v in data['test']['regular']['knn_score']['values']], 'r')
ax9.plot(data['test']['regular']['svm_score']['steps'], [v * 100 for v in data['test']['regular']['svm_score']['values']], 'b')
ax9.plot(data['test']['regular']['lr_score']['steps'] , [v * 100 for v in data['test']['regular']['lr_score']['values']] , 'g')

ax9.set_ylim(bottom=0, top=30)
ax9.yaxis.grid()
ax9.tick_params(labelsize=13)
# ax9.add_patch(patches.Polygon(xy=np.array([[40000, 25], [27800, 18], [50000, 18], [50000, 9.1], [50000, 22.2]]), closed=True, color='silver'))
# ax9.add_patch(patches.Rectangle(xy=(40000, 22), width=10000, height=3, facecolor='moccasin'))

ax2.legend(['DNN', '$k$-NN', 'SVM', 'LR'], loc=(0.65, 0.1), fontsize=15)
ax3.legend(['DNN', '$k$-NN', 'SVM', 'LR'], loc=(0.65, 0.1), fontsize=15)
ax5.legend(['DNN', '$k$-NN', 'SVM', 'LR'], loc=(0.65, 0.1), fontsize=15)
ax6.legend(['DNN', '$k$-NN', 'SVM', 'LR'], loc=(0.65, 0.1), fontsize=15)
ax8.legend(['DNN', '$k$-NN', 'SVM', 'LR'], loc=(0.65, 0.1), fontsize=15)
ax9.legend(['DNN', '$k$-NN', 'SVM', 'LR'], loc=(0.65, 0.1), fontsize=15)

# fix subax position
# pos11 = subax1.get_position() # [[0.250367647059, 0.721470588235], [0.341544117647, 0.789411764706]]
# pos12 = [pos11.x0-0.052, pos11.y0+0.05, pos11.width, pos11.height]
# subax1.set_position(pos12)
# pos41 = subax4.get_position() # [[0.250367647059, 0.449705882353], [0.364338235294, 0.540294117647]]
# pos42 = [pos41.x0-0.052, pos41.y0, pos41.width, pos41.height]
# subax4.set_position(pos42)
plt.tight_layout()
plt.savefig('test_acc_vs_iter.png', dpi=350)


import json

from telegram.ext import Updater, CommandHandler

def start(update, context):
    """Команда подключения к боту."""
    user_list = {}
    with open("users.json", "r") as users:
        try:
            user_list = json.load(users)
        except:
            pass
    user_list[update.message.chat.username] = update.message.chat.to_dict()
    with open("users.json", "w") as users:
        users.write(json.dumps(user_list))
    context.bot.send_message(
        chat_id = update.effective_chat.id,
        text = "Привет!"
    )

def help(update, context):
    """Стандартная команда подсказки по использованию бота."""
    context.bot.send_message(
        chat_id = update.effective_chat.id,
        text = "Это учебный бот, он собирает данные о пользователях!"
    )

def stop(update, context):
    """Команда отключения от бота."""
    user_list = {}
    with open("users.json", "r") as users:
        user_list = json.load(users)
    user_list.pop(update.message.chat.username, None)
    with open("users.json", "w") as users:
        users.write(json.dumps(user_list))
    context.bot.send_message(
        chat_id = update.effective_chat.id,
        text = f'Прощай, {update.message.chat.first_name}! :('
    )

def user_count(update, context):
    """Команда выводящая количество подключенных пользователей."""
    user_list = {}
    with open("users.json", "r") as users:
        user_list = json.load(users)
    context.bot.send_message(
        chat_id = update.effective_chat.id,
        text = f'Пользователей подключено: {len(user_list)}'
    )

def describe_me(update, context):
    """Команда выводящая информацию о пользователе."""
    user_list = {}
    with open("users.json", "r") as users:
        user_list = json.load(users)
    me = user_list.get(update.message.chat.username)
    context.bot.send_message(
        chat_id = update.effective_chat.id,
        text = f'Ваш айди - {me.get("id")}, ваш ник - {me.get("username")}, ваше имя = {me.get("first_name")}!'
    )

def main():
    updater = Updater("") # сюда нужно вставить токен бота
    updater.dispatcher.add_handler(CommandHandler('start', start))
    updater.dispatcher.add_handler(CommandHandler('help', help))
    updater.dispatcher.add_handler(CommandHandler('stop', stop))
    updater.dispatcher.add_handler(CommandHandler('count', user_count))
    updater.dispatcher.add_handler(CommandHandler('describe', describe_me))
    updater.start_polling(poll_interval=10.0)

if __name__ == "__main__":
    main()


from typing import Union
from mltk.core.tflite_model.tflite_schema import BuiltinOperator
from .layer import Layer, flat_size, KerasLayer, TfliteLayer

class Quantize(Layer):

    def __init__(self):
        Layer.__init__(self, ('Quantize', BuiltinOperator.QUANTIZE))

    def process(self, layer:Union[TfliteLayer,KerasLayer]):
        self.name = layer.name
        self.macs = 0
        # https://github.com/tensorflow/tflite-micro/blob/main/tensorflow/lite/kernels/internal/reference/quantize.h#L31
        # 1 divide
        # 1 add
        # 1 min
        # 1 max
        if isinstance(layer, KerasLayer):
            input_shape = layer.input_shape
        else:
            input_shape = layer.inputs[0].shape

        self.ops = flat_size(input_shape[1:]) * 4

class Dequantize(Layer):

    def __init__(self):
        Layer.__init__(self, ('Dequantize', BuiltinOperator.DEQUANTIZE))

    def process(self, layer:Union[TfliteLayer,KerasLayer]):
        self.name = layer.name
        self.macs = 0
        # https://github.com/tensorflow/tflite-micro/blob/main/tensorflow/lite/kernels/internal/reference/dequantize.h
        # 1 multiply
        # 1 substract

        if isinstance(layer, KerasLayer):
            input_shape = layer.input_shape
        else:
            input_shape = layer.inputs[0].shape

        self.ops = flat_size(input_shape[1:]) * 2

Quantize()
Dequantize()


import unittest
from collections import deque

class Solution:
    def numBusesToDestination(self, routes, S, T):
        """
        :type routes: List[List[int]]
        :type S: int
        :type T: int
        :rtype: int
        """
        if S == T:
            return 0

        if not routes or not routes[0]:
            return -1

        table = self.stopBoard(routes)
        if S not in table or T not in table:
            return -1

        ret = 0
        busTaken = set()
        queue = deque()
        queue.append(S)
        while queue:
            ret += 1

            size = len(queue)
            for _ in range(size):
                currStop = queue.popleft()
                buses = table[currStop]
                for bus in buses:
                    if bus in busTaken:
                        continue
                    busTaken.add(bus)
                    for stop in routes[bus]:
                        if stop == T:
                            return ret
                        queue.append(stop)
            ret += 1
        return -1

    def stopBoard(self, routes):
        table = dict()  # key is bus stop, value is a list of buses that go though the list
        numOfBuses = len(routes)
        for busIdx in range(numOfBuses):
            for stop in routes[busIdx]:
                if stop not in table:
                    table[stop] = list()
                table[stop].append(busIdx)
        return table

class TestSolution(unittest.TestCase):
    sol = Solution()

    def testNumBusesToDestination(self):
        routes = [[1, 2, 7], [3, 6, 7]]
        S = 1
        T = 6
        self.assertEqual(2, self.sol.numBusesToDestination(routes, S, T))

if __name__ == '__main__':
    unittest.main()


import random
from django.core.management.base import BaseCommand
from main.models import Player

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument('count', type=int, help='Total number of Players to create')
        parser.add_argument('-p', '--prefix', type=str, help='Prefix for Players to create')
        parser.add_argument('-l', '--local', action='store_true', help='Create local Players')
        parser.add_argument('-f', '--foreign', action='store_true', help='Create foreign Players')

    def handle(self, *args, **options):
        count = options.get('count')
        prefix = options.get('prefix')
        local = options.get('local')
        foreign = options.get('foreign')

        if prefix is None:
            prefix = 'Player'

        for i in range(count):
            if local:
                Player.objects.create(first_name=f'{prefix} {i}', age=random.randint(18, 40), is_local=True)
            elif foreign:
                Player.objects.create(first_name=f'{prefix} {i}', age=random.randint(18, 40), is_foreign=True)
            else:
                Player.objects.create(first_name=f'{prefix} {i}', age=random.randint(18, 40))
            self.stdout.write(f'{prefix} {i} created')


# xray.py

import pandas as pd

from ..fields.xray_emissivity import get_xray_emissivity
from ..load_sim import LoadSim

class Xray:

    @LoadSim.Decorators.check_pickle
    def read_xray_all(self, nums=None, prefix='xray_all',
                      savdir=None, force_override=False):
        rr = dict()
        if nums is None:
            nums = self.nums

        print('num:', end=' ')
        for i,num in enumerate(nums):
            print(num, end=' ')
            r = self.read_xray(num=num, savdir=savdir, force_override=False)
            if i == 0:
                for k in r.keys():
                    rr[k] = []

            for k in r.keys():
                try:
                    rr[k].append(r[k].value.item())
                except:
                    rr[k].append(r[k])

        rr = pd.DataFrame(rr)
        return rr

    @LoadSim.Decorators.check_pickle
    def read_xray(self, num, Z_gas=1.0, emin_keV=0.5, emax_keV=7.0, prefix='L_X',
                  savdir=None, force_override=False):
        """
        Function to calculate x-ray luminosity of the snapshot
        """

        ds = self.load_vtk(num)
        dV = ds.domain['dx'].prod()*(self.u.length.cgs.value)**3
        d = ds.get_field(['density','temperature'])
        em = get_xray_emissivity(d['temperature'].data, Z_gas,
                                 emin_keV, emax_keV, energy=True)
        d['j_X'] = d['density']**2*em
        #dd['I_X'] = d['j_X'].sum(dim='z')*d.domain['dx'][2]*self.u.length.cgs.value

        res = dict()
        res['time'] = ds.domain['time']
        res['L_X'] = float(d['j_X'].sum()*dV)

        return res


from flask import Flask, Response, request, jsonify, abort
import json
import requests

URL = "http://web2:5000/"

application = Flask(__name__)

books = [
    {"id" : 1,
    "author" : "J. K. Rowling",
    "title" : "Harry Potter and the Philosopher's stone",
    "year" : 1997,
    "isbn" : 9780590353403,
    "part" : 3
    },
    {"id" : 2,
    "author" : "J. K. Rowling",
    "title" : "Harry Potter and the Chamber of secrets",
    "year" : 1998,
    "isbn" : 9788498387650,
    "part" : 2
    },
    {"id" : 3,
    "author" : "Charles Dickens",
    "title" : "A tale of Two Cities",
    "year" : 1859,
    "isbn" : 9780721407104,
    "part" : 2
    },
    {"id" : 4,
    "author" : "Antoine de Saint-Exupery",
    "title" : "The Little Prince",
    "year" : 1943,
    "isbn" : 9788854172388,
    "part" : 1
    }]

@application.route("/")
def homePage():
    return "
Welcome to virtual library!
To see all books"

@application.route("/api/books", methods = ["GET"])
def getBookList():

    booksToReturn = []

    args = request.args
    expand = args.get('expand')

    if (expand == "part"):
        for book in books:
            booksWithPart = book.copy()
            try:
                get = requests.get(URL + "api/parts/" + str(book["part"]))
                getPart = {
                    "manufacturer" : get.json()[0]["manufacturer"],
                    "name" : get.json()[0]["name"],
                    "type" : get.json()[0]["type"],
                    "price" : get.json()[0]["price"]
                }
                booksWithPart["part"] = getPart

            except requests.exceptions.RequestException as ex:
                print(ex)
            booksToReturn.append(booksWithPart)
        return Response(json.dumps(booksToReturn),status=200,mimetype="application/json")

    else:
        return Response(json.dumps(books),status=200,mimetype="application/json")

@application.route("/api/books", methods = ["POST"])
def postBookList():
    newRecord = request.get_json("force: True")

    if ("expand" in request.args):
        if ("author" in newRecord) and ("title" in newRecord) and ("year" in newRecord) and ("isbn" in newRecord) and ("part" in newRecord):
            if type(newRecord["part"]) == type(0):
                return Response(json.dumps({"Failure" : "Request expects json type part"}),status=400,mimetype="application/json")
            if ("manufacturer" in newRecord["part"]) and ("name" in newRecord["part"]) and ("price" in newRecord["part"]) and ("type" in newRecord["part"]):
                postPart = {
                    "manufacturer" : newRecord["part"]["manufacturer"],
                    "name" : newRecord["part"]["name"],
                    "price" : newRecord["part"]["price"],
                    "type" : newRecord["part"]["type"]
                }
                try:
                    post = requests.post(URL + "api/parts", json = postPart)
                    getPart = requests.get(URL + "/api/parts")
                    returnPart = {
                        "id" : getPart.json()[len(getPart.json())-1]["id"],
                        "manufacturer" : getPart.json()[len(getPart.json())-1]["manufacturer"],
                        "name" : getPart.json()[len(getPart.json())-1]["name"],
                        "price" : getPart.json()[len(getPart.json())-1]["price"],
                        "type" : getPart.json()[len(getPart.json())-1]["type"]
                    }
                except requests.exceptions.RequestException as ex:
                    return Response(json.dumps({"Failure" : "Can not connect to the server"}),status="503",mimetype="application/json")

                if(str(post.status_code) == "400" or str(post.status_code) == "404"):
                    return Response(json.dumps({"Failure" : "error code: " +str(post.status_code)}),status=post.status_code,mimetype="application/json")

                else:
                    newBook = {
                        "id" : books[len(books)-1]["id"] + 1,
                        "author" : newRecord["author"],
                        "title" : newRecord["title"],
                        "year" : newRecord["year"],
                        "isbn" : newRecord["isbn"],
                        "part" : getPart.json()[len(getPart.json())-1]["id"]
                    }

                    returnBook = newBook.copy()
                    returnBook["part"] = returnPart

                    books.append(newBook)
                    bookID = returnBook["id"]
                    return Response((json.dumps({"Success":"Book was added; "})+json.dumps(returnBook)), status=201, headers={"location": "/api/extendedBookList/"+str(bookID)},mimetype="application/json")

            else:
                error = "This data has not been given: "
                if "manufacturer" not in newRecord["part"]:
                    error += "manufacturer; "
                if "name" not in newRecord["part"]:
                    error += "name; "
                if "price" not in newRecord["part"]:
                    error += "price; "
                if "type" not in newRecord["part"]:
                    error += "type."
                return Response(json.dumps({"Failure" : error}),status=400,mimetype="application/json")

        else:
            error = "This data has not been given: "
            if "author" not in newRecord:
                error += "author; "
            if "title" not in newRecord:
                error += "title; "
            if "year" not in newRecord:
                error += "year; "
            if "isbn" not in newRecord:
                error += "isbn;"
            if "part" not in newRecord:
                error += "part."
            return Response(json.dumps({"Failure" : error}),status=400,mimetype="application/json")

    else:
        if ("author" in newRecord) and ("title" in newRecord) and ("year" in newRecord) and ("isbn" in newRecord) and ("part" in newRecord):
            if type(newRecord["part"]) == type(0):
                newBook = {
                    "id" : books[len(books)-1]["id"] + 1,
                    "author" : newRecord["author"],
                    "title" : newRecord["title"],
                    "year" : newRecord["year"],
                    "isbn" : newRecord["isbn"],
                    "part" : newRecord["part"]
                }
                books.append(newBook)

                return Response((json.dumps({"Success":"Book was added; "})+json.dumps(newBook)), status=201, headers={"location": "/api/bookList/"+str(books[len(books)-1]["id"])}, mimetype="application/json")
            error = "Request expects part id"
            return Response(json.dumps({"Failure" : error}),status=400,mimetype="application/json")

        else:
            error = "This data has not been given: "
            if "author" not in newRecord:
                error += "author; "
            if "title" not in newRecord:
                error += "title; "
            if "year" not in newRecord:
                error += "year; "
            if "isbn" not in newRecord:
                error += "isbn;"
            if "part" not in newRecord:
                error += "part."
            return Response(json.dumps({"Failure" : error}),status=400,mimetype="application/json")

# ---------------------------------------------------------------------------------------------------------------------------------------

@application.route("/api/books/<int:bookID>", methods = ["GET"])
def getBookByID(bookID):

    book = [book for book in books if book["id"] == bookID]

    if len(book) != 1:
        abort(404)

    args = request.args
    expand = args.get('expand')

    if (expand == "part"):
        returnBook = {
            "author" : book[0]["author"],
            "title" : book[0]["title"],
            "year" : book[0]["year"],
            "isbn" : book[0]["isbn"],
            "part" : book[0]["part"]
        }
        try:
            get = requests.get(URL + "api/parts/" + str(book[0]["part"]))
            getPart = {
            "manufacturer" : get.json()[0]["manufacturer"],
            "name" : get.json()[0]["name"],
            "type" : get.json()[0]["type"],
            "price" : get.json()[0]["price"]
            }
            returnBook["part"] = getPart
        except requests.exceptions.RequestException as ex:
            print(ex)

        return Response(json.dumps(returnBook), status=200,mimetype="application/json")

    else:
        returnBook = {
            "author" : book[0]["author"],
            "title" : book[0]["title"],
            "year" : book[0]["year"],
            "isbn" : book[0]["isbn"],
            "part" : book[0]["part"]
        }
        return Response((json.dumps(returnBook)), status=200,mimetype="application/json")

@application.route("/api/books/<int:bookID>", methods = ["PUT"])
def putBookByID(bookID):
    book = [book for book in books if book["id"] == bookID]

    if len(book) != 1:
        abort(404)

    updateBook = request.get_json("force: True")

    if ("author" not in updateBook) and ("title" not in updateBook) and ("year" not in updateBook) and ("isbn" not in updateBook) and ("part" not in updateBook):
        return Response(json.dumps({"Failure" : "No data is given for update"}),status=400,mimetype="application/json")

    update = "This data have been changed: "

    if ("expand" in request.args):
        if type(updateBook["part"]) == type(0):
            return Response(json.dumps({"Failure" : "Request expects json type part"}),status=400,mimetype="application/json")

        if ("manufacturer" not in updateBook["part"]) and ("name" not in updateBook["part"]) and ("price" not in updateBook["part"]) and ("type" not in updateBook["part"]):
            return Response(json.dumps({"Failure" : "No data is given for book part"}),status=400,mimetype="application/json")

        if "author" in updateBook:
            book[0]["author"] = updateBook["author"]
            update += "author; "
        if "title" in updateBook:
            book[0]["title"] = updateBook["title"]
            update += "title; "
        if "year" in updateBook:
            book[0]["year"] = updateBook["year"]
            update += "year; "
        if "isbn" in updateBook:
            book[0]["isbn"] = updateBook["isbn"]
            update += "isbn; "
        if "part" in updateBook:

            returnBook = {
                "author" : book[0]["author"],
                "title" : book[0]["title"],
                "year" : book[0]["year"],
                "isbn" : book[0]["isbn"],
                "part" :  book[0]["part"]
            }
            try:
                put = requests.put(URL + "api/parts/" + str(book[0]["part"]), json = updateBook["part"])
                update += "part; "
                get = requests.get(URL + "api/parts/" + str(book[0]["part"]))
                getPart = {
                    "manufacturer" : get.json()[0]["manufacturer"],
                    "name" : get.json()[0]["name"],
                    "type" : get.json()[0]["type"],
                    "price" : get.json()[0]["price"]
                }
                returnBook["part"] = getPart
            except requests.exceptions.RequestException as ex:
                update += "(part could not be changed because of connection error)"

        return Response(json.dumps({"Success" : str(update)})+json.dumps(returnBook),status="200",mimetype="application/json")

    else:
        if type(updateBook["part"]) == type(0):
            if "author" in updateBook:
                book[0]["author"] = updateBook["author"]
                update += "author; "
            if "title" in updateBook:
                book[0]["title"] = updateBook["title"]
                update += "title; "
            if "year" in updateBook:
                book[0]["year"] = updateBook["year"]
                update += "year; "
            if "isbn" in updateBook:
                book[0]["isbn"] = updateBook["isbn"]
                update += "isbn; "
            if "part" in updateBook:
                book[0]["part"] = updateBook["part"]

            returnBook = {
                "author" : book[0]["author"],
                "title" : book[0]["title"],
                "year" : book[0]["year"],
                "isbn" : book[0]["isbn"],
                "part" : book[0]["part"]
            }

            return Response(json.dumps({"Success" : update})+json.dumps(returnBook),status="200",mimetype="application/json")

        error = "Request expects part id"
        return Response(json.dumps({"Failure" : error}),status=400,mimetype="application/json")

@application.route("/api/books/<int:bookID>", methods = ["DELETE"])
def deleteBookByID(bookID):

    for bookDEL in books:
        if bookDEL["id"] == bookID:
            books.remove(bookDEL)
            return Response(json.dumps({"Success" : "Book with id "+str(bookID)+" have been deleted"}),status=200,mimetype="application/json")

# ---------------------------------------------------------------------------------------------------------------------------------------

@application.route("/api/books/<int:bookID>/part", methods = ["GET"])
def getBooksPartByID(bookID):
    book = [book for book in books if book["id"] == bookID]

    if len(book) == 0:
        abort(404)

    get = requests.get(URL + "/api/parts/" + str(book[0]["part"]))
    part = {
        "manufacturer" : get.json()[0]["manufacturer"],
        "name" : get.json()[0]["name"],
        "type" : get.json()[0]["type"],
        "price" : get.json()[0]["price"]
    }
    return Response(json.dumps(part),status="200",mimetype="application/json")

if __name__ == "__main__":
    application.run(host = "0.0.0.0", port = 80, debug = True)


import json

from pymongo import MongoClient

from pydatalab.config import CONFIG

client = MongoClient(CONFIG.MONGO_URI)

db = client.datalabvue

all_documents = db.items.find()

nodes = []
edges = []
for document in all_documents:
    if ("parent_items" not in document) and ("child_items" not in document):
        continue
    nodes.append(
        {"data": {"id": document["item_id"], "name": document["name"], "type": document["type"]}}
    )
    if "parent_items" not in document:
        continue
    for parent_id in document["parent_items"]:
        target = document["item_id"]
        source = parent_id
        edges.append(
            {
                "data": {
                    "id": f"{source}->{target}",
                    "source": source,
                    "target": target,
                    "value": 1,
                }
            }
        )

with open("cy_links_production.json", "w") as f:
    json.dump({"nodes": nodes, "edges": edges}, f)


import cv2
import matplotlib.pyplot as plt
import numpy as np
from Word_segmentation import word
from segment_lines import line_segment
from questionHistogram import projection
from Question_segmentation import question

question_partitions = []
line_partitions = []
character_partitions = []
word_partitions = []

def segmentation(Thresh_Image, axis_value):

    final_Image = Thresh_Image.copy();

    height, width = Thresh_Image.shape
    print("Height : ", height)
    print("Width : ", width)

    Thresh_Image[Thresh_Image == 0] = 1
    Thresh_Image[Thresh_Image == 255] = 0

    horizontal_projection = np.sum(Thresh_Image, axis_value)
    print(horizontal_projection)
    #
    blankImage = np.zeros((height, width, 3), np.uint8)

    for row in range(height):
        cv2.line(blankImage, (0, row), (int(horizontal_projection[row] * width / height), row), (255, 255, 255), 1)

############################################################################################################
# for loop for segmenting
    # segmentingLoop(horizontal_projection, row_index, limit, start_list, end_list)

    limit = 0
    if(axis_value == 1):
        limit = height
    if(axis_value == 0):
        limit = width

    row_index = 0
    start_list = []
    end_list = []

    line_segment(horizontal_projection, row_index, limit, start_list, end_list)

#############################################################################################################
    # kadena images tika wena wenama array ekakata append karagaththa
    if axis_value == 1:

        # print("\n- - - - - line cordinations > > > > > > > > ")
        # print(start_list)
        #  print(end_list)

        # question - - - -

        name = plt.imshow(blankImage, cmap='gray')
        plt.title("Horizontal Projections")
        plt.show()

        line_slices = []
        for x in range(len(start_list)):
                line_slices.append(final_Image[start_list[x]:end_list[x], 0:width])
                name = "line - " + str(x+1)
                Imgname = plt.imshow(line_slices[x], cmap='gray')
                plt.title(name)
                plt.show()

        global line_partitions
        line_partitions = line_slices

    if axis_value == 0:

        word_start = [start_list[0]]
        word_end = []
        word(start_list, end_list, word_start, word_end)

        word_slices = []
        for t in range(0, len(word_start)):
            word_slices.append(final_Image[0:height, word_start[t]:word_end[t]])
            name = "Word - " + str(t+1)
            Imgname = plt.imshow(word_slices[t], cmap='gray')
            plt.title(name)
            plt.show()

        character_slices = []
        for i in range(0, len(start_list)):
            character_slices.append(final_Image[0:height, start_list[i]:end_list[i]])
            charName = "character - " + str(i+1)
            # cv2.imshow(name, character_slices[i])
            name = plt.imshow(character_slices[i], cmap='gray')
            plt.title(charName)
            plt.show()

        global character_partitions
        character_partitions = character_slices

    ###############################################################################################################


from __future__ import print_function

#   SDK modules to import

import boto3
from decimal import Decimal
import json
import urllib
import sys

#   Service assignments

rekognition = boto3.client('rekognition')
sns = boto3.client('sns')

#   Declare variables

jobID = ""
key = ""
bucket = ""
queue_url = ''
bgfFaceSearchRekognitionServiceRole = ''
bgfFaceSearchSNS = ''
matchthreshold = 80

#   Main function

def lambda_handler(event, context):

#   Get variables from event

    bucket = (event['Records'][0]['s3']['bucket']['name'])
    key = (event['Records'][0]['s3']['object']['key'])

#   Call Rekognition with event and credentials
#   Pass event data to be included in SNS message

    response = rekognition.start_face_search(
    Video={'S3Object':{'Bucket':bucket,'Name':key}},
    FaceMatchThreshold=matchthreshold,
    JobTag=key,
    CollectionId="ourfacescollection",
    NotificationChannel={
        'SNSTopicArn': bgfFaceSearchSNS,
        'RoleArn': bgfFaceSearchRekognitionServiceRole})
    print ("Processed video: " + key)
    return response