from tkinter import ttk, Frame, Entry, Scrollbar, messagebox
from ttkbootstrap import Style
from ttkbootstrap.constants import *
from pandas import ExcelWriter, DataFrame
from selenium.webdriver import Chrome
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from time import sleep
from datetime import datetime, date
from clipboard import copy
from json import load

class Funcoes:
    def iniciar_testes(self):
        # Coletando opções
        self.resp_squad = self.box_squad.get() if self.box_squad.get() != 'Todos os Squads' else 0
        self.salvar = self.box_salvar.get()
        self.sites = self.url_lps()
        self.resultado_etapas = []
        self.resultado_geral = dict()
        self.resultado_final = []

        # Desativando botões da interface
        self.box_salvar.configure(state='disable')
        self.box_squad.configure(state='disable')
        self.bt_iniciar.destroy()
        try:
            self.entry_destino_arquivo.configure(state='disable')
        except:
            pass
        # Meios técnicos para recriar o botão
        self.bt_iniciar = ttk.Button(self.frame_um, text='Testes Finalizados!', width=50, style=SUCCESS)
        self.bt_iniciar.pack(pady=(164, 0))

        # Alterando e inserindo elementos
        self.root.geometry('450x530')
        self.frame_um.place(relx=0.02, rely=0.01, relwidth=0.96, relheight=0.4985)
        self.frame_2()
        self.frame_de_lista()

        #Iniciando o fluxo
        self.fluxo_completo()
        self.salvar_arquivo()
        self.bt_copiar = ttk.Button(self.frame_dois, text='Copiar Mensagem Selecionada', width=46, style=OUTLINE,
                                    command=self.copiar_dados_lista)
        self.bt_copiar.place(relx=0.01, rely=0.745, relheight=0.14)

    def tratar_cidade(self, navegador):
        try:
            navegador.find_element(By.XPATH, '//*[@id="modal-portal"]/div/div/form/div/div/select').click()
            navegador.find_element(By.XPATH, '//*[@id="modal-portal"]/div/div/form/div/div/select/option[2]').click()
            navegador.find_element(By.XPATH, '//*[@id="modal-portal"]/div/div/form/button').click()
            return 'Cidade OK'
        except:
            try:
                navegador.find_element(By.XPATH, '//*[@id="__next"]/section/div/div/main/section/div/select').click()
                navegador.find_element(By.XPATH,
                                       '//*[@id="__next"]/section/div/div/main/section/div/select/option[2]').click()
                return 'Cidade OK'
            except:
                try:
                    navegador.find_element(By.XPATH, '//*[@id="modal-portal"]/div/div/form/div/div/div/select').click()
                    navegador.find_element(By.XPATH,
                                           '//*[@id="modal-portal"]/div/div/form/div/div/div/select/option[2]').click()
                    navegador.find_element(By.XPATH, '//*[@id="modal-portal"]/div/div/form/button').click()
                    return 'Cidade OK'
                except:
                    return 'N/A'
                    pass

    def abrir_snippet(self, navegador):
        try:
            sleep(1)
            navegador.execute_script('document.querySelector("#dots-cta > img").click()')  # clica snippet
            navegador.execute_script('document.querySelector("#dots-chat-cta > img").click()')  # clica no ícone webchat
            return 'Snippet OK'
        except:
            return 'Snippet Não OK'

    def url_lps(self):
        with open('url_lps.json', encoding='utf-8') as arquivo_urls:
            sites = load(arquivo_urls)
        return sites.copy()

    def mapeamento_steps(self, cep='30000000', numero='01', dt_vencimento='não sei'):
        with open('mapeamento_steps.json', encoding='utf-8') as mapeamento:
            palavras_chave = load(mapeamento)
        palavras_chave['CEP'].append(cep)
        palavras_chave['CEP_2'].append(cep)
        palavras_chave['Número Ende.'].append(numero)
        palavras_chave['Data Vencimento'].append(dt_vencimento)
        palavras_chave['Data Vencimento2'].append(dt_vencimento)

        return palavras_chave.copy()

    def encontra_chave_step(self, navegador, cep='30000000', numero='01', dt_vencimento='não sei'):
        steps_local = self.mapeamento_steps(cep, numero, dt_vencimento)
        cont = 0
        while True:
            n_bloco_atual = len(navegador.find_elements(By.XPATH, '//*[@id="messages-list"]/div[1]/div/div/div[2]/div'))
            if n_bloco_atual % 2 == 0 and n_bloco_atual > 0:
                try:
                    WebDriverWait(navegador, 1).until(
                        expected_conditions.presence_of_element_located(
                            (By.XPATH, f'//*[@id="messages-list"]/div[1]/div/div/div[2]/div[{n_bloco_atual + 1}]')))
                except:
                    break
                else:
                    continue
            else:
                sleep(0.5)
                cont += 0.5
                if cont >= 1:
                    break

        n_ultima_msg = len(navegador.find_elements(
            By.XPATH, f'//*[@id="messages-list"]/div[1]/div/div/div[2]/div[{n_bloco_atual}]/div[2]/div'))
        try:
            for cont in range(1, n_ultima_msg + 1):
                mensagem = str(navegador.find_element(
                    By.XPATH, f'//*[@id="messages-list"]/div[1]/div/div/div[2]/div[{n_bloco_atual}]'
                              f'/div[2]/div[{cont}]/div/div/div/div/div[1]/div[1]').text)
                for key, valor in steps_local.items():
                    if valor[0].upper() in mensagem.upper():
                        return key
        except:
            return '1'
        else:
            return '1'

    def interacao_chat(self, navegador, operacao, CEP='30000000', num='01', dt_vencimento='não sei', squad_atual='Null'):
        steps = self.mapeamento_steps(CEP, num, dt_vencimento)
        lista_aux_chat = []
        tempo_erro = apoio = 0

        while True:
            chave_step = self.encontra_chave_step(navegador, CEP, num, dt_vencimento)
            if chave_step.isnumeric():
                tempo_erro += int(chave_step)
            elif apoio == chave_step:
                tempo_erro += 1
            else:
                lista_aux_chat.append(f'{chave_step} - OK')
                tempo_erro = 0

            if chave_step in 'Oferta Planos':
                sleep(3.5)

            try:
                navegador.find_element(By.ID, 'msg-textarea').send_keys(steps[chave_step][1], Keys.ENTER)
            except:
                pass
            n_bloco_atual = len(navegador.find_elements(By.XPATH, '//*[@id="messages-list"]/div[1]/div/div/div[2]/div'))
            n_ultima_msg = len(navegador.find_elements(
                By.XPATH, f'//*[@id="messages-list"]/div[1]/div/div/div[2]/div[{n_bloco_atual}]/div[2]/div'))
            if chave_step.split()[0].upper() in 'ERRO':
                mensagem = str(navegador.find_element(
                    By.XPATH, f'//*[@id="messages-list"]/div[1]/div/div/div[2]/div[{n_bloco_atual}]'
                              f'/div[2]/div[{n_ultima_msg - 1}]/div/div/div/div/div[1]/div[1]').text)
                self.lista_observacoes.insert('', END, values=(squad_atual, operacao, chave_step, mensagem))
                if self.salvar == 'Sim':
                    pasta_arquivo = self.entry_destino_arquivo.get()
                    navegador.save_screenshot(f'{pasta_arquivo}/{operacao} - {chave_step} ({date.today()}).png')
                break
            if chave_step in 'Consultor Indisponível, Finalização' or chave_step.split()[0].upper() in 'TRANSBORDO_ATH':
                break
            elif chave_step == '1' and n_bloco_atual % 2 == 0 and n_bloco_atual > 0 and tempo_erro >= 15:
                chave_step = 'Chave não mapeada'
                lista_aux_chat.append(chave_step)
                mensagem = str(navegador.find_element(
                    By.XPATH, f'//*[@id="messages-list"]/div[1]/div/div/div[2]/div[{n_bloco_atual}]'
                              f'/div[2]/div[{n_ultima_msg - 1}]/div/div/div/div/div[1]/div[1]').text)
                self.lista_observacoes.insert('', END, values=(squad_atual, operacao, chave_step, mensagem))
                if self.salvar == 'Sim':
                    pasta_arquivo = self.entry_destino_arquivo.get()
                    navegador.save_screenshot(f'{pasta_arquivo}/{operacao} - {chave_step} ({date.today()}).png')
                break
            elif n_bloco_atual > 0 and tempo_erro >= 15:
                chave_step = f'ERROR TIMEOUT'
                lista_aux_chat.append(chave_step)
                mensagem = str(navegador.find_element(
                    By.XPATH, f'//*[@id="messages-list"]/div[1]/div/div/div[2]/div[{n_bloco_atual}]'
                              f'/div[2]/div[{n_ultima_msg - 1}]/div/div/div/div/div[1]/div[1]').text)
                self.lista_observacoes.insert('', END, values=(squad_atual, operacao, chave_step, mensagem))
                if self.salvar == 'Sim':
                    pasta_arquivo = self.entry_destino_arquivo.get()
                    navegador.save_screenshot(f'{pasta_arquivo}/{operacao} - {chave_step} ({date.today()}).png')
                break
            apoio = chave_step
        return lista_aux_chat[:]

    def fluxo_completo(self):
        try:
            self.navegador = Chrome()
            lista_auxiliar = []
            if self.resp_squad == 0:
                nome_squad_1 = list(self.sites)[0]
                nome_lp_1 = list(self.sites[nome_squad_1])[0]
                primeira_url = self.sites[nome_squad_1][nome_lp_1][0]
                for k, squad in self.sites.items():
                    for k2, lp in squad.items():
                        self.resultado_geral[k2] = []
                        try:
                            if self.sites[k][k2][0] == primeira_url:
                                self.navegador.get(lp[0])
                            else:
                                self.navegador.switch_to.new_window('tab')
                                self.navegador.get(lp[0])
                        except:
                            self.resultado_geral[k2] = ['LP Fora do Ar']
                            self.lista_observacoes.insert('', END, values=(k, k2,
                                                                           'LP Fora do Ar', 'LP Fora do Ar'))
                            continue
                        else:
                            c = self.sites[k][k2][1]
                            n = self.sites[k][k2][2]
                            venci = 'Não sei'
                            try:
                                venci = self.sites[k][k2][3]
                            except:
                                pass
                            self.resultado_etapas.append('LP OK')
                            self.resultado_etapas.append(self.tratar_cidade(self.navegador))
                            self.resultado_etapas.append(self.abrir_snippet(self.navegador))
                            try:
                                self.navegador.switch_to.frame(self.navegador.find_element(By.ID, 'blip-chat-iframe'))
                                WebDriverWait(self.navegador, 30).until(
                                    expected_conditions.presence_of_element_located(
                                        (By.XPATH,
                                         '//*[@id="messages-list"]/div[1]/div/div/div[2]/div[2]/div[2]/div[1]/div/div/div/div/div[1]/div')))
                            except:
                                self.resultado_etapas.append('CHAT TIMEOUT')
                                for cont in range(0, len(lista_auxiliar)):
                                    self.resultado_geral[k2].append(self.resultado_etapas[cont])
                                if self.resultado_etapas[-1] == 'Snippet Não OK':
                                    self.lista_observacoes.insert('', END, values=(k, k2,
                                                                                'Snippet Não OK', 'CHAT TIMEOUT'))
                                self.resultado_etapas.clear()
                                continue
                            else:
                                self.resultado_etapas.append('CHAT OK')

                            lista_auxiliar = self.interacao_chat(self.navegador, k2, c, n, venci, k)
                            for cont in range(0, len(lista_auxiliar)):
                                self.resultado_etapas.append(lista_auxiliar[cont])

                            self.resultado_geral[k2] = self.resultado_etapas[:]
                            self.resultado_etapas.clear()
                            lista_auxiliar.clear()

                    tamanho_maximo_etapas = 0
                    for operacao in self.resultado_geral.values():
                        if len(operacao) > tamanho_maximo_etapas:
                            tamanho_maximo_etapas = len(operacao)

                    for operacao in self.resultado_geral.values():
                        if len(operacao) < tamanho_maximo_etapas:
                            dif = tamanho_maximo_etapas - len(operacao)
                            for cont in range(0, dif):
                                operacao.append('-')

                    vars()[f'df_{k}'] = DataFrame(data=self.resultado_geral)
                    self.resultado_geral.clear()
                    self.resultado_final.append(vars()[f'df_{k}'])

            else:
                nome_lp_1 = list(self.sites[self.resp_squad])[0]
                primeira_url = self.sites[self.resp_squad][nome_lp_1][0]
                for k2, lp in self.sites[self.resp_squad].items():
                    self.resultado_geral[k2] = []
                    try:
                        if lp[0] == primeira_url:
                            self.navegador.get(lp[0])
                        else:
                            self.navegador.switch_to.new_window('tab')
                            self.navegador.get(lp[0])
                    except:
                        self.resultado_geral[k2] = ['LP Fora do Ar']
                        self.lista_observacoes.insert('', END, values=(self.resp_squad, k2,
                                                                       'LP Fora do Ar', 'LP Fora do Ar'))
                    else:
                        c = self.sites[self.resp_squad][k2][1]
                        n = self.sites[self.resp_squad][k2][2]
                        venci = 'Não sei'
                        try:
                            venci = self.sites[self.resp_squad][k2][3]
                        except:
                            pass
                        self.resultado_etapas.append('LP OK')
                        self.resultado_etapas.append(self.tratar_cidade(self.navegador))
                        self.resultado_etapas.append(self.abrir_snippet(self.navegador))
                        try:
                            self.navegador.switch_to.frame(self.navegador.find_element(By.ID, 'blip-chat-iframe'))
                            WebDriverWait(self.navegador, 30).until(
                                expected_conditions.presence_of_element_located(
                                    (By.XPATH,
                                     '//*[@id="messages-list"]/div[1]/div/div/div[2]/div[2]/div[2]/div[1]/div/div/div/div/div[1]/div')))
                        except:
                            self.resultado_etapas.append('CHAT TIMEOUT')
                            for cont in range(0, len(lista_auxiliar)):
                                self.resultado_geral[k2].append(self.resultado_etapas[cont])
                            if self.resultado_etapas[-1] == 'Snippet Não OK':
                                self.lista_observacoes.insert('', END, values=(self.resp_squad, k2,
                                                                               'Snippet Não OK', 'CHAT TIMEOUT'))
                            self.resultado_etapas.clear()
                            continue
                        else:
                            self.resultado_etapas.append('CHAT OK')

                        lista_auxiliar = self.interacao_chat(self.navegador, k2, c, n, venci, self.resp_squad)
                        for cont in range(0, len(lista_auxiliar)):
                            self.resultado_etapas.append(lista_auxiliar[cont])

                        self.resultado_geral[k2] = self.resultado_etapas[:]
                        self.resultado_etapas.clear()
                        lista_auxiliar.clear()

                tamanho_maximo_etapas = 0
                for operacao in self.resultado_geral.values():
                    if len(operacao) > tamanho_maximo_etapas:
                        tamanho_maximo_etapas = len(operacao)

                for operacao in self.resultado_geral.values():
                    if len(operacao) < tamanho_maximo_etapas:
                        dif = tamanho_maximo_etapas - len(operacao)
                        for cont in range(0, dif):
                            operacao.append('-')

                vars()[f'df_{self.resp_squad}'] = DataFrame(data=self.resultado_geral)
                self.resultado_geral.clear()
                self.resultado_final.append(vars()[f'df_{self.resp_squad}'])
            messagebox.showinfo(title='Status Testes', message='Testes finalizados com êxito!')
        except Exception as erro:
            messagebox.showerror(title='Erro no fluxo', message=f'Houve erro no fluxo do chat!\n\n'
                                                                f'Erro: {erro.__class__}\n\n'
                                                                f'Descrição: {erro.__context__}\n'
                                                                f'{erro.__str__()}')

    def salvar_arquivo(self):
        try:
            if self.salvar == 'Sim':
                hora = int(str(datetime.time(datetime.today()))[:2])
                if 12 > hora >= 6:
                    turno = 'Manhã'
                elif 18 > hora >= 12:
                    turno = 'Tarde'
                else:
                    turno = 'Noite'
                pasta_arquivo = self.entry_destino_arquivo.get() + '/'
                pasta_arquivo = f'{pasta_arquivo}/Testes de Fluxo {self.resp_squad if self.resp_squad != 0 else ""}' \
                                f' {date.today().day}-{date.today().month} ({turno}).xlsx'
                arquivo = ExcelWriter(pasta_arquivo, engine='xlsxwriter')
                if self.resp_squad == 0:
                    for i, resultado in enumerate(self.resultado_final):
                        nome = list(self.sites)[i]
                        self.resultado_final[i].to_excel(arquivo, sheet_name=nome, index=False)
                else:
                    self.resultado_final[0].to_excel(arquivo, sheet_name=self.resp_squad, index=False)
                arquivo.save()
            self.resultado_final.clear()
        except Exception as erro:
            messagebox.showerror(title='Erro no Salvamento', message=f'Houve erro ao salvar os testes!\n\n'
                                                                f'Erro: {erro.__class__}\n\n'
                                                                f'Descrição: {erro.__context__}\n'
                                                                f'{erro.__str__()}')

    def copiar_dados_lista(self, evento=None):
        valores = list()
        itens_selecionados = self.lista_observacoes.selection()
        if len(itens_selecionados) >=1:
            for i_item in range(0, len(itens_selecionados)):
                valores.append(self.lista_observacoes.item(itens_selecionados[i_item], 'values'))
            valores = str(valores)
            copy(valores)
            messagebox.showinfo(title='Valor Copiado',
                                message='Os itens selecionados foram copiados \npara a área de transferência com êxito!')
        else:
            messagebox.showwarning(title='Seleção inválida', message='Selecione um elemento para ser copiado.')

class Janela_principal(Funcoes):
    def __init__(self):
        # Atributos
        #Estilo
        self.estilo_1 = Style(theme='cosmo')
        self.estilo_1.configure('TButton', font='sans-serif 11')
        self.estilo_1.configure('Treeview', foreground='#3D3D3D', rowheight=25)
        self.estilo_1.map('Treeview', background=[('selected', 'green')])
        #Opções comboBox squad
        self.squads = ['Todos os Squads']
        aux = list(Funcoes.url_lps(self))
        num_squads = len(aux)
        for x in range(0, num_squads):
            self.squads.append(aux[x])
        # Define tema e cria o objeto janela
        self.root = self.estilo_1.master
        # Funções
        self.janela()
        self.frame_1()
        self.titulo()
        self.campos()
        self.botoes()
        # Mantendo janela aberta
        self.root.mainloop()

    def janela(self):
        self.root.title('BOT de Teste de Fluxo ISPs Automatizado')
        self.larg_janela = 450
        self.compr_janela = 277
        self.root.geometry(f'{self.larg_janela}x{self.compr_janela}')
        self.root.resizable(False, False)
        self.root.config(background='#257DFE')

    def frame_1(self):
        self.frame_um = Frame(self.root)
        self.frame_um.place(relx=0.02, rely=0.027, relwidth=0.96, relheight=0.9461)

    def frame_2(self):
        self.frame_dois = ttk.LabelFrame(self.root, text=' Observações ')
        self.frame_dois.place(relx=0.02, rely=0.52, relwidth=0.96, relheight=0.465)

    def frame_de_lista(self):
        # Barra de Rolagem vertical
        self.scroll_vertical = Scrollbar(self.frame_dois, orient='vertical')
        self.scroll_vertical.place(relx=0.92, rely=0.01, relwidth=0.07, relheight=0.88)
        # Barra de Rolagem horizontal
        self.scroll_horizontal = Scrollbar(self.frame_dois, orient='horizontal')
        self.scroll_horizontal.place(relx=0.01, rely=0.9, relwidth=0.9, relheight=0.08)
        #Criando o frame com o nome das colunas e linkando-o aos scrolls
        self.lista_observacoes = ttk.Treeview(self.frame_dois, height=2, columns=('squad', 'operação', 'observação', 'mensagem'),
                                              show='headings',
                                              yscrollcommand=self.scroll_vertical.set,
                                              xscrollcommand=self.scroll_horizontal.set)
        self.lista_observacoes.place(relx=0.01, rely=0.01, relwidth=0.9, relheight=0.73)
        # Linkando a barra de rolagem e a lista
        self.scroll_vertical.config(command=self.lista_observacoes.yview)
        self.scroll_horizontal.config(command=self.lista_observacoes.xview)
        #Cabeçalho das colunas
        self.lista_observacoes.heading('#1', text='Squad', anchor=W)
        self.lista_observacoes.heading('#2', text='Operação', anchor=W)
        self.lista_observacoes.heading('#3', text='Observação', anchor=W)
        self.lista_observacoes.heading('#4', text='Mensagem', anchor=W)
        #Tamanho das colunas
        self.lista_observacoes.column('#1', width=80, anchor=W)
        self.lista_observacoes.column('#2', width=80, anchor=W)
        self.lista_observacoes.column('#3', width=100, anchor=W)
        self.lista_observacoes.column('#4', width=240, anchor=W)

    def titulo(self):
        self.titulo_lb = ttk.Label(self.frame_um, text='BOT de Teste de Fluxo LP', font=('sans-serif', '18', 'bold'),
                                   foreground='#257DFE')
        self.titulo_lb.pack(pady=8)

    def campos(self):
        # Campo Squad
        self.lb_squad = ttk.Label(self.frame_um, text='Escolha o Squad:', font=('sans-serif', '13'), foreground='black')
        self.lb_squad.place(relx=0.02, rely=0.265)
        # Combobox
        self.box_squad = ttk.Combobox(
            self.frame_um, values=self.squads, font=('sans-serif', '13'),
            foreground='black', state='readonly', takefocus=False
        )
        self.box_squad.current(0)
        self.box_squad.place(relx=0.35, rely=0.24, relwidth=0.625)

        # Campo "Salvar""
        self.lb_salvar = ttk.Label(self.frame_um, text='Deseja Salvar o resultado?', font=('sans-serif', '13'),
                                   foreground='black')
        self.lb_salvar.place(relx=0.02, rely=0.45)
        # Combobox
        self.box_salvar = ttk.Combobox(
            self.frame_um, values=['Sim', 'Não'], font=('sans-serif', '13'), foreground='black', state='readonly',
            takefocus=False
        )
        self.box_salvar.current(1)
        self.box_salvar.place(relx=0.50, rely=0.425, relwidth=0.475)
        self.box_salvar.bind('<<ComboboxSelected>>', self.reacao_box_salvar)

    def botoes(self):
        self.bt_iniciar = ttk.Button(self.frame_um, text='Iniciar Testes', width=50, command=self.iniciar_testes)
        self.bt_iniciar.pack(pady=(162, 0))

    def reacao_box_salvar(self, evento=None):
        if self.box_salvar.get() == "Sim":
            self.lb_destino_arquivo = ttk.Label(self.frame_um, text='Destino do arquivo: ', font=('sans-serif', '13'),
                                                foreground='black')
            self.lb_destino_arquivo.place(relx=0.02, rely=0.625)
            # Campo de destino do arquivo
            self.entry_destino_arquivo = Entry(self.frame_um, font=('sans-serif', 9), fg='gray', state='normal')
            self.entry_destino_arquivo.place(relx=0.375, rely=0.615, relwidth=0.60, relheight=0.12)
            self.entry_destino_arquivo.insert('1',
                                            'S:/Inovação/Planejamento/3 - MIS/Gerencial/Acompanhamento das ISPS - Semanal/Testes de Fluxo')
        else:
            try:
                self.lb_destino_arquivo.destroy()
                self.entry_destino_arquivo.destroy()
            except:
                pass

Janela_principal()

Mundiale Projects/Automatização Testes Fluxo ISPs/BOT LP com interface/Interface_BOT.py


from typing import List, Optional, Callable, Tuple, Any, Dict
import numpy as np
from dataclasses import dataclass, field
import math
from om.algorithms.crystallography import TypePeakList, Peakfinder8PeakDetection

## Some abstractions to call peakfinder8 in Python

def build_pixel_map(row: int, col: int, y0: int, x0: int) -> Dict[str, int]:
    """
    Calculate radius pixels map for a given center x0,y0.

    Parameters
    ----------
    row: int
        Number of rows of data.
    col: int
        Number of columns of data.
    y0:
        Radial center position in y axis (row).
    x0:
        Radial center position in x axis (column).

    Returns
    ----------
    radius_pixel_map: Dict
        "radius": radius pixel map in realtion to the give center. It has same size of data given by row and col.
    """
    radius_pixel_map = np.ones((row, col)).astype(int)
    for idy, i in enumerate(radius_pixel_map):
        for idx, j in enumerate(i):
            radius_pixel_map[idy, idx] = int(
                math.sqrt((idx - x0) ** 2 + (idy - y0) ** 2)
            )
    return dict(radius=radius_pixel_map)

@dataclass
class PF8Info:
    max_num_peaks: int
    pf8_detector_info: dict
    adc_threshold: float
    minimum_snr: int
    min_pixel_count: int
    max_pixel_count: int
    local_bg_radius: int
    min_res: float
    max_res: float
    _bad_pixel_map: np.array
    _pixelmaps: np.array = field(init=False)

    def __post_init__(self):

        self._pixelmaps = build_pixel_map(
            (self._bad_pixel_map).shape[0],
            (self._bad_pixel_map.shape[1]),
            int((self._bad_pixel_map).shape[0] / 2),
            int((self._bad_pixel_map).shape[1] / 2),
        )

    def modify_mask(self, mask):
        self._bad_pixel_map = mask

    def modify_radius(self, center_x, center_y):
        self._pixelmaps = build_pixel_map(
            (self._bad_pixel_map).shape[0],
            (self._bad_pixel_map.shape[1]),
            center_y,
            center_x,
        )

class PF8:
    def __init__(self, info):
        assert isinstance(
            info, PF8Info
        ), f"Info object expected type PF8Info, found {type(info)}."
        self.pf8_param = info

    def get_peaks_pf8(self, data):
        detector_layout = self.pf8_param.pf8_detector_info

        peak_detection = Peakfinder8PeakDetection(
            self.pf8_param.max_num_peaks,
            self.pf8_param.pf8_detector_info["asic_nx"],
            self.pf8_param.pf8_detector_info["asic_ny"],
            self.pf8_param.pf8_detector_info["nasics_x"],
            self.pf8_param.pf8_detector_info["nasics_y"],
            self.pf8_param.adc_threshold,
            self.pf8_param.minimum_snr,
            self.pf8_param.min_pixel_count,
            self.pf8_param.max_pixel_count,
            self.pf8_param.local_bg_radius,
            self.pf8_param.min_res,
            self.pf8_param.max_res,
            self.pf8_param._bad_pixel_map.astype(np.float32),
            (self.pf8_param._pixelmaps["radius"]).astype(np.float32),
        )
        peaks_list = peak_detection.find_peaks(data)
        return peaks_list


import unittest
from BBTools.BBToolsImpl import BBTools
from BBTools.BBToolsServer import MethodContext
from installed_clients.authclient import KBaseAuth as _KBaseAuth
from BBTools.utils.MemEstimatorRunner import MemEstimatorRunner
import os
try:
    from ConfigParser import ConfigParser  # py2
except:
    from configparser import ConfigParser  # py3

class MemEstimatorTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        token = os.environ.get('KB_AUTH_TOKEN', None)
        config_file = os.environ.get('KB_DEPLOYMENT_CONFIG', None)
        cls.cfg = {}
        config = ConfigParser()
        config.read(config_file)
        for nameval in config.items('BBTools'):
            cls.cfg[nameval[0]] = nameval[1]
        # Getting username from Auth profile for token
        authServiceUrl = cls.cfg['auth-service-url']
        auth_client = _KBaseAuth(authServiceUrl)
        user_id = auth_client.get_user(token)
        # WARNING: don't call any logging methods on the context object,
        # it'll result in a NoneType error
        cls.ctx = MethodContext(None)
        cls.ctx.update({'token': token,
                        'user_id': user_id,
                        'provenance': [
                            {'service': 'BBTools',
                             'method': 'please_never_use_it_in_production',
                             'method_params': []
                             }],
                        'authenticated': 1})
        cls.serviceImpl = BBTools(cls.cfg)
        cls.scratch = cls.cfg['scratch']
        cls.callback_url = os.environ['SDK_CALLBACK_URL']

    def test_mem_estimator_ok_unit(self):
        in_file = os.path.join('data','reads','interleaved.fastq')
        params = {
            "reads_file": in_file
        }
        runner = MemEstimatorRunner(params)
        estimate = runner.run()
        self.assertGreater(estimate["estimate"], 0)
        self.assertGreater(estimate["size"], 0)

    def test_mem_estimator_ok_pair(self):
        params = {
            "reads_file": os.path.join('data','reads','small.forward.fq'),
            "reads_file2": os.path.join('data','reads','small.reverse.fq')
        }
        runner = MemEstimatorRunner(params)
        estimate = runner.run()
        self.assertGreater(estimate["estimate"], 0)
        self.assertGreater(estimate["size"], 0)

    def test_mem_estimator_pair_dups(self):
        in_file = os.path.join('data','reads','interleaved.fastq')
        params = {
            "reads_file": in_file,
            "reads_file2": in_file
        }
        with self.assertRaises(ValueError) as e:
            MemEstimatorRunner(params)
        self.assertIn("If two files are present, they must be different.", str(e.exception))

    def test_mem_estimator_bad_file(self):
        in_file = os.path.join('data','reads','bad_reads.txt')
        params = {
            "reads_file": in_file
        }
        with self.assertRaises(ValueError) as e:
            MemEstimatorRunner(params).run()
        self.assertIn("Count of unique 31-mers not found.", str(e.exception))
        self.assertNotIn("31-mers processed", str(e.exception))
        self.assertNotIn("Correct 31-mers", str(e.exception))
        self.assertNotIn("Unique 31-mers", str(e.exception))

    def getImpl(self):
        return self.__class__.serviceImpl

    def getContext(self):
        return self.__class__.ctx

    def test_mem_estimator_e2e(self):
        impl = self.getImpl()
        ctx = self.getContext()
        in_file = os.path.join('data','reads','interleaved.fastq')
        params = {
            "reads_file": in_file
        }
        res = impl.run_mem_estimator(ctx, params)[0]
        self.assertIn("estimate", res)
        self.assertGreater(res["estimate"], 0)

        params = {
            "reads_file": os.path.join('data','reads','small.forward.fq'),
            "reads_file2": os.path.join('data','reads','small.reverse.fq')
        }
        res = impl.run_mem_estimator(ctx, params)[0]
        self.assertIn("estimate", res)
        self.assertGreater(res["estimate"], 0)
        self.assertIn("size", res)
        self.assertGreater(res["size"], 0)

    def test_bad_inputs(self):
        impl = self.getImpl()
        ctx = self.getContext()
        with self.assertRaises(ValueError) as e:
            MemEstimatorRunner({})
        self.assertIn('Parameter "reads_file" must be present!', str(e.exception))

        with self.assertRaises(ValueError) as e:
            impl.run_mem_estimator(ctx, {"reads_file": "not_real"})
        self.assertIn("The file not_real does not seem to exist, or is a directory", str(e.exception))

        with self.assertRaises(ValueError) as e:
            impl.run_mem_estimator(ctx, {
                "reads_file": os.path.join('data','reads','interleaved.fastq'),
                "reads_file2": "not_real2"}
            )
        self.assertIn("The file not_real2 does not seem to exist, or is a directory", str(e.exception))


import pygame
import sys
import pandas as pd

from settings import Settings
from tile import Tile
from sudoku import Solver

class SudokuVisualizer:
    """Overall class to manage algorithm visualizer"""

    def __init__(self):
        """Initialize game assets"""

        # Pygame basic settings
        pygame.init()
        self.settings = Settings()

        self.screen = pygame.display.set_mode(
            (self.settings.screen_size,
            self.settings.screen_size)
        )
        pygame.display.set_caption("Sudoku Solver")

        # Matrix to represent board
        self.board, self.solution = self._create_board()
        self.tiles = pygame.sprite.Group()
        self._create_tiles()
        self.selected_tile = None

        # Solver object
        self.solver = Solver(self, self.board)

    def run_visualizer(self):
        """Main loop of visualizer"""
        while True:
            self._check_events()
            self.update_screen()

    def _check_events(self):
        """Respond to events"""
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                sys.exit()

            elif event.type == pygame.MOUSEBUTTONDOWN:
                mouse_pos = pygame.mouse.get_pos()
                self._check_mouse_click(mouse_pos)

            elif event.type == pygame.KEYDOWN:
                self._check_keydown_event(event)

    def _check_keydown_event(self, event):
        """Respond to key presses"""

        # Solve button clicked
        if event.key == pygame.K_s:
            self.solver.solve((0,0))

        # Number clicked
        if self.selected_tile:
            self._check_number_click(event)

            # Try to insert number into final board
            if event.key == pygame.K_RETURN:
                self._check_enter_press()

            self.selected_tile.prep_number()

    def _check_number_click(self, event):
        """Respond to number presses"""
        if event.key == pygame.K_0:
            self.selected_tile.user_number = 0
        elif event.key == pygame.K_1:
            self.selected_tile.user_number = 1
        elif event.key == pygame.K_2:
            self.selected_tile.user_number = 2
        elif event.key == pygame.K_3:
            self.selected_tile.user_number = 3
        elif event.key == pygame.K_4:
            self.selected_tile.user_number = 4
        elif event.key == pygame.K_5:
            self.selected_tile.user_number = 5
        elif event.key == pygame.K_6:
            self.selected_tile.user_number = 6
        elif event.key == pygame.K_7:
            self.selected_tile.user_number = 7
        elif event.key == pygame.K_8:
            self.selected_tile.user_number = 8
        elif event.key == pygame.K_9:
            self.selected_tile.user_number = 9

    def _check_enter_press(self):
        """Determine if user number is correct number"""

        # Get correct number from solution board
        correct_number = (self.solution[self.selected_tile.position[0]]
                         [self.selected_tile.position[1]])

        # Correct guess
        if (self.selected_tile.user_number == correct_number):
            self.selected_tile.number = self.selected_tile.user_number
            self.selected_tile.final = True

            # Update board number
            self.board[self.selected_tile.position[0]] \
                [self.selected_tile.position[1]] = self.selected_tile.number

        # Incorrect guess
        else:
            self.selected_tile.user_number = 0

    def _check_mouse_click(self, mouse_pos):
        """Respond to click on tile"""
        tiles_clicked = [
            t for t in self.tiles if t.rect.collidepoint(mouse_pos)
        ]

        if tiles_clicked and not tiles_clicked[0].final:
            # Deselect last clicked tile
            if self.selected_tile:
                self.selected_tile.selected = False

            # Select new tile
            self.selected_tile = tiles_clicked[0]
            self.selected_tile.selected = True

    def _create_board(self):
        """Choose random sudoku and create board along with solution"""
        with open("sudoku.csv", "r") as csv_file:
            sudokus = pd.read_csv("sudoku.csv")
            board_data = sudokus.sample()
            board_str = board_data.iloc[0, 0]
            solution_str = board_data.iloc[0, 1]

        return self._parse_board(board_str), self._parse_board(solution_str)

    def _parse_board(self, board_str):
        """Parse a string to sudoku board"""
        # print(board_str)
        board = [[0 for j in range(9)] for i in range(9)]
        counter = 0
        for i, row in enumerate(board):
            for j, element in enumerate(row):
                board[i][j] = int(board_str[counter])
                counter += 1

        return board

    def _create_tiles(self):
        """Create and place sudoku tiles"""
        for i in range(9):
            for j in range(9):
                x_coord = j * self.settings.tile_size
                y_coord = i * self.settings.tile_size
                number = self.board[i][j]
                tile = Tile(self, (i, j), number)

                if number:
                    tile.final = True

                tile.prep_number()
                self.tiles.add(tile)

    def _draw_separation_lines(self):
        """Draw the board lines that separate tiles"""
        for i in range(9):
            width = 4 if i % 3 == 0 else 1
            pygame.draw.line(
                self.screen,
                (0, 0, 0),
                (self.settings.tile_size*i, 0),
                (self.settings.tile_size*i, self.settings.screen_size),
                width=width
            )
            pygame.draw.line(
                self.screen,
                (0, 0, 0),
                (0, self.settings.tile_size*i),
                (self.settings.screen_size, self.settings.tile_size*i),
                width=width
            )

    def update_screen(self):
        """Draw Sudoku board"""
        self.screen.fill(self.settings.bg_color)

        # Draw tiles
        for tile in self.tiles.sprites():
            tile.draw_tile()

        # Draw board lines
        self._draw_separation_lines()

        # Switch to newest screen
        pygame.display.flip()

if __name__ == '__main__':
    sv = SudokuVisualizer()
    sv.run_visualizer()


# -*- coding: utf-8 -*-
"""
Created on Thu Jul 15 18:18:40 2021

@author: Hengheng Zhang

E-Mail: hengheng.zhang@kit.edu

Function：

"""
import requests
import json
import time
from datetime import datetime
import numpy as np
import pandas as pd
import os
DataOut = []
DateTimes = [datetime.now()]
Count = 0

while True:
    try:
        Channelr =requests.get('http://141.52.172.253/websites/ab/index.php/json/api/getchannels',timeout = 5)
        success = json.loads(Channelr.text)
        Channels=json.loads(Channelr.text)['channels']
        VariableNames = []
        valueids = ''
        for channel in Channels:
            if channel['app'] == 'wsserial':
                VariableNames.append(channel['measurementname'] +'('+channel['unitname'] +')')
                valueids = valueids + channel['valueid']+','
        valueids = valueids[:-1]

        Valuesr =requests.get('http://141.52.172.253/websites/ab/index.php/json/api/getvalues?valueids='+valueids,timeout = 5)
        Values=json.loads(Valuesr.text)['values']
        SenOut = []
        for values in Values:
            SenOut.append(values['value'])
        DateTimesingle =  datetime.strptime(Values[-1]['timestamp'], '%Y-%m-%d %H:%M:%S')
        if not DateTimesingle == DateTimes[-1]:
            DateTimes.append(DateTimesingle)
            DataOut.append(SenOut)
            Count = Count + 1
            print('system is Running')
        if Count == 360:
            DataOut = np.array(DataOut)
            DateTimes = DateTimes[1:]
            Datadf = pd.DataFrame(data = DataOut,index = DateTimes,columns = VariableNames)
            Datadf.index.name = 'DateTime'
            if not os.path.exists('./Data/'):
                os.makedirs('./Data/')
            SaveFileName = 'WS700_'+DateTimes[-1].strftime('%Y_%m_%d_%H_%M_%S')+'.txt'
            Datadf.to_csv('./Data/'+SaveFileName, sep='\t')
            print(str(len(DateTimes))+' recored has been save to file at'+datetime.now().strftime('%Y-%m-%d %H:%M:%S'))

            DataOut = []
            DateTimes = [datetime.now()]
            Count = 0
    except requests.exceptions.RequestException as e:
        if len(DateTimes)>1:
            DataOut = np.array(DataOut)
            DateTimes = DateTimes[1:]
            Datadf = pd.DataFrame(data = DataOut,index = DateTimes,columns = VariableNames)
            Datadf.index.name = 'DateTime'
            if not os.path.exists('./Data/'):
                os.makedirs('./Data/')
            SaveFileName = 'WS700_'+DateTimes[-1].strftime('%Y_%m_%d_%H_%M_%S')+'.txt'
            Datadf.to_csv('./Data/'+SaveFileName, sep='\t')
            print(str(len(DateTimes))+' recored has been save to file at'+datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
            DataOut = []
            DateTimes = [datetime.now()]
            Count = 0
        print('cannot connect to serve')

      # DateTimes = datetime.strptime(value., '')


import sys
import time
import pyspark
from operator import add
from collections import defaultdict, deque
from itertools import combinations
from pyspark import SparkContext, SQLContext, StorageLevel, SparkConf
from pyspark.sql import functions
from graphframes import *

threshold, inpath, btw_outpath, comm_outpath = int(sys.argv[-4]), sys.argv[-3], sys.argv[-2], sys.argv[-1]
start = time.time()
sc = SparkContext(appName="task2")
sc.setLogLevel("WARN")

#graph
def load(x):
    arr = x.split(',')
    return (str(arr[1]), str(arr[0]))
data = sc.textFile(inpath).filter(lambda x : x != ('user_id,business_id'))\
        .map(load).groupByKey().mapValues(list)\
        .map(lambda x: sorted(x[1]))\
        .flatMap(lambda x: [(com, 1) for com in combinations(x, 2)])\
        .reduceByKey(add).filter(lambda x: x[1] >= threshold)\
        .flatMap(lambda x: [(x[0][0], x[0][1]), (x[0][1], x[0][0])])
graph = defaultdict(set)
for n1, n2 in data.collect():
    graph[n1].add(n2)
    graph[n2].add(n1)

#Between
def get_btw(node):
    edgeVal = defaultdict(float)
    q = deque([node])
    parent = defaultdict(set)
    parent[node] = set()
    degree = {node: 0}
    node_val = {}
    mx = 0

    while q:
        n = q.popleft()
        for child in graph[n]:
            if child not in parent:
                parent[child].add(n)
                degree[child] = degree[n]+1
                mx = max(mx, degree[child])
                q.append(child)
            else:
                if degree[child] > degree[n]:
                    parent[child].add(n)

    degree = sorted(list(degree.items()), key=lambda x: x[1])
    q = deque([i[0] for i in degree][::-1])
    path = {node: 1}
    for arr in degree[1:]:
        n = arr[0]
        path[n] = sum([path[p] for p in parent[n]])

    while q:
        cur = q.popleft()
        for par in parent[cur]:
            if par not in node_val:
                node_val[par] = 1
            val = node_val.get(cur, 1)*path[par]/path[cur]
            node_val[par] += val
            pair = tuple(sorted([cur, par]))
            edgeVal[pair] += val

    return edgeVal.items()

nodes = sc.parallelize(graph.keys())
btw = nodes.flatMap(get_btw).reduceByKey(add).map(lambda x: (x[0], x[1]/2))
sort_btw = sorted(btw.collect(), key=lambda x: (-x[1], x[0]))

with open(btw_outpath, 'w') as f:
    for row in sort_btw:
        f.write(str(row[0])+','+str(round(row[1], 5))+'\n')

#Community
def get_community_btw(comm_idx):
    edgeVal = defaultdict(float)
    for node in graph.keys():
        if node not in comm[comm_idx]:
            comm_idx += 1
            comm[comm_idx] = set([node])

        q = deque([node])
        parent = defaultdict(set)
        parent[node] = set()
        degree = {node: 0}
        node_val = {}
        mx = 0

        while q:
            n = q.popleft()
            for child in graph[n]:
                if child not in parent:
                    comm[comm_idx].add(child)
                    parent[child].add(n)
                    degree[child] = degree[n]+1
                    mx = max(mx, degree[child])
                    q.append(child)
                else:
                    if degree[child] > degree[n]:
                        parent[child].add(n)

        degree = sorted(list(degree.items()), key=lambda x: x[1])
        q = deque([i[0] for i in degree][::-1])
        path = {node: 1}
        for arr in degree[1:]:
            n = arr[0]
            path[n] = sum([path[p] for p in parent[n]])

        while q:
            cur = q.popleft()
            for par in parent[cur]:
                if par not in node_val:
                    node_val[par] = 1
                val = node_val.get(cur, 1)*path[par]/path[cur]
                node_val[par] += val
                pair = tuple(sorted([cur, par]))
                edgeVal[pair] += val

    return sorted(list(edgeVal.items()), key=lambda x: (-x[1], x[0]))

def del_high_btw():
    if not sort_btw: return False
    pair, high = sort_btw.popleft()
    graph[pair[0]].remove(pair[1])
    graph[pair[1]].remove(pair[0])

    if not sort_btw: return False
    while sort_btw[0][1] == high:
        pair, high = sort_btw.popleft()
        graph[pair[0]].remove(pair[1])
        graph[pair[1]].remove(pair[0])
        if not sort_btw: return False
    return True

def get_Q():
    def get_sum(pair):
        n1, n2 = pair
        A = 1 if n2 in org_graph[n1] else 0
        return A-(len(graph[n1])*len(graph[n2])/(2*m))

    q_val = sc.parallelize(list(comm.values()))\
               .flatMap(lambda x: [(com, 1) for com in combinations(list(x), 2)])\
               .distinct().map(lambda x: x[0])\
               .map(get_sum).reduce(add)

    return q_val

mxQ = -1
res_comm = {}
comm = {0:set()}
m = len(sort_btw)
org_graph = graph
sort_btw = deque(sort_btw)
while del_high_btw():
    sort_btw = deque(get_community_btw(0))
    temp_q = get_Q()/(2*m)
    if temp_q > mxQ:
        mxQ = temp_q
        res_comm = comm
    comm = {0:set()}

del res_comm[0]
res = []
visited = set()
for idx, arr in res_comm.items():
    arr = sorted(arr)
    temp = arr[0]
    if temp not in visited:
        visited.add(temp)
        res.append(arr)
out = sorted(res, key=lambda x: (len(x), x[0]))
with open(comm_outpath, 'w') as f:
    for arr in out:
        f.write("'" + "', '".join(arr) + "'\n")

print("------------Time: ", time.time()-start)


from __future__ import division
from sklearn.metrics import accuracy_score, precision_score, recall_score

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

class CifarLogisticRegression:
    def __init__(self,
                 trainingDataSet,
                 testingDataSet,
                 learningRate=0.001,
                 tolerance=0.001,
                 maxIterations=1000):
        self.trainingDataSet = trainingDataSet
        self.testingDataSet = testingDataSet
        self.learningRate = learningRate
        self.tolerance = tolerance
        self.maxIterations = maxIterations

    def sigmoid(self, z):
        return 1.0 / (1.0 + np.exp(-z))

    def hypothesis(self, X, theta):
        return self.sigmoid(np.dot(X, theta))

    def cost(self, X, Y, theta):
        probabilities = self.hypothesis(X, theta)

        return (np.multiply(-Y, np.log(probabilities)) - np.multiply(
            (1 - Y), np.log(1 - probabilities))).mean()

    def gradient(self, X, Y, theta):
        prediction = self.hypothesis(X, theta)
        error = prediction - Y

        return np.dot(X.T, error) / X.shape[0]

    def logisticRegression(self, dataSet):
        X = np.matrix(dataSet.iloc[:, :-1])
        Y = np.matrix(dataSet.iloc[:, -1]).T

        attributes = dataSet.shape[1] - 1
        theta = np.matrix(np.random.uniform(-1, 1, attributes)).T

        logisticLoss = [self.cost(X, Y, theta)]
        iterations = 0
        for i in range(self.maxIterations):
            iterations = i + 1
            gradient = self.gradient(X, Y, theta)
            newTheta = theta - self.learningRate * gradient
            newCost = self.cost(X, Y, newTheta)
            if (logisticLoss[i] - newCost < self.tolerance):
                break
            else:
                logisticLoss.append(newCost)
                theta = newTheta

        plt.plot(logisticLoss)
        plt.xlabel('Iteration')
        plt.ylabel('Logistic Loss')
        plt.title('Logistic Regression')
        plt.show()

        return theta, iterations

    def predict(self, X, theta):
        probabilities = self.hypothesis(X, theta)

        return [
            1 if probability >= 0.5 else 0 for probability in probabilities
        ]

    def accuracy(self, dataSet, theta):
        X = np.matrix(dataSet.iloc[:, :-1])
        Y = dataSet.iloc[:, -1]

        prediction = self.predict(X, theta)

        accuracy = accuracy_score(Y, prediction)
        precision = precision_score(Y, prediction, average="weighted")
        recall = recall_score(Y, prediction, average="weighted")

        return accuracy, precision, recall

    def validate(self):
        print("Training Data Set:")
        theta, iterations = self.logisticRegression(self.trainingDataSet)
        print("\tIterations: {}".format(iterations))
        trainAccuracy, trainPrecision, trainRecall = self.accuracy(
            self.trainingDataSet, theta)
        print("\tAccuracy: {}\n\tPrecision: {}\n\tRecall: {}".format(
            trainAccuracy, trainPrecision, trainRecall))

        print("Test Data Set:")
        testAccuracy, testPrecision, testRecall = self.accuracy(
            self.testingDataSet, theta)

        print("\tAccuracy:{}\n\tPrecision:{}\n\tRecall:{}".format(
            testAccuracy, testPrecision, testRecall))

def importData(fileName):
    dataSet = pd.read_pickle(fileName)

    return dataSet

if __name__ == "__main__":
    cifarTrainFileName = 'TrainData.pkl'
    cifarTrainDataSet = importData(cifarTrainFileName)

    cifarTestFileName = 'TestData.pkl'
    cifarTestDataSet = importData(cifarTestFileName)

    for learningRate in [0.1]:
        print("Learning Rate: {}".format(learningRate))
        for tolerance in [0.0001]:
            print("Tolerance: {}".format(tolerance))
            cifarLogisticRegression = CifarLogisticRegression(
                cifarTrainDataSet, cifarTestDataSet, learningRate, tolerance)
            cifarLogisticRegression.validate()


#tutorial/03-run-hello.py
from azureml.core import Workspace, Experiment, Environment, ScriptRunConfig

ws = Workspace.from_config()
experiment = Experiment(workspace=ws, name='ml-workspace')

config = ScriptRunConfig(source_directory='./motion-sense-har', script='hello.py', compute_target='cpu-cluster')

run = experiment.submit(config)
aml_url = run.get_portal_url()
print(aml_url)


# flask imports
from flask import Flask, request, jsonify, render_template, redirect, url_for, make_response
from flask_cors import CORS, cross_origin

# general imports
import json
import requests
from datetime import datetime
import os

# speech imports
import azure.cognitiveservices.speech as speechsdk

# recommender imports
import numpy as np
import pandas as pd
from RecommenderSystem import *
from sklearn.metrics.pairwise import pairwise_distances
import torch

##################################################################################################################################################
# load secrets
with open("secrets.json", "r") as f:
    secrets = json.load(f)

# load keys and urls
getEmbeddingUrl = secrets['azure_albert']
watcher_url = secrets['watcher_url']
watcher_auth_token = secrets["watcher_auth_token"]
MIDDLEWARE_KEY = secrets['MIDDLEWARE_KEY']

# config speech2text
speech_config = speechsdk.SpeechConfig(
    subscription=secrets["S2T_KEY"], region=secrets["S2T_REGION"])
speech_config.speech_recognition_language = "en-US"

# start app
app = Flask(__name__)
CORS(app)

##################################################################################################################################################

# load data for recommender
descriptions = np.array(torch.load(
    "data/avgpooled_albert_sentence_embeddings_float32.pt", map_location=torch.device("cpu")))
description_matrix = np.array(torch.load(
    "data/pca_albert_description.pt", map_location=torch.device("cpu")))
title_matrix = np.array(torch.load(
    "data/pca_albert_title.pt", map_location=torch.device("cpu")))
cast_matrix = np.array(torch.load(
    "data/pca_albert_cast.pt", map_location=torch.device("cpu")))
# df = pd.read_csv("data/data_tmbd_cleaned.csv", delimiter=";", lineterminator="\n")
df = pd.read_pickle("data/df.pkl")
df = df[df.popularity >= list(df.popularity.sort_values(ascending=False))[10000]]

# filter matrices
description_matrix = description_matrix[list(df.index),:]
descriptions = descriptions[list(df.index),:]
title_matrix = title_matrix[list(df.index),:]
cast_matrix = cast_matrix[list(df.index),:]

# build matrices
movie_info = ContentBasedRecommender.build_movie_info_matrix(df)
mapping_matrix = ContentBasedRecommender.build_mapping_matrix(df)

# build input
matrix_dict = {
    'movie_info_matrix': movie_info,
    'description_matrix': descriptions,
    'title_matrix': title_matrix,
    'cast_info_matrix': cast_matrix
}

# build recommender system
RS = ContentBasedRecommender(
    mapping_matrix=mapping_matrix,
    **matrix_dict,
    scaling_kwargs=None
)

# define function for checking balance
def check_balance():
    # send request
    response = requests.get(watcher_url + "?key=" + watcher_auth_token)
    # parse response
    if "true" in response.text:
        return True
    else:
        return False

# prepare dataframe and function for drawing samples
# define top genres
top_genres = ['Adventure', 'Animation', 'Fantasy', 'Action',
              'Sci-Fi', 'Horror', 'Thriller', 'Romance', 'Drama', 'Comedy']

# get most rated movies
top_movies = df.sort_values(by='vote_count', ascending=False).head(1000)

def draw_samples(top_movies: pd.DataFrame, top_genres: list) -> pd.DataFrame:
    '''
    This function draws 15 samples from the top 1000 most-rated movies. Ten samples are drawn from the most popular genres, five more at random.

    Args:
    -----
        top_movies:pd.DataFrame - A pandas dataframe containing the top 1000 most-rated movies
        top_genres:list - List containing all genres to select from

    Returns:
    --------
        random_samples:pd.DataFrame - A pandas dataframe containing fifteen samples

    '''
    # prepare dataframe for samples
    random_samples = pd.DataFrame(columns=top_movies.columns)

    # loop over genres to draw samples
    for genre in top_genres:
        genre_movies = top_movies[top_movies[genre] == 1]
        drawn_sample = genre_movies.sample(n=1)

        # check if sample not already in samples
        while random_samples[random_samples.title == drawn_sample.title.values[0]].shape[0] != 0:
            drawn_sample = genre_movies.sample(n=1)

        random_samples = pd.concat((random_samples, drawn_sample))

    # add five more random samples
    random_samples = pd.concat((random_samples, top_movies.sample(n=5)))

    # postprocess titles
    random_samples.title = random_samples.title.apply(
        lambda x: "The " + x.rstrip(", The") if x.endswith(", The") else x)
    random_samples.title = random_samples.title.apply(
        lambda x: "A " + x.rstrip(", A") if x.endswith(", The") else x)

    return random_samples

##################################################################################################################################################

@app.route('/')
@cross_origin()
def index():
    return '''
Middleware works fine
'''

@app.route("/getRecommendationsByIds", methods=['POST'])
@cross_origin()
def getRecommendationsByIds():
    # get data
    data = request.json

    if not check_balance():
        return {"message": "Cost limit reached"}, 403

    # check key
    if data['key'] != MIDDLEWARE_KEY:
        return {"message": "Forbidden Request"}, 403

    ids = data['ids']
    if type(ids) != list:
        return {"message": "Forbidden Request"}, 403

    # build ratings
    ratings = pd.DataFrame(columns=['userId', 'movieId', 'rating'])
    ratings['movieId'] = pd.Series(ids)
    ratings['userId'] = 21
    ratings['rating'] = 5

    # get recommendation
    recommendation_titles, recommendation_movieIds = RS.get_recommendations_for_user(
        ratings_user=ratings, n_rec=20)

    # prepare paths, title and description
    response = df[df.movieId.isin(recommendation_movieIds)]

    # turn dummy encoding back
    genres = []

    for i in range(response.shape[0]):
        df_tmp = response.iloc[i, :].T[['Adventure', 'Animation', 'Children', 'Comedy', 'Fantasy', 'Romance', 'Drama', 'Action',
                                        'Crime', 'Thriller', 'Horror', 'Mystery', 'Sci-Fi', 'IMAX', 'Documentary', 'War', 'Musical', 'Western', 'Film-Noir']]
        genres.append(list(df_tmp[df_tmp == 1].index))

    response['genres'] = pd.Series(genres)

    # filter columns
    response = response[['title', 'description', 'poster_path',
                         'vote_average', 'actor1', 'actor2', 'actor3', 'year', 'genres']]
    response = response.to_json(orient="index")
    return response

@app.route("/getRandomMovies", methods=['GET'])
@cross_origin()
def getRandomMovies():
    # check balance
    if not check_balance():
        return {"message": "Cost limit reached"}, 403

    print(request.args)
    # authentication
    if request.args.get("key") != MIDDLEWARE_KEY:
        return {"message": "Forbidden Request"}, 403

    # draw samples
    random_recommendations = draw_samples(
        top_movies=top_movies, top_genres=top_genres)

    # turn dummy encoding back
    genres = []

    for i in range(random_recommendations.shape[0]):
        df_tmp = random_recommendations.iloc[i, :].T[['Adventure', 'Animation', 'Children', 'Comedy', 'Fantasy', 'Romance', 'Drama',
                                                      'Action', 'Crime', 'Thriller', 'Horror', 'Mystery', 'Sci-Fi', 'IMAX', 'Documentary', 'War', 'Musical', 'Western', 'Film-Noir']]
        genres.append(list(df_tmp[df_tmp == 1].index))

    # add genres as column to df
    random_recommendations = random_recommendations.copy()
    random_recommendations.loc[:, 'genres'] = pd.Series(genres)

    # filter columns
    random_recommendations = random_recommendations[[
        'title', 'description', 'poster_path', 'vote_average', 'actor1', 'actor2', 'actor3', 'year', 'genres', 'movieId']]
    response = random_recommendations.to_json(orient="index")

    return response

@app.route("/speech2text", methods=['POST'])
@cross_origin()
def speech2text():
    if not check_balance():
        return {"message": "Cost limit reached"}, 403

    if request.method != 'POST' or request.form['key'] != MIDDLEWARE_KEY:
        return {"message": "Forbidden request"}, 403

    if 'audiofile' not in request.files.keys():
        return {"message": "Audiofile missing!"}, 400

    if not request.files['audiofile'].filename.endswith('.wav'):
        return {"message": "Audiofile must be in .wav format!"}, 400

    # tmp store file
    save_path = str(datetime.now()) + "temp.wav"
    request.files['audiofile'].save(save_path)

    # continue processing...
    audio_config = speechsdk.audio.AudioConfig(filename=save_path)
    speech_recognizer = speechsdk.SpeechRecognizer(
        speech_config=speech_config, audio_config=audio_config)

    # get text
    result = speech_recognizer.recognize_once_async().get()
    text = result.text

    print(text)

    # delete tmp file
    os.remove(save_path)

    # make request to azure albert
    r = requests.post(getEmbeddingUrl, json={'text': text})

    print(r)
    data = r.json()

    # transform response
    embedding_flattened, shape = data['embedding'], data['shape']
    embedding = torch.tensor(embedding_flattened).reshape(shape)

    # calculate cosine similarity between descriptions
    distances = pairwise_distances(
        descriptions, embedding.reshape(1, 768), metric='cosine')
    indices = np.argsort(distances, axis=0).tolist()
    indices = indices[:10]

    # get recommendations
    recommendations = pd.DataFrame(columns=df.columns)
    for index in indices:
        recommendations = pd.concat((recommendations, df.iloc[index, :]))

    # turn dummy encoding back
    genres = []

    for i in range(recommendations.shape[0]):
        df_tmp = recommendations.iloc[i, :].T[['Adventure', 'Animation', 'Children', 'Comedy', 'Fantasy', 'Romance', 'Drama', 'Action',
                                               'Crime', 'Thriller', 'Horror', 'Mystery', 'Sci-Fi', 'IMAX', 'Documentary', 'War', 'Musical', 'Western', 'Film-Noir']]
        genres.append(list(df_tmp[df_tmp == 1].index))

    genres = pd.Series(genres)
    genres.index = recommendations.index
    recommendations['genres'] = genres

    # filter columns
    recommendations = recommendations[['title', 'description', 'poster_path',
                                       'vote_average', 'actor1', 'actor2', 'actor3', 'year', 'genres']]
    recommendations = recommendations.to_json(orient="index")
    return recommendations

@app.route("/testSpeech2Text", methods=['GET'])
@cross_origin()
def testSpeech2Text():
    if not check_balance():
        return {"message": "Cost limit reached"}, 403

    # send audio to speech2text
    audio_config = speechsdk.audio.AudioConfig(filename="test.wav")
    speech_recognizer = speechsdk.SpeechRecognizer(
        speech_config=speech_config, audio_config=audio_config)

    # get text
    result = speech_recognizer.recognize_once_async().get()
    text = result.text
    return text

# start application
if __name__ == "__main__":
    app.run(host='0.0.0.0', port=5501)


import os
import requests

from webdriver_manager.chrome import ChromeDriverManager
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.select import Select

from selenium.webdriver.chrome.options import Options
#https://stackoverflow.com/questions/16180428/can-selenium-webdriver-open-browser-windows-silently-in-background

from PBL import application

class Controller:
    def __init__(self):
        self.imagesFolder = os.path.join(application.resourcesFolder, 'images')#PBL/Resources/images
        self.createDriver()

    def createDriver(self):
        #self.driver = webdriver.Chrome(os.path.join(self.resourcesFolder,'chromedriver'))
        self.driver = webdriver.Chrome(ChromeDriverManager().install())

    def closeDriver(self):
         self.driver.close()

    def makeImagesFolder(self):
        if not os.path.isdir(self.imagesFolder):
            os.makedirs(self.imagesFolder)

    def getImage(self,url):
        self.driver.get(url)
        xpath2 = """//li[@class='image item itemNo0 maintain-height selected']//img"""
        xpath = """//li[@class='a-spacing-small item imageThumbnail a-declarative']//img"""
        WebDriverWait(self.driver,10).until(EC.presence_of_element_located(
            (By.XPATH, xpath2)))
        img = self.driver.find_elements_by_xpath(xpath2)[0]
        print(img)
        src = img.get_attribute('src')
        return src

    def download(self,url,productName):
        self.makeImagesFolder()
        response = requests.get(url, stream=True)
        filename = os.path.join(self.imagesFolder, productName+'.png')
        progress = response.iter_content(1024)
        with open(filename, "wb") as f:
            for data in progress:
                # write data read to the file
                f.write(data)

    def getItemPrice(self):
        price_xpath = "//*[@id='priceblock_ourprice']"
        item_price = self.driver.find_element_by_xpath(price_xpath).text
        #item starts with currency symbol $,¥,...
        item_price = item_price[1:]
        print(item_price)
        return item_price

    def getQuantities(self):
        # Click on one time purchase box
        one_time_purchase_div_xpath = "//*[@id='newAccordionRow']"
        one_time_purchase_alternate_xpath = "//*[@id='oneTimeBuyBox']"
        dropdown_menu = None
        try:
            self.naviguate(one_time_purchase_div_xpath, one_time_purchase_alternate_xpath)
        except TimeoutException:
            try:
                one_time_purchase_button = self.driver.find_element_by_id("oneTimeBuyBox").click()
            except:
                #Sometimes there are no boxes to click, the menu is already visible
                quantity_dropdown_menu_xpath = "//*[@id='quantity']"
                dropdown_menu = WebDriverWait(self.driver, 5).until(EC.presence_of_element_located(
                    (By.XPATH, quantity_dropdown_menu_xpath)))

        if not dropdown_menu:
            # Select quantity from dropdown menu
            quantity_dropdown_menu_xpath = "//*[@id='quantity']"
            dropdown_menu = WebDriverWait(self.driver, 5).until(EC.presence_of_element_located(
                (By.XPATH, quantity_dropdown_menu_xpath)))

        options_element = [x for x in dropdown_menu.find_elements_by_tag_name("option")]
        options = []
        for element in options_element:
            options.append(element.get_attribute("value"))

        return options

    def naviguate(self,xpath,alternative_xpath=None):
        try:
            WebDriverWait(self.driver, 10).until(EC.presence_of_element_located(
                (By.XPATH, xpath)))
            self.driver.find_element_by_xpath(xpath).click()
        except TimeoutException:
            WebDriverWait(self.driver, 10).until(EC.presence_of_element_located(
                (By.XPATH, xpath)))
            self.driver.find_element_by_xpath(alternative_xpath).click()

# if __name__ == "__main__":
#     c = Controller()
#     url = "https://www.amazon.co.jp/%E5%92%8C%E5%85%89%E5%A0%82-FQ4-%E3%81%AF%E3%81%98%E3%82%81%E3%81%A6%E3%81%AE%E9%9B%A2%E4%B9%B3%E9%A3%9F-%E8%A3%8F%E3%81%94%E3%81%97%E3%81%8A%E3%81%95%E3%81%8B%E3%81%AA-2-6g%C3%976%E5%80%8B/dp/B0052VL6RI/ref=sr_1_12?__mk_ja_JP=%E3%82%AB%E3%82%BF%E3%82%AB%E3%83%8A&dchild=1&keywords=baby+food&qid=1592376143&sr=8-12"
#     c.getImage(url)
#
#     c.closeDriver()


# Basic script to toggle the LED state on GPIO pin 7
# For Raspberry Pi pin mapping, see:
# https://www.raspberrypi.org/documentation/usage/gpio/README.md
#
# Insipired by:
# https://www.raspberrypi.org/learning/python-quick-reaction-game/worksheet/
#

from gpiozero import LED
from time import sleep

flag = True
led = LED(7)
while True:
    if flag:
        print("on");
        led.on()
    else:
        print("off");
        led.off()
    flag = not flag
    sleep(1)


#!/usr/bin/env python3
from git import Repo, GitCommandError
import subprocess, signal

def kill_all_process():
    p = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)
    out, err = p.communicate()

    for line in out.splitlines():
        line = str(line)
        if 'receiver.py' in line or 'manage.py' in line or 'sender.py' in line or 'homebridge' in line:
            pid = int(line.split()[1])
            os.kill(pid, signal.SIGKILL)

if __name__ == "__main__":
    try:
        g = Repo('.').git
        res = g.pull()
        if res != 'Already up to date.':
            print('updated:   :D\n%s' % res)
            kill_all_process()
            subprocess.call(['/home/pi/BMS-Core/watch'])
    except GitCommandError as exp:
        print('pull exception. because: %s' % exp)


from dataclasses import dataclass
from functools import lru_cache

import numpy as np

from piafedit.model.geometry.point import PointAbs
from piafedit.model.geometry.rect import RectAbs, RawRect
from piafedit.model.geometry.size import SizeAbs
from piafedit.model.libs.operator import Buffer
from piafedit.model.source.data_source import DataSource

@dataclass(frozen=True)
class TileConfig:
    size: SizeAbs

    @property
    def width(self):
        return self.size.width

    @property
    def height(self):
        return self.size.height

class TileSource(DataSource):
    def __init__(self, source: DataSource, config: TileConfig):
        self.source = source
        self.config = config
        self.cache = dict()

    def infos(self):
        return self.source.infos()

    def read(self, window: RectAbs = None, output_size: SizeAbs = None) -> Buffer:
        window = self.update_window(window)
        tiles = self._tiles(window)
        lines = []

        h = len(tiles) * self.config.width
        w = len(tiles[0]) * self.config.height
        b = self.source.infos().bands
        shape = (h, w, b)
        buffer = np.zeros(shape)

        for line in tiles:
            for row in line:
                tile = RectAbs.from_raw(row)
                tile_buffer = self._read_tile(row)
                b = tile.crop(buffer)
                if b.shape == tile_buffer.shape:
                    b[...] = tile_buffer
        return buffer[:window.item_per_page.height, :window.item_per_page.width, ...]

    def write(self, buffer: Buffer, window: RectAbs = None):
        self.source.write(buffer, window)

    def _tiles(self, rect: RectAbs):
        start = self.point_to_tile(rect.pos)
        end = self.point_to_tile(rect.pos.copy().move(*rect.size.raw()))
        lines = [
            [
                RectAbs(PointAbs(x, y), self.config.size).raw()
                for x in range(start.pos.x, end.pos.x, self.config.width)
            ]
            for y in range(start.pos.y, end.pos.y, self.config.height)
        ]
        return lines

    @lru_cache()
    def _read_tile(self, tile: RawRect):
        return self.source.read(RectAbs.from_raw(tile))

    def point_to_tile(self, pos: PointAbs):
        dx = pos.x % self.config.width
        dy = pos.y % self.config.height
        return RectAbs(
            pos=PointAbs(pos.x - dx, pos.y - dy),
            size=self.config.size
        )


import re

from PySide2.QtCore import Qt, QSortFilterProxyModel, QModelIndex
from MightyUseful.PandaTables.AbstractFilter import AbstractFilter

class MultiFilterMode:
    AND = 0
    OR = 1

class MultiFilterProxyModel(QSortFilterProxyModel):
    def __init__(self, *args, **kwargs):
        QSortFilterProxyModel.__init__(self, *args, **kwargs)
        self.filters = {}
        self.multi_filter_mode = MultiFilterMode.AND

    def lessThan(self, source_left: QModelIndex, source_right: QModelIndex) -> bool:
        col = source_left.column()
        if col in [2, 3, 4, 9, 10]:
            leftInt = int(str(source_left.data()).replace(',', ''))
            rightInt = int(str(source_right.data()).replace(',', ''))
            return leftInt < rightInt
        elif col == 0:
            return False
        else:
            return source_left.data() < source_right.data()

    def setFilterByColumn(self, column, regex):
        if isinstance(regex, str):
            regex = re.compile(regex)
        self.filters[column] = regex
        self.invalidateFilter()

    def clearFilter(self, column):
        del self.filters[column]
        self.invalidateFilter()

    def clearFilters(self):
        self.filters = {}
        self.invalidateFilter()

    def filterAcceptsRow(self, source_row, source_parent):
        if not self.filters:
            return True

        results = []
        for key, regex in self.filters.items():
            text = ''
            index = self.sourceModel().index(source_row, key, source_parent)
            if index.isValid():
                text = self.sourceModel().data(index, Qt.DisplayRole)
                if text is None:
                    text = ''
                if isinstance(regex, AbstractFilter):
                    # print("Is filter")
                    results.append(regex.matches(text))
                else:
                    results.append(regex.match(text))

        if self.multi_filter_mode == MultiFilterMode.OR:
            return any(results)
        return all(results)


# %% load in libraries
from bs4 import BeautifulSoup
import pandas as pd
import time
from selenium import webdriver

# %% set up selenium and login
from selenium import webdriver
driver = webdriver.Firefox()
url1 = 'https://freida.ama-assn.org/search/list?spec=43236&page=1'
driver.get(url1)

# %% get program names for each specialty
specialties = pd.read_csv('specialties.csv')
df = pd.DataFrame({'Specialty':[],'School': [], 'Link': []})
for index, row in specialties.iterrows():
    urlbase = 'https://freida.ama-assn.org/search/list?spec=' + str(row[1]) + '&page='
    site = 'https://freida.ama-assn.org'
    for i in range(1,100):
        url = urlbase + str(i)
        # load page
        driver.get(url)
        # wait for it to load
        time.sleep(3)

        #soup page
        soup = BeautifulSoup(driver.page_source,'html.parser')
        # check for blank pages
        for div in soup.find_all('div',{'class':['search-list__count']}):
            count_text = div.text.strip().split()

        else:
            links = []
            for a in soup.find_all('a',{'class':['search-result-card__title']}):
                links += [site + a['href']]
            schools = []
            for a in soup.find_all('a',{'class':['search-result-card__title']}):
                schools += [a.text]
            dict_ = {'Specialty':[row[0]]*len(schools),'School': schools,'Link':links}
            df_it = pd.DataFrame(dict_)
            df = df.append(df_it)

            if int(count_text[5]) == int(count_text[3]):
                break

# %% save program names for each specialty
df.to_csv('names.csv')


from data import dp, db
from aiogram.types import Message, CallbackQuery
from Keyboards.inline_keyboards import beginning

@dp.message_handler(commands=["start"])
async def start_keyboard(message: Message):
    user_id = message.from_user.id
    existing_user = await db.select_user(user_id)
    if not existing_user:
        username = message.from_user.username
        first_name = message.from_user.last_name
        last_name = message.from_user.first_name
        await db.add_user(username=username, user_id=user_id, first_name=first_name, last_name=last_name)
    await message.answer("Нажмите на меню чтобы увидеть меню", reply_markup=beginning)

@dp.callback_query_handler(text="cancel")
async def cancel(call: CallbackQuery):
    await call.answer("Все заказы были отменены!", show_alert=True)


from typing import Optional, List
import logging
import datetime
import pandas as pd
from us_stock_wizard.database.db_utils import StockDbUtils, DbTable

logging.basicConfig(level=logging.INFO)

class FundamentalMeasurements:
    CRITERIA_1 = "cret_1"

class FundamentalAnalyzer:
    """

    Usage:
    >>> analyzer = FundamentalAnalyzer(ticker="AAPL")
    >>> await analyzer.get_fundamental()
    >>> res: bool = analyzer.get_result([FundamentalMeasurements.CRITERIA_1])

    """

    def __init__(self, ticker: str) -> None:
        self.ticker = ticker
        self.db_utils = StockDbUtils()
        self.data: Optional[pd.DataFrame] = pd.DataFrame()

    async def get_fundamental(self) -> None:
        """
        Get fundamental data from database
        """
        fundamental = await StockDbUtils.read(
            DbTable.FUNDAMENTALS, where={"ticker": self.ticker}, output="df"
        )
        self.data = fundamental

    def analyze(self) -> pd.DataFrame:
        """
        Analyze fundamental data
        """
        if self.data.empty:
            logging.info("No fundamental data found")
            return
        fundamental = self.data.copy()

        # Handle Quarterly Report only
        fundamental["reportDate"] = pd.to_datetime(fundamental["reportDate"])
        fundamental = fundamental[fundamental["reportType"] == "QUARTERLY"]
        fundamental = fundamental.sort_values(by="reportDate", ascending=True)

        # Calculate the YoY increasement
        fundamental["sales_YoY"] = fundamental["sales"].pct_change(periods=4)
        fundamental["netIncome_YoY"] = fundamental["netIncome"].pct_change(periods=4)
        fundamental["grossMarginRatio_YoY"] = fundamental[
            "grossMarginRatio"
        ].pct_change(periods=4)

        # Calculate the QoQ increasement
        fundamental["sales_QoQ"] = fundamental["sales"].pct_change(periods=1)
        fundamental["netIncome_QoQ"] = fundamental["netIncome"].pct_change(periods=1)
        fundamental["grossMarginRatio_QoQ"] = fundamental[
            "grossMarginRatio"
        ].pct_change(periods=1)

        return fundamental

    def get_result(self, criteria: List[FundamentalMeasurements]) -> bool:
        """
        Get result of fundamental analysis
        """
        if self.data.empty:
            return False
        combined_bool = []
        for cret in criteria:
            if cret == FundamentalMeasurements.CRITERIA_1:
                _ = self.cret_1()
                combined_bool.append(_)
            else:
                raise ValueError(f"Unknown criteria: {cret}")
        return all(combined_bool)

    def cret_1(self) -> bool:
        """
        Criteria
        1. Sales YoY growth rate > 30%
        """
        try:
            data = self.analyze()
            if data.empty:
                logging.warning("No fundamental data found")
                return False

            sales_yoy = data.iloc[-1]["sales_YoY"] > 0.3
            # Disable for now
            # netIncome_yoy = data.iloc[-1]["netIncome_YoY"] > 0.2

            _ = sales_yoy
            return _
        except Exception as e:
            logging.error(e)
            return False

class FundamentalScreener:
    """
    Screen a list of stocks based on fundamental analysis

    Example:
        >>> screener = FundamentalScreener()
        >>> await screener.initialize()
    """

    def __init__(self) -> None:
        self.stocks_df = pd.DataFrame()

        self.preferred_keywords = [
            "tech",
            "bio",
            "health",
            "pharm",
            "software",
            "medic",
            "consumer",
        ]
        self.blacklist_keywords = [
            "bank",
            "insurance",
            "estate",
            "oil",
            "energy",
            "gas",
            "power",
        ]

    async def initialize(self):
        await self.get_stocks()

    async def get_stocks(self) -> None:
        """
        Get stocks!
        """
        all_stocks = await StockDbUtils.read(table=DbTable.TICKERS, output="df")
        all_stocks = all_stocks[all_stocks["ticker"].str.len() < 5]
        self.stocks_df = all_stocks

    def filter_preferred(self) -> List[str]:
        """
        Return the preferred stock list
        """
        if self.stocks_df.empty:
            raise ValueError("No stocks in the dataframe!")
        df = self.stocks_df.copy()
        df["joint_kw"] = df["sector"] + " " + df["industry"]
        df["joint_kw"] = df["joint_kw"].str.lower()
        pattern = "|".join(self.preferred_keywords)
        df["contains_preferred"] = df["joint_kw"].str.contains(pattern)
        df = df[df["contains_preferred"] == True]
        return df["ticker"].tolist()

    def filter_blacklist(self) -> List[str]:
        """
        Return the blacklist stock list
        """
        if self.stocks_df.empty:
            raise ValueError("No stocks in the dataframe!")
        df = self.stocks_df.copy()
        df["joint_kw"] = df["sector"] + " " + df["industry"]
        df["joint_kw"] = df["joint_kw"].str.lower()
        pattern = "|".join(self.blacklist_keywords)
        df["contains_blacklist"] = df["joint_kw"].str.contains(pattern)
        df = df[df["contains_blacklist"] == True]
        return df["ticker"].tolist()

    async def filter_fundamentals(
        self,
        crets: List[FundamentalMeasurements] = [FundamentalMeasurements.CRITERIA_1],
    ) -> List[str]:
        """
        Filter stocks based on fundamental analysis
        """
        if self.stocks_df.empty:
            raise ValueError("No stocks in the dataframe!")
        df = self.stocks_df.copy()
        tickers = df["ticker"].tolist()
        res = []
        for ticker in tickers:
            _: bool = False
            try:
                analyzer = FundamentalAnalyzer(ticker=ticker)
                await analyzer.get_fundamental()
                _ = analyzer.get_result(crets)
            except Exception as e:
                logging.error(f"Analyse {ticker} failed: {e}")
            if _:
                res.append(ticker)
        return res


import logging
from bs4 import BeautifulSoup
from logging import StreamHandler
from logging.handlers import RotatingFileHandler
from utils.get_pages import get_page, get_page_selenium
from utils.helpers import remove_whitespace

# https://youtrack.jetbrains.com/issue/PY-39762
# noinspection PyArgumentList
logging.basicConfig(
    format="%(asctime)s %(funcName)20s() %(levelname)-8s  %(message)s",
    level=logging.INFO,
    datefmt="%Y-%m-%d %H:%M:%S",
    handlers=[
        RotatingFileHandler(
            filename="logs/pre-order-tracker.log",
            mode="a",
            maxBytes=10485760,
            backupCount=3,
        ),
        StreamHandler(),
    ],
)

logger = logging.getLogger(__name__)

def scrape_walmart_canada(path):
    logger.info(f"Scraping Walmart Canada availability for {path}")

    OUT_OF_STOCK_ONLINE = "Out of stock online"

    url = f"https://www.walmart.ca/{path}"

    availability = {
        "available": False,
        "availability_description": None,
        "company": "Walmart Canada",
        "product_title": None,
        "url": url,
    }

    # get html
    walmart_page = get_page_selenium(url)

    try:
        soup = BeautifulSoup(walmart_page, "html.parser")

        # get relevant text
        raw_product_title = soup.find(
            attrs={"data-automation": "product-title"}
        ).get_text()
        raw_availability_description = soup.find(
            attrs={"data-automation": "online-only-label"}
        ).get_text()

        # clean up text
        product_title = remove_whitespace(raw_product_title)
        availability_description = remove_whitespace(raw_availability_description)

        availability["product_title"] = product_title
        availability["availability_description"] = availability_description

        if availability_description != OUT_OF_STOCK_ONLINE:
            availability["available"] = True
    except Exception as error:
        logger.error(error)

    return availability