# import kubernetes
#
# kubernetes.config.load_kube_config()
#
# v1 = kubernetes.client.CoreV1Api
# pods = v1.list_pod_for_all_namespaces(watch=False)
# for i in pods.items():
#     print(i)

from kubernetes import config, client,watch

config.load_config()
pods = client.CoreV1Api().list_pod_for_all_namespaces()
for i in pods.items:
    print(i.metadata.name)


"""Module with the classes that manage the authorStats
and paperStats plots.

This file is part of the physbiblio package.
"""
import datetime
import traceback

import matplotlib
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.backends.backend_qtagg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.lines import Line2D
from matplotlib.patches import Rectangle
from PySide6.QtCore import Qt
from PySide6.QtGui import QFont
from PySide6.QtWidgets import QDialog, QGridLayout, QLineEdit, QPushButton

plt.switch_backend("QtAgg")

try:
    from physbiblio.config import pbConfig
    from physbiblio.database import pBDB
    from physbiblio.gui.basicDialogs import askDirName, infoMessage
    from physbiblio.gui.commonClasses import PBDialog, PBLabel
    from physbiblio.gui.errorManager import pBGUILogger
    from physbiblio.inspireStats import pBStats
    from physbiblio.strings.gui import InspireStatsGUIStrings as igstr
except ImportError:
    print("Could not find physbiblio and its modules!")
    print(traceback.format_exc())

figTitles = [
    igstr.paperNumber,
    igstr.paperYear,
    igstr.totalCitations,
    igstr.citationsYear,
    igstr.meanCitations,
    igstr.citationsPaper,
]

class AuthorStatsPlots(PBDialog):
    """Class that constructs a window to show
    the results of `authorStats`
    """

    def __init__(self, figs, title=None, parent=None):
        """Constructor.

        Parameters:
            figs: the list of figures
            title (optional): the window title
            parent (default None): the parent object,
                which should have a property `lastAuthorStats`
        """
        super(AuthorStatsPlots, self).__init__(parent)
        self.figs = figs
        self.canvas = None
        if title is not None:
            self.setWindowTitle(title)
        layout = QGridLayout(self)
        layout.setSpacing(1)
        self.setLayout(layout)
        self.updatePlots()

        nlines = int(len(figs) / 2)
        self.layout().addWidget(PBLabel(igstr.linesMoreInfo), nlines + 1, 0)

        try:
            hIndex = "%d" % self.parent().lastAuthorStats["h"]
        except (TypeError, AttributeError):
            hIndex = igstr.hIndexE
        self.hIndex = PBLabel(igstr.hIndexV % hIndex)
        largerFont = QFont("Times", 15, QFont.Bold)
        self.hIndex.setFont(largerFont)
        self.layout().addWidget(self.hIndex, nlines + 1, 2, 1, 2)

        self.textBox = QLineEdit("")
        self.textBox.setReadOnly(True)
        self.layout().addWidget(self.textBox, nlines + 2, 0, 1, 4)

        self.allcits = np.array(pBStats.allCitations)
        self.layout().addWidget(PBLabel(igstr.yearsForAvgY), nlines + 3, 0)
        defyrs = "3"
        self.yearsBox = QLineEdit(defyrs)
        self.yearsBox.textChanged.connect(self.onUpdateYears)
        self.layout().addWidget(self.yearsBox, nlines + 3, 1)
        self.layout().addWidget(PBLabel(igstr.yearsForAvgC), nlines + 3, 2)
        self.citperyearBox = QLineEdit("")
        self.citperyearBox.setReadOnly(True)
        self.layout().addWidget(self.citperyearBox, nlines + 3, 3)

        self.saveButton = QPushButton(igstr.save, self)
        self.saveButton.clicked.connect(self.saveAction)
        self.layout().addWidget(self.saveButton, nlines + 4, 0, 1, 2)

        self.clButton = QPushButton(igstr.close, self)
        self.clButton.clicked.connect(self.onClose)
        self.clButton.setAutoDefault(True)
        self.layout().addWidget(self.clButton, nlines + 4, 2, 1, 2)

        self.onUpdateYears(defyrs)

    def onUpdateYears(self, string):
        """When the number of years is changed,
        update the text reporting
        the average citations per year and total citations
        in the given amount of years

        Parameters:
            string: the new number of years to consider
        """
        try:
            yrs = int(string)
        except ValueError:
            self.citperyearBox.setText(igstr.yearsForAvgI)
            return
        if len(self.allcits) == 0:
            self.citperyearBox.setText(igstr.yearsForAvgT % (0, 0, yrs))
        else:
            today = datetime.datetime.now(self.allcits[0].tzinfo)
            ref = today.replace(year=today.year - yrs)
            tot = len(np.where(self.allcits >= ref)[0])
            self.citperyearBox.setText(igstr.yearsForAvgT % (tot / yrs, tot, yrs))

    def onClose(self):
        """Close dialog"""
        QDialog.close(self)

    def saveAction(self):
        """Save the plot into a file,
        after asking the directory where to save them
        """
        savePath = askDirName(self, igstr.whereSavePlots)
        if savePath != "":
            try:
                self.parent().lastAuthorStats["figs"] = pBStats.plotStats(
                    author=True, save=True, path=savePath
                )
            except AttributeError:
                pBGUILogger.warning("", exc_info=True)
            else:
                infoMessage(igstr.plotsSaved)
                self.saveButton.setDisabled(True)

    def onPress(self, event):
        """Print the plot coordinates where a click was performed.
        To be connected through `mpl_connect`.
        Used for testing.

        Parameter:
            a `matplotlib.backend_bases.Event`
        """
        print(event.xdata, event.ydata)

    def pickEvent(self, event):
        """Save into `self.textBox` the coordinates
        of the clicked object (`Line2D` or `Rectangle`) in the plot.

        Parameter:
            a `matplotlib.backend_bases.Event`
        """
        ob = event.artist
        ix = -1
        for i, f in enumerate(self.figs):
            if f == ob.figure:
                ix = i
        if isinstance(ob, Rectangle):
            self.textBox.setText(
                igstr.xInYearIs % (figTitles[ix], int(ob.get_x()), int(ob.get_height()))
            )
        elif isinstance(ob, Line2D):
            xdata = ob.get_xdata()
            ydata = ob.get_ydata()
            ind = event.ind
            if ix == 4:
                formatString = igstr.xInDateIsF
            else:
                formatString = igstr.xInDateIsD
            self.textBox.setText(
                formatString
                % (
                    figTitles[ix],
                    np.take(xdata, ind)[0].strftime(igstr.datefmt),
                    np.take(ydata, ind)[0],
                )
            )

    def updatePlots(self):
        """Reset the dialog window removing all the previous items
        and create new canvas for the figures.
        """
        i = 0
        while True:
            item = self.layout().takeAt(i)
            if item is None:
                break
            del item
        if hasattr(self, "canvas"):
            del self.canvas
        self.canvas = []
        for fig in self.figs:
            if fig is not None:
                self.canvas.append(FigureCanvas(fig))
                self.layout().addWidget(self.canvas[-1], int(i / 2), (i % 2) * 2, 1, 2)
                self.canvas[-1].mpl_connect("pick_event", self.pickEvent)
                self.canvas[-1].draw()
                i += 1

class PaperStatsPlots(PBDialog):
    """Class that constructs a window
    to show the results of `paperStats`
    """

    def __init__(self, fig, title=None, parent=None):
        """Constructor.
        Defines some properties and buttons

        Parameters:
            fig: the figure to be shown
            title (optional): the window title
            parent (default None): the parent object,
                which should have a property lastPaperStats
        """
        super(PaperStatsPlots, self).__init__(parent)
        self.fig = fig
        self.canvas = None
        if title is not None:
            self.setWindowTitle(title)
        layout = QGridLayout(self)
        layout.setSpacing(1)
        self.setLayout(layout)
        self.updatePlots()

        nlines = 1
        self.layout().addWidget(PBLabel(igstr.lineMoreInfo), nlines + 1, 0)

        self.textBox = QLineEdit("")
        self.textBox.setReadOnly(True)
        self.layout().addWidget(self.textBox, nlines + 2, 0, 1, 2)
        self.saveButton = QPushButton(igstr.save, self)
        self.saveButton.clicked.connect(self.saveAction)
        self.layout().addWidget(self.saveButton, nlines + 3, 0)

        self.clButton = QPushButton(igstr.close, self)
        self.clButton.clicked.connect(self.onClose)
        self.clButton.setAutoDefault(True)
        self.layout().addWidget(self.clButton, nlines + 3, 1)

    def onClose(self):
        """Close dialog"""
        QDialog.close(self)

    def saveAction(self):
        """Save the plot into a file,
        after asking the directory where to save them
        """
        savePath = askDirName(self, igstr.whereSavePlot)
        if savePath != "":
            try:
                self.parent().lastPaperStats["fig"] = pBStats.plotStats(
                    paper=True, save=True, path=savePath
                )
            except AttributeError:
                pBGUILogger.warning("", exc_info=True)
            else:
                infoMessage(igstr.plotSaved)
                self.saveButton.setDisabled(True)

    def pickEvent(self, event):
        """Save into `self.textBox` the coordinates
        of the clicked object (`Line2D`) in the plot.

        Parameter:
            a `matplotlib.backend_bases.Event`
        """
        ob = event.artist
        ix = -1
        if isinstance(ob, Line2D):
            xdata = ob.get_xdata()
            ydata = ob.get_ydata()
            ind = event.ind
            formatString = igstr.xInDateIsD
            self.textBox.setText(
                formatString
                % (
                    igstr.citations,
                    np.take(xdata, ind)[0].strftime(igstr.datefmt),
                    np.take(ydata, ind)[0],
                )
            )

    def updatePlots(self):
        """Reset the dialog window removing all the previous items
        and create a new canvas for the figure.
        """
        i = 0
        while True:
            item = self.layout().takeAt(i)
            if item is None:
                break
            del item
        if hasattr(self, "canvas"):
            del self.canvas
        if self.fig is not None:
            self.canvas = FigureCanvas(self.fig)
            self.layout().addWidget(self.canvas, 0, 0, 1, 2)
            self.canvas.mpl_connect("pick_event", self.pickEvent)
            self.canvas.draw()


####
# Le parser permet d'aller chercher les éléments qui nous interessent dans la page web

####
from bs4 import BeautifulSoup
import bs4
from itertools import chain
from selenium.webdriver.firefox.webdriver import WebDriver
from selenium.common.exceptions import StaleElementReferenceException, ElementNotInteractableException, \
    ElementNotSelectableException, ElementNotVisibleException
from selenium.webdriver.common.keys import Keys
import traceback
import time
import re

from webcrawler.utils import reorgPaquetGenerator
from webcrawler.log import parse_log
from webcrawler.mapper import mapp
from webcrawler.utils import xpath

class Parse():

    webdriver = WebDriver
    PAUSE = 4
    DOCSTRING_LIST_WEBELEMENT = re.compile('^.*:Returns:.*- (list of WebElement).*$')
    DOCSTRING_WEBELEMENT = re.compile('^.*:Returns:.*- (WebElement).*$')

    def __init__(self, session):
        '''
        Le parser permet de parser une page web en fonction d'une liste de balise et de retourner une selection
        nous pouvons soit parser une liste de balise, soit une seule.
        voir les méthodes : parse et list_parse
        :param session: l'objet session
        '''

        parse_log.debug("Nous parsons la page {}".format(type(session)))
        if isinstance(session, self.webdriver):
            parse_log.debug("Recherche de type selenium")
            self.typeRecherche = self.rechercheSelenium
            self.page = session
        elif isinstance(session, str):
            parse_log.debug("Recherche de type BeautifulSoup")
            self.typeRecherche = self.rechercheBF
            self.page = BeautifulSoup(self.page, 'html.parser')

    def list_parse(self, list_baliz ):
        '''
        Permet de parser une liste de balise en appelant la methode parse

        :param list: parse une liste de kwargs
        :return:

        List_resultat
        '''
        return [self.parse(**baliz) for baliz in list_baliz] if list_baliz else self.page

    def __mapp(self, orig, results):
        '''
        Méthode permettant de mettre en relation les résultats avec les attributs finaux demandés
        dans le dictionnaire de données
        :param orig:
        :param results:
        :return:
        '''
        assert isinstance(orig, dict) == True
        assert isinstance(results, dict) == True
        try:
            return [(v, results[k]) for k, v in orig.items()]
        except(KeyError):
            print("le mapping est incorrect \n-orig : {0}\n-results : {1}".format(orig, results))

    def __getBFmethod(self, resu, cle):
        '''
        Méthode permettant d'accéder aux méthodes de beautifulSoup
        :param resu:
        :param cle:
        :return:
        '''
        if hasattr(resu, cle):
            return getattr(resu, cle)
        return

    def __getSeleniumMethod(self, resu, cle):
        '''
        Méthode permettant d'accéder aux méthodes de beautifulSoup
        :param resu:
        :param cle:
        :return:
        '''
        print(set(map(type, resu)), " : " , cle)
        if hasattr(resu, cle):
            #print('l\'element {} possede la methode {}'.format(resu, cle))
            return getattr(resu, cle)
        else:
            return getattr(self.page, cle)
        return

    def resultSetIter(self, resultSet):
        for result in resultSet:
            yield result
    #
    # def rechercheSelenium(self, motif, element):
    #     continue
    # def rechercheSelenium(self, motif, element):

    @xpath
    def rechercheBF(self, motif, element):
        '''
        Recherche BF est un generateur permettant de lancer
        de manière recursive des coroutines de recherche beautiful soup

        :param motif: tuple typeRecherche, valeursDeRecherche
        :param element: bs4.BeautifulSoup or bs4.element.ResultSet
        :param objetRetour: Variable a laquelle assigner une liste de resultat permettant de
        continuer la recherche
        :return: resultat de recherche
        '''

        typeRecherche, valeursDeRecherche = list(motif.items())[0]
        parse_log.debug("Nous sommes dans la fonction {0} valeursDeRecherche:{1}, element: {2} ".format(typeRecherche, valeursDeRecherche, type(element)))

        try:
            if isinstance(element, bs4.element.ResultSet) or isinstance(element, list):
                parse_log.debug('BeautifulSoup ! 1 ' + type(element) + 'typeRecherche : ' + typeRecherche)
                obj = [ self.rechercheBF(motif, result) for result in element ]
                objetRetour = obj if not isinstance(obj[0], list) else list(chain.from_iterable(obj))#list(chain.from_iterable(obj))
            elif isinstance(element, bs4.BeautifulSoup) or isinstance(element, bs4.element.Tag):
                parse_log.debug('BeautifulSoup ! 2 ' + str(type(element)) + 'typeRecherche : ' + typeRecherche ) #"" element bs :" + str(element))
                if isinstance(valeursDeRecherche, dict):
                    objetRetour = self.__getBFmethod(element, typeRecherche)(**valeursDeRecherche)
                if isinstance(valeursDeRecherche, str):
                    objetRetour = self.__getBFmethod(element, typeRecherche)(valeursDeRecherche)
            return objetRetour
        except(Exception) as e:
            print("Nous sommes dans une exception du parseur {}".format(e))
            raise
        #print(" !!! \n\n Nous sommes dans un cas special ", type(element))

    # def infiniteScrollSearch(self, item, action, args=None):
    #     '''
    #     La methode infinite scroll search permet de faire des recherches d'éléments
    #     en faisant un scroll infini dans toutes la page
    #     :param item:
    #     :param action:
    #     :param args:
    #     :return:
    #     '''
    #     pause = 4 ## 4 secondes de pause pour le chargement
    #     onContinue = True
    #     inter = set([])
    #     ## Nous remontons en haut de la page
    #     self.page.execute_script("window.scrollTo(0, 0)")
    #
    #     #print('nous cherchons dans les éléments {}'.format(type(item), item.text if type(item) != type(self.page) else ''))
    #     while onContinue:
    #         try:
    #             trouve = self.seleniumRechercheBase(item, action, args)
    #             if trouve:
    #                 inter.update(set(trouve))
    #             lastHeight = self.page.execute_script("return document.body.scrollHeight")
    #             self.page.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    #             time.sleep(self.PAUSE)
    #             newHeight = self.page.execute_script("return document.body.scrollHeight")
    #             if newHeight == lastHeight:
    #                 onContinue = False
    #             lastHeight = newHeight
    #
    #         except(Exception) as e:
    #             print(e, traceback.format_exc())
    #             raise
    #
    #     return list(inter)

    # def rechercheNestedSelenium(self, selection, resultats):
    #     '''
    #     Permet d'effecteur une recherche sur les éléments inclus dans des recherches
    #     :param selection:
    #     :param resultats:
    #     :return:
    #     '''
    #
    #     #select = (i for i in selection)
    #     resultat = resultats
    #     parse_log.debug("resultats nested: " + str(resultat))
    #     # try:
    #     for item in resultat:
    #         print("Nous sommes à l'item nested {}".format(item))
    #         select = (i for i in selection)
    #         resultat_inter = ''
    #         while select:
    #             print("PETITE PAUSE \n\n")
    #             time.sleep(self.PAUSE)
    #             try:
    #                 action, args = next(select)
    #                 print("Nous faisons l'action {} {} ".format(action, args))
    #                 if action == "nested":
    #                     self.rechercheNestedSelenium(args, resultat if resultat else None)
    #                 else:
    #                     trouve = self.infiniteScrollLocalize(item, action, args)
    #                 if trouve:
    #                     resultat_inter += trouve
    #             except(TypeError) as e:
    #                 print(e, traceback.format_exc())
    #                 raise
    #             except(StaleElementReferenceException):
    #                 inter = self.infiniteScrollLocalize(item, action, args)
    #                 if not inter:
    #                     print("\n ## Nous n'avons pas trouvé l'élément \n ")
    #             except(StopIteration):
    #                 print('Nous avons chopé l\'exception de sortie')
    #                 break
    #
    #     ## Retour à la page principale
    #     self.page.switch_to_window(self.pagePrincipale)
    #
    #     return resultat_inter

    def scroll(self, lastHeight):
        '''
        Permet le scroll
        :return:
        '''
        try:
            self.page.execute_script("window.scrollTo(0, document.documentElement.scrollHeight);")
            #self.page.implicitly_wait(self.PAUSE)
            time.sleep(self.PAUSE)
            newHeight = self.page.execute_script("return document.documentElement.scrollHeight")
            if newHeight == lastHeight:
                self.page.execute_script("window.scrollTo(0, 0)")
                return False, newHeight
            return True, newHeight
        except(Exception) as e:
            print(e, traceback.format_exc())
            raise

    def seleniumRechercheBase(self, item, action, args=None):

        try:
            methodeSelenium = self.__getSeleniumMethod(item, action)
        except(Exception) as e:
            print(e, traceback.format_exc())
            print("L'objet {} n'a pas de méthode {}".format(item, action))
            raise
        return methodeSelenium(args) if args else methodeSelenium()

    def infiniteScrollLocalize(self, item, action, args=None):
        '''
        Methode permettant de rechercher un élément dans la page en scrollant
        :param item: element selenium
        :param action: Methode selenium a appliquer
        :param args: Arguments accompagnant la methode selenium
        :return:
        '''

        onContinue = True
        inter = set([])
        try:
            itemDoc = item.__doc__.replace('\n', '')
        except(AttributeError):
            itemDoc = item.__doc__
        finally:
            if not itemDoc:
                itemDoc = ''

        ## Nous remontons en haut de la page
        self.page.execute_script("window.scrollTo(0, 0)")
        #print("Remontée en haut de page")
        #print('nous cherchons dans l\'élément {} l\'élément {}'.format(type(item), item.text if type(item)!=type(self.page) else '' ))
        lastHeight = self.page.execute_script("return document.documentElement.scrollHeight")
        while onContinue:
            try:
                trouve = [self.__mapp(self.resultat, {cle: item.get_attribute(cle) or getattr(item, cle, '') if cle != 'text' else item.getText().strip() if hasattr(item, 'getText') else item.text
                                                       for cle in self.resultat.keys()}) for item in self.seleniumRechercheBase(item, action, args) ]

                #print(trouve, self.DOCSTRING_LIST_WEBELEMENT.match(itemDoc), self.DOCSTRING_WEBELEMENT.match(itemDoc))
                if (self.DOCSTRING_LIST_WEBELEMENT.match(itemDoc) or itemDoc is '') and trouve:
                    for item in trouve:
                        inter.update(item)
                elif self.DOCSTRING_WEBELEMENT.match(itemDoc) and trouve:
                    return trouve
                elif item in ('click', 'drag_and_drop'):
                    return None
            except(Exception) as e:
                print(e, traceback.format_exc())
                raise

            onContinue, newHeight = self.scroll(lastHeight)
            lastHeight = newHeight

        #print("List inter :", [i.tag_name for i in inter])
        return list(inter) if inter else None

    def rechercheSelenium(self, selection, resultats=None):
        """
        Nous chainons les methodes selenium les unes aux autres
        Plusieurs recherches différentes peuvent être réalisées
        -1 recherche classiques de plusieurs éléments
        -2 Recherches classique puis à partir des cette dernière -- > recherche d'éléments imbriqués -
        cad que nous allons aller vers un groupe d'éléments puis revenir au principal

        :param methodes: Dictionnaires des méthodes
        :return:
        """
        parse_log.debug('Nous chainons les méthodes selenium')
        # methodes_chaines = "self.page."+ ".".join(["{}(**{})".format(func, args) if isinstance(args, dict) else \
        #     "{}({})".format(func, "'{}'".format(str(args)) if args else '')  for item in selection for func, args in item.items()])
        # parse_log.debug(methodes_chaines)

        ## Page principale de référence
        self.pagePrincipale = self.page.current_window_handle

        # nous faisons un generateur
        select = (i for i in selection)

        if not resultats:
            action_prem, args = next(select)
            resultat = self.infiniteScrollLocalize(self.page, action_prem, args)
        else:
            resultat = resultats
        # try:

        while select:
            resultat_inter = ''
            try:
                action, args = next(select)
                parse_log.debug("action args : {} {}".format(action, args))

                if isinstance(resultat, list) or isinstance(resultat, set):
                    resultat_inter = set([])
                    #parse_log.debug('Nous avons plusieurs resultats :\n{} {}'.format(args, str([i.text for i in resultat])))
                    for result_uniq in resultat:
                        try:
                            resultat_inter.update(self.infiniteScrollLocalize(result_uniq, action, args))
                        except(TypeError):
                            resultat_inter.update([])
                            ## si la liste est vide, nous ne pouvons pas itérer dessus
                elif isinstance(resultat, self.page._web_element_cls):
                    parse_log.debug(
                        'Nous avons un resultat unique :\n{} {}'.format(args, resultat.text))
                    resultat_inter = self.infiniteScrollLocalize(self.page, action, args)
                resultat = resultat_inter if resultat_inter else resultat
                parse_log.debug(
                    'Resultat :\n{}'.format(resultat))
            except(StopIteration):
                parse_log.debug(
                    '''Signal d\'arrêt d\'iteration' :{}\n{}'''.format(args, resultat))
                resultat = resultat_inter if resultat_inter else resultat
                break
            except(Exception) as e:
                print(e, traceback.format_exc())
                raise

        return resultat
        # except(Exception) as e:
        #     print(e, traceback.format_exc())
        #     raise

    def parse(self, **kwargs):

        '''

        Permet de faire de recherche de balise et de les retourner sous forme de dictionnaire
        :param objet: recherche un objet contenu dans une balise
               kwargs : {'selection':{'type':None, 'classe':None, 'value':None, 'regex':None},
                         'resultat':{'text':'', 'attr':['liste des attributs']}}}
               Le selection permet de fair eune selection des balises et le résultat permet
               de faire remonter les données attendues
        :return:

        '''

        # try:
        #     assert type(self.page)==str, "La page doit être de type string"
        # except AssertionError:
        #     return

        selection, self.resultat, with_parents = kwargs['selection'].copy(), kwargs['results'].copy(),\
                                  kwargs.get('with_parents', None)

        #page_bf = self.soup
        result = []
        parse_log.debug("selection :\n" + str(selection))
        #parse_log.debug("page affichée :\n" + self.page)
        #     ##faire un iterateur qui renvoi une exception en cas de fin d'iteration
        self.result_partiel = None
        if isinstance(self.page, self.webdriver):
            self.result_partiel = self.rechercheSelenium(selection)
        elif isinstance(self.page, str):
            for selectionMotif in selection:
                ## Si il existe un resultat partiel alors celui-ci est affecté à l'élément sinon
                ## nous prennons la page bf4
                element = self.result_partiel if self.result_partiel else self.page
                #print("Objet en retour dans boucle des motifs {}, result partiel : {}".format(selectionMotif, type(self.result_partiel)))
                self.result_partiel = self.typeRecherche(selectionMotif, element)

        #Permet de renvoyer des résultats non dupliqués si des balises similaires ressortent
        if kwargs.get('duplicates', None):
            #TODO tester si cela passe au niveau du duplicates
            for item in self.result_partiel:
                if not item in result:
                    result.append(item)
        else:
            result = self.result_partiel

        # for i in result:
        #     print(type(i), i, len(i))
            # for g in i:
            #     print(type(g), g)
        ## TODO : tester avec betclick pour que l'on retourne toutes les valeurs des matchs et developper le mapping fields pour que nous puissions injecter directement avec django
        ##
        if result:
            # print(result)
            #parse_log.debug("resultat du parseur" + str(result) +str([i.__dict__ for i in result])+ " type : "+ str(type(result)))
            #parse_log.debug("{}".format([[(item.__dict__, item.get(cle)) if cle != 'text' else item.getText().strip() for cle in resultat.keys()] for item in result ]))
            # resultat = [self.__mapp(resultat, {cle: item.get_attribute("href") or getattr(item, cle, '') if cle != 'text' else item.getText().strip() if hasattr(item, 'getText') else item.text
            #                                            for cle in resultat.keys()}) for item in result ]
            print(self.resultat)
            return self.resultat
        else:
            parse_log.debug("Le parseur ne trouve pas de résultat")
            return

    def getList(self, list_baliz):
        '''
        Permet d'obtenir une liste des balises demandées
        :param list_baliz:
        :return:
        '''
        recherche = self.list_parse(list_baliz)
        resultante = min([len([item.values() for item in  item_recherche if item.values()]) for item_recherche in recherche])
        return list(zip(*[[j  for j in reorgPaquetGenerator(i, resultante) if j ] for i in recherche]))

class rechercheSelenium():

    '''
    Permet de faire une recherche en chaine d'éléments selenium
    la recherche selenium produit des objets chaineDeRecherche
    '''

    def __init__(self, page, recherche):
        pass

class chaineDeRecherche():
    '''
    La chaîne de recherche permet d'avoir un objet contenant tous les éléments d'une chaîne de recherche
    Le chaine permet de faire toute la recherche en une fois. CAD nous allons tout de suite a l'élément le plus profond.
    Une chaîne est produite par élément recherché. Chaque élément est de type webElement au nous allons ajouter des méthodes
    et attributs.
    La chaîne de recherche permet de rechercher dans une page les éléments et de les comparer
    '''

    CHAINE = set([])

    def scroll(self):
        pass

    def __init__(self, recherches, webdriver):
        self.recherches
        self.webdriver

from types import MethodType
# https://www.ianlewis.org/en/dynamically-adding-method-classes-or-class-instanc

class webElement():
    '''
    L'objet webElement est une classe contenant des méthodes qui seront ajoutés aux objet webelement de selenium
    '''
    def __init__(self):
        self.pere = None
        self.fils = None

    def __eq__(self, objet):
        if self.getText == objet.getText:
            return True
        return False

if __name__=='__main__':
    # heure_debut_rencontre = {'selection':{'type':'span', 'class':'KambiBC-event-item__start-time--time'},
    #                  'resultat':{'attrs':['class', 'text']}}
    # sport = {'selection':{'type':'span', 'class':'KambiBC - modularized - event - path__fragment'},
    #                  'resultat':{'attrs':['class', 'text']}
    #          }
    # competition = {'selection':{'type':'span', 'class':'KambiBC-modularized-event-path__fragment'},
    #                  'resultat':{'attrs':['class', 'text']}
    #          }
    # participants = {'selection': {'type': 'div', 'class':'KambiBC-event-participants__name'},
    #                  'resultat': {'text': '','attrs': ['text']}}
    # paris = {'selection': {'type': 'span', 'class': 'KambiBC-mod-outcome__odds'},
    #                'resultat': {'attrs': ['class', 'text']}}
    # nb_offre_de_paris = {'selection': {'type': 'span', 'class': 'KambiBC-event-item__bet-offer-count'},
    #          'resultat': {'attrs': ['class', 'text']}}
    html ='''
    <html>
       <body>
           <div class="test 1">

                    <h1>Titre esssai</h1>
                    <h2>Sous  titre</h2>

           </div>
           <div class='balise1'>
                <span>
                    <h1>Titre esssai 1</h1>
                    <h2>Sous  titre 1</h2>
                </span>
                <span>
                    <h1>Titre esssai 2</h1>
                    <p><h3 class="type1">La ferme de pigeons</h3><h3 class="type1">Le projet des teubés</h3></p>
                    <h2>Sous  titre 2</h2>
                </span>
           </div>
           <div class="test3">
                <span>
                    <h1>Titre esssai 3</h1>
                    <p><h3 class="type1">La ferme de pigeons 2</h3><h3 class="type1">Le projet des teubés 3</h3></p>
                    <h2>Sous  titre 3</h2>
                </span>
           </div>
           <div></div>
       </body>
    </html>
    '''
    # soup = BeautifulSoup(html, 'html.parser')
    # print(soup.html.body.div.span.__getattribute__('parent'))
    a = Parse(html).parse(duplicates=True, **{'selection': [{'find_all': {'name': 'div',}},
                                           {'find_all': {'name': 'span'}},
                                           {'find_all': {'name': 'h2',}},
                                           {'find_parent': 'div'},
                                           {'find_all': {'name': 'h3'}},
                                                       ],
                                         'results': {'class':'classe', 'name':'name', 'xpath':'xpath', 'text': 'texte'},
                                         'mapping_fields': [],
                                         }
                        )
    print(a)
    # for i in a:
    #     print(i)


# Code to simulate the environment when trained
import gym

from stable_baselines3 import PPO
from stable_baselines3.ppo import MlpPolicy
from stable_baselines3.common.env_util import make_vec_env
from quadcoptV1 import QuadcoptEnvV1

env = QuadcoptEnvV1()

model = PPO.load("ppo_Quad_1Attempt")

obs = env.reset()
info_u=[0]
info_w=[0]
info_Z=[0]
for i in range(1000):
    action, _state = model.predict(obs, deterministic=True) # Add deterministic true for PPO to achieve better performane

    obs, reward, done, info = env.step(action)
    info_u.append(info["u"])
    info_w.append(info["w"])
    info_Z.append(info["Z"])

    #env.render()
    if done:
      # obs = env.reset()
      break

#print(info_u)
print(info_w)
print(info_w.__len__())


import io
import sys

_INPUT = """\
6
7 0 6 30
7 30 7 30
0 0 23 59
"""

sys.stdin = io.StringIO(_INPUT)
case_no=int(input())
for __ in range(case_no):
  A,B,C,D=map(int,input().split())
  if A*60+B<=C*60+D: print('Takahashi')
  else: print('Aoki')


"""
There are a total of n courses you have to take, labeled from 0 to n-1.
Some courses may have prerequisites, for example to take course 0 you have
to first take course 1, which is expressed as a pair: [0,1]
Given the total number of courses and a list of prerequisite pairs, is it
possible for you to finish all courses?
LeetCode Rating: Medium
Link: https://leetcode.com/problems/course-schedule/
"""

from collections import defaultdict
from collections import deque

# this is basically a topological sort question
# top sort is a cycle detection algo
def canFinish(numCourses, prerequisites):
    # using the adjacency list rep of a graph (note the use if defualtdict)
    graph = defaultdict(list)
    # map to track the indegrees of the courses
    indegrees = [0 for i in range(numCourses)]
    # populate the above two data structures
    for pair in prerequisites:
        graph[pair[0]].append(pair[1])
        indegrees[pair[0]] += 1
    # init topSorted list
    topSorted = []
    # init queue of nodes with zero degrees
    zeroDegree = deque()
    for i in range(numCourses):
        if indegrees[i] == 0:
            zeroDegree.append(i)
    while zeroDegree:
        node = zeroDegree.popleft()
        topSorted.append(node)
        # now for each node M with an edge E from N to M do
        for m in graph.keys():
            if graph[m] is not None:
                if node in graph[m]:
                    lst = graph[m]
                    lst.remove(node)
                    graph[m] = lst
                    if graph[m] == []:
                        zeroDegree.append(m)
    # now if the graph has even a single edge --> it has a cycle
    for m in graph.keys():
        if graph[m] != []:
            return False
    return True

if __name__ == "__main__":
    num, pres = 3, [[1, 0], [2, 0]]
    print(canFinish(num, pres))


from typing import Dict, List, Set
from pipert2.core.base.wire import Wire
from pipert2.core.base.routine import Routine
from pipert2.utils.consts import GENERATOR_ROUTINE, INNER_ROUTINE, FINAL_ROUTINE

def infer_routines_types(wires: List[Wire]) -> Dict[str, Set[Routine]]:
    """
    Group routines by their inferred type.
    Types:
        GENERATOR_ROUTINE - Routine that only produces data and send via queue
        INNER_ROUTINE - Routine that receives data from another source, produces new data and send it via queue
        FINAL_ROUTINE - Routine that receives data from another source, performs manipulation and doesn't send
        it to another routine.

    Args:
        wires: List of wires which describes connections in the Pipeline

    Returns:
        Dictionary of key-string and value-Set[Routine] where key is one of three:
            GENERATOR_ROUTINE
            INNER_ROUTINE
            FINAL_ROUTINE
        and value is a unique set of routines that matches that key by method conditions.

    """
    sources = set()
    destinations = set()

    for wire in wires:
        sources.add(wire.source)

        for routine in wire.destinations:
            destinations.add(routine)

    generator = sources - destinations
    inner = sources.intersection(destinations)
    final = destinations - sources

    return {GENERATOR_ROUTINE: generator,
            INNER_ROUTINE: inner,
            FINAL_ROUTINE: final}


# title: ORXOR
# problem link: https://atcoder.jp/contests/abc197/tasks/abc197_c

from functools import reduce
from operator import or_
from typing import List, Union

def bit_or(xs: List[int]) -> int:
    return reduce(or_, xs, 0)

def main() -> Union[list, int, str]:
    n = int(input())
    xs = [*map(int, input().split())]

    m = 2 ** 30
    for i in range(2 ** n):
        xored = 0
        ored = 0
        for j in range(n + 1):
            if j < n:
                ored |= xs[j]
            if j == n or (i >> j) & 1:
                xored ^= ored
                ored = 0
        m = min(m, xored)
    return m

if __name__ == "__main__":
    res = main()
    if not isinstance(res, list):
        res = [res]
    [*map(print, res)]


import os
import sys
sys.path.append(os.getcwd())
from matplotlib import pyplot as plt
import numpy as np
from main import *

        #Top 10 famous programming languages

# X and Y co-ordinates
languages=np.array(list())
user1=np.array(list())

for langs in c.most_common(15):
    languages=np.append(languages,langs[0])
    user1=np.append(user1,langs[1])

# Create labels and title
plt.style.use('seaborn')
plt.title('Top 10 Famous programming languages')
plt.ylabel('No. of user')

plt.bar(languages[0:11:],user1[0:11:],width=.4,color='#5a7d9a')

plt.tight_layout()
plt.show()

            #Top 15 Famous Programming Languages

# Create labels and title
plt.style.use('seaborn')
plt.title('Top 15 Famous Programming Languages')
plt.ylabel('No. of user')

# X and Y co-ordinate
languages=languages[::-1]
user1=user1[::-1]

plt.barh(languages,user1,height=.6,color='#5a7d9a')

plt.tight_layout()
plt.show()

Analysis of programming languages popularity/bar plot/bar.py


from re import match
from mytts import SpeechConfig, AudioOutputConfig, SpeechSynthesizer
from html import escape
from rich import print

def read(filename):
    rst:list[str] = []
    with open(filename, 'r',encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            delay = match("\{delay\s([\d\.]+)\}",line)
            if delay is not None:
                # rst.append(f'<break time="{delay.group(1)}" />')
                t = float(delay.group(1))
                delay_text = ""
                if t>5:
                    delay_text = '<break time="5s" />'*int(t//5)+f'<break time="{t%5}s" />'
                else:
                    delay_text = f'<break time="{t}s" />'
                if "</voice>" in rst[-1]:
                    rst[-1] = rst[-1].replace("</voice>",delay_text+'</voice>')
                else:
                    rst.append(f'<voice name="zh-CN-YunfengNeural"><break time="{delay_text}s" /</voice>')
                    raise RuntimeWarning("Why there is a delay at the very beginning?")
                continue
            who, con = line.split(': ')
            if who == "B":
                fmt = f'<voice name="zh-CN-YunfengNeural"><prosody rate="-30%">{con}</prosody></voice>'
            elif who == "W":
                fmt = f'<voice name="en-US-JennyMultilingualNeural"><prosody rate="-5%" pitch="0%">{con}</prosody></voice>'
            elif who == "M":
                fmt = f'<voice name="en-US-EricNeural"><mstts:silence  type="Sentenceboundary" value="100ms"/><prosody rate="10%" pitch="0%">{con}</prosody></voice>'
            else:
                raise AssertionError("Not a valid text format: {}->{}".format(who,con))
            rst.append(fmt)
    SSML_MODEL='''<speak xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="http://www.w3.org/2001/mstts" xmlns:emo="http://www.w3.org/2009/10/emotionml" version="1.0" xml:lang="en-US">{}</speak>'''
    lines = rst
    totLines = len(lines)
    content = []
    i = 0
    while i < totLines:
        tem = ""
        while tem.count("<voice name=") < 49 and i < totLines:
            tem += lines[i]
            tem += "\n"
            i += 1
        # tem = escape(tem)
        content.append(SSML_MODEL.format(tem))
    return content

def syn(content:list[str]):
    spe=SpeechConfig()
    for i,con in enumerate(content):
        opt_cfg=AudioOutputConfig(filename="%02d.mp3" % i)
        print(SpeechSynthesizer(spe,opt_cfg,debug=True).speak_ssml(con))

if __name__ == '__main__':
    print(read("input.txt")[0])
    syn(read("input.txt"))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import array
import binascii

from test_utils import timeit

@timeit
def _test2():

    a = array.array('i', range(3))
    print("Original    : ", a)
    print("Hex Version : ", binascii.hexlify(a))

    a.extend(range(3))
    print("extended    : ", binascii.hexlify(a))

    print("Sliced : ", a[2:5])
    print("Sliced Hex  : ", binascii.hexlify(a[2:5]))

    print("Iterator    : ", list(enumerate(a)))

@timeit
def _test_file():
    import tempfile

    a = array.array('i', range(5))
    print("Original    ", a)

    output = tempfile.NamedTemporaryFile()

    a.tofile(output.file)

    output.flush()

    with open(output.name, 'rb') as f:

        raw_s = f.read()
        print('Readed content :', binascii.hexlify(raw_s))

        f.seek(0)

        a2 = array.array('i')

        a2.fromfile(f, len(a))
        print('Input is   ', a2)

@timeit
def _test_bytes():

    a = array.array('i', range(5))

    raw_b = a.tobytes()
    print(raw_b)
    print('Hex : ', binascii.hexlify(raw_b))

    a2 = array.array('i')
    a2.frombytes(raw_b)
    print('bytes: ', a2)
    print('Hex : ', binascii.hexlify(a2))

@timeit
def _test_ordering():

    def to_hex(a):

        size_per_item = a.itemsize * 2
        chunk = binascii.hexlify(a)
        chunk_size = len(chunk)

        for pos in range(0, chunk_size, size_per_item):
            yield chunk[pos:pos + size_per_item]

    start = int('0x12345678', 16)
    end   = start + 5

    a1 = array.array('i', range(start, end))
    a2 = array.array('i', range(start, end))
    a2.byteswap()

    fmt = '{:>12}' * 4
    print(fmt.format('A1 Hex', 'A1', 'A2 Hex', 'A2'))
    print(fmt.format('-'*12,'-'*12,'-'*12,'-'*12))

    fmt = '{!r:>12}{:>12}' * 2
    for value in zip(to_hex(a1), a1, to_hex(a2), a2):
        print(fmt.format(*value))

@timeit
def _test1():

    s = b'This is a array'
    a = array.array('b', s)

    print('As byte string :', s)
    print('As Array :',     a)
    print('As Hex :', binascii.hexlify(a))

if __name__ == '__main__':
    print("hello, test array...")
    _test1()
    _test2()
    _test_file()
    _test_bytes()
    _test_ordering()


from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class MercadoPage(object):
    def __init__(self, driver):
        self._driver = driver
        self._url = 'https://www.mercadolibre.com/'
        self.search_locator = 'as_word'

    @property
    def is_loaded(self):
        WebDriverWait(self._driver, 5).until(EC.presence_of_element_located((By.NAME, 'as_word')))
        return True

    @property
    def keyword(self):
        input_field = self._driver.find_element(By.NAME, 'as_word')
        return input_field.get_attribute('value')

    @property
    def word_searched(self):
        word_searched = self._driver.find_element(By.XPATH, '//*[@id="root-app"]/div/div[1]/aside/div[1]/h1')
        return word_searched.text.lower()

    # This fuunction receives a list of possible working xpaths
    def _search_by_xpaths(self, xpaths):
            # WebDriverWait(self._driver, 10).until(EC.element_to_be_clickable((By.XPATH, xpath)))
            # elemento = self._driver.find_element(By.XPATH, xpath)
            # elemento.click()
        for xpath in xpaths:
            try:
                elemento = self._driver.find_element(By.XPATH, xpath)
                elemento.click()
                return None
            except:
                continue

        raise ValueError("None of the registered XPATHs matches, the list shoud be actualized")

    def open(self):
        self._driver.get(self._url)

    def search(self, keyword):
        input_field = self._driver.find_element(By.NAME, 'as_word')
        input_field.clear()
        input_field.send_keys(keyword)

    def click_submit(self):
        input_field = self._driver.find_element(By.NAME, 'as_word')
        input_field.submit()

    def select_country(self, country_ID):
        country_button = self._driver.find_element(By.ID, country_ID)
        country_button.click()
        xpath1 = '/html/body/div[2]/div[1]/div[2]/button[1]'
        close_coockie_disc = [xpath1]
        MercadoPage._search_by_xpaths(self, close_coockie_disc)

    def see_all_locations(self):

        xpath1 = '//*[@id="root-app"]/div/div[1]/aside/section/div[15]/ul/li[10]/a'
        xpath2 = '//*[@id="root-app"]/div/div[1]/aside/section/div[16]/ul/li[10]/a'
        xpath3 = '//*[@id="root-app"]/div/div[1]/aside/form/div[15]/ul/li[10]/a'
        xpath4 = '//*[@id="root-app"]/div/div[1]/aside/form/div[16]/ul/li[10]/a'
        xpath5 = '//*[@id="root-app"]/div/div[1]/aside/form/div[14]/ul/li[10]/a'
        xpath6 = '//*[@id="modal"]/div[2]/a[13]'
        xpath7 = '//*[@id="root-app"]/div/div[1]/aside/form/div[16]/ul/li[10]/a'

        xpaths = [xpath1, xpath2, xpath3, xpath4, xpath5, xpath6, xpath7]
        c = 0
        while True:
            try:
                MercadoPage._search_by_xpaths(self, xpaths)
                return None
            except:
                self._driver.refresh()
                c += 5
            if c == 5:
                raise ValueError('Too many failed attempts at see_all_locations func')

    def select_location_guerrero(self):

        MercadoPage.see_all_locations(self)

        xpath1 = '//*[@id="modal"]/div[2]/a[13]'
        xpath2 = '//*[@id="modal"]/div[2]/div[6]/div[2]/a[2]'

        xpaths = [xpath1, xpath2]

        MercadoPage._search_by_xpaths(self, xpaths)

    # TODO: make one method 'select_location_(location)' for every state using XPATH

    def select_condition_nuevo(self):

        xpath1 = '/html/body/main/div/div[1]/aside/section/div[6]/ul/li[1]/a'
        xpath2 = '//*[@id="root-app"]/div/div[1]/aside/form/div[6]/ul/li[1]/button'
        xpath3 = '//*[@id="root-app"]/div/div[1]/aside/form/div[3]/ul/li[1]/button'

        xpaths = [xpath1, xpath2, xpath3]

        MercadoPage._search_by_xpaths(self, xpaths)

    # TODO: make a method simmilar to the one above for the "used" condition

    def sorting_option(self, option):
        xpath_s_s_button = '//*[@id="root-app"]/div/div[1]/section/div[1]/div/div/div/div[2]/div/div/button'
        xpath_s_o_mayor_precio = '//*[@id="root-app"]/div/div/section/div[1]/div/div/div/div[2]/div/div/div/ul/a[2]'
        xpath_confirm_option = '//*[@id="root-app"]/div/div/section/div[1]/div/div/div/div[2]/div/div/button/span'

        sort_select_button = self._driver.find_element(By.XPATH, xpath_s_s_button)
        sort_select_button.click()

        # TODO add a way to click in the different options
        if option == 'importance':
            pass
        elif option == 'asc_price':
            pass
        elif option == 'desc_price':
            select_mayor_precio = self._driver.find_element(By.XPATH, xpath_s_o_mayor_precio)
            select_mayor_precio.click()
            # WebDriverWait(self._driver, 10).until(EC.text_to_be_present_in_element((By.XPATH, xpath_confirm_option), 'Mayor precio'))
            # self._driver.implicitly_wait(3)
        else:
            raise ValueError('Unexpected option\n\nValid options: \n* importance\n* asc_price\n* desc_price')

    def scrape_results(self):
        xpath1 = '//*[@id="root-app"]/div/div[1]/section/ol[1]/li[1]/div/div/a/div/div[3]/h2'
        xpath2 = '//*[@id="root-app"]/div/div[1]/section/ol[1]/li[2]/div/div/a/div/div[3]/h2'
        xpath3 = '//*[@id="root-app"]/div/div[1]/section/ol[2]/li[1]/div/div/a/div/div[3]/h2'
        xpath3 = '//*[@id="root-app"]/div/div[1]/section/ol[1]/li[3]/div/div/a/div/div[3]/h2'
        #         //*[@id="root-app"]/div/div[1]/section/ol[1]/li[1]/div/div/a/div/div[3]/h2

        result, price = '', ''
        while result == '' or price == '':
            try:
                result = self._driver.find_element(By.XPATH, '//*[@id="root-app"]/div/div[1]/section/ol[1]/li[1]/div/div/a/div/div[3]/h2')
                texto = result.text
                price = self._driver.find_element(By.XPATH, '//*[@id="root-app"]/div/div[1]/section/ol[1]/li[1]/div/div/a/div/div[1]/div/div/div/span[1]/span[2]/span[2]')
                price_t = price.text
            except:
                # raise ValueError("None of the registered XPATHs matches, the list shoud be actualized")
                self._driver.refresh()

        print('----->' + texto + ' ' + price_t + '<------')

        c = 0
        results = []
        prices = []
        for j in range(24):
            try:

                for i in range(2):
                    try:
                        result = self._driver.find_element(By.XPATH, f'//*[@id="root-app"]/div/div[1]/section/ol[{j+1}]/li[{i+1}]/div/div/a/div/div[3]/h2')
                        print(str(result.text), end='')
                        results.append(result.text)
                        price = self._driver.find_element(By.XPATH, f'//*[@id="root-app"]/div/div[1]/section/ol[{j+1}]/li[{i+1}]/div/div/a/div/div[1]/div/div/div/span[1]/span[2]/span[2]')
                        print(' ' + str(price.text))
                        prices.append(price.text)
                        c += 1
                    except:
                        continue
            except:
                continue
        print('Elementos encontrados: ' + str(c))
        print(results)
        print(prices)

        # ol, li = 1, 1
        # names = []
        # for _ in range(50):
        #     try:
        #         product = self._driver.find_element(By.XPATH, f'//*[@id="root-app"]/div/div[1]/section/ol[{ol}]/li[{li}]/div/div/a/div/div[3]/h2').text
        #         names.append(product)
        #         li += 1
        #     except:
        #         ol += 1

        # print('Se encontraron' + str(len(names)))


from django.db import models

# Create your models here.
class Question(models.Model) :
    # 질문 모델
    subject = models.CharField(max_length=200)
    # 제목 최대 200줄로 설정
    content = models.TextField()
    # 글자 수 제한 없음
    create_date = models.DateTimeField()
    # 날짜 * 시간 속성

    def __str__(self):  # 데이터 조회시 나오는 값
        return self.subject # 데이터 조회시 제목 표시

class Answer(models.Model) :
    question = models.ForeignKey(Question, on_delete=models.CASCADE)
    # 왜래키로 다른 모델 속성 가지기(모델 연결), on_delete=models.CASCADE는 답변의 질문이 삭제시 답변도 함께 삭제
    content = models.TextField()
    create_date = models.DateTimeField()


import sys
import getopt

sys.path.insert(0,'/home/pi/Adafruit-Raspberry-Pi-Python-Code/Adafruit_PWM_Servo_Driver')
sys.path.append('.')

import RTIMU
import os.path
#import time
import math
#from datetime import datetime
from Adafruit_PWM_Servo_Driver import PWM
#import matplotlib.pyplot as plt
#import numpy as np
import pygame
from pygame.locals import *

#initializing variables
pygame.init()
screen=pygame.display.set_mode((640,480),0,24)
pygame.display.set_caption("Key Press Test")
f1=pygame.font.SysFont("comicsansms",24)

pwm = PWM(0x40)
pwm.setPWMFreq(60)

SETTINGS_FILE = "RTIMULib"

def computeHeight(pressure):
    return 44330.8 * (1 - pow(pressure / 1013.25, 0.190263));

print("Using settings file " + SETTINGS_FILE + ".ini")
if not os.path.exists(SETTINGS_FILE + ".ini"):
  print("Settings file does not exist, will be created")

s = RTIMU.Settings(SETTINGS_FILE)
imu = RTIMU.RTIMU(s)
pressure = RTIMU.RTPressure(s)
humidity = RTIMU.RTHumidity(s)

print("IMU Name: " + imu.IMUName())
print("Pressure Name: " + pressure.pressureName())
print("Humidity Name: " + humidity.humidityName())

if (not imu.IMUInit()):
    print("IMU Init Failed")
    sys.exit(1)
else:
    print("IMU Init Succeeded");

# this is a good time to set any fusion parameters

imu.setSlerpPower(0.02)
imu.setGyroEnable(True)
imu.setAccelEnable(True)
imu.setCompassEnable(True)

if (not pressure.pressureInit()):
    print("Pressure sensor Init Failed")
else:
    print("Pressure sensor Init Succeeded")

if (not humidity.humidityInit()):
    print("Humidity sensor Init Failed")
else:
    print("Humidity sensor Init Succeeded")

poll_interval = imu.IMUGetPollInterval()
print("Recommended Poll Interval: %dmS\n" % poll_interval)

#set stopped and centred initially
pwm.setPWM(1,0,400)
pwm.setPWM(0,0,400)

speed=400
steer=400
flag=0
fusionPose=0
name="null"
screen.fill((255,255,255))

#main loop which drives the vehicle
while flag==0:
    for i in pygame.event.get():
        if pygame.key.get_focused():
            press=pygame.key.get_pressed()
            for i in xrange(0,len(press)):
                if press[i]==1:
                    name=pygame.key.name(i)
                    if name=="left":
                        steer -=120
                    elif name=="right":
                        steer +=120
                    elif name=="up":
                        speed +=50
                    elif name=="down":
                        speed -=50
                    elif name=="space":
                        steer=400
                    elif name=="x":
                        speed=400
                    elif name=="escape":
                        flag=1

            pwm.setPWM(0,0,speed)
            pwm.setPWM(1,0,steer)

    if imu.IMURead():
        data = imu.getIMUData()
        fusionPose = data["fusionPose"]

    screen.fill((255,255,255))
    text=f1.render(name,True,(0,0,0))
    screen.blit(text,(100,100))
    text2=f1.render("Steering %s" % str(steer),True,(0,0,0))
    screen.blit(text2,(100,150))
    text3=f1.render("Speed %s" % str(speed),True,(0,0,0))
    screen.blit(text3,(100,200))
    text4=f1.render("Heading %s" % str(int(math.degrees(fusionPose[2]))),True,(0,0,0))
    screen.blit(text4,(100,250))
    pygame.display.update()

#t=datetime.now()
#timu=0
#vel=0

##for loop in range(1,100000):
    #if imu.IMURead():
        #data = imu.getIMUData()
        #fusionPose = data["fusionPose"]
        ##print("r: %f p: %f y: %f" % (math.degrees(fusionPose[0]),math.degrees(fusionPose[1]), math.degrees(fusionPose[2])))
        #deltapi=datetime.now()-t
        ##print("deltapi %f" % deltapi.microseconds)
        #t=datetime.now()
        #deltaimu=data["timestamp"]-timu
        ##print("deltaimu: %f" % deltaimu)
        #timu=data["timestamp"]
        #acc=data["accel"][0]
        #vel=vel+(acc*(deltapi.microseconds))/1000000
        #print(data)
        ##print(vel)

#while True:
#  if imu.IMURead():
#    # x, y, z = imu.getFusionData()
#    # print("%f %f %f" % (x,y,z))
#    data = imu.getIMUData()
#    (data["pressureValid"], data["pressure"], data["pressureTemperatureValid"], data["pressureTemperature"]) = pressure.pressureRead()
#    (data["humidityValid"], data["humidity"], data["humidityTemperatureValid"], data["humidityTemperature"]) = humidity.humidityRead()
#    fusionPose = data["fusionPose"]
#    print("r: %f p: %f y: %f" % (math.degrees(fusionPose[0]),
#        math.degrees(fusionPose[1]), math.degrees(fusionPose[2])))
#    if (data["pressureValid"]):
#        print("Pressure: %f, height above sea level: %f" % (data["pressure"], computeHeight(data["pressure"])))
#    if (data["pressureTemperatureValid"]):
#        print("Pressure temperature: %f" % (data["pressureTemperature"]))
#    if (data["humidityValid"]):
#        print("Humidity: %f" % (data["humidity"]))
#    if (data["humidityTemperatureValid"]):
#        print("Humidity temperature: %f" % (data["humidityTemperature"]))
#    time.sleep(poll_interval*1.0/1000.0)

# Disable PWM outputs
pwm.setPWM(0,4096,0)
pwm.setPWM(1,4096,0)


import io
import sys

_INPUT = """\
6
4
-WWW
L-DD
LD-W
LDW-
2
-D
D-
"""

sys.stdin = io.StringIO(_INPUT)
case_no=int(input())
for __ in range(case_no):
  N=int(input())
  A=[input() for _ in range(N)]
  ans='correct'
  for i in range(N):
    for j in range(N):
      if i==j: continue
      if A[i][j]=='W' and A[j][i]!='L': ans='incorrect'
      if A[i][j]=='L' and A[j][i]!='W': ans='incorrect'
      if A[i][j]=='D' and A[j][i]!='D': ans='incorrect'
  print(ans)


import json

from nltk.corpus import stopwords
from nltk.stem.snowball import SnowballStemmer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import SVC

# Evaluation visualisation library
import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix

analyzer = TfidfVectorizer().build_analyzer()
sb_stemmer = SnowballStemmer('english')

def stemmed_words(doc):
    return (sb_stemmer.stem(w) for w in analyzer(doc))

def build_intent_matching_classifier():
    # Load all the required dataset
    with open('./data/intent_matching/small_talk_intent.json') as f:
        small_talk_intent = json.load(f)

    with open('./data/intent_matching/information_retrieval_game_intent.json') as f:
        information_retrieval_game_intent = json.load(f)

    with open('./data/intent_matching/information_retrieval_recommendation_intent.json') as f:
        information_retrieval_recommendation_intent = json.load(f)

    with open('./data/intent_matching/identity_management_intent.json') as f:
        identity_management_intent = json.load(f)

    with open('./data/intent_matching/question_answer_intent.json') as f:
        question_answer_intent = json.load(f)

    intent_label = {
        'small talk': small_talk_intent,
        'game': information_retrieval_game_intent,
        'movie recommendation': information_retrieval_recommendation_intent,
        'identity management': identity_management_intent,
        'question answer': question_answer_intent
    }

    data_inputs = []
    data_intents = []

    # Append the data inputs & intent into a list
    for label in intent_label.keys():
        for intent in intent_label[label]['intents']:
            for text in intent['text']:
                data_inputs.append(text)
                data_intents.append(label)

    intent_tfidf_vectorizer = TfidfVectorizer(lowercase=True, stop_words=stopwords.words('english'), ngram_range=(1, 2),
                                              analyzer=stemmed_words)

    X_train_input, X_test_input, y_train_intent, y_test_intent = train_test_split(data_inputs, data_intents, stratify=data_intents, test_size=0.2, random_state=42)

    # x_train_tf = intent_tfidf_vectorizer.fit_transform(data_inputs)
    X_train_tf = intent_tfidf_vectorizer.fit_transform(X_train_input)

    # Train the classifier
    intent_classifier = SVC(random_state=42, probability=True).fit(X_train_tf, y_train_intent)

    # Classifier evaluation section
    # References: Code adapted from COMP 3009 Machine Learning module
    # X_test_tf = intent_tfidf_vectorizer.transform(X_test_input)
    # evaluation_accuracy = intent_classifier.score(X_test_tf, y_test_intent)
    # print("SVM intent classification accuracy: {:.2f}%".format(evaluation_accuracy * 100))
    #
    # # Plot confusion matrix,
    # y_pred = intent_classifier.predict(X_test_tf)
    # print(y_pred)
    # cm = confusion_matrix(y_test_intent, y_pred)
    # fig, ax = plt.subplots(figsize=(1, 1))
    # fig.set_size_inches(7, 4)
    # ax.matshow(cm, cmap=plt.cm.Blues, alpha=0.3)
    #
    # for i in range(cm.shape[0]):
    #     for j in range(cm.shape[1]):
    #         ax.text(x=j, y=i,
    #                 s=cm[i, j],
    #                 va='center', ha='center')
    #
    # classes = intent_classifier.classes_
    #
    # tick_marks = np.arange(len(classes))
    # plt.xticks(tick_marks, classes, rotation=45)
    # plt.yticks(tick_marks, classes)
    # plt.xlabel('Predicted Values')
    # plt.ylabel('Actual Values')
    # plt.show()
    #
    # # Save the figure to evaluation folder
    # fig.savefig('./evaluation/intent_matching_confusion_matrix.png', dpi=100)
    #
    # print(classification_report(y_test_intent, y_pred))

    return [intent_label, intent_classifier, intent_tfidf_vectorizer]

# Evaluation of intent classifier
# [intent_label, intent_classifier, intent_tfidf_vectorizer] = build_intent_matching_classifier()


"""empty message

Revision ID: 5a293e1dc753
Revises: 788a53b697f4
Create Date: 2019-03-10 17:33:18.275204

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '5a293e1dc753'
down_revision = '788a53b697f4'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('tb_menu', sa.Column('image_name', sa.String(length=128), nullable=True))
    op.create_index(op.f('ix_tb_menu_image_name'), 'tb_menu', ['image_name'], unique=False)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_tb_menu_image_name'), table_name='tb_menu')
    op.drop_column('tb_menu', 'image_name')
    # ### end Alembic commands ###


#!/usr/bin/env python
# coding: utf-8

# # Modelo - Pipeline classificação
# - Marketing campaign

# In[1]:

from platform import python_version

print('Versão python neste Jupyter Notebook:', python_version())

# In[2]:

import sklearn
import matplotlib
import pandas as pd
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
import matplotlib as m
import warnings

import missingno as msno
import plotly.graph_objects as go
import plotly.express as px
from yellowbrick.cluster import KElbowVisualizer

warnings.filterwarnings("ignore")

# In[3]:

get_ipython().run_line_magic('reload_ext', 'watermark')

get_ipython().run_line_magic('watermark', '-a "Rafael Gallo" --iversions')

# In[5]:

# Configuração para os gráficos largura e layout dos graficos

plt.rcParams["figure.figsize"] = (18, 5)

plt.style.use('fivethirtyeight')
pd.set_option('display.expand_frame_repr', False)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

m.rcParams['axes.labelsize'] = 25
m.rcParams['xtick.labelsize'] = 25
m.rcParams['ytick.labelsize'] = 25
m.rcParams['text.color'] = 'k'

# # Base dados

# In[6]:

# Carregando base dados
data = pd.read_csv("marketing_campaign.csv", sep='\t')
data

# In[7]:

# Visualizando os 5 primeiros dados
data.head()

# In[8]:

# Visualizando os 5 cinco últimos dados
data.tail()

# In[9]:

# Visualizando os infos dados
data.info()

# In[10]:

# Tipos de dados
data.dtypes

# In[11]:

# Visualizando linhas colunas
data.shape

# In[12]:

# Total de colunas e linhas - data_train

print("Números de linhas: {}" .format(data.shape[0]))
print("Números de colunas: {}" .format(data.shape[1]))

# In[13]:

# Exibindo valores ausentes e valores únicos

print("\nMissing values :  ", data.isnull().sum().values.sum())
print("\nUnique values :  \n",data.nunique())

# In[14]:

# Verificando dados duplicados
data.duplicated()

# In[15]:

# Visualizando dados nulos
data.isnull().sum()

# # Estatística descritiva

# In[16]:

# Exibindo estatísticas descritivas visualizar alguns detalhes estatísticos básicos como percentil, média, padrão, etc.
# De um quadro de dados ou uma série de valores numéricos.

data.describe().T

# In[17]:

# Matriz correlação de pares de colunas, excluindo NA / valores nulos.

corr = data.corr()
corr

# In[18]:

plt.figure(figsize=(18.2, 8))

ax = sns.distplot(data['Year_Birth']);
plt.title("Distribuição normal", fontsize=20)
plt.axvline(data['Year_Birth'].mean(), color='k')
plt.axvline(data['Year_Birth'].median(), color='r')
plt.axvline(data['Year_Birth'].mode()[0], color='g');

# In[19]:

# Verificando os dados no boxplot valor total verificando possíveis outliers
plt.figure(figsize=(18.2, 8))

sns.boxplot(x="Income", y="Education", data = data)
plt.title("Educação dos consumidores")
plt.xlabel("Total")
plt.ylabel("Educação")

# In[20]:

plt.figure(figsize=(14, 10))
sns.heatmap(data.corr(), cmap="Blues")
plt.title("Correlations", size=16)
plt.show()

# In[21]:

# Gráfico da matriz de correlação

plt.figure(figsize=(35.5,30))
ax = sns.heatmap(corr, annot=True, cmap='YlGnBu');
plt.title("Matriz de correlação")

# In[22]:

plt.figure(figsize=(30.5,20))

mask = np.triu(np.ones_like(corr, dtype = bool))
sns.heatmap(corr, mask = mask, annot = True, fmt = '.2g', linewidths = 1)
plt.show()

# # Análise dados

# In[23]:

plt.figure(figsize=(20, 10))

plt.title("Educação dos consumidores")
sns.countplot(data["Education"])
plt.xlabel("Educação")
plt.ylabel("Total")

# In[24]:

plt.figure(figsize=(20, 10))

sns.histplot(data["Year_Birth"])
plt.title("Ano de nascimento dos consumidores")
plt.xlabel("Ano")
plt.ylabel("Idades")

# In[25]:

plt.figure(figsize=(20, 10))

sns.histplot(data["Income"])
plt.title("Renda dos consumidores")
plt.xlabel("Total da renda")
plt.ylabel("Total")

# In[26]:

# Gráfico condições de vida por valor do imóvel
plt.figure(figsize=(35.5, 15))

plt.title("Formações dos consumidores")
ax = sns.barplot(x="Education", y="Income", data = data, hue="Marital_Status")
plt.ylabel("Valor")
plt.xlabel("Formações")

# In[27]:

# Região das vendas dos imóveis pela área
plt.figure(figsize=(18.2, 8))

plt.title("Renda dos consumidores")
ax = sns.scatterplot(x="MntWines", y="Income", data = data, hue = "Marital_Status")
plt.xlabel("Renda ")
plt.ylabel("Valor")

# In[28]:

plt.figure(figsize=(20, 10))

# Graduation, PHD, master, basic, 2n cycle

plt.pie(data.groupby("Education")['Education'].count(), labels=["Graduation", "PHD", "master", "basic", "2n_cycle"], autopct = "%1.1f%%");
plt.title("Perfil de escolariedade dos consumidores")
plt.xlabel("Total consumidores")
plt.ylabel("Total")

# # Análise de dados = Univariada

# In[29]:

data.hist(bins = 40, figsize=(20.2, 20))
plt.show()

# In[30]:

fig1 , axes = plt.subplots(nrows=3,ncols=3 , figsize = (20,20))

sns.distplot(data["NumWebVisitsMonth"] , ax=axes[0, 0] )
sns.distplot(data["AcceptedCmp1"] ,  ax=axes[0, 1])
sns.distplot(data["AcceptedCmp2"] , ax=axes[0, 2])
sns.distplot(data["AcceptedCmp3"], ax=axes[1, 0])
sns.distplot(data["AcceptedCmp4"] , ax=axes[1, 1])
sns.distplot(data["AcceptedCmp5"] , ax=axes[1, 2])
sns.distplot(data["Recency"] , ax=axes[2, 0])
sns.distplot(data["MntWines"], ax=axes[2, 1])
sns.countplot(data["Education"], ax=axes[2, 2])

plt.show()

# In[31]:

# Nós imputamos os valores omissos na coluna "Renda" com a mediana dessa coluna em particular

data["Income"].fillna(data["Income"].median(), inplace=True)
data

# # Feature Engineering

# In[32]:

data["Age"] = 2022 - data["Year_Birth"]

data["Money_Spent"] = (data["MntWines"] + data["MntFruits"] + data["MntMeatProducts"] + data["MntFishProducts"] + data["MntSweetProducts"] + data["MntGoldProds"])
data["PurchaseNum"] = data["NumWebPurchases"] + data["NumCatalogPurchases"] + data["NumStorePurchases"]
data["Children"] = data["Kidhome"] + data["Teenhome"]
data["Marital_Status"] = data["Marital_Status"].replace({"Married": "Together", "Alone": "Single", "Absurd": "Single", "Divorced": "Single", "Widow": "Single", "Divorced": "Single", "YOLO": "Single"})

# exluindo coluna
data.drop(["Education", "Marital_Status", "Dt_Customer"], axis=1, inplace=True)
data

# # Treino e Teste
#
# - Treino e teste da base de dados da coluna price e idade

# In[47]:

x = data.drop('Response',axis=1).values
y = data['Response'].values

# In[48]:

# Total de linhas e colunas dados variável x
x.shape

# In[49]:

# Total de linhas e colunas dados variável y
y.shape

# # Escalonamento
#
# - Escalonamento uma forma de contornar os problemas relacionados à escala, mantendo a informação estatística dos dados. O procedimento consiste em realizar uma transformação sobre o conjunto original dos dados de modo que cada variável apresente média zero e variância unitária.

# In[50]:

# Escalonamento dos dados

# Importando biblioteca a biblioteca
from sklearn.preprocessing import StandardScaler

# Criando o escalonamento
model_scaler = StandardScaler()

# Treinamneto do escalonamento
model_scaler_fit = model_scaler.fit_transform(x)

# Visualizando linhas e colunas do escalonamento
model_scaler_fit.shape

# In[51]:

# Importação da biblioteca sklearn para treino e teste do modelo

from sklearn.model_selection import train_test_split

x_train, x_test, y_train, y_test = train_test_split(x, # Variável x
                                                    y, # Variável y
                                                    test_size=0.2, # Divivindo os dados em 20% para treino e 80% para teste
                                                    random_state = 0) # Random state igual a zero

# In[42]:

# Total de linhas e colunas e linhas dos dados de treino x
x_train.shape

# In[43]:

# Total de linhas dos dados de treino y
y_train.shape

# In[44]:

# Total de linhas e colunas dos dados de treino x teste
x_test.shape

# In[45]:

# Total de linhas e colunas dos dados de treino y teste
y_test.shape

# # Model pipeline

# In[52]:

# Modelo pipeline

from sklearn.feature_selection import VarianceThreshold
from sklearn.pipeline import Pipeline
from sklearn.neighbors import KNeighborsClassifier

data_pipeline = Pipeline([
    ("scaler", StandardScaler()), # Scaler : Para pré-processamento de dados, ou seja, transforme os dados em média zero e variância de unidade usando o StandardScaler ().
    ("selector", VarianceThreshold()), # Seletor de recurso : Use VarianceThreshold () para descartar recursos cuja variação seja menor que um determinado limite definido.
    ("classifier", KNeighborsClassifier()) # Classificador : KNeighborsClassifier (), que implementa o classificador de k-vizinho mais próximo e seleciona a classe dos k pontos principais, que estão mais próximos do exemplo de teste.
])

data_pipeline_fit = data_pipeline.fit(x_train, y_train)
data_pipeline_score = data_pipeline.score(x_train, y_train)

print('Treinamento base treino - Pipeline: ' + str(data_pipeline.score(x_train,y_train)))
print('Treinamento base teste - Pipeline: ' + str(data_pipeline.score(x_test,y_test)))

# In[54]:

# Previsão do pipeline do modelo
data_pipeline_pred_1 = data_pipeline.predict(x_test)

# In[55]:

# Matrix confusion ou Matriz de Confusão
from sklearn.metrics import confusion_matrix
from mlxtend.plotting import plot_confusion_matrix

matrix_1 = confusion_matrix(y_test, data_pipeline_pred_1)
plot_confusion_matrix(matrix_1, show_normed=True, colorbar=False, class_names=['Acesso', 'Não acesso'])

# In[56]:

# Accuracy do pipeline
from sklearn.metrics import accuracy_score

accuracy_pipeline_1 = accuracy_score(y_test, data_pipeline_pred_1)
print("Accuracy KNN - Pipeline: %.2f" % (accuracy_pipeline_1 * 100))

# In[57]:

# Curva ROC do modelo
from sklearn.metrics import roc_curve, roc_auc_score

roc = data_pipeline.predict_proba(x_test)[:,1]
tfp, tvp, limite = roc_curve(y_test, roc)
print('roc_auc', roc_auc_score(y_test, roc))

plt.subplots(1, figsize=(5,5))
plt.title('Curva ROC')
plt.plot(tfp,tvp)
plt.xlabel('Especifidade')
plt.ylabel('Sensibilidade')
plt.plot([0, 1], ls="--", c = 'red')
plt.plot([0, 0], [1, 0], ls="--", c = 'green'), plt.plot([1, 1], ls="--", c = 'green')
plt.show()

# In[58]:

# Classification report
from sklearn.metrics import classification_report

classification = classification_report(y_test, data_pipeline_pred_1)
print("Modelo - Pipeline 1")
print()
print(classification)

# In[59]:

# Métricas do modelo
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score

precision = precision_score(y_test, data_pipeline_pred_1)
Recall = recall_score(y_test, data_pipeline_pred_1)
Accuracy = accuracy_score(y_test, data_pipeline_pred_1)
F1_Score = f1_score(y_test, data_pipeline_pred_1)

precisao = pd.DataFrame({

    "Metricas" : ["precision",
                 "Recall",
                  "Accuracy",
                  "F1_Score"],

    "Resultado": [precision,
                Recall,
                Accuracy,
                F1_Score]})

precisao.sort_values(by = "Resultado", ascending = False)

# # Pipeline 2
# **Pipeline 2 - Decision Tree Classifier**

# In[60]:

# Pipeline decision Tree Classifier
from sklearn.tree import DecisionTreeClassifier

data_pipeline_2 = Pipeline([
    ("scaler", StandardScaler()), # Scaler : Para pré-processamento de dados, ou seja, transforme os dados em média zero e variância de unidade usando o StandardScaler ().
    ("selector", VarianceThreshold()), # Seletor de recurso : Use VarianceThreshold () para descartar recursos cuja variação seja menor que um determinado limite definido.
    ("classifier", DecisionTreeClassifier(max_depth=4, random_state=0)) # Classificador : DecisionTreeClassifier (), que implementa o classificador de árvore decisão Árvores de decisão são métodos de classificação que podem extrair regras simples sobre os recursos de dados que são inferidos do conjunto de dados de entrada
])

data_pipeline2_fit = data_pipeline_2.fit(x_train, y_train)
data_pipeline2_score = data_pipeline_2.score(x_train, y_train)

print('Treinamento base treino - Pipeline: ' + str(data_pipeline_2.score(x_train,y_train)))
print('Treinamento base teste - Pipeline: ' + str(data_pipeline_2.score(x_test,y_test)))

# In[61]:

# Previsão do pipeline
data_pipeline_pred_2 = data_pipeline_2.predict(x_test)
data_pipeline_pred_2

# In[62]:

# Accuracy do pipeline
accuracy_pipeline_2 = accuracy_score(y_test, data_pipeline_pred_2)
print("Accuracy Pipeline 2: %.2f" % (accuracy_pipeline_2 * 100))

# In[63]:

# A matrix confusion do modelo
matrix_confusion_1 = confusion_matrix(y_test, data_pipeline_pred_2)
plot_confusion_matrix(matrix_confusion_1, show_normed=True, colorbar=False, class_names=['Acesso', 'Não acesso'])

# In[64]:

# Curva ROC do modelo
roc = data_pipeline_2.predict_proba(x_test)[:,1]
tfp, tvp, limite = roc_curve(y_test, roc)
print('roc_auc', roc_auc_score(y_test, roc))

plt.subplots(1, figsize=(5,5))
plt.title('Curva ROC')
plt.plot(tfp,tvp)
plt.xlabel('Especifidade')
plt.ylabel('Sensibilidade')
plt.plot([0, 1], ls="--", c = 'red')
plt.plot([0, 0], [1, 0], ls="--", c = 'green'), plt.plot([1, 1], ls="--", c = 'green')
plt.show()

# In[65]:

# Classification_report
classification = classification_report(y_test, data_pipeline_pred_2)
print("Modelo Pipeline 2")
print()
print(classification)

# In[67]:

# Méricas do modelo
precision = precision_score(y_test, data_pipeline_pred_2)
Recall = recall_score(y_test, data_pipeline_pred_2)
Accuracy = accuracy_score(y_test, data_pipeline_pred_2)
F1_Score = f1_score(y_test, data_pipeline_pred_2)

precisao = pd.DataFrame({

    "Metricas" : ["precision",
                 "Recall",
                  "Accuracy",
                  "F1_Score"],

    "Resultado": [precision,
                Recall,
                Accuracy,
                F1_Score]})

precisao.sort_values(by = "Resultado", ascending = False)

# **Pipeline 3 - Naive bayes**

# In[68]:

# Modelo naive bayes

# Importando biblioteca ML naive bayes
from sklearn.naive_bayes import GaussianNB

# Pipeline Naive bayes
data_pipeline_3 = Pipeline([
    ("scaler", StandardScaler()),
    ("selector", VarianceThreshold()),
    ("classifier", GaussianNB())])

data_pipeline3_fit = data_pipeline_3.fit(x_train, y_train)
data_pipeline3_score = data_pipeline_3.score(x_train, y_train)

print('Treinamento base treino - Pipeline: ' + str(data_pipeline_3.score(x_train,y_train)))
print('Treinamento base teste - Pipeline: ' + str(data_pipeline_3.score(x_test,y_test)))

# In[69]:

# Previsão do modelo - Naive bayes

# Previsão do pipeline
data_pipeline_pred_3 = data_pipeline_3.predict(x_test)
data_pipeline_pred_3

# In[70]:

# Accuracy do pipeline
accuracy_pipeline_3 = accuracy_score(y_test, data_pipeline_pred_3)
print("Accuracy pipeline 3: %.2f" % (accuracy_pipeline_3 * 100))

# In[71]:

# A matrix confusion pipeline
matrix_confusion_4 = confusion_matrix(y_test, data_pipeline_pred_3)
plot_confusion_matrix(matrix_confusion_4, show_normed=True, colorbar=False, class_names=['Acesso', 'Não acesso'])

# In[72]:

# Curva ROC do pipeline
roc = data_pipeline_3.predict_proba(x_test)[:,1]
tfp, tvp, limite = roc_curve(y_test, roc)
print('roc_auc', roc_auc_score(y_test, roc))

plt.subplots(1, figsize=(5,5))
plt.title('Curva ROC')
plt.plot(tfp,tvp)
plt.xlabel('Especifidade')
plt.ylabel('Sensibilidade')
plt.plot([0, 1], ls="--", c = 'red')
plt.plot([0, 0], [1, 0], ls="--", c = 'green'), plt.plot([1, 1], ls="--", c = 'green')
plt.show()

# In[73]:

# Classification report do modelo
class_report = classification_report(y_test, data_pipeline_pred_3)
print("Modelo 03 - Pipeline")
print("\n")
print(class_report)

# In[74]:

# Metricas do pipeline
precision = precision_score(y_test, data_pipeline_pred_3)
Recall = recall_score(y_test, data_pipeline_pred_3)
Accuracy = accuracy_score(y_test, data_pipeline_pred_3)
F1_Score = f1_score(y_test, data_pipeline_pred_3)

precisao = pd.DataFrame({

    "Metricas" : ["precision",
                 "Recall",
                  "Accuracy",
                  "F1_Score"],

    "Resultado": [precision,
                Recall,
                Accuracy,
                F1_Score]})

precisao.sort_values(by = "Resultado", ascending = False)

# In[75]:

# Resultados - Modelos machine learning

modelos = pd.DataFrame({

    "Models" :["Pipeline 1: K-NN",
               "Pipeline 2: Decision tree",
               "Pipeline 3: Naive bayes"],

    "Acurácia" :[accuracy_pipeline_1,
                      accuracy_pipeline_2,
                      accuracy_pipeline_3]})

modelos_2 = modelos.sort_values(by = "Acurácia", ascending = False)
modelos_2.to_csv("modelos_2.csv")
modelos_2

# In[76]:

# Salvando pipeline Machine learning

import pickle

with open('data_pipeline_pred_1.pkl', 'wb') as file:
    pickle.dump(data_pipeline_pred_1, file)

with open('data_pipeline_pred_2.pkl', 'wb') as file:
    pickle.dump(data_pipeline_pred_2, file)

with open('data_pipeline_pred_3.pkl', 'wb') as file:
    pickle.dump(data_pipeline_pred_3, file)

Jupyter Notebook/Customer Personality Analysis/SRC/pipeline_model.py


import matplotlib.pyplot as plt

# Initialiser des listes pour stocker les données
nb_line = []
naive_duration = []
lps_duration = []
optimized_lps_duration = []

# Ouvrir le fichier en mode lecture
with open('../xp_result.csv', 'r') as file:
    # Lire les lignes du fichier
    lines = file.readlines()

# Parcourir les lignes et extraire les données
for line in lines[1:]:  # Ignorer la première ligne (en-tête)
    parts = line.strip().split()  # Diviser la ligne en valeurs en utilisant des espaces
    nb_line.append(int(parts[0]))
    naive_duration.append(int(parts[1]))
    lps_duration.append(int(parts[2]))
    optimized_lps_duration.append(int(parts[3]))

# Créer la figure et les sous-graphiques
fig, ax = plt.subplots()

# Tracer les données
ax.plot(nb_line, naive_duration, label='naive_duration')
ax.plot(nb_line, lps_duration, label='lps_duration')
ax.plot(nb_line, optimized_lps_duration, label='optimized_lps_duration')

# Ajouter des titres et une légende
ax.set_title('Comparaison des Durées')
ax.set_xlabel('nb_line')
ax.set_ylabel('Durée')
ax.legend()

# Afficher le graphique
plt.show()


from flask import Flask, jsonify, json, request, render_template
from flask_mysql_connector import MySQL
from flask_bootstrap import Bootstrap

app = Flask( __name__)
app.config['MYSQL_HOST'] = 'localhost'
# A PORTA APONTA PARA A BASE CORRETA ( USERS 3306 )
app.config['MYSQL_PORT'] = '3306'
app.config['MYSQL_USER'] = 'root'
app.config['MYSQL_PASSWORD'] = 'root'
app.config['MYSQL_DB'] = 'sys'
app.config['MYSQL_CHARSET'] = 'utf8'

mysql = MySQL(app)
Bootstrap(app)

@app.errorhandler(404)
def page_not_found(error):
    return render_template('page_not_found.html'), 404

@app.route('/', methods=['GET'])
def index():
    return render_template('index.html')

# @app.route("/")
# def hello():
#df = mysql.execute_sql(SQL, to_pandas=True)
    #return str(df.to_dict())
    #return jsonify({"message":"HelloJson!"})

    #df = mysql.execute_sql(SQL, to_pandas=True)
    #user = df.to_dict()

def execsql(sql):
    cur = mysql.new_cursor(dictionary=True)
    cur.execute(str(sql))
    output  = cur.fetchall()
    cur.close()
    return json.dumps(str(output))

def execsqlone(sql):
    cur = mysql.new_cursor(dictionary=True)
    cur.execute(str(sql))
    output  = cur.fetchone()
    cur.close()
    return json.dumps(str(output))

@app.route('/user/<id>', methods=['GET'])
def user(id):
    SQL = "select * from sys.users where id=%s and active = 1" % id
    #print(SQL)
    user = json.loads(execsqlone(SQL))
    #return str(user)
    print(user)
    if user:
        return user
    else:
        return str("USER NOT FOUND")
    #return str("USER NOT FOUND")

@app.route('/allusers', methods=['GET'])
def allusers():
    SQL = "select * from sys.users"
    return execsql(SQL)

@app.route('/auth', methods=['POST'])
def autenticate():
    if (request.data):
        params = json.loads(request.data)
        print(params)
        # print("user", params['user'])
        # print("user", params['pws'])
        SQL = "select * from sys.users where login='%s' and pws='%s' and active = 1" % (str(params['user']), str(params['pws']))
        #print(SQL)
        user = json.loads(execsqlone(SQL))
        #return str(user)
        print(user)
        if user:
            return user
        else:
            return str("AUTH FAIL")
    return str("AUTH FAIL")

if  __name__ == '__main__':
    # app.run()
    app.run(port=5000)


import pathlib

from fastapi import FastAPI
from fastapi.staticfiles import StaticFiles
from . import route

def create_app():
    _app = FastAPI()
    _app.mount(
        path="/static",
        app=StaticFiles(
            directory=pathlib.Path(__file__).parent.parent / "static",
            html=True
        ),
        name="static"
    )
    _app.include_router(route.rt)
    return _app


from ..models.MongoConnector import *
from bson.objectid import ObjectId
from ..models.UserModel import User
from ..models.GroupModel import Group

class MongoManager(object):
	"""docstring for MongoManager"""
	def __init__(self, conn):
		super(MongoManager, self).__init__()
		self.conn = conn

	def insertOrUpdate(self,jsonobj):
			if "_id" in jsonobj:
				self.conn.replace_one({"_id":ObjectId(jsonobj['_id'])},jsonobj,True)
			else:
				self.conn.insert_one(jsonobj)

	def getId(self,id):
		return self.toModel(self.conn.find_one({"_id":ObjectId(id)}))

	def getDocs(self,querys):
		result=[]
		for query in querys:
			result+=[i for i in self.conn.find(query)]
		result_models = [self.toModel(i) for i in result]
		return result_models

	def toModel(self,result):
		new_object=None
		print(result)
		if "type" in result:
			if result['type']['id'] == 'User':
				new_object = User()
				new_object.setDict(result)
			if result['type']['id'] == 'group':
				new_object = Group()
				new_object.setDict(result)
		return new_object

	def toModels(self,resultset):
		result=[]
		for _row in resultset:
			result.append(self.toModel(_row))
		return result


import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
import numpy as np
import pandas as pd
from datetime import datetime as dt
from dash_auth import BasicAuth

username_passwords = [['admin','lucafaloni']]

# Load and process data

def month_year(col):
    m = col[0]; y = col[1]
    return m + ' ' + str(y)

df = pd.read_csv('data.csv')
df.fillna('__N/A__', inplace=True)

df['Date'] = pd.to_datetime(df['day'])
df['WeekofYear'] = df['Date'].apply(lambda d: d.weekofyear)
df['Year'] = df['Date'].apply(lambda d: d.year)
df['Month_Name'] = df['Date'].apply(lambda d: d.month_name())
df['Month_Year'] = df[['Month_Name','Year']].apply(month_year,axis=1)
df['color'] = df['product_title'].apply(lambda p: p.split()[0])
df['Dummy'] = 1

gdf_month = df.groupby(by='Month_Year').sum().reset_index()
gdf_month['Month_Start'] = pd.to_datetime(gdf_month['Month_Year'])
gdf_month.sort_values(by='Month_Start',inplace=True)

def gdf_month_filter(item,feature):
    global df
    gdf_month = df[df[feature]==item].groupby(by='Month_Year').sum().reset_index()
    gdf_month['Month_Start'] = pd.to_datetime(gdf_month['Month_Year'])
    gdf_month.sort_values(by='Month_Start',inplace=True)
    return gdf_month

def gdf_wd_filter(item,feature,wd,sd,ed):
    global df
    gdf_wd = df[(df[feature]==item) & (df['Date']>=sd) & (df['Date']<=ed)].groupby(by=wd).sum().reset_index()
    gdf_wd.sort_values(by=wd,inplace=True)
    return gdf_wd

external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']

# Build Dashboard

app = dash.Dash(__name__, external_stylesheets=external_stylesheets)
server = app.server

auth = BasicAuth(app,username_passwords)

# Defining Dash Core Components

graph_bymonth = dcc.Graph(
    id='graph_bymonth',
    hoverData={'points': [{'x': 'January 2019'}]}
    )

graph_byweekorday = dcc.Graph(id='graph_byweekorday')

pie_bymonth = dcc.Graph(id='pie_bymonth')

byweekorday = dcc.Dropdown(id='byweekorday', options=[
    {'label':'Week','value':'Week'},
    {'label':'Day','value':'Day'}],
    value='Week')

itemlist = dcc.Dropdown(id='itemlist',multi=True)

value_dict={'total_sales':'Total Sales', 'net_quantity':'Net Quantity'}

feature_dict={'product_type':'Product',
    'variant_title':'Size',
    'color':'Color',
    'shipping_country':'Shipping Country',
    'shipping_city':'Shipping City',
    'shipping_region':'Shipping Region'
}

featurelist = dcc.Dropdown(id='featurelist',options=[
    {'label':feature_dict[k],'value':k} for k in feature_dict
], value='product_type')

value_op = dcc.RadioItems(id='value_op',
    options=[
        {'label':'Total Sales','value':'total_sales'},
        {'label':'Net Quantity','value':'net_quantity'}
    ], value='net_quantity',
    labelStyle={'display': 'inline-block'}
)

date_range = dcc.DatePickerRange(
        id='date_range',
        min_date_allowed=df['Date'].min(),
        max_date_allowed=df['Date'].max(),
        start_date=df['Date'].min(),
        end_date=df['Date'].max(),
        display_format='DD/MM/YYYY',
        clearable=False
    )
# Defining Dash Layout

app.layout = html.Div(children=[
    html.H1(children='Luca Faloni Dashboard',style={'display':'inline-block'}),
    featurelist,
    itemlist,
    html.Div(value_op),
    html.Div(children=[
        graph_bymonth
    ], style={'width':'60%','display':'inline-block'}),
    html.Div(children=[
        pie_bymonth
    ], style={'display':'inline-block','float':'right'}),
    html.Div(),
    html.Div(children=[date_range],style={'display':'inline-block','width':'30%'}),
    html.Div(children=[byweekorday],style={'display':'inline-block','width':'30%'}),
    html.Div(children='Please hover on the main plot to update the Pie Chart data',style={'display':'inline-block','float':'right'}),
    graph_byweekorday
    ])

# Linking Interactive Components

@app.callback(
    Output(component_id='itemlist',component_property='options'),
    [Input(component_id='featurelist',component_property='value')]
)
def set_itemlist_options(feature):
    return [{'label': i, 'value': i} for i in df[feature].unique()]

@app.callback(
    Output(component_id='itemlist',component_property='value'),
    [Input(component_id='featurelist',component_property='value')]
)
def set_itemlist_value(feature):
    return df[feature].unique()

@app.callback(
    Output(component_id='graph_bymonth',component_property='figure'),
    [Input(component_id='itemlist',component_property='value'),
    Input(component_id='featurelist',component_property='value'),
    Input(component_id='value_op', component_property='value')]
)
def update_bymonth(items,feature,value_op):
    global value_dict
    trace=[{'x': gdf_month['Month_Year'], 'y': gdf_month[value_op], 'type':'line','name':'All'}]
    datadict_bymonth_extend = [{'x': gdf_month_filter(item,feature)['Month_Year'], 'y': gdf_month_filter(item,feature)[value_op],
                'type': 'line', 'name': item} for item in items]
    trace.extend(datadict_bymonth_extend)
    return {
        'data':trace,
        'layout': {
            'title': value_dict[value_op] + ' by Month',
            'barmode': 'stack',
            'hovermode': 'closest'
        }
    }

@app.callback(
    Output(component_id='graph_byweekorday',component_property='figure'),
    [Input(component_id='itemlist',component_property='value'),
    Input(component_id='featurelist',component_property='value'),
    Input(component_id='value_op', component_property='value'),
    Input(component_id='date_range', component_property='start_date'),
    Input(component_id='date_range', component_property='end_date'),
    Input(component_id='byweekorday',component_property='value')]
)
def update_byweekorday(items,feature,value_op,start_date,end_date,weekorday):
    global value_dict, df
    xis = 'WeekofYear' if weekorday =='Week' else 'Date'
    trace=[{
        'x': gdf_wd_filter(1,'Dummy',xis,start_date,end_date)[xis],
        'y': gdf_wd_filter(1,'Dummy',xis,start_date,end_date)[value_op],
        'type':'line','name':'All'
    }]
    datadict_bywd_extend = [{
        'x': gdf_wd_filter(item,feature,xis,start_date,end_date)[xis],
        'y': gdf_wd_filter(item,feature,xis,start_date,end_date)[value_op],
        'type': 'lines+marker',
        'name': item} for item in items]
    trace.extend(datadict_bywd_extend)
    return {
        'data':trace,
        'layout': {
            'title': value_dict[value_op] + ' by ' + weekorday,
            'barmode': 'stack',
            'hovermode': 'closest'
        }
    }

@app.callback(
    Output(component_id='pie_bymonth',component_property='figure'),
    [Input(component_id='itemlist',component_property='value'),
    Input(component_id='featurelist',component_property='value'),
    Input(component_id='value_op', component_property='value'),
    Input(component_id='graph_bymonth',component_property='hoverData')]
)
def update_bymonth(items,feature,value_op,month):
    global value_dict, feature_dict, df
    pivotdf = df.pivot_table(values=value_op,index=feature,columns='Month_Year',aggfunc='sum')
    pivotdf.fillna(0,inplace=True)
    # trace_all = [{'labels':['All'], 'values':[0], 'type':'pie', 'hole':0.4}]
    trace = [{'labels':pivotdf.index, 'values':pivotdf[month['points'][0]['x']].get_values(), 'type':'pie', 'hole':0.4}] # [{'x': gdf_month['Month_Year'], 'y': gdf_month[value_op], 'type':'line','name':'All'}]
    return {
        'data':trace,
        'layout': {
            'title': value_dict[value_op]+' Distribution by '+feature_dict[feature]+' for '+month['points'][0]['x'],
            'hovermode': 'closest'
        }
    }

if __name__ == '__main__':
    app.run_server(debug=True)


import argparse
import urllib.request

def download_image():
    parser = argparse.ArgumentParser(description='Web上から画像をダウンロードし、処理を行う。')
    parser.add_argument('--url', '-u', default='https://images-na.ssl-images-amazon.com/images/G/01/img15/pet-products/small-tiles/23695_pets_vertical_store_dogs_small_tile_8._CB312176604_.jpg', help='ダウンロードするイメージのURLを指定する')
    args = parser.parse_args()

    print('Download Image From {0} ....'.format(args.url))
    image_file_path = './sample_images/sample.jpg'
    urllib.request.urlretrieve(args.url, image_file_path)

    return image_file_path


import os,json

lfddir='../apidata_lfd'

# Looks very much like the erroneous 2021-03-26 file (published 2021-03-27) contains the true full LFDcases
#captureddates=sorted(os.listdir(lfddir))
captureddates=sorted(x for x in os.listdir(lfddir) if x[:2]=='20')

dd={}
for fn in captureddates:
  with open(os.path.join(lfddir,fn),'r') as fp:
    dd[fn]={y['date']: y for y in json.load(fp)}

# dd[publishdate][historicaldate]={historical data}

PPVLB={date:0 for date in captureddates}
n=len(captureddates)
for i0 in range(n-1):
  date0=captureddates[i0]
  for i1 in range(i0+1,n):
    date1=captureddates[i1]
    for date2 in captureddates:
      if date2 in dd[date0] and date2 in dd[date1]:
        ppv=1-dd[date1][date2]['newCasesLFDOnlyBySpecimenDate']/dd[date0][date2]['newCasesLFDOnlyBySpecimenDate']
        if ppv>PPVLB[date2]: PPVLB[date2]=ppv

d=dd[captureddates[-1]]

for x in sorted(list(PPVLB)):
  a=d[x]['newCasesLFDOnlyBySpecimenDate']
  b=d[x]['newCasesLFDConfirmedPCRBySpecimenDate']
  print(x,"PPV >= %5.1f%% %5.1f%%"%(PPVLB[x]*100,b/(a+b)*100))