from __future__ import unicode_literals, division, absolute_import, print_function

__license__   = 'GPL v3'
__copyright__ = '2011, Grant Drake'

import re, copy, sys, traceback
from collections import OrderedDict
from functools import partial
from uuid import uuid4
from six import text_type as unicode

from lxml.html import fromstring, tostring
from lxml.html.clean import Cleaner

# Maintain backwards compatibility with older versions of Qt and calibre.
try:
    from qt.core import (QApplication, Qt, QWidget, QTextEdit, QGridLayout,
                        QLabel, QGroupBox, QVBoxLayout, QPushButton, QHBoxLayout,
                        QVBoxLayout, QLineEdit, QToolButton, QMenu, QCheckBox, QSpinBox,
                        QTextCursor, QBrush, QUrl, QDialogButtonBox,
                        QRadioButton, QScrollArea, QComboBox)
except ImportError:
    from PyQt5.Qt import (QApplication, Qt, QWidget, QTextEdit, QGridLayout,
                        QLabel, QGroupBox, QVBoxLayout, QPushButton, QHBoxLayout,
                        QVBoxLayout, QLineEdit, QToolButton, QMenu, QCheckBox, QSpinBox,
                        QTextCursor, QBrush, QUrl, QDialogButtonBox,
                        QRadioButton, QScrollArea, QComboBox)

try:
    qTextCursor_KeepAnchor = QTextCursor.MoveMode.KeepAnchor
    qTextCursor_MoveAnchor = QTextCursor.MoveMode.MoveAnchor
    qTextCursor_Right = QTextCursor.MoveOperation.Right
except:
    qTextCursor_KeepAnchor = QTextCursor.KeepAnchor
    qTextCursor_MoveAnchor = QTextCursor.MoveAnchor
    qTextCursor_Right = QTextCursor.Right

from calibre import browser, random_user_agent, prints
from calibre.ebooks.metadata import fmt_sidx
from calibre.ebooks.BeautifulSoup import BeautifulSoup
from calibre.gui2 import error_dialog, open_url
from calibre.utils.cleantext import clean_ascii_chars
from calibre.utils.ipc.simple_worker import fork_job, WorkerError

import calibre_plugins.import_list.config as cfg
from calibre_plugins.import_list.common_compatibility import qSizePolicy_Preferred, qSizePolicy_Expanding
from calibre_plugins.import_list.common_icons import get_icon
from calibre_plugins.import_list.common_dialogs import SizePersistedDialog
from calibre_plugins.import_list.page_common import AUTHOR_SEPARATOR, TAGS_SEPARATOR
from calibre_plugins.import_list.tab_common import (DragDropComboBox, get_templated_url,
                                tidy_title, tidy_author, tidy_pubdate, tidy_field,
                                DEFAULT_STRIP_PATTERNS, create_standard_columns,
                                AddRemoveFieldDialog, MatchGroupBox)

try:
    load_translations()
except NameError:
    pass

WEB_ENCODINGS = ['utf-8','iso-8859-1','iso-8859-2','latin-1','ascii']

def load_page_in_browser(url, delay):
    raw = None
    try:
        br = browser(random_user_agent())
        raw = br.open_novisit(url).read()
    except:
        pass
    return raw

class RexexStripDialog(SizePersistedDialog):

    def __init__(self, parent, field_name, regex, regex_is_strip):
        SizePersistedDialog.__init__(self, parent, 'import list plugin:regex strip dialog')

        self.setWindowTitle(field_name)
        layout = QVBoxLayout(self)
        self.setLayout(layout)

        hrl = QHBoxLayout()
        layout.addLayout(hrl)
        self.regxex_is_strip_radio = QRadioButton(_('Strip matching text'), self)
        self.regex_is_inclusive_radio = QRadioButton(_('Include matching text'), self)
        hrl.addWidget(self.regxex_is_strip_radio)
        hrl.addWidget(self.regex_is_inclusive_radio)
        hrl.addStretch(1)
        if regex_is_strip:
            self.regxex_is_strip_radio.setChecked(True)
        else:
            self.regex_is_inclusive_radio.setChecked(True)

        hl = QHBoxLayout()
        layout.addLayout(hl)

        self.strip_regex_ledit = QLineEdit(regex, self)
        self.strip_regex_ledit.setMinimumWidth(150)
        self.standard_pat_button = QToolButton(self)
        self.standard_pat_button.setToolTip(_('Choose a predefined pattern'))
        self.standard_pat_button.setMenu(self._create_standard_pat_menu())
        self.standard_pat_button.setIcon(get_icon('images/script.png'))
        self.standard_pat_button.setPopupMode(QToolButton.InstantPopup)
        self.clear_button = QToolButton(self)
        self.clear_button.setToolTip(_('Clear the regular expression'))
        self.clear_button.setIcon(get_icon('trash.png'))
        self.clear_button.clicked.connect(self._clear_regex)
        hl.addWidget(self.strip_regex_ledit, 1)
        hl.addWidget(self.standard_pat_button)
        hl.addWidget(self.clear_button)

        button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
        button_box.accepted.connect(self.accept)
        button_box.rejected.connect(self.reject)
        layout.addWidget(button_box)

        # Cause our dialog size to be restored from prefs or created on first usage
        self.resize_dialog()

    def _create_standard_pat_menu(self):
        menu = QMenu(self)
        for name, regex in DEFAULT_STRIP_PATTERNS:
            action = menu.addAction(name)
            action.triggered.connect(partial(self._assign_standard_pattern, regex))
        return menu

    def _assign_standard_pattern(self, regex):
        self.strip_regex_ledit.setText(regex)

    def _clear_regex(self):
        self.strip_regex_ledit.clear()
        self.raw_list = []

    def regex_text(self):
        return unicode(self.strip_regex_ledit.text())

    def regex_is_strip(self):
        return self.regxex_is_strip_radio.isChecked()

class ListComboBox(QComboBox):

    def __init__(self, parent, values, selected_value=None):
        QComboBox.__init__(self, parent)
        self.values = values
        if selected_value is not None:
            self.populate_combo(selected_value)

    def populate_combo(self, selected_value):
        self.blockSignals(True)
        self.clear()
        selected_idx = idx = -1
        for value in self.values:
            idx = idx + 1
            self.addItem(value)
            if value == selected_value:
                selected_idx = idx
        self.setCurrentIndex(selected_idx)
        self.blockSignals(False)

    def selected_value(self):
        return unicode(self.currentText())

class ImportWebPageTab(QWidget):

    def __init__(self, parent_page):
        self.parent_page = parent_page
        QWidget.__init__(self)
        self.is_changed_text = False
        self.xpath_row_controls = OrderedDict()
        self._init_controls()
        self._load_column_metadata()
        self.raw_list = []

    def _init_controls(self):
        self.block_events = False

        vl = QVBoxLayout()
        self.setLayout(vl)

        l = self.l = QGridLayout()
        vl.addLayout(l)

        url_lbl = QLabel(_('&Download from url:'), self)
        url_lbl.setAlignment(Qt.AlignLeft | Qt.AlignBottom)
        self.setting_lbl = QLabel('', self)
        self.setting_lbl.setAlignment(Qt.AlignRight | Qt.AlignBottom)
        self.url_combo = DragDropComboBox(self, drop_mode='url')
        self.url_combo.editTextChanged.connect(self._on_url_changed)
        url_lbl.setBuddy(self.url_combo)
        l.addWidget(url_lbl, 0, 0, 1, 1)
        l.addWidget(self.setting_lbl, 0, 1, 1, 1)
        cul = QHBoxLayout()
        l.addLayout(cul, 1, 0, 1, 2)
        cul.addWidget(self.url_combo, 1)
        self.paste_button = QToolButton(self)
        self.paste_button.setToolTip(_('Paste a URL from your clipboard'))
        self.paste_button.setIcon(get_icon('edit-paste.png'))
        self.paste_button.clicked.connect(self._paste_url)
        cul.addWidget(self.paste_button)
        self.open_browser_button = QToolButton(self)
        self.open_browser_button.setToolTip(_('Open this url in your web browser'))
        self.open_browser_button.setIcon(get_icon('images/browser.png'))
        self.open_browser_button.clicked.connect(self._open_in_browser)
        cul.addWidget(self.open_browser_button)

        wl = QHBoxLayout()
        l.addLayout(wl, 2, 0, 1, 2)
        source_lbl = QLabel(_('&Source html:'), self)
        encoding_lbl = QLabel(_('&Encoding:'), self)
        encoding_lbl.setToolTip(_('Specify the web page encoding if not utf-8'))
        self.encoding_combo = ListComboBox(self, WEB_ENCODINGS, WEB_ENCODINGS[0])
        encoding_lbl.setBuddy(self.encoding_combo)
        self.javascript_checkbox = QCheckBox(_('Load javascript'), self)
        self.javascript_checkbox.setToolTip(_('Enable this option if the web page renders its content using javascript'))
        delay_lbl = QLabel(_('&Delay:'), self)
        delay_lbl.setToolTip(_('Additional delay in seconds to wait for web page javascript to execute'))
        self.delay_spin = QSpinBox(self)
        self.delay_spin.setRange(0, 60)
        delay_lbl.setBuddy(self.delay_spin)
        wl.addWidget(source_lbl, 1)
        wl.addWidget(encoding_lbl)
        wl.addWidget(self.encoding_combo)
        wl.addWidget(self.javascript_checkbox)
        wl.addWidget(delay_lbl)
        wl.addWidget(self.delay_spin)

        self.source_tedit = QTextEdit(self)
        if hasattr(self.source_tedit, 'setTabStopDistance'):
            self.source_tedit.setTabStopDistance(24)
        else:
            self.source_tedit.setTabStopWidth(24)
        self.source_tedit.textChanged.connect(self._source_text_changed)
        source_lbl.setBuddy(self.source_tedit)
        l.addWidget(self.source_tedit, 3, 0, 1, 2)
        l.setRowStretch(3, 1)

        fgb = QGroupBox(' '+_('Find:')+' ', self)
        fgb.setStyleSheet('QGroupBox { font-weight: bold; }')
        l.addWidget(fgb, 4, 0, 1, 2)
        fgbl = QHBoxLayout()
        fgb.setLayout(fgbl)
        find_lbl = QLabel(_('&Text:'), self)
        find_lbl.setToolTip(_('Search for text to help locate the region of interest such as\n'
                            'a book name when designing your xpath expressions'))
        self.find_ledit = QLineEdit('', self)
        self.find_ledit.setPlaceholderText(_('Search for specific text'))
        find_lbl.setBuddy(self.find_ledit)
        self.find_button = QPushButton(get_icon('search.png'), _('&Find'), self)
        self.find_button.clicked.connect(self._find_text)
        fgbl.addWidget(find_lbl)
        fgbl.addWidget(self.find_ledit, 1)
        fgbl.addWidget(self.find_button)

        self.mgb = MatchGroupBox(self)
        self.match_layout = QVBoxLayout()
        self.match_layout.addWidget(self.mgb)
        l.addLayout(self.match_layout, 5, 0, 1, 2)

        gb = QGroupBox(' '+_('XPath:')+' ', self)
        gb.setStyleSheet('QGroupBox { font-weight: bold; }')

        gbl = QVBoxLayout()
        gb.setLayout(gbl)
        xpath_lbl = QLabel(_('Specify XPath expressions to identify the parent '
                           'rows and title / authors within each row in the source html.'), self)
        xpath_lbl.setWordWrap(True)
        xpath_lbl.linkActivated.connect(self.parent_page.open_external_link)
        gbl.addWidget(xpath_lbl)

        self.xpath_layout = QGridLayout()
        gbl.addLayout(self.xpath_layout)

        scroll = QScrollArea()
        scroll.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded)
        scroll.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)
        scroll.setWidgetResizable(True)
        scroll.setSizePolicy(qSizePolicy_Expanding, qSizePolicy_Preferred)
        scroll.setWidget(gb)
        l.addWidget(scroll, 6, 0, 1, 2)
        l.setRowStretch(6, 1)
        # Controls will be added dynamically when grid is populated

        xpbl = QHBoxLayout()
        l.addLayout(xpbl, 7, 0, 1, 2)
        self.row_count_lbl = QLabel(_('0 occurrences'), self)
        self.row_count_lbl.setToolTip(_('Number of matches using Test'))
        self.test_previous_button = QToolButton(self)
        self.test_previous_button.setToolTip(_('Previous test result'))
        self.test_previous_button.setIcon(get_icon('arrow-up.png'))
        self.test_previous_button.setEnabled(False)
        self.test_previous_button.clicked.connect(self._goto_previous)
        self.test_next_button = QToolButton(self)
        self.test_next_button.setToolTip(_('Next test result'))
        self.test_next_button.setIcon(get_icon('arrow-down.png'))
        self.test_next_button.setEnabled(False)
        self.test_next_button.clicked.connect(self._goto_next)
        xpbl.addStretch(1)
        xpbl.addWidget(self.row_count_lbl)
        xpbl.addWidget(self.test_previous_button)
        xpbl.addWidget(self.test_next_button)

        butl = QHBoxLayout()
        vl.addLayout(butl)

        self.clear_button = QPushButton(get_icon('trash.png'), _('&Clear'), self)
        self.clear_button.setToolTip(_('Clear all settings back to the defaults on this tab'))
        self.clear_button.clicked.connect(self._clear_to_defaults)
        self.add_field_button = QPushButton(get_icon('column.png'), _('&Fields')+'...', self)
        self.add_field_button.setToolTip(_('Select field(s) to import'))
        self.add_field_button.clicked.connect(self._add_field)
        self.reverse_list_checkbox = QCheckBox(_('Reverse order'), self)
        self.reverse_list_checkbox.setToolTip(_('Display the books in the opposite order to the source'))
        self.preview_button = QPushButton(get_icon('wizard.png'), _('&Preview'), self)
        self.preview_button.setToolTip(_('Preview the results in the books grid'))
        self.preview_button.clicked.connect(self._preview_rows)
        butl.addWidget(self.clear_button)
        butl.addWidget(self.add_field_button)
        butl.addStretch(1)
        butl.addWidget(self.reverse_list_checkbox)
        butl.addWidget(self.preview_button)

        self.block_events = False

    def _load_column_metadata(self):
        self.possible_columns = OrderedDict()
        self.possible_columns['rows'] ='Rows'
        for k,v in create_standard_columns(self.parent_page.db).items():
            self.possible_columns[k] = v

    def _add_field(self):
        used_fields = list(self.xpath_row_controls.keys())
        d = AddRemoveFieldDialog(self, self.possible_columns, used_fields)
        if d.exec_() == d.Accepted:
            for remove_field in d.removed_names:
                self._clear_xpath_controls_for_row(remove_field)
            for new_field in d.added_names:
                self._append_xpath_row_controls(new_field)
        self.mgb._refresh_match_opts()

    def _source_text_changed(self):
        if self.block_events:
            return
        self.is_changed_text = True

    def _find_text(self):
        find_text = unicode(self.find_ledit.text()).lower()
        txt = unicode(self.source_tedit.toPlainText()).lower()
        if not txt or not find_text:
            return
        cursor_idx = self.source_tedit.textCursor().position() + 1
        next_match_idx = -1
        if cursor_idx + len(find_text) < len(txt):
            next_match_idx = txt.find(find_text, cursor_idx + 1)
        if next_match_idx < 0:
            next_match_idx = txt.find(find_text)
        if next_match_idx < 0:
            return error_dialog(self, _('No matches'), _('No matches for:') + ' ' + find_text, show=True, show_copy_button=False)
        self._goto_loc(next_match_idx, n=len(find_text))
        self.source_tedit.setFocus()

    def _clear_xpath_controls_for_row(self, key):
        if key not in self.xpath_row_controls:
            return
        row_controls = self.xpath_row_controls[key]
        self.xpath_layout.removeWidget(row_controls['label'])
        row_controls['label'].setParent(None)
        self.xpath_layout.removeWidget(row_controls['ledit'])
        row_controls['ledit'].setParent(None)
        self.xpath_layout.removeWidget(row_controls['test_xpath_button'])
        row_controls['test_xpath_button'].clicked.disconnect()
        row_controls['test_xpath_button'].setParent(None)
        if 'regex_button' in row_controls:
            self.xpath_layout.removeWidget(row_controls['regex_button'])
            row_controls['regex_button'].clicked.disconnect()
            row_controls['regex_button'].setParent(None)
        del self.xpath_row_controls[key]

    def _clear_xpath_controls(self, add_default_fields=True):
        for key in list(self.xpath_row_controls.keys()):
            self._clear_xpath_controls_for_row(key)
        if add_default_fields:
            self._append_xpath_row_controls('rows')
            self._append_xpath_row_controls('title')
            self._append_xpath_row_controls('authors')

    def _append_xpath_row_controls(self, field_name, xpath_text='', regex_text='', is_strip_regex=True):
        row_controls = {}
        row = self.xpath_layout.rowCount()
        # It should be possible to load a saved setting for an identifier column
        # even if that column is not one of our "known" ones.
        if field_name not in self.possible_columns:
            if field_name.startswith('identifier:'):
                self.possible_columns[field_name] = field_name.replace('identifier:', 'ID:')
        display_name = self.possible_columns.get(field_name, '')

        lbl = QLabel(display_name + ':', self)
        ledit = QLineEdit(xpath_text, self)
        test_xpath_button = QToolButton(self)
        test_xpath_button.setIcon(get_icon('images/color.png'))
        test_xpath_button.clicked.connect(partial(self._test_xpath, field_name))
        self.xpath_layout.addWidget(lbl, row, 0, 1, 1)
        self.xpath_layout.addWidget(ledit, row, 1, 1, 1)
        self.xpath_layout.addWidget(test_xpath_button, row, 2, 1, 1)
        row_controls['label'] = lbl
        row_controls['ledit'] = ledit
        row_controls['test_xpath_button'] = test_xpath_button
        # Special cases hard-coded
        if field_name == 'rows':
            lbl.setToolTip(_('Identify the parent node in the html source that contains a single book'))
        else:
            regex_button = QToolButton(self)
            self._set_regex_button_visual(regex_button, regex_text, is_strip_regex)
            regex_button.clicked.connect(partial(self._modify_regex, regex_button, field_name))
            self.xpath_layout.addWidget(regex_button, row, 3, 1, 1)
            row_controls['regex_button'] = regex_button
            row_controls['regex'] = regex_text
            row_controls['regex_is_strip'] = is_strip_regex
        self.xpath_row_controls[field_name] = row_controls

    def _clear_to_defaults(self, clear_last=True, add_default_fields=True):
        self.block_events = True
        self._clear_setting_name(clear_last)
        self.url_combo.clearEditText()
        self.encoding_combo.setCurrentIndex(0)
        self.javascript_checkbox.setChecked(False)
        self.source_tedit.clear()
        self.raw_list = []
        self.find_ledit.setText('')
        self._clear_xpath_controls(add_default_fields)
        self.is_changed_text = False
        self.match_locs = []
        self.test_previous_button.setEnabled(False)
        self.test_next_button.setEnabled(False)
        self.reverse_list_checkbox.setChecked(False)
        self.block_events = False
        self.parent_page.clear_preview_books()
        self.mgb._refresh_match_opts()

    def _modify_regex(self, regex_button, field_name):
        regex = self.xpath_row_controls[field_name]['regex']
        regex_is_strip = self.xpath_row_controls[field_name]['regex_is_strip']
        d = RexexStripDialog(self, field_name, regex, regex_is_strip)
        if d.exec_() == d.Accepted:
            self.xpath_row_controls[field_name]['regex'] = d.regex_text()
            self.xpath_row_controls[field_name]['regex_is_strip'] = d.regex_is_strip()
            self._set_regex_button_visual(regex_button, d.regex_text(), d.regex_is_strip())

    def _set_regex_button_visual(self, regex_button, regex, is_strip_regex):
        if regex:
            regex_button.setIcon(get_icon('images/regex_specified.png'))
            if is_strip_regex:
                regex_button.setToolTip(_('Strip: %s')% regex)
            else:
                regex_button.setToolTip(_('Include: %s')% regex)
        else:
            regex_button.setIcon(get_icon('images/regex.png'))
            regex_button.setToolTip(_('No expression'))

    def _clear_setting_name(self, clear_last=True):
        if clear_last:
            self.parent_page.library_config[cfg.KEY_LAST_WEB_SETTING] = ''
        self.setting_lbl.setText('')
        self.parent_page.info['current_setting'] = ''

    def _paste_url(self):
        cb = QApplication.instance().clipboard()
        self._clear_setting_name()
        txt = unicode(cb.text()).strip()
        self.url_combo.setEditText(txt)

    def _on_url_changed(self):
        self.block_events = True
        self.source_tedit.clear()
        self.raw_list = []
        self.block_events = False

    def _open_in_browser(self):
        url = get_templated_url(unicode(self.url_combo.currentText()).strip())
        if url:
            open_url(QUrl(url))

    def _load_html(self, url, encoding, render_javascript):
        if url.lower().startswith('file://'):
            fname = QUrl(url).toLocalFile()
            with open(fname, 'rb') as f:
                raw = f.read()
        else:
            if render_javascript:
                try:
                    res = fork_job('calibre_plugins.import_list.tab_webpage', 'load_page_in_browser',
                            (url, int(unicode(self.delay_spin.value()))))
                except WorkerError as e:
                    prints(e.orig_tb)
                    raise RuntimeError('Failed to run load_page_in_browser')
                raw = res['result']
            else:
                br = browser(url, user_agent=random_user_agent())
                raw = br.open_novisit(url).read()
        if raw:
            raw = raw.decode(encoding, errors='replace').encode('utf-8')
            raw = clean_ascii_chars(raw).replace(b'\r', b'')
            #open('''D:\\import-list-raw.html''','wb').write(raw)
        return raw

    def get_page_range(self, url):
        page_range = re.sub(r'.*\{(\d+)\-(\d+)\}.*', r'\1,\2', url)
        try:
            page_range = [ int(x) for x in page_range.split(',') ]
            first, last = page_range
            return first, last
        except:
            return None

    def url_with_page_no(self, url, page_no):
        url = re.sub(r'(.*)\{\d+\-\d+\}(.*)', r'\1__page_no__\2'.format(page_no), url)
        url = url.replace('__page_no__', unicode(page_no))
        return url

    def _download_source(self):
        url = unicode(self.url_combo.currentText()).strip()
        if not url:
            self.block_events = True
            self.source_tedit.clear()
            self.raw_list = []
            self.block_events = False
            self.parent_page.clear_preview_books()
            return None

        page_range = self.get_page_range(url)
        if page_range:
            first, last = page_range
            urls = [self.url_with_page_no(url, x) for x in range(first, last+1)]
        else:
            urls = [ get_templated_url(unicode(self.url_combo.currentText()).strip()) ]

        self.url_combo.reorder_items()
        encoding = self.encoding_combo.selected_value()
        render_javascript = self.javascript_checkbox.isChecked()

        QApplication.setOverrideCursor(Qt.WaitCursor)
        try:
            for idx, url in enumerate(urls):
                try:
                    raw = self._load_html(url, encoding, render_javascript)

                    # Now prettify the html
                    soup = BeautifulSoup(raw)
                    raw = soup.prettify()#.decode('utf-8')
                    #open('''D:\\raw_prettified.html''','wb').write(raw)
                    root = fromstring(raw)
                    # Strip out loads of nonsense like Javascript and CSS which won't be used to parse
                    cleaner = Cleaner(style=True, page_structure=False)
                    root = cleaner.clean_html(root)
                    raw = tostring(root, method='html')
                    #open('D:\\raw_cleaned.html','wb').write(raw)
                    # We convert another time between root and raw to be 100% sure we can highlight correctly
                    # It's a pain from a performance perspective but only way to be sure due to issues found
                    # with Fantastic Fiction not showing the right results
                    root = fromstring(raw)
                    raw = tostring(root, method='html')
                    #open('D:\\raw_reconverted.html','wb').write(raw)
                    self.raw_list.append(raw)

                    if idx == 0:
                        self.block_events = True
                        self.source_tedit.setPlainText(unicode(raw, 'utf-8'))
                        self.block_events = False

                        self.test_next_button.setEnabled(False)
                        self.test_previous_button.setEnabled(False)
                except:
                    traceback.print_exc()
                    msg = _('Failed to download page')+':\n' + unicode(sys.exc_info()[1])
                    error_dialog(self.parent_page, _('Failed to download page'), msg, show=True)
                    #return None
        finally:
            QApplication.restoreOverrideCursor()
            return self.raw_list

    def _get_matching_elements(self, root, row_xpath, field_name, xpath):
        elements = []

        def iterate_matches(parent):
            # Any special cases up the top
            if field_name == 'series_index':
                series_indexes = parent.xpath(xpath)
                for series_index in series_indexes:
                    try:
                        _si = float(series_index)
                        elements.append(series_index)
                    except:
                        pass
            else:
                values = parent.xpath(xpath)
                for value_name in values:
                    if value_name is not None and len(value_name):
                        elements.append(value_name)

        if row_xpath:
            rows = root.xpath(row_xpath)
            for row in rows:
                if field_name == 'rows':
                    elements.append(row)
                else:
                    iterate_matches(row)
        else:
            iterate_matches(root)
        return elements

    def _test_xpath(self, field_name='rows'):
        self.test_next_button.setEnabled(False)
        self.test_previous_button.setEnabled(False)

        rows_xpath = unicode(self.xpath_row_controls['rows']['ledit'].text()).strip()
        xpath = unicode(self.xpath_row_controls[field_name]['ledit'].text()).strip()
        if not xpath:
            return error_dialog(self, _('Test failed'), _('You must specify an xpath'), show=True)

        raw = unicode(self.source_tedit.toPlainText())
        if not self.raw_list:
            raw_list = self._download_source()
            raw = raw_list[0]
        if not raw:
            return error_dialog(self, _('Test failed'), _('No html source available to match against'), show=True)

        QApplication.setOverrideCursor(Qt.WaitCursor)
        root = fromstring(raw)
        if self.is_changed_text:
            # Have to replace the text in case the tostring/fromstring changed nodes
            new_raw = tostring(root, method='html')
            self.source_tedit.setPlainText(new_raw)
            self.is_changed_text = False
        #Remove any previous selections
        self.source_tedit.setExtraSelections([])
        # Find all our matching elements
        try:
            elements = self._get_matching_elements(root, rows_xpath, field_name, xpath)
        except:
            QApplication.restoreOverrideCursor()
            error_dialog(self, _('Invalid xpath'),
                         _('Your xpath expression is invalid')+':\n'+unicode(sys.exc_info()[1]),
                         show=True, det_msg=traceback.format_exc())
            return
        finally:
            QApplication.restoreOverrideCursor()

        color = Qt.yellow
        matches, uuid_raw = self._search_for_matches(root, elements)
        self.row_count_lbl.setText(unicode(len(matches)) + ' '+_('occurrences'))
        if matches:
            self._highlight_matches(matches, uuid_raw, color)

    def _search_for_matches(self, root, matching_elements):
        matches = []
        prefix = ''
        # Insert a uuid before and after the opening/closing of the matching tag
        # We do this so able to find this text again afterwards
        for elem in matching_elements:
            open_id = unicode(uuid4())
            close_id = unicode(uuid4())
            if hasattr(elem, 'tag'):
                #print(open_id, close_id, 'has a tag')
                prev = elem.getprevious()
                if prev is not None:
                    #print('Prev=', prev.tag)
                    if prev.tail is None:
                        prev.tail = open_id
                    else:
                        prev.tail = prev.tail + open_id
                else:
                    prev = elem.getparent()
                    if prev is not None:
                        #print('Parent=', prev.tag)
                        if prev.text is None:
                            prev.text = open_id
                        else:
                            prev.text = prev.text + open_id
                    else:
                        # Special case of the first element
                        prefix += open_id

                if elem.tail is None:
                    elem.tail = close_id
                else:
                    elem.tail = close_id + elem.tail
            else:
                #print(open_id, close_id, 'has no tag')
                parent_elem = elem.getparent()
                if parent_elem.text is None:
                    # Assume this is a self-closing element
                    parent_elem.tail = open_id + parent_elem.tail + close_id
                else:
                    parent_elem.text = open_id + parent_elem.text + close_id
            matches.append((open_id, close_id))
        uuid_raw = prefix + tostring(root, method='html').decode(encoding='utf-8')
        return matches, uuid_raw

    def _highlight_matches(self, matches, uuid_raw, color=Qt.yellow):
        selections = []
        self.match_locs = []
        cursor = QTextCursor(self.source_tedit.document())
        extsel = QTextEdit.ExtraSelection()
        extsel.cursor = cursor
        extsel.format.setBackground(QBrush(color))
        #open('d:\\uuid_raw.html','wb').write(uuid_raw)
        try:
            offset = 0
            for open_id, close_id in matches:
                start_pos = uuid_raw.find(open_id, offset) - offset
                offset += len(open_id)
                end_pos = uuid_raw.find(close_id, start_pos) - offset
                offset += len(close_id)
                #print('Open', open_id,'Close',close_id,'Start=',start_pos,'End=',end_pos,'Offset=',offset)
                es = QTextEdit.ExtraSelection(extsel)
                es.cursor.setPosition(start_pos, qTextCursor_MoveAnchor)
                es.cursor.setPosition(end_pos, qTextCursor_KeepAnchor)
                selections.append(es)
                self.match_locs.append((start_pos, end_pos))
        except Exception as e:
            pass
        self.source_tedit.setExtraSelections(selections)
        if self.match_locs:
            self.test_next_button.setEnabled(True)
            self.test_previous_button.setEnabled(True)
            self._goto_loc(0)
            self._goto_next()
        else:
            self.test_next_button.setEnabled(False)
            self.test_previous_button.setEnabled(False)

    def _goto_previous(self):
        pos = self.source_tedit.textCursor().position()
        if self.match_locs:
            match_loc = len(self.match_locs) - 1
            for i in range(len(self.match_locs) - 1, -1, -1):
                loc = self.match_locs[i][1]
                if pos > loc:
                    match_loc = i
                    break
            self._goto_loc(self.match_locs[match_loc][0])

    def _goto_next(self):
        pos = self.source_tedit.textCursor().position()
        if self.match_locs:
            match_loc = 0
            for i in range(len(self.match_locs)):
                loc = self.match_locs[i][0]
                if pos < loc:
                    match_loc = i
                    break
            self._goto_loc(self.match_locs[match_loc][0])

    def _goto_loc(self, loc, operation=qTextCursor_Right, mode=qTextCursor_KeepAnchor, n=0):
        doc = self.source_tedit.document()
        cursor = QTextCursor(doc)
        if n:
            cursor.setPosition(loc)
            cursor.movePosition(operation, mode, n)
            self.source_tedit.setTextCursor(cursor)
        else:
            cursor.setPosition(doc.characterCount() - 1)
            self.source_tedit.setTextCursor(cursor)
            cursor.setPosition(loc)
            self.source_tedit.setTextCursor(cursor)

    def _preview_rows(self):
        if self.block_events:
            return

        raw = unicode(self.source_tedit.toPlainText()).strip()
        raw_list = self.raw_list
        if not raw_list:
            raw_list = self._download_source()
        if not raw_list:
            self.parent_page.clear_preview_books()
            return

        xpaths = {}
        has_xpath = False
        for field_name, controls in self.xpath_row_controls.items():
            xpath = unicode(controls['ledit'].text()).strip()
            if xpath:
                has_xpath = True
            xpaths[field_name] = xpath
        if not has_xpath:
            return error_dialog(self, _('Invalid xpath'), _('You have not specified any xpath expressions'), show=True)
        if not xpaths['title']:
            return error_dialog(self, _('Invalid xpath'), _('You have not specified a title xpath expression'), show=True)
        if 'series' in xpaths and 'series_index' not in xpaths:
            return error_dialog(self, _('Invalid xpath'), _('You must add a series_index xpath field if using series'), show=True)
        if 'series_index' in xpaths and 'series' not in xpaths:
            return error_dialog(self, _('Invalid xpath'), _('You must add a series xpath field if using series index'), show=True)
        rows_xpath = xpaths['rows']
        del xpaths['rows']

        field_regexes = {}
        for field_name, controls in self.xpath_row_controls.items():
            if 'regex' in controls:
                regex_text = controls['regex'].strip()
                if regex_text:
                    regex = re.compile(regex_text)
                    field_regexes[field_name] = (regex, controls['regex_is_strip'])

        QApplication.setOverrideCursor(Qt.WaitCursor)
        try:
            #root = fromstring(raw)
            books = []
            custom_columns = self.parent_page.db.field_metadata.custom_field_metadata()

            def append_matches(row_name_values):
                max_values = 0
                for values in row_name_values.values():
                    if len(values) > max_values:
                        max_values = len(values)
                for i in range(0, max_values):
                    book = {}
                    for field_name, values in row_name_values.items():
                        # Special case for series/series_index which are paired together
                        if field_name == 'series_index' or (field_name.startswith('#') and field_name.endswith('_index')):
                            continue
                        if i >= len(values) and field_name not in ['authors', 'series']:
                            # Special cases for assume same series/author on each row
                            book[field_name] = ''
                            continue
                        regex, regex_is_strip = field_regexes.get(field_name, (None, True))
                        value_text = ''
                        col = {}
                        if field_name.startswith('#') and field_name in custom_columns:
                            col = custom_columns[field_name]
                        # Special cases for some fields
                        if field_name == 'authors':
                            if len(values) == 1:
                                value_text = tidy_author(values[0], regex, regex_is_strip)
                            elif i < len(values):
                                value_text = tidy_author(values[i], regex, regex_is_strip)
                        elif field_name == 'series' or col.get('datatype','') == 'series':
                            series_name = ''
                            series_index = ''
                            if len(values) == 1:
                                series_name = tidy_field(values[0], regex, regex_is_strip)
                            elif i < len(values):
                                series_name = tidy_field(values[i], regex, regex_is_strip)
                            si_regex, si_regex_is_strip = field_regexes.get(field_name+'_index', (None, True))
                            if i >= len(row_name_values[field_name+'_index']):
                                series_index = '0'
                            else:
                                series_index = tidy_field(row_name_values[field_name+'_index'][i], si_regex, si_regex_is_strip)
                            if series_name and series_index:
                                try:
                                    value_text = '%s [%s]'%(series_name, fmt_sidx(series_index))
                                except:
                                    value_text = series_name + ' [0]'
                        elif field_name == 'title':
                            value_text = tidy_title(values[i], regex, regex_is_strip)
                        elif field_name == 'pubdate':
                            # May be able to remove this special case if put a regex
                            value_text = tidy_pubdate(values[i], regex, regex_is_strip)
                        elif col.get('datatype','') == 'bool':
                            value_text = tidy_field(values[i], regex, regex_is_strip)
                            if value_text.lower() in ['true','yes','y','1']:
                                value_text = 'Yes'
                            elif value_text.lower() in ['n/a','undefined','']:
                                value_text = 'n/a'
                            else:
                                value_text = 'No'
                        else:
                            value_text = tidy_field(values[i], regex, regex_is_strip)
                        book[field_name] = value_text
                    # Only append book if has a title
                    if len(book.get('title','')):
                        books.append(book)

            for raw in raw_list:
                root = fromstring(raw)
                if rows_xpath:
                    rows = root.xpath(rows_xpath)
                    for row in rows:
                        row_name_values = {}
                        for field_name, xpath in xpaths.items():
                            field_values = []
                            if xpath:
                                matches = row.xpath(xpath)
                                for match in matches:
                                    field_value = match.strip()
                                    if field_value:
                                        field_values.append(field_value)
                            # If multiple values for a field, append them into a single value for this row.
                            if len(field_values) > 1 and field_name == 'authors':
                                row_name_values[field_name] = [AUTHOR_SEPARATOR.join(field_values)]
                            elif len(field_values) > 1 and field_name == 'tags':
                                row_name_values[field_name] = [TAGS_SEPARATOR.join(field_values)]
                            elif len(field_values) > 1 and field_name in custom_columns and custom_columns[field_name]['is_multiple']:
                                if 'is_names' in custom_columns[field_name]['display'] and custom_columns[field_name]['display']['is_names']:
                                    row_name_values[field_name] = [AUTHOR_SEPARATOR.join(field_values)]
                                else:
                                    row_name_values[field_name] = [TAGS_SEPARATOR.join(field_values)]
                            else:
                                row_name_values[field_name] = field_values
                        append_matches(row_name_values)
                else:
                    row_name_values = {}
                    for field_name, xpath in xpaths.items():
                        if xpath:
                            row_name_values[field_name] = root.xpath(xpath)
                        else:
                            row_name_values[field_name] = []
                    append_matches(row_name_values)

            if self.reverse_list_checkbox.isChecked():
                books.reverse()
            columns = self._get_current_columns()
            self.parent_page.refresh_preview_books(columns, books, cfg.KEY_IMPORT_TYPE_WEB)
        except:
            QApplication.restoreOverrideCursor()
            msg = _('Failed to process page')+':\n' + unicode(sys.exc_info()[1])
            return error_dialog(self.parent_page, _('Failed to process page'), msg, show=True,
                         det_msg=traceback.format_exc())
        else:
            QApplication.restoreOverrideCursor()

    def _get_current_columns(self):
        # Identify which columns the user has configured
        columns = ['title', 'authors']
        for field_name in self.xpath_row_controls.keys():
            if field_name not in columns and field_name != 'rows':
                columns.append(field_name)
        return columns

    def restore_settings(self, library_config):
        self._clear_to_defaults(clear_last=False, add_default_fields=False)
        context = library_config[cfg.KEY_CURRENT].get(cfg.KEY_IMPORT_TYPE_WEB, None)
        if context is None:
            context = copy.deepcopy(cfg.DEFAULT_WEB_SETTING_VALUES)
        self.url_combo.populate_items(library_config[cfg.KEY_WEB_URLS],
                                      context[cfg.KEY_WEB_URL])
        # When populating, have to cater for our "default" rows.
        for data in context[cfg.KEY_WEB_XPATH_DATA]:
            field_name = data[cfg.KEY_WEB_FIELD]
            field_xpath = data[cfg.KEY_WEB_XPATH]
            strip_regex = data.get(cfg.KEY_WEB_REGEX, '')
            is_strip_regex = data.get(cfg.KEY_WEB_REGEX_IS_STRIP, True)
            self._append_xpath_row_controls(field_name, field_xpath, strip_regex, is_strip_regex)
        # Update: add match by identifier {
        # this must be done after restoring csv rows to be able to restore match_by_identifier option
        match_settings = context[cfg.KEY_MATCH_SETTINGS]
        self.mgb.set_match_opts(match_settings)
        #}
        self.reverse_list_checkbox.setChecked(context.get(cfg.KEY_WEB_REVERSE_LIST, False))
        self.encoding_combo.populate_combo(context.get(cfg.KEY_WEB_ENCODING, 'utf-8'))
        self.javascript_checkbox.setChecked(context.get(cfg.KEY_WEB_JAVASCRIPT, False))
        self.delay_spin.setValue(library_config.get(cfg.KEY_JAVASCRIPT_DELAY, cfg.DEFAULT_DELAY))
        last_setting_name = library_config.get(cfg.KEY_LAST_WEB_SETTING, '')
        self.setting_lbl.setText('<b>%s</b>'%last_setting_name)

    def save_settings(self, library_config):
        library_config[cfg.KEY_WEB_URLS] = self.url_combo.get_items_list()
        library_config[cfg.KEY_JAVASCRIPT_DELAY] = int(unicode(self.delay_spin.value()))
        context = library_config[cfg.KEY_CURRENT].get(cfg.KEY_IMPORT_TYPE_WEB, None)
        if context is None:
            context = copy.deepcopy(cfg.DEFAULT_WEB_SETTING_VALUES)
        context[cfg.KEY_WEB_URL] = unicode(self.url_combo.currentText())
        # Update: add match by identifier {
        context[cfg.KEY_MATCH_SETTINGS] = self.mgb.get_match_opts()
        #}
        data_items = []
        for field_name, controls in self.xpath_row_controls.items():
            data = {}
            data[cfg.KEY_WEB_FIELD] = field_name
            data[cfg.KEY_WEB_XPATH] = unicode(controls['ledit'].text()).strip()
            if 'regex' in controls:
                data[cfg.KEY_WEB_REGEX] = controls['regex']
                data[cfg.KEY_WEB_REGEX_IS_STRIP] = controls['regex_is_strip']
            data_items.append(data)
        context[cfg.KEY_WEB_XPATH_DATA] = data_items
        context[cfg.KEY_WEB_REVERSE_LIST] = self.reverse_list_checkbox.isChecked()
        context[cfg.KEY_WEB_JAVASCRIPT] = self.javascript_checkbox.isChecked()
        context[cfg.KEY_WEB_ENCODING] = self.encoding_combo.selected_value()


import json

class Pokemon:
    def __init__(self, name, weight, height):
        self.name = name
        self.weight = weight
        self.height = height

def load_pikachu():
    pika_file = open("pika.json")
    pikachu_dict = json.load(pika_file)
    return Pokemon(pikachu_dict["name"],
                   pikachu_dict["weight"],
                   pikachu_dict["height"])

pikachu = load_pikachu()

# Stworzenie klasy Pokemon ułatwia nam czytanie danych z
# załadowanego jsona:
print(pikachu.name)


from django.shortcuts import render, get_object_or_404
from .models import CoffeeMachine
from .filters import MachineFilter

# Create your views here.

def all_machines(request):
    all_machines = CoffeeMachine.objects.all()

    myfilter = MachineFilter(request.GET,queryset=all_machines)
    all_machines = myfilter.qs
    context = {
        'all_machines' : all_machines,
        'myfilter' : myfilter
    }
    return render(request, 'machines.html', context)

def home(request):

    return render(request, 'home.html')


import sqlite3
import random
def insertTask(ix,name, des):

    print("task.py created")

    conn = sqlite3.connect('home.db')
    c =  conn.cursor()

    c.execute('insert into Tasks (TaskID, name, description) values (?,?,?)', (int(ix),name,des))
    conn.commit()
    conn.close()


"""
Provides abstractions for making calls to Ushahidi instances

* Request: Instance of this type encapsulates all the data required for a http request
* RequestBuilder: Builder pattern based abstraction which returns an instance of Request type
* HttpClient: Encapsulates Base url for all requests, authorization headers

"""
import json
import os

import requests

class ImmutableRequest(Exception):
    pass

class Request:
    """
    Encapsulates suburl, query_params (url params), request-body, headers

    """

    def __init__(self, suburl: str, query_params: dict, headers: dict, request_body: dict):
        self.suburl = suburl
        self.query_params = query_params
        self.headers = headers
        self.request_body: dict = request_body

class RequestBuilder:
    """
    Provides a readable api for creating Request objects using builder pattern
    request = (RequestBuilder()
                .suburl('api/v3')
                .suburl('forms')
                .query_param('limit', 4)
                .build()
               )
    """

    def __init__(self):
        self._sub_urls = []
        self._query_params = dict()
        self._headers = dict()
        self._request_body = None
        self.__built = False

    def query_param(self, key: str, value: str):
        """
        :param key:
        :param value:
        :return:
        """
        if self.__built:
            raise ImmutableRequest
        self._query_params[key] = value
        return self

    def header(self, key: str, value: str):
        """
        :param key:
        :param value:
        :return:
        """
        if self.__built:
            raise ImmutableRequest
        self._headers[key] = value
        return self

    def suburl(self, url: str):
        """
        :param url:
        :return:
        """
        if self.__built:
            raise ImmutableRequest
        self._sub_urls.append(str(url).strip('/'))
        return self

    def key(self, key: any):
        """
        Alias for suburl, provide more readable api for specifying keys for http resources
        :param key:
        :return:
        """
        self.suburl(str(key))
        return self

    def set_content_type_as_json(self):
        self.header('Content-Type', 'application/json')
        return self

    def request_body(self, content: any):
        if type(content) == dict:
            self._request_body = json.dumps(content)
        else:
            self._request_body = content
        return self

    def build(self) -> Request:
        """
        :return:
        """
        if self.__built:
            raise ImmutableRequest
        self.__built = True
        return Request(suburl=os.path.join(*self._sub_urls),
                       query_params=self._query_params,
                       headers=self._headers,
                       request_body=self._request_body)

class HttpClient:
    """
    Make http calls using Request objects
    """

    def __init__(self, base_url: str):
        self._base_url = base_url

    def get(self, request: Request) -> requests.Response:
        """
        :return:
        """
        return self._request(method='get', request=request)

    def post(self, request: Request) -> requests.Response:
        """
        :return:
        """
        return self._request(method='post', request=request)

    def put(self, request: Request) -> requests.Response:
        """
        :return:
        """
        return self._request(method='put', request=request)

    def _request(self, method: str, request: Request):
        """
        :param method:
        :param request:
        :return:
        """
        request_params = {
            'url': os.path.join(self._base_url, request.suburl),
            'params': request.query_params,
            'headers': request.headers,
            'data': str(request.request_body)
        }
        if method == 'get':
            return requests.get(**request_params)
        elif method == 'post':
            return requests.post(**request_params)
        else:
            raise NotImplementedError('Method: {} is not implemented yet')


# -*- coding: utf-8 -*-
"""
Created on Thu Apr  5 20:35:31 2018

@author: Guido Salimbeni originally developed by Lorenzo

designed to work on mutation and black background

run from command line:
python fractEvo_v2.py -p D:\aaa -c D:\aaa\test.csv

This code take a picture as an input and write a Json file as an output
the csv file contains the name of the file, the score total score and the individual scores of the 6 features
"""
# import the library for run the script from command
import argparse
# import the library
import cv2
import numpy as np
from scipy.spatial import distance as dist # if complex to install the library I can write another distance algorithm
import os, os.path
import csv
from skimage.measure import compare_ssim as ssim

# construct the argument parse and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-p","--path", required = True, help = "path to the images to load")
ap.add_argument("-c","--csvOutputPath", required = True, help = "path to the csv file to write")
args = vars(ap.parse_args())

# the class that contains the 8 features extractions

class FractEvo ():

    def __init__ (self, imagepath):
        self.image = cv2.imread(imagepath)
        self.imagepath = imagepath
        self.totalPixels = self.image.size
        self.gray = cv2.cvtColor (self.image, cv2.COLOR_BGR2GRAY)

    def analyze_area_Lorenzo(self, thresMinLorenzo = 1 , thresMaxLorenzo = 256,
                             maxArea = 60000,
                             minArea = 5000,
                             penalty = 4,
                             minMeanAreaLitlleCnt = 10,
                             minLenCnts = 3,
                             FractEvoConstraint = True,
                             fractionFractEVO = 3/4):

        #copyImage = self.gray.copy()
        blurred = cv2.GaussianBlur(self.gray, (5, 5), 0)
        ret,threshImage = cv2.threshold(blurred, thresMinLorenzo, thresMaxLorenzo, cv2.THRESH_BINARY)
        # count number of not black pixels
        totalAreaLorenzo = (threshImage > 0).sum()
        # determine the external silhoutte
        ing2,contours, hierarchy = cv2.findContours(threshImage, cv2.RETR_LIST,
                                               cv2.CHAIN_APPROX_NONE)

        areas = []
        for cnt in contours:
            area = cv2.contourArea(cnt)
            areas.append(area)
            #cv2.drawContours(self.image, [cnt], -1, (255, 0, 255), 1)

        # penalise the shape with many little contours
        meanAreaLittleCnt = (sum(areas) - areas[-1]) / len(contours)
        #print (meanAreaLittleCnt, len(contours) )
        # Lorenzo means probably another set of images
        # since there is no relation to 2/3 of the whole
        # in his code

        totalArea = self.image.shape[0] * self.image.shape[1]
        if FractEvoConstraint == False:

            if totalAreaLorenzo < minArea or totalAreaLorenzo > maxArea :
                return (totalAreaLorenzo/totalArea) / penalty
            elif (meanAreaLittleCnt < minMeanAreaLitlleCnt and len(contours) > minLenCnts):
                return (totalAreaLorenzo/totalArea) / penalty
            else:
                return totalAreaLorenzo/totalArea
        if FractEvoConstraint == True:

            if totalAreaLorenzo < minArea or totalAreaLorenzo > maxArea :
                return 0
            else:
                mid = (maxArea - minArea ) * (fractionFractEVO)
                mid += minArea
                r = (totalAreaLorenzo - mid) / mid
                return 1 - abs(r)

    def calculateAsymmetryIndex (self, image = None, extraWeight = False,
                                 MinAreaSilh = 10000,
                                 MaxAreaSilh = 21000,
                                 penalty = 5):
        if image is None:
            image = self.image

        imageCopy = image.copy()
        imageFlipLfRt = cv2.flip(imageCopy, 1)
        imageFlipLf = imageFlipLfRt.copy()
        imageFlipLfHalfCols = int(imageFlipLf.shape [1] / 2)
        imageHalfCols = int(imageCopy.shape [1] / 2)

        imageFlipLf [:,0 : imageFlipLfHalfCols ] = 0
        imageCopy [:, 0 : imageHalfCols] = 0

        imageFlipLf = imageFlipLf [0 : image.shape[0], imageFlipLfHalfCols : image.shape[1]]
        imageCopy = imageCopy [0 : image.shape[0], imageHalfCols : image.shape[1]]

        similaritySSIM = ssim(imageFlipLf,imageCopy, win_size=None, gradient=False, data_range=None,
                 multichannel = True, gaussian_weights=False, full=False, dynamic_range = None)
        ssimAsymmetry = 1 - similaritySSIM

        # for weighting the result of asymmetry to the area of the shape
        ImageCopy = self.image.copy()
        gray = cv2.cvtColor(ImageCopy, cv2.COLOR_BGR2GRAY)
        #a get the outer silhouette and max one shape only
        ret,thresh = cv2.threshold(gray, 1, 255, cv2.THRESH_BINARY)
        # area foreground
        area = (thresh > 0).sum()
        # aread right and area left of the silhoutte (half orizzontally)
        row = thresh.shape[0]
        col = thresh.shape[1]
        areaDX = (thresh[0:row, 0: int(col/2)]>0).sum()
        areaSX = (thresh[0:row, int(col/2): col]>0).sum()

        absDiff = abs(areaDX-areaSX)
        absDiffRatio = absDiff / areaDX
        # calculate what weight to give to the score
        # penalty big or too little silhoutte
        # add extra weight for area diffeence in
        # horizz pixel count along the half of the figure
        if area < MinAreaSilh or area > MaxAreaSilh:
            return ssimAsymmetry / penalty
        else:
            if extraWeight:
                return ssimAsymmetry +  absDiffRatio
            else:
                return ssimAsymmetry

    def analyze_blobs_Lorenzo(self, thresMinLorenzo = 100, maxBlobs = 350,
                              fractionFractEVO = 0.5, minBlobs = 0):
        #convert to gray
        grayBlob = self.gray.copy()
        # threshold
        ret,thresh = cv2.threshold(grayBlob, thresMinLorenzo, 255, cv2.THRESH_BINARY)
        # find contours
        ing2, contours, hierarchy = cv2.findContours(thresh,
                                                     cv2.RETR_TREE,
                                                     cv2.CHAIN_APPROX_SIMPLE)

        contours = sorted(contours, key = cv2.contourArea)
        # Approximates a polygonal curve(s) with the specified precision
        contours_poly = []
        radius_list = []
        for contour in contours[:-2]:
            contour_poly = cv2.approxPolyDP(contour, 3, True)
            contours_poly.append(contour_poly)
            (x,y),radius = cv2.minEnclosingCircle(contour_poly)
            radius_list.append(radius)
            #cv2.drawContours(self.image, [contour], -1, (255, 0, 255), 1)
            radius = (int(radius))
            #cv2.circle(self.image,(int(x),int(y)),radius,(0, 255, 255), 1)

        n_blobs_Lorenzo = len(radius_list)
        # Lorenzo wrote min 0 max 10 and scale factor 0.5
        # but this values not matches with the set of image that
        # now the values of n_blobs are from 150 to 450 circa

        if n_blobs_Lorenzo > maxBlobs:
            return 0
        else:
            mid = (maxBlobs - minBlobs ) * (fractionFractEVO)
            mid += minBlobs
            r = (n_blobs_Lorenzo - mid) / mid
            return  1 - abs(r)

    def findCentreOfMass(self,distMINWeight = 0.20, distMAXWeight = 1,
                         fractionFractEVO = 0.5, FractEvoConstraint = False):
        gray = self.gray
        #a get the outer silhouette and max one shape only
        blurred = cv2.GaussianBlur(gray, (5, 5), 0)

        ret,thresh = cv2.threshold(blurred, 1, 255, cv2.THRESH_BINARY)
        ing2, contours, hierarchy = cv2.findContours(thresh,cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)

        # since we look only at the foreground
        # we take only the bigger contour

        sorted_contours = sorted(contours, key = cv2.contourArea, reverse = True)
        OneBigContour = sorted_contours [:1] # hardcoded for having one only result
        # loop left open to implementation for the FUTURE TODO superclass that has
        # also more contours or background not black
        # loop over the contours - replaced with OneBigContour that is one only contour

        for c in OneBigContour:

        	# compute the center of the contour
            M = cv2.moments(c)
            cX = int(M["m10"] / M["m00"])
            cY = int(M["m01"] / M["m00"])
            # area for weight
            area = cv2.contourArea(c)

        # how far is the centroid to the image centre
        ImageCentreX = self.image.shape[1] / 2
        ImageCentreY = self.image.shape[0] / 2

        distanceCentreOfMasstoCentreofImage = dist.euclidean((ImageCentreX, ImageCentreY), (cX, cY))

        # higher number means more interesting shape since the shape will be in
        # the frame however but its centre of mass not in the centre of the image frame
        # more interesting since less trivial
        ratio = area/self.totalPixels
        weightedDistance = distanceCentreOfMasstoCentreofImage * ratio

        if weightedDistance < distMINWeight:
            return 0
        elif weightedDistance > distMAXWeight:
            return 0
        else:
            if FractEvoConstraint == True:
                return weightedDistance * fractionFractEVO
            else:
                return weightedDistance

    def analyse_nfeatures_FAST(self, threshLorenzoDetector = 100, maxNum = 1000,
                               minNum = 100,
                               FractEvoConstraint = True,
                               fractionFractEVO = 0.7):
        grayFast = self.gray
        # create fast detector object
        fast = cv2.FastFeatureDetector_create(threshLorenzoDetector)
        # Obtain Key Points, by default non max suppression is On
        # to turn off set fast.setBool('nonmaxSuppression', False)

        keypoints = fast.detect(grayFast, None)

        numberOfFeaturesLor = len(keypoints)
        if FractEvoConstraint == False:
            if numberOfFeaturesLor > maxNum:
                return 0
            else:
                return  (1 - (maxNum - numberOfFeaturesLor)/maxNum)*0.7
        elif FractEvoConstraint == True:
                mid = (maxNum - minNum ) * (fractionFractEVO)
                mid += minNum
                r = (numberOfFeaturesLor - mid) / mid
                return 1 - abs(r)
        else:
            return 0

    def analyze_holes_Lorenzo (self,  thresMinLorenzo = 20, holeMinArea = 10,
                               maxNumOfHoles = 20,
                               minNumofHoles = 1,
                               FractEvoConstraint = True,
                               fractionFractEVO = 0.5 ):

        grayHoles = self.gray
        ret,thresh = cv2.threshold(grayHoles, thresMinLorenzo, 255, cv2.THRESH_BINARY)
        # find contours
        threshCopy = thresh.copy()
        ing2, contours, hierarchies = cv2.findContours(threshCopy,
                                                     cv2.RETR_CCOMP,
                                                     cv2.CHAIN_APPROX_SIMPLE)
        # get the total area of contours
        # loop over the contorurs and count if their area is bigger than
        # of a fixed value holeMinArea
        all_contous_areas = self.utils_get_contour_areas(contours)

        counted = []
        noncounted = []
        for  cnt ,area, hierarchy in zip (contours,all_contous_areas, hierarchies[0]):

            if hierarchy[2] == -1 and area > holeMinArea:
                counted.append(area)
                #cv2.drawContours(self.image, [cnt], -1, (0, 255, 0), 1)
            else:
                noncounted.append(area)

        numberOfHolesLorenzo = (len(counted))
        # if there are too many holes it means that the mesh is too noise
        # an alternative would be to add gaussian blur to the initial gray image
        if FractEvoConstraint == False:
            if numberOfHolesLorenzo > maxNumOfHoles:
                return 0
            else:
                return numberOfHolesLorenzo
        if FractEvoConstraint == True:
            if numberOfHolesLorenzo < minNumofHoles or numberOfHolesLorenzo > maxNumOfHoles :

                return 0
            else:
                mid = (maxNumOfHoles - minNumofHoles ) * (fractionFractEVO)
                mid += minNumofHoles
                mid = int(mid)
                r = (numberOfHolesLorenzo - mid) / mid

                return 1 - abs(r)

    def clearSilhoutte (self, extraConstraint = False,
                        minDist = 40 , maxDist = 80):

        graySilhoutte = cv2.cvtColor(self.image, cv2.COLOR_BGR2GRAY)
        ImageCopy = self.image.copy()
        #a get the outer silhouette and max one shape only
        Imgblurred = cv2.GaussianBlur(graySilhoutte, (5, 5), 0)
        ret,thresh = cv2.threshold(Imgblurred, 1, 255, cv2.THRESH_BINARY)
        ing2, contours, hierarchy = cv2.findContours(thresh,cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)

        # draw a bounding rectangle
        x = 0 ; y = 0 ; w = 0 ; h = 0
        if len(contours)> 1:
            # no single clear silhoutte
            sorted_contours = sorted(contours, key = cv2.contourArea, reverse = True)

            x,y,w,h = cv2.boundingRect(sorted_contours[0])
            #cv2.rectangle(ImageCopy,(x,y),(x+w,y+h),(0,255,0),2)

        else:
            x,y,w,h = cv2.boundingRect(contours[0])
            #cv2.rectangle(ImageCopy,(x,y),(x+w,y+h),(0,255,0),2)

        # check the distances of the corners and the frames
        distLeftHighBoundingBox2LeftHighFrameCorner = dist.euclidean((0,0), (x, y))
        #distRightHighBoundingBox2RightHighFrameCorner = dist.euclidean ((0, self.image.shape[1]), (x, y + w))
        #distLeftLowBoundingBox2LeftLowFrameCorner = dist.euclidean((self.image.shape[0],0), (x + h, y))
        distRightLowBoundingBox2RightLowFrameCorner = dist.euclidean ((self.image.shape[0], self.image.shape[1]), (x + w, y + h))
        # calcualter the ratio of distances
        #DistCentre2Corner = dist.euclidean((self.image.shape[0]/2, self.image.shape[1]/2), (0,0))
        A = distLeftHighBoundingBox2LeftHighFrameCorner
        #B = distRightHighBoundingBox2RightHighFrameCorner
        #C = distLeftLowBoundingBox2LeftLowFrameCorner
        D = distRightLowBoundingBox2RightLowFrameCorner

        ratioSilhoutte = (A  + D ) /2
        # the rect touches the border it gives zero score
        if x == 0 or y == 0 or (x+w) == self.image.shape[1] or (y+h) == self.image.shape[0]:

            return 0
        # alternative method to give more info to the method BUT not used
        if extraConstraint == True:
            if ratioSilhoutte > maxDist:

                return  0
            elif ratioSilhoutte < minDist:

                return 0
            else:
                mid = (maxDist + minDist) / 2
                diff = abs(ratioSilhoutte - mid) / mid

                return 1 - diff
        else:
            return 1

    def perimeterCntVsHoles(self,thresMinLorenzo = 10, holeMinArea = 10,
                            perimeterThresh = 750, numberOfHolesThresh = 5):
        # If Length of Silhouette is long, then reward high no of holes Else
        # If Length of Silhouette is short, then reward low no of holes.

        Gray = cv2.cvtColor (self.image, cv2.COLOR_BGR2GRAY)
        # threshold
        ret,ThresPerim = cv2.threshold(Gray, 1, 255, cv2.THRESH_BINARY)
        threshCopy = ThresPerim.copy()
        ing2, contours, hierarchy = cv2.findContours(threshCopy,
                                                     cv2.RETR_EXTERNAL,
                                                     cv2.CHAIN_APPROX_SIMPLE)
        sorted_contours = sorted(contours, key = cv2.contourArea, reverse = True)
        OneBigContour = sorted_contours [:1]

        numberOfHoles,copyfordisplay = self._get_holes_Lorenzo ( thresMinLorenzo = thresMinLorenzo, holeMinArea = holeMinArea)

        for cnt in OneBigContour:

            perimeter = cv2.arcLength(cnt,True)
            #print ('perimeter' , perimeter)

        #print ('number of holes', numberOfHoles)

        if perimeter > perimeterThresh:
            if numberOfHoles > numberOfHolesThresh:
                scorePerVsHoles = 1
            else:
                scorePerVsHoles = 0

        if perimeter < perimeterThresh:
            if numberOfHoles < numberOfHolesThresh:
                scorePerVsHoles = 1
            else:
                scorePerVsHoles = 0

        return scorePerVsHoles

    def _get_holes_Lorenzo (self,  thresMinLorenzo = 10, holeMinArea = 10):

        copyfordisplay = self.image.copy()
        grayHoles = self.gray
        ret,thresh = cv2.threshold(grayHoles, thresMinLorenzo, 255, cv2.THRESH_BINARY)
        # find contours
        threshCopy = thresh.copy()
        ing2, contours, hierarchies = cv2.findContours(threshCopy,
                                                     cv2.RETR_CCOMP,
                                                     cv2.CHAIN_APPROX_SIMPLE)
        # get the total area of contours
        # loop over the contorurs and count if their area is bigger than
        # of a fixed value holeMinArea
        all_contous_areas = self.utils_get_contour_areas(contours)

        counted = []
        noncounted = [] # not used
        for  cnt ,area, hierarchy in zip (contours,all_contous_areas, hierarchies[0]):

            if hierarchy[2] == -1 and area > holeMinArea:
                counted.append(area)

                cv2.drawContours(copyfordisplay, [cnt], -1, (0, 255, 0), 1)

            else:
                noncounted.append(area)

        numberOfHoles = len(counted)

        return numberOfHoles, copyfordisplay

    def fractalDimMinkowskiBoxCount(self, target = 1.5):

        Z = self.gray

        # calculate the threshold on the mean
        mean, standardDeviation = cv2.meanStdDev(Z)
        threshold = mean[0][0]

        # Only for 2d image
        assert(len(Z.shape) == 2)

        # From https://github.com/rougier/numpy-100 (#87)
        def boxcount(Z, k):
            S = np.add.reduceat(
                np.add.reduceat(Z, np.arange(0, Z.shape[0], k), axis=0),
                                   np.arange(0, Z.shape[1], k), axis=1)

            # We count non-empty (0) and non-full boxes (k*k)
            return len(np.where((S > 0) & (S < k*k))[0])

        # Transform Z into a binary array
        Z = (Z < threshold)

        # Minimal dimension of image
        p = min(Z.shape)

        # Greatest power of 2 less than or equal to p
        n = 2**np.floor(np.log(p)/np.log(2))

        # Extract the exponent
        n = int(np.log(n)/np.log(2))

        # Build successive box sizes (from 2**n down to 2**1)
        sizes = 2**np.arange(n, 1, -1)

        # Actual box counting with decreasing size
        counts = []
        for size in sizes:
            counts.append(boxcount(Z, size))

        # Fit the successive log(sizes) with log (counts)
        coeffs = np.polyfit(np.log(sizes), np.log(counts), 1)

        fractal = -coeffs[0]

        distFromTarget = abs(target - fractal)
        # avoiding nan results
        if str(distFromTarget) == 'nan':
            distFromTarget = 1

        return 1 - distFromTarget

    def utils_get_contour_areas (self, contours):
        all_areas = []
        for cnt in contours:
            area = cv2.contourArea(cnt)
            all_areas.append(area)
        return all_areas

# this is the code to load the image and run the class to extract the score

# debugging path
#path = "D:\\aaa"
path = args["path"]
#☼ original path
#path = "D:\\google drive\\A PhD Project at Godlsmiths\\ArtistSupervisionMaya\\images"
#imageOutputPath = 'D:\\google drive\\A PhD Project at Godlsmiths\\Artist supervision code\\images'
#csvOutputPath = 'FeaturesScoresWtest.csv'
csvOutputPath = args["csvOutputPath"]

# load the images
imagepaths = []
imageNames = []
valid_images = [".jpg", ".png", ".tga", ".gif"]
for f in os.listdir(path):

    ext = os.path.splitext(f)[1]
    name = os.path.splitext(f)[0]
    if ext.lower() not in valid_images:
        continue
    imagepaths.append(os.path.join(path,f))
    imageNames.append(name)
# store the total number of images
totalNumOfImages = len(imagepaths)

# list of the features to write
areaLorenzoList= []
nFeaturesList = []
nBlobsList = []
nHolesList = []
asymmetryScoreList = []
clearSilhoutteList = []
centreOfMassList = []
fractalScoreList = []
perimeterVsHolesScoreList = []
totalScoreList = []
# make the calculation
for i in range(totalNumOfImages):
    imagepath = imagepaths[i]
    imageName = imageNames[i]

    frac = FractEvo(imagepath)
    areaLorenzo = frac.analyze_area_Lorenzo( thresMinLorenzo = 1 , thresMaxLorenzo = 256,
                             maxArea = 60000,
                             minArea = 5000,
                             penalty = 4,
                             minMeanAreaLitlleCnt = 10,
                             minLenCnts = 3,
                             FractEvoConstraint = True,
                             fractionFractEVO = 3/4)
# =============================================================================
#     nFeatures = frac.analyse_nfeatures_FAST(threshLorenzoDetector = 20, maxNum = 1000,
#                                minNum = 100,
#                                FractEvoConstraint = True,
#                                fractionFractEVO = 0.7)
# =============================================================================
    nBlobs = frac.analyze_blobs_Lorenzo(thresMinLorenzo = 100, maxBlobs = 350,
                              fractionFractEVO = 0.5, minBlobs = 0)
    nHoles = frac.analyze_holes_Lorenzo(thresMinLorenzo = 10, holeMinArea = 10,
                               maxNumOfHoles = 20,
                               minNumofHoles = 1,
                               FractEvoConstraint = True,
                               fractionFractEVO = 0.5 )
    asymmetryScore = frac.calculateAsymmetryIndex(image = None, extraWeight = False,
                                 MinAreaSilh = 5000,
                                 MaxAreaSilh = 45000,
                                 penalty = 5)
    clearSilhoutte = frac.clearSilhoutte(extraConstraint =  False, minDist = 20 , maxDist = 80)
    centreOfMass = frac.findCentreOfMass(distMINWeight = 0.20, distMAXWeight = 1,
                         fractionFractEVO = 0.5, FractEvoConstraint = True)
    fractalScore = frac.fractalDimMinkowskiBoxCount(target = 1.5)
    perimeterVsHolesScore = frac.perimeterCntVsHoles(thresMinLorenzo = 20, holeMinArea = 10,
                            perimeterThresh = 750, numberOfHolesThresh = 5)

    areaLorenzoList.append(areaLorenzo)
    #nFeaturesList.append(nFeatures)
    nBlobsList.append(nBlobs)
    nHolesList.append(nHoles)
    asymmetryScoreList.append(asymmetryScore)
    clearSilhoutteList.append(clearSilhoutte)
    centreOfMassList.append(centreOfMass)
    fractalScoreList.append(fractalScore)
    perimeterVsHolesScoreList.append(perimeterVsHolesScore)
    A = 1; B = 1 ; C = 1 ; D = 1 ; E = 1 ; F = 1 ; G = 1
    totalScore = A * areaLorenzo
    #+ B * nFeatures
    + C * nBlobs + D * asymmetryScore + E * clearSilhoutte + F * centreOfMass + G * fractalScore + perimeterVsHolesScore
    totalScoreList.append(totalScore)

    del(frac)
# selected features csv file output

#os.chmod(csvOutputPath, 0777)
file_to_output = open(csvOutputPath, 'w', newline='')
csv_writer = csv.writer(file_to_output, delimiter = ',')
csv_writer.writerow (['file',
                      'areaLorenzo',
                      #'nFeatures',
                      'nBlobs',
                      'nHoles',
                      'asymmetryScore',
                      'clearSilhoutte',
                      'centreOfMass',
                      'fractalScore',
                      'totalScore'
                      ])

for i in range(totalNumOfImages):
    csv_writer.writerow ([imageNames[i],
                          areaLorenzoList[i],
                          #nFeaturesList[i],
                          nBlobsList[i],
                          nHolesList[i],
                          asymmetryScoreList[i],
                          clearSilhoutteList[i],
                          centreOfMassList[i],
                          fractalScoreList[i],
                          totalScoreList[i]
                         ])
file_to_output.close()


import pytest
import logging
import os
import shutil
import glob

@pytest.fixture
def base_url(request):
    return request.config.getoption('--url')

def pytest_sessionstart():
    if os.path.isdir(os.path.dirname(__file__) + '/.pytest_cache'):
        shutil.rmtree(os.path.dirname(__file__) + '/.pytest_cache')
    files = glob.glob('allure-results/*') + glob.glob('logs/*')
    for f in files:
        try:
            os.remove(f)
        except OSError as e:
            print("Error: %s : %s" % (f, e.strerror))

@pytest.fixture(scope='module')
def logger(request):
    if not os.path.exists(os.path.dirname(__file__) + '/logs'):
        os.mkdir(os.path.dirname(__file__) + '/logs')
    log_path = os.path.dirname(__file__) + r'/logs'
    logger = logging.getLogger(request.node.name)
    logger.setLevel(logging.DEBUG)
    create_handler = logging.FileHandler(os.path.join(log_path, f'{request.node.module.__name__}.log'))
    create_handler.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(asctime)s| %(module)s |%(name)s | %(levelname)s | %(message)s')
    create_handler.setFormatter(formatter)
    logger.addHandler(create_handler)
    return logger


# macro 적용 - gpt api and MultinomialNB_labeling

from sklearn.metrics import f1_score, accuracy_score, precision_score, recall_score
from sklearn.metrics import classification_report
import pandas as pd

df_result = pd.read_csv('/home/beerlab/result_run_Binary_class_v2.csv')
y_true = df_result['original']
y_pred = df_result['predicted']

f1 = f1_score(y_true, y_pred, average='macro')
acc = accuracy_score(y_true, y_pred)
precision = precision_score(y_true, y_pred, average='macro')
recall = recall_score(y_true, y_pred, average='macro')
report = classification_report(y_true, y_pred)

print('result_run_Binary_class_v2 classification_report의 결과')
print(f"f1_score: {f1:.3f}, accuracy: {acc:.3f}, precision: {precision:.3f}, recall: {recall:.3f}")
print(len(df_result))
print(df_result['original'].value_counts())
print(df_result['predicted'].value_counts())
print(report)


from playwright.sync_api import Playwright, sync_playwright, expect

def run(playwright: Playwright) -> None:
    browser = playwright.chromium.launch(headless=False)
    context = browser.new_context()

    # Open new page
    page = context.new_page()

    # Go to https://www.wikipedia.org/
    page.goto("https://www.wikipedia.org/")

    # Click strong:has-text("日本語")
    page.locator("strong:has-text(\"日本語\")").click()
    page.wait_for_url("https://ja.wikipedia.org/wiki/%E3%83%A1%E3%82%A4%E3%83%B3%E3%83%9A%E3%83%BC%E3%82%B8")

    # Click text=名鉄3400系電車
    page.locator("text=名鉄3400系電車").click()
    page.wait_for_url("https://ja.wikipedia.org/wiki/%E5%90%8D%E9%89%843400%E7%B3%BB%E9%9B%BB%E8%BB%8A")

    # Close page
    page.close()

    # ---------------------
    context.close()
    browser.close()

with sync_playwright() as playwright:
    run(playwright)


from .core import (
    TempReadOnlyFile,
)
from dataclasses import (
    dataclass,
)
from fa_purity import (
    Cmd,
    FrozenDict,
    FrozenList,
    Stream,
)
from fa_purity.cmd.core import (
    CmdUnwrapper,
    new_cmd,
)
from fa_purity.frozen import (
    freeze,
)
from fa_singer_io.singer import (
    SingerMessage,
    SingerRecord,
    SingerSchema,
)
from fa_singer_io.singer.emitter import (
    emit,
)
import logging
from tempfile import (
    NamedTemporaryFile,
)
from typing import (
    Dict,
    IO,
    Tuple,
)

LOG = logging.getLogger(__name__)

@dataclass
class GroupedRecords:
    stream: str
    file: TempReadOnlyFile

def group_records(
    data: Stream[SingerMessage],
) -> Cmd[Tuple[FrozenDict[str, SingerSchema], FrozenList[GroupedRecords]]]:
    def _action(
        act: CmdUnwrapper,
    ) -> Tuple[FrozenDict[str, SingerSchema], FrozenList[GroupedRecords]]:
        LOG.info("Grouping records...")
        schemas: Dict[str, SingerSchema] = {}
        files: Dict[str, IO[str]] = {}
        file_paths: Dict[str, str] = {}  # stream - file_path dict
        for s in act.unwrap(data.unsafe_to_iter()):
            if isinstance(s, SingerSchema):
                LOG.debug("Saving SingerSchema of stream %s", s.stream)
                schemas[s.stream] = s
            elif isinstance(s, SingerRecord):
                if files.get(s.stream) is None:
                    LOG.debug("Creating file for stream %s", s.stream)
                    files[s.stream] = NamedTemporaryFile(
                        "w+", delete=False
                    )  # implicit cmd
                act.unwrap(emit(files[s.stream], s))
        for k, f in files.items():
            LOG.debug("Closing file of stream %s", k)
            f.close()  # for getting data flushed
            LOG.debug("Closed file of stream %s!", k)
            file_paths[k] = f.name
        LOG.debug("Generating GroupedRecords objs...")
        groups = tuple(
            GroupedRecords(s, act.unwrap(TempReadOnlyFile.freeze(f)))
            for s, f in file_paths.items()
        )
        LOG.debug("GroupedRecords generated!")
        LOG.debug("Freezing schema")
        _schema = freeze(schemas)
        LOG.debug("Schema frozen!")
        return (_schema, groups)

    end = Cmd.from_cmd(lambda: LOG.info("Records grouping completed!"))
    return new_cmd(_action).bind(lambda i: end.map(lambda _: i))


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import time
import os
import string
import numpy as np
import regex as re

py_name = os.path.basename(__file__)
file = "day" + re.findall(r'\d+', py_name)[0] + ".txt"
file_path = os.path.join("inputs", file)

# opens the file
with open(file_path) as f:
    lines = [line.rstrip() for line in f]  # takes the \n

def bothparts():
    best_scenic_score = 0
    visible_trees = 0
    matrix = np.zeros((len(lines), len(lines[0])))
    for idx, l in enumerate(lines):
        row = [int(d) for d in str(l)]
        matrix[idx] = row

    border_trees = len(lines[0]) + 2 * (len(lines) - 1) + (len(lines[0]) - 2)
    visible_trees += border_trees
    for x, element in enumerate(matrix):
        if x == 0 or x == len(matrix) - 1:
            pass
        else:
            inner_element = element[1:-1]
            i = 1

            for el in inner_element:
                visibles, scenic_score, visible_trees_list = check_visibility(m=matrix, coords=[x, i], el=el)

                if scenic_score >= best_scenic_score:
                    best_scenic_score = scenic_score

                i += 1

                if any(visibles):
                    visible_trees += 1

    return visible_trees, best_scenic_score

def check_visibility(m, coords, el):
    visibles = []
    visible_trees_list = []
    top = m[:coords[0], coords[1]].tolist()
    left = m[coords[0], :coords[1]].tolist()
    top.reverse()
    left.reverse()
    right = m[coords[0], coords[1] + 1:].tolist()
    down = m[coords[0] + 1:, coords[1]].tolist()

    visibility_lines = [top, left, right, down]

    for j in visibility_lines:
        visible, lower_trees = check_visibility_line(el, j)
        visibles.append(visible)
        visible_trees_list.append(lower_trees)

    scenic_score = np.prod(visible_trees_list)

    return visibles, scenic_score, visible_trees_list

def check_visibility_line(el, line):
    lower_trees = 0
    for i in line:
        lower_trees += 1
        if i >= el:
            return False, max(lower_trees, 1)

    return True, lower_trees

result, scenic_score = bothparts()
print(result, scenic_score)


import torch
import transformers

name = "mosaicml/mpt-7b"

config = transformers.AutoConfig.from_pretrained(name, trust_remote_code=True)
config.attn_config["attn_impl"] = "triton"
config.init_device = "cuda:0"  # For fast initialization directly on GPU!

model = transformers.AutoModelForCausalLM.from_pretrained(
    name,
    config=config,
    trust_remote_code=True,
)

tokenizer = transformers.AutoTokenizer.from_pretrained(name, trust_remote_code=True)

# Generate text
input_ids = tokenizer("The dog", return_tensors="pt").input_ids.to("cuda:0")
output = model.generate(
    input_ids, do_sample=True, max_length=50, num_return_sequences=1
)
print(tokenizer.decode(output[0], skip_special_tokens=True))


import tkinter as tk
import trame,paquetip,tcp,httpmodule
import source as src
import sqlite3 as sql

#Dans cette version il faut mettre la liste condition en paramètre et la liste retournée est la liste de résultats filtrés
def database_from_trame(dbft):
    cursor = dbft.cursor()
    cursor.execute("CREATE TABLE IF NOT EXISTS ipdb(num_trame INTEGER, protocol TEXT, ipsrc TEXT, ipaddr TEXT)")
    dbft.commit()
    cursor.execute("CREATE TABLE IF NOT EXISTS trame(num_trame INTEGER, ethsrc TEXT, ethaddr TEXT)")
    dbft.commit()
    cursor.execute("CREATE TABLE IF NOT EXISTS tcpdb(num_trame INTEGER, tcpsrc TEXT, tcpaddr TEXT)")
    dbft.commit()
    cursor.execute("CREATE TABLE IF NOT EXISTS httpdb(num_trame INTEGER, httpversion TEXT)")
    dbft.commit()
    for i in range(src.nbtrames):
        i += 1
        if (((paquetip.Source_Address).get(i) is not None) and ((paquetip.Destination_Address).get(i) is not None)):
            cursor.execute("""
            INSERT INTO ipdb(num_trame, protocol, ipsrc, ipaddr) VALUES (?,?,?,?)""",(i, (paquetip.Protocol).get(i), (paquetip.Source_Address).get(i), (paquetip.Destination_Address).get(i))
            )
        if(((trame.mac_adress_source).get(i) is not None) and ((trame.mac_adress_distination).get(i) is not None)):
            cursor.execute("""
            INSERT INTO trame(num_trame, ethsrc, ethaddr) VALUES (?,?,?)""",(i, (trame.mac_adress_source).get(i), (trame.mac_adress_distination).get(i))
            )
        if(((tcp.Port_Source).get(i) is not None) and ((tcp.Port_destination).get(i) is not None)):
            cursor.execute("""
            INSERT INTO tcpdb(num_trame, tcpsrc, tcpaddr) VALUES (?,?,?)""",(i, (tcp.Port_Source).get(i), (tcp.Port_destination).get(i))
            )
        if(((httpmodule.versiondic).get(i) is not None)):
            cursor.execute("""
            INSERT INTO httpdb(num_trame, httpversion) VALUES (?,?)""",(i, (httpmodule.versiondic).get(i))
            )

    return dbft

def filtre(condition):
    try :
        if len(condition)>=3:
            len_con = len(condition)
            temp = condition[0]
            temp = temp.replace('.','')
            temp1 = condition[1]
            temp2 = condition[2]
            requete = 'SELECT * FROM '
            if (temp.lower() == 'ipsrc' or temp.lower() == 'ipaddr'):
                requete = requete + 'ipdb'
                listip = temp2.split('.')
                for eight_bit in listip:
                    try :
                        if ((int(eight_bit) < 0) or (int(eight_bit) >255)):
                            return "echec"
                    except :
                        return "echec"
            try :
                if (temp.lower() == 'protocol'):
                    requete = requete + 'ipdb'
            except :
                return "echec"
            try :
                if (temp.lower() == 'tcpsrc' or temp.lower() == 'tcpaddr'):
                    requete = requete + 'tcpdb'
                    port = int(temp2)
                    if ((port < 0) or (port > 65535)):
                        return "echec"
            except :
                return "echec "
            try :
                if (temp.lower() == 'ethsrc' or temp.lower() == 'ethaddr'):
                    requete = requete + 'trame'
            except :
                return "echec"
            try :
                if (temp.lower() == 'httpversion'):
                    requete = requete + 'httpdb'
            except :
                return "echec"

            try :
                if (temp.lower() != 'ipsrc' and temp.lower() != 'ipaddr' and temp.lower() != 'protocol' and temp.lower() != 'tcpsrc' and temp.lower() != 'tcpaddr' and temp.lower() != 'ethsrc' and temp.lower() != 'ethaddr' and temp.lower() != 'httpversion'):
                    return "echec"
            except :
                return "echec"
            requete = requete + ' WHERE '
            if (temp1 == '=='):
                requete = requete + temp + ' = ' + "('"+temp2+"')"
            if (temp1 == '!='):
                requete = requete + temp + ' != ' + "('"+temp2+"')"
            while(len_con > 3):
                if((condition[3].lower() == 'and') or (condition[3] == '&&')):
                    requete = requete + ' AND ('
                if((condition[3].lower() == 'or') or (condition[3] == '||')):
                    requete = requete + ' OR ('
                condition = condition[4:len(condition)]
                temp = condition[0]
                temp = temp.replace('.','')
                temp1 = condition[1]
                temp2 = condition[2]
                if (temp.lower() == 'ipsrc' or temp.lower() == 'ipaddr'):
                    listip = temp2.split('.')
                    for eight_bit in listip:
                        try:
                            if ((int(eight_bit) < 0) or (int(eight_bit) >255)):
                                return "echec"
                        except :
                            return "echec"

                if (temp.lower() == 'tcpsrc' or temp.lower() == 'tcpaddr'):
                    try :
                        port = int(temp2)
                        if ((port < 0) or (port > 65535)):
                            return "echec"
                    except :
                        return "echec"

                requete = requete + temp.lower()
                if (temp1 == '=='):
                    requete = requete + ' = ' + "'"+temp2+"')"
                if (temp1 == '!='):
                    requete = requete + ' != ' + "'"+temp2+"')"
                len_con = len(condition)
            global dbft
            dbft = sql.connect('./trames/ma_base.db') #Crée une base de donnée en mémoire, mettre db.close() quand on en aura plus besoin
            database_from_trame(dbft)
            cursor = dbft.cursor()
            cursor.execute(requete)
            return cursor
        return "echec"
    except :
        return "echec"


import requests
import pandas as pd
import time
from config import TOKEN

# define valid boston zip codes and yelp cuisine tags
ZIPS = {'East Boston':['02128'], 'Charlestown': ['02129'], 'Allston': ['02163','02134'], 'Brighton':['02135'], 'Beacon Hill': ['02108'],
        'Back Bay': ['02116', '02199'],'Chinatown': ['02111'], 'Dorchester': ['02121', '02122', '02124', '02125'], 'Fenway': ['02115', '02215'],
        'Hyde Park': ['02136'], 'Jamaica Plain': ['02130'], 'Mattapan': ['02126'], 'Mission Hill': ['02120'],'North End': ['02113', '02109'],
        'Roslindale': ['02131'], 'Roxbury': ['02119'], 'South Boston': ['02127', '02210'], 'South End': ['02118'], 'West End': ['02114'],
        'West Roxbury': ['02132'], 'Wharf District': ['02110'],'Downtown': ['02203', '02201']
        }

VALID_CUISINES = (
    "afghan", "african", "senegalese", "southafrican", "newamerican", "tradamerican", "andalusian", "arabian",
    "arabpizza", "argentine", "armenian", "asianfusion", "asturian", "australian", "austrian", "baguettes",
    "bangladeshi", "bbq", "basque", "bavarian", "beergarden", "beerhall", "beisl", "belgian", "flemish", "bistros",
    "blacksea", "brasseries", "brazilian", "brazilianempanadas", "centralbrazilian", "northeasternbrazilian",
    "northernbrazilian", "rodizios", "breakfast_brunch", "pancakes", "british", "buffets", "bulgarian", "burgers",
    "burmese", "cafes", "themedcafes", "cafeteria", "cajun", "cambodian", "newcanadian", "canteen", "caribbean",
    "dominican", "haitian", "puertorican", "trinidadian", "catalan", "cheesesteaks", "chickenshop", "chicken_wings",
    "chilean", "chinese", "cantonese", "congee", "dimsum", "fuzhou", "hainan", "hakka", "henghwa", "hokkien", "hunan",
    "pekinese", "shanghainese", "szechuan", "teochew", "comfortfood", "corsican", "creperies", "cuban", "currysausage",
    "cypriot", "czech", "czechslovakian", "danish", "delis", "diners", "dinnertheater", "dumplings", "eastern_european",
    "eritrean", "ethiopian", "hotdogs", "filipino", "fischbroetchen", "fishnchips", "flatbread", "fondue", "food_court",
    "foodstands", "freiduria", "french", "alsatian", "auvergnat", "berrichon", "bourguignon", "mauritius", "nicois",
    "provencal", "reunion", "sud_ouest", "galician", "gamemeat", "gastropubs", "georgian", "german", "baden",
    "easterngerman", "franconian", "hessian", "northerngerman", "palatine", "rhinelandian", "giblets", "gluten_free",
    "greek", "guamanian", "halal", "hawaiian", "heuriger", "himalayan", "honduran", "hkcafe", "hotdog", "hotpot",
    "hungarian", "iberian", "indpak", "indonesian", "international", "irish", "island_pub", "israeli", "italian",
    "abruzzese", "altoatesine", "apulian", "calabrian", "cucinacampana", "emilian", "friulan", "ligurian", "lumbard",
    "napoletana", "piemonte", "roman", "sardinian", "sicilian", "tuscan", "venetian", "japanese", "blowfish",
    "conveyorsushi", "donburi", "gyudon", "oyakodon", "handrolls", "horumon", "izakaya", "japacurry", "kaiseki",
    "kushikatsu", "oden", "okinawan", "okonomiyaki", "onigiri", "ramen", "robatayaki", "soba", "sukiyaki", "takoyaki",
    "tempura", "teppanyaki", "tonkatsu", "udon", "unagi", "westernjapanese", "yakiniku", "yakitori", "jewish", "kebab",
    "kopitiam", "korean", "kosher", "kurdish", "laos", "laotian", "latin", "colombian", "salvadoran", "venezuelan",
    "raw_food", "lyonnais", "malaysian", "mamak", "nyonya", "meatballs", "mediterranean", "falafel", "mexican",
    "easternmexican", "jaliscan", "northernmexican", "oaxacan", "pueblan", "tacos", "tamales", "yucatan",
    "mideastern", "egyptian", "lebanese", "milkbars", "modern_australian", "modern_european", "mongolian",
    "moroccan", "newmexican", "newzealand", "nicaraguan", "nightfood", "nikkei", "noodles", "norcinerie",
    "opensandwiches", "oriental", "pfcomercial", "pakistani", "panasian", "eltern_cafes", "parma", "persian",
    "peruvian", "pita", "pizza", "polish", "pierogis", "polynesian", "popuprestaurants", "portuguese", "alentejo",
    "algarve", "azores", "beira", "fado_houses", "madeira", "minho", "ribatejo", "tras_os_montes", "potatoes",
    "poutineries", "pubfood", "riceshop", "romanian", "rotisserie_chicken", "russian", "salad", "sandwiches",
    "scandinavian", "schnitzel", "scottish", "seafood", "serbocroatian", "signature_cuisine", "singaporean",
    "slovakian", "somali", "soulfood", "soup", "southern", "spanish", "arroceria_paella", "sri_lankan",
    "steakhouses", "swabian", "swedish", "swiss", "syrian", "tabernas", "taiwanese", "teaparlor", "teppanyaki",
    "tex-mex", "thai", "traditional", "tradamerican", "turkish", "doner", "gokuniku", "homemade", "lahmacun", "oygur",
    "pide", "trinidadian", "turkishravioli", "uyghur", "uzbek", "vegan", "vegetarian", "venison", "vietnamese",
    "waffles", "wok", "wraps", "yugoslav"
)

if __name__ == "__main__":
    # initiate a list for data to be added to
    valid_data = []
    # keep track of number of calls to api and how many businesses yelp returned that are considered invalid
    calls = 0
    invalid_count = 0

    # Handle the API request error here
    try:
        # loop through the neighborhoods we are interested in
        for neighborhood, zip_codes in ZIPS.items():
            for code in zip_codes:
                # offset will increase each api call per neighborhood to ensure we get most of the businesses
                offset = 0
                # add_count will remain above zero as long as an api call returns at least one valid business
                add_count = 1

                while add_count > 0 and offset < 1000:
                    add_count = 0
                    # retrieve data from the api using the zip code and offset
                    url = 'https://api.yelp.com/v3/businesses/search?location=Boston%2C%20MA%2C%20' + code + '&term=restaurants&sort_by=distance&limit=50&offset=' + str(offset)
                    headers = {
                                'accept': 'application/json',
                                'Authorization': 'Bearer ' + TOKEN
                            }

                    response = requests.get(url, headers=headers)
                    json_data = response.json()

                    for bus in json_data['businesses']:
                        # verify that the business is located within the target neighborhood
                        if bus['location']['zip_code'] == code:
                            # verify that yelp has returned a valid restaurant category
                            for cuisine_dict in bus['categories']:
                                if cuisine_dict['alias'] in VALID_CUISINES:
                                    # add business to valid data list
                                    bus['neighborhood'] = neighborhood
                                    valid_data.append(bus)
                                    add_count += 1
                                else:
                                    invalid_count += 1
                                    break
                        else:
                            invalid_count += 1

                    # increase the offset and calls before next iteration
                    offset += 50
                    calls += 1
                    # do not call from the api too fast
                    time.sleep(1)
    except:
        print('Failed to retrieve data from the API.')
        valid_data = False