import csv
import datetime
import hashlib
import logging
import os
import re
import sys
import urllib.parse
from dataclasses import dataclass
from collections import defaultdict, deque
from concurrent.futures import ProcessPoolExecutor
from contextlib import contextmanager
from decimal import Decimal
from itertools import chain
from io import BytesIO, IOBase
from pathlib import Path, PurePosixPath
from typing import Optional, Callable

import dateutil
import dateutil.parser
from bs4 import BeautifulSoup
import httpx
from lxml import etree
from lxml.etree import XMLSyntaxError
from stream_unzip import stream_unzip

_COLUMNS = (
    'run_code',
    'company_id',
    'date',
    'file_type',
    'taxonomy',
    'balance_sheet_date',
    'companies_house_registered_number',
    'entity_current_legal_name',
    'company_dormant',
    'average_number_employees_during_period',
    'period_start',
    'period_end',
    'tangible_fixed_assets',
    'debtors',
    'cash_bank_in_hand',
    'current_assets',
    'creditors_due_within_one_year',
    'creditors_due_after_one_year',
    'net_current_assets_liabilities',
    'total_assets_less_current_liabilities',
    'net_assets_liabilities_including_pension_asset_liability',
    'called_up_share_capital',
    'profit_loss_account_reserve',
    'shareholder_funds',
    'turnover_gross_operating_revenue',
    'other_operating_income',
    'cost_sales',
    'gross_profit_loss',
    'administrative_expenses',
    'raw_materials_consumables',
    'staff_costs',
    'depreciation_other_amounts_written_off_tangible_intangible_fixed_assets',
    'other_operating_charges_format2',
    'operating_profit_loss',
    'profit_loss_on_ordinary_activities_before_tax',
    'tax_on_profit_or_loss_on_ordinary_activities',
    'profit_loss_for_period',
    'error',
    'zip_url',
)

logger = logging.getLogger(__name__)

def _xbrl_to_rows(name_xbrl_xml_str_orig):
    name, xbrl_xml_str_orig = name_xbrl_xml_str_orig

    # Slightly hacky way to remove BOM, which is present in some older data
    xbrl_xml_str = BytesIO(xbrl_xml_str_orig[xbrl_xml_str_orig.find(b'<'):])

    # Low level value parsers

    def _date(text):
        return dateutil.parser.parse(text).date()

    def _parse(element, text, parser):
        return \
            parser(element, text.strip()) if text and text.strip() not in ['', '-'] else \
            None

    def _parse_str(element, text):
        return str(text).replace('\n', ' ').replace('"', '')

    def _parse_absolute(element, text):
        # Some cases where employee numbers have a negative sign attached,
        # seemingly indicating negative employee numbers
        return abs(_parse_decimal_with_colon_or_dash(element, text))

    def _parse_decimal(element, text):
        sign = -1 if element.get('sign', '') == '-' else +1
        text_without_thousands_separator = \
            text.replace('.', '').replace(',', '.') if element.get('format', '').rpartition(':')[2] == 'numdotcomma' else \
            text.replace(' ', '') if element.get('format', '').rpartition(':')[2] == 'numspacedot' else \
            text.replace(',', '')

        return sign * Decimal(text_without_thousands_separator) * Decimal(10) ** Decimal(element.get('scale', '0'))

    def _parse_decimal_with_colon_or_dash(element, text):
        # Values seem to have a human readble prefix that isn't part of the value,
        # like "2017 - 2" to mean 2 employees. So we strip the prefix.
        return _parse(element, re.sub(r'(.*:)|(.+- )', '', text), _parse_decimal)

    def _parse_date(element, text):
        format = element.get('format','').rpartition(':')[2].lower()
        day_first = format in ('datedaymonthyear', 'dateslasheu', 'datedoteu')
        if format == 'datedaymonthyearen':
            text = text.replace(' ','')
        text = re.sub(r"(?i)(\d)((st)|(nd)|(rd)|(th))", r"\1", text)
        try:
            return dateutil.parser.parse(text, dayfirst=day_first).date()
        except dateutil.parser.ParserError:
            # Try to parse mis-spellings that still have the first 3 characters right
            return dateutil.parser.parse(re.sub(r'([a-zA-Z]+)', lambda m: m.group(0)[:3], text), dayfirst=day_first).date()

    def _parse_bool(element, text):
        return False if text == 'false' else True if text == 'true' else None

    def _parse_reversed_bool(element, text):
        return False if text == 'true' else True if text == 'false' else None

    # Parsing strategy
    #
    # The XBRL format is a "tagging" format that can tag elements in any order with machine readable metadata.
    # While flexible, this means that it's difficult to efficiently convert to a dataframe.
    #
    # The simplest way to do this would XPath repeatedly to find extract the data for each columnn. This was
    # done in previous versions, but took about 3 times as long as the current solution. The current solution
    # leverages the fact that dictionary lookups are fast, and so constructs dictionaries that can be looked up
    # while iterating through all the elements in the document.

    # Although in some cases a dictionary lookup doesn't seem possible, and so a custom matcher can be defined

    @dataclass
    class _test():
        name: Optional[str]
        search: Callable = lambda element, local_name, attribute_name, context_ref: (element,)

    @dataclass
    class _tn(_test):
        # (Local) Tag name, i.e. withoout namespace
        pass

    @dataclass
    class _av(_test):
        # Attribute value. Matches on the "name" attribute, but stripping off the namespace prefix
        pass

    @dataclass
    class _custom(_test):
        # Custom test when matching on tag name or name attribute isn't enought
        pass

    GENERAL_XPATH_MAPPINGS = {
        'balance_sheet_date': (
            [
                (_av('BalanceSheetDate'), _parse_date),
                (_tn('BalanceSheetDate'), _parse_date),
            ]
        ),
        'companies_house_registered_number': (
            [
                (_av('UKCompaniesHouseRegisteredNumber'), _parse_str),
                (_tn('CompaniesHouseRegisteredNumber'), _parse_str),
            ]
        ),
        'entity_current_legal_name': (
            [
                (_av('EntityCurrentLegalOrRegisteredName', lambda element, local_name, attribute_name, context_ref: chain((element,), element.xpath("./*[local-name()='span'][1]"),)), _parse_str),
                (_tn('EntityCurrentLegalName', lambda element, local_name, attribute_name, context_ref: chain((element,), element.xpath("./*[local-name()='span'][1]"),)), _parse_str),
            ]
        ),
        'company_dormant': (
            [
                (_av('EntityDormantTruefalse'), _parse_bool),
                (_av('EntityDormant'), _parse_bool),
                (_tn('CompanyDormant'), _parse_bool),
                (_tn('CompanyNotDormant'), _parse_reversed_bool),
            ]
        ),
        'average_number_employees_during_period': (
            [
                (_av('AverageNumberEmployeesDuringPeriod'), _parse_absolute),
                (_av('EmployeesTotal'), _parse_absolute),
                (_tn('AverageNumberEmployeesDuringPeriod'), _parse_absolute),
                (_tn('EmployeesTotal'), _parse_absolute),
            ]
        ),
    }

    PERIODICAL_XPATH_MAPPINGS = {
        # balance sheet
        'tangible_fixed_assets': (
            [
                (_tn('FixedAssets'), _parse_decimal),
                (_av('FixedAssets'), _parse_decimal),
                (_tn('TangibleFixedAssets'), _parse_decimal),
                (_av('TangibleFixedAssets'), _parse_decimal),
                (_av('PropertyPlantEquipment'), _parse_decimal),
            ]
        ),
        'debtors': (
            [
                (_tn('Debtors'), _parse_decimal),
                (_av('Debtors'), _parse_decimal),
            ]
        ),
        'cash_bank_in_hand': (
            [
                (_tn('CashBankInHand'), _parse_decimal),
                (_av('CashBankInHand'), _parse_decimal),
                (_av('CashBankOnHand'), _parse_decimal),
            ]
        ),
        'current_assets': (
            [
                (_tn('CurrentAssets'), _parse_decimal),
                (_av('CurrentAssets'), _parse_decimal),
            ]
        ),
        'creditors_due_within_one_year': (
            [
                (_av('CreditorsDueWithinOneYear'), _parse_decimal),
                (_av('Creditors', lambda element, local_name, attribute_name, context_ref: (element,) if 'WithinOneYear' in element.get('contextRef') else ()), _parse_decimal),
            ]
        ),
        'creditors_due_after_one_year': (
            [
                (_av('CreditorsDueAfterOneYear'), _parse_decimal),
                (_custom(None, lambda element, local_name, attribute_name, context_ref: (element,) if 'Creditors' == local_name and 'AfterOneYear' in context_ref else ()), _parse_decimal)
            ]
        ),
        'net_current_assets_liabilities': (
            [
                (_tn('NetCurrentAssetsLiabilities'), _parse_decimal),
                (_av('NetCurrentAssetsLiabilities'), _parse_decimal),
            ]
        ),
        'total_assets_less_current_liabilities': (
            [
                (_tn('TotalAssetsLessCurrentLiabilities'), _parse_decimal),
                (_av('TotalAssetsLessCurrentLiabilities'), _parse_decimal),
            ]
        ),
        'net_assets_liabilities_including_pension_asset_liability': (
            [
                (_tn('NetAssetsLiabilitiesIncludingPensionAssetLiability'), _parse_decimal),
                (_av('NetAssetsLiabilitiesIncludingPensionAssetLiability'), _parse_decimal),
                (_tn('NetAssetsLiabilities'), _parse_decimal),
                (_av('NetAssetsLiabilities'), _parse_decimal),
            ]
        ),
        'called_up_share_capital': (
            [
                (_tn('CalledUpShareCapital'), _parse_decimal),
                (_av('CalledUpShareCapital'), _parse_decimal),
                (_custom(None, lambda element, local_name, attribute_name, context_ref: (element,) if 'Equity' == attribute_name and 'ShareCapital' in element.get('contextRef', '') else ()), _parse_decimal),
            ]
        ),
        'profit_loss_account_reserve': (
            [
                (_tn('ProfitLossAccountReserve'), _parse_decimal),
                (_av('ProfitLossAccountReserve'), _parse_decimal),
                (_custom(None, lambda element, local_name, attribute_name, context_ref: (element,) if 'Equity' == attribute_name and 'RetainedEarningsAccumulatedLosses' in element.get('contextRef', '') else ()), _parse_decimal),
            ]
        ),
        'shareholder_funds': (
            [
                (_tn('ShareholderFunds'), _parse_decimal),
                (_av('ShareholderFunds'), _parse_decimal),
                (_custom(None,  lambda element, local_name, attribute_name, context_ref: (element,) if 'Equity' == attribute_name and 'segment' not in context_ref else ()), _parse_decimal),
            ]
        ),
        # income statement
        'turnover_gross_operating_revenue': (
            [
                (_tn('TurnoverGrossOperatingRevenue'), _parse_decimal),
                (_av('TurnoverGrossOperatingRevenue'), _parse_decimal),
                (_tn('TurnoverRevenue'), _parse_decimal),
                (_av('TurnoverRevenue'), _parse_decimal),
            ]
        ),
        'other_operating_income': (
            [
                (_tn('OtherOperatingIncome'), _parse_decimal),
                (_av('OtherOperatingIncome'), _parse_decimal),
                (_tn('OtherOperatingIncomeFormat2'), _parse_decimal),
                (_av('OtherOperatingIncomeFormat2'), _parse_decimal),
            ]
        ),
        'cost_sales': (
            [
                (_tn('CostSales'), _parse_decimal),
                (_av('CostSales'), _parse_decimal),
            ]
        ),
        'gross_profit_loss': (
            [
                (_tn('GrossProfitLoss'), _parse_decimal),
                (_av('GrossProfitLoss'), _parse_decimal),
            ]
        ),
        'administrative_expenses': (
            [
                (_tn('AdministrativeExpenses'), _parse_decimal),
                (_av('AdministrativeExpenses'), _parse_decimal),
            ]
        ),
        'raw_materials_consumables': (
            [
                (_tn('RawMaterialsConsumables'), _parse_decimal),
                (_av('RawMaterialsConsumables'), _parse_decimal),
                (_tn('RawMaterialsConsumablesUsed'), _parse_decimal),
                (_av('RawMaterialsConsumablesUsed'), _parse_decimal),
            ]
        ),
        'staff_costs': (
            [
                (_tn('StaffCosts'), _parse_decimal),
                (_av('StaffCosts'), _parse_decimal),
                (_tn('StaffCostsEmployeeBenefitsExpense'), _parse_decimal),
                (_av('StaffCostsEmployeeBenefitsExpense'), _parse_decimal),
            ]
        ),
        'depreciation_other_amounts_written_off_tangible_intangible_fixed_assets': (
            [
                (_tn('DepreciationOtherAmountsWrittenOffTangibleIntangibleFixedAssets'), _parse_decimal),
                (_av('DepreciationOtherAmountsWrittenOffTangibleIntangibleFixedAssets'), _parse_decimal),
                (_tn('DepreciationAmortisationImpairmentExpense'), _parse_decimal),
                (_av('DepreciationAmortisationImpairmentExpense'), _parse_decimal),
            ]
        ),
        'other_operating_charges_format2': (
            [
                (_tn('OtherOperatingChargesFormat2'), _parse_decimal),
                (_av('OtherOperatingChargesFormat2'), _parse_decimal),
                (_tn('OtherOperatingExpensesFormat2'), _parse_decimal),
                (_av('OtherOperatingExpensesFormat2'), _parse_decimal),
            ]
        ),
        'operating_profit_loss': (
            [
                (_tn('OperatingProfitLoss'), _parse_decimal),
                (_av('OperatingProfitLoss'), _parse_decimal),
            ]
        ),
        'profit_loss_on_ordinary_activities_before_tax': (
            [
                (_tn('ProfitLossOnOrdinaryActivitiesBeforeTax'), _parse_decimal),
                (_av('ProfitLossOnOrdinaryActivitiesBeforeTax'), _parse_decimal),
            ]
        ),
        'tax_on_profit_or_loss_on_ordinary_activities': (
            [
                (_tn('TaxOnProfitOrLossOnOrdinaryActivities'), _parse_decimal),
                (_av('TaxOnProfitOrLossOnOrdinaryActivities'), _parse_decimal),
                (_tn('TaxTaxCreditOnProfitOrLossOnOrdinaryActivities'), _parse_decimal),
                (_av('TaxTaxCreditOnProfitOrLossOnOrdinaryActivities'), _parse_decimal),
            ]
        ),
        'profit_loss_for_period': (
            [
                (_tn('ProfitLoss'), _parse_decimal),
                (_av('ProfitLoss'), _parse_decimal),
                (_tn('ProfitLossForPeriod'), _parse_decimal),
                (_av('ProfitLossForPeriod'), _parse_decimal),
            ]
        ),
    }

    ALL_MAPPINGS = dict(**GENERAL_XPATH_MAPPINGS, **PERIODICAL_XPATH_MAPPINGS)

    TAG_NAME_TESTS = {
        test.name: (name, priority, test, parser)
        for (name, tests) in ALL_MAPPINGS.items()
        for (priority, (test, parser)) in enumerate(tests)
        if isinstance(test, _tn)
    }

    ATTRIBUTE_VALUE_TESTS = {
        test.name: (name, priority, test, parser)
        for (name, tests) in ALL_MAPPINGS.items()
        for (priority, (test, parser)) in enumerate(tests)
        if isinstance(test, _av)
    }

    CUSTOM_TESTS = tuple(
        (name, priority, test, parser)
        for (name, tests) in ALL_MAPPINGS.items()
        for (priority, (test, parser)) in enumerate(tests)
        if isinstance(test, _custom)
    )

    def _get_dates(context):
        instant_elements = context.xpath("./*[local-name()='instant']")
        start_date_text_nodes = context.xpath("./*[local-name()='startDate']/text()")
        end_date_text_nodes = context.xpath("./*[local-name()='endDate']/text()")
        return \
            (None, None) if context is None else \
            (instant_elements[0].text.strip(), instant_elements[0].text.strip()) if instant_elements else \
            (None, None) if start_date_text_nodes[0] is None or end_date_text_nodes[0] is None else \
            (start_date_text_nodes[0].strip(), end_date_text_nodes[0].strip())

    try:
        document = etree.parse(xbrl_xml_str, etree.XMLParser(ns_clean=True, recover=True))
        root = document.getroot()
        document.xpath("//*[0]")
    except:
        # In at least one case - Prod224_9956_04944372_20100331.xml, the XML seems very badly formed.
        # Suspect this is before Companies House had better validation. The best we can do is log and
        # carry on. We can at least still get a row in the data
        logger.warning("Bad XML. Name: %s XML: %s", name, xbrl_xml_str_orig)
        document = etree.parse(BytesIO(b'<?xml version="1.0" encoding="UTF-8"?><root></root>'), etree.XMLParser(ns_clean=True, recover=True))
        root = document.getroot()

    context_dates = {
        e.get('id'): _get_dates(period)
        for e in document.xpath("//*[local-name()='context']")
        for period in e.xpath("./*[local-name()='period']")[:1]
    }

    fn = os.path.basename(name)
    # Some April 2021 data files end in .zip, but seem to really be html
    mo = re.match(r'^(Prod\d+_\d+)_([^_]+)_(\d\d\d\d\d\d\d\d)\.(html|xml|zip)', fn)
    run_code, company_id, date, filetype = mo.groups()
    allowed_taxonomies = [
        'http://www.xbrl.org/uk/fr/gaap/pt/2004-12-01',
        'http://www.xbrl.org/uk/gaap/core/2009-09-01',
        'http://xbrl.frc.org.uk/fr/2014-09-01/core',
    ]

    core_attributes = (
        run_code,
        company_id,
        _date(date),
        filetype,
        ';'.join(set(allowed_taxonomies) & set(root.nsmap.values())),
    )

    # Mutable dictionaries to store the "priority" (lower is better) of a found value
    general_attributes_with_priorities = {
        name: (10, None)
        for name in GENERAL_XPATH_MAPPINGS.keys()
    }
    periodic_attributes_with_priorities = defaultdict(lambda: {
        name: (10, None)
        for name in PERIODICAL_XPATH_MAPPINGS.keys()
    })

    def tag_name_tests(local_name):
        try:
            yield from (TAG_NAME_TESTS[local_name],)
        except KeyError:
            pass

    def attribute_value_tests(attribute_value):
        try:
            yield from (ATTRIBUTE_VALUE_TESTS[attribute_value],)
        except KeyError:
            pass

    def handle_general(element, local_name, attribute_value, context_ref, name, priority, test, parse):
        best_priority, best_value = general_attributes_with_priorities[name]

        if priority > best_priority:
            return

        for element in test.search(element, local_name, attribute_value, context_ref):
            filtered = ((e.text or '') for e in element.iter() if e.tag.rpartition('}')[2] != "exclude")
            value = _parse(element, ''.join(filtered), parse)
            if value is not None:
                general_attributes_with_priorities[name] = (priority, value)
                break

    def handle_periodic(element, local_name, attribute_value, context_ref, name, priority, test, parse):
        if not context_ref:
            return
        dates = context_dates.get(context_ref)
        if not dates:
            return

        for element in test.search(element, local_name, attribute_value, context_ref):
            best_priority, best_value = periodic_attributes_with_priorities[dates][name]

            if priority >= best_priority:
                return

            filtered = ((e.text or '') for e in element.iter() if etree.QName(e).localname != "exclude")
            value = _parse(element, ''.join(filtered), parse)
            if value is not None:
                periodic_attributes_with_priorities[dates][name] = (priority, value)
                break

    error = None
    try:
        for element in document.xpath('//*'):
            _, _, local_name = element.tag.rpartition('}')
            _, _, attribute_value = element.get('name', '').rpartition(':')
            context_ref = element.get('contextRef', '')

            for name, priority, test, parse in chain(tag_name_tests(local_name), attribute_value_tests(attribute_value), CUSTOM_TESTS):
                handler = \
                    handle_general if name in general_attributes_with_priorities else \
                    handle_periodic

                handler(element, local_name, attribute_value, context_ref, name, priority, test, parse)

        general_attributes = tuple(
            general_attributes_with_priorities[name][1]
            for name in GENERAL_XPATH_MAPPINGS.keys()
        )

        periods = tuple(
            (datetime.date.fromisoformat(period_start_end[0]), datetime.date.fromisoformat(period_start_end[1]))
            + tuple(
                periodic_attributes[name][1]
                for name in PERIODICAL_XPATH_MAPPINGS.keys()
            )
            for period_start_end, periodic_attributes in periodic_attributes_with_priorities.items()
        )
        sorted_periods = sorted(periods, key=lambda period: (period[0], period[1]), reverse=True)
    except ValueError as e:
        error = str(e)

    return \
        ((core_attributes + (None,) * (2 + len(GENERAL_XPATH_MAPPINGS) + len(PERIODICAL_XPATH_MAPPINGS)) + (error,)),) if error is not None else \
        tuple((core_attributes + general_attributes + period + (None,)) for period in sorted_periods) if sorted_periods else \
        ((core_attributes + general_attributes + (None,) * (3 + len(PERIODICAL_XPATH_MAPPINGS))),)

@contextmanager
def stream_read_xbrl_zip(
    zip_bytes_iter,
    zip_url=None,
):
    queue = deque()
    num_workers = max(os.cpu_count() - 1, 1)

    def imap(executor, func, param_iterables):
        for params in param_iterables:
            if len(queue) == num_workers:
                yield queue.popleft().result()

            queue.append(executor.submit(func, params))

        while queue:
            yield queue.popleft().result()

    with ProcessPoolExecutor(max_workers=num_workers) as executor:
        yield _COLUMNS, (
            row + (zip_url,)
            for results in imap(executor, _xbrl_to_rows, ((name.decode(), b''.join(chunks)) for name, _, chunks in stream_unzip(zip_bytes_iter)))
            for row in results
        )

@contextmanager
def stream_read_xbrl_sync(
    ingest_data_after_date=datetime.date(datetime.MINYEAR, 1, 1),
    data_urls=(
        'https://download.companieshouse.gov.uk/en_accountsdata.html',
        'https://download.companieshouse.gov.uk/en_monthlyaccountsdata.html',
        'https://download.companieshouse.gov.uk/historicmonthlyaccountsdata.html',
    ),
    get_client = lambda: httpx.Client(timeout=60.0, transport=httpx.HTTPTransport(retries=3)),
    chunk_size = 100 * 1048576  # 100 MiB
):
    def extract_start_end_dates(url):
        file_basename = os.path.basename(url)
        file_name_no_ext = os.path.splitext(file_basename)[0]

        if 'JanToDec' in file_name_no_ext or 'JanuaryToDecember' in file_name_no_ext:
            file_name_no_ext = os.path.splitext(url)[0]
            year = file_name_no_ext[-4:]
            return datetime.date(int(year), 1, 1), datetime.date(int(year), 12, 31)
        elif 'Accounts_Monthly_Data' in file_name_no_ext and file_name_no_ext[-4:].isnumeric():
            year = int(file_name_no_ext[-4:])
            month_name = file_name_no_ext.split('-')[1][:-4]
            # Convert the month name to a month number
            month_num = datetime.datetime.strptime(month_name, '%B').month
            # Calculate the last date of the month
            first_day_of_month = datetime.date(year, month_num, 1)
            next_month = datetime.date(year, month_num, 28) + datetime.timedelta(days=4)
            last_day_of_month = next_month - datetime.timedelta(days=next_month.day)
            return (first_day_of_month, last_day_of_month)
        elif 'Accounts_Bulk_Data' in file_name_no_ext:
            date_str = file_name_no_ext.split('-', 1)[1]
            day = datetime.datetime.strptime(date_str, '%Y-%m-%d').date()
            return (day, day)
        else:
            return (None, None)

    def get_content(client, url):
        r = client.get(url)
        r.raise_for_status()
        return r.content

    @contextmanager
    def get_content_streamed(client, url):

        def get_chunks():
            start = 0
            end = chunk_size - 1
            etag = None
            remaining = None

            while remaining is None or remaining > 0:
                with client.stream('GET', url, headers={
                    'range': f'bytes={start}-{end}'
                }) as r:
                    r.raise_for_status()
                    if etag is None:
                        etag = r.headers['etag']
                    else:
                        if etag != r.headers['etag']:
                            raise Exception('etag has changed since beginning requests')
                    if remaining is None:
                        remaining = int(r.headers['content-range'].split('/')[1])
                    content_length =int(r.headers['content-length'])
                    assert content_length > 0
                    remaining -= content_length
                    yield from r.iter_bytes(chunk_size=65536)
                start += chunk_size
                end += chunk_size

        chunks = get_chunks()
        try:
            yield chunks
        finally:
            # This is for the case of unfinished iteration. It raises a GeneratorExit in get_chunks, so any
            # open context in "get_chunks" gets properly closed, i.e. to close its open HTTP connection
            chunks.close()

    dummy_list_to_ingest = [
        (datetime.date(2021, 5, 2), (('1', '2'), ('3', '4'))),
        (datetime.date(2022, 2, 8), (('5', '6'), ('7', '8'))),
    ]

    with get_client() as client:
        pages_of_links = [
            (data_url, BeautifulSoup(get_content(client, data_url), 'html.parser').find_all('a'))
            for data_url in data_urls
        ]

        all_zip_urls = [
            link.attrs['href'].strip() if link.attrs['href'].strip().startswith('http://') or link.attrs['href'].strip().startswith('https://') else
            urllib.parse.urljoin(data_url, link.attrs['href'].strip())
            for (data_url, page_of_links) in pages_of_links
            for link in page_of_links
            if link.attrs.get('href', '').endswith('.zip')
        ]

        all_zip_urls_with_dates = [
            (zip_url, extract_start_end_dates(zip_url))
            for zip_url in all_zip_urls
        ]

        all_zip_urls_with_parseable_dates = [
            (zip_url, dates)
            for (zip_url, dates) in all_zip_urls_with_dates
            if dates != (None, None)
        ]

        all_zip_urls_with_dates_oldest_first = sorted(
            all_zip_urls_with_parseable_dates, key=lambda zip_start_end: (zip_start_end[1][0], zip_start_end[1][1])
        )

        zip_urls_with_date_in_range_to_ingest = [
            (zip_url, (start_date, end_date))
            for (zip_url, (start_date, end_date)) in all_zip_urls_with_dates_oldest_first
            if (start_date, end_date) != (None, None) and end_date > ingest_data_after_date
        ]

        def _final_date_and_rows():
            for zip_url, (start_date, end_date) in zip_urls_with_date_in_range_to_ingest:
                with get_content_streamed(client, zip_url) as chunks:
                    with stream_read_xbrl_zip(chunks, zip_url=zip_url) as (_, rows):
                        yield (start_date, end_date), rows

        yield (_COLUMNS, _final_date_and_rows())

def stream_read_xbrl_sync_s3_csv(s3_client, bucket_name, key_prefix):

    def _to_file_like_obj(iterable):
        chunk = b''
        offset = 0
        it = iter(iterable)

        def up_to_iter(size):
            nonlocal chunk, offset

            while size:
                if offset == len(chunk):
                    try:
                        chunk = next(it)
                    except StopIteration:
                        break
                    else:
                        offset = 0
                to_yield = min(size, len(chunk) - offset)
                offset = offset + to_yield
                size -= to_yield
                yield chunk[offset - to_yield : offset]

        class FileLikeObj(IOBase):
            def readable(self):
                return True

            def read(self, size=-1):
                return b''.join(
                    up_to_iter(float('inf') if size is None or size < 0 else size)
                )

        return FileLikeObj()

    def _convert_to_csv(columns, rows):
        class PseudoBuffer:
            def write(self, value):
                return value.encode("utf-8")

        pseudo_buffer = PseudoBuffer()
        csv_writer = csv.writer(pseudo_buffer, quoting=csv.QUOTE_NONNUMERIC)
        yield csv_writer.writerow(columns)
        yield from (csv_writer.writerow(row) for row in rows)

    s3_paginator = s3_client.get_paginator('list_objects_v2')
    dates = (
        # The -10: is to support older versions where only the end date was in the file name
        datetime.date.fromisoformat(PurePosixPath(content['Key']).stem[-10:])
        for page in s3_paginator.paginate(Bucket=bucket_name, Prefix=key_prefix)
        for content in page.get('Contents', ())
    )
    latest_completed_date = max(dates, default=datetime.date(datetime.MINYEAR, 1, 1))

    with stream_read_xbrl_sync(latest_completed_date) as (columns, final_date_and_rows):
        for ((start_date, final_date), rows) in final_date_and_rows:
            key = f'{key_prefix}{start_date}--{final_date}.csv'
            logger.info('Saving Companies House accounts data to %s/%s ...', bucket_name, key)
            csv_file = _to_file_like_obj(_convert_to_csv(columns, rows))
            s3_client.upload_fileobj(Bucket=bucket_name, Key=key, Fileobj=csv_file)
            logger.info('Saving Companies House accounts data to %s/%s (done)', bucket_name, key)

def stream_read_xbrl_debug(zip_url, run_code, company_id, date, debug_cache_folder=".debug-cache"):
    Path(debug_cache_folder).mkdir(parents=True, exist_ok=True)

    # Hashing so we have a filesystem-safe URL
    hashed_zip_url = hashlib.sha256(zip_url.encode('utf-8')).hexdigest()
    local_zip_file = Path(debug_cache_folder).joinpath(hashed_zip_url)

    if not local_zip_file.exists():
        print('The ZIP', zip_url, 'does not exist in local cache. Downloading...', file=sys.stderr)
        done = False
        try:
            with \
                    httpx.stream('GET', zip_url) as r, \
                    open(local_zip_file, 'wb') as f:
                r.raise_for_status()
                for chunk in r.iter_bytes():
                    f.write(chunk)
            done = True
        finally:
            if not done:
                local_zip_file.unlink(missing_ok=True)
        print('Downloaded', file=sys.stderr)
    else:
        print('Found', zip_url, 'in local cache', file=sys.stderr)

    def local_chunks():
        with open(local_zip_file, 'rb') as f:
            while True:
                chunk = f.read(65536)
                if not chunk:
                    break
                yield chunk

    print('Searching ZIP for member file matching', run_code, company_id, date, file=sys.stderr)
    found = False
    for name, _, chunks in stream_unzip(local_chunks()):
        fn = os.path.basename(name.decode('utf-8'))
        mo = re.match(r'^(Prod\d+_\d+)_([^_]+)_(\d\d\d\d\d\d\d\d)\.(html|xml)', fn)
        _run_code, _company_id, _date, _ = mo.groups()
        # print(_run_code, _company_id, _date)

        if _run_code == run_code and _company_id == company_id and _date == date.isoformat().replace('-', ''):
            print('Found matching file', name, file=sys.stderr)
            found = True
            for chunk in chunks:
                sys.stdout.buffer.write(chunk)
        else:
            for chunk in chunks:
                pass

    if not found:
        print('No matching member file found', file=sys.stderr)

    print('Finished', file=sys.stderr)


import os.path
from psycopg2 import sql
import io

from odoo import SUPERUSER_ID, api
from odoo.tools import misc, pycompat
from odoo import api, tools
from odoo.tools.misc import file_open

from . import models

def _import_sem_keyword_ads_historical_metrics_csv(env):
    fname = misc.file_path(os.path.join('viin_mkp_support', 'data',
                                        'sem.keyword.ads.historical.metrics.csv'))
    with file_open(fname, mode='rb') as fp:
        reader = pycompat.csv_reader(io.BytesIO(fp.read()), quotechar='"', delimiter=',')
        columns = next(reader)
        list_index_ref = []
        list_index_ref.append(columns.index('keyword_id'))
        list_index_ref.append(columns.index('country_id'))
        list_index_ref.append(columns.index('language_id'))
        list_index_ref.append(columns.index('currency_id'))
        datas = []
        for row in reader:
            for index in list_index_ref:
                row[index] = env.ref(row[index]).id
            datas.append(tuple(row))

        if datas:
            for small_batch in env['to.base'].splittor(datas, 10000):
                query = sql.SQL("INSERT INTO {} ({}) VALUES {}").format(
                    sql.Identifier('sem_keyword_ads_historical_metrics'),
                    sql.SQL(", ").join(sql.Identifier(name) for name in columns),
                    sql.SQL(", ".join([str(row).replace('None', 'Null') for row in small_batch])),
                )
            env.cr.execute(query)

def post_init_hook(cr, registry):
    env = api.Environment(cr, SUPERUSER_ID, {})
    # active VietNam
    # tools.convert_file(env.cr, 'viin_mkp_support', 'data/sem.keyword.csv', None,
    #                    mode='init', noupdate=True, kind='init')
    # _import_sem_keyword_ads_historical_metrics_csv(env)

def pre_init_hook(cr):
    env = api.Environment(cr, SUPERUSER_ID, {})
    # active VietNam
    lang = env.ref('base.lang_vi_VN')
    env.ref('base.lang_vi_VN')._activate_lang(lang.code)
    mods = env['ir.module.module'].search([('state', '=', 'installed')])
    mods._update_translations(lang.code, True)
    env.cr.execute('ANALYZE ir_translation')


from random import choice

import pygame as pg
from .. import tools, prepare
from ..components.labels import Label, ButtonGroup, Button

SONGS = [
    ("main_stem", "Main Stem - U.S. Army Blues", 0.2),
    ("betcha_nickel",
        "Betcha' Nickel - Ella Fitzgerald and Her Famous Orchestra", 0.2),
    ("buckin_the_dice", "Buckin' The Dice - Fats Waller and His Rhythm", 0.2),
    ("money_burns_a_hole_in_my_pocket",
        "Money Burns A Hole In My Pocket - Dean Martin", 0.2),
    ("ace_in_the_hole", "Ace In The Hole - Ella Fitzgerald", 0.25),
    ("gospel_truth", "The Gospel Truth - U.S. Army Blues", 0.2),
    ("world_on_a_string",
        "I've Got The World On A String - Frank Sinatra", 0.3),
    ("im_shooting_high", "I'm Shooting High - Chris Connor", 0.5),
    ("anything_for_you", ("Anything For You - "
        "Kay Starr with Van Alexander's Orchestra"), 0.35)
    ]

class MusicHandler(object):
    def __init__(self):
        self.current_song = "main_stem"
        self.song_index = 0
        self.songs = {s[0]: (prepare.MUSIC[s[0]], s[1], s[2]) for s in SONGS}
        self.rect = pg.Rect(0, 0, 300, 70)
        self.rect.right = prepare.RENDER_SIZE[0]
        self.music_on = not prepare.ARGS['music_off']
        self.volume_mod = 1.0
        self.prep_volume_icons()
        self.buttons = self.make_buttons()
        font = prepare.FONTS["Saniretro"]
        self.volume_label = Label(font, 32, "Volume", "goldenrod3",
                                 {"midtop": (self.rect.centerx,self.rect.y+3)})
        self.change_song("main_stem")

    def make_buttons(self):
        icon_sheet = prepare.GFX["audio_icon_strip"]
        icon_size = (64, 64)
        icons = tools.strip_from_sheet(icon_sheet, (0, 0), icon_size, 3, 2)
        buttons = ButtonGroup()
        mute_rect = pg.Rect((0,0), icon_size)
        mute_rect.bottomleft = (self.rect.x+5, self.rect.bottom)
        skip_rect = mute_rect.copy()
        skip_rect.bottomright = (self.rect.right-5, self.rect.bottom)
        skip_kwargs = {"idle_image" : icons[0], "hover_image" : icons[3],
                       "call" : self.change_song,
                       "bindings" : [pg.K_PERIOD]}
        mute_kwargs = {"idle_image" : icons[2], "hover_image" : icons[5],
                       "call" : self.mute_unmute_music,
                       "visible" : self.music_on,
                       "bindings" : [pg.K_m]}
        unmute_kwargs = {"idle_image" : icons[1], "hover_image" : icons[4],
                         "call" : self.mute_unmute_music,
                         "visible" : not self.music_on,
                         "bindings" : [pg.K_m]}
        Button(skip_rect, buttons, **skip_kwargs)
        mute = Button(mute_rect, buttons, **mute_kwargs)
        unmute = Button(mute_rect, buttons, **unmute_kwargs)
        mute.args, unmute.args = (mute, unmute), (unmute, mute)
        return buttons

    def prep_volume_icons(self):
        """
        Strip needed icons off needed sheets and place them in data
        structures as needed.
        """
        volume_sheet = prepare.GFX["volumeicons"]
        self.gray_icon = prepare.GFX["volumeiconsgray"]
        self.gray_rect = self.gray_icon.get_rect()
        self.gray_rect.midbottom=(self.rect.centerx, self.rect.bottom)
        volume_icons = tools.strip_from_sheet(volume_sheet, (0,0), (24,27), 5)
        vol_levels = [0.3, 0.7, 1.0, 1.5, 2.0]
        left = self.gray_rect.left
        x_space = 24
        top = self.gray_rect.top
        self.volume_icons = []
        for i, icon_level in enumerate(zip(volume_icons, vol_levels)):
            vol_icon = VolumeIcon((left+(i*x_space),top),
                                  icon_level[0], icon_level[1])
            self.volume_icons.append(vol_icon)

    def change_song(self, song_name=None):
        """
        Change to a song given by song_name or change to the next song
        if song_name is None.
        """
        if self.music_on:
            if song_name:
                for i,song in enumerate(SONGS):
                    if song_name == song[0]:
                        self.song_index = i
                        break
            else:
                self.song_index = (self.song_index+1)%len(SONGS)
            self.current_song = SONGS[self.song_index][0]
            song_info = self.songs[self.current_song]
            pg.mixer.music.load(song_info[0])
            pg.mixer.music.set_volume(song_info[2] * self.volume_mod)
            pg.mixer.music.play()
            caption_args = prepare.CAPTION, song_info[1]
            caption = "{} - Now Playing: {}".format(*caption_args)
            pg.display.set_caption(caption)

    def change_volume(self, volume_level):
        self.volume_mod = volume_level
        song_volume = self.songs[self.current_song][2]
        pg.mixer.music.set_volume(self.volume_mod*song_volume)

    def get_event(self, event, scale=(1,1)):
        if event.type == pg.MOUSEBUTTONDOWN:
            pos = tools.scaled_mouse_pos(event.pos, scale)
            if self.music_on:
                for icon in self.volume_icons:
                    if icon.rect.collidepoint(pos):
                        self.change_volume(icon.value)
        self.buttons.get_event(event)

    def mute_unmute_music(self, args=None):
        if args:
            visible, invisible = args
            visible.visible = False
            invisible.visible = True
        self.music_on = not self.music_on
        pg.mixer.music.play() if self.music_on else pg.mixer.music.stop()

    def update(self, scale):
        pos = tools.scaled_mouse_pos(pg.mouse.get_pos(), scale)
        self.buttons.update(pos)
        if self.music_on:
            if not pg.mixer.music.get_busy():
                self.change_song()

    def draw(self, surface):
        pg.draw.rect(surface, pg.Color("gray10"), self.rect)
        pg.draw.rect(surface, pg.Color("gold3"), self.rect, 4)
        self.volume_label.draw(surface)
        self.buttons.draw(surface)
        surface.blit(self.gray_icon, self.gray_rect)
        if self.music_on:
            for icon in self.volume_icons:
                if icon.value <= self.volume_mod:
                    icon.draw(surface)

class VolumeIcon(object):
    def __init__(self, topleft, image, value):
        self.value = value
        self.image = image
        self.rect = self.image.get_rect(topleft=topleft)

    def draw(self, surface):
        surface.blit(self.image, self.rect)


import numpy as np
import numpy.random as random
import cv2

import Queue

def compute_prob(map, threshold=0.1):
	w,h = map.shape

	sigma = 1.0*np.count_nonzero(map)/np.size(map)

	return np.abs(random.Light.normal(0,sigma))<threshold

random.seed()

w = 200
h = 200

map = np.zeros((w,h))
walked_places = np.zeros((w,h))

# earth should be 60% of the map
maxSize = 0.6*w*h

seed = [w/2,h/2]#[random.randint(w/3,w-w/3),random.randint(h/5,h-h/5)]
map[seed[0],seed[1]] = 1
walked_places[seed[0],seed[1]] = 1

earthSize = np.count_nonzero(map)

#seed_point // starting point
#visited // boolean array/matrix, same size as image
#point_queue // empty queue

#point_queue.enqueue( seed_point )
#visited( seed_point ) = true
#
#while( point_queue is not empty ) {
#    this_point = point_queue.dequeue()
#    for each neighbour of this_point {
#        if not visited( neighbour ) and neighbour is black/red/whatever
#            point_queue.enqueue( neighbour )
#            visited( neighbour ) = true
#    }
#}

points = Queue.Queue()
points.put(seed)

while earthSize<maxSize and not points.empty():

	cur_pix = points.get()

	# orientations
	orient = [(1, 0), (0, 1), (-1, 0), (0, -1), (1, 1), (1, -1), (-1, -1), (-1, 1)]

	for j in range(len(orient)):
		temp_pix = [cur_pix[0] +orient[j][0], cur_pix[1] +orient[j][1]]

		# are we still in range?
		is_on_map = w>temp_pix[0]>0 and h>temp_pix[1]>0

		if is_on_map and walked_places[temp_pix[0],temp_pix[1]] == 0:
			walked_places[temp_pix[0],temp_pix[1]] = 1
			if compute_prob(map):
				map[temp_pix[0],temp_pix[1]] = 1
				points.put(temp_pix)

	earthSize = np.count_nonzero(map)

kernel = np.ones((5,5),np.uint8)
closing = cv2.morphologyEx(map*255, cv2.MORPH_CLOSE, kernel)

cv2.imshow( "Display window", closing);
cv2.waitKey(0)


import torch
import torch
if False:
    def solve_standard_regression(X,y,alpha=1000):
        # if True:
        # Add a column of ones to X for the bias term
        X = torch.cat((X, torch.ones(X.shape[0], 1,device=X.device)), dim=1)

        # Remove redundant columns from X
        u, s, v = torch.svd(X)
        tol = s.max() * X.shape[1] * torch.finfo(s.dtype).eps
        mask = s > tol
        """
        X = torch.mm(u[:, mask], torch.diag(s[mask]).mm(v[:, mask].t()))

        # Calculate the OLS solution for W and b
        X_t = torch.transpose(X, 0, 1)  # Transpose of X
        XX_t_inv = torch.inverse(torch.matmul(X_t, X))  # Inverse of (X^T * X)
        """
        """
        Xt_X = torch.mm(v[ mask,:].t(), torch.diag(s[mask]**2).mm(v[ mask,:]))
        XX_t_inv = torch.inverse(Xt_X)
        W = torch.matmul(torch.matmul(XX_t_inv, X_t), y.unsqueeze(1))[:-1]  # W = (X^T * X)^-1 * X^T * y, excluding the last element (bias term)
        """
        u = u[:, mask]
        s = s[mask]
        v = v[:, mask]

        # calculate weights using OLS solution
        XtX_inv = torch.mm(v, torch.diag(1/s)).mm(v.t())
        X = torch.mm(u[:, mask], torch.diag(s[mask]).mm(v[:, mask].t()))
        Xt = X.T
        W,b = solve_lasso_regression_(XtX_inv, Xt,y, alpha)
        return W,b

    def solve_lasso_regression_(XtX_inv, Xt,y, alpha):
        # Compute the number of features
        num_features = XtX_inv.shape[0]

        Xty = torch.matmul(Xt, y.unsqueeze(1))

        # Initialize the coefficients with zeros
        W = torch.zeros(num_features, 1, device=XtX_inv.device)

        for _ in range(num_features):
            # Compute the correlation between features and the current residual
            correlations = torch.matmul(XtX_inv, W) - Xty

            # Find the feature with the highest positive correlation
            max_correlation_idx = torch.argmax(correlations)

            # Update the corresponding coefficient
            if correlations[max_correlation_idx] > alpha:
                W[max_correlation_idx] = (correlations[max_correlation_idx] - alpha) / XtX_inv[max_correlation_idx, max_correlation_idx]
            elif correlations[max_correlation_idx] < -alpha:
                W[max_correlation_idx] = (correlations[max_correlation_idx] + alpha) / XtX_inv[max_correlation_idx, max_correlation_idx]
            else:
                W[max_correlation_idx] = 0.0

        # Extract the bias term
        b = W[-1]
        W = W[:-1]
        W = W.squeeze()
        # Print the value of W and b
        import ipdb;ipdb.set_trace()
        print(f"W = {W}")
        print(f"b = {b.item()}")

        return W, b

elif False:
    def solve_standard_regression(X,y,device=None,alpha=None):
        # if True:
        # Add a column of ones to X for the bias term
        X = torch.cat((X, torch.ones(X.shape[0], 1,device=X.device)), dim=1)

        # Remove redundant columns from X
        u, s, v = torch.svd(X)
        tol = s.max() * X.shape[1] * torch.finfo(s.dtype).eps
        mask = s > tol
        """
        X = torch.mm(u[:, mask], torch.diag(s[mask]).mm(v[:, mask].t()))

        # Calculate the OLS solution for W and b
        X_t = torch.transpose(X, 0, 1)  # Transpose of X
        XX_t_inv = torch.inverse(torch.matmul(X_t, X))  # Inverse of (X^T * X)
        """
        """
        Xt_X = torch.mm(v[ mask,:].t(), torch.diag(s[mask]**2).mm(v[ mask,:]))
        XX_t_inv = torch.inverse(Xt_X)
        W = torch.matmul(torch.matmul(XX_t_inv, X_t), y.unsqueeze(1))[:-1]  # W = (X^T * X)^-1 * X^T * y, excluding the last element (bias term)
        """
        u = u[:, mask]
        s = s[mask]
        v = v[:, mask]

        # calculate weights using OLS solution
        XtX_inv = torch.mm(v, torch.diag(1/s)).mm(v.t())
        # W = XtX_inv.mm(X.t()).mm(y)
        W_and_b = XtX_inv.mm(X.t()).mm(y[:,None])
        W_and_b = W_and_b.squeeze()
        b = W_and_b[-1]  # Extract the bias term from the last element of W
        W = W_and_b[:-1]
        if False:
            # Print the value of W and b
            print(f"W = {W.squeeze()}")
            print(f"b = {b.item()}")
        return W,b
elif 'sklearn regression' and True:
    def solve_standard_regression(X,Y,device='cpu',alpha=0,sample_weight=None):
        # alpha=0.1
        from sklearn import linear_model
        if alpha != 0:
            print('alpha',alpha)
            # clf = linear_model.Lasso(alpha=alpha)
            clf = linear_model.Ridge(alpha=alpha)
            # import ipdb;ipdb.set_trace()
            # clf = linear_model.ElasticNet(alpha=alpha,l1_ratio=0.5)
        else:
            clf = linear_model.LinearRegression()
        # clf.fit([[0,0], [1, 1], [2, 2]], [0, 1, 2])
        clf.fit(X.cpu().numpy(),Y.cpu().numpy(),sample_weight=sample_weight)
        w = clf.coef_
        b = clf.intercept_
        w,b = torch.tensor(w,device=device),torch.tensor(b,device=device)
        return w,b
elif "sklearn random forest" and False:
    def solve_standard_regression(X,Y,device='cpu',alpha=0):
        # alpha=0.1
        from sklearn.ensemble import RandomForestRegressor
        regr = RandomForestRegressor(max_depth=2, random_state=0)
        regr.fit(X.cpu().numpy(),Y.cpu().numpy())
        w = regr
        b = None
        # w,b = torch.tensor(w,device=device),torch.tensor(b,device=device)
        return w,b
elif 'ridge-copied from github' and False:
    import torch
    from torch import nn
    import torch.nn.functional as F

    class Ridge:
        def __init__(self, alpha = 0, fit_intercept = True,):
            alpha = 0
            self.alpha = alpha
            self.fit_intercept = fit_intercept

        def fit(self, X: torch.tensor, y: torch.tensor) -> None:

            device = X.device
            X = X.rename(None)
            y = y.rename(None).view(-1,1)
            assert X.shape[0] == y.shape[0], "Number of X and y rows don't match"
            if self.fit_intercept:
                X = torch.cat([torch.ones(X.shape[0], 1,device=device), X], dim = 1)
            # Solving X*w = y with Normal equations:
            # X^{T}*X*w = X^{T}*y
            lhs = X.T @ X
            rhs = X.T @ y
            # if rhs.ndim == 2:
            #     rhs= rhs.squeeze()
            if self.alpha == 0:
                self.w, residual,rank,singular_values = torch.linalg.lstsq(rhs, lhs)
            else:
                ridge = self.alpha*torch.eye(lhs.shape[0],device=device)
                self.w, residual,rank,singular_values = torch.linalg.lstsq(rhs, lhs + ridge)
            if self.w.ndim == 2:
                assert self.w.shape[0] == 1
                self.w = self.w.squeeze(0)
                # import ipdb;ipdb.set_trace()
        def predict(self, X: torch.tensor) -> None:
            X = X.rename(None)
            if self.fit_intercept:
                X = torch.cat([torch.ones(X.shape[0], 1,device=X.device), X], dim = 1)
            return X @ self.w
    def solve_standard_regression(X,y,device,alpha):
        reg = Ridge(alpha)
        reg.fit(X,y)
        return reg.w[:-1],reg.w[-1]

if __name__ == "__main__":
    ## demo
    X = torch.randn(100,3)
    y = torch.randn(100,1) # supports only single outputs

    model = Ridge(alpha = 1e-3, fit_intercept = True)
    model.fit(X,y)
    model.predict(X)

def solve_shared_regression(X,Y,lambda_ = 0.01):
    # Y = torch.log(prob)
    # Define the L1 regularization coefficient lambda
    # lambda_ = 0.01
    # lambda_ = 0.01
    # Flatten the Y tensor to shape (50, 1)
    Y = Y.reshape(-1, 1)

    # Compute the closed-form solution for w with L1 regularization
    X_2_sum = torch.sum(X ** 2, dim=(0, 2))
    X_Y_sum = torch.sum(X * Y[:, None, :], dim=(0, 2))
    X_sum = torch.sum(X, dim=(0, 2))
    Y_sum = torch.sum(Y, dim=0)
    n_samples = X.shape[0]
    w = torch.zeros_like(X[0, :, 0])
    for i in range(w.shape[0]):
        denominator = X_2_sum[i]
        numerator = X_Y_sum[i] - ((1 / n_samples) * X_sum[i] * Y_sum)
        if numerator > lambda_:
            w[i] = (numerator - lambda_) / denominator
        elif numerator < -lambda_:
            w[i] = (numerator + lambda_) / denominator
        else:
            w[i] = 0

    # Compute the bias b
    b = torch.mean(Y - torch.einsum('ijk,j->i', X, w))
    if False:
        # Print the weight vector w and the bias b
        print(w)
        print(b)
    # import ipdb;ipdb.set_trace()
    return w,b


from django.contrib import admin
from django.urls import path
from .views import home,postcreate, detail, new_freecomment, new_comment,freehome,freedetail,freepostcreate
urlpatterns = [
    path('', home, name='home' ),
    path('detail/<int:post_id>', detail, name='detail' ),
    path('postcreate/', postcreate, name='postcreate' ),
    path('new_comment/', new_comment, name='new_comment' ),

    path('freehome/', freehome, name='freehome' ),
    path('freedetail/<int:post_id>', freedetail, name='freedetail' ),
    path('freepostcreate/', freepostcreate, name='freepostcreate' ),
    path('new_freecomment/', new_freecomment, name='new_freecomment' ),

]


from keras.models import load_model,model_from_json
import cv2 as cv2
import numpy as np
import operator
import sys, os
directory='model_5_99/'
json_file = open(directory+"model-bw_digit.json", "r")
model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(model_json)
loaded_model.load_weights(directory+"model-bw_digit.h5")
print("Loaded model from disk")
#categories = {1: 'ONE', 2: 'TWO', 3: 'THREE', 4: 'FOUR', 5: 'FIVE',6:'SIX',7:'SEVEN',8:'EIGHT',9:'NINE',A:'A',B:'B',C:'C',D:'D',E:'E',F:'F',G:'G',H:'H',I:'I',J:'J',K:'K',L:'L',M:'M',N:'N',O:'O',P:'P',Q:'Q',R:'R',S:'S',T:'T',U:'U',V:'V',W:'W',X:'X',Y:'Y',Z:'Z'}

cap = cv2.VideoCapture(0)

while True:
  ret,frame = cap.read()
  frame = cv2.flip(frame,1)
  roi = frame[100:400,320:620]
  cv2.imshow('roi',roi)
  gray = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY)
  blur = cv2.GaussianBlur(gray,(5,5),2)
  th3 = cv2.adaptiveThreshold(blur,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY_INV,11,2)
  ret, test_image = cv2.threshold(th3, 70, 255, cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)
  test_image = cv2.resize(test_image,(64,64))
  cv2.imshow('roi scaled and grey',test_image)
  copy = frame.copy()
  cv2.rectangle(copy,(320,100),(620,400),(255,0,0),5)
  roi=roi/255
  result = loaded_model.predict(test_image.reshape(1,64,64,1))
  prediction={'1':result[0][0],
              '2':result[0][1],
              '3':result[0][2],
              '4':result[0][3],
              '5':result[0][4],
              '6':result[0][5],
              '7':result[0][6],
              '8':result[0][7],
              '9':result[0][8],}
  prediction=sorted(prediction.items(),key=operator.itemgetter(1),reverse=True)
  current_predict = prediction[0][0]
  cv2.putText(copy,current_predict,(300,100),cv2.FONT_HERSHEY_COMPLEX,2,(0,255,0,2))
  cv2.imshow('frame',copy)

  if cv2.waitKey(1) == 13:
    break

cap.release()
cv2.destroyAllWindows()


import struct_estimator_dataratio_zerobrainlevel_weighted

import os.path as op
import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm, datasets
from sklearn.metrics import roc_curve, auc
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import label_binarize
from sklearn.multiclass import OneVsRestClassifier
from scipy import interp
import joblib

n_classes = 18
plt.close('all')

READ_DIR = 'srne_benchmark_dataratioFIXED100_zerobrainlevel_weighted'

REGS = ['tree-l2', 'l1',
        'elastic-net',
        'sparse-group-lasso-l2', 'group-lasso-l2', 'trace-norm']
REGS_STR = ['Hierarchical Tree Sparsity', 'Lasso',
            'Elastic-Net',
            'Sparse Group Lasso', 'Group Lasso', 'Trace-Norm']

auc_mat = np.zeros((len(REGS), n_classes))

# load HCP task data
print('Loading data...')
from sklearn.cross_validation import StratifiedShuffleSplit
if on_server:
    X_task, Y = joblib.load('/storage/workspace/danilo/prni2015/preload_HT_3mm')
else:
    X_task, Y = joblib.load('/git/prni2015/preload_HT_3mm')
Y = np.array(Y)
folder = StratifiedShuffleSplit(Y, n_iter=10, test_size=0.1,
                                random_state=42)
inds_train, inds_test = iter(folder).next()
X_train = X_task[inds_train]
# all estimators will have the same training set due to random_state
Y_train = Y[inds_train]
X_test = X_task[inds_test]
Y_test = Y[inds_test]

for i_r, r in enumerate(REGS):
    print('-' * 80)
    print(r)
    dump_path = op.join(READ_DIR, '%s_dataratio100_maxit100' % r)
    clf = joblib.load(dump_path)

    print(len(clf.best_estimator_.estimators_))
    print('Mean Accuracy: %.3f' % np.mean(clf.test_acc))
    print('Mean Precision: %.3f' % np.mean(clf.test_prfs[0]))
    print('Mean Recall: %.3f' % np.mean(clf.test_prfs[1]))

    coef_ = np.array([est.W_.ravel() for
                      est in clf.best_estimator_.estimators_]).T

    y_score = np.dot(X_test, coef_)

    # Compute ROC curve and ROC area for each class

    Y_test_bin = label_binarize(Y_test, classes=np.arange(n_classes))

    fpr = dict()
    tpr = dict()
    roc_auc = dict()
    for i in range(n_classes):
        fpr[i], tpr[i], _ = roc_curve(Y_test_bin[:, i], y_score[:, i])
        cur_auc = auc(fpr[i], tpr[i])
        roc_auc[i] = cur_auc
        auc_mat[i_r, i] = cur_auc

    # Compute micro-average ROC curve and ROC area
    fpr["micro"], tpr["micro"], _ = roc_curve(
        Y_test_bin.ravel(), y_score.ravel())
    roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])

    plt.figure()
    for i in range(n_classes):
        plt.plot(fpr[i], tpr[i], label='ROC curve (area = %0.2f)' % roc_auc[i])
    plt.plot([0, 1], [0, 1], 'k--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC: %s' % r)
    # plt.legend(loc="lower right")
    plt.show()
    plt.savefig(dump_path + '_ROC.png')

for i_r, r in enumerate(REGS):
    print('%s: mean %.3f std: %.3f' % (
        r, np.mean(auc_mat[i_r, :]), np.std(auc_mat[i_r, :])
    ))

# summary AUC plot: per classifier
plt.figure()
for i_c in range(n_classes):
    plt.plot(np.arange(len(REGS)), auc_mat[:, i_c] * 100, linewidth=2)
plt.ylim([40., 102.])
REGS_STR_auc = ['' + r + '\nmean AUC=%.1f' % np.mean(auc_mat[i_r, :] * 100) for i_r, r in enumerate(REGS_STR)]
plt.xticks(np.arange(len(REGS)), REGS_STR_auc, rotation=35)
plt.ylabel('class AUC')
# plt.title('Comparing classifications of 18 tasks')
plt.tight_layout()
plt.show()
plt.savefig(op.join(READ_DIR, 'ROC_ALL_perclf.png'))

# summary AUC plot: per task
my_width = 5
colors = ['r', 'y', 'b', 'g', 'k', 'c']
plt.figure()
for i_r, c in zip([0, 1, 2, 3, 4, 5], colors):
    plt.plot(np.arange(n_classes),
             auc_mat[i_r, :] * 100,
             label=REGS_STR[i_r] + ' (mean AUC=%.1f)' % np.mean(auc_mat[i_r, :] * 100),
             color=c,
             linewidth=my_width)
i_r = 0
plt.plot(np.arange(n_classes),
    auc_mat[i_r, :] * 100, color='r',
    linewidth=my_width)
plt.ylim([35., 102.])
plt.xticks(np.arange(n_classes), np.arange(n_classes) + 1)
plt.ylabel('AUC')
plt.title('Comparing classifications of 18 classes')
plt.xlabel('psychological task')
plt.tight_layout()
plt.legend(loc="middle right")
plt.show()
plt.savefig(op.join(READ_DIR, 'ROC_ALL_perclass.png'))


#!/usr/bin/env python

import sys
import xml.dom.minidom as minidom
import codecs

def usage():
    pass

def convert_pre(filename):
    global dom
    dom = minidom.parse(filename)
    root = dom.documentElement
    for pre in root.getElementsByTagName('pre'):
        convert(pre)
        pre.tagName='div'
        pre.setAttribute('id','codeText')
        pre.setAttribute('class','codeText')
    xml_str = dom.toxml()
    xml_str = xml_str.replace('nbsp',' ')
    f = codecs.open(filename,'w','utf-8');
    f.write(xml_str)
    f.close()

def convert(ele):
    if ele.nodeType == ele.ELEMENT_NODE:
        tmp = list(ele.childNodes)
        for child in tmp:
            convert(child)
    elif ele.nodeType == ele.TEXT_NODE:
        xml_str = ele.toxml().replace(' ','nbsp')
        xml_str = xml_str.replace('\n','<br/>\n')
        xml_str = "<data>" + xml_str + "</data>"
        xml_str = xml_str.encode('utf-8')
        new_ele = minidom.parseString(xml_str)
        nodes = list(new_ele.documentElement.childNodes)
        for node in nodes:
            ele.parentNode.insertBefore(node,ele)
        ele.parentNode.removeChild(ele)

if __name__ == "__main__":
    if len(sys.argv) < 2:
        usage()
        sys.exit(0)
    convert_pre(sys.argv[1])


from datetime import datetime
from .utils import MaxPQ, MemcacheWrapper, FirebaseRequest
DATETIME_FORMAT = '%m/%d/%Y %H:%M:%S'

class Bid():
    def __init__(self, saleIn=None):
        if saleIn is not None:
            self.sale = saleIn
            self.cacheKey = str(self.sale.pk) + "_bid"
            self.bidData = {
                'sale_id': self.sale.pk,
                'sale_buy_now': self.sale.price
            }
        self.bidderPQ = MaxPQ()

    def place_bid(self, user, bidPrice=1):
        """
        This method takes a user and bidPrice and checks if its the highest bid.
        Then adds it to the bid data and serializes and sends a notification to
        the required user.

        +++ New feature +++
        place_bid will now also push information to firebase in order to get
        real time information in the app.

        Arguments:
        user: a instance of the User model
        bidPirce: an int depicting the bidPrice

        :return: boolean
        """
        bidUsr = BidUser(user.pk, bidPrice)

        if "highest_bidder" not in self.bidData.keys():
            # This is the first bid for this sale
            self.bidData['highest_bidder'] = bidUsr.serialize()
            self.bidderPQ.enqueue(bidUsr)
            return True

        # Checking if the max in the bidderPQ is less than the argument price
        maxBidderPQPrice = self.bidderPQ.peek().bidPrice
        if int(bidPrice) > int(maxBidderPQPrice):
            # new price is the highest price
            self.bidData['highest_bidder'] = bidUsr.serialize()
        self.bidderPQ.enqueue(bidUsr)

        #payload for firebase request
        firebase_payload = {
            'id' : self.cacheKey,
            'highest_bidder': bidUsr.serialize(),
            'timestamp': datetime.now().strftime(DATETIME_FORMAT)
        }
        endpoint = "/bid/" + self.cacheKey
        #creating a firebase request
        firebase_request = FirebaseRequest(endpoint, payload=firebase_payload)
        firebase_request.post()
        return True

    def stats(self):
        """
        This method returns a dict with some statistics regarding the current bid.

        :return: dict
        """
        # Checking if bidData is available
        if self.bidData is None:
            return None

        # Checking if there is a highest bidder
        if "highest_bidder" not in list(self.bidData.keys()):
            return None

        statData = dict(self.bidData)
        statData['bidders_count'] = self.bidderPQ.size
        users = []
        prices = []
        n = self.bidderPQ.max
        while n is not None:
            users.append(n.element.pk)
            prices.append(n.element.bidPrice)
            n = n.next
        statData['users'] = users
        statData['prices'] = prices
        userId, userBidPrice = statData['highest_bidder'].split('-')
        statData['highest_bidder'] = {
            'user_id': userId,
            'bid_price': userBidPrice
        }
        return statData

    def serialize(self, mc):
        # serialize the priority queue
        pqList = []
        n = self.bidderPQ.max
        while n is not None:
            userStr = str(n.element.pk) + "-" + str(n.element.bidPrice)
            pqList.append(userStr)
            n = n.next
        self.bidData['bidders'] = pqList
        return mc.set_key_value(self.cacheKey, self.bidData)

    def deserialize(self, cacheKey, mc):
        self.bidData = mc.get_val(cacheKey)
        for user in self.bidData['bidders']:
            userId, bidPrice = user.split('-')
            self.bidderPQ.enqueue(BidUser(userId, bidPrice))
        self.cacheKey = cacheKey
        del self.bidData['bidders']
        return True

class BidUser():
    def __init__(self, pk, bidPrice):
        self.pk = pk
        self.bidPrice = int(bidPrice)

    def getPk(self):
        return self.pk

    def getBidPrice(self):
        return self.bidPrice

    def serialize(self):
        return str(self.pk) + "-" + str(self.bidPrice)

    def comparePriceTo(self, that):
        if self.bidPrice > that.bidPrice:
            return 1
        elif self.bidPrice < that.bidPrice:
            return -1
        else:
            return 0


import time
import os
from datetime import datetime
import sys

from Src_to_Dataframe import SrcToDataframe as sd
from Data_Extraction import DataExtraction as de

class Driver:

    def __init__(self, src_type, src_file_path, output_root_folder, primary_key):
        self.src_type = src_type
        self.src_file_path = src_file_path
        self.output_root_folder = output_root_folder
        self.source = 0
        self.output_path = 0
        self.start_insight()

    def start_insight(self):
        try:
            if self.src_type == 1:
                self.source = sd.load_sqlserver_data(self.src_file_path)
            elif self.src_type == 2:
                self.source = sd.load_mysql_data(self.src_file_path)
            elif self.src_type == 3:
                self.source = sd.load_oracle_data(self.src_file_path)
            elif self.src_type == 4:
                self.source = sd.load_msaccess_data(self.src_file_path)
            elif self.src_type == 5:
                self.source = sd.load_netezza_data(self.src_file_path)
            elif self.src_type == 6:
                print("Other sources are not supported now. Kindly contact Admin.")
                time.sleep(30)
                sys.exit()
            else:
                print("Choosen Invalid option, kindly choose correct option.")
                time.sleep(30)
                sys.exit()
        except Exception as err:
            print(str(err))
            sys.exit()
        report = "Report_" + datetime.now().strftime("%Y%m%d_%H%M%S%f")
        self.output_path = self.output_root_folder + report
        try:
            os.mkdir(self.output_path)
        except Exception as err:
            print(err)
        try:
            de.extract_data(self.source, self.output_path)
        except Exception as err:
            print(err)


import tensorflow as tf
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras import initializers
import matplotlib.pyplot as plt

#function for choosing point of rounding up or down numbers
def rounder(x):
  index = 0
  while index < x.shape[0]:
    value =  x[index]
    remainer = value - int(value)
    if remainer > .62:
      remainder = 1
    else:
      remainder = 0
    x[index] = int(value) + remainder
    index+=1
  return x

raw_data = pd.read_csv("../../3rd_year_project/2019-ca326-eharkin-soccer-score-predictor-and-fantasy-league-web-application-using-machine-learning/code/results_and_players/training_results.csv")
data = raw_data.copy()
train_dataset = data.sample(frac=0.90,random_state=0)
test_dataset = data.drop(train_dataset.index)
train_labels_home = train_dataset.pop("home_result_score")
train_labels_away = train_dataset.pop("away_result_score")
test_labels_home = test_dataset.pop("home_result_score")
test_labels_away = test_dataset.pop("away_result_score")

train_stats = train_dataset.describe()
train_stats = train_stats.transpose()

#function for normalizing all the various inputs
def norm(x):
  return (x - train_stats['mean']) / train_stats['std']

normed_train_data = norm(train_dataset)
normed_test_data = norm(test_dataset)

def build_model():
  model = keras.Sequential([
    layers.Dense(4, activation=tf.nn.leaky_relu,input_shape=[len(train_dataset.keys())]),
    layers.Dense(1)
  ])

  optimizer = tf.keras.optimizers.Adam(0.001)

  model.compile(loss='mse',
                optimizer=optimizer,
                metrics=['mae', 'mse'])
  return model

class PrintDot(keras.callbacks.Callback):
  #function for visualizing how long training is taking
  def on_epoch_end(self, epoch, logs):
    if epoch % 100 == 0: print('')
    print('.', end='')

home_model = build_model()
away_model = build_model()
home_model.summary()

EPOCHS = 2000

# The patience argument is the amount of epochs to check for improvement before exiting training
early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', patience=100)

history_home = home_model.fit(
  normed_train_data, train_labels_home,
  epochs=EPOCHS, validation_split = 0.2, shuffle=True,verbose=0,
  callbacks=[early_stop,PrintDot()])
history_away = away_model.fit(
  normed_train_data, train_labels_away,
  epochs=EPOCHS, validation_split = 0.2, shuffle=True,verbose=0,
  callbacks=[early_stop,PrintDot()])

home_model.save('home_score_NN.model')
away_model.save('away_score_NN.model')

#history shows the improvement over time
def plot_history(history):
  hist = pd.DataFrame(history.history)
  hist['epoch'] = history.epoch

  #drawing the graph of improvement over time
  plt.figure()
  plt.xlabel('Epoch')
  plt.ylabel('Mean Abs Error [home_result_score]')
  plt.plot(hist['epoch'], hist['mean_absolute_error'],
           label='Train Error')
  plt.plot(hist['epoch'], hist['val_mean_absolute_error'],
           label = 'Val Error')
  plt.legend()
  plt.ylim([0,5])

plot_history(history_home)
plt.show()
plot_history(history_away)
plt.show()

loss, mae, mse = home_model.evaluate(normed_test_data, test_labels_home, verbose=0)

print("Testing set Mean Abs Error: {:5.2f} score".format(mae))

home_test_predictions = home_model.predict(normed_test_data).flatten()
away_test_predictions = away_model.predict(normed_test_data).flatten()

#how far off home score was from predicted
error = home_test_predictions - test_labels_home
#how far off rounded for home
error1 = abs(np.around(home_test_predictions) - test_labels_home)
#how many goals total it was off
total_error =(abs(rounder(home_test_predictions) - test_labels_home)) +  (abs(rounder(away_test_predictions) - test_labels_away))
#if was correct outcome or not, home prediction multiplied by 2 to ensure a predicted loss and an actual win did not result in return a correctly predicted draw
result_error =(2 * np.sign((rounder(home_test_predictions) - rounder(away_test_predictions)))) + (np.sign(test_labels_home - test_labels_away))
plt.hist(total_error, bins = 25)
plt.hist(result_error, bins = 25)
plt.xlabel("Prediction Error [home_result_score]")
_ = plt.ylabel("Count")
plt.show()
error2= error1
error1 = error1.describe()
error1 = error1.transpose()

print(error1["mean"])
plt.hist(error2, bins = 25)
plt.xlabel("Prediction Error [home_result_score]")
_ = plt.ylabel("Count")

#generate predictions for upcoming games
game = pd.read_csv("../results_and_players/test_games.csv")
normed_game = norm(game)
home_test_predictions = home_model.predict(normed_game).flatten()
away_test_predictions = away_model.predict(normed_game).flatten()

file_name = "../../3rd_year_project/2019-ca326-eharkin-soccer-score-predictor-and-fantasy-league-web-application-using-machine-learning/code/fixtures/fixtures.csv"
file = open(file_name,"r")
file = file.readlines()
file = file[1:]
i = 0
predicted = ""
while i < len(home_test_predictions):
  predicted += file[i].strip() + "," + str(round(home_test_predictions[i])) + "," +  str(round(away_test_predictions[i])) + "\n"
  i+=1
file_write = open("../../3rd_year_project/2019-ca326-eharkin-soccer-score-predictor-and-fantasy-league-web-application-using-machine-learning/code/fixtures/predictions.txt","w")


import yaml
from argparse import Namespace

def parse_args(parser):
    """Parse the arguments.
    Args:
        parser: An instance of argparse.ArgumentParser.
    Returns:
        args: An instance of argparse.Namespace.
    """
    args = parser.parse_args()
    if not args.conf_path:
        raise ValueError(f"--conf_path can't be None.")
    args_dict = yaml.load(open(args.conf_path).read())
    args = vars(args)
    for k, v in args.items():
        if v:
            args_dict[k] = v
    args = dict2namespace(args_dict)
    return args

def dict2namespace(dic):
    """Convert a `dict` to an `argparse.Namespace`.
    Args:
        dic: A `dict`.
    Returns:
        args: An instance of argparse.Namespace.
    """
    for k, v in dic.items():
        if isinstance(v, dict):
            v = dict2namespace(v)
            dic[k] = v
    return Namespace(**dic)


from selenium import webdriver
from common.base import Base
import time
login_url='http://192.168.0.181:9001/cas/login?service=http://192.168.0.181:9003/haems-web/cas'
# login_url='http://192.168.0.181:9001/cas/login?service=http://192.168.0.181:9000/base-web/cas'

class LoginPage(Base):#继承

    #定位登录
    loc1_user = ('xpath', '//*[@id="username"]')
    loc2_psw= ('xpath', '//*[@id="password"]')
    loc3_button = ('xpath', '//*[@id="fm1"]/div[4]/div/input')

    # loc4_keep=('xpath','//*[@id="keeplogin"]/label')
    # loc5_forget_psw=('xpath','//*[@id="login-form"]/form/table/tbody/tr[4]/td/a')
    '''判断user是否存在'''
    loc6_get_user=('xpath','//*[@id="userInfo"]/a')
    '''点击不良事件图标'''
    loc7_blsj=('xpath','//ul[@id="systemIconList"]/li[5]/a/p')
    '''获取登录用户'''
    loc8_admin=('xpath','//p[@role="button"]')
    '''事件上报管理元素'''
    loc9_addinfo=('xpath',"//span[contains(text(),'事件上报管理')]")
    '''事件上报元素'''
    loc10_addinfo1=('xpath','//*[@id="app"]/div/section/section/aside/div[1]/ul/div/li[2]/ul/div/li[1]')

    '''新增上报元素'''
    loc11_addinfo2=('xpath','.//main[@class="el-main"]//span[contains(text(),"新增上报")]')
    # loc11_addinfo2=('//*[@id="pane-list"]/form/div[6]/div/button/span')
    '''选择事件反反复复'''
    loc12_ffff=('xpath',"//div[@class='cell']//span[contains(text(),'反反复复')]")
    '''点击其它'''
    loc13_qt=('xpath',"//span[contains(text(),'其它')]")
    '''选择日期'''
    loc14_evtime=('xpath','//input[@name="evTime"]')
    '''点击此刻'''
    loc15_chike=('xpath',"//span[contains(text(),'此刻')]")
    '''事件伤害级别'''
    loc16_shcd=('xpath','//input[@name="aeLevel"]')
    '''选择事件伤害级别O级'''
    loc17_addshcd=('xpath',"//span[contains(text(),'〇级')]")
    '''选择单选1'''
    loc18_xyx1=('xpath',"//label[@id='option_radio']//span[contains(text(),'选项1')]")
    '''点击下一项'''
    loc19_xyx2=('xpath','//button[@type="button"]//span[contains(text(),"下一项 (表单未完成)")]')
    # loc20_xyx3=('xpath','')
    # loc21_xyx4 = ('xpath', '')
    # loc21_xyx5 = ('xpath', '')
    '''选择正常上报'''
    loc20_zcsb=('xpath',"//span[contains(text(),'正常上报')]")
    '''点击提交按钮'''
    loc21_tj=('xpath','//button[@class="el-button el-button--primary el-button--mini"]//span[contains(text(),"提交")]')
    '''选择审核人'''
    loc22_djsh=('xpath','//*[@id="pane-add"]/div[1]/div/div[7]/div/div[2]/form/div[3]/div/div/div/input')
    '''选择护士'''
    loc23_djshryxz=('xpath','//div[@class="el-cascader-panel"]//span[contains(text(),"护士")]')
    # loc23_djshryxz=('xpath','//*[@id="cascader-menu-8395-1-0"]/span')
    '''选择管理员'''
    loc24_gly=('xpath','//li[@role="menuitem"]//span[contains(text(),"管*员")]')
    '''点击确定'''
    # frame1=("xpath",'//iframe[@id="xh-bar"]')
    # loc25_qd=('xpath','//div[@class="el-picker-panel__footer"]//span[contains(text(),"确定")]')
    loc25_qd =('xpath','//*[@id="pane-add"]/div[1]/div/div[7]/div/div[3]/div/button[2]/span')
    '''清除状态'''
    loc26_qczt=('xpath','//*[@id="pane-list"]/form/div[3]/div/div/div[2]/span/span/i')
    '''点击查询'''
    loc27_djcx=('xpath','//div[@id="pane-list"]//span[contains(text(),"查询")]')

    # loc7_forget_psw_page=('xpath',"/html/body/div/div/div[2]/p/a")
    '''获取事件名称文本'''
    loc28=('xpath','//*[@id="pane-list"]/div[1]/div[3]/table/tbody/tr[1]/td[6]/div')
    def input_user(self,text=""):
        self.sendKeys(self.loc1_user,text)
    def input_pwd(self,text=""):
        self.sendKeys(self.loc2_psw,text)
    def click_login(self):
        self.click(self.loc3_button)

    # def keep_login(self):
    #     self.click(self.loc4_keep)
    # def click_forget_psw(self):
    #     self.click(self.loc5_forget_psw)
    # #def login(self,user='admin',psw='123456'):
    #     self.driver.get('http://192.168.0.159/zentao/user-login-L3plbnRhby8=.html')
    #     self.sendKeys(self.loc1,user)
    #     self.sendKeys(self.loc2,psw)
    #     self.click(self.loc3)
    # a=('xpth','//*[@id="mainmenu"]/ul/li[1]/a/span')
    # def is_add_bug_sucess(self,_text):
    #     return self.is_text_in_element(self.a, _text)
    def get_login_name(self):
        user=self.get_text(self.loc6_get_user)
        return  user
    def get_login_result(self,user):
        result=self.is_text_in_element(self.loc6_get_user,user)
        return result
    def is_add_bug_sucess(self,_text):
        return self.is_text_in_element(self.loc28,_text)

    # def is_alert_exist(self):
    #     '''判断alert是不是存在'''
    #     a = self.is_alert()
    #     if a:
    #         print(a.text)
    #         a.accept()

    # def is_refresh_exist(self):
    #     # '''判断忘记密码页没刷新按钮是否存在'''
    #     m= self.isElementExist(self.loc7_forget_psw_page)
    #     return m

    # def is_add_bug_sucess(self, _text):
    #     return self.is_text_in_element(self.loc6_get_user, _text)

    def login(self,user='thinkgem',psw='password'):#keep开关
        '''二次封装登录流程'''
        self.driver.get(login_url)
        self.input_user(user)
        self.input_pwd(psw)
        # if keep_login1:self.keep_login()
        self.click_login()
        # frame = self.findElement(("name","mainFrame"))
        # self.driver.switch_to.frame(frame)
        # self.click(self.loc7_blsj)
        # self.driver.switch_to.default_content()
    def add_aeinfo(self):
        self.click(self.loc9_addinfo)
        time.sleep(3)
        self.click(self.loc10_addinfo1)
        time.sleep(4)
        self.click(self.loc11_addinfo2)
        time.sleep(3)
        self.click(self.loc12_ffff)
        self.click(self.loc13_qt)
        self.click(self.loc14_evtime)
        time.sleep(3)
        self.click(self.loc15_chike)
        self.click(self.loc16_shcd)
        time.sleep(3)
        self.click(self.loc17_addshcd)
        time.sleep(3)
        self.click(self.loc18_xyx1)
        time.sleep(3)
        self.click(self.loc19_xyx2)
        self.click(self.loc20_zcsb)
        self.click(self.loc21_tj)
        self.click(self.loc22_djsh)
        time.sleep(3)
        self.click(self.loc23_djshryxz)
        self.click(self.loc24_gly)
        # # id_1=()
        # frame = self.findElement(("class name", "ke-edit-iframe"))
        # # self.driver.switch_to.frame(frame)
        # frame1 = ("xpath", '//iframe[@id="xh-bar"]')
        # self.driver.switch_to.frame("xh-bar")
        # self.driver.switch_to("xh-bar")
        self.click(self.loc25_qd)
        # self.driver.switch_to.default_content()
        time.sleep(5)
        self.click(self.loc26_qczt)
        self.click(self.loc27_djcx)

if __name__=='__main__':
    driver=webdriver.Chrome()
    login_page=LoginPage(driver)
    # driver.get(login_url)
    # login_page.input_user('admin')
    # login_page.input_pwd('123456')
    # login_page.keep_login()
    # login_page.click_login()
    # login_page.login(keep_login1=True)#keep开关，ture为保持登录，False为不点击保持登录
    login_page.login()
    #js处理不良事件时间
    #有readOnly用readonly=false，没有直接改
    # time.sleep(11)
    # js = "a=document.getElementsByName('')[0];" \
    #        "a.value='2020-05';"
    # driver.execute_script(js)
    #