import inspect
import sys
import textwrap
import typing
from typing import get_type_hints, TypeVar, Any, AnyStr, Generic, Union

from sphinx.util import logging
from sphinx.util.inspect import signature as Signature, stringify_signature

import type_globals

try:
    from typing_extensions import Protocol
except ImportError:
    Protocol = None

logger = logging.getLogger(__name__)
pydata_annotations = {'Any', 'AnyStr', 'Callable', 'ClassVar', 'NoReturn', 'Optional', 'Tuple',
                      'Union'}

def format_annotation(annotation, fully_qualified=False):
    if inspect.isclass(annotation) and annotation.__module__ == 'builtins':
        if annotation.__qualname__ == 'NoneType':
            return '``None``'
        else:
            return ':py:class:`{}`'.format(annotation.__qualname__)

    annotation_cls = annotation if inspect.isclass(annotation) else type(annotation)
    if annotation_cls.__module__ == 'typing':
        class_name = str(annotation).split('[')[0].split('.')[-1]
        params = None
        module = 'typing'
        extra = ''

        origin = getattr(annotation, '__origin__', None)
        if inspect.isclass(origin):
            annotation_cls = annotation.__origin__
            try:
                mro = annotation_cls.mro()
                if Generic in mro or (Protocol and Protocol in mro):
                    module = annotation_cls.__module__
            except TypeError:
                pass  # annotation_cls was either the "type" object or typing.Type

        if annotation is Any:
            return ':py:data:`{}typing.Any`'.format("" if fully_qualified else "~")
        elif annotation is AnyStr:
            return ':py:data:`{}typing.AnyStr`'.format("" if fully_qualified else "~")
        elif isinstance(annotation, TypeVar):
            bound = annotation.__bound__
            if bound:
                if 'ForwardRef(' in str(bound):
                    try:
                        bound = bound._evaluate(sys.modules[annotation.__module__].__dict__, None)
                    except:
                        try:
                            bound = bound._evaluate(type_globals.__dict__, None)
                        except:
                            bound = bound.__forward_arg__
                return format_annotation(bound, fully_qualified)
            return '\\%r' % annotation
        elif (annotation is Union or getattr(annotation, '__origin__', None) is Union or
              hasattr(annotation, '__union_params__')):
            if hasattr(annotation, '__union_params__'):
                params = annotation.__union_params__
            elif hasattr(annotation, '__args__'):
                params = annotation.__args__

            if params and len(params) == 2 and (hasattr(params[1], '__qualname__') and
                                                params[1].__qualname__ == 'NoneType'):
                class_name = 'Optional'
                params = (params[0],)
        elif annotation_cls.__qualname__ == 'Tuple' and hasattr(annotation, '__tuple_params__'):
            params = annotation.__tuple_params__
            if annotation.__tuple_use_ellipsis__:
                params += (Ellipsis,)
        elif annotation_cls.__qualname__ == 'Callable':
            arg_annotations = result_annotation = None
            if hasattr(annotation, '__result__'):
                arg_annotations = annotation.__args__
                result_annotation = annotation.__result__
            elif getattr(annotation, '__args__', None):
                arg_annotations = annotation.__args__[:-1]
                result_annotation = annotation.__args__[-1]

            if arg_annotations in (Ellipsis, (Ellipsis,)):
                params = [Ellipsis, result_annotation]
            elif arg_annotations is not None:
                params = [
                    '\\[{}]'.format(
                        ', '.join(
                            format_annotation(param, fully_qualified)
                            for param in arg_annotations)),
                    result_annotation
                ]
        elif str(annotation).startswith('typing.ClassVar[') and hasattr(annotation, '__type__'):
            # < py3.7
            params = (annotation.__type__,)
        elif hasattr(annotation, 'type_var'):
            # Type alias
            class_name = annotation.name
            params = (annotation.type_var,)
        elif getattr(annotation, '__args__', None) is not None:
            params = annotation.__args__
        elif hasattr(annotation, '__parameters__'):
            params = annotation.__parameters__

        if params:
            extra = '\\[{}]'.format(', '.join(
                format_annotation(param, fully_qualified) for param in params))

        return '{prefix}`{qualify}{module}.{name}`{extra}'.format(
            prefix=':py:data:' if class_name in pydata_annotations else ':py:class:',
            qualify="" if fully_qualified else "~",
            module=module,
            name=class_name,
            extra=extra
        )
    elif annotation is Ellipsis:
        return '...'
    elif (inspect.isfunction(annotation) and annotation.__module__ == 'typing' and
          hasattr(annotation, '__name__') and hasattr(annotation, '__supertype__')):
        return ':py:func:`{qualify}typing.NewType`\\(:py:data:`~{name}`, {extra})'.format(
            qualify="" if fully_qualified else "~",
            name=annotation.__name__,
            extra=format_annotation(annotation.__supertype__, fully_qualified),
        )
    elif inspect.isclass(annotation) or inspect.isclass(getattr(annotation, '__origin__', None)):
        if not inspect.isclass(annotation):
            annotation_cls = annotation.__origin__

        extra = ''
        try:
            mro = annotation_cls.mro()
        except TypeError:
            pass
        else:
            if Generic in mro or (Protocol and Protocol in mro):
                params = (getattr(annotation, '__parameters__', None) or
                          getattr(annotation, '__args__', None))
                if params:
                    extra = '\\[{}]'.format(', '.join(
                        format_annotation(param, fully_qualified) for param in params))

        return ':py:class:`{qualify}{module}.{name}`{extra}'.format(
            qualify="" if fully_qualified else "~",
            module=annotation.__module__,
            name=annotation_cls.__qualname__,
            extra=extra
        )

    return str(annotation)

def process_signature(app, what: str, name: str, obj, options, signature, return_annotation):
    if not callable(obj):
        return

    if what in ('class', 'exception'):
        obj = getattr(obj, '__init__', getattr(obj, '__new__', None))

    if not getattr(obj, '__annotations__', None):
        return

    obj = inspect.unwrap(obj)
    signature = Signature(obj)
    parameters = [
        param.replace(annotation=inspect.Parameter.empty)
        for param in signature.parameters.values()
    ]

    if '<locals>' in obj.__qualname__:
        logger.warning(
            'Cannot treat a function defined as a local function: "%s"  (use @functools.wraps)',
            name)
        return

    if parameters:
        if what in ('class', 'exception'):
            del parameters[0]
        elif what == 'method':
            outer = inspect.getmodule(obj)
            for clsname in obj.__qualname__.split('.')[:-1]:
                outer = getattr(outer, clsname)

            method_name = obj.__name__
            if method_name.startswith("__") and not method_name.endswith("__"):
                # If the method starts with double underscore (dunder)
                # Python applies mangling so we need to prepend the class name.
                # This doesn't happen if it always ends with double underscore.
                class_name = obj.__qualname__.split('.')[-2]
                method_name = "_{c}{m}".format(c=class_name, m=method_name)

            method_object = outer.__dict__[method_name] if outer else obj
            if not isinstance(method_object, (classmethod, staticmethod)):
                del parameters[0]

    signature = signature.replace(
        parameters=parameters,
        return_annotation=inspect.Signature.empty)

    return stringify_signature(signature).replace('\\', '\\\\'), None

def get_all_type_hints(obj, name):
    rv = {}

    try:
        rv = get_type_hints(obj)
    except (AttributeError, TypeError, RecursionError):
        # Introspecting a slot wrapper will raise TypeError, and and some recursive type
        # definitions will cause a RecursionError (https://github.com/python/typing/issues/574).
        pass
    except NameError as exc:
        try:
            rv = get_type_hints(obj, localns=type_globals.__dict__)
        except Exception as exc:
            logger.warning('Cannot resolve forward reference in type annotations of "%s": %s',
                           name, exc)
            rv = obj.__annotations__

    if rv:
        return rv

    rv = backfill_type_hints(obj, name)

    try:
        obj.__annotations__ = rv
    except (AttributeError, TypeError):
        return rv

    try:
        rv = get_type_hints(obj)
    except (AttributeError, TypeError):
        pass
    except NameError as exc:
        try:
            rv = get_type_hints(obj, localns=type_globals.__dict__)
        except:
            logger.warning('Cannot resolve forward reference in type annotations of "%s": %s',
                           name, exc)
            rv = obj.__annotations__

    return rv

def backfill_type_hints(obj, name):
    parse_kwargs = {}
    if sys.version_info < (3, 8):
        try:
            import typed_ast.ast3 as ast
        except ImportError:
            return {}
    else:
        import ast
        parse_kwargs = {'type_comments': True}

    def _one_child(module):
        children = module.body  # use the body to ignore type comments

        if len(children) != 1:
            logger.warning(
                'Did not get exactly one node from AST for "%s", got %s', name, len(children))
            return

        return children[0]

    try:
        obj_ast = ast.parse(textwrap.dedent(inspect.getsource(obj)), **parse_kwargs)
    except TypeError:
        return {}

    obj_ast = _one_child(obj_ast)
    if obj_ast is None:
        return {}

    try:
        type_comment = obj_ast.type_comment
    except AttributeError:
        return {}

    if not type_comment:
        return {}

    try:
        comment_args_str, comment_returns = type_comment.split(' -> ')
    except ValueError:
        logger.warning('Unparsable type hint comment for "%s": Expected to contain ` -> `', name)
        return {}

    rv = {}
    if comment_returns:
        rv['return'] = comment_returns

    args = load_args(obj_ast)
    comment_args = split_type_comment_args(comment_args_str)
    is_inline = len(comment_args) == 1 and comment_args[0] == "..."
    if not is_inline:
        if args and args[0].arg in ("self", "cls") and len(comment_args) != len(args):
            comment_args.insert(0, None)  # self/cls may be omitted in type comments, insert blank

        if len(args) != len(comment_args):
            logger.warning('Not enough type comments found on "%s"', name)
            return rv

    for at, arg in enumerate(args):
        arg_key = getattr(arg, "arg", None)
        if arg_key is None:
            continue

        if is_inline:  # the type information now is tied to the argument
            value = getattr(arg, "type_comment", None)
        else:  # type data from comment
            value = comment_args[at]

        if value is not None:
            rv[arg_key] = value

    return rv

def load_args(obj_ast):
    func_args = obj_ast.args
    args = []
    pos_only = getattr(func_args, 'posonlyargs', None)
    if pos_only:
        args.extend(pos_only)

    args.extend(func_args.args)
    if func_args.vararg:
        args.append(func_args.vararg)

    args.extend(func_args.kwonlyargs)
    if func_args.kwarg:
        args.append(func_args.kwarg)

    return args

def split_type_comment_args(comment):
    def add(val):
        result.append(val.strip().lstrip("*"))  # remove spaces, and var/kw arg marker

    comment = comment.strip().lstrip("(").rstrip(")")
    result = []
    if not comment:
        return result

    brackets, start_arg_at, at = 0, 0, 0
    for at, char in enumerate(comment):
        if char in ("[", "("):
            brackets += 1
        elif char in ("]", ")"):
            brackets -= 1
        elif char == "," and brackets == 0:
            add(comment[start_arg_at:at])
            start_arg_at = at + 1

    add(comment[start_arg_at: at + 1])
    return result

def process_docstring(app, what, name, obj, options, lines):
    if isinstance(obj, property):
        obj = obj.fget

    if callable(obj):
        if what in ('class', 'exception'):
            obj = getattr(obj, '__init__')

        obj = inspect.unwrap(obj)
        type_hints = get_all_type_hints(obj, name)

        for argname, annotation in type_hints.items():
            if argname == 'return':
                continue  # this is handled separately later
            if argname.endswith('_'):
                argname = '{}\\_'.format(argname[:-1])

            formatted_annotation = format_annotation(
                annotation, fully_qualified=app.config.typehints_fully_qualified)

            searchfor = ':param {}:'.format(argname)
            insert_index = None

            for i, line in enumerate(lines):
                if line.startswith(searchfor):
                    insert_index = i
                    break

            if insert_index is None and app.config.always_document_param_types:
                lines.append(searchfor)
                insert_index = len(lines)

            if insert_index is not None:
                lines.insert(
                    insert_index,
                    ':type {}: {}'.format(argname, formatted_annotation)
                )

        if 'return' in type_hints and what not in ('class', 'exception'):
            formatted_annotation = format_annotation(
                type_hints['return'], fully_qualified=app.config.typehints_fully_qualified)

            insert_index = len(lines)
            for i, line in enumerate(lines):
                if line.startswith(':rtype:'):
                    insert_index = None
                    break
                elif line.startswith(':return:') or line.startswith(':returns:'):
                    insert_index = i

            if insert_index is not None:
                if insert_index == len(lines):
                    # Ensure that :rtype: doesn't get joined with a paragraph of text, which
                    # prevents it being interpreted.
                    lines.append('')
                    insert_index += 1

                lines.insert(insert_index, ':rtype: {}'.format(formatted_annotation))

def builder_ready(app):
    if app.config.set_type_checking_flag:
        typing.TYPE_CHECKING = True

def setup(app):
    app.add_config_value('set_type_checking_flag', False, 'html')
    app.add_config_value('always_document_param_types', False, 'html')
    app.add_config_value('typehints_fully_qualified', False, 'env')
    app.connect('builder-inited', builder_ready)
    app.connect('autodoc-process-signature', process_signature)
    app.connect('autodoc-process-docstring', process_docstring)
    return dict(parallel_read_safe=True)


from builtins import object
import logging
import os
import sys

from collections import deque

from django.urls import reverse
from mako.lookup import TemplateLookup

from desktop.models import Document2
from notebook.connectors.base import get_api
from notebook.models import Notebook, make_notebook
from useradmin.models import User

from indexer.conf import CONFIG_INDEXING_TEMPLATES_PATH, CONFIG_INDEXER_LIBS_PATH
from indexer.fields import get_field_type
from indexer.file_format import get_file_format_instance, get_file_format_class
from indexer.indexers.morphline_operations import get_checked_args
from indexer.solr_client import SolrClient

if sys.version_info[0] > 2:
  from django.utils.translation import gettext as _
else:
  from django.utils.translation import ugettext as _

LOG = logging.getLogger()

class MorphlineIndexer(object):

  def __init__(self, user, fs=None, jt=None, solr_client=None):
    self.fs = fs
    self.jt = jt
    self.user = user
    self.username = user.username
    self.solr_client = solr_client if solr_client is not None else SolrClient(self.user)

  def _upload_workspace(self, morphline):
    from oozie.models2 import Job

    hdfs_workspace_path = Job.get_workspace(self.username)
    hdfs_morphline_path = os.path.join(hdfs_workspace_path, "morphline.conf")
    hdfs_log4j_properties_path = os.path.join(hdfs_workspace_path, "log4j.properties")

    log4j_template_path = os.path.join(CONFIG_INDEXING_TEMPLATES_PATH.get(), "log4j.properties")

    # Create workspace on hdfs
    self.fs.do_as_user(self.username, self.fs.mkdir, hdfs_workspace_path)

    self.fs.do_as_user(self.username, self.fs.create, hdfs_morphline_path, data=morphline)
    self.fs.do_as_user(self.username, self.fs.create, hdfs_log4j_properties_path, data=open(log4j_template_path).read())

    return hdfs_workspace_path

  def run_morphline(self, request, collection_name, morphline, input_path, query=None, start_time=None, lib_path=None):
    workspace_path = self._upload_workspace(morphline)

    task = make_notebook(
      name=_('Indexing into %s') % collection_name,
      editor_type='notebook',
      on_success_url=reverse('search:browse', kwargs={'name': collection_name}),
      pub_sub_url='assist.collections.refresh',
      is_task=True,
      is_notebook=True,
      last_executed=start_time
    )

    if query:
      q = Notebook(document=Document2.objects.get_by_uuid(user=self.user, uuid=query))
      notebook_data = q.get_data()
      snippet = notebook_data['snippets'][0]

      api = get_api(request, snippet)

      destination = '__hue_%s' % notebook_data['uuid'][:4]
      location = '/user/%s/__hue-%s' % (request.user, notebook_data['uuid'][:4])
      sql, _success_url = api.export_data_as_table(notebook_data, snippet, destination, is_temporary=True, location=location)
      input_path = '${nameNode}%s' % location

      task.add_hive_snippet(snippet['database'], sql)

    client = SolrClient(self.user)

    extra_args = ['-Dmapreduce.job.user.classpath.first=true'] if client.is_solr_six_or_more() else []

    task.add_java_snippet(
      clazz='org.apache.solr.hadoop.MapReduceIndexerTool',
      app_jar=lib_path if lib_path is not None else CONFIG_INDEXER_LIBS_PATH.get(),
      arguments=extra_args + [
          u'--morphline-file',
          u'morphline.conf',
          u'--output-dir',
          u'${nameNode}/user/%s/indexer' % self.username,
          u'--log4j',
          u'log4j.properties',
          u'--go-live',
          u'--zk-host',
          client.get_zookeeper_host(),
          u'--collection',
          collection_name,
          input_path,
      ],
      files=[
          {u'path': u'%s/log4j.properties' % workspace_path, u'type': u'file'},
          {u'path': u'%s/morphline.conf' % workspace_path, u'type': u'file'}
      ]
    )

    return task.execute(request, batch=True)

  def guess_format(self, data):
    file_format = get_file_format_instance(data['file'])
    return file_format.get_format()

  def guess_field_types(self, data):
    file_format = get_file_format_instance(data['file'], data['format'])
    return file_format.get_fields() if file_format else {'columns': []}

  # Breadth first ordering of fields
  @classmethod
  def get_field_list(self, field_data, is_converting_types=False):
    fields = []

    queue = deque(field_data)

    while len(queue):
      curr_field = queue.popleft()
      curr_field['type'] = curr_field['type']
      if is_converting_types:
        SolrClient._port_field_types(curr_field)
      fields.append(curr_field)

      for operation in curr_field["operations"]:
        for field in operation["fields"]:
          queue.append(field)

    return fields

  @classmethod
  def get_kept_field_list(cls, field_data):
    return [field for field in cls.get_field_list(field_data) if field['keep']]

  def get_unique_field(self, format_):
    unique_fields = [column['name'] for column in format_['columns'] if column['unique']]

    if unique_fields:
      return unique_fields[0]
    else:
      base_name = "_uuid"
      field_names = set([column['name'] for column in format_['columns']])

      while base_name in field_names:
        base_name = '_' + base_name

      return base_name

  def is_unique_generated(self, format_):
    unique_fields = [column['name'] for column in format_['columns'] if column['unique']]

    return len(unique_fields) == 0

  @staticmethod
  def _get_regex_for_type(type_name):
    field_type = get_field_type(type_name)

    return field_type.regex.replace('\\', '\\\\')

  def generate_morphline_config(self, collection_name, data, uuid_name=None, lib_path=None):
    if lib_path is None:
      lib_path = CONFIG_INDEXER_LIBS_PATH.get()

    geolite_loc = os.path.join(lib_path, "GeoLite2-City.mmdb")
    grok_dicts_loc = os.path.join(lib_path, "grok_dictionaries")

    properties = {
      "collection_name": collection_name,
      "fields": self.get_field_list(data['columns'], is_converting_types=True),
      "num_base_fields": len(data['columns']),
      "uuid_name": uuid_name,
      "get_regex": MorphlineIndexer._get_regex_for_type,
      "format_settings": data['format'],
      "format_class": get_file_format_class(data['format']['type']),
      "get_kept_args": get_checked_args,
      "grok_dictionaries_location": grok_dicts_loc if self.fs and self.fs.exists(grok_dicts_loc) else None,
      "geolite_db_location": geolite_loc if self.fs and self.fs.exists(geolite_loc) else None,
      "zk_host": self.solr_client.get_zookeeper_host() ## offline test?
    }

    oozie_workspace = CONFIG_INDEXING_TEMPLATES_PATH.get()

    lookup = TemplateLookup(directories=[oozie_workspace])
    morphline = lookup.get_template("morphline_template.conf").render(**properties)

    return morphline

def _create_solr_collection(user, fs, client, destination, index_name, kwargs):
  unique_key_field = destination['indexerPrimaryKey'] and destination['indexerPrimaryKey'][0] or None
  df = destination['indexerDefaultField'] and destination['indexerDefaultField'][0] or None

  indexer = MorphlineIndexer(user, fs)
  fields = indexer.get_field_list(destination['columns'])
  skip_fields = [field['name'] for field in fields if not field['keep']]

  kwargs['fieldnames'] = ','.join([field['name'] for field in fields])
  for field in fields:
    for operation in field['operations']:
      if operation['type'] == 'split':
        field['multiValued'] = True # Solr requires multiValued to be set when splitting
        kwargs['f.%(name)s.split' % field] = 'true'
        kwargs['f.%(name)s.separator' % field] = operation['settings']['splitChar'] or ','

  if skip_fields:
    kwargs['skip'] = ','.join(skip_fields)
    fields = [field for field in fields if field['name'] not in skip_fields]

  if not unique_key_field:
    unique_key_field = 'hue_id'
    fields += [{"name": unique_key_field, "type": "string"}]
    kwargs['rowid'] = unique_key_field

  if not destination['hasHeader']:
    kwargs['header'] = 'false'
  else:
    kwargs['skipLines'] = 1

  if not client.exists(index_name):
    client.create_index(
        name=index_name,
        config_name=destination.get('indexerConfigSet'),
        fields=fields,
        unique_key_field=unique_key_field,
        df=df,
        shards=destination['indexerNumShards'],
        replication=destination['indexerReplicationFactor']
    )


import streamlit as st
from utilities.pose import components
st.set_page_config(layout="wide",initial_sidebar_state='collapsed')
def main():
    create_col, display_col = st.columns([1, 1])
    with create_col:
        form = components.create_pose()
    with display_col:
        components.list_poses()
if __name__ == "__main__":
    main()


#!/usr/bin/python

import os
import sys
import argparse

# Jam Master W. Because it's a Wrapper.

build_methods = []
run_methods = []

def add_build_method(method):
    build_methods.append(method)

def add_run_method(method):
    run_methods.append(method)

def main():
    parser = argparse.ArgumentParser(description="do some stuff")
    parser.add_argument('--build', action='store_true')
    parser.add_argument('command', help='command to wrap', nargs="*")
    args = parser.parse_args()

    if args.build:
        for m in build_methods:
            m()
    else:
        for m in run_methods:
            m()
        os.execvp(args.command[0], args.command)


from fastapi import Depends, APIRouter,FastAPI, HTTPException
from typing import List, Optional
from app import crud

from sqlalchemy.orm.session import Session
from app.api.deps import get_db
from app.schemas import Category, CategoryCreate, CategoryUpdate

router = APIRouter()

@router.get("/", response_model=List[Category])
async def get_accounts(user_id: int, db: Session = Depends(get_db)):
    db_accounts = crud.categories.get_user_categories(db, user_id)
    if not db_accounts:
        raise HTTPException(status_code=404, detail="Account not found")
    return db_accounts

@router.get("/{category_id}", response_model=Category)
def get_category(category_id: int, db: Session = Depends(get_db)):
    db_category = crud.categories.get_category(db, category_id)
    if not db_category:
        raise HTTPException(status_code=404, detail="Category not found")
    return db_category

@router.post("/", response_model=Category)
def create_category(user_id: int, data: CategoryCreate, db:Session = Depends(get_db)):
    db_category = crud.categories.create_category(db, user_id, data)
    if not db_category:
        raise HTTPException(status_code=404, detail="Error while creating category")
    return db_category

@router.put("/{category_id}", response_model=Category)
def update_category(category_id: int, data: CategoryUpdate, db:Session = Depends(get_db)):
    db_category = crud.categories.update_category(db, category_id, data)
    if not db_category:
        raise HTTPException(status_code=404, detail="Error while updating category")
    return db_category


import datetime
import json
import time
import pendulum
import pandas as pd
import requests
from io import StringIO
import numpy as np

from GeoJsonService import GeoJsonService
from Scraper import Scraper
from utils import write_record_to_output, cleanup, move_to_final

columns = ["country", "region", "infected", "deaths", "recoveries", "long", "lat", "last_updated"]
hopkins_csv = "output/john_hopkins_output.csv"
taiwan_csv = "output/taiwan.csv"
japan_csv = "output/japan.csv"
china_csv = "output/china.csv"
philip_csv = "output/philippines.csv"

class MiscScrapers:
    def __init__(self):
        self.last_updated = datetime.datetime.utcnow()
        self.geojson_service = GeoJsonService()

    def write_to_file(self, country, region, infected="", deaths="", recoveries="", long="", lat="", filename=None):
        record = dict(country=country,
                      region=region,
                      infected=infected,
                      deaths=deaths,
                      recoveries=recoveries,
                      long=long,
                      lat=lat,
                      last_updated=self.last_updated.strftime("%Y-%m-%d %H:%M:%S"))
        write_record_to_output(record, filename)

    @staticmethod
    def scrape_china_geojson(file_name):
        from googletrans import Translator
        translator = Translator()
        with open(file_name) as fin, open('output/china.csv', 'w') as fout:
            fout.write('"Country","Address","Address_Translated","Total","Recovered","Deaths","Lat","Long"\n')
            json_dict = json.load(fin)
            for feature in json_dict["features"]:
                while True:
                    try:
                        properties = feature["properties"]
                        geometry = feature["geometry"]
                        lat, long = geometry["coordinates"]
                        address = properties["Address"]
                        translation = translator.translate(address)
                        address_translated = translation.text
                        total = properties["cities__confirmedCount"]
                        recovered = ""
                        deaths = ""
                        fout.write(
                            f'"China","{address}","{address_translated}","{total}","{recovered}","{deaths}","{lat}","{long}"\n')
                        print(address, address_translated, total, recovered, deaths, lat, long, sep="\t", end="\n")
                        break
                    except Exception as e:
                        print(e, "Retrying in 30 seconds.")
                        time.sleep(30)

    def scrape_philipines_data(self):
        r = requests.get(
            "https://services5.arcgis.com/mnYJ21GiFTR97WFg/arcgis/rest/services/municitycent/FeatureServer/0"
            "/query?f=json&where=count_%3E%3D1&returnGeometry=false&spatialRel=esriSpatialRelIntersects"
            "&outFields=*&orderByFields=count_%20desc&resultOffset=0&resultRecordCount=100&cacheHint=true")
        response = r.json()
        for feature in response["features"]:
            attributes = feature["attributes"]
            city = attributes["ADM3_EN"]
            count = attributes["count_"]
            latitude = attributes["latitude"]
            longitude = attributes["longitude"]
            self.write_to_file(country="Philippines", region=city, infected=count,
                               long=longitude, lat=latitude, filename=philip_csv)
            print(f'"Philippines","{city}","{latitude}","{longitude}"')

    def get_city_name(self, city):
        city = city[:-16]
        return city

    def scrape_philippines_data(self):
        s = Scraper(url="https://covid19stats.ph/stats/by-location")
        table = s.find_table_with_text("Quezon")
        lst = s.table_to_2d_list(table)
        if not lst:
            return []
        country = "Philippines"
        cleanup(philip_csv)
        for row in lst[1:]:
            city = self.get_city_name(row[0])
            infected = row[1]
            recovered = row[3]
            deaths = row[5]
            latitude, longitude = self.geojson_service.get_lat_long(country, city)
            self.write_to_file(country=country, region=city, infected=infected, recoveries=recovered, deaths=deaths,
                               long=longitude, lat=latitude, filename=philip_csv)
        move_to_final(philip_csv)

    def scrape_taiwan_data(self):
        print("Starting taiwan scrape")
        country = "Taiwan"
        r = requests.get("https://services7.arcgis.com/HYXUMO0l0lNCIifa/arcgis/rest/services"
                         "/Wuhan_Coronavirus_Taiwan_County/FeatureServer/0/query?f=json&where=Confirmed%3E0"
                         "&returnGeometry=false&spatialRel=esriSpatialRelIntersects&outFields=*&orderByFields=Confirmed"
                         "%20desc&resultOffset=0&resultRecordCount=25&cacheHint=true")
        response = r.json()
        cleanup(taiwan_csv)
        for feature in response["features"]:
            attributes = feature["attributes"]
            city = attributes["COUNTYENG"]
            infected = attributes["Confirmed"]
            recovered = attributes["recover"]
            deaths = attributes["death"]
            latitude, longitude = self.geojson_service.get_lat_long(country, city)
            self.write_to_file(country=country, region=city, infected=infected, recoveries=recovered, deaths=deaths,
                               long=longitude, lat=latitude, filename=taiwan_csv)
        move_to_final(taiwan_csv)

    def scrape_china_data(self, date_str=None):
        if date_str is None:
            date_str = datetime.datetime.now().strftime("%d-%m-%Y")
            print(date_str)
        r = requests.get("https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data"
                         f"/csse_covid_19_daily_reports/{date_str}.csv")
        if r.status_code == 200:
            req = StringIO(r.text)
            df = pd.read_csv(req)
            china_df = df.loc[df["Country_Region"] == "China"]
        else:
            print(r)
            return

        for index, row in china_df.iterrows():
            country = row["Country_Region"]
            region = row["Province_State"]
            confirmed = row["Confirmed"]
            deaths = row["Deaths"]
            recoveries = row["Recovered"]
            active = row["Active"]
            latitude = row["Lat"]
            longitude = row["Long_"]
            self.write_to_file(country=country, region=region, infected=confirmed, deaths=deaths, recoveries=recoveries,
                               long=longitude, lat=latitude, filename=china_csv)

    def copy_static_files(self):
        move_to_final(china_csv)

    def scrape_japan_data(self):
        print("Starting japan scrape")
        country = "Japan"
        r = requests.get('https://data.covid19japan.com/summary/latest.json')
        response = r.json()
        prefectures = response["prefectures"]
        cleanup(japan_csv)
        for prefecture in prefectures:
            region = prefecture["name"]
            confirmed = prefecture["confirmed"]
            deaths = prefecture["deaths"]
            recoveries = prefecture["recovered"]
            latitude, longitude = self.geojson_service.get_lat_long(country, region)
            self.write_to_file(country=country, region=region, infected=confirmed, deaths=deaths, recoveries=recoveries,
                               long=longitude, lat=latitude, filename=japan_csv)
        move_to_final(japan_csv)

    def scrape_john_hopkins_data(self, date_str=None):
        print("Starting john hopkins scrape")
        excluded_countries = ["China"]
        countries = ["Australia", "Canada", "China"]
        curr_date = pendulum.now()
        processed = False
        if not date_str:
            date_str = curr_date.strftime("%m-%d-%Y")
        while not processed:
            print(f"Trying {date_str}...")
            r = requests.get("https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data"
                             f"/csse_covid_19_daily_reports/{date_str}.csv")
            if r.status_code == 200:
                processed = True
            else:
                print(f"Error! Status Code: {r.status_code}. Data still not updated.")
                curr_date = curr_date.subtract(days=1)
                date_str = curr_date.strftime("%m-%d-%Y")
                move_to_final(hopkins_csv)
                return
        if r.status_code == 200:
            req = StringIO(r.text)
            df = pd.read_csv(req).replace(np.nan, '', regex=True)
            df = df[df["Country_Region"].isin(countries)]
        else:
            print(r)
            return
        cleanup(hopkins_csv)
        for index, row in df.iterrows():
            country = row["Country_Region"]
            if country in excluded_countries:
                continue
            city = row["Admin2"]
            region = row["Province_State"] if not city else f'{city}, {row["Province_State"]}'
            confirmed = row["Confirmed"]
            deaths = row["Deaths"]
            recoveries = row["Recovered"]
            active = row["Active"]
            latitude = row["Lat"]
            longitude = row["Long_"]
            if "Unassigned" in region:
                continue
            self.write_to_file(country=country, region=region, infected=confirmed, deaths=deaths, recoveries=recoveries,
                               long=longitude, lat=latitude, filename=hopkins_csv)
        move_to_final(hopkins_csv)
        return df

    def run(self):
        print("Starting world scrape")
        self.scrape_john_hopkins_data()
        print("Starting japan scrape")
        self.scrape_japan_data()
        print("Starting taiwan scrape")
        self.scrape_taiwan_data()
        print("Starting philippines scrape")
        self.scrape_philippines_data()
        print("Scraping complete")

if __name__ == '__main__':
    misc_scrapers = MiscScrapers()
    misc_scrapers.run()


from typing import Tuple
import httpx, datetime, time
from dotenv import dotenv_values
from web3 import Web3, HTTPProvider
from web3.middleware.geth_poa import geth_poa_middleware
from tenacity import retry
from tenacity.wait import wait_random
from tenacity.stop import stop_after_attempt

api_key = dotenv_values()["API_KEY"]
if api_key is None:
    raise ValueError("API_KEY is required")
# api = "https://api.etherscan.io/api"
# api = 'https://api.bscscan.com/api'
# api = 'https://api-test.bscscan.com/api'
# endpoint = dotenv_values()['ANKR_BSC_ENDPOINT']
# endpoint = dotenv_values()["ALCHEMY_ETH_API"]
# endpoint = 'https://rpc.ankr.com/eth'
endpoint = dotenv_values()["ENDPOINT"]
assert endpoint is not None, "endpoint not config"
api = endpoint + "api"
rpc = dotenv_values()["RPC"]
assert rpc is not None, "rpc not set"
proxy = "http://127.0.0.1:10809"

w3 = Web3(HTTPProvider(rpc))
w3.middleware_onion.inject(geth_poa_middleware, layer=0)

class Scanner:
    def __init__(self, api=api, key=api_key, proxies=proxy):
        self.api = api
        self.key = key
        self.proxies = proxies

    @retry(stop=stop_after_attempt(10), wait=wait_random(min=1, max=1.5), reraise=True)
    @staticmethod
    def get_start_end_block_of_date(
        date: datetime.date, proxies=None
    ) -> Tuple[int, int]:
        """
        The block starts after the date starts
        """

        start_timestamp_of_date = int(time.mktime(time.strptime(str(date), "%Y-%m-%d")))
        end_timestamp_of_date = (
            int(
                time.mktime(
                    time.strptime(str(date + datetime.timedelta(days=1)), "%Y-%m-%d")
                )
            )
            - 1
        )

        with Scanner(proxies=proxy) as scanner:
            start_block_no = int(
                scanner.scan(
                    "block",
                    "getblocknobytime",
                    timestamp=start_timestamp_of_date,
                    closest="after",
                )
            )
            end_block_no = (
                int(
                    scanner.scan(
                        "block",
                        "getblocknobytime",
                        timestamp=end_timestamp_of_date,
                        closest="before",
                    )
                )
                if end_timestamp_of_date < time.time()
                else w3.eth.block_number
            )

        return start_block_no, end_block_no

    @retry(stop=stop_after_attempt(1), wait=wait_random(min=1, max=1.5), reraise=True)
    def scan(self, module: str, action: str, **kwargs):
        params = {"module": module, "action": action, "apiKey": self.key, **kwargs}

        return self.client.get(url=self.api, params=params).json()["result"]

    @retry(stop=stop_after_attempt(1), wait=wait_random(min=1, max=1.5), reraise=True)
    async def scan_async(self, module: str, action: str, **kwargs):
        params = {"module": module, "action": action, "apiKey": self.key, **kwargs}

        res = await self.async_client.get(url=self.api, params=params)

        return res.json()["result"]

    def __enter__(self):
        self.client = httpx.Client(proxies=self.proxies, timeout=None)
        return self

    async def __aenter__(self):
        self.async_client = httpx.AsyncClient(proxies=self.proxies, timeout=None)
        return self

    def __exit__(self, exc_type, exc_value, trace):
        self.client.close()
        return False

    async def __aexit__(self, exc_type, exc_value, trace):
        await self.async_client.aclose()
        return False


"""
Tests for fast version of ARMA innovations algorithm
"""

import numpy as np
import pytest
from numpy.testing import assert_equal, assert_allclose

from statsmodels.tsa.arima_process import arma_acovf
from statsmodels.tsa.innovations import _arma_innovations, arma_innovations
from statsmodels.tsa.statespace.sarimax import SARIMAX

def test_brockwell_davis_ex533():
    # See Brockwell and Davis (2009) - Time Series Theory and Methods
    # Example 5.3.3: ARMA(1, 1) process, p.g. 177
    nobs = 10

    ar_params = np.array([0.2])
    ma_params = np.array([0.4])
    sigma2 = 8.92
    p = len(ar_params)
    q = len(ma_params)
    m = max(p, q)

    ar = np.r_[1, -ar_params]
    ma = np.r_[1, ma_params]

    # First, get the autocovariance of the process
    arma_process_acovf = arma_acovf(ar, ma, nobs=nobs, sigma2=sigma2)
    unconditional_variance = (
        sigma2 * (1 + 2 * ar_params[0] * ma_params[0] + ma_params[0]**2) /
        (1 - ar_params[0]**2))
    assert_allclose(arma_process_acovf[0], unconditional_variance)

    # Next, get the autocovariance of the transformed process
    # Note: as required by {{prefix}}arma_transformed_acovf, we first divide
    # through by sigma^2
    arma_process_acovf /= sigma2
    unconditional_variance /= sigma2
    transformed_acovf = _arma_innovations.darma_transformed_acovf_fast(
        ar, ma, arma_process_acovf)
    acovf, acovf2 = (np.array(arr) for arr in transformed_acovf)

    # `acovf` is an m^2 x m^2 matrix, where m = max(p, q)
    # but it is only valid for the autocovariances of the first m observations
    # (this means in particular that the block `acovf[m:, m:]` should *not* be
    # used)
    # `acovf2` then contains the (time invariant) autocovariance terms for
    # the observations m + 1, ..., nobs - since the autocovariance is the same
    # for these terms, to save space we do not construct the autocovariance
    # matrix as we did for the first m terms. Thus `acovf2[0]` is the variance,
    # `acovf2[1]` is the first autocovariance, etc.

    # Test the autocovariance function for observations m + 1, ..., nobs
    # (it is time invariant here)
    assert_equal(acovf2.shape, (nobs - m,))
    assert_allclose(acovf2[0], 1 + ma_params[0]**2)
    assert_allclose(acovf2[1], ma_params[0])
    assert_allclose(acovf2[2:], 0)

    # Test the autocovariance function for observations 1, ..., m
    # (it is time varying here)
    assert_equal(acovf.shape, (m * 2, m * 2))

    # (we need to check `acovf[:m * 2, :m]`, i.e. `acovf[:2, :1])`
    ix = np.diag_indices_from(acovf)
    ix_lower = (ix[0][:-1] + 1, ix[1][:-1])

    # acovf[ix] is the diagonal, and we want to check the first m
    # elements of the diagonal
    assert_allclose(acovf[ix][:m], unconditional_variance)

    # acovf[ix_lower] is the first lower off-diagonal
    assert_allclose(acovf[ix_lower][:m], ma_params[0])

    # Now, check that we compute the moving average coefficients and the
    # associated variances correctly
    out = _arma_innovations.darma_innovations_algo_fast(
        nobs, ar_params, ma_params, acovf, acovf2)
    theta = np.array(out[0])
    v = np.array(out[1])

    # Test v (see eq. 5.3.13)
    desired_v = np.zeros(nobs)
    desired_v[0] = unconditional_variance
    for i in range(1, nobs):
        desired_v[i] = 1 + (1 - 1 / desired_v[i - 1]) * ma_params[0]**2
    assert_allclose(v, desired_v)

    # Test theta (see eq. 5.3.13)
    # Note that they will have shape (nobs, m + 1) here, not (nobs, nobs - 1)
    # as in the original (non-fast) version
    assert_equal(theta.shape, (nobs, m + 1))
    desired_theta = np.zeros(nobs)
    for i in range(1, nobs):
        desired_theta[i] = ma_params[0] / desired_v[i - 1]
    assert_allclose(theta[:, 0], desired_theta)
    assert_allclose(theta[:, 1:], 0)

    # Test against Table 5.3.1
    endog = np.array([
        -1.1, 0.514, 0.116, -0.845, 0.872, -0.467, -0.977, -1.699, -1.228,
        -1.093])
    u = _arma_innovations.darma_innovations_filter(endog, ar_params, ma_params,
                                                   theta)

    # Note: Table 5.3.1 has \hat X_n+1 = -0.5340 for n = 1, but this seems to
    # be a typo, since equation 5.3.12 gives the form of the prediction
    # equation as \hat X_n+1 = \phi X_n + \theta_n1 (X_n - \hat X_n)
    # Then for n = 1 we have:
    # \hat X_n+1 = 0.2 (-1.1) + (0.2909) (-1.1 - 0) = -0.5399
    # And for n = 2 if we use what we have computed, then we get:
    # \hat X_n+1 = 0.2 (0.514) + (0.3833) (0.514 - (-0.54)) = 0.5068
    # as desired, whereas if we used the book's number for n=1 we would get:
    # \hat X_n+1 = 0.2 (0.514) + (0.3833) (0.514 - (-0.534)) = 0.5045
    # which is not what Table 5.3.1 shows.
    desired_hat = np.array([
        0, -0.540, 0.5068, -0.1321, -0.4539, 0.7046, -0.5620, -0.3614,
        -0.8748, -0.3869])
    desired_u = endog - desired_hat
    assert_allclose(u, desired_u, atol=1e-4)

def test_brockwell_davis_ex534():
    # See Brockwell and Davis (2009) - Time Series Theory and Methods
    # Example 5.3.4: ARMA(1, 1) process, p.g. 178
    nobs = 10

    ar_params = np.array([1, -0.24])
    ma_params = np.array([0.4, 0.2, 0.1])
    sigma2 = 1
    p = len(ar_params)
    q = len(ma_params)
    m = max(p, q)

    ar = np.r_[1, -ar_params]
    ma = np.r_[1, ma_params]

    # First, get the autocovariance of the process
    arma_process_acovf = arma_acovf(ar, ma, nobs=nobs, sigma2=sigma2)
    assert_allclose(arma_process_acovf[:3],
                    [7.17133, 6.44139, 5.06027], atol=1e-5)

    # Next, get the autocovariance of the transformed process
    transformed_acovf = _arma_innovations.darma_transformed_acovf_fast(
        ar, ma, arma_process_acovf)
    acovf, acovf2 = (np.array(arr) for arr in transformed_acovf)
    # See test_brockwell_davis_ex533 for details on acovf vs acovf2

    # Test acovf
    assert_equal(acovf.shape, (m * 2, m * 2))

    ix = np.diag_indices_from(acovf)
    ix_lower1 = (ix[0][:-1] + 1, ix[1][:-1])
    ix_lower2 = (ix[0][:-2] + 2, ix[1][:-2])
    ix_lower3 = (ix[0][:-3] + 3, ix[1][:-3])
    ix_lower4 = (ix[0][:-4] + 4, ix[1][:-4])

    assert_allclose(acovf[ix][:m], 7.17133, atol=1e-5)
    desired = [6.44139, 6.44139, 0.816]
    assert_allclose(acovf[ix_lower1][:m], desired, atol=1e-5)
    assert_allclose(acovf[ix_lower2][0], 5.06027, atol=1e-5)
    assert_allclose(acovf[ix_lower2][1:m], 0.34, atol=1e-5)
    assert_allclose(acovf[ix_lower3][:m], 0.1, atol=1e-5)
    assert_allclose(acovf[ix_lower4][:m], 0, atol=1e-5)

    # Test acovf2
    assert_equal(acovf2.shape, (nobs - m,))
    assert_allclose(acovf2[:4], [1.21, 0.5, 0.24, 0.1])
    assert_allclose(acovf2[4:], 0)

    # Test innovations algorithm output
    out = _arma_innovations.darma_innovations_algo_fast(
        nobs, ar_params, ma_params, acovf, acovf2)
    theta = np.array(out[0])
    v = np.array(out[1])

    # Test v (see Table 5.3.2)
    desired_v = [7.1713, 1.3856, 1.0057, 1.0019, 1.0016, 1.0005, 1.0000,
                 1.0000, 1.0000, 1.0000]
    assert_allclose(v, desired_v, atol=1e-4)

    # Test theta (see Table 5.3.2)
    assert_equal(theta.shape, (nobs, m + 1))
    desired_theta = np.array([
        [0, 0.8982, 1.3685, 0.4008, 0.3998, 0.3992, 0.4000, 0.4000, 0.4000,
         0.4000],
        [0, 0, 0.7056, 0.1806, 0.2020, 0.1995, 0.1997, 0.2000, 0.2000, 0.2000],
        [0, 0, 0, 0.0139, 0.0722, 0.0994, 0.0998, 0.0998, 0.0999, 0.1]]).T
    assert_allclose(theta[:, :m], desired_theta, atol=1e-4)
    assert_allclose(theta[:, m:], 0)

    # Test innovations filter output
    endog = np.array([1.704, 0.527, 1.041, 0.942, 0.555, -1.002, -0.585, 0.010,
                      -0.638, 0.525])
    u = _arma_innovations.darma_innovations_filter(endog, ar_params, ma_params,
                                                   theta)

    desired_hat = np.array([
        0, 1.5305, -0.1710, 1.2428, 0.7443, 0.3138, -1.7293, -0.1688,
        0.3193, -0.8731])
    desired_u = endog - desired_hat
    assert_allclose(u, desired_u, atol=1e-4)

@pytest.mark.parametrize("ar_params,ma_params,sigma2", [
    (np.array([]), np.array([]), 1),
    (np.array([0.]), np.array([0.]), 1),
    (np.array([0.9]), np.array([]), 1),
    (np.array([]), np.array([0.9]), 1),
    (np.array([0.2, -0.4, 0.1, 0.1]), np.array([0.5, 0.1]), 1.123),
    (np.array([0.5, 0.1]), np.array([0.2, -0.4, 0.1, 0.1]), 1.123),
])
def test_innovations_algo_filter_kalman_filter(ar_params, ma_params, sigma2):
    # Test the innovations algorithm and filter against the Kalman filter
    # for exact likelihood evaluation of an ARMA process

    ar = np.r_[1, -ar_params]
    ma = np.r_[1, ma_params]

    endog = np.random.normal(size=10)
    nobs = len(endog)

    # Innovations algorithm approach
    arma_process_acovf = arma_acovf(ar, ma, nobs=nobs, sigma2=sigma2)
    transformed_acov = _arma_innovations.darma_transformed_acovf_fast(
        ar, ma, arma_process_acovf / sigma2)
    acovf, acovf2 = (np.array(mv) for mv in transformed_acov)
    theta, r = _arma_innovations.darma_innovations_algo_fast(
        nobs, ar_params, ma_params, acovf, acovf2)
    u = _arma_innovations.darma_innovations_filter(endog, ar_params, ma_params,
                                                   theta)

    v = np.array(r) * sigma2
    u = np.array(u)

    llf_obs = -0.5 * u**2 / v - 0.5 * np.log(2 * np.pi * v)

    # Kalman filter apparoach
    mod = SARIMAX(endog, order=(len(ar_params), 0, len(ma_params)))
    res = mod.filter(np.r_[ar_params, ma_params, sigma2])

    # Test that the two approaches are identical
    assert_allclose(u, res.forecasts_error[0])
    # assert_allclose(theta[1:, 0], res.filter_results.kalman_gain[0, 0, :-1])
    assert_allclose(llf_obs, res.llf_obs)

    # Get llf_obs directly
    llf_obs2 = _arma_innovations.darma_loglikeobs_fast(
        endog, ar_params, ma_params, sigma2)

    assert_allclose(llf_obs2, res.llf_obs)

@pytest.mark.parametrize("ar_params,ma_params,sigma2", [
    (np.array([]), np.array([]), 1),
    (np.array([0.]), np.array([0.]), 1),
    (np.array([0.9]), np.array([]), 1),
    (np.array([]), np.array([0.9]), 1),
    (np.array([0.2, -0.4, 0.1, 0.1]), np.array([0.5, 0.1]), 1.123),
    (np.array([0.5, 0.1]), np.array([0.2, -0.4, 0.1, 0.1]), 1.123),
])
def test_innovations_algo_direct_filter_kalman_filter(ar_params, ma_params,
                                                      sigma2):
    # Test the innovations algorithm and filter against the Kalman filter
    # for exact likelihood evaluation of an ARMA process, using the direct
    # function.

    endog = np.random.normal(size=10)

    # Innovations algorithm approach
    u, r = arma_innovations.arma_innovations(endog, ar_params, ma_params,
                                             sigma2)

    v = np.array(r) * sigma2
    u = np.array(u)

    llf_obs = -0.5 * u**2 / v - 0.5 * np.log(2 * np.pi * v)

    # Kalman filter apparoach
    mod = SARIMAX(endog, order=(len(ar_params), 0, len(ma_params)))
    res = mod.filter(np.r_[ar_params, ma_params, sigma2])

    # Test that the two approaches are identical
    assert_allclose(u, res.forecasts_error[0])
    # assert_allclose(theta[1:, 0], res.filter_results.kalman_gain[0, 0, :-1])
    assert_allclose(llf_obs, res.llf_obs)

    # Get llf_obs directly
    llf_obs2 = _arma_innovations.darma_loglikeobs_fast(
        endog, ar_params, ma_params, sigma2)

    assert_allclose(llf_obs2, res.llf_obs)

@pytest.mark.parametrize("ar_params,diff,ma_params,sigma2", [
    (np.array([]), 1, np.array([]), 1),
    (np.array([0.]), 1, np.array([0.]), 1),
    (np.array([0.9]), 1, np.array([]), 1),
    (np.array([]), 1, np.array([0.9]), 1),
    (np.array([0.2, -0.4, 0.1, 0.1]), 1, np.array([0.5, 0.1]), 1.123),
    (np.array([0.5, 0.1]), 1, np.array([0.2, -0.4, 0.1, 0.1]), 1.123),
    (np.array([0.5, 0.1]), 2, np.array([0.2, -0.4, 0.1, 0.1]), 1.123),
])
def test_integrated_process(ar_params, diff, ma_params, sigma2):
    # Test loglikelihood computation when model has integration

    nobs = 100

    endog = np.cumsum(np.random.normal(size=nobs))

    # Innovations algorithm approach
    llf_obs = arma_innovations.arma_loglikeobs(
        np.diff(endog, diff), ar_params, ma_params, sigma2)

    # Kalman filter apparoach
    mod = SARIMAX(endog, order=(len(ar_params), diff, len(ma_params)),
                  simple_differencing=True)
    res = mod.filter(np.r_[ar_params, ma_params, sigma2])

    # Test that the two approaches are identical
    assert_allclose(llf_obs, res.llf_obs)

@pytest.mark.parametrize("ar_params,ma_params,sigma2", [
    (np.array([]), np.array([]), 1),
    (np.array([0.]), np.array([0.]), 1),
    (np.array([0.9]), np.array([]), 1),
    (np.array([]), np.array([0.9]), 1),
    (np.array([0.2, -0.4, 0.1, 0.1]), np.array([0.5, 0.1]), 1.123),
    (np.array([0.5, 0.1]), np.array([0.2, -0.4, 0.1, 0.1]), 1.123),
])
def test_regression_with_arma_errors(ar_params, ma_params, sigma2):
    # Test loglikelihood computation when model has regressors
    nobs = 100

    eps = np.random.normal(nobs)
    exog = np.c_[np.ones(nobs), np.random.uniform(size=nobs)]
    beta = [5, -0.2]
    endog = np.dot(exog, beta) + eps

    # Innovations algorithm approach
    beta_hat = np.squeeze(np.linalg.pinv(exog).dot(endog))
    demeaned = endog - np.dot(exog, beta_hat)
    llf_obs = arma_innovations.arma_loglikeobs(
        demeaned, ar_params, ma_params, sigma2)

    # Kalman filter approach
    # (this works since we impose here that the regression coefficients are
    # beta_hat - in practice, the MLE estimates will not necessarily match
    # the OLS estimates beta_hat)
    mod = SARIMAX(endog, exog=exog, order=(len(ar_params), 0, len(ma_params)))
    res = mod.filter(np.r_[beta_hat, ar_params, ma_params, sigma2])

    # Test that the two approaches are identical
    assert_allclose(llf_obs, res.llf_obs)


#!/usr/bin/env python3

from qtpy.QtWidgets import *

from tts.edge_tts_engine import EdgeTtsEngine
from tts.WidgetBase import WidgetBase

class EdgeTtsWidget(WidgetBase):
    rate_defult = 0
    volume_defult = 0
    voice_defult = "Microsoft Server Speech Text to Speech Voice (zh-CN, XiaoxiaoNeural)"
    file_defult_suffix = '.mp3'
    config_section_name = 'edge_tts'

    def __init__(self, *args, **kwargs):
        super(EdgeTtsWidget, self).__init__(*args, **kwargs)
        self.class_now = EdgeTtsWidget
        self.setObjectName(self.config_section_name)

    def init_range(self):
        self.rate_slider.setMaximum(100)
        self.rate_slider.setMinimum(-99)
        self.volume_slider.setMaximum(100)
        self.volume_slider.setMinimum(-99)

    def init_engine(self):
        self.engine_obj = EdgeTtsEngine(
            voice=self.voice_defult, rate=self.rate, volume=self.volume)

    def init_voices_combox(self):
        for i, item in enumerate(self.engine_obj.voice_all):
            self.voices_combox.addItem(item["FriendlyName"], item)
            if item["Name"] == self.voice_defult:
                self.voices_combox.setCurrentIndex(i)

    def voices_combox_change(self):
        data = self.voices_combox.currentData()
        self.voice = data["Name"]
        self.engine_obj.voice = self.voice

    def rate_slider_change(self, value):
        value = int(value)
        if value >= 0:
            flag = "+"
        else:
            flag = ""
        self.rate = value
        self.rate_up_value_label.setText(f"{flag}{value}%")
        self.engine_obj.rate = f"{flag}{value}%"

    def volume_slider_change(self, value):
        value = int(value)
        if value >= 0:
            flag = "+"
        else:
            flag = ""
        self.volume = value
        self.volume_up_value_label.setText(f"{flag}{value}%")
        self.engine_obj.volume = f"{flag}{value}%"

    def about_voice(self):
        data = self.voices_combox.currentData()
        aboutdialog = AboutDialog(data=data)
        aboutdialog.exec_()

    def play_btn_func(self, state):
        # print(state)
        pass

class AboutDialog(QDialog):
    def __init__(self, data: dict = None, *args, **kwargs):
        super(AboutDialog, self).__init__(*args, **kwargs)
        self.data = data
        self._Name = data.get("Name")
        self._ShortName = data.get("ShortName")
        self._FriendlyName = data.get("FriendlyName")
        self._Gender = data.get("Gender")
        self._Locale = data.get("Locale")
        self._Status = data.get("Status")
        self._Language = data.get("Language")

        self.setWindowTitle("关于")
        self.init_ui()

    def init_ui(self):
        layout = QFormLayout(self)

        if self._Name:
            name_label = QLabel("Name:")
            name_detail_label = QLabel(self._Name)
            layout.addRow(name_label, name_detail_label)

        if self._ShortName:
            shortname_label = QLabel("ShortName:")
            shortname_detail_label = QLabel(self._ShortName)
            layout.addRow(shortname_label, shortname_detail_label)

        if self._FriendlyName:
            FriendlyName_label = QLabel("FriendlyName:")
            FriendlyName_detail_label = QLabel(self._FriendlyName)
            layout.addRow(FriendlyName_label, FriendlyName_detail_label)

        if self._Gender:
            Gender_label = QLabel("Gender:")
            Gender_detail_label = QLabel(self._Gender)
            layout.addRow(Gender_label, Gender_detail_label)

        if self._Locale:
            Locale_label = QLabel("Locale:")
            Locale_detail_label = QLabel(self._Locale)
            layout.addRow(Locale_label, Locale_detail_label)

        if self._Status:
            Status_label = QLabel("Status:")
            Status_detail_label = QLabel(self._Status)
            layout.addRow(Status_label, Status_detail_label)

        if self._Language:
            Language_label = QLabel("Language:")
            Language_detail_label = QLabel(self._Language)
            layout.addRow(Language_label, Language_detail_label)

if __name__ == "__main__":

    import sys
    from mainapplication import appctxt
    main_appctxt = appctxt
    mainWin = EdgeTtsWidget()
    mainWin.show()
    exit_code = main_appctxt.app.exec_()
    sys.exit(exit_code)


from typing import Tuple

import pytorch_lightning as ptl
import torch
from torch import nn

# pylint: disable=abstract-method
# pylint: disable=too-many-ancestors
class SegmentationModel(ptl.LightningModule):
    """Segmentation model.

    This class contains the used segmentation model along with
    parameters relevant for model training, such as optimizer, loss function,
    learning rate.
    """

    def __init__(self, model: nn.Module, learning_rate: float, loss: nn.Module):
        """Ran once when instantiating the SegmentationModel object."""
        super().__init__()
        self.model = model
        self.learning_rate = learning_rate
        self.loss = loss
        self.train_loss = []

    # pylint: disable=arguments-differ
    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """Forward step.

        Args:
            x: an image
        Returns:
            Predicted mask.
        """
        return self.model(x)

    # pylint: disable=arguments-differ
    # pylint: disable=unused-argument
    def training_step(self, batch: Tuple[torch.Tensor], batch_idx: int) -> float:
        """Training step.

        Args:
            batch: tuple of images and masks
            batch_idx: batch index
        Returns:
            Loss value.
        """
        x, y = batch
        y_hat = self.forward(x)
        loss = self.loss(y_hat, y.float())
        self.log(
            "train_loss", loss, on_step=False, on_epoch=True, prog_bar=True, logger=True
        )
        self.train_loss.append(loss)
        return loss

    # pylint: disable=arguments-differ
    # pylint: disable=unused-argument
    def validation_step(self, batch: Tuple[torch.Tensor], batch_idx: int):
        """Validation step.

        Args:
            batch: tuple of images and masks
            batch_idx: batch index.
        """
        x, y = batch
        y_hat = self.forward(x)
        loss = self.loss(y_hat, y.float())
        self.log(
            "val_loss", loss, on_step=False, on_epoch=True, prog_bar=True, logger=True
        )

    def configure_optimizers(self):
        """Optimizer configuration."""
        return torch.optim.Adam(self.parameters(), lr=self.learning_rate)


import numpy as np
import matplotlib.pyplot as plt
from scipy import pi
from scipy.fftpack import fft

def plot_fft(filename, number, name, color):

	plt.figure(number)
	utc, the, energy, ra, dec = np.loadtxt(filename, unpack=True)
	N= len(utc)
	T= 3600
	cos_ra=np.cos(ra*pi/180.0)
	sin_ra=np.sin(ra*pi/180.0)
	yauger=  np.sqrt(sin_ra**2 + cos_ra**2)#fft(ra)
	xauger=np.linspace(0.0,1.0/(2.0*T), N/2)
	plt.yscale("log")
	plt.plot(xauger,2.0/N * np.abs(yauger[:N//2]), color=color)
	plt.title(name)
	pass

filename="../../Herald/Central/Modified/Energy_above_1EeV/Herald_8EeV.dat"

plot_fft(filename, 1, "test", "red")
plt.show()


from app import inputValidation, sqlite3, redirect, url_for, request, session
from datetime import datetime
from logRegister import logRegister

def updateUserr():
    try:
        conn = sqlite3.connect('auditoria.db')
        cursor = conn.execute("SELECT USER from UTILIZADOR where ID = %s" % (session['user'],) )
        for row in cursor:
            user = row[0]
        if inputValidation(request.form['name'], 'alphanumeric') and inputValidation(request.form['id'], 'digit'):
            cursor = conn.execute("SELECT USER from UTILIZADOR where USER = '%s'" % (request.form['name'],) )
            result = 0
            for row in cursor:
                result = result + 1
            if result == 0:
                conn = sqlite3.connect('auditoria.db')
                conn.execute("UPDATE UTILIZADOR set USER = '%s' where ID = %s" % (request.form['name'], request.form['id'] ))
                conn.commit()
                conn.close()
                logRegister(user, 'updateUser', 'success', datetime.now().strftime("%m/%d/%Y,%H:%M:%S"))
                return redirect(url_for('mainpageAdmin'))
            else:
                logRegister(user, 'updateUser', 'failure', datetime.now().strftime("%m/%d/%Y,%H:%M:%S"))
                return "User already exist"
        else:
            logRegister(user, 'updateUser', 'failure', datetime.now().strftime("%m/%d/%Y,%H:%M:%S"))
            return "Input Error"
    except:
        logRegister(user, 'updateUser', 'failure', datetime.now().strftime("%m/%d/%Y,%H:%M:%S"))
        return "Something went wrong"


from SongManager.models import Composer, Tag, Song, SongFile, FileType

from rest_framework import serializers

class ComposerSerializer(serializers.ModelSerializer):
    class Meta:
        model = Composer
        fields = ['id', 'first_name', 'last_name']

class TagSerializer(serializers.ModelSerializer):
    class Meta:
        model = Tag
        fields = ['id', 'tag_name']

class SongFileListingField(serializers.RelatedField):
    def to_representation(self, value):
        url = value.file.url
        return url

class FileTypeSerializer(serializers.ModelSerializer):
    class Meta:
        model = FileType
        fields = ['id', 'type']

class DynamicFieldsModelSerializer(serializers.ModelSerializer):
    """
    A ModelSerializer that takes an additional `fields` argument that
    controls which fields should be displayed.
    """

    def __init__(self, *args, **kwargs):
        # Don't pass the 'fields' arg up to the superclass
        fields = kwargs.pop('fields', None)

        # Instantiate the superclass normally
        super(DynamicFieldsModelSerializer, self).__init__(*args, **kwargs)

        if fields is not None:
            # Drop any fields that are not specified in the `fields` argument.
            allowed = set(fields)
            existing = set(self.fields)
            for field_name in existing - allowed:
                self.fields.pop(field_name)

class SongFileSerializer(serializers.ModelSerializer):
    filetypes = FileTypeSerializer(many=True, required=False)

    def to_representation(self, value):

        if value.thumbnail:
            self.context['thumbnail'] = value.thumbnail
        return super().to_representation(value)

    def create(self, validated_data):
        filetypes = None
        if 'filetypes' in validated_data:
            filetypes = validated_data.pop('filetypes')
        songfile = SongFile.objects.create(**validated_data)
        if filetypes:
            add_all(filetypes, songfile, FileType, 'filetypes')
        return songfile

    def update(self, instance, validated_data, *args, **kwargs):
        if 'filetypes' in validated_data:
            filetypes = validated_data.pop('filetypes')
            add_all(filetypes, instance, FileType, 'filetypes')
        return super().update(instance, validated_data)
    class Meta:
        model = SongFile
        fields = ['id', 'file', 'song', 'comments', 'filetypes', 'thumbnail']

def add_all(collection, obj, Cls, attr):
    getattr(obj, attr).clear()
    for item_data in collection:
        item, created = Cls.objects.get_or_create(**item_data)
        getattr(obj, attr).add(item)

class SongListSerializer(DynamicFieldsModelSerializer):
    composers = ComposerSerializer(required=True, many=True)
    tags = TagSerializer(required=False, many=True)
    #files = SongFileSerializer(many=True, read_only=True, required=False)
    #files = SongFileListingField(many=True, read_only=True)
    class Meta:
        model = Song
        fields = ['id', 'title', 'composers', 'tags', 'first_line',]
class SongSerializer(DynamicFieldsModelSerializer):
    composers = ComposerSerializer(required=False, many=True)
    tags = TagSerializer(required=False, many=True)
    files = SongFileSerializer(many=True, read_only=True, required=False)
    #files = SongFileListingField(many=True, read_only=True)
    class Meta:
        model = Song
        fields = ['id', 'title', 'composers', 'tags', 'first_line', 'files']

    def create(self, validated_data):
        print("In create")
        composers_data = validated_data.pop('composers') if 'composers' in validated_data else None
        tags_data = validated_data.pop('tags') if 'tags' in validated_data else None
        song = Song.objects.create(**validated_data)

        if composers_data:
            add_all(composers_data, song, Composer, 'composers')
        if tags_data:
            add_all(tags_data, song, Tag, 'tags')

        return song

    def update(self, instance, validated_data):
        if 'composers' in validated_data:
            composers_data = validated_data.pop('composers')
            instance.composers.clear()
            add_all(composers_data, instance, Composer, 'composers')
        if 'tags' in validated_data:
            tags_data = validated_data.pop('tags')
            instance.tags.clear()
            add_all(tags_data, instance, Tag, 'tags')
        return super().update(instance, validated_data)
"""

tests

from SongManager.models import Song
from SongManager.serializers import SongSerializer
from rest_framework.renderers import JSONRenderer
from rest_framework.parsers import JSONParser
song = Song.objects.get(id=653)

serializer = SongSerializer(song)
print(serializer.data)
print(JSONRenderer().render(serializer.data))

"""


import numpy as np
import sys
import math
import datetime
import os
import random
# import own modules #
sys.path.append('../public')
from my_modules import *
from constants  import *

class ShipDemand:
    def __init__(self, history_data=None, neu=None, sigma=None, u=None, d=None, p=None):
        if history_data is None:
            #self.yearly_history_data = load_history_data(YEAR,DEMAND_TYPE)
            self.monthly_history_data = load_history_data(MONTH,DEMAND_TYPE)
        else:
            self.yearly_history_data = history_data
        # initialize parameters
        if (neu is None or sigma is None or u is None or d is None or p is None):
            self.calc_params_from_history()
        else:
            self.neu, self.sigma, self.u, self.d, self.p = neu, sigma, u, d, p

    # calc neu and sigma from history data
    def calc_params_from_history(self):
        index   = 0
        delta_t = 1.0 / DELTA_T_DAY
        values  = np.array([])
        for date, ship_demand in self.monthly_history_data:
            if index == 0:
                # initialize the price
                s_0 = ship_demand
            else:
                s_t      = ship_demand
                base_val = math.log(s_t / s_0)
                values   = np.append(values, base_val)
                # update the price
                s_0      = ship_demand
            index += 1

        # substitute inf to nan in values
        values = inf_to_nan_in_array(values)
        self.neu    = np.nanmean(values)
        self.sigma  = np.nanstd(values)
        self.u      = np.exp(self.sigma * np.sqrt(delta_t))
        self.d      = np.exp(self.sigma * (-1) * np.sqrt(delta_t))
        self.p      = 0.5 + 0.5 * (self.neu / self.sigma) * np.sqrt(delta_t)
        return

    def calc_ship_demand(self, current_ship_demand):
        return self.u * current_ship_demand if random.randint(0,1) < self.p else self.d * current_ship_demand

    # generate predicted scenario
    def generate_scenario(self,predict_years=DEFAULT_PREDICT_YEARS,predict_pattern_number=DEFAULT_PREDICT_PATTERN_NUMBER):
        # default predict_years is 15 years [180 months]
        self.predict_years  = predict_years

        # predicted data type
        self.predicted_data = []
        for p_num in range(predict_pattern_number):
            self.predicted_data.append([])

        predict_months_num = int(self.predict_years * 12)

        # latest date from history_data
        latest_history_date_str, latest_ship_demand = self.monthly_history_data[-1]
        latest_history_date                      = datetime.datetime.strptime(latest_history_date_str.decode('UTF-8'), '%Y/%m/%d')
        for pattern in range(predict_pattern_number):
            current_date  = latest_history_date
            current_ship_demand = latest_ship_demand
            for predict_month_num in range(predict_months_num):
                current_date        = add_month(current_date)
                current_date_str    = datetime.datetime.strftime(current_date, '%Y/%m/%d')
                for time in range(DELTA_T_DAY):
                    current_ship_demand    = self.calc_ship_demand(current_ship_demand)
                self.predicted_data[pattern].append({'date':current_date_str, 'price':current_ship_demand})
        return


#%%
import numpy as np
import sys
sys.path.append('../0')
import os
sys.path.append(os.path.abspath('../0'))

from importlib import reload

from numpy import array, c_, matrix,  zeros
from numpy.linalg import norm

#%%

#from time import time

filename0 = 'q8_patch_s11'

filename0 = 'Cook_8_CPE8R_ccx_plast'

filename = 'ccx_examples/'+filename0

import FE_io
reload(FE_io)

#bvp_dict =  FE_io.bvp_dict(filename)
#bvp_dict.steps[1]['*nlgeom'] = False

bvp_dict =  FE_io.bvp_dict(filename)
bvp_dict.steps[1]['*nlgeom'] = True

#%%

import FE_bvp
reload(FE_bvp)

# initialise solution
#soln = FE_bvp.solution(bvp_dict)

soln = FE_bvp.solution(bvp_dict)

#%%
timef=1

# Solve unknowns for full time step
#dU,new_ISVs,is_converged = soln.solve_increment(time=timef)
#
dU,new_ISVs,is_converged = soln.solve_increment(time=timef)
#

#%%
# again
#over 10 steps

bvp_dict =  FE_io.bvp_dict(filename)
bvp_dict.steps[1]['*nlgeom'] = True# False
soln = FE_bvp.solution(bvp_dict)
dtime = 0.1

dU,new_ISVs,is_converged,n_iter = soln.solve_increment(time=0.001)
dU*=100

timef = 0
for i in range(10):
    timef += dtime
    dU,new_ISVs,is_converged,n_iter = soln.solve_increment(dU,time=timef)
    #
    if is_converged:
        print("*************** TIME %.2f converged\n\n"%timef)
        soln.statev = new_ISVs.copy()
        soln.U += dU
        soln.solved_time = timef
        soln.time_steps += [soln.solved_time]

        soln.archive_results(update_ISVs=True)

# write to output:
#FE_io.write_to_vtu(soln,'patch_lagrange_strain')
FE_io.write_to_vtu(soln,filename0+'_nlgeom')

#%%

import FE_bvp
reload(FE_bvp)
bvp_dict =  FE_io.bvp_dict(filename)
soln = FE_bvp.solution(bvp_dict)
soln.solve_step()
FE_io.write_to_vtu(soln,filename0)

#%%

suse = soln2
# test full residual tangent matrix
residual,k_global,new_ISVs = suse.assemble(dU,time=timef)

pertval = 1e-6

k_global_FD = []
for i in range(residual.size):
    dU_FD = dU[suse.active_dofs]
    dU_FD[i] += pertval
    residualFw = suse.calc_residual(dU_FD,time=timef)

    dU_FD = dU[suse.active_dofs]
    dU_FD[i] -= pertval
    residualBw = suse.calc_residual(dU_FD,time=timef)

    k_global_FD += [0.5*array(residualFw-residualBw).flatten()/pertval]

KA = k_global.todense()
KB = matrix(k_global_FD)

compared_K = KA - KB

maxkb = np.max(KB-KB.T)
print('maxkb  = ',maxkb)

k_an_norm = norm(k_global.todense())
k_fd_norm = norm(k_global_FD)
k_cm_norm = norm(compared_K)

print('norm of AN : %.6e, FD : %.6e, compared : %.6e'%(k_an_norm,k_fd_norm,k_cm_norm))

#%%
#
#def assemble_FD(dU,time):
#    residual = soln.calc_residual(dU,time=time)
#    #
#    k_global_FD = []
#    for i in range(residual.size):
#        dU_FD = dU[soln.active_dofs]
#        dU_FD[i] += 1e-8
#        residualFD = soln.calc_residual(dU_FD,time=timef)
#        k_global_FD += [array(residualFD-residual).flatten()/1e-8]
#
#    return residual, matrix(k_global_FD)

#%%

#
## prescribed displacements and contribution to dU:
#U_i, U_val = soln.disp_ext(time,from_step)
#if U_i.size>0:
#    dU[U_i] = U_val-soln.U[U_i]
##
##free degrees of freedom
#fdof = [i for i in soln.active_dofs if i not in U_i]
##
#
#Ra,Ka = assemble_FD(dU,timef)
#
#residual[fdof] = Ra
#
## add external foces to resdual:
#F_i, F_val = soln.force_ext(time,from_step)
#if F_i.size>0:
#    residual[F_i,0] = F_val

#%%
# try Newton with line search:
#
#from_step = 1
#do_sparse = True
#only_active = False
#
#from scipy import sparse
#from scipy.sparse import linalg
#
#
#stepTime = soln.steps[from_step]['*time'][1]
#time = timef
#dtime = time-soln.solved_time
##
## prescribed displacements and contribution to dU:
#U_i, U_val = soln.disp_ext(time,from_step)
#if U_i.size>0:
#    dU[U_i] = U_val-soln.U[U_i]
##
##free degrees of freedom
#fdof = [i for i in soln.active_dofs if i not in U_i]
##
#residual,k_global,new_ISVs = soln.assemble(dU,time,dtime,from_step,do_sparse,only_active)
##
## add external foces to resdual:
#F_i, F_val = soln.force_ext(time,from_step)
#if F_i.size>0:
#    residual[F_i,0] = F_val
##
## residual norm
## relative residual norm
#ResNrm = norm(residual[fdof,0]);
#print('Normalized residual at start of iteration  = %10.6e'%ResNrm)
## solve
#Kaa = k_global[fdof,:].T[fdof,:].T;
#Pa  = residual[fdof];
#Kaa_LU = sparse.linalg.splu(Kaa.tocsc())
#deltaUf = -array(Kaa_LU.solve(Pa)).flatten()
#dUNrm = norm(deltaUf)
#print('Normalized update to Unknowns              = %10.6e'%dUNrm)
##
#def linefn(alpha):
#    dU_new = dU.copy()
#    dU_new[fdof] += alpha*deltaUf
#    residual_new = array(soln.calc_residual(dU_new,time,dtime,from_step,do_sparse,only_active)).flatten()
#    obj = sum(deltaUf*residual_new[fdof])
#    return obj
##lower
#a_L = 0
#s_0 = sum(deltaUf*array(residual[fdof]).flatten())
#s_L = s_0
##upper
#a_U = 1
#s_N = linefn(a_U)
#s_U = s_N
#kount = 0
#while abs(s_N/s_0)>1e-15 and kount<20:
#    #
#    kount += 1
#    # next guess
#    a_N = a_U - s_U*(a_L - a_U)/(s_L - s_U)
#    # test and update bounds:
#    s_N = linefn(a_N)
#    if s_N*s_L<0:
#        a_U = a_N
#        s_U = s_N
#    else:
#        a_L = a_N
#        s_L = s_N
#
## new guess for the update
#dU[fdof] += a_N*deltaUf

# full newton:
#dU[fdof] += deltaUf

#%%
#def assemble_system(bvp,U0,dU):
#
#
#
#    return bvp
#
#
#
#
##outputall = True
#bvp = FE_solver.solve(filename,returnbvp=True,outputall=True)

#FE_solver.solve(filename,resultsfile='results/'+filename0,outputall=True)

### Create Boundary Value Problems (dictionar(y)/(ies))
#tic = time()

#bvp = FE_io.bvp_dict(filename)
##
##
#print('\n** BOUNDARY VALUE DICTIONARY CREATED FROM INPUT FILE << %s >> '%(filename.split('/')[-1].split('.')[0]))
#print('\t\t>> done in %.4s seconds'%(time()-tic))
##
##
##
##
## SOLVE step 1:

#from scipy import sparse
#from scipy.sparse import linalg
#import pickle

##
##
## CRATE A LIST TO LINK dof AND NODES:
##
## number of dofs per node:
#n_dofs = 2
## conversion from node_id to dof: (node_id's start at 1)
#def t_dof(node_id):
    #return 0#n_dofs*(node_id-1)
#def x_dof(node_id):
    #return n_dofs*(node_id-1)
#def y_dof(node_id):
    #return n_dofs*(node_id-1)+1
##
##
##

## list of nodes and elements
#nd_lst = list(bvp.nodes.keys())
#el_lst = list(bvp.elements.keys())
##
#n_nds = nd_lst.__len__()

##solve for step 1
#nstep = 1
##
## current step:
#cstep = bvp.steps[nstep]
#SolvedStep = 0 # start of the step

## maximum number of increments

#inc_count = 0
#try:
    #maxinc = cstep['*inc']
#except:
    #maxinc = 1000

#dTime0 = cstep['*time'][0]
#Timescale = 1
#stepTime = cstep['*time'][1]

#time_format = bvp.time_format

#while (inc_count<maxinc)&(SolvedStep<stepTime):

    #inc_count += 1

    ### Time stamp (also used for internal state variables)
    #PrevTime = bvp.solved_time
    #time_prevs = time_format%PrevTime
    ## Time increment:
    #dTime = dTime0*Timescale
    #if SolvedStep + dTime>stepTime:
        #print('*** Time increment reduced to reach total step time')
        #dTime = stepTime-SolvedStep
    #sTime = SolvedStep + dTime
    #Time = bvp.solved_time+dTime
    #time_stamp = time_format%Time
    #print('** Attempting to solve for time at %s seconds'%time_stamp)
    ##
    ##
    ##
    ##
    ## All nodal Unknowns
    #U_all = np.zeros(n_nds*n_dofs)
    #dU_all = np.zeros(n_nds*n_dofs)
    ##
    ## initial values (from bvp.NodeVals)
    #if 'temperature' in bvp.NodeVals[time_prevs].keys():
        #prevTemp = bvp.NodeVals[time_prevs]['temperature']
        #for nd_Id in prevTemp.keys():
            #U_all[t_dof(nd_Id)] = prevTemp[nd_Id]

    #if 'displacement' in bvp.NodeVals[time_prevs].keys():
        #prevDisp = bvp.NodeVals[time_prevs]['displacement']
        #for nd_Id in prevDisp.keys():
            #nd_U = prevDisp[nd_Id]
            ## check if a list of values (i.e. x and y):
            #if type(nd_U) is list:
                #U_all[x_dof(nd_Id)],U_all[y_dof(nd_Id)] = nd_U

            #elif type(nd_U) is dict:
                #if 1 in prevDisp[nd_Id].keys():
                    #U_all[x_dof(nd_Id)] = prevDisp[nd_Id][1]
                #if 2 in prevDisp[nd_Id].keys():
                    #U_all[y_dof(nd_Id)] = prevDisp[nd_Id][2]
    ##
##
    ##
    ## at this time step, what is the known boundary condition:
    ##
    ##
    #U_known = []
    #U_k_idx = []
    ##
    ## first apply potential initial "fixed" / "homogeneous" case dofs:
    #if '*boundary' in bvp.steps[0].keys():
        #for nd_Id in bvp.steps[0]['*boundary'].keys():
            #for dof_nr in bvp.steps[0]['*boundary'][nd_Id].keys():
                ## values fixed at 0

                ##if dof_nr == 11: # Temperature:
                    ##U_k_idx += [t_dof(nd_Id)]
                    ##U_known += [scalef*fullvalue]
                #if dof_nr == 1: # X-dof:
                    #U_k_idx += [x_dof(nd_Id)]
                    #U_known += [0.]
                #if dof_nr == 2: # Y-dof:
                    #U_k_idx += [y_dof(nd_Id)]
                    #U_known += [0.]

    ## now for the particular step
    #if '*boundary' in cstep.keys():
        #for nd_Id in cstep['*boundary'].keys():
            #for dof_nr in cstep['*boundary'][nd_Id].keys():
                #fullvalue = cstep['*boundary'][nd_Id][dof_nr]['value']
                #ampname = cstep['*boundary'][nd_Id][dof_nr]['amplitude']
                ##check if there is an amplitude to take into account, else ramp
                #scalef = sTime/stepTime

                ##if dof_nr == 11: # Temperature:
                    ##U_k_idx += [t_dof(nd_Id)]
                    ##U_known += [scalef*fullvalue]
                #if dof_nr == 1: # X-dof:
                    #U_k_idx += [x_dof(nd_Id)]
                    #U_known += [scalef*fullvalue]
                #if dof_nr == 2: # Y-dof:
                    #U_k_idx += [y_dof(nd_Id)]
                    #U_known += [scalef*fullvalue]
    ##
    ##
    ##
    ## EXTERNAL FORCES:
    #F_known = []
    #F_k_idx = []
    #if '*cload' in cstep.keys():
        #for nd_Id in cstep['*cload'].keys():
            #for dof_nr in cstep['*cload'][nd_Id].keys():
                #fullvalue = cstep['*cload'][nd_Id][dof_nr]['value']
                #ampname = cstep['*cload'][nd_Id][dof_nr]['amplitude']
                ##check if there is an amplitude to take into account, else ramp
                #scalef = sTime/stepTime

                #if dof_nr == 1: # X-dof:
                    #F_k_idx += [x_dof(nd_Id)]
                    #F_known += [scalef*fullvalue]
                #if dof_nr == 2: # Y-dof:
                    #F_k_idx += [y_dof(nd_Id)]
                    #F_known += [scalef*fullvalue]
    ##

    ##
    ## initialise dictionary for ISVs
    #bvp.ISVs[time_format%Time] = {}
    ##
    ## initialise dictinary for nodal values:
    #bvp.NodeVals[time_format%Time] = {}
    ##
    ## for a specific section definition (loop over sections)
    ##section_name = 'eall'
    ##current_section = bvp.sections[section_name]
    ##current_material = bvp.materials[current_section['*material']]
    ##
    ##
    ##
    ##

    ## check only active dofs and free degrees of freedom:
    ##adof = [] # active degrees of freedom
    ##fdof = [] # free degrees of freedom
    ##for nd_Id in nd_lst:
        ##cdof = t_dof(nd_Id)
        ##if cdof in row_vec:
            ##adof+=[cdof]
            ##if cdof not in U_k_idx: fdof +=[cdof]

    #adof = [] # active degrees of freedom
    #fdof = [] # free degrees of freedom
    ##
    ##
    ##
    #print('\nSOLVE STATIC DISPLACEMENT')
    #U0 = np.matrix(U_all.reshape(-1,1))
    #dU = np.matrix(dU_all.reshape(-1,1))
    ## assign known values
    #dU[U_k_idx] = np.matrix(U_known).reshape(-1,1) - U0[U_k_idx]
    ##
    ## Residuals and counters
    #tol    = 1.e-6;
    #ResNrm,dUNrm = 1.,1.
    #ii   = -1
    #while (ResNrm>tol)|(dUNrm>tol):
        #tic = time()
        #ii += 1;
        ##Main loop over elements. Compute k_elem and assemble
        ##Initialize global stiffness matrix vectors;
        #row_vec   = []# np.zeros((TotKvec*nelem,1));
        #col_vec   = []#np.zeros((TotKvec*nelem,1);
        #stiff_vec = []#np.zeros(TotKvec*nelem,1);
        #Residual  = np.zeros((n_dofs*n_nds,1));
        ##Initialize global load vector
        #F_ext = np.zeros((n_dofs*n_nds,1));
        ##
        ##pos_vec = 0;
        ## loop over sections:
        #for section_name in bvp.sections.keys():
            #current_section = bvp.sections[section_name]
            #current_material = bvp.materials[current_section['*material']]
            ## loop over each element in the set
            #for el_nr in bvp.el_set[section_name]:
                ##
                ## element copnnectivity:
                #el_connect = bvp.elements[el_nr]
                ## Find reference coordinates of element nodes
                #X = [bvp.nodes[nd_Id][0] for nd_Id in el_connect]
                #Y = [bvp.nodes[nd_Id][1] for nd_Id in el_connect]
                #XY   = np.c_[X, Y];
                ##
                ## element internal state variables
                #try:
                    #EL_ISVs = bvp.ISVs[time_prevs][el_nr]
                #except:
                    #EL_ISVs = {}
                ## global dof's associated with element:
                ## for temperature:
                ##pg_temp = [t_dof(nd_Id) for nd_Id in el_connect]
                ## for displacement and temperature
                #pg_disp = []
                #Temp = []
                #for nd_Id in el_connect:
                    #pg_disp += [x_dof(nd_Id),y_dof(nd_Id)]
                    #try:
                        #Temp += [bvp.NodeVals[time_prevs]['temperature'][nd_Id]]
                    #except:
                        #Temp += [0.]
                ##
                ## GET CURRENT GUESS FOR NODAL DISPLACEMENTS:
                #U0_el = U0[pg_disp]
                #dU_el = dU[pg_disp]
                ##
                ## displacement residual:
                #[U_residual,U_tangent,EN_ISVs] = FE_element.Element_Disp(XY,U0_el,dU_el,Temp,dTime,EL_ISVs,bvp._el_type[el_nr],current_section,current_material,cstep['*nlgeom'])
                ##
                #bvp.ISVs[time_stamp][el_nr] = EN_ISVs
                ##
                ## Assemble residual
                #Residual[pg_disp,0] += U_residual;
                ## Assemble T_tangent into sparse k_global using vectors
                #col_vec += pg_disp*pg_disp.__len__()
                #row_vec += list(np.array([pg_disp]*pg_disp.__len__()).T.flatten())
                #stiff_vec += list(U_tangent)

        ##
        ##

        ## Assemble k_global from vectors
        #k_global = sparse.csr_matrix((stiff_vec,(row_vec,col_vec)),shape=(n_dofs*n_nds,n_dofs*n_nds));
        ##
        ## clear memory
        ##row_vec,col_vec,stiff_vec = [],[],[]

        #finish = time()-tic;
        #print(['Done assembling stiffness matrix: %.4f seconds.'%finish])

        #tic = time()

        ### check only active dofs:
        ##adof = [] # active degrees of freedom
        ##fdof = [] # free degrees of freedom
        #if fdof.__len__()==0:
            #adof = [] # active degrees of freedom
            #fdof = [] # free degrees of freedom
            #for nd_Id in nd_lst:
                #cdof = x_dof(nd_Id)
                #if cdof in row_vec:
                    #adof+=[cdof]
                    #if cdof not in U_k_idx: fdof +=[cdof]
                #cdof = y_dof(nd_Id)
                #if cdof in row_vec:
                    #adof+=[cdof]
                    #if cdof not in U_k_idx: fdof +=[cdof]
            #finish = time()-tic;
            #print(['Sorted through active and inactive degrees of freedom : %.4f seconds.'%finish])
            #tic = time()

        ##
        ## Add nodal loads to global load vector
        #F_ext[F_k_idx] = np.matrix(F_known).reshape(-1,1)

        ## Subtract internal nodal loads
        #F = Residual - F_ext;

        ## relative residual norm
        #ResNrm = np.linalg.norm(F[fdof,0]);
        #if ii == 0:
            #if ResNrm > 1e-4:
                #ResNrm0 = ResNrm;
            #else:
                #ResNrm0 = 1;
        #ResNrm = ResNrm/ResNrm0;

        #print('Normalized residual at start of iteration %i   = %10.6e'%(ii+1,ResNrm))

        ### Solve update of free dof's
        ### Solution for non-symmetric stiffness matrix

        #Kaa = k_global[fdof,:].T[fdof,:].T;
        #Pa  = F[fdof];

        #finish = time()-tic;
        #print('Done assembling stiffness matrix: %.4f seconds.'%finish)

        #tic = time();

        ## factorise: (incomplete LU):
        #Kaa_LU = sparse.linalg.spilu(Kaa.tocsc())
        ##
        ## solve using native solve?
        #deltaUf = -Kaa_LU.solve(Pa)
        ## gmres or something else?

        #finish = time()-tic;
        #print('Done solving system             : %.4f seconds.'%finish)

        #dUNrm = np.linalg.norm(deltaUf);
        #if ii == 0:
            #if dUNrm > 1e-4:
                #dUNrm0 = dUNrm;
            #else:
                #dUNrm0 = 1;
        #dUNrm = dUNrm/dUNrm0;
        #print('Normalized update to Unknowns            = %10.6e'%dUNrm)
        #print('                    --------------------')
        ## Sort Uf and Ub into A
        ##U0[fdof,0] += deltaUf[:,0]#;

        #dU[fdof] += deltaUf

        ##AllResNrm[ii] = ResNrm;
        ##AlldUNrm[ii]  = dUNrm;

    ### Get support reactions
    ###Fp = F[U_k_idx];

    ### update ISVs:
    ##ISVs = new_ISVs
    ##new_ISVs = {}

    ##print('Load increment %i converged after %i iterations.'%(iter_load+1,ii+1))
    ###All_iter(iter_load) = ii;
    ###All_soln(1+n_nds*(iter_load-1):n_nds*iter_load,:) = [U0(1:2:n_dofs*n_nds) U0(2:2:n_dofs*n_nds)];

    ##
    ## time step:
    #SolvedStep += dTime
    #bvp.solved_time += dTime
    #bvp.time_steps += [bvp.solved_time]

    ### save solution (node values to BVP dictionary
    ##nd_vals = bvp.NodeVals[time_stamp]['temperature'] = {}
    ##for nd_Id in bvp.nd_set['000']:
        ##dof = t_dof(nd_Id)
        ##if dof in adof:
            ##nd_vals[nd_Id] = U0[dof][0,0]

    ## add displacement to bvp nodal values
    #nd_vals = bvp.NodeVals[time_stamp]['displacement'] = {}
    #for nd_Id in bvp.nd_set['000']:
        #dof = x_dof(nd_Id)
        #x_disp = U0[dof][0,0]
        #if dof in adof: x_disp += dU[dof][0,0]
        #dof = y_dof(nd_Id)
        #y_disp = U0[dof][0,0]
        #if dof in adof: y_disp += dU[dof][0,0]
        ##
        #nd_vals[nd_Id] = [x_disp,y_disp]

    #if outputall:
        #FE_io.write_to_vtu(bvp,'results/'+filename0)

##if outputall:
    ##FE_io.write_to_vtu(bvp,filename0,bvp.NodeVals.keys())

##else:

#if not outputall:
    #FE_io.write_to_vtu(bvp,'results/'+filename0)


from metabolic_network import MetabolicNetwork
import matplotlib.pyplot as plot

class TP1:
    def perg_1(file_meta, file_reac, file_mat):
        mn = MetabolicNetwork('metabolite-reaction', {})
        mn.load_from_files(file_reac, file_meta, file_mat)
        return mn

    def perg_2(graph):
        reac=[]
        meta=[]
        graph_list = list(graph.g)
        for i in range(0, len(graph_list)):
            if(graph_list[i].startswith('R_')):
                reac.append(graph_list[i])
            elif(graph_list[i].startswith('M_')):
                meta.append(graph_list[i])
        return len(reac), len(meta)

    def perg_3(graph):
        meta={}
        graph_list = dict(graph.all_degrees())
        for k,v in graph_list.items():
            if(k.startswith('M_')):
                meta[k[2:]] = v
        sort = sorted(meta.items(), key=lambda x:x[1], reverse=True)
        return sort[:10]

    def perg_4(graph):
        d = {}
        graph_list = list(graph.g)
        for i in range(0, len(graph_list)):
            if(graph_list[i].startswith('R_')):
                degree = graph.degree(graph_list[i])
                if degree in d:
                    d[degree] = d[degree] + 1
                else:
                    d[degree] = 1
        plot.bar(range(len(d)), d.values(), align='center')
        plot.xticks(range(len(d)), d.keys())
        plot.show()

    def perg_5(graph):
        dead_ends = []
        graph_list = list(graph.g)
        for i in range(0, len(graph_list)):
            if(graph_list[i].startswith('M_')):
                if(graph.out_degree(graph_list[i])==0):
                    dead_ends.append(graph_list[i][2:])
        return dead_ends

    def perg_6(graph,uptake):
        suc= []
        uptake = input("Insira os valores pretendidos semparados por ';' (ex: M_2dhguln_c;M_maltttr_c) \n>")
        uptake =  uptake.split(";")
        metain = list(set(uptake))
        reacin = list(set([]))
        for i in range (0, len(metain)):
            reacin = graph.get_successors([metain[i]])
            for j in range (0, len (reacin)):
                if (graph.get_predecessors(reacin[j]) in metain):
                    metain = graph.get_successors(reacin[j])

    def perg_7(graph,meta):
        reac=[]
        graph_list = list(graph.g)
        for i in range(0, len(graph_list)):
            if(graph_list[i].startswith('R_')):
                if(meta in graph.get_successors(graph_list[i])):
                    reac.append(graph_list[i][2:])
        return reac

    def perg_8(graph):
        reac=set()
        graph_list = list(graph.g)
        for i in range(0, len(graph_list)):
            if(graph_list[i].startswith('R_')):
                for meta in graph.get_successors(graph_list[i]):
                    if(graph_list[i] in graph.get_successors(meta)):
                        reac.add(graph_list[i][2:])
        return reac

    def perg_9 (graph, orig, dest):
        paths = []
        past_elems = []
        orig = 'M_' + orig
        dest = 'M_' + dest
        if(orig != dest):
            if(graph.distance(orig,dest)>0):
                l = [(orig, [])]
                while(l):
                    cur, path = l.pop(0)
                    for elem in graph.g[cur]:
                        if(elem == dest):
                            paths.append([orig[2:]] + path + [elem[2:]])
                        elif(elem not in past_elems):
                            l.append((elem, path + [elem[2:]]))
                            past_elems.append(elem)
                return paths
            else: #nao existe caminho ate la
                return None
        else: #origem = destino
            return None

    def perg_10 (graph):
        ciclos = set()
        graph_list = list(graph.g)
        for i in range(0, len(graph_list)):
            if(graph.node_has_cycle(graph_list[i])):
                ciclos.add(graph_list[i][2:])
        return ciclos

    if __name__ == '__main__':
        while True:
            print()
            print('1. Cria um grafo com base na rede metabólica presente nos 3 ficheiros.')
            print('2. Qual o número de reações e metabolitos.')
            print('3. Identifica os 10 metabolitos que participam num maior número de reações.')
            print('4. Cria um gráfico com a distribuição do número de reações por cada um dos possíveis graus do tipo inout.')
            print('5. Identifica os metabolitos que são dead ends.')
            print('6. Considerando uma lista de metabolitos como uptake dado pelo utilizador, implementa uma função que retorna quais os produtos excretados pelo modelo.')
            print('7. Cria uma função que dado um metabolito dado pelo utilizador devolve a lista de reações que produzem esse metabolito.')
            print('8. Valida que no modelo não existem reações que contenham o mesmo metabolito como reagente e produto.')
            print('9. Implementa uma função que dado o metabolito origem e metabolito destino retorne todos os caminhos possíveis entre estes dois nós.')
            print('10. Implementa uma função que retorne a lista de ciclos existentes na rede.')
            print('0. Sair')
            escolha=input('Escolha a pergunta a executar: ')
            if escolha=='1':
                graph = perg_1('ijr904-metab.txt', 'ijr904-reac.txt', 'ijr904-matrix.txt')
                graph.print_graph()
            elif escolha=='2':
                graph = perg_1('ijr904-metab.txt', 'ijr904-reac.txt', 'ijr904-matrix.txt')
                print('Numero reações: ', perg_2(graph)[0])
                print('Numero metabolitos: ', perg_2(graph)[1])
            elif escolha=='3':
                graph = perg_1('ijr904-metab.txt', 'ijr904-reac.txt', 'ijr904-matrix.txt')
                print(perg_3(graph))
            elif escolha=='4':
                graph = perg_1('ijr904-metab.txt', 'ijr904-reac.txt', 'ijr904-matrix.txt')
                perg_4(graph)
            elif escolha=='5':
                graph = perg_1('ijr904-metab.txt', 'ijr904-reac.txt', 'ijr904-matrix.txt')
                print('Detetados ' , len(perg_5(graph)), ' deadends: ', perg_5(graph) )
            elif escolha=='6':
                graph = perg_1('ijr904-metab.txt', 'ijr904-reac.txt', 'ijr904-matrix.txt')
                perg_6(graph,'')
            elif escolha=='7':
                graph = perg_1('ijr904-metab.txt', 'ijr904-reac.txt', 'ijr904-matrix.txt')
                meta = input('Metabolito a pesquisar: ')
                print('Reações produtoras do metabolito: ' , perg_7(graph,meta))
            elif escolha=='8':
                graph = perg_1('ijr904-metab.txt', 'ijr904-reac.txt', 'ijr904-matrix.txt')
                print('Existem ', len(perg_8(graph)), ' reações que contem os mesmos metabolitos como reagente e produto:' , perg_8(graph))
            elif escolha=='9':
                graph = perg_1('ijr904-metab.txt', 'ijr904-reac.txt', 'ijr904-matrix.txt')
                meta_orig = input('Metabolito de origem: ')
                meta_dest = input('Metabolito de destino: ')
                paths = perg_9(graph,meta_orig,meta_dest)
                for i in range(len(paths)):
                    print("Caminho", end='')
                    for j in range(len(paths[i])):
                        print(' -> ' + paths[i][j], end='')
                    print()
            elif escolha=='10':
                graph = perg_1('ijr904-metab.txt', 'ijr904-reac.txt', 'ijr904-matrix.txt')
                ciclos = perg_10(graph)
                print('Existem ' , len(ciclos) , ' ciclos na rede: ', ciclos)
            elif escolha=='0':
                print('\n Adeus!')
                exit()