from __future__ import absolute_import
import argparse
import logging
import shutil
import sys
import tempfile
import six

from telemetry import benchmark
from telemetry import story
from telemetry.internal.browser import browser_options
from telemetry.internal.results import results_options
from telemetry.internal import story_runner
from telemetry.internal.util import binary_manager
from telemetry.page import legacy_page_test
from telemetry.util import matching
from telemetry.util import wpr_modes

from py_utils import discover
import py_utils

DEFAULT_LOG_FORMAT = (
    '(%(levelname)s) %(asctime)s %(module)s.%(funcName)s:%(lineno)d  '
    '%(message)s')

class RecorderPageTest(legacy_page_test.LegacyPageTest):
  def __init__(self, page_test):
    super().__init__()
    self._page_test = page_test
    self._platform = None

  @property
  def platform(self):
    return self._platform

  def CustomizeBrowserOptions(self, options):
    if self._page_test:
      self._page_test.CustomizeBrowserOptions(options)

  def WillStartBrowser(self, platform):
    if self.platform is not None:
      assert platform.GetOSName() == self.platform
    self._platform = platform.GetOSName()  # Record platform name from browser.
    if self._page_test:
      self._page_test.WillStartBrowser(platform)

  def DidStartBrowser(self, browser):
    if self._page_test:
      self._page_test.DidStartBrowser(browser)

  def WillNavigateToPage(self, page, tab):
    """Override to ensure all resources are fetched from network."""
    tab.ClearCache(force=False)
    if self._page_test:
      self._page_test.WillNavigateToPage(page, tab)

  def DidNavigateToPage(self, page, tab):
    if self._page_test:
      self._page_test.DidNavigateToPage(page, tab)
    tab.WaitForDocumentReadyStateToBeComplete()
    py_utils.WaitFor(tab.HasReachedQuiescence, 30)

  def CleanUpAfterPage(self, page, tab):
    if self._page_test:
      self._page_test.CleanUpAfterPage(page, tab)

  def ValidateAndMeasurePage(self, page, tab, results):
    if self._page_test:
      self._page_test.ValidateAndMeasurePage(page, tab, results)

def _GetSubclasses(base_dir, cls):
  """Returns all subclasses of |cls| in |base_dir|.

  Args:
    cls: a class

  Returns:
    dict of {underscored_class_name: benchmark class}
  """
  return discover.DiscoverClasses(base_dir, base_dir, cls,
                                  index_by_class_name=True)

def _MaybeGetInstanceOfClass(target, base_dir, cls):
  if isinstance(target, cls):
    return target
  classes = _GetSubclasses(base_dir, cls)
  return classes[target]() if target in classes else None

def _PrintAllImpl(all_items, item_name, output_stream):
  output_stream.write('Available %s\' names with descriptions:\n' % item_name)
  keys = sorted(all_items.keys())
  key_description = [(k, all_items[k].Description()) for k in keys]
  _PrintPairs(key_description, output_stream)
  output_stream.write('\n')

def _PrintAllBenchmarks(base_dir, output_stream):
  # TODO: reuse the logic of finding supported benchmarks in benchmark_runner.py
  # so this only prints out benchmarks that are supported by the recording
  # platform.
  _PrintAllImpl(_GetSubclasses(base_dir, benchmark.Benchmark), 'benchmarks',
                output_stream)

def _PrintAllStories(base_dir, output_stream):
  # TODO: actually print all stories once record_wpr support general
  # stories recording.
  _PrintAllImpl(_GetSubclasses(base_dir, story.StorySet), 'story sets',
                output_stream)

def _PrintPairs(pairs, output_stream, prefix=''):
  """Prints a list of string pairs with alignment."""
  first_column_length = max(len(a) for a, _ in pairs)
  format_string = '%s%%-%ds  %%s\n' % (prefix, first_column_length)
  for a, b in pairs:
    output_stream.write(format_string % (a, b.strip()))

class WprRecorder():

  def __init__(self, environment, target, args=None, parse_flags=True):
    self._base_dir = environment.top_level_dir
    self._output_dir = tempfile.mkdtemp()
    try:
      self._options = self._CreateOptions()
      self._benchmark = _MaybeGetInstanceOfClass(target, self._base_dir,
                                                 benchmark.Benchmark)
      if parse_flags:
        self._parser = self._options.CreateParser(usage='See %prog --help')
        self._AddCommandLineArgs()
        self._ParseArgs(args)
        self._ProcessCommandLineArgs(environment)
        self._SetExtraBrowserOptions()
      page_test = None
      if self._benchmark is not None:
        test = self._benchmark.CreatePageTest(self.options)
        # Object only needed for legacy pages; newer benchmarks don't need this.
        if isinstance(test, legacy_page_test.LegacyPageTest):
          page_test = test

      self._record_page_test = RecorderPageTest(page_test)
      options = self._options
      self._page_set_base_dir = (
          self._options.page_set_base_dir if
          (hasattr('options', 'page_set_base_dir') and
           options.page_set_base_dir)
          else self._base_dir)
      self._story_set = self._GetStorySet(target)
    except:
      self._CleanUp()
      raise

  def __enter__(self):
    return self

  def __exit__(self, *args):
    self._CleanUp()

  @property
  def options(self):
    return self._options

  @property
  def story_set(self):
    return self._story_set

  def _CreateOptions(self):
    options = browser_options.BrowserFinderOptions()
    options.browser_options.wpr_mode = wpr_modes.WPR_RECORD
    options.intermediate_dir = self._output_dir
    return options

  def _CleanUp(self):
    shutil.rmtree(self._output_dir)

  def CreateResults(self):
    if self._benchmark is not None:
      benchmark_name = self._benchmark.Name()
      benchmark_description = self._benchmark.Description()
    else:
      benchmark_name = 'record_wpr'
      benchmark_description = None

    return results_options.CreateResults(
        self._options,
        benchmark_name=benchmark_name,
        benchmark_description=benchmark_description,
        report_progress=True)

  def _AddCommandLineArgs(self):
    self._parser.add_option('--page-set-base-dir', action='store',
                            type='string')
    story_runner.AddCommandLineArgs(self._parser)
    if self._benchmark is not None:
      self._benchmark.AddCommandLineArgs(self._parser)
      self._benchmark.SetArgumentDefaults(self._parser)
    self._parser.add_option('--upload', action='store_true')
    self._parser.add_option('--use-local-wpr', action='store_true',
                            help='Builds and runs WPR from Catapult. '
                            'Also enables WPR debug output to STDOUT.')
    self._SetArgumentDefaults()

  def _SetArgumentDefaults(self):
    self._parser.set_defaults(output_formats=['none'])

  def _ParseArgs(self, args=None):
    args_to_parse = sys.argv[1:] if args is None else args
    self._parser.parse_args(args_to_parse)

  def _ProcessCommandLineArgs(self, environment):
    story_runner.ProcessCommandLineArgs(self._parser, self._options,
                                        environment)

    if self._options.use_live_sites:
      self._parser.error("Can't --use-live-sites while recording")

    if self._benchmark is not None:
      self._benchmark.ProcessCommandLineArgs(self._parser, self._options)

  def _SetExtraBrowserOptions(self):
    if self._benchmark is not None:
      self._benchmark.SetExtraBrowserOptions(self._options)

  def _GetStorySet(self, target):
    if self._benchmark is not None:
      return self._benchmark.CreateStorySet(self._options)
    story_set = _MaybeGetInstanceOfClass(target, self._page_set_base_dir,
                                         story.StorySet)
    if story_set is None:
      sys.stderr.write('Target %s is neither benchmark nor story set.\n'
                       % target)
      if not self._HintMostLikelyBenchmarksStories(target):
        sys.stderr.write(
            'Found no similar benchmark or story. Please use '
            '--list-benchmarks or --list-stories to list candidates.\n')
        self._parser.print_usage()
      sys.exit(1)
    return story_set

  def _HintMostLikelyBenchmarksStories(self, target):
    def _Impl(all_items, category_name):
      candidates = matching.GetMostLikelyMatchedObject(
          six.iteritems(all_items), target, name_func=lambda kv: kv[1].Name())
      if candidates:
        sys.stderr.write('\nDo you mean any of those %s below?\n' %
                         category_name)
        _PrintPairs([(k, v.Description()) for k, v in candidates], sys.stderr)
        return True
      return False

    has_benchmark_hint = _Impl(
        _GetSubclasses(self._base_dir, benchmark.Benchmark), 'benchmarks')
    has_story_hint = _Impl(
        _GetSubclasses(self._base_dir, story.StorySet), 'stories')
    return has_benchmark_hint or has_story_hint

  def Record(self, results):
    assert self._story_set.wpr_archive_info, (
        'Pageset archive_data_file path must be specified.')

    # Always record the benchmark one time only.
    self._options.pageset_repeat = 1
    self._story_set.wpr_archive_info.AddNewTemporaryRecording()
    self._record_page_test.CustomizeBrowserOptions(self._options)
    story_runner.RunStorySet(
        self._record_page_test,
        self._story_set,
        self._options,
        results)

  def HandleResults(self, results, upload_to_cloud_storage):
    if results.had_failures or results.had_skips:
      logging.warning('Some pages failed and/or were skipped. The recording '
                      'has not been updated for these pages.')
    results.Finalize()
    self._story_set.wpr_archive_info.AddRecordedStories(
        [run.story for run in results.IterStoryRuns() if run.ok],
        upload_to_cloud_storage,
        target_platform=self._record_page_test.platform)

def Main(environment, **log_config_kwargs):
  # the log level is set in browser_options
  log_config_kwargs.pop('level', None)
  log_config_kwargs.setdefault('format', DEFAULT_LOG_FORMAT)
  logging.basicConfig(**log_config_kwargs)

  parser = argparse.ArgumentParser(
      usage='Record a benchmark or a story (page set).')
  parser.add_argument(
      'benchmark_or_story_set', # Note, positional.
      help=('benchmark or story set name. This argument might be optional. '
            'If both benchmark name '
            'and story name are specified, this takes precedence as the '
            'target of the recording.'),
      nargs='?')
  parser.add_argument('--story', help='story (page set) name')
  parser.add_argument('--list-stories', dest='list_stories',
                      action='store_true', help='list all story names.')
  parser.add_argument('--list-benchmarks', dest='list_benchmarks',
                      action='store_true', help='list all benchmark names.')
  parser.add_argument('--upload', action='store_true',
                      help='upload to cloud storage.')

  args, extra_args = parser.parse_known_args()

  if args.list_benchmarks or args.list_stories:
    if args.list_benchmarks:
      _PrintAllBenchmarks(environment.top_level_dir, sys.stderr)
    if args.list_stories:
      _PrintAllStories(environment.top_level_dir, sys.stderr)
    return 0

  target = args.benchmark_or_story_set or args.story

  if not target:
    sys.stderr.write('Please specify target (benchmark or story). Please refer '
                     'usage below\n\n')
    parser.print_help()
    return 0

  binary_manager.InitDependencyManager(environment.client_configs)

  # TODO(crbug.com/1111556): update WprRecorder so that it handles the
  # difference between recording a benchmark vs recording a story better based
  # on the distinction between args.benchmark & args.story
  with WprRecorder(environment, target, extra_args) as wpr_recorder:
    results = wpr_recorder.CreateResults()
    wpr_recorder.Record(results)
    wpr_recorder.HandleResults(results, args.upload)
    return min(255, results.num_failed)


import time
import logging
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver import ActionChains
from selenium.webdriver.support.select import Select
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import StaleElementReferenceException
from utilities.utils import Utils

class TestReviewPage:
    log = Utils.custlogger(logLevel=logging.WARNING)

    def __init__(self, driver, wait):
        self.driver = driver
        self.wait = wait

    def test_sort(self, sortby, sort_id):
        #sort_id = 'wc-block-components-sort-select__select-0'
        self.wait.until(EC.element_to_be_clickable((By.ID, sort_id)))
        dropdown = self.driver.find_element(By.ID, sort_id)
        dd = Select(dropdown)
        dd.select_by_index(sortby)
        self.log.warning("Sorted the Reviews")
        time.sleep(5)

    def backtotop(self):
        backtotop = self.driver.find_element(By.XPATH, "//span[contains(text(),'Back to top')]")
        self.driver.execute_script("arguments[0].scrollIntoView();", backtotop)
        backtotop.click()
        self.log.warning("Moved to Top of page")

    def loadmorereviews(self):
        current_reviews = self.driver.find_elements(By.XPATH,
                                                    "//div[@class='main-content clear-fix boxed-wrapper']//li")
        loadmore = self.driver.find_element(By.XPATH, "//button[@class='wp-block-button__link']")
        while bool(loadmore):
            try:
                loadmore.click()
                loadmore = self.driver.find_element(By.XPATH, "//button[@class='wp-block-button__link']")
                self.driver.execute_script("arguments[0].scrollIntoView();", loadmore)
            except (NoSuchElementException, StaleElementReferenceException):
                loadmore = False
        self.log.warning("Loaded all reviews ")
        load_reviews = self.driver.find_elements(By.XPATH,
                                                 "//div[@class='main-content clear-fix boxed-wrapper']//li")
        list1 = [len(load_reviews), len(current_reviews)]
        return list1

    def review_checkbox(self):
        review_checkbox = self.driver.find_element(By.ID, "wp-comment-cookies-consent")
        review_checkbox.click()
        self.driver.execute_script("arguments[0].scrollIntoView();", review_checkbox)
        self.driver.implicitly_wait(10)
        checkbox = self.driver.find_element(By.ID, "wp-comment-cookies-consent").is_selected()
        return checkbox


import sys
import re
import argparse
import os
import json

regex_flags = re.MULTILINE | re.VERBOSE

syscall_regex = re.compile(r'''
__syscall\s+                    # __syscall attribute, must be first
([^(]+)                         # type and name of system call (split later)
[(]                             # Function opening parenthesis
([^)]*)                         # Arg list (split later)
[)]                             # Closing parenthesis
''', regex_flags)

struct_tags = ["__subsystem", "__net_socket"]

tagged_struct_decl_template = r'''
%s\s+                           # tag, must be first
struct\s+                       # struct keyword is next
([^{]+)                         # name of subsystem
[{]                             # Open curly bracket
'''

def tagged_struct_update(target_list, tag, contents):
    regex = re.compile(tagged_struct_decl_template % tag, regex_flags)
    items = [mo.groups()[0].strip() for mo in regex.finditer(contents)]
    target_list.extend(items)

def analyze_headers(multiple_directories):
    syscall_ret = []
    tagged_ret = {}

    for tag in struct_tags:
        tagged_ret[tag] = []

    for base_path in multiple_directories:
        for root, dirs, files in os.walk(base_path, topdown=True):
            dirs.sort()
            files.sort()
            for fn in files:

                # toolchain/common.h has the definitions of these tags which we
                # don't want to trip over
                path = os.path.join(root, fn)
                if (not (path.endswith(".h") or path.endswith(".c")) or
                        path.endswith(os.path.join(os.sep, 'toolchain',
                                                   'common.h'))):
                    continue

                with open(path, "r", encoding="utf-8") as fp:
                    contents = fp.read()

                try:
                    syscall_result = [(mo.groups(), fn)
                                      for mo in syscall_regex.finditer(contents)]
                    for tag in struct_tags:
                        tagged_struct_update(tagged_ret[tag], tag, contents)
                except Exception:
                    sys.stderr.write("While parsing %s\n" % fn)
                    raise

                syscall_ret.extend(syscall_result)

    return syscall_ret, tagged_ret

def update_file_if_changed(path, new):
    if os.path.exists(path):
        with open(path, 'r') as fp:
            old = fp.read()

        if new != old:
            with open(path, 'w') as fp:
                fp.write(new)
    else:
        with open(path, 'w') as fp:
            fp.write(new)

def parse_args():
    global args
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter, allow_abbrev=False)

    parser.add_argument("-i", "--include", required=True, action='append',
                        help='''include directories recursively scanned
                        for .h files.  Can be specified multiple times:
                        -i topdir1 -i topdir2 ...''')
    parser.add_argument(
        "-j", "--json-file", required=True,
        help="Write system call prototype information as json to file")
    parser.add_argument(
        "-t", "--tag-struct-file", required=True,
        help="Write tagged struct name information as json to file")

    args = parser.parse_args()

def main():
    parse_args()

    syscalls, tagged = analyze_headers(args.include)

    # Only write json files if they don't exist or have changes since
    # they will force an incremental rebuild.

    syscalls_in_json = json.dumps(
        syscalls,
        indent=4,
        sort_keys=True
    )
    update_file_if_changed(args.json_file, syscalls_in_json)

    tagged_struct_in_json = json.dumps(
        tagged,
        indent=4,
        sort_keys=True
    )
    update_file_if_changed(args.tag_struct_file, tagged_struct_in_json)

if __name__ == "__main__":
    main()


import pytest
from pathlib import Path
import tempfile
from django.conf import settings
from snoop import digest, models, pst
from snoop.content_types import guess_content_type

pytestmark = pytest.mark.skipif(not settings.SNOOP_READPST_BINARY,
    reason="SNOOP_READPST_BINARY not set")

PST_JANE_AND_DOE = {
    'parent': None,
    'path': "pst/flags_jane_doe.pst",
    'files': [
        "pst-test-2@aranetic.com/5.eml",
        "pst-test-2@aranetic.com/Inbox/1.eml",
        "pst-test-2@aranetic.com/Inbox/2.eml",
        "pst-test-2@aranetic.com/Inbox/3.eml",
        "pst-test-2@aranetic.com/Inbox/4.eml",
        "pst-test-2@aranetic.com/Inbox/5.eml",
        "pst-test-2@aranetic.com/Inbox/6.eml",
        "pst-test-2@aranetic.com/Sent Items/1.eml",
        "pst-test-2@aranetic.com/Sent Items/2.eml",
    ],
    'md5': '699502e6b8c005e0f2b3d523cf9479cc',
    'sha1': '7638280179f243ad10fcb4cd651bbfb5d979f5ab',
    'filename': 'flags_jane_doe.pst',
    'type': 'email-archive',
}

EMAIL_TWO = {
    'parent': PST_JANE_AND_DOE,
    'path': 'pst-test-2@aranetic.com/Inbox/2.eml',
    'md5': '46e5c10ee1603086e6fcf20acc4d1581',
    'sha1': '23e41ddfb84491bfc593fc7cc9d4b76b5a15e130',
    'type': 'email',
}

@pytest.fixture(autouse=True)
def no_models(monkeypatch):
    def disable(item, attr, default):
        def dummy(*a, **k):
            return default
        monkeypatch.setattr(item, attr, dummy)
    class DummyDocument:
        id = 0
    disable(models.Ocr.objects, 'filter', [])
    disable(models.Ocr.objects, 'all', [])
    disable(models.Document.objects, 'get_or_create', (DummyDocument(), False))

@pytest.yield_fixture(autouse=True)
def archive_dir(monkeypatch):
    with tempfile.TemporaryDirectory() as tmp:
        monkeypatch.setattr(pst, 'CACHE_ROOT', Path(tmp))
        yield

def doc_obj(obj, collection):
    filename = obj.get('filename') or obj['path'].split('/')[-1]
    content_type = guess_content_type(filename)
    doc = models.Document(
        path=obj['path'],
        content_type=content_type,
        filename=filename,
        collection=collection,
    )
    doc.save = lambda *a, **k: None
    if obj.get('sha1'):
        doc.sha1 = obj['sha1']
    if obj.get('md5'):
        doc.md5 = obj['md5']
    if obj.get('parent'):
        doc.container = doc_obj(obj['parent'], collection)
    return doc

def digest_obj(obj, collection):
    return digest.digest(doc_obj(obj, collection))

def assert_archive_consistence(obj, collection):
    data = digest_obj(obj, collection)
    print(data)
    keys = ['filename', 'sha1', 'md5', 'type', 'text']
    for key in keys:
        if key in obj:
            assert obj[key] == data[key]

def test_simple_pst_data(document_collection):
    assert_archive_consistence(PST_JANE_AND_DOE, document_collection)
    assert_archive_consistence(EMAIL_TWO, document_collection)

def test_pst_email(document_collection):
    data = digest_obj(EMAIL_TWO, document_collection)
    assert "This email has never been read." in data['text']


from transformers import AutoTokenizer, TFAutoModelForSeq2SeqLM

def translate(text: str):
    model_name = "hcho22/opus-mt-ko-en-finetuned-kr-to-en"
    #model_name = "../model/" #if running local
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = TFAutoModelForSeq2SeqLM.from_pretrained(model_name)

    input_text = text
    tokenized = tokenizer([input_text], return_tensors='np')
    out = model.generate(**tokenized, max_length=128)

    with tokenizer.as_target_tokenizer():
        result = tokenizer.decode(out[0], skip_special_tokens = True)

    return result


import os

from wagtail import __version__ as WAGTAIL_VERSION

PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

ALLOWED_HOSTS = ["*"]

DATABASES = {
    "default": {
        "NAME": "wagtailpurge-test.sqlite",
        "ENGINE": "django.db.backends.sqlite3",
    }
}

CACHES = {
    "default": {
        "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
        "LOCATION": "default-location",
    },
    "secondary": {
        "BACKEND": "django.core.cache.backends.dummy.DummyCache",
    },
    "tertiary": {
        "BACKEND": "django.core.cache.backends.dummy.DummyCache",
    },
}

TEMPLATES = [
    {
        "BACKEND": "django.template.backends.django.DjangoTemplates",
        "DIRS": [os.path.join(PROJECT_DIR, "templates")],
        "APP_DIRS": True,
        "OPTIONS": {
            "context_processors": [
                "django.template.context_processors.debug",
                "django.template.context_processors.request",
                "django.contrib.auth.context_processors.auth",
                "django.contrib.messages.context_processors.messages",
            ],
        },
    }
]

INSTALLED_APPS = [
    "wagtailpurge",
    "wagtailpurge.testapp",
    "modelcluster",
    "taggit",
    "django.contrib.auth",
    "django.contrib.contenttypes",
    "django.contrib.sessions",
    "django.contrib.messages",
    "django.contrib.staticfiles",
    "django.contrib.sites",
    "wagtail.admin",
    "wagtail.sites",
    "wagtail.search",
    "wagtail.users",
    "wagtail.images",
    "wagtail.documents",
    "wagtail.contrib.frontend_cache",
    "wagtail.contrib.modeladmin",
]
if WAGTAIL_VERSION >= "3":
    INSTALLED_APPS.append("wagtail")
else:
    INSTALLED_APPS.append("wagtail.core")

MIDDLEWARE = [
    "django.contrib.sessions.middleware.SessionMiddleware",
    "django.middleware.common.CommonMiddleware",
    "django.contrib.auth.middleware.AuthenticationMiddleware",
    "django.contrib.messages.middleware.MessageMiddleware",
]

ROOT_URLCONF = "wagtailpurge.testapp.urls"
WAGTAIL_SITE_NAME = "Wagtailpurge"
LOGIN_URL = "wagtailadmin_login"
LOGIN_REDIRECT_URL = "wagtailadmin_home"
SECRET_KEY = "fake-key"

# Django i18n
TIME_ZONE = "Europe/London"
USE_TZ = True

# Used in Wagtail emails
BASE_URL = "https://localhost:8000"

# Don't redirect to HTTPS in tests
SECURE_SSL_REDIRECT = False

# Use default static files storage for tests
STATICFILES_STORAGE = "django.contrib.staticfiles.storage.StaticFilesStorage"
STATIC_ROOT = os.path.join(PROJECT_DIR, "static")
STATIC_URL = "/static/"

# By default, Django uses a computationally difficult algorithm for passwords hashing.
# We don't need such a strong algorithm in tests, so use MD5
PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]

WAGTAILFRONTENDCACHE = {
    "dummy": {
        "BACKEND": "wagtailpurge.utils.DummyFECacheBackend",
        "LOCATION": "http://localhost:8000",
    },
}

WAGTAILIMAGES_IMAGE_MODEL = "testapp.CustomImage"


import glob

import click
import yamale

from atacac._utils import log
from atacac import validators

@click.command()
@click.option('-v', '--validators', 'custom_validators', multiple=True,
              help='Python scripts with additional validators')
@click.argument('assets_glob', envvar='ASSETS_GLOB')
@click.argument('assets_schema', envvar='ASSETS_SCHEMA', required=False)
def main(assets_glob, assets_schema, custom_validators):
    """
    Validate assets files.
    \f
    Validates the assets with rules defined in the schema.

    !! ASSETS_GLOB must be in single quotes to prevent glob pattern expansion in
    shell.

    \b
    Folloving arguments can be passed via environment variables:
        * ASSETS_GLOB
        * ASSETS_SCHEMA
    """
    # Run only if schema is set
    if assets_schema:
        schema = yamale.make_schema(assets_schema,
                                    validators=validators.load(custom_validators))

        for f in sorted(glob.glob(assets_glob, recursive=True)):
            log('INFO', f"Validating {f} against schema {assets_schema}")
            yamale.validate(schema, yamale.make_data(f))

        log('INFO', "... finished")

if __name__ == '__main__':
    # pylint: disable=unexpected-keyword-arg,no-value-for-parameter
    main()


from src.metric.CompoundMetric import CompoundMetric
from src.metric.SampleMetricManager import SampleMetricManager
from src.core.Setupable import SetupMode
from src.metric.CompoundMetricManager import CompoundMetricManager
from typing import Any
from cleanfid import fid
from src.population.Population import Population
from src.dataset.FFHQDataset import FFHQDataset
import numpy as np
from pathlib import Path

FID_NAME = "FID"
FID_CALC_MODES = ["clean", "legacy_tensorflow", "legacy_pytorch"]
FID_DEFAULT_CALC_MODE = "clean"

# TODO: Fix this:
#! Only works when equal to zero, gets pickle error otherwise
NUM_WORKERS = 0

class FIDCompoundMetric(CompoundMetric):
    def __init__(
        self,
        cmm: CompoundMetricManager,
        smm: SampleMetricManager = None,
    ):
        """
        Constructor for FIDCompoundMetric class, subclass of the CompoundMetric class.

        Args:
            cmm (CompoundMetricManager): Manager used by metrics. Population and dataset is derived
                from this manager.
            smm (SampleMetricManager, optional): Not used for this metric. Defaults to None.
        """
        super(FIDCompoundMetric, self).__init__(FID_NAME, cmm, smm)

        # Init storage structure for this metric
        self._fid = dict()

    def reg_setup_modes(self) -> dict[str, SetupMode]:
        ds = self.get_dataset()
        return {
            f"statistics_{fcm}_{ds.get_name(ds.get_resolution())}": SetupMode(
                True,
                lambda _, fcm=fcm: self._setup(fcm),
                lambda fcm=fcm: self._is_ready(fcm),
            )
            for fcm in FID_CALC_MODES
        }

    def _setup(self, calc_mode: str = FID_DEFAULT_CALC_MODE) -> None:
        """
        Setup the needed statistics to calculate the metric.

        Note that each dataset and `calc_mode` combination needs a calculated statistic.

        For more information regarding `calc_mode`, see the documentation for `calc()`.

        Args:
            calc_mode (str, optional): Calc mode determines FID implementation, different statistics
                needed for different implementations. See documentation on `calc()` for more information.
                Defaults to `FID_DEFAULT_CALC_MODE` ("clean").

        Raises:
            ValueError: Error when non-valid `calc_mode`, valid modes are defined by `FID_CALC_MODES`.
        """
        # Check calc_mode
        if calc_mode not in FID_CALC_MODES:
            raise ValueError(
                f"{calc_mode} not supported, supported modes: {FID_CALC_MODES}"
            )

        # Calculate custom statistics
        ds = self.get_dataset()
        fid.make_custom_stats(
            ds.get_name(ds.get_resolution()), str(ds.get_image_dir()), mode=calc_mode
        )

    def _is_ready(self, calc_mode=FID_DEFAULT_CALC_MODE) -> bool:
        """
        Checks if compound metric is ready for calculations.

        Args:
            calc_mode (str, optional): Calc mode determines FID implementation, different statistics
                needed for different implementations. See documentation on `calc()` for more information.
                Defaults to `FID_DEFAULT_CALC_MODE` ("clean").

        Raises:
            ValueError: Error when non-valid `calc_mode`, valid modes are defined by `FID_CALC_MODES`.

        Returns:
            bool: True if the compound metrics is ready for calculations.
        """
        # Check calc_mode
        if calc_mode not in FID_CALC_MODES:
            raise ValueError(
                f"{calc_mode} not supported, supported modes: {FID_CALC_MODES}"
            )

        ds = self.get_dataset()

        if type(ds).get_resolution_invariant_name() == "FFHQ" and (
            ds.get_resolution() == 256 or ds.get_resolution() == 1024
        ):
            # pre-computed statistic by clean-fid
            return True
        else:
            return fid.test_stats_exists(ds.get_name(), calc_mode)

    def _move_filtered_files(self, source_files: list[Path]) -> Path:
        # Look for temp name not taken
        while True:
            target = (
                Population.POPULATION_ROOT_DIR
                / f"temp_filtered_population{np.random.randint(10000,99999)}"
            )
            if not (target.is_file() or target.is_dir()):
                break

        # Create temp directory
        Path.mkdir(target)

        # Move filtered files to temp directory
        for image_file in source_files:
            image_file.rename(target.joinpath(image_file.name))

        return target

    def _move_filtered_files_back(self, source: Path) -> None:
        target = Population.POPULATION_ROOT_DIR / self.get_population().get_name()
        # Check if source file exists
        if not source.exists():
            raise FileNotFoundError(f"Could not find file: '{source.absolute()}'")

        # Check if target file exists
        if not target.exists():
            raise FileNotFoundError(f"Could not find file: '{target.absolute()}'")

        # Move filtered files back to population directory
        for image_file in source.glob("*"):
            image_file.rename(target.joinpath(image_file.name))

        # Remove temp directory
        source.rmdir()

    def calc(self, filter_bit: int = 1, **parameters: Any) -> Any:
        """
        Calculates the FID given the dataset and the population.

        No setup is needed for FFHQ 256/1024, if using other custom datasets `calc()`
        requires the user to run `setup()` first.

        Args:
            filter_bit (int, optional): Filter bit used to select a subset of the
                population. Defaults to 1 (IdentityFilter).
            calc_mode (str, optional): Either "clean", "legacy_tensorflow, or "legacy_pytorch".
                This decides how the FID score should be calculated, i.e., using clean-fid,
                regular tensorflow implementation, or pytorch implementation. Default is "clean" (clean-fid).
        Raises:
            ValueError: Error when non-valid `calc_mode`, valid modes are defined by `FID_CALC_MODES`.
            ValueError: Error when the name of the dataset in conjunction with the
                specified `calc_mode` don't have a pre-computed statistic.
            RuntimeError: When clean-fid library gets an error.
            FileNotFoundError: When incorrect path is provided when moving files.
        Returns:
            Any: The FID value.
        """
        # Fetch parameters
        calc_mode = self._check_calc_mode(parameters)

        # Move files to temp folder
        uris = self._population.get_filtered_data(filter_bit)[
            self._population.COLUMN_URI
        ]
        uris = [Path(str_path) for str_path in list(uris)]
        pop_path = self._move_filtered_files(uris)

        # Get variables for use in FID
        ds = self.get_dataset()
        resolution = ds.get_resolution()
        fid_score = None

        if type(ds).get_resolution_invariant_name() == "FFHQ" and (
            ds.get_resolution() == 256 or ds.get_resolution() == 1024
        ):
            # Use pre-computed statistic by clean-fid
            try:
                fid_score = fid.compute_fid(
                    str(pop_path),
                    dataset_name=type(ds).get_resolution_invariant_name(),
                    dataset_res=resolution,
                    mode=calc_mode,
                    dataset_split="trainval70k",
                    num_workers=NUM_WORKERS,
                )
            except Exception as error:
                self._move_filtered_files_back(pop_path)
                print("Something went wrong when calculating FID using clean-fid.")
                print(repr(error))
                raise
        else:
            # Use custom pre-computed statistic
            dataset_name = ds.get_name(resolution)

            # Check if statistic exists
            if fid.test_stats_exists(dataset_name, calc_mode):
                try:
                    fid_score = fid.compute_fid(
                        str(pop_path),
                        dataset_name=dataset_name,
                        mode=calc_mode,
                        dataset_split="custom",
                        num_workers=NUM_WORKERS,
                    )
                except Exception as error:
                    # Move back files
                    self._move_filtered_files_back(pop_path)
                    print("Something went when calculating FID using clean-fid.")
                    print(repr(error))
                    raise

            else:
                # Move back files
                self._move_filtered_files_back(pop_path)
                raise ValueError(
                    f"Statistic named '{dataset_name}' with `calc_mode` '{calc_mode}'"
                    " has no statistic. Double check `calc_mode` or run 'setup()'"
                )

        # Move back files
        self._move_filtered_files_back(pop_path)

        # Save result
        self._fid[calc_mode] = fid_score
        return fid_score

    def get(self, calc_if_missing: bool = False, **parameters: Any) -> Any:
        # Check parameters
        calc_mode = self._check_calc_mode(parameters)

        # Check if metric already calculated
        if calc_mode in self._fid.keys() and self._fid[calc_mode] is not None:
            return self._fid[calc_mode]

        # Check if calculate when missing
        elif calc_if_missing:
            return self.calc(**parameters)

        else:
            return None

    def print_result(self) -> None:
        for calc_mode, fid_score in self._fid.items():
            print(calc_mode + " FID: ", fid_score)

    def plot_result(self) -> None:
        pass

    def _check_calc_mode(self, parameters) -> str:
        # Fetch parameters
        if "calc_mode" in parameters.keys():
            calc_mode = parameters["calc_mode"]

            # Check calc_mode
            if calc_mode not in FID_CALC_MODES:
                raise ValueError(
                    f"{calc_mode} not supported, supported modes: {FID_CALC_MODES}"
                )
        else:
            calc_mode = FID_DEFAULT_CALC_MODE
        return calc_mode


import logging
from threading import Lock
from time import time

logging.basicConfig(format='%(message)s', filename="../result/GHS.log", level=logging.INFO, filemode='w')
lock = Lock()
def log(time, string):
    with lock:
        logging.info(f'{time:03.3f}) {string}')

class Channel:
    def __init__(self, node):
        self.node = node
        self.times = []
        self.messages = []
        self.mutex = Lock()

    def search(self, left, right, receiveTime):

        while right > left + 1:
            middle = (right + left) // 2

            if receiveTime >= self.times[middle - 1]:
                left = middle
            else:
                right = middle

        if right == left + 1 and receiveTime >= self.times[left]:
            return right

        return left

    def addToQueue(self, receiveTime, message, new=True):
        self.mutex.acquire()

        index = self.search(0, len(self.times), receiveTime)
        self.times.insert(index, receiveTime)
        self.messages.insert(index, message)

        self.mutex.release()

        if new:
            self.node.increaseMessage(message.getSize())
            log((time() - self.node.getTime()), f"node {self.node.getId()} {message}")

    def getMessage(self):

        if len(self.times) > 0 and self.times[0] <= time():
            self.mutex.acquire()
            message = self.messages.pop(0)
            self.times.pop(0)
            self.mutex.release()
            return message

        return None


r"""
Welcome to
   ___  _______   ____  __  _____
  / _ |/ ___/ /  / __ \/ / / / _ \
 / __ / /__/ /__/ /_/ / /_/ / // /
/_/ |_\___/____/\____/\____/____/

This a tool to create Android Virtual Devices locally/remotely.

- Prerequisites:
 The manual will be available at
 https://android.googlesource.com/platform/tools/acloud/+/master/README.md

- To get started:
 - Create instances:
    1) To create a remote cuttlefish instance with the local built image.
       Example:
       $ acloud create --local-image
       Or specify built image dir:
       $ acloud create --local-image /tmp/image_dir
    2) To create a local cuttlefish instance using the image which has been
       built out in your workspace.
       Example:
       $ acloud create --local-instance --local-image

 - Delete instances:
   $ acloud delete

 - Reconnect:
   To reconnect adb/vnc to an existing instance that's been disconnected:
   $ acloud reconnect
   Or to specify a specific instance:
   $ acloud reconnect --instance-names <instance_name like ins-123-cf-x86-phone>

 - List:
   List will retrieve all the remote instances you've created in addition to any
   local instances created as well.
   To show device IP address, adb port and instance name:
   $ acloud list
   To show more detail info on the list.
   $ acloud list -vv

-  Pull:
   Pull will download log files or show the log file in screen from one remote
   cuttlefish instance:
   $ acloud pull
   Pull from a specified instance:
   $ acloud pull --instance-name "your_instance_name"

Try $acloud [cmd] --help for further details.

"""

from __future__ import print_function
import argparse
import logging
import os
import platform
import sys
import sysconfig
import traceback

# TODO: Remove this once we switch over to embedded launcher.
# Exit out if python version is < 2.7.13 due to b/120883119.
if (sys.version_info.major == 2
        and sys.version_info.minor == 7
        and sys.version_info.micro < 13):
    print("Acloud requires python version 2.7.13+ (currently @ %d.%d.%d)" %
          (sys.version_info.major, sys.version_info.minor,
           sys.version_info.micro))
    print("Update your 2.7 python with:")
    # pylint: disable=invalid-name
    os_type = platform.system().lower()
    if os_type == "linux":
        print("  apt-get install python2.7")
    elif os_type == "darwin":
        print("  brew install python@2 (and then follow instructions at "
              "https://docs.python-guide.org/starting/install/osx/)")
        print("  - or -")
        print("  POSIXLY_CORRECT=1 port -N install python27")
    sys.exit(1)
# This is a workaround to put '/usr/lib/python3.X' ahead of googleapiclient of
# build system path list to fix python3 issue of http.client(b/144743252)
# that googleapiclient existed http.py conflict with python3 build-in lib.
# Using embedded_launcher(b/135639220) perhaps work whereas it didn't solve yet.
if sys.version_info.major == 3:
    sys.path.insert(0, os.path.dirname(sysconfig.get_paths()['purelib']))

# By Default silence root logger's stream handler since 3p lib may initial
# root logger no matter what level we're using. The acloud logger behavior will
# be defined in _SetupLogging(). This also could workaround to get rid of below
# oauth2client warning:
# 'No handlers could be found for logger "oauth2client.contrib.multistore_file'
DEFAULT_STREAM_HANDLER = logging.StreamHandler()
DEFAULT_STREAM_HANDLER.setLevel(logging.CRITICAL)
logging.getLogger().addHandler(DEFAULT_STREAM_HANDLER)

# pylint: disable=wrong-import-position
from acloud import errors
from acloud.create import create
from acloud.create import create_args
from acloud.delete import delete
from acloud.delete import delete_args
from acloud.internal import constants
from acloud.reconnect import reconnect
from acloud.reconnect import reconnect_args
from acloud.list import list as list_instances
from acloud.list import list_args
from acloud.metrics import metrics
from acloud.powerwash import powerwash
from acloud.powerwash import powerwash_args
from acloud.public import acloud_common
from acloud.public import config
from acloud.public import report
from acloud.public.actions import create_cuttlefish_action
from acloud.public.actions import create_goldfish_action
from acloud.pull import pull
from acloud.pull import pull_args
from acloud.restart import restart
from acloud.restart import restart_args
from acloud.setup import setup
from acloud.setup import setup_args

LOGGING_FMT = "%(asctime)s |%(levelname)s| %(module)s:%(lineno)s| %(message)s"
ACLOUD_LOGGER = "acloud"
_LOGGER = logging.getLogger(ACLOUD_LOGGER)
NO_ERROR_MESSAGE = ""
PROG = "acloud"
_ACLOUD_CONFIG_ERROR = "ACLOUD_CONFIG_ERROR"

# Commands
CMD_CREATE_CUTTLEFISH = "create_cf"
CMD_CREATE_GOLDFISH = "create_gf"

# Config requires fields.
_CREATE_REQUIRE_FIELDS = ["project", "zone", "machine_type"]
_CREATE_CF_REQUIRE_FIELDS = ["resolution"]
# show contact info to user.
_CONTACT_INFO = ("If you have any question or need acloud team support, "
                 "please feel free to contact us by email at "
                 "buganizer-system+419709@google.com")
_LOG_INFO = " and attach those log files from %s"

# pylint: disable=too-many-statements
def _ParseArgs(args):
    """Parse args.

    Args:
        args: Argument list passed from main.

    Returns:
        Parsed args.
    """
    usage = ",".join([
        setup_args.CMD_SETUP,
        create_args.CMD_CREATE,
        list_args.CMD_LIST,
        delete_args.CMD_DELETE,
        reconnect_args.CMD_RECONNECT,
        pull_args.CMD_PULL,
        restart_args.CMD_RESTART,
    ])
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter,
        usage="acloud {" + usage + "} ...")
    parser = argparse.ArgumentParser(prog=PROG)
    parser.add_argument('--version', action='version', version=(
        '%(prog)s ' + config.GetVersion()))
    subparsers = parser.add_subparsers(metavar="{" + usage + "}")
    subparser_list = []

    # Command "create_cf", create cuttlefish instances
    create_cf_parser = subparsers.add_parser(CMD_CREATE_CUTTLEFISH)
    create_cf_parser.required = False
    create_cf_parser.set_defaults(which=CMD_CREATE_CUTTLEFISH)
    create_args.AddCommonCreateArgs(create_cf_parser)
    subparser_list.append(create_cf_parser)

    # Command "create_gf", create goldfish instances
    # In order to create a goldfish device we need the following parameters:
    # 1. The emulator build we wish to use, this is the binary that emulates
    #    an android device. See go/emu-dev for more
    # 2. A system-image. This is the android release we wish to run on the
    #    emulated hardware.
    create_gf_parser = subparsers.add_parser(CMD_CREATE_GOLDFISH)
    create_gf_parser.required = False
    create_gf_parser.set_defaults(which=CMD_CREATE_GOLDFISH)
    create_gf_parser.add_argument(
        "--emulator_build_id",
        type=str,
        dest="emulator_build_id",
        required=False,
        help="Emulator build used to run the images. e.g. 4669466.")
    create_gf_parser.add_argument(
        "--emulator_branch",
        type=str,
        dest="emulator_branch",
        required=False,
        help="Emulator build branch name, e.g. aosp-emu-master-dev. If specified"
        " without emulator_build_id, the last green build will be used.")
    create_gf_parser.add_argument(
        "--base_image",
        type=str,
        dest="base_image",
        required=False,
        help="Name of the goldfish base image to be used to create the instance. "
        "This will override stable_goldfish_host_image_name from config. "
        "e.g. emu-dev-cts-061118")
    create_gf_parser.add_argument(
        "--tags",
        dest="tags",
        nargs="*",
        required=False,
        default=None,
        help="Tags to be set on to the created instance. e.g. https-server.")

    create_args.AddCommonCreateArgs(create_gf_parser)
    subparser_list.append(create_gf_parser)

    # Command "create"
    subparser_list.append(create_args.GetCreateArgParser(subparsers))

    # Command "setup"
    subparser_list.append(setup_args.GetSetupArgParser(subparsers))

    # Command "delete"
    subparser_list.append(delete_args.GetDeleteArgParser(subparsers))

    # Command "list"
    subparser_list.append(list_args.GetListArgParser(subparsers))

    # Command "reconnect"
    subparser_list.append(reconnect_args.GetReconnectArgParser(subparsers))

    # Command "restart"
    subparser_list.append(restart_args.GetRestartArgParser(subparsers))

    # Command "powerwash"
    subparser_list.append(powerwash_args.GetPowerwashArgParser(subparsers))

    # Command "pull"
    subparser_list.append(pull_args.GetPullArgParser(subparsers))

    # Add common arguments.
    for subparser in subparser_list:
        acloud_common.AddCommonArguments(subparser)

    if not args:
        parser.print_help()
        sys.exit(constants.EXIT_BY_WRONG_CMD)

    return parser.parse_args(args)

# pylint: disable=too-many-branches
def _VerifyArgs(parsed_args):
    """Verify args.

    Args:
        parsed_args: Parsed args.

    Raises:
        errors.CommandArgError: If args are invalid.
        errors.UnsupportedCreateArgs: When a create arg is specified but
                                      unsupported for a particular avd type.
                                      (e.g. --system-build-id for gf)
    """
    if parsed_args.which == create_args.CMD_CREATE:
        create_args.VerifyArgs(parsed_args)
    if parsed_args.which == setup_args.CMD_SETUP:
        setup_args.VerifyArgs(parsed_args)
    if parsed_args.which == CMD_CREATE_CUTTLEFISH:
        if not parsed_args.build_id and not parsed_args.branch:
            raise errors.CommandArgError(
                "Must specify --build_id or --branch")
    if parsed_args.which == CMD_CREATE_GOLDFISH:
        if not parsed_args.emulator_build_id and not parsed_args.build_id and (
                not parsed_args.emulator_branch and not parsed_args.branch):
            raise errors.CommandArgError(
                "Must specify either --build_id or --branch or "
                "--emulator_branch or --emulator_build_id")
        if not parsed_args.build_target:
            raise errors.CommandArgError("Must specify --build_target")
        if (parsed_args.system_branch
                or parsed_args.system_build_id
                or parsed_args.system_build_target):
            raise errors.UnsupportedCreateArgs(
                "--system-* args are not supported for AVD type: %s"
                % constants.TYPE_GF)

    if parsed_args.which in [
            create_args.CMD_CREATE, CMD_CREATE_CUTTLEFISH, CMD_CREATE_GOLDFISH
    ]:
        if (parsed_args.serial_log_file
                and not parsed_args.serial_log_file.endswith(".tar.gz")):
            raise errors.CommandArgError(
                "--serial_log_file must ends with .tar.gz")

def _ParsingConfig(args, cfg):
    """Parse config to check if missing any field.

    Args:
        args: Namespace object from argparse.parse_args.
        cfg: AcloudConfig object.

    Returns:
        error message about list of missing config fields.
    """
    missing_fields = []
    if args.which == create_args.CMD_CREATE and args.local_instance is None:
        missing_fields = cfg.GetMissingFields(_CREATE_REQUIRE_FIELDS)
    if args.which == CMD_CREATE_CUTTLEFISH:
        missing_fields.extend(cfg.GetMissingFields(_CREATE_CF_REQUIRE_FIELDS))
    if missing_fields:
        return "Missing required configuration fields: %s" % missing_fields
    return None

def _SetupLogging(log_file, verbose):
    """Setup logging.

    This function define the logging policy in below manners.
    - without -v , -vv ,--log_file:
    Only display critical log and print() message on screen.

    - with -v:
    Display INFO log and set StreamHandler to acloud parent logger to turn on
    ONLY acloud modules logging.(silence all 3p libraries)

    - with -vv:
    Display INFO/DEBUG log and set StreamHandler to root logger to turn on all
    acloud modules and 3p libraries logging.

    - with --log_file.
    Dump logs to FileHandler with DEBUG level.

    Args:
        log_file: String, if not None, dump the log to log file.
        verbose: Int, if verbose = 1(-v), log at INFO level and turn on
                 logging on libraries to a StreamHandler.
                 If verbose = 2(-vv), log at DEBUG level and turn on logging on
                 all libraries and 3rd party libraries to a StreamHandler.
    """
    # Define logging level and hierarchy by verbosity.
    shandler_level = None
    logger = None
    if verbose == 0:
        shandler_level = logging.CRITICAL
        logger = logging.getLogger(ACLOUD_LOGGER)
    elif verbose == 1:
        shandler_level = logging.INFO
        logger = logging.getLogger(ACLOUD_LOGGER)
    elif verbose > 1:
        shandler_level = logging.DEBUG
        logger = logging.getLogger()

    # Add StreamHandler by default.
    shandler = logging.StreamHandler()
    shandler.setFormatter(logging.Formatter(LOGGING_FMT))
    shandler.setLevel(shandler_level)
    logger.addHandler(shandler)
    # Set the default level to DEBUG, the other handlers will handle
    # their own levels via the args supplied (-v and --log_file).
    logger.setLevel(logging.DEBUG)

    # Add FileHandler if log_file is provided.
    if log_file:
        fhandler = logging.FileHandler(filename=log_file)
        fhandler.setFormatter(logging.Formatter(LOGGING_FMT))
        fhandler.setLevel(logging.DEBUG)
        logger.addHandler(fhandler)

def main(argv=None):
    """Main entry.

    Args:
        argv: A list of system arguments.

    Returns:
        Job status: Integer, 0 if success. None-zero if fails.
        Stack trace: String of errors.
    """
    args = _ParseArgs(argv)
    _SetupLogging(args.log_file, args.verbose)
    _VerifyArgs(args)
    _LOGGER.info("Acloud version: %s", config.GetVersion())

    cfg = config.GetAcloudConfig(args)
    parsing_config_error = _ParsingConfig(args, cfg)
    # TODO: Move this check into the functions it is actually needed.
    # Check access.
    # device_driver.CheckAccess(cfg)

    reporter = None
    if parsing_config_error:
        reporter = report.Report(command=args.which)
        reporter.UpdateFailure(parsing_config_error, _ACLOUD_CONFIG_ERROR)
    elif args.which == create_args.CMD_CREATE:
        reporter = create.Run(args)
    elif args.which == CMD_CREATE_CUTTLEFISH:
        reporter = create_cuttlefish_action.CreateDevices(
            cfg=cfg,
            build_target=args.build_target,
            build_id=args.build_id,
            branch=args.branch,
            kernel_build_id=args.kernel_build_id,
            kernel_branch=args.kernel_branch,
            kernel_build_target=args.kernel_build_target,
            system_branch=args.system_branch,
            system_build_id=args.system_build_id,
            system_build_target=args.system_build_target,
            bootloader_branch=args.bootloader_branch,
            bootloader_build_id=args.bootloader_build_id,
            bootloader_build_target=args.bootloader_build_target,
            gpu=args.gpu,
            num=args.num,
            serial_log_file=args.serial_log_file,
            autoconnect=args.autoconnect,
            report_internal_ip=args.report_internal_ip,
            boot_timeout_secs=args.boot_timeout_secs,
            ins_timeout_secs=args.ins_timeout_secs)
    elif args.which == CMD_CREATE_GOLDFISH:
        reporter = create_goldfish_action.CreateDevices(
            cfg=cfg,
            build_target=args.build_target,
            build_id=args.build_id,
            emulator_build_id=args.emulator_build_id,
            branch=args.branch,
            emulator_branch=args.emulator_branch,
            kernel_build_id=args.kernel_build_id,
            kernel_branch=args.kernel_branch,
            kernel_build_target=args.kernel_build_target,
            gpu=args.gpu,
            num=args.num,
            serial_log_file=args.serial_log_file,
            autoconnect=args.autoconnect,
            tags=args.tags,
            report_internal_ip=args.report_internal_ip,
            boot_timeout_secs=args.boot_timeout_secs)
    elif args.which == delete_args.CMD_DELETE:
        reporter = delete.Run(args)
    elif args.which == list_args.CMD_LIST:
        list_instances.Run(args)
    elif args.which == reconnect_args.CMD_RECONNECT:
        reconnect.Run(args)
    elif args.which == restart_args.CMD_RESTART:
        reporter = restart.Run(args)
    elif args.which == powerwash_args.CMD_POWERWASH:
        reporter = powerwash.Run(args)
    elif args.which == pull_args.CMD_PULL:
        reporter = pull.Run(args)
    elif args.which == setup_args.CMD_SETUP:
        setup.Run(args)
    else:
        error_msg = "Invalid command %s" % args.which
        sys.stderr.write(error_msg)
        return constants.EXIT_BY_WRONG_CMD, error_msg

    if reporter and args.report_file:
        reporter.Dump(args.report_file)
    if reporter and reporter.errors:
        error_msg = "\n".join(reporter.errors)
        help_msg = _CONTACT_INFO
        if reporter.data.get(constants.ERROR_LOG_FOLDER):
            help_msg += _LOG_INFO % reporter.data.get(constants.ERROR_LOG_FOLDER)
        sys.stderr.write("Encountered the following errors:\n%s\n\n%s.\n" %
                         (error_msg, help_msg))
        return constants.EXIT_BY_FAIL_REPORT, error_msg
    return constants.EXIT_SUCCESS, NO_ERROR_MESSAGE

if __name__ == "__main__":
    EXIT_CODE = None
    EXCEPTION_STACKTRACE = None
    EXCEPTION_LOG = None
    LOG_METRICS = metrics.LogUsage(sys.argv[1:])
    try:
        EXIT_CODE, EXCEPTION_STACKTRACE = main(sys.argv[1:])
    except Exception as e:
        EXIT_CODE = constants.EXIT_BY_ERROR
        EXCEPTION_STACKTRACE = traceback.format_exc()
        EXCEPTION_LOG = str(e)
        raise
    finally:
        # Log Exit event here to calculate the consuming time.
        if LOG_METRICS:
            metrics.LogExitEvent(EXIT_CODE,
                                 stacktrace=EXCEPTION_STACKTRACE,
                                 logs=EXCEPTION_LOG)


from __future__ import print_function
import numpy as np
import tensorflow as tf
from causal_controller.CausalController import CausalController
from tqdm import trange
import os
import pandas as pd

from utils import make_summary,distribute_input_data,get_available_gpus
from utils import save_image

from data_loader import DataLoader
from figure_scripts.pairwise import crosstab

class Trainer(object):

    def __init__(self, config, cc_config, model_config=None):
        self.config=config
        self.cc_config=cc_config
        self.model_dir = config.model_dir
        self.cc_config.model_dir=config.model_dir

        self.model_config=model_config
        if self.model_config:
            self.model_config.model_dir=config.model_dir

        self.save_model_dir=os.path.join(self.model_dir,'checkpoints')
        if not os.path.exists(self.save_model_dir):
            os.mkdir(self.save_model_dir)

        self.summary_dir=os.path.join(self.model_dir,'summaries')
        if not os.path.exists(self.summary_dir):
            os.mkdir(self.summary_dir)

        self.load_path = config.load_path
        self.use_gpu = config.use_gpu

        #This tensor controls batch_size for all models
        #Not expected to change during training, but during testing it can be
        #helpful to change it

        self.batch_size=tf.placeholder_with_default(self.config.batch_size,[],name='batch_size')

        loader_batch_size=config.num_devices*config.batch_size

        #Always need to build CC
        print('setting up CausalController')
        cc_batch_size=config.num_devices*self.batch_size#Tensor/placeholder
        self.cc=CausalController(cc_batch_size,cc_config)
        self.step=self.cc.step

        #Data
        print('setting up data')
        self.data=DataLoader(self.cc.label_names,config)

        if self.cc_config.is_pretrain or self.config.build_pretrain:
            print('setup pretrain')
            #queue system to feed labels quickly. This does not queue images
            label_queue= self.data.get_label_queue(loader_batch_size)
            self.cc.build_pretrain(label_queue)

        #Build Model
        if self.model_config:
            #Will build both gen and discrim
            self.model=self.config.Model(self.batch_size,self.model_config)

            #Trainer step is defined as cc.step+model.step
            #e.g. 10k iter pretrain and 100k iter image model
            #will have image summaries at 100k but trainer model saved at Model-110k
            self.step+=self.model.step

            # This queue holds (image,label) pairs, and is used for training conditional GANs
            data_queue=self.data.get_data_queue(loader_batch_size)

            self.real_data_by_gpu = distribute_input_data(data_queue,config.num_gpu)
            self.fake_data_by_gpu = distribute_input_data(self.cc.label_dict,config.num_gpu)

            with tf.variable_scope('tower'):
                for gpu in get_available_gpus():
                    print('using device:',gpu)

                    real_data=self.real_data_by_gpu[gpu]
                    fake_data=self.fake_data_by_gpu[gpu]
                    tower=gpu.replace('/','').replace(':','_')

                    with tf.device(gpu),tf.name_scope(tower):
                        #Build num_gpu copies of graph: inputs->gradient
                        #Updates self.tower_dict
                        self.model(real_data,fake_data)

                    #allow future gpu to use same variables
                    tf.get_variable_scope().reuse_variables()

            if self.model_config.is_train or self.config.build_train:
                self.model.build_train_op()
                self.model.build_summary_op()

        else:
            print('Image model not built')

        self.saver = tf.train.Saver(keep_checkpoint_every_n_hours=2)
        self.summary_writer = tf.summary.FileWriter(self.summary_dir)

        print('trainer.model_dir:',self.model_dir)
        gpu_options = tf.GPUOptions(allow_growth=True,
                                  per_process_gpu_memory_fraction=0.333)
        sess_config = tf.ConfigProto(allow_soft_placement=True,
                                    gpu_options=gpu_options)

        sv = tf.train.Supervisor(
                                logdir=self.save_model_dir,
                                is_chief=True,
                                saver=self.saver,
                                summary_op=None,
                                summary_writer=self.summary_writer,
                                save_model_secs=300,
                                global_step=self.step,
                                ready_for_local_init_op=None
                                )
        self.sess = sv.prepare_or_wait_for_session(config=sess_config)

        if cc_config.pt_load_path:
            print('Attempting to load pretrain model:',cc_config.pt_load_path)
            self.cc.load(self.sess,cc_config.pt_load_path)

            print('Check tvd after restore')
            info=crosstab(self,report_tvd=True)
            print('tvd after load:',info['tvd'])

            #save copy of cc model in new dir
            cc_step=self.sess.run(self.cc.step)
            self.cc.saver.save(self.sess,self.cc.save_model_name,cc_step)

        if config.load_path:#Declare loading point
            pnt_str='Loaded variables at ccStep:{}'
            cc_step=self.sess.run(self.cc.step)
            pnt_str=pnt_str.format(cc_step)
            print('pntstr',pnt_str)
            if self.model_config:
                pnt_str+=' imagemodelStep:{}'
                model_step=self.sess.run
                pnt_str=pnt_str.format(model_step)
            print(pnt_str)

        #PREPARE training:
        #TODO save as Variables so they are restored to same values when load model
        fixed_batch_size=256 #get this many fixed z values

        self.fetch_fixed_z={n.z:n.z for n in self.cc.nodes}
        if model_config:
            self.fetch_fixed_z[self.model.z_gen]=self.model.z_gen

        #feed_dict that ensures constant inputs
        #add feed_fixed_z[self.cc.Male.label]=1*ones() to intervene
        self.feed_fixed_z=self.sess.run(self.fetch_fixed_z,{self.batch_size:fixed_batch_size})

    def pretrain_loop(self,num_iter=None):
        '''
        num_iter : is the number of *additional* iterations to do
        baring one of the quit conditions (the model may already be
        trained for some number of iterations). Defaults to
        cc_config.pretrain_iter.

        '''
        #TODO: potentially should be moved into CausalController for consistency

        num_iter = num_iter or self.cc.config.pretrain_iter

        if hasattr(self,'model'):
            model_step=self.sess.run(self.model.step)
            assert model_step==0,'if pretraining, model should not be trained already'

        cc_step=self.sess.run(self.cc.step)
        if cc_step>0:
            print('Resuming training of already optimized CC model at\
                  step:',cc_step)

        label_stats=crosstab(self,report_tvd=True)

        def break_pretrain(label_stats,counter):
            c1=counter>=self.cc.config.min_pretrain_iter
            c2= (label_stats['tvd']<self.cc.config.min_tvd)
            return (c1 and c2)

        for counter in trange(cc_step,cc_step+num_iter):
            #Check for early exit
            if counter %(10*self.cc.config.log_step)==0:
                label_stats=crosstab(self,report_tvd=True)
                print('ptstep:',counter,'  TVD:',label_stats['tvd'])
                if break_pretrain(label_stats,counter):
                    print('Completed Pretrain by TVD Qualification')
                    break

            #Optimize critic
            self.cc.critic_update(self.sess)

            #one iter causal controller
            fetch_dict = {
                "pretrain_op": self.cc.train_op,
                'cc_step':self.cc.step,
                'step':self.step,
            }

            #update what to run
            if counter % self.cc.config.log_step == 0:
                fetch_dict.update({
                    "summary": self.cc.summary_op,
                    "c_loss": self.cc.c_loss,
                    "dcc_loss": self.cc.dcc_loss,
                })
            result = self.sess.run(fetch_dict)

            #update summaries
            if counter % self.cc.config.log_step == 0:
                if counter %(10*self.cc.config.log_step)==0:
                    sum_tvd=make_summary('misc/tvd', label_stats['tvd'])
                    self.summary_writer.add_summary(sum_tvd,result['cc_step'])

                self.summary_writer.add_summary(result['summary'],result['cc_step'])
                self.summary_writer.flush()

                c_loss = result['c_loss']
                dcc_loss = result['dcc_loss']
                print("[{}/{}] Loss_C: {:.6f} Loss_DCC: {:.6f}".\
                      format(counter, cc_step+ num_iter, c_loss, dcc_loss))

            if counter %(10*self.cc.config.log_step)==0:
                self.cc.saver.save(self.sess,self.cc.save_model_name,result['cc_step'])

        else:
            label_stats=crosstab(self,report_tvd=True)
            self.cc.saver.save(self.sess,self.cc.save_model_name,self.cc.step)
            print('Completed Pretrain by Exhausting all Pretrain Steps!')

        print('step:',result['cc_step'],'  TVD:',label_stats['tvd'])

    def train_loop(self,num_iter=None):
        '''
        This is a function for handling the training of either CausalBEGAN or
        CausalGAN models. The python function Model.train_step() is called
        num_iter times and some general image save features: intervening,
        conditioning, etc are done here too.
        '''
        num_iter=num_iter or self.model_config.num_iter

        #Train loop
        print('Entering train loop..')
        for counter in trange(num_iter):

            self.model.train_step(self.sess,counter)

            #scalar and histogram summaries
            if counter % self.config.log_step == 0:
                step,summ=self.sess.run([self.model.step,self.model.summary_op])
                self.summary_writer.add_summary(summ,step)
                self.summary_writer.flush()

            #expensive summaries
            if counter % (self.config.log_step * 50) == 0:
                self.causal_sampling([8,16])
                self.label_interpolation()
                self.sample_diversity()

            #more rare events
            if counter % (self.config.log_step * 100) == 0:
                self.causal_sampling([2,10])

    ##Wrapper methods
    def sample_label(self, cond_dict=None, do_dict=None,N=None):
        return self.cc.sample_label(self.sess,cond_dict=cond_dict,do_dict=do_dict,N=N)
    ##

    ##Sampling and figure methods
    def label_interpolation(self,inputs=None,save_dir=None,ext='.pdf'):
        '''
        Holding all other inputs the same, move a causal controller
        labels between 0 and 1. Recalculate the downstream effects to capture the causal effect.

        For each label, this makes an 8x8 image with each row being
        an instance of z_fixed with varying label
        '''

        interpolation_dir=os.path.join(self.model_dir,'label_interpolation')
        save_dir=save_dir or interpolation_dir
        if not os.path.exists(save_dir):
            os.mkdir(save_dir)

        inputs=inputs or {}

        #use the first 8 values
        #contrasting np.repeat and np.tile to get all combinations
        fixed_z=inputs or {k:np.repeat(v[:8],8,axis=0) for k,v in self.feed_fixed_z.items()}
        setval=np.tile(np.linspace(0,1,8),8).reshape([64,1])

        fixed_z.update({self.batch_size:64})
        save_name='{}/{}_G_interp_{}'+ext

        #make 8x8 image
        for node in self.cc.nodes:

            fd=fixed_z.copy()
            fd[node.label]=setval
            images,step=self.sess.run([self.model.G,self.model.step],fd)
            interp_path=save_name.format(save_dir,step,node.name)
            save_image(images,interp_path,nrow=8)

        out_str="[*] Interpolation Samples saved: "+save_name
        print(save_name.format(save_dir,step,'*'))

    def causal_sampling(self, img_shape ,ext='.pdf'):
        '''
        sampling new noise inputs each time, draw samples from
        interventional distributions.
        Recalculate downstream effects given a label value

        img_shape must have rows divisible by 2

        This function implements the following three sampling techniques:
        1) Images where
            Top half is sampled from the intervention do(label=1)
            Bottom half is sampled from the intervention do(label=0)
        2) Images where
            Top half is sampled from the intervention do(label=1/0)
            Bottom half is sampled conditioned on |label = 1/0
        3) Image where
            Top half is sampled conditioned on |label = 1
            Bottom half is sampled conditioned on |label = 0
        '''

        assert len(img_shape)==2,'2d shape for output'
        assert img_shape[0]%2==0,'should have equal top and bot half'

        shape_str='_'+'x'.join(map(str,img_shape))

        #sample given(Label=1/0)
        conditioning_dir=os.path.join(self.model_dir,'label_conditioning')
        if not os.path.exists(conditioning_dir):
            os.mkdir(conditioning_dir)

        #sample do(Label=1/0)
        intervention_dir=os.path.join(self.model_dir,'label_intervention')
        if not os.path.exists(intervention_dir):
            os.mkdir(intervention_dir)

        #sample do(Label=1)/given(Label=1)
        #sample do(Label=0)/given(Label=0)
        intv_v_conditioning_dir=os.path.join(self.model_dir,'label_intv_v_conditioning')
        if not os.path.exists(intv_v_conditioning_dir):
            os.mkdir(intv_v_conditioning_dir)

        save_name_cond =os.path.join(conditioning_dir,'{}_condition_{}'+shape_str+ext)
        save_name_intv =os.path.join(intervention_dir,'{}_interv_{}'+shape_str+ext)
        save_name_intvcond=os.path.join(intv_v_conditioning_dir,'{}_intvcond_{}={}'+shape_str+ext)

        half_shape=[img_shape[0]//2, img_shape[1]]
        N=np.prod(half_shape)

        for name in self.cc.node_names:
            #First sample labels (two step more efficient)
            #ex:{'Male':1}
            c0=self.sample_label(cond_dict={name:0},N=N)
            c1=self.sample_label(cond_dict={name:1},N=N)
            d0=self.sample_label(do_dict=  {name:0},N=N)
            d1=self.sample_label(do_dict=  {name:1},N=N)

            feed_c0={self.cc.label_dict[k]:v for k,v in c0.iteritems()}
            feed_c1={self.cc.label_dict[k]:v for k,v in c1.iteritems()}
            feed_d0={self.cc.label_dict[k]:v for k,v in d0.iteritems()}
            feed_d1={self.cc.label_dict[k]:v for k,v in d1.iteritems()}

            feed_c0[self.batch_size]=N
            feed_c1[self.batch_size]=N
            feed_d0[self.batch_size]=N
            feed_d1[self.batch_size]=N

            step=self.sess.run(self.model.step)
            c0_images=self.sess.run(self.model.G,feed_c0)
            c1_images=self.sess.run(self.model.G,feed_c1)
            d0_images=self.sess.run(self.model.G,feed_d0)
            d1_images=self.sess.run(self.model.G,feed_d1)

            save_path_cond      = save_name_cond.format(step,name)
            save_path_intv      = save_name_intv.format(step,name)
            save_path_intvcond0 = save_name_intvcond.format(step,name,0)
            save_path_intvcond1 = save_name_intvcond.format(step,name,1)

            #saveimage fills row by row from top left
            save_image(np.concatenate([c1_images,c0_images]),save_path_cond,nrow=img_shape[0])
            save_image(np.concatenate([d1_images,d0_images]),save_path_intv,nrow=img_shape[0])
            save_image(np.concatenate([d0_images,c0_images]),save_path_intvcond0,nrow=img_shape[0])
            save_image(np.concatenate([d1_images,c1_images]),save_path_intvcond1,nrow=img_shape[0])

        print("[*] Conditioning Samples saved: "+conditioning_dir)
        print("[*] Intervention Samples saved: "+intervention_dir)
        print("[*] Intervention vs Condition Samples saved: "+intv_v_conditioning_dir)

    def sample_diversity(self,save_dir=None,ext='.pdf'):
        '''
        This is to make a 16x16 image from fixed inputs
        to examine the image diversity over time
        '''
        #Make 16x16 image
        nrow=16
        diversity_dir=os.path.join(self.model_dir,'image_diversity')
        save_dir=save_dir or diversity_dir
        if not os.path.exists(save_dir):
            os.mkdir(save_dir)
        save_name=os.path.join(save_dir,'{}_G_diversity'+ext)

        feed_fixed={k:v[:256] for k,v in self.feed_fixed_z.items()}
        feed_fixed.update({self.batch_size:256})

        step,images = self.sess.run([self.model.step,self.model.G], feed_dict=feed_fixed)

        print('image shape',images.shape)

        save_path=save_name.format(step)
        save_image(images, save_path, nrow=nrow)
        print("[*] Diversity Sample saved: {}".format(save_path))


import numpy as np
from matplotlib import pyplot as plt
from matplotlib import animation
from time import time

class Particle:
    def __init__(self, dt, r0, b, v0):
        self.m = 1
        self.alpha = -1000

        r = np.sqrt(b*b + r0*r0)
        theta = np.pi-np.arctan(b/r0)

        self.dt = dt
        self.r = r
        self.theta = theta
        self.r_ = v0 * np.cos(theta)
        self.theta_ = -v0 * np.sin(theta) / r

    def getCoords(self):
        return [self.r * np.cos(self.theta), self.r * np.sin(self.theta)]

    def timestep(self):
        r = self.r
        r_ = self.r_
        m = self.m
        alpha = self.alpha
        th = self.theta
        th_ = self.theta_
        r__ = alpha / (m*r*r) + r * th_ * th_
        th__ = -2*r_*th_/r

        self.r_ += r__ * self.dt
        self.r += self.r_ * self.dt
        self.theta_ += th__ * self.dt
        self.theta += self.theta_ * self.dt

def main():
    fig = plt.figure()
    ax = plt.axes(xlim=(-50, 50), ylim=(-50, 50), aspect='equal')
    ax.grid()
    linep, = ax.plot([], [], lw=1)

    ptcl = Particle(0.001, 0.00001, 20, 7.1)

    x = []
    y = []
    def init():
        linep.set_data([], [])
        return linep,

    def animate(i):
        ptcl.timestep()
        coords = ptcl.getCoords()
        x.append(coords[0])
        y.append(coords[1])
        linep.set_data(x, y)
        return linep,
    t0 = time()
    animate(0)
    t1 = time()
    intv = ptcl.dt * 1000 - (t1-t0)
    anim = animation.FuncAnimation(fig, animate, init_func=init, frames=300, interval=intv, blit=True)
    plt.show()

main()


from typing import Any
from airflow.models import BaseOperator
from airflow.utils.context import Context
from airflow.utils.decorators import apply_defaults
from custom.Movielens_hook import MovielensHook
import os
import json

class MovielensFetchRatingsOperator(BaseOperator):
    template_fields = ("_start_date", "_end_date", "_output_path")

    @apply_defaults
    def __init__(
        self,
        conn_id,
        output_path,
        start_date="{{ds}}",
        end_date="{{next_ds}}",
        **kwargs,
    ):
        super(MovielensFetchRatingsOperator, self).__init__(**kwargs)

        self._conn_id = conn_id
        self._output_path = output_path
        self._start_date = start_date
        self._end_date = end_date

    def execute(self, context: Context) -> Any:
        hook = MovielensHook(self._conn_id)

        try:
            self.log.info(
                f"Fetching ratings for {self._start_date} to {self._end_date}"
            )
            ratings = list(
                hook.get_ratings(start_date=self._start_date, end_date=self._end_date)
            )
            self.log.info(f"Fetched {len(ratings)} ratings")
        finally:
            hook.close()

        self.log.info(f"Writing ratings to {self._output_path}")
        output_dir = os.path.dirname(self._output_path)
        os.makedirs(output_dir, exist_ok=True)

        with open(self._output_path, "w") as file_:
            json.dump(ratings, fp=file_)


import tkinter as tk
import requests as rq
from bs4 import BeautifulSoup

search = 'Weather in São Bernardo do Campo'
url = f'https://www.google.com/search?&q={search}'

r = rq.get(url)
s = BeautifulSoup(r.text,'html.parser')
update = s.find('div',class_='BNeawe').text

class MinhaGUI:
    def __init__(self):
        self.main_window = tk.Tk()
        self.main_window.minsize(60,50)
        self.main_window.title('Wheater Today!')
        self.label1 = tk.Label(self.main_window,text= '   Olá Jhonatas, essa é previsão do tempo hoje!   ')
        self.label2 = tk.Label(self.main_window,text= update)
        self.label1.pack()
        self.label2.pack()
        tk.mainloop()

minha_gui = MinhaGUI()


from flask import Flask, request
from test import getsepal
from flask_cors import CORS, cross_origin

app = Flask("__name__")
cors = CORS(app)
app.config['CORS_HEADERS'] = 'Content-Type'

@app.route("/", methods=["GET"])
@cross_origin()
def get():
    return "hello"

@app.route("/model", methods=["POST"])
@cross_origin()
def slr():
    s = request.json["x"]
    r = getsepal(s)
    return str(r)

if __name__ == "__main__":
    print("server running")
    app.run(debug=True)


import scrapy

from scrapy.loader import ItemLoader
from ..items import AprilanoItem
from itemloaders.processors import TakeFirst

class AprilanoSpider(scrapy.Spider):
	name = 'aprilano'
	start_urls = ['https://www.aprila.no/no/newsfeed?hsLang=no']

	def parse(self, response):
		post_links = response.xpath('//article/a/@href').getall()
		yield from response.follow_all(post_links, self.parse_post)

	def parse_post(self, response):
		title = response.xpath('//h1/span/text()').get()
		description = response.xpath('//article//text()[normalize-space() and not(ancestor::header)]').getall()
		description = [p.strip() for p in description]
		description = ' '.join(description).strip()
		date = response.xpath('//time/text()').get()

		item = ItemLoader(item=AprilanoItem(), response=response)
		item.default_output_processor = TakeFirst()
		item.add_value('title', title)
		item.add_value('description', description)
		item.add_value('date', date)

		return item.load_item()


#!/usr/bin/env python3
import sounddevice as sd
import os
import sys
from tkinter import *
import pathlib

def resource_path(relative_path):
    """ Get absolute path to resource, works for dev and for PyInstaller """
    try:
        # PyInstaller creates a temp folder and stores path in _MEIPASS
        base_path = sys._MEIPASS
    except Exception:
        base_path = os.path.abspath(".")

    return os.path.join(base_path, relative_path)

# Create main window
mainWindow =Tk()
mainWindow.geometry("500x100")
mainWindow.title("formuls")
mainWindow.configure(bg='green')

# Query the available audio devices
audioOptions = sd.query_devices()
deviceName = []
devConcat = str
x = 0
for i in audioOptions:
    devConcat = str(x) + " " + (i['name'])
    deviceName.append(devConcat)
    x = x + 1

#Access the Menu Widget using StringVar function
clicked = StringVar()
clicked.set("Select audio output...") # set optionMenu initial value

#Create an instance of Menu in the frame
audioDeviceMenu = OptionMenu(mainWindow, clicked, *deviceName)
audioDeviceMenu.pack()
audioDeviceMenu.configure(bg='green')
audioDeviceMenu.place(relx = 0.2, rely = 0.5, anchor = 'center')

# function to access the selected menu item, then run formulsengine and node/open stage control
runx = 1
def RUN():
    deviceGet = clicked.get()
    audioOutput = deviceGet[0]
    # command0 = "cd " + resource_path("")
    command1 = resource_path("gui/node") + " " + resource_path("gui/open-stage-control/") + " -s 127.0.0.1:9000 -o 9001 -l " + resource_path("gui/_main.json") + " --client-options framerate=25 hdpi=0 &" # run o-s-c interface
    command2 = "cd " + resource_path("") + "; " + resource_path("pd/formulsengine") + " 1 " + audioOutput + " 16 &" # audio process
    command3 = "cd " + resource_path("") + "; " + resource_path("pd/formulsengine") + " 0 " + audioOutput + " 1 &" # control process
    # os.system(command0)
    os.system(command1)
    os.system(command2)
    os.system(command3)
    os.system("pwd")

# function to stop formulsengines and node/open stage control
def STOP():
    os.system("killall formulsengine")
    os.system("killall node")

# function to kill all subprocesses and exit the program
def QUIT():
    os.system("killall formulsengine")
    os.system("killall node")
    sys.exit()

# load images for run, stop and quit buttons
runicon = PhotoImage(file = resource_path("icons/run.png"))
runicon = runicon.subsample(2,2)
stopicon = PhotoImage(file = resource_path("icons/stop.png"))
stopicon = stopicon.subsample(2,2)
quiticon = PhotoImage(file = resource_path("icons/quit.png"))
quiticon = quiticon.subsample(3,3)

# create run, stop and quit buttons
runButton = Button(mainWindow, fg = 'green', border = 0, image = runicon, command=RUN)
runButton.pack()
runButton.place(relx = 0.5, rely = 0.5, anchor = 'center')

stopButton = Button(mainWindow, fg = 'green', border = 0, image = stopicon, command=STOP)
stopButton.pack()
stopButton.place(relx = 0.62, rely = 0.5, anchor = 'center')

quitButton = Button(mainWindow, fg = 'green', border = 0, image = quiticon, command=QUIT)
quitButton.pack()
quitButton.place(relx = 0.9, rely = 0.5, anchor = 'center')

mainWindow.mainloop()


import json
from nltk.stem.snowball import SnowballStemmer
from collections import OrderedDict
import time
from operator import itemgetter
from nltk.corpus import stopwords

def singleWordSearch(searchInput):
 try:
   docs=[]
   docs1=[]
   #  if the word exist in our data then show the docs
   searchQuery= ss.stem(searchInput)

   start=time.time()
   if searchQuery in titlelexFileData:
    #  if the word is in title
       wordID1=0
       wordID1= titlelexFileData[searchQuery]
      #  print(searchQuery)
       numberOfDocuments1=titleinvertedFileData[str(wordID1)]
       docs1=OrderedDict(sorted(numberOfDocuments1.items(), key=lambda item: len(item[1]), reverse=True))
       for i in docs1:

        dataArray1=(metaData[str(i)])
        print("Title")
        print(dataArray1[0])
        print("URL")
        print(dataArray1[1])

   if searchQuery in lexFileData:
    #  if the word is in content
      wordID=0

      wordID = lexFileData[searchQuery]

      numberOfDocuments=invertedFileData[str(wordID)]
      docs = OrderedDict(sorted(numberOfDocuments.items(), key=lambda item: len(item[1]), reverse=True))
      # print(docs)
      # print(docs1)

      for i in docs:
        if i not in docs1:
          # The document shouldn't be first printed
         dataArray=(metaData[str(i)])
         print("Title")
         print(dataArray[0])
         print("URL")
         print(dataArray[1])
      end = time.time()
      print(end-start)

 except:
   #  Else show that the words didn't exist in our data
    print("The word didn't exist in our data")

def scoreMaker(hitlists):
  print(hitlists)
  # Returning the length of the hits

  return (len(hitlists))

def multiWordSearch(search):
  try:
      docList=[]
      for i in search:
        # print("helo")
        if i in titlelexFileData:

        # if the word didn't find in lexicon then it would show the exception
          search_word_ids1 = [titlelexFileData[word] for word in search if ((word not in stop_words))]
          # Having the word Id's from lexicon of the word entered by user
          inverted_index_entries1 = [titleinvertedFileData[str(word_id)] for word_id in search_word_ids1 if word_id != -1]
          # Having the indexes and docs from inverted Index

          docs_with_score1 = []
          # iie is for inverted index entries (IIE)
          for ind, iie in enumerate(inverted_index_entries1):
            # Looping the inverted index enteries
            for doc1 in iie:
              # Traversing the docs in dictionary as it will have the format {docID:[indexes]}

              current_doc_hitlists1 = [iie[doc1]]

              # having the indexes of current document in the loop in current doc hitlist

              for remaining_iie1 in inverted_index_entries1[ind + 1:]:
                # for the next document to check the gap between words
                if doc1 in remaining_iie1:
                  # if the doc is in in the remaining inverted index entries
                  current_doc_hitlists1.append(remaining_iie1[doc1])
                  # so append the hilist with that index of the next doc
                  del remaining_iie1[doc1]
                  # and delete the doc value so that we are done with that doc
              # print(current_doc_hitlists1)
              docs_with_score1.append((doc1,scoreMaker(current_doc_hitlists1)))

            # if the two words are in the same doc it will append the indexes of two docs in one list
            # Appending score with that doc so that we can make sure the search quality
              sorted(docs_with_score1, key=lambda x: x[1])
              docs_with_score1.sort(key=lambda x: x[1], reverse=True)
              # print(docs_with_score1)
              # Sorting it on the base of the score(descending order)
              # print(docs_with_score1)
              # print(1)
              # print(docs_with_score1)
          for index, values in enumerate(docs_with_score1):
            # print(index)
            docList.append(docs_with_score1[index][0])

            # print(values)
          # print(docList)
          for i in docs_with_score1:
              #  if i not in docs_with_score1:
                # print(i)
                dataArray1=(metaData[str(i[0])])
                # print(dataArray1)
                print("Title")
                print(dataArray1[0])
                print("URL")
                print(dataArray1[1])
                # break
          break
      for i in search:

        if i in lexFileData:

         # if the word didn't find in lexicon then it would show the exception
         search_word_ids = [lexFileData[word] for word in search if ((word not in stop_words))]
         # Having the word Id's from lexicon of the word entered by user
         inverted_index_entries = [invertedFileData[str(word_id)] for word_id in search_word_ids if word_id != -1]
         # Having the indexes and docs from inverted Index

         docs_with_score = []
         # iie is for inverted index entries (IIE)
         print(inverted_index_entries)
         for i, iie in enumerate(inverted_index_entries):
           # Looping the inverted index enteries
           for doc in iie:
             # Traversing the docs in dictionary as it will have the format {docID:[indexes]}

             current_doc_hitlists = [iie[doc]]

             # having the indexes of current document in the loop in current doc hitlist

             for remaining_iie in inverted_index_entries[i + 1:]:
               # for the next document to check the gap between words
               if doc in remaining_iie:
                 # if the doc is in in the remaining inverted index entries
                 current_doc_hitlists.append(remaining_iie[doc])
                 # so append the hilist with that index of the next doc
                 del remaining_iie[doc]
                 # and delete the doc value so that we are done with that doc
            #  print(current_doc_hitlists)
            #  print(doc)
             docs_with_score.append((doc,scoreMaker(current_doc_hitlists)))
            #  scoreMaker(current_doc_hitlists)
             # if the two words are in the same doc it will append the indexes of two docs in one list
             # Appending score with that doc so that we can make sure the search quality
         sorted(docs_with_score, key=lambda x: x[1])
         docs_with_score.sort(key=lambda x: x[1], reverse=True)
         # Sorting it on the base of the score(descending order)

        # print(docs_with_score1)
        for i in docs_with_score:
          # print(docs_with_score1)

          if i[0] not in docList:
          #  print(i)
           dataArray=(metaData[str(i[0])])

           print("Title")
           print(dataArray[0])
           print("URL")
           print(dataArray[1])
        break
  except:
      print("The word is not in our data set!")

ss = SnowballStemmer("english")
stop_words = set(stopwords.words("english"))

lexiconFile = open("lexicon.json", "r")
# forwardIndexFile = open("forwardIndex.json", "r")
reverseIndexFile = open("reverseIndex.json", "r")
metaDataFile = open("metaData.json","r")
titleLexiconFile = open("titleLexicon.json", "r")
# forwardIndexFile = open("forwardIndex.json", "r")
titleReverseIndexFile = open("titleReverseIndex.json", "r")

lexFileData= json.load(lexiconFile)

# forwardFileData= json.load(forwardIndexFile)

invertedFileData= json.load(reverseIndexFile)
titlelexFileData= json.load(titleLexiconFile)

# forwardFileData= json.load(forwardIndexFile)

titleinvertedFileData= json.load(titleReverseIndexFile)
# print(titlelexFileData)
# print(titleinvertedFileData)

metaData = json.load(metaDataFile)
innerDict = {}
docList=[]
searchInput= input("Enter the word to search in document\n")

searchInput=searchInput.lower()
# Converting the word into lower case alphabets
search= searchInput.split()
# Splittting the word so that it would be easy to go for mulit and single word approach
for s in range(len(search)):
  search[s]=ss.stem(search[s])
  # Converting all the words in their stemmed form to process in multiword Query

if(len(search)>  1):

        multiWordSearch(search)

   # Multiword Search Query will be if it is greater than length 1
else:
   # Else it will be a single word
   singleWordSearch(searchInput)


from collections import defaultdict

class Solution:
    def majorityElement(self, nums: List[int]) -> int:
        length = len(nums)
        if length%2:
            maj_count = length//2 + 1
        else:
            maj_count = length//2

        number_count = defaultdict(int)
        for index in range(length):
            number_count[nums[index]] += 1
            if number_count[nums[index]] >= maj_count:
                return nums[index]


from visionManager.visionModule import KinematicModule
from newbie_hanuman.msg import suchartPostDictMsg, suchartVisionMsg

from newbie_hanuman.msg import SuchartFeedback

from geometry_msgs.msg import Polygon, Point32

import rospy

import numpy as np
import cv2

import sys

PATH = sys.path[0]

def getJsPosFromName(Js, name):
	indexJs = Js.name.index(name)
	return Js.position[indexJs]

class Kinematic(KinematicModule):
	# OFF1 = 7.6
	OFF1 = 7.6
	L1 = 0.2
	L2 = 0.3
	H1 = 0.6
	H2 = 0.5
	def __init__(self):
		super(Kinematic, self).__init__()

		path = "/".join(PATH.split("/")[:]+["output_matrix.npz"])
		intrinMat = np.load(path)["camera_matrix"]
		self.set_IntrinsicCameraMatrix(intrinMat)

		self.objectsMsgType = suchartVisionMsg
		self.posDictMsgType = suchartPostDictMsg

		self.motorCortexTopicName = "/spinalcord/suchart_status"
		self.motorCortexMsgType = SuchartFeedback

		## Add plane
		## TODO : re-create plane after known a exact position of robot base.

		tranVec = np.array([-2.5,0,0.25],float)
		rotVec = np.array([0,0,0],float)
		Hplane1 = self.create_transformationMatrix(tranVec, rotVec, 'zyz')

		self.add_plane(	"ground", Hplane1,
						(0,10), (-5,5), (-1,1))

		tranVec = np.array([-2.5,4.75,5],float)
		rotVec = np.array([-np.pi/2,np.pi/2,np.pi/2],float)
		Hplane2 = self.create_transformationMatrix(tranVec, rotVec, 'zyz')

		self.add_plane(	"left", Hplane2,
						(0,10), (-5,5), (-1,1))

		tranVec = np.array([-2.5,-4.75,5],float)
		rotVec = np.array([-np.pi/2,-np.pi/2,np.pi/2],float)
		Hplane3 = self.create_transformationMatrix(tranVec, rotVec, 'zyz')

		self.add_plane(	"right", Hplane3,
						(0,10), (-5,5), (-1,1))

		# self.H1_left = self.getInverseHomoMat(Hplane2)
		# self.H1_right = self.getInverseHomoMat(Hplane3)
		self.H1_ground = Hplane1
		self.H1_left = Hplane2
		self.H1_right = Hplane3

		## for visualize
		x, y = np.indices((5,5))
		z = np.zeros((25,1), float)
		points = np.hstack((x.reshape(-1,1), y.reshape(-1,1))).astype(float)
		points[:,0] *= 0.25
		points[:,1] = 0.25*points[:,1] - 0.5
		points = np.hstack((points,z))
		self.points = points.copy()*10

		# print self.points

		self.point2D1 = None
		self.point2D2 = None
		self.point2D3 = None

		self.pattern = [	[0, 4],
							[5, 9],
							[10, 14],
							[15, 19],
							[20, 24],
							[0, 20],
							[1, 21],
							[2, 22],
							[3, 23],
							[4, 24]
						]

		self.worldCoors = []
		self.labels = []
		self.rects = []

	def __processRConfig(self, rconfig):
		if rconfig is None:
			return 0.0
		if not "joint1" in rconfig.currPosition.name:
			return 0.0
		indx = rconfig.currPosition.name.index("joint1")
		j1 = rconfig.currPosition.position[indx]
		return j1

	def __createHomoCam(self, Js, rconfig=None):
		## FK for camera
		q1 = getJsPosFromName(Js, "pan")
		q2 = getJsPosFromName(Js, "tilt")

		j1 = self.__processRConfig(rconfig)
		# j1 = np.pi

		tranOffset = np.array([2.5,-0.0,self.OFF1],float)
		rotOffset = np.array([j1-np.pi,0,0],float)
		HOffset = self.create_transformationMatrix(tranOffset, rotOffset, 'zyz', order = "rot-first")
		# print j1

		rot =[	[np.sin(q1), 	np.cos(q1)*np.sin(q2),	np.cos(q1)*np.cos(q2)],
				[-np.cos(q1),	np.sin(q1)*np.sin(q2),	np.sin(q1)*np.cos(q2)],
				[0,				-np.cos(q2),			np.sin(q2)]]
		rot = np.array(rot, float)
		tran = [[self.L2*np.cos(q1)*np.cos(q2)+self.L1*np.cos(q1)-self.H2*np.cos(q1)*np.sin(q2)],
				[self.L2*np.sin(q1)*np.cos(q2)-self.H2*np.sin(q1)*np.sin(q1)+self.L1*np.sin(q1)],
				[self.L2*np.sin(q2)+self.H2*np.cos(q2)+self.H1]]
		tran = np.array(tran, float)
		Homo = np.hstack((rot, tran))
		Homo = np.vstack((Homo, np.array([0,0,0,1])))
		return np.matmul(HOffset, Homo)

	def __rect2CntAndCenter(self, rect):
		cnt = np.zeros((4,2), dtype = float)
		center = np.zeros((2), dtype = float)
		for i,p in enumerate(rect.points):
			cnt[i,0] = p.x
			cnt[i,1] = p.y
			center[0] += 0.25*p.x
			center[1] += 0.25*p.y
		return cnt, center

	def clipLine(self, point1, point2, shape):
		if point1 is None or point2 is None:
			return False, point1, point2

		if (-1000000>=point1).any() or (point1>=1000000).any():
			return False, point1, point2

		if (-1000000>=point2).any() or (point2>=1000000).any():
			return False, point1, point2

		point1 = point1.astype(int)
		point2 = point2.astype(int)
		ret, pt1, pt2 = cv2.clipLine( (0,0,shape[1],shape[0]), tuple(point1), tuple(point2))
		# print ret
		return ret, pt1, pt2

	def kinematicCalculation(self, objMsg, Js, rconfig=None):
		H = self.__createHomoCam(Js, rconfig)

		self.worldCoors = []
		self.labels = []
		self.rects = []
		for l, rect in zip(objMsg.labels, objMsg.rects):
			cnt, center = self.__rect2CntAndCenter(rect)
			temp = self.calculate3DCoor(cnt, HCamera=H)
			# r = [self.calculate2DCoor(i[1], i[0], HCamera=H) for i in temp if i[0] is not None]
			# print temp
			poly = Polygon()
			for n, p in temp:
				# print p
				if n is None:
					continue
				p2D = self.calculate2DCoor([p], n, HCamera=H)[0]
				if p2D is not None:
					poly.points.append(Point32(x=p2D[0], y=p2D[1]))

			center = self.calculate3DCoor([center], HCamera=H)[0]
			fromBase = None
			if center[0] is None:
				continue
			elif center[0] == "ground":
				# center[1] = np.hstack((center[1],1))
				center_aux = np.hstack((center[1],1)).T
				fromBase = np.matmul(self.H1_ground, center_aux)[:-1]/10.0
			elif center[0] == "right":
				center_aux = np.hstack((center[1],1)).T
				fromBase = np.matmul(self.H1_right, center_aux)[:-1]/10.0
			elif center[0] == "left":
				center_aux = np.hstack((center[1],1)).T
				fromBase = np.matmul(self.H1_left, center_aux)[:-1]/10.0
			self.worldCoors.append(Point32(x=fromBase[0],y=fromBase[1],z=fromBase[2]))
			# print self.H1_right
			# print center_aux
			# print fromBase
			self.labels.append(l)
			self.rects.append(poly)

		self.point2D1 = self.calculate2DCoor(self.points, "ground", HCamera=H)
		self.point2D1 = np.array(self.point2D1)

		self.point2D2 = self.calculate2DCoor(self.points, "left", HCamera=H)
		self.point2D2 = np.array(self.point2D2)

		self.point2D3 = self.calculate2DCoor(self.points, "right", HCamera=H)
		self.point2D3 = np.array(self.point2D3)

		msg = suchartPostDictMsg(rects = objMsg.rects,
								labels = objMsg.labels,
								world_coor = self.worldCoors)
		msg.header.stamp = rospy.Time.now()

		return msg

	def loop(self):
		blank = np.zeros((480,640,3), dtype=np.uint8)
		if self.point2D1 is not None:
			for i in self.pattern:
				point1 = self.point2D1[i[0]]
				point2 = self.point2D1[i[1]]
				# print point1, point2
				ret, pt1, pt2 = self.clipLine(point1, point2, blank.shape)
				if ret:
					# print "Draw"
					cv2.line(blank, pt1, pt2, (255,0,0), 4)

		if self.point2D2 is not None:
			for i in self.pattern:
				point1 = self.point2D2[i[0]]
				point2 = self.point2D2[i[1]]
				ret, pt1, pt2 = self.clipLine(point1, point2, blank.shape)
				if ret:
					cv2.line(blank, pt1, pt2, (0,0,255), 4)

		if self.point2D3 is not None:
			for i in self.pattern:
				point1 = self.point2D3[i[0]]
				point2 = self.point2D3[i[1]]
				ret, pt1, pt2 = self.clipLine(point1, point2, blank.shape)
				# print ret
				if ret:
					cv2.line(blank, pt1, pt2, (0,255,0), 4)

		for rect,label in zip(self.rects, self.labels):
			cen_x = 0
			cen_y = 0
			cnt = np.zeros((4,1,2))
			for i,p in enumerate(rect.points):
				cen_x += 0.25*p.x
				cen_y += 0.25*p.y
				cnt[i,0,0] = p.x
				cnt[i,0,1] = p.y
				# p1 = (int(p.x), int(p.y))
				# p2 = (int(rect.points[(i+1)%4].x), int(rect.points[(i+1)%4].y))
				# cv2.line(blank, p1, p2, (255,255,255),-1)
			# print cnt
			cv2.drawContours(blank, [cnt.astype(int)], 0, (255,255,255), -1)
			cv2.putText(blank, str(label), (int(cen_x),int(cen_y)),
					cv2. FONT_HERSHEY_SIMPLEX,
					1, (0,255,0), 7)
			cv2.putText(blank, str(label), (int(cen_x),int(cen_y)),
					cv2. FONT_HERSHEY_SIMPLEX,
					1, (0,0,255), 3)

		cv2.imshow("img", blank)
		cv2.waitKey(1)

	def end(self):
		cv2.destroyAllWindows()