import numpy as np
import scipy, scipy.io
import matplotlib
import matplotlib.pyplot as plt
from matplotlib import image
import os, os.path
import cv2
from tqdm import tqdm

from torchvision.transforms import RandomCrop

def find_images():
	# This will search the data directory to get the filenames of all images,
	# and their corresponding data filenames.
	#
	# Directory structure should be
	# data
	# |-- 300W_LP
	# |   |-- AFW
	# |   |   |-- AFW_#####.jpg
	# |   |   |-- AFW_#####.mat
	# |   |
	# |   |-- AFW_Flip
	# |   |   |-- AFW_#####.jpg
	# |   |   |-- AFW_#####.mat

	dataset_path = "./data/300W_LP/"
	directories = ["AFW", "HELEN", "IBUG", "LFPW"]
	# We don't include the *_Flip directories, since it's easier to just flip the images in python

	image_fnames = []
	data_fnames = []
	for d in directories:
		raw_fnames = os.listdir(dataset_path + d)
		for fname in raw_fnames:
			if fname[-6:] == "_0.jpg":
				# We don't want to accidentally load an image multiple times, but there's a .jpg and .mat file
				# The _0 at the end ensures that it's not one of the "rotated" images included in the dataset
				image_fname = dataset_path + d + "/" + fname[:-4] + ".jpg"
				data_fname = dataset_path + "landmarks/" + d + "/" + fname[:-4] + "_pts.mat"
				image_fnames.append(image_fname)
				data_fnames.append(data_fname)

	print("Found %d images" % len(image_fnames))
	return image_fnames, data_fnames

def load_data(image_fnames, data_fnames, use_loading_bar=True):
	# Load the image and landmark data for each listed filename above.
	# Note that the indices must correspond -- don't shuffle them!
	print("Loading images...")
	images = [cv2.cvtColor(matplotlib.image.imread(image_fname), cv2.COLOR_RGB2GRAY) for image_fname in (tqdm(image_fnames) if use_loading_bar else image_fnames)]
	print("Loading landmarks...")
	datas = [scipy.io.loadmat(data_fname) for data_fname in (tqdm(data_fnames) if use_loading_bar else data_fnames)]
	landmarks_2d = [data["pts_2d"] for data in datas]
	landmarks_3d = [data["pts_3d"] for data in datas]
	print("Loaded %d images" % len(images))
	# tx=transforms.Compose([Rescale(250),RandomCrop(224),Normalize()])
	# images = np.array(images)
	# images=tx(images)
	# images=RandomCrop(250)(images)
	return images, landmarks_2d, landmarks_3d

def augment_flip(images, landmarks_2d, landmarks_3d):
	# Doubles the number of images by flipping each one horizontally,
	# and also producing the appropriate corresponding landmark locations
	for i in range(len(images)):
		images.append(np.flip(images[i], axis=1))
		landmarks_2d.append(landmarks_2d[i].copy())
		landmarks_2d[-1][:,0] = images[-1].shape[1] - landmarks_2d[-1][:,0]
		landmarks_3d.append(landmarks_3d[i].copy())
		landmarks_3d[-1][:,0] = images[-1].shape[1] - landmarks_3d[-1][:,0]

def get_color_image(image_fnames, idx):
	image_fname = image_fnames[idx % len(image_fnames)]
	if idx // len(image_fnames) == 1:
		return np.flip(matplotlib.image.imread(image_fname), axis=1)
	else:
		return matplotlib.image.imread(image_fname)

if __name__ == "__main__":
	image_fnames, data_fnames = find_images()
	images, landmarks_2d, landmarks_3d = load_data(image_fnames, data_fnames)
	augment_flip(images, landmarks_2d, landmarks_3d)
	# tx=transforms.Compose([Rescale(250),RandomCrop(224),Normalize()])
	images = np.array(images)
	# images=RandomCrop(200)(images)
	landmarks_2d = np.array(landmarks_2d)
	landmarks_3d = np.array(landmarks_3d)
	try:
		while True:
			idx = np.random.randint(len(images))
			if idx > len(image_fnames):
				print("Displaying image: %s (flipped)" % image_fnames[idx - len(image_fnames)])
				print("            data: %s" % data_fnames[idx - len(data_fnames)])
			else:
				print("Displaying image: %s (flipped)" % image_fnames[idx])
				print("            data: %s" % data_fnames[idx])
			print("           index: %d" % idx)

			img = images[idx]
			landmarks = landmarks_2d[idx].flatten()
			fig, ax = plt.subplots()
			# ax.imshow(img, cmap="gray")
			ax.imshow(get_color_image(image_fnames, idx))
			ax.scatter(landmarks[0::2], landmarks[1::2])
			plt.show()
	except KeyboardInterrupt:
		pass


import torch
import torch.nn as nn
import probtorch

class Encoder(nn.Module):
    def __init__(self, num_pixels=784, num_hidden=50, num_digits=10, num_style=2):
        super(self.__class__, self).__init__()
        self.h = nn.Sequential(
                    nn.Linear(num_pixels, num_hidden),
                    nn.ReLU())
        self.y_log_weights = nn.Linear(num_hidden, num_digits)
        self.z_mean = nn.Linear(num_hidden + num_digits, num_style)
        self.z_log_std = nn.Linear(num_hidden + num_digits, num_style)

    def forward(self, x, y_values=None, num_samples=10):
        q = probtorch.Trace()
        x = x.expand(num_samples, *x.size())
        if y_values is not None:
            y_values = y_values.expand(num_samples, *y_values.size())
        h = self.h(x)
        y = q.concrete(logits=self.y_log_weights(h), temperature=0.66,
                       value=y_values, name='y')
        h2 = torch.cat([y, h], -1)
        z = q.normal(loc=self.z_mean(h2),
                     scale=torch.exp(self.z_log_std(h2)),
                     name='z')
        return q


import codecs
import os

from setuptools import find_packages, setup

# single sourcing package version strategy taken from
# https://packaging.python.org/guides/single-sourcing-package-version

PACKAGE_NAME = "kwnlp_sql_parser"

def read_file(filename: str) -> str:
    """Read package file as text to get name and version."""
    here = os.path.abspath(os.path.dirname(__file__))
    with codecs.open(os.path.join(here, PACKAGE_NAME, filename), "r") as f:
        return f.read()

def find_version() -> str:
    """Only define version in one place."""
    for line in read_file("__init__.py").splitlines():
        if line.startswith("__version__"):
            delim = '"' if '"' in line else "'"
            return line.split(delim)[1]
    else:
        raise RuntimeError("Unable to find version string.")

def find_long_description() -> str:
    """Return the content of the README.rst file."""
    return read_file("../README.md")

setup(
    name=PACKAGE_NAME,
    version=find_version(),
    description="Utility for parsing Wikipedia SQL dumps into CSVs.",
    long_description=find_long_description(),
    long_description_content_type="text/markdown",
    url="https://github.com/kensho-technologies/kwnlp-sql-parser",
    author="Kensho Technologies LLC.",
    author_email="kwnlp@kensho.com",
    license="Apache 2.0",
    packages=find_packages(exclude=["tests*"]),
    package_data={"": []},
    install_requires=[],
    extras_require={
        "dev": [
            "pre-commit",
        ]
    },
    classifiers=[
        "Development Status :: 4 - Beta",
        "Intended Audience :: Developers",
        "License :: OSI Approved :: Apache Software License",
        "Programming Language :: Python :: 3 :: Only",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
    ],
    keywords="wikipedia sql dump open data",
    python_requires=">=3.6",
)


from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

from datetime import datetime
import json
import logging
import random
import shlex

from dashboard.common import cloud_metric
from dashboard.common import utils
from dashboard.pinpoint.models import errors
from dashboard.pinpoint.models.quest import execution as execution_module
from dashboard.pinpoint.models.quest import quest
from dashboard.services import swarming

_TESTER_SERVICE_ACCOUNT = (
    'chrome-tester@chops-service-accounts.iam.gserviceaccount.com')
_CAS_DEFAULT_INSTANCE = (
    'projects/chromium-swarm/instances/default_instance'
)

def SwarmingTagsFromJob(job):
  ret = {
      'pinpoint_job_id': job.job_id,
      'url': job.url,
      'comparison_mode': job.comparison_mode,
      'pinpoint_task_kind': 'test',
      'pinpoint_user': job.user,
  }

  if job.batch_id is not None:
    ret['pinpoint_batch_id'] = job.batch_id

  return ret

class RunTest(quest.Quest):

  def __init__(self, swarming_server, dimensions, extra_args, swarming_tags,
               command, relative_cwd):
    """RunTest Quest

    Args:
      swarming_server: a string indicating the swarming server.
      dimensions: a list of dimensions.
      extra_args: a list of strings treated as additional arguments to
          provide to the task in Swarming.
      swarming_tags: a dict of swarming tags.
      command: a list of strings to be provided to the Swarming task command.
      relative_cwd: a string indicating the working directory in the isolate.
    """
    self._swarming_server = swarming_server
    self._dimensions = dimensions
    self._extra_args = extra_args
    self._swarming_tags = {} if not swarming_tags else swarming_tags
    self._command = command
    self._relative_cwd = relative_cwd

    # We want subsequent executions use the same bot as the first one.
    self._canonical_executions = []

    self._bots = None
    self._comparison_mode = None
    self._attempt_count = None

    self._started_executions = {}

  def __eq__(self, other):
    return (isinstance(other, type(self))
            and self._swarming_server == other._swarming_server
            and self._dimensions == other._dimensions
            and self._extra_args == other._extra_args
            and self._canonical_executions == other._canonical_executions
            and self._command == other._command
            and self._relative_cwd == other._relative_cwd
            and self._bots == other._bots
            and self._comparison_mode == other._comparison_mode
            and self._attempt_count == other._attempt_count
            and self._started_executions == other._started_executions)

  def __hash__(self):
    return hash(self.__str__())

  def __str__(self):
    return 'Test'

  def __setstate__(self, state):
    self.__dict__ = state  # pylint: disable=attribute-defined-outside-init
    if not hasattr(self, '_swarming_tags'):
      self._swarming_tags = {}
    if not hasattr(self, '_command'):
      self._command = None
    if not hasattr(self, '_relative_cwd'):
      self._relative_cwd = None

  @property
  def command(self):
    return getattr(self, '_command')

  @property
  def relative_cwd(self):
    return getattr(self, '_relative_cwd', 'out/Release')

  def PropagateJob(self, job):
    if not hasattr(self, '_swarming_tags'):
      self._swarming_tags = {}
    self._swarming_tags.update(SwarmingTagsFromJob(job))
    self._comparison_mode = job.comparison_mode
    self._attempt_count = job.state.attempt_count
    self._bots = [str(b) for b in job.bots]

  def Start(self, change, isolate_server, isolate_hash):
    return self._Start(change, isolate_server, isolate_hash, self._extra_args,
                       {}, None)

  def _Start(self, change, isolate_server, isolate_hash, extra_args,
             swarming_tags, execution_timeout_secs):
    if not hasattr(self, '_started_executions'):
      self._started_executions = {}
    if change not in self._started_executions:
      self._started_executions[change] = []
    index = len(self._started_executions[change])

    if self._swarming_tags:
      swarming_tags.update(self._swarming_tags)

    dimensions = self._GetDimensions(index)

    test_execution = _RunTestExecution(
        self,
        self._swarming_server,
        dimensions,
        extra_args,
        isolate_server,
        isolate_hash,
        swarming_tags,
        index,
        change,
        command=self.command,
        relative_cwd=self.relative_cwd,
        execution_timeout_secs=execution_timeout_secs)
    self._started_executions[change].append(test_execution)
    return test_execution

  def _StartAllTasks(self):
    # We need to wait for all of the builds to be complete.
    if len(self._started_executions) != 2:
      return
    logging.debug('bots in quest: %s', [str(b) for b in self._bots])
    a_list, b_list = self._started_executions.values()
    if len(a_list) != self._attempt_count or len(b_list) != self._attempt_count:
      return

    orderings = self._GetABOrderings(self._attempt_count)
    for i in range(self._attempt_count):
      if orderings[i]:
        a_list[i]._StartTask()
        b_list[i]._StartTask()
      else:
        b_list[i]._StartTask()
        a_list[i]._StartTask()

  def _GetABOrderings(self, attempt_count):
    # Get a list of if a/0 or b/1 should go first, such that
    # - half of As and half of Bs go first
    # - which half is random
    half = attempt_count // 2
    orderings = [0] * half + [1] * half
    if attempt_count % 2:
      orderings.append(0)
    random.shuffle(orderings)
    return orderings

  def _GetDimensions(self, index):
    # Adds a bot_id to dimensions
    if not hasattr(
        self,
        '_comparison_mode') or self._comparison_mode != 'try' or not hasattr(
            self, '_bots'):
      return self._dimensions
    dimensions = list(self._dimensions)

    bot_id = self._bots[index % len(self._bots)]
    if bot_id:
      logging.debug('add bot_id for index %s and bots %s', str(index), bot_id)
      dimensions.append({'key': 'id', 'value': bot_id})
    return dimensions

  @classmethod
  def _ComputeCommand(cls, arguments):
    """Computes the relative_cwd and command properties for Swarming tasks.

    This can be overridden in the derived classes to allow custom computation
    of the relative working directory and the command to be provided to the
    Swarming task.

    Args:
      arguments: a dict of arguments provided to a Pinpoint job.

    Returns a tuple of (relative current working dir, command)."""
    return arguments.get('relative_cwd'), arguments.get('command')

  @classmethod
  def FromDict(cls, arguments):
    swarming_server = arguments.get('swarming_server')
    if not swarming_server:
      raise TypeError('Missing a "swarming_server" argument.')

    dimensions = arguments.get('dimensions')
    if not dimensions:
      raise TypeError('Missing a "dimensions" argument.')
    if isinstance(dimensions, str):
      dimensions = json.loads(dimensions)
    if not any(dimension['key'] == 'pool' for dimension in dimensions):
      raise ValueError('Missing a "pool" dimension.')
    relative_cwd, command = cls._ComputeCommand(arguments)
    extra_test_args = cls._ExtraTestArgs(arguments)
    swarming_tags = cls._GetSwarmingTags(arguments)
    return cls(swarming_server, dimensions, extra_test_args, swarming_tags,
               command, relative_cwd)

  @classmethod
  def _ExtraTestArgs(cls, arguments):
    extra_test_args = arguments.get('extra_test_args')
    if not extra_test_args:
      return []

    # We accept a json list or a string. If it can't be loaded as json, we
    # fall back to assuming it's a string argument.
    try:
      extra_test_args = json.loads(extra_test_args)
    except ValueError:
      extra_test_args = shlex.split(extra_test_args)
    if not isinstance(extra_test_args, list):
      raise TypeError('extra_test_args must be a list: %s' % extra_test_args)
    return extra_test_args

  @classmethod
  def _GetSwarmingTags(cls, arguments):
    pass

class _RunTestExecution(execution_module.Execution):

  def __init__(self,
               containing_quest,
               swarming_server,
               dimensions,
               extra_args,
               isolate_server,
               isolate_hash,
               swarming_tags,
               index,
               change,
               command=None,
               relative_cwd='out/Release',
               execution_timeout_secs=None):
    super().__init__()
    self._quest = containing_quest
    self._bot_id = None
    self._command = command
    self._dimensions = dimensions
    self._extra_args = extra_args
    self._isolate_hash = isolate_hash
    self._isolate_server = isolate_server
    self._relative_cwd = relative_cwd
    self._swarming_server = swarming_server
    self._swarming_tags = swarming_tags
    self._index = index
    self._change = change
    self._execution_timeout_secs = execution_timeout_secs
    self._task_id = None

  def __setstate__(self, state):
    self.__dict__ = state  # pylint: disable=attribute-defined-outside-init
    if not hasattr(self, '_swarming_tags'):
      self._swarming_tags = {}
    if not hasattr(self, '_command'):
      self._command = None
    if not hasattr(self, '_relative_cwd'):
      self._relative_cwd = 'out/Release'
    if not hasattr(self, '_execution_timeout_secs'):
      self._execution_timeout_secs = None

  @property
  def bot_id(self):
    return self._bot_id

  @property
  def command(self):
    return getattr(self, '_command')

  @property
  def relative_cwd(self):
    return getattr(self, '_relative_cwd', 'out/Release')

  @property
  def execution_timeout_secs(self):
    return getattr(self, '_execution_timeout_secs')

  def _AsDict(self):
    details = []
    if self._bot_id:
      details.append({
          'key': 'bot',
          'value': self._bot_id,
          'url': self._swarming_server + '/bot?id=' + self._bot_id,
      })
    if self._task_id:
      details.append({
          'key': 'task',
          'value': self._task_id,
          'url': self._swarming_server + '/task?id=' + self._task_id,
      })
    if self._result_arguments:
      cas_root_ref = self._result_arguments.get('cas_root_ref')
      if cas_root_ref is not None:
        digest = cas_root_ref['digest']
        url = 'https://cas-viewer.appspot.com/{}/blobs/{}/{}/tree'.format(
            cas_root_ref['cas_instance'], digest['hash'], digest['size_bytes'])
        value = '{}/{}'.format(digest['hash'], digest['size_bytes'])
      else:
        url = (self._result_arguments['isolate_server'] + '/browse?digest=' +
               self._result_arguments['isolate_hash'])
        value = self._result_arguments['isolate_hash']
      details.append({
          'key': 'isolate',
          'value': value,
          'url': url,
      })
    return details

  def _Poll(self):
    if not self._task_id:
      self._StartTasksIfMasterOrNotTry()
      return

    logging.debug('_RunTestExecution Polling swarming: %s', self._task_id)
    swarming_task = swarming.Swarming(self._swarming_server).Task(self._task_id)

    result = swarming_task.Result()
    logging.debug('swarming response: %s', result)

    if 'bot_id' in result:
      # For bisects, this will be set after the task is allocated to a bot.
      # A/Bs will set this elsewhere.
      self._bot_id = result['bot_id']

    if self._bot_id:
      if not swarming.IsBotAlive(self._bot_id, self._swarming_server):
        raise errors.SwarmingTaskError('Bot is dead.')

    if result['state'] == 'PENDING' and self._bot_id:
      pool = 'chrome.tests.pinpoint-staging' if utils.IsStagingEnvironment(
      ) else 'chrome.tests.pinpoint'
      pending_count = swarming.Swarming(self._swarming_server).Tasks().Count(
          bot_id=self._bot_id, state='PENDING', pool=pool).get('count', 0)
      logging.debug('Bot %s has %s jobs in the pending queue.', self._bot_id,
                    pending_count)
      cloud_metric.PublishSwarmingBotPendingTasksMetric(
          bot_id=self._bot_id, pool=pool, count=int(pending_count))

    if result['state'] == 'PENDING' or result['state'] == 'RUNNING':
      return

    if result['state'] == 'EXPIRED':
      raise errors.SwarmingExpired()

    if result['state'] != 'COMPLETED':
      raise errors.SwarmingTaskError(result['state'])

    # The swarming task is completed. Report the pending time
    self._ReportSwarmingJobsMetric(result)

    if result['failure']:
      if 'outputs_ref' not in result:
        task_url = '%s/task?id=%s' % (self._swarming_server, self._task_id)
        raise errors.SwarmingTaskFailed('%s' % (task_url,))

      isolate_output_url = '%s/browse?digest=%s' % (
          result['outputs_ref']['isolatedserver'],
          result['outputs_ref']['isolated'])
      raise errors.SwarmingTaskFailed('%s' % (isolate_output_url,))

    if 'cas_output_root' in result:
      result_arguments = {
          'cas_root_ref': result['cas_output_root'],
      }
    else:
      result_arguments = {
          'isolate_server': result['outputs_ref']['isolatedserver'],
          'isolate_hash': result['outputs_ref']['isolated'],
      }

    self._Complete(result_arguments=result_arguments)

  @staticmethod
  def _ReportSwarmingJobsMetric(result):
    """Report metrics on the completed swarming task.

    Including:
      pending_time: how long the task has waited, such as for a busy bot
      running_time: how long the task runs.
    Dimensions are:
      task_id: id of the swarming task
      bot_os: the detailed os version of the bot.
      bot_id: bot id.
      pinpoint_job_type: try or bisect. Expecting longer wait on try jobs
        as one of the two branches will wait for the same bot from the
        other branch.
      pinpont_job_id: id of the pinpoint job which launches the swarming task
    """
    created_ts = result.get('created_ts')
    started_ts = result.get('started_ts')
    completed_ts = result.get('completed_ts')
    if created_ts and started_ts and completed_ts:
      pending_time = datetime.fromisoformat(
          started_ts) - datetime.fromisoformat(created_ts)
      running_time = datetime.fromisoformat(
          completed_ts) - datetime.fromisoformat(started_ts)

      bot_os = pinpoint_job_type = pinpoint_job_id = None
      for tag in result.get('tags', ''):
        key, value = tag.split(':', 1)
        if key == 'os':
          bot_os = value
        if key == 'comparison_mode':
          pinpoint_job_type = value
        if key == 'pinpoint_job_id':
          pinpoint_job_id = value

      # debug infor for crbug/1422306
      if None in (bot_os, pinpoint_job_type, pinpoint_job_id):
        logging.debug('Missing value in swarming result: %s', result)

      cloud_metric.PublishPinpointSwarmingPendingMetric(
          task_id=result.get('task_id'),
          pinpoint_job_type=pinpoint_job_type,
          pinpoint_job_id=pinpoint_job_id,
          bot_id=result.get('bot_id'),
          bot_os=bot_os,
          pending_time=pending_time.total_seconds())

      cloud_metric.PublishPinpointSwarmingRuntimeMetric(
          task_id=result.get('task_id'),
          pinpoint_job_type=pinpoint_job_type,
          pinpoint_job_id=pinpoint_job_id,
          bot_id=result.get('bot_id'),
          bot_os=bot_os,
          running_time=running_time.total_seconds())

  @staticmethod
  def _IsCasDigest(d):
    return '/' in d

  def _StartTasksIfMasterOrNotTry(self):
    if not hasattr(self, '_quest') or self._quest._comparison_mode != 'try':
      self._StartTask()
      return

    if self._change.variant != 0 or self._index != 0:
      # This task is not responsible for kicking off all other tasks
      return

    # This will call _StartTask on all tasks, including this one
    self._quest._StartAllTasks()

  def _StartTask(self):
    """Kick off a Swarming task to run a test."""
    # TODO(fancl): Seperate cas input from isolate (including endpoint and
    # datastore module)
    if self._IsCasDigest(self._isolate_hash):
      cas_hash, cas_size = self._isolate_hash.split('/', 1)
      instance = self._isolate_server
      # This is a workaround for build cached uploaded before crrev/c/2964515
      # landed. We can delete it after all caches expired.
      if instance.startswith('https://'):
        instance = _CAS_DEFAULT_INSTANCE
      input_ref = {
          'cas_input_root': {
              'cas_instance': instance,
              'digest': {
                  'hash': cas_hash,
                  'size_bytes': int(cas_size),
              }
          }
      }
    else:
      input_ref = {
          'inputs_ref': {
              'isolatedserver': self._isolate_server,
              'isolated': self._isolate_hash,
          }
      }

    properties = {
        'extra_args': self._extra_args,
        'dimensions': self._dimensions,
        'execution_timeout_secs': str(self.execution_timeout_secs or 2700),
        'io_timeout_secs': str(self.execution_timeout_secs or 2700),
    }
    properties.update(**input_ref)

    if self.command:
      properties.update({
          # Set the relative current working directory to be the root of the
          # isolate.
          'relative_cwd': self.relative_cwd,

          # Use the command provided in the creation of the execution.
          'command': self.command + self._extra_args,
      })

      # Swarming requires that if 'command' is present in the request, that we
      # not provide 'extra_args'.
      del properties['extra_args']

    body = {
        'realm':
            'chrome:pinpoint',
        'name':
            'Pinpoint job',
        'user':
            'Pinpoint',
        'priority':
            '100',
        'service_account':
            _TESTER_SERVICE_ACCOUNT,
        'task_slices': [{
            'properties': properties,
            'expiration_secs': '86400',  # 1 day.
        }],
    }

    if self._swarming_tags:
      # This means we have additional information available about the Pinpoint
      # tags, and we should add those to the Swarming Pub/Sub updates.
      body.update({
          'tags': ['%s:%s' % (k, v) for k, v in self._swarming_tags.items()],
          # TODO(dberris): Consolidate constants in environment vars?
          'pubsub_topic':
              'projects/chromeperf/topics/pinpoint-swarming-updates',
          'pubsub_auth_token':
              'UNUSED',
          'pubsub_userdata':
              json.dumps({
                  'job_id': self._swarming_tags.get('pinpoint_job_id'),
                  'task': {
                      'type': 'test',
                      'id': self._swarming_tags.get('pinpoint_task_id'),
                  },
              }),
      })

    for d in self._dimensions:
      if d['key'] == 'id':
        self._bot_id = d['value']

    logging.debug('Requesting swarming task with parameters: %s', body)
    response = swarming.Swarming(self._swarming_server).Tasks().New(body)
    logging.debug('Response: %s', response)
    self._task_id = response['task_id']


#!/usr/local/bin/python
# encoding: utf-8
"""
*A function to quickly add multiprocessing to any program*

:Author:
    David Young
"""
from __future__ import division
import inspect
from functools import partial
from fundamentals import tools
from past.utils import old_div
import sys
import os
os.environ['TERM'] = 'vt100'

def fmultiprocess(
        log,
        function,
        inputArray,
        poolSize=False,
        timeout=3600,
        turnOffMP=False,
        **kwargs):
    """multiprocess pool

    **Key Arguments**

    - ``log`` -- logger
    - ``function`` -- the function to multiprocess
    - ``inputArray`` -- the array to be iterated over
    - ``poolSize`` -- limit the number of CPU that are used in multiprocess job
    - ``timeout`` -- time in sec after which to raise a timeout error if the processes have not completed
    - ``turnOffMP`` -- turn off multiprocessing. Useful for profiling and debugging. Default **False**

    **Return**

    - ``resultArray`` -- the array of results

    **Usage**

    ```python
    from fundamentals import multiprocess
    # DEFINE AN INPUT ARRAY
    inputArray = range(10000)
    results = multiprocess(log=log, function=functionName, poolSize=10, timeout=300,
                          inputArray=inputArray, otherFunctionKeyword="cheese")
    ```

    """
    log.debug('starting the ``multiprocess`` function')

    logFound = False
    # PYTHON 3 VS 2 ..
    try:
        if "log" in inspect.getfullargspec(function)[0]:
            logFound = True
    except:
        if "log" in inspect.getargspec(function)[0]:
            logFound = True

    if turnOffMP == False:
        import psutil
        # import multiprocess as mp
        # mp.set_start_method('forkserver')
        from multiprocess import cpu_count, Pool

        # DEFINTE POOL SIZE - NUMBER OF CPU CORES TO USE (BEST = ALL - 1)
        if not poolSize:
            poolSize = psutil.cpu_count()

        if poolSize:
            p = Pool(processes=poolSize)
        else:
            p = Pool()

        cpuCount = psutil.cpu_count()
        chunksize = int(old_div((len(inputArray) + 1), (cpuCount * 3)))

        if chunksize == 0:
            chunksize = 1

        # chunksize = 1
        # MAP-REDUCE THE WORK OVER MULTIPLE CPU CORES
        if logFound:
            mapfunc = partial(function, log=log, **kwargs)
            resultArray = p.map_async(mapfunc, inputArray, chunksize=chunksize)
        else:
            mapfunc = partial(function, **kwargs)
            resultArray = p.map_async(mapfunc, inputArray, chunksize=chunksize)

        resultArray = resultArray.get(timeout=timeout)

        p.close()
        p.join()
        # p.terminate()

    else:
        resultArray = []

        if logFound:
            for i in inputArray:
                r = function(log, i, **kwargs)
                resultArray.append(r)
        else:
            for i in inputArray:
                r = function(i, **kwargs)
                resultArray.append(r)

    log.debug('completed the ``multiprocess`` function')
    return resultArray


import asyncio
import random

import orjson
import simple_dataclass_settings
import websockets

import libs.logger

@simple_dataclass_settings.settings
class _Log:
    name: str = "local-e2e-test"
    level: str = "INFO"
    root_level: str = "ERROR"

@simple_dataclass_settings.settings
class Settings:
    log: _Log

def _get_worker_registration_data() -> dict:
    return {
        "ip": "127.0.0.1",
        "address": "127.0.0.1",
        "hardware": "Test worker",
        "hardware_id": "test-worker",
        "caption": "E2E_TEST",
    }

def _get_job_input(
    proof_target: int = None,
    block_height: int = None,
) -> dict:
    if proof_target is None:
        proof_target = random.randint(1000, 10000)
    if block_height is None:
        block_height = random.randint(1, 100)

    return {
        "type": "job",
        "epoch_challenge": {
            "epoch_number": 123,
            "epoch_block_hash": "abc",
            "degree": 8191,
        },
        "proof_target": proof_target,
        "block_height": block_height,
    }

def _get_solution_input(
    task_id: str,
    solution_target: int = None,
    nonce: int = None,
) -> dict:
    if solution_target is None:
        solution_target = random.randint(1000, 10000)
    if nonce is None:
        nonce = random.randint(1, 100)

    return {
        "type": "solution",
        "task_id": task_id,
        "solution": {
            "partial_solution": {
                "address": "def",
                "nonce": nonce,
                "commitment": "ghi",
            },
            "proof.w": {
                "x": "123",
                "y": "456",
                "infinity": False,
            },
        },
        "solution_target": solution_target,
    }

async def main(
    cfg: Settings,
) -> None:
    logger = libs.logger.get(
        name=cfg.log.name,
        level=cfg.log.level,
    )

    try:
        logger.info("Starting tests")

        async with websockets.connect(f"ws://localhost:8001") as node:
            async with websockets.connect(f"ws://localhost:8002") as worker:
                logger.info("Registering worker and waiting for registration")
                await worker.send(orjson.dumps(_get_worker_registration_data()))
                await asyncio.sleep(1.5)
                logger.info("Message sent")

                logger.info("Sending message to node server")
                job_input = _get_job_input()
                await node.send(orjson.dumps(job_input))
                logger.info("Message sent")

                logger.info("Waiting message from worker server")
                worker_task = orjson.loads(await asyncio.wait_for(worker.recv(), 1.5))
                logger.info("Message received, checking content")
                assert len(worker_task.keys()) == 2
                assert "task_id" in worker_task
                assert worker_task["epoch_challenge"] == job_input["epoch_challenge"]
                logger.info("Message checked")

                logger.info("Sending solution to worker server")
                solution_input = _get_solution_input(
                    task_id=worker_task["task_id"],
                    solution_target=job_input["proof_target"],
                )
                await worker.send(orjson.dumps(solution_input))
                logger.info("Message sent")

                logger.info("Waiting message from node server")
                solution_output = orjson.loads(await asyncio.wait_for(node.recv(), 1.5))
                logger.info("Message received, checking content")
                assert solution_output == solution_input["solution"]
                logger.info("Message checked")
    except Exception:
        logger.info("Tests failed")
        raise
    else:
        logger.info("Tests done")

if __name__ == "__main__":
    simple_dataclass_settings.read_envfile('./.env.local_e2e_test')

    settings = simple_dataclass_settings.populate(Settings)
    libs.logger.configure(
        level=settings.log.root_level,
    )

    asyncio.run(main(
        cfg=settings,
    ))


import random

import numpy as np
import torch

def set_seed(seed: int, cuda_deterministic: bool = False):
    """Set seed for random, numpy, torch.

    Args:
        seed (int): Random seed.
        cuda_deterministic (bool, optional): Deterministic for cuda. Defaults to False.
    """
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed(seed)
        torch.cuda.manual_seed_all(seed)
    if cuda_deterministic:  # slower, more reproducible
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
    else:
        torch.backends.cudnn.deterministic = False
        torch.backends.cudnn.benchmark = True


import sqlite3
import pandas as pd
class Database:
    def create_connection():
        conn = sqlite3.connect('test_db.sqlite')
        conn.row_factory = sqlite3.Row
        return conn

    def criar_tabela(conn):
        c = conn.cursor()
        print('Criando tabela...')
        c.execute("""CREATE TABLE IF NOT EXISTS tabelaIPCA (
            ano varchar(4) NOT NULL,
            mes varchar(3) NOT NULL,
            baseIPCA float NOT NULL,
            createdAt timestamp NOT NULL DEFAULT current_timestamp,
            updatedAt timestamp,
            PRIMARY KEY (ano,mes)
            );
        """)
        print('Tabela criada')
        conn.commit()
        return True

    def alimentar_primeiros_dados(conn,df):
        c = conn.cursor()
        c.execute("""select * from tabelaIPCA""")
        records = c.fetchall()
        if(len(records) == 0):
            print('Atualizei a base')
            df.to_sql("tabelaIPCA", conn, if_exists="append", index=False)
        else: print('Nao atualizei nada.')

        return True

    def inserir_registro(conn, ano, mes, baseIPCA):
        try:
            c = conn.cursor()
            sql = "INSERT INTO tabelaIPCA (ano, mes, baseIPCA) VALUES ('{0}', '{1}',{2})".format(ano,mes,baseIPCA)
            c.execute(sql)
            c.execute("select * from tabelaIPCA where rowid=last_insert_rowid()")
            records = c.fetchone()

            if(not records):
                return {}

            conn.commit()
            return dict(records)
        except sqlite3.IntegrityError:
            return {}

    def consultar_todos(conn):
        # c = conn.cursor()
        # c.execute("SELECT * FROM tabelaIPCA")
        df = pd.read_sql_query("SELECT * FROM tabelaIPCA order by ano desc", conn)
        data = df.to_dict('r')
        # dict = {}
        # for linha in data:
        #     ano = linha['ANO']
        #     linha.pop('ANO')
        #     dict[ano] = linha
        return data

    def consultar_mes_ano(conn:any,ano:str, mes:str):
        sql = ''
        if(ano != '' and mes != ''): sql = "Select * from tabelaIPCA where ano='"+ano+"' and mes='"+mes+"' order by ano desc"
        elif(ano != ''): sql = "Select * from tabelaIPCA where ano='"+ano+"' order by ano desc"
        elif(mes != ''): sql = "Select * from tabelaIPCA where mes='"+mes+"'order by ano desc"
        else: sql = 'Select * from tabelaIPCA order by ano desc'

        df = pd.read_sql_query(sql, conn)
        data = df.to_dict('r')
        return data

    def excluir_registro(conn,ano,mes):
        c = conn.cursor()
        c.execute("select * from tabelaIPCA where mes='{0}' and ano='{1}'".format(mes, ano))
        records = c.fetchone()

        if(not records):
            return {}

        c.execute("DELETE FROM tabelaIPCA where mes='{0}' and ano='{1}'".format(mes,ano))
        conn.commit()
        return dict(records)

    def atualizar_registro(conn,baseIPCA,ano,mes):
        print(baseIPCA,ano,mes)
        c = conn.cursor()
        c.execute("update tabelaIPCA set baseIPCA={0},updatedAt=current_timestamp where mes='{1}' and ano='{2}'".format(baseIPCA, mes, ano))
        c.execute("Select * from tabelaIPCA where mes='{0}' and ano='{1}'".format(mes, ano))
        records = c.fetchone()
        print('Achei isso:',records)

        if(not records):
            return {}

        conn.commit()
        return dict(records)


from django.forms import DateInput, ModelForm, Select, Textarea
from simple_task_manager.main.models import Event, ArchiveReport

class EventForm(ModelForm):
    class Meta:
        model = Event
        fields = ['category', 'description']
        widgets = {
            'category': Select(attrs={
                'class': 'form-select',
                'size': '4',
                'aria-label': 'category selection',
                'required': '',
            }),
            'description': Textarea(attrs={
                'class': 'form-control',
                'rows': '8',
                'aria-label': 'category description',
            })}

    def __init__(self, *args, **kwargs):
        super(EventForm, self).__init__(*args, **kwargs)
        self.fields['category'].empty_label = None

class DateInput(DateInput):
    input_type = 'date'

class ReportForm(ModelForm):
    class Meta:
        model = ArchiveReport
        fields = ['start_date', 'end_date']
        widgets = {
            'start_date': DateInput(attrs={
                'class': 'form-control',
                'type': 'date',
            }),
            'end_date': DateInput(attrs={
                'class': 'form-control',
                'type': 'date',
            }),
        }


import numpy as np
import logging

from spuq.utils.testing import *
from spuq.application.egsz.multi_vector import MultiVector, MultiVectorWithProjection
from spuq.math_utils.multiindex import Multiindex
from spuq.math_utils.multiindex_set import MultiindexSet
from spuq.linalg.vector import FlatVector

# setup logging
logging.basicConfig(filename=__file__[:-2] + 'log', level=logging.INFO)

def test_init():
    MultiVector()

def test_get_set():
    mv = MultiVector()
    mv[Multiindex([1, 2, 3])] = FlatVector([3, 4, 5])
    mv[Multiindex([0, 1, 2, 3])] = FlatVector([3, 4, 6])
    mv[Multiindex([1, 2, 3, 0])] = FlatVector([3, 4, 7])
    assert_equal(mv[Multiindex([1, 2, 3])], FlatVector([3, 4, 7]))
    assert_equal(mv[Multiindex([0, 1, 2, 3])], FlatVector([3, 4, 6]))

def test_keys():
    mv = MultiVector()
    mi1 = Multiindex([1, 2, 3])
    mi2 = Multiindex([1, 2, 4])
    mv[mi1] = FlatVector([3, 4, 5])
    mv[mi2] = FlatVector([3, 4, 7])
    keys = mv.keys()
    assert_equal(sorted(keys), sorted([mi1, mi2]))
    inds = mv.active_indices()
    assert_equal(sorted(inds), sorted([mi1, mi2]))

def test_repr():
    mv = MultiVector()
    mv[Multiindex([1, 2, 3])] = FlatVector([3, 4, 5])
    mv[Multiindex([0, 1, 2, 3])] = FlatVector([3, 4, 6])
    assert_equal(str(mv), "<MultiVector keys=[<Multiindex inds=[0 1 2 3]>, <Multiindex inds=[1 2 3]>]>")

def test_copy():
    # make sure the vectors are copied and each have their own copy of
    # the data
    mv = MultiVector()
    mv[Multiindex([1, 2, 3])] = FlatVector([3, 4, 5])
    mv[Multiindex([1, 2, 4])] = FlatVector([3, 4, 7])
    assert_equal(mv[Multiindex([1, 2, 3])], FlatVector([3, 4, 5]))
    assert_equal(mv[Multiindex([1, 2, 4])], FlatVector([3, 4, 7]))
    mv2 = mv.copy()
    mv[Multiindex([1, 2, 3])].coeffs[2] = 8
    assert_equal(mv2[Multiindex([1, 2, 3])], FlatVector([3, 4, 5]))
    assert_equal(mv2[Multiindex([1, 2, 4])], FlatVector([3, 4, 7]))
    assert_equal(mv[Multiindex([1, 2, 3])], FlatVector([3, 4, 8]))

def test_set_defaults():
    mv = MultiVector()
    mis = MultiindexSet.createCompleteOrderSet(3, 4)
    mv.set_defaults(mis, FlatVector([3, 4, 5]))
    assert_equal(mv[Multiindex([1, 2, 1])], FlatVector([3, 4, 5]))
    assert_equal(mv[Multiindex()], FlatVector([3, 4, 5]))
    assert_raises(KeyError, mv.__getitem__, Multiindex([1, 2, 2]))

def test_equality():
    mv1 = MultiVector()
    mv2 = MultiVector()
    mv3 = MultiVector()
    mv4 = MultiVector()
    mis1 = MultiindexSet.createCompleteOrderSet(3, 4)
    mis2 = MultiindexSet.createCompleteOrderSet(3, 5)
    mv1.set_defaults(mis1, FlatVector([3, 4, 5]))
    mv2.set_defaults(mis1, FlatVector([3, 4, 5]))
    mv3.set_defaults(mis2, FlatVector([3, 4, 5]))
    mv4.set_defaults(mis1, FlatVector([3, 4, 6]))
    assert_true(mv1 == mv2)
    assert_false(mv1 != mv2)
    assert_true(mv1 != mv3)
    assert_false(mv1 == mv3)
    assert_true(mv1 != mv4)
    assert_false(mv1 == mv4)

def test_add():
    mv1 = MultiVector()
    mv2 = MultiVector()
    mv3 = MultiVector()
    mis1 = MultiindexSet.createCompleteOrderSet(3, 4)
    mv1.set_defaults(mis1, FlatVector([3, 4, 5]))
    mv2.set_defaults(mis1, FlatVector([6, 8, 12]))
    mv3.set_defaults(mis1, FlatVector([9, 12, 17]))
    assert_equal(mv1 + mv2, mv3)

def test_neg():
    mv1 = MultiVector()
    mis1 = MultiindexSet.createCompleteOrderSet(3, 4)
    mv1.set_defaults(mis1, FlatVector([3, 4, 5]))
    mv = -mv1
    assert_equal(mv[Multiindex(mis1[-1])], FlatVector([-3, -4, -5]))
    assert_equal(mv1[Multiindex(mis1[-1])], FlatVector([3, 4, 5]))

def test_mul():
    mv1 = MultiVector()
    mv2 = MultiVector()
    mis1 = MultiindexSet.createCompleteOrderSet(3, 4)
    #mis2 = MultiindexSet.createCompleteOrderSet(3, 5)
    mv1.set_defaults(mis1, FlatVector([3, 4, 5]))
    mv2.set_defaults(mis1, FlatVector([6, 8, 10]))
    assert_equal(2 * mv1, mv2)
    assert_equal(mv1 * 2, mv2)
    assert_equal(2.0 * mv1, mv2)
    assert_equal(mv1 * 2.0, mv2)
    assert_equal(mv1[Multiindex()], FlatVector([3, 4, 5]))
    assert_equal(mv1[Multiindex([1, 2, 1])], FlatVector([3, 4, 5]))
    mv1 *= 2.0
    assert_equal(mv1, mv2)

def test_sub():
    mv1 = MultiVector()
    mv2 = MultiVector()
    mi1 = Multiindex([1, 2, 1])
    mi2 = Multiindex([3, 2, 1, 7])
    mis1 = MultiindexSet.createCompleteOrderSet(3, 4)
    mv1.set_defaults(mis1, FlatVector([3, 4, 5]))
    mv2.set_defaults(mis1, FlatVector([6, 8, 10]))
    mv1[mi2] = FlatVector([7, 10, 6])
    mv2[mi2] = FlatVector([2, 6, 13])
    mv3 = mv1 - mv2
    assert_equal(mv3[mi1], FlatVector([-3, -4, -5]))
    assert_equal(mv3[mi2], FlatVector([5, 4, -7]))

def test_mvwp_repr():
    mv = MultiVectorWithProjection()
    mv[Multiindex([1, 2, 3])] = FlatVector([3, 4, 5])
    assert_equal(str(mv), "<MultiVectorWithProjection keys=[<Multiindex inds=[1 2 3]>]>")

def test_mvwp_copy():
    # compares equal to copied MultiVectorWP but not to MultiVector
    mv0 = MultiVector()
    mv1 = MultiVectorWithProjection()
    mis1 = MultiindexSet.createCompleteOrderSet(3, 4)
    mv0.set_defaults(mis1, FlatVector([3, 4, 5]))
    mv1.set_defaults(mis1, FlatVector([3, 4, 5]))
    mv2 = mv1.copy()
    assert_equal(mv2, mv1)
    assert_not_equal(mv2, mv0)

    # compares equal if project methods match, make sure project method is copied
    mv3 = MultiVectorWithProjection(project=lambda : None)
    mv3.set_defaults(mis1, FlatVector([3, 4, 5]))
    mv4 = mv3.copy()
    assert_not_equal(mv2, mv3)
    assert_equal(mv4, mv3)

def test_mvwp_project():
    def pr(src, dest):
        return 2 * src + dest
    mv1 = MultiVectorWithProjection(project=pr)
    mi1 = Multiindex([1, 2, 1])
    mi2 = Multiindex([3, 2, 1, 7])
    v1 = FlatVector([7, 10, 6])
    v2 = FlatVector([2, 6, 13])
    mv1[mi1] = v1
    mv1[mi2] = v2

    assert_equal(mv1.get_projection(mi1, mi2), pr(v1, v2))
    assert_equal(mv1.get_back_projection(mi1, mi2), pr(pr(v1, v2), v1))

    # TODO: clearing of the cache needs to tested

test_main()


from django.contrib import admin
from django.urls import path,include

urlpatterns = [
    path('admin/', admin.site.urls),
    path('cube/',include('cube.urls')),
    path('cars/',include('cars.urls')),
    path('core/', include('core.urls'))

]


import configparser
import json
from os import path

DATA_FILE = 'data.ini'

file_exists = path.exists(DATA_FILE)

def convert_to_dict(string):
    list_ = []
    while True:
        i = string.find('},{') + 1
        if i == 0:
            break
        list_.append(string[:i])
        string = string[i+1:]
    list_.append(string)
    return [json.loads(l) for l in list_]

def add_to_data_file(section_name, **kwargs):
    try:
        config = configparser.ConfigParser()
        config[section_name] = kwargs
        with open(DATA_FILE, 'a') as configfile:
            config.write(configfile)
    except Exception as e:
        print(e)

def update_data_file(section_name, **kwargs):
    p = configparser.ConfigParser()
    with open(DATA_FILE, 'r+') as s:
        p.read_file(s)
        p.remove_section(section_name)
        s.seek(0)
        p.write(s)
        s.truncate()
    add_to_data_file(section_name, **kwargs)

def get_value_in_data_file(section, key):
    config = configparser.ConfigParser()
    config.read(DATA_FILE)
    return config.get(section, key)

def section_in_data_file():
    config = configparser.ConfigParser()
    config.read(DATA_FILE)
    return config.sections()


# file to do expirements without losing past ideas
# put current expirement at bottom and run

import os
import sys

current_dir = os.path.dirname(os.path.abspath(__file__))
parent_dir = os.path.dirname(current_dir)
sys.path.append(parent_dir)

import numpy as np
import time
import helpers.pre_processing as pre
import helpers.helper as helper
import matplotlib.pyplot as plt
import helpers.seed_corr as seed
import helpers.stats as stats
from sklearn.decomposition import FastICA, PCA
from scipy.signal import hilbert

# test SVD and global signal regression
def test_SVD():
    data = helper.load_data_np(
        relative_path="matlab_files/Time_Series_Data/ROI_CBV_Data/reg_cbv_salket_m.mat")
    region_data = pre.bandpass_filter(data, 0.01, 0.1, 4)
    final_data = pre.global_signal_regression(region_data[:, :, :])
    # plt.plot(final_data[9, :])
    # plt.show()

    # ask gpt how to align brain atlas without compressing time series into single image
    corr = helper.get_corr_matrix(final_data[:, :, :], start=2700, end=2800)[0]
    helper.plot_correlation(corr)

# test_SVD()

# test ROI plots,

def test_ROI_plots():
    data = helper.load_data_np(
        relative_path="matlab_files/Time_Series_Data/ROI_CBV_Data/rel_cbv_nalket_m.mat")
    # region_data = pre.bandpass_filter(data, 0.01, 0.1, 4)
    corr_matrix = helper.get_corr_matrix(data, start=1800, end=2400)[1]
    helper.plot_correlation_ROI(corr_matrix)

# test_ROI_plots()

# saving large matlab as np
# tranposed, needs to be tranposed again upon loading for some weird reason

def save_large_np():
    cbv_data = helper.load_large_data(
        'matlab_files/Time_Series_Data/pixel_data/nalket_m_pix.mat')
    print('half way there')
    print(cbv_data.shape)
    np.save('python_data/time_series_data/pixel_data', cbv_data)

# save_large_np()

# is the pixel file even being loaded correctly???

def test_seed():
    pixel = helper.load_from_np(
        "python_data/time_series_data/pixel_data/nalket_m_pix.npy")
    pixel = pixel[0, :, :, :]
    pixel = pre.bandpass_filter(pixel, 0.01, 0.1, 4)
    # pixel = pre.global_signal_regression_proj(pixel)
    roi = helper.load_data_np(
        "matlab_files/Time_Series_Data/ROI_CBV_Data/rel_cbv_nalket_m.mat")[0, :, :]
    roi = pre.bandpass_filter(roi, 0.01, 0.1, 4)
    # roi = pre.global_signal_regression_proj(roi)
    corr = seed.mouse_pixel_corr(pixel, roi, 17, start=1600, end=2000)
    helper.plot_correlation(corr, min=-1, max=1)

# test_seed()

# checkingmaskdimensions

def maskTest():
    a = helper.load_data_np(
        "matlab_files/Time_Series_Data/segment_masks/nalket_m_mask.mat")
    print(a.shape)
# maskTest()

def save_pixel():
    helper.save_large_np("matlab_files/Time_Series_Data/pixel_data/nalket_f_pix.mat",
                         "python_data/time_series_data/pixel_data")
    print("done")
    helper.save_large_np("matlab_files/Time_Series_Data/pixel_data/nalket_m_pix.mat",
                         "python_data/time_series_data/pixel_data")
    print("done")
    helper.save_large_np("matlab_files/Time_Series_Data/pixel_data/salket_f_pix.mat",
                         "python_data/time_series_data/pixel_data")
    print("done")
    helper.save_large_np("matlab_files/Time_Series_Data/pixel_data/salket_m_pix.mat",
                         "python_data/time_series_data/pixel_data")
    print("done")
    helper.save_large_np("matlab_files/Time_Series_Data/pixel_data/nalsal_f_pix.mat",
                         "python_data/time_series_data/pixel_data")
    print("done")
    helper.save_large_np("matlab_files/Time_Series_Data/pixel_data/nalsal_m_pix.mat",
                         "python_data/time_series_data/pixel_data")
    print("done")
# save_pixel()

def test_tscore():
    data = helper.load_data_np(
        "matlab_files/Time_Series_Data/ROI_CBV_Data/rel_cbv_nalket_m.mat")
    # data = helper.load_data_np("matlab_files/Time_Series_Data/ROI_CBV_Data/nalket_m_roi.mat")

    region_data = pre.bandpass_filter(data[:, :, 1200:1800], 0.01, 0.1, 4)
    corr_matrix = helper.get_corr_matrix(region_data)
    sig_matrix = stats.get_sig_val_matrix(
        0.005, 9, corr_matrix[0], corr_matrix[1])
    helper.plot_correlation_ROI(sig_matrix)
# test_tscore()

def test_fisher_tscore():
    data = helper.load_data_np(
        "matlab_files/Time_Series_Data/ROI_CBV_Data/rel_cbv_nalket_m.mat")
    region_data = pre.bandpass_filter(data[:, :, 1200:1800], 0.01, 0.1, 4)
    corr_matrix = helper.get_all_corr_matrix(region_data)
    sig_matrix = stats.fisher_sig_val(corr_matrix, 0.01)
    helper.plot_correlation_ROI(sig_matrix)

# test_fisher_tscore()

def test_load_pix():
    data = helper.load_data_np(
        "matlab_files/Time_Series_Data/pix_area/pix_area_nalket_m.mat")
    print(data[2, :].shape)
    # data = data[1, 60, 57, :]
    # cleaned_data = pre.bandpass_filter(data, 0.01, 0.1, 4)
    # plt.plot(cleaned_data)
    # plt.show()

# test_load_pix()

def test_ROI_segmentation():
    # roi = helper.load_data_np("matlab_files/Time_Series_Data/ROI_CBV_Data/rel_cbv_nalket_m.mat")[5, :, :]
    a = pre.pixel_to_ROI_vectorized("python_data/time_series_data/pixel_data/nalket_m_pix.npy",
                                    "matlab_files/Time_Series_Data/segment_masks/nalket_m_mask.mat",
                                    "matlab_files/Time_Series_Data/pix_area/pix_area_nalket_m.mat", 5)
    # print(np.sum(roi - a))
    a = pre.bandpass_filter(a[:, 1200:1800], 0.01, 0.1, 4)
    a = pre.global_signal_regression_proj(a)
    corr = helper.get_corr_matrix_mouse(a)
    helper.plot_correlation_ROI(corr)

# test_ROI_segmentation()

def gsr_pixel_test():
    t = time.time()
    gsr_pixel = helper.load_from_np(
        "python_data/time_series_data/pixel_data/nalket_m_pix.npy")[3, :, :, :]
    # gsr_pixel = pre.bandpass_filter(gsr_pixel, 0.01, 0.1, 4)
    # gsr_pixel = pre.global_signal_regression_proj(pixel)
    # gsr_pixel = pre.bandpass_filter(gsr_pixel, 0.01, 0.1, 4)
    roi = pre.pixel_to_ROI_pixdata(gsr_pixel, "matlab_files/Time_Series_Data/segment_masks/nalket_m_mask.mat",
                                   "matlab_files/Time_Series_Data/pix_area/pix_area_nalket_m.mat", 3)
    roi = pre.bandpass_filter(roi, 0.01, 0.1, 4)
    corr = helper.get_corr_matrix_mouse(roi)
    print(time.time() - t)
    helper.plot_correlation_ROI(corr)

# gsr_pixel_test()

def test_group_roi():
    two_pixels = helper.load_from_np(
        "python_data/time_series_data/pixel_data/nalket_m_pix.npy")[:2, :, :, :]
    mask = helper.load_data_np(
        "matlab_files/Time_Series_Data/segment_masks/nalket_m_mask.mat")[:2, :, :, :]
    pix_areas = helper.load_data_np(
        "matlab_files/Time_Series_Data/pix_area/pix_area_nalket_m.mat")[:2, :]
    two_pixels = pre.group_to_ROI(two_pixels, mask, pix_areas)

    rois = helper.load_from_np(
        "matlab_files/Time_Series_Data/ROI_CBV_Data/rel_cbv_nalket_m.mat")[:2, :, :]

    print(rois.shape)
    print(two_pixels.shape)
    print(np.sum(two_pixels - rois))

# test_group_roi()

# looping through bandpass is much faster
# vectorization optimizes within function but not for huge datasets and between subjects
# my datset is 40 million withoyt vectorization, 360 million with

def band_time_test():
    pix_data = helper.load_from_np(
        "python_data/time_series_data/pixel_data/nalket_m_pix.npy")

    t1 = time.time()
    pre.global_signal_regression_proj(pix_data)
    t2 = time.time()
    print(t2-t1)
    for i in range(pix_data.shape[0]):
        pre.global_signal_regression_proj(pix_data[i, :, :, :])
    t3 = time.time()
    print(t3 - t2)

# band_time_test()

# succesful, manual works as well as keep

def test_keep_roi():
    pix_data = helper.load_from_np(
        "python_data/time_series_data/pixel_data/nalket_m_pix.npy")[4, :, :, :]
    pix_area = helper.load_data_np(
        "matlab_files/Time_Series_Data/pix_area/pix_area_nalket_m.mat")[4, :]
    roi_data = helper.load_data_np(
        "matlab_files/Time_Series_Data/ROI_CBV_Data/rel_cbv_nalket_m.mat")[4, :, :]
    mask = helper.load_data_np(
        "matlab_files/Time_Series_Data/segment_masks/nalket_m_mask.mat")[4, :, :, :]
    keep = pre.keep_roi_pixels(pix_data, mask)
    new_roi = pre.manual_pixel_to_ROI(keep, mask, pix_area)
    print(np.sum(new_roi - roi_data))

# test_keep_roi()

def plot_region():
    data = helper.load_data_np(
        "matlab_files/Time_Series_Data/ROI_CBV_Data/rel_cbv_nalket_m.mat")
    # compare 4 and 7 to bandpass

    # data[7] = pre.bandpass_filter(data[7], 0.01, 0.1, 4)
    plt.plot(data[2, 12, :])
    plt.show()

# plot_region()

def test_dianni():
    data = helper.load_data_np('matlab_files/dianni_data/nalket_dianni.mat')
    hamming = helper.load_data_np('matlab_files/dianni_data/hamm_win.mat')
    windowed_data = hamming.T * data
    corr = helper.get_corr_matrix(windowed_data)[1]
    helper.plot_correlation_ROI(corr)

# test_dianni()

def dianni_center():
    data = helper.load_data_np('matlab_files/dianni_data/nalket_dianni.mat')
    hamming = helper.load_data_np('matlab_files/dianni_data/hamm_win.mat')
    means = np.mean(data, axis=2, keepdims=True)
    data = (data - means)/means
    windowed_data = hamming.T * data
    corr = helper.get_corr_matrix(windowed_data)[1]
    helper.plot_correlation_ROI(corr)

# dianni_center()

def add_hamming():
    data = helper.load_data_np(
        'matlab_files/Time_Series_Data/ROI_CBV_Data/rel_cbv_nalket_m.mat')
    hamming = helper.load_data_np('matlab_files/dianni_data/hamm_win.mat')
    data = data[:, :, 150:1051]
    data = pre.bandpass_filter(data, 0.01, 0.1, 4)
    data = data * hamming.T
    corr = helper.get_corr_matrix(data)[1]
    helper.plot_correlation_ROI(corr)

# add_hamming()
def gsr_dianni():
    data = helper.load_data_np('matlab_files/dianni_data/nalket_dianni.mat')
    for i in range(data.shape[0]):
        data[i] = pre.global_signal_regression_proj(data[i])
    hamming = helper.load_data_np('matlab_files/dianni_data/hamm_win.mat')
    windowed_data = hamming.T * data
    corr = helper.get_corr_matrix(windowed_data)[1]
    helper.plot_correlation_ROI(corr)
# gsr_dianni()

def ica_test():
    data = helper.load_data_np('matlab_files/dianni_data/nalket_dianni.mat')
    data = data[4, :, :].T
    data_centered = data - data.mean(axis=0)
    pca = PCA(whiten=True)
    data_whitened = pca.fit_transform(data_centered)
    num_components = 5
    ica = FastICA(n_components=num_components)
    ica.fit(data_whitened)
    components = ica.components_
    np.save('python_data/', components)
# ica_test()

# def ica_load():
#     ica = np.load('python_data/ica.npy')
#     print(ica[0])
# ica_load()

def signal():
    data = helper.load_data_np(
        'matlab_files/Time_Series_Data/ROI_CBV_Data/rel_cbv_nalket_m.mat')
    plt.figure()

def phase():
    data = helper.load_data_np('matlab_files/dianni_data/nalket_dianni.mat')[:, :, 0:600]
    data2 = helper.load_data_np('matlab_files/dianni_data/nalket_dianni_baseline.mat')[:, :, 0:600]
    data = pre.hamming(data2)
    hilbert_transform = hilbert(data, axis=-1)
    instantaneous_phase = np.angle(hilbert_transform) * 180/3.14
    plt.plot(instantaneous_phase[3, 4, :])
    plt.show()

#plots between groups, differences need to be plotted
phase()


import logging

import redis
import hashlib
import json

import powergslb.database
import powergslb.system

__all__ = ['RedisCache']

class RedisCache(redis.Redis):
    Error = redis.RedisError
    _status_key = "contents:check"
    _base_key = "record"

    def __enter__(self):
        return self

    def __exit__(self, *_):
      pass

    def __init__(self, **kwargs):
      if 'update_interval' in kwargs: kwargs.pop('update_interval')
      super(RedisCache, self).__init__(**kwargs)

    def reload(self):
      records_hash_key = "records:check"

      with powergslb.database.Database(**powergslb.system.get_config().items('database')) as db:
        rows = db.gslb_records()

      rows_hash = hashlib.sha1(json.dumps(rows, sort_keys=True)).hexdigest()
      cache_hash = self.get(records_hash_key)

      if rows_hash != cache_hash:
        logging.info('{}: update cache from the database'.format(type(self).__name__))
        pipe = self.pipeline()

        for key in self.scan_iter(self._status_key + ":*"):
          pipe.delete(key)
        for key in self.scan_iter(self._base_key + ":*"):
          pipe.delete(key)
        pipe.delete(self._base_key)

        for row in rows:
          #pipe.sadd(self._base_key, row['qname'])

          name_key = self._base_key + ":" + row['qname']
          pipe.sadd(name_key, row['qtype'])

          name_type_key = name_key + ":" + row['qtype']
          pipe.sadd(name_type_key, row['recid'])

          row_key = name_type_key + ":" + str(row['recid'])
          pipe.hmset(row_key, row)

          monitor_key = self._status_key + ":" + str(row['id'])
          pipe.sadd(monitor_key, row['qname'])

        self.set(records_hash_key, rows_hash)
        pipe.execute()
      else:
        logging.info('{}: no rows update needed'.format(type(self).__name__))

    def gslb_check_names(self, content_monitor_id):
      monitor_key = self._status_key + ":" + str(content_monitor_id)
      return self.smembers(monitor_key)

    def gslb_records(self, qname, qtype):
      logging.debug('Redis request: [%s, %s]' % (qname, qtype))
      rows = []
      name_key = self._base_key + ":" + qname

      for row_type in self.smembers(name_key):
        if (qtype == row_type or qtype == "ANY"):
          name_type_key = name_key + ":" + row_type

          for recid in self.smembers(name_type_key):
            row_key = name_type_key + ":" + recid
            rows.append(self.hgetall(row_key))

      return rows

    def get_status(self):
      return self.smembers(self._status_key)

    def check_status(self, member):
      return self.sismember(self._status_key, member)

    def clean_status(self, *members):
      pipe = self.pipeline()
      for member in members:
        self.srem(self._status_key, member)
      pipe.execute()

    def add_status(self, *members):
      pipe = self.pipeline()
      for member in members:
        self.sadd(self._status_key, member)
      pipe.execute()


# -*- coding: utf-8 -*-
# @Time    : 2020/3/7 13:30
# @Author  : Deng Wenxing
# @Email   : dengwenxingae86@163.com
# @File    : person_relation_car.py
# @Software: PyCharm
# @content : 人车关联信息

import sys, os
from pyspark import SparkConf
from pyspark.sql import SparkSession
# from pyspark.sql import functions as fun
from pyspark.sql.functions import *
from pyspark.sql.types import *
import logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s: %(message)s')
logger = logging.getLogger(__name__)

reload(sys)
sys.setdefaultencoding('utf-8')
#warehouse_location = '/user/hive/warehouse/'
conf=SparkConf().set('spark.driver.maxResultSize', '2g')
conf.set('spark.yarn.executor.memoryOverhead', '30g')
conf.set('spark.yarn.am.cores', 5)
conf.set('spark.executor.memory', '2g')
conf.set('spark.executor.instances', 10)
conf.set('spark.executor.cores', 4)
conf.set('spark.executor.extraJavaOptions', '-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+UseG1GC')
#conf.set("spark.sql.warehouse.dir", warehouse_location)

spark = SparkSession \
    .builder \
    .config(conf=conf) \
    .enableHiveSupport() \
    .getOrCreate()

from common import *
commonUtil = CommonUdf(spark)

path_prefix = '/phoebus/_fileservice/users/slmp/shulianmingpin/midfile/company'
save_root = 'relation_theme_extenddir'

def create_uniondf(spark,table_list,tmp_sql):
    '''
    :param spark:
    :param table_list: 所需原始表及其字段对应信息
    :param tmp_sql: 原始sql语句，根据table_list进行格式化
    :return: table_List所有df的合集
    '''
    dfs = []
    for info in table_list:
        sql = tmp_sql.format(**info)
        df = read_orc(spark,add_save_path(info['tablename'],root=save_root))
        if df:
            df.createOrReplaceTempView(info['tablename'])
            df = spark.sql(sql)
            dfs.append(df)
    union_df = reduce(lambda x,y:x.unionAll(y),dfs).dropDuplicates([table_list[0]['col']])
    write_orc(union_df,add_save_path('vertex_'+table_list[0]['type'],root=save_root))

def vertex_im_account():

    # nb_app_dws_per_res_his_accmobile	用户账号与手机号的关系历史全量汇总
    # vt_type	虚拟身份类型
    # user_acc	用户账号
    # mob	手机号码
    # rel_type	关系类型
    # todo: 暂无数据

    table_list = [
        {'tablename':'edge_phone_link_qq','account':'user_account','type':'qq','col':'qq'},
        {'tablename':'edge_case_link_qq','account':'qq','type':'qq','col':'qq'},
        {'tablename':'edge_person_link_qq','account':'user_account','type':'qq','col':'qq'},
        {'tablename':'edge_qq_link_qq','account':'qq1','type':'qq','col':'qq'},
        {'tablename':'edge_qq_link_qq','account':'qq2','type':'qq','col':'qq'},
        {'tablename':'edge_phone_link_wechat','account':'user_account','type':'wechat','col':'wechat'},
        {'tablename':'edge_case_link_wechat','account':'wechat','type':'wechat','col':'wechat'},
        {'tablename':'edge_person_link_wechat','account':'user_account','type':'wechat','col':'wechat'},
        {'tablename':'edge_wechat_link_wechat','account':'wechat1','type':'wechat','col':'wechat'},
        {'tablename':'edge_wechat_link_wechat','account':'wechat2','type':'wechat','col':'wechat'},
        {'tablename':'edge_phone_link_sinablog','account':'user_account','type':'sina','col':'sina'},
        {'tablename':'edge_person_link_sinablog','account':'user_account','type':'sina','col':'sina'},

    ]

    qq_list = filter(lambda row:row.get('type')=='qq',table_list)
    we_list = filter(lambda row:row.get('type')=='wechat',table_list)
    sina_list = filter(lambda row:row.get('type')=='sina',table_list)

    tmp_sql = '''
        select {account} {col} from {tablename} group by {account}
    '''

    create_uniondf(spark,qq_list,tmp_sql)
    create_uniondf(spark,we_list,tmp_sql)
    create_uniondf(spark,sina_list,tmp_sql)

if __name__ == "__main__":
    logger.info('========================start time:%s==========================' % (
        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))

    vertex_im_account()

    logger.info('========================end time:%s==========================' % (
        time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))


from flask import Flask
from flask import render_template
import requests
import json

app = Flask(__name__)

@app.get("/")
def index():
    return render_template('base.html')

def get_meme():
    url = "https://meme-api.com/gimme/wholesomememes"
    response = json.loads(requests.request("GET", url).text)
    meme_large = response["preview"][-2]
    subreddit = response["subreddit"]
    return meme_large, subreddit

@app.get("/about/")
def about():
    meme_pic, subreddit = get_meme()
    return render_template('meme.html', meme_pic=meme_pic, subreddit=subreddit)

if __name__ == '__main__':
    app.run(debug=False)


import pygame
pygame.init() # 반드시 해줘야 되는 부분

# 화면 크기
screen_width = 480 # 가로
screen_height = 640 # 세로
screeen = pygame.display.set_mode((screen_width, screen_height)) # 480 * 640

#화면 타이틀 설정
pygame.display.set_caption("Nado Game") #게임 이름

# 이벤트 루프
running = True # 게임이 진행중인지 아닌지 확인하는거 = flag
while running :
    for event in pygame.event.get(): # 어떤 이벤트가 발생하는지
        if event.type == pygame.QUIT: # 창이 닫히는 이벤트가 발생하면 ( 안쓰면 꺼지지 않음 )
            running = False
# 파이게임 종료
pygame.quit()


"""
Atomic facts.
"""
from __future__ import annotations

import collections
from dataclasses import dataclass
import inspect
import logging
from typing import Any, List, Type
from uuid import UUID

from fact import AttributeFact, RelationshipFact
from fact_store import FactStore, MemoryFactStore
from entities import *
from attribute import *
from attribute import _MESSAGE_TYPE_CLS_LIST_DICT, _MESSAGE_TYPE_FUNCTION_TO_DICT
from message import UserTableMessageType, DictAttributeMapping
from route import Route, MessageRoundabout

logging.basicConfig(level=logging.DEBUG)

#_MESSAGE_TYPE_CLS_LIST_DICT: dict = collections.defaultdict(list)
#_MESSAGE_TYPE_FUNCTION_TO_DICT: dict = collections.defaultdict(dict)

__MISSING__ = "__MISSING__"

def get_entity_parameters(func):
    """
    Inspect the signature etc.
    """
    sig = dict(func._function_signature.parameters.items())
    out = {}
    for parameter in sig.keys():
        entity_name, attribute_name = parameter.split("__")
        entity_obj = globals()[entity_name]
        attribute_obj = globals()[attribute_name]
        logging.debug(entity_obj)
        logging.debug(attribute_obj)
        out[parameter] = (
            entity_obj,
            attribute_obj,
        )
    return out

class Session:
    """
    Holds state and so on.
    """

    def __init__(
        self,
        fact_store_cls: Type = None,
        fact_store_kwargs: dict = None,
        message_roundabout: MessageRoundabout = None,
    ):
        self.fact_store = fact_store_cls(**(fact_store_kwargs or {}))
        self.message_roundabout = message_roundabout
        self.fact_store.session = self

        def _callback_dict():
            function_update_callback_dict = collections.defaultdict(list)

            attribute_function_dict = {}
            obj_names = globals().keys()
            all_obj_dict = {
                obj_name: globals()[obj_name] for obj_name in obj_names
            }
            for obj_name, obj in all_obj_dict.items():
                if (
                    hasattr(obj, "_attribute_function")
                    and obj._attribute_function
                ):
                    attribute_function_dict[obj_name] = obj
                    logging.info(f"Found attribute function: {obj_name}")
                    logging.debug(f"Signature: {inspect.signature(obj)}")
                    logging.debug(f"function name: {obj._function_name}")
                    logging.debug(f"signature: {obj._function_signature}")

            for func in attribute_function_dict.values():
                for (
                    parameter_name,
                    _,
                ) in func._function_signature.parameters.items():
                    if parameter_name in attribute_function_dict.keys():
                        logging.debug(parameter_name)
                        # Set callback
                        entity_name, attribute_name = parameter_name.split(
                            "__"
                        )
                        entity_cls = globals()[entity_name]
                        attribute_cls = globals()[attribute_name]
                        function_update_callback_dict[
                            (
                                entity_cls,
                                attribute_cls,
                            )
                        ].append(func)
            return function_update_callback_dict

        self.callback_dict = _callback_dict()

    def __enter__(self, *args, **kwargs):
        return self

    def __exit__(self, *args, **kwargs):
        pass

    def __call__(self, message):
        # Route sample message to identify its type.
        message_type = self.message_roundabout(message)
        logging.debug(message_type)
        logging.debug(_MESSAGE_TYPE_CLS_LIST_DICT[message_type])
        kwargs = message_type().get_kwargs(message)
        logging.debug(f"message_type kwargs: {kwargs}")
        for attribute_function in _MESSAGE_TYPE_CLS_LIST_DICT[message_type]:
            value, attribute_cls = attribute_function(**kwargs)
            logging.debug(value)
            id_keypath = _MESSAGE_TYPE_FUNCTION_TO_DICT[attribute_function][
                message_type
            ]["id_keypath"]
            entity_type = _MESSAGE_TYPE_FUNCTION_TO_DICT[attribute_function][
                message_type
            ]["entity_type"]
            id_value = DictAttributeMapping.lookup_keypath(message, id_keypath)
            logging.debug(id_value)
            logging.debug(entity_type)
            _attribute = AttributeFact(
                entity_type=entity_type,
                entity_id=id_value,
                attribute=attribute_cls,
                value=value,
            )
            logging.debug(_attribute)
            session.fact_store(_attribute)

        for (
            relationship_name,
            relationship_config,
        ) in message_type.relationship_mapping.items():
            source_entity_type = globals()[
                relationship_config["source"]["entity_type"]
            ]
            target_entity_type = globals()[
                relationship_config["target"]["entity_type"]
            ]
            relationship_cls = globals()[relationship_name]
            source_entity_id = DictAttributeMapping.lookup_keypath(
                message,
                message_type.argument_mapping[
                    message_type.relationship_mapping[relationship_name][
                        "source"
                    ]["entity_id_keypath"]
                ],
            )
            target_entity_id = DictAttributeMapping.lookup_keypath(
                message,
                message_type.argument_mapping[
                    message_type.relationship_mapping[relationship_name][
                        "target"
                    ]["entity_id_keypath"]
                ],
            )
            relationship_fact = RelationshipFact(
                source_entity_type=source_entity_type,
                target_entity_type=target_entity_type,
                source_entity_id=source_entity_id,
                target_entity_id=target_entity_id,
                relationship=relationship_cls,
            )
            self.fact_store(relationship_fact)

if __name__ == "__main__":
    _message_roundabout = MessageRoundabout()

    ROUTE = Route(
        message_type=UserTableMessageType,
        keypath=[
            "metadata",
            "table",
        ],
        value_match="users",
    )
    _message_roundabout.add_route(ROUTE)

    SAMPLE_MESSAGE = {
        "cdc": {
            "columns": {
                "id": 4,
                "name": "Bob Smith",
                "age": 48,
                "state": "FL",
            }
        },
        "metadata": {"foo": 1, "bar": 2, "table": "users"},
    }

    with Session(
        fact_store_cls=MemoryFactStore, message_roundabout=_message_roundabout
    ) as session:
        session(SAMPLE_MESSAGE)
        for i in session.fact_store:
            print(i)


import unittest
import trytond.tests.test_tryton
import datetime
from decimal import Decimal
from trytond.tests.test_tryton import ModuleTestCase, with_transaction
from trytond.modules.company.tests import create_company, set_company
from trytond.modules.account.tests import create_chart, get_fiscalyear
from trytond.pool import Pool
from trytond.transaction import Transaction
from trytond.exceptions import UserError
from trytond.model.modelsql import SQLConstraintError, RequiredValidationError

__all__ = [
    'create_bank_account',
    'create_cash_bank',
    'create_receipt',
    'create_sequence',
    'create_journal',
    'create_fiscalyear'
    ]

class CashBankTestCase(ModuleTestCase):
    'Test CashBank module'
    module = 'cash_bank'

    @with_transaction()
    def test_cash_bank(self):
        pool = Pool()
        Account = pool.get('account.account')
        Config = pool.get('cash_bank.configuration')
        Receipt = pool.get('cash_bank.receipt')
        ReceiptLine = pool.get('cash_bank.receipt.line')
        DocumentType = pool.get('cash_bank.document.type')
        Document = pool.get('cash_bank.document')
        Docs = pool.get('cash_bank.document-cash_bank.receipt')
        Transfer = pool.get('cash_bank.transfer')
        Convertion = pool.get('cash_bank.convertion')
        CashBank = pool.get('cash_bank.cash_bank')
        ReceiptType = pool.get('cash_bank.receipt_type')

        party = self._create_party('Party test', None)

        transaction = Transaction()

        company = create_company()
        with set_company(company):
            create_chart(company)
            create_fiscalyear(company)

            account_transfer, = Account.search([
                    ('name', '=', 'Main Expense'),
                    ])
            account_cash, = Account.search([
                    ('name', '=', 'Main Cash'),
                    ])
            account_revenue, = Account.search([
                    ('name', '=', 'Main Revenue'),
                    ])
            account_expense, = Account.search([
                    ('name', '=', 'Main Expense'),
                    ])

            journal = create_journal(company, 'journal_cash')

            config = Config(
                account_transfer=account_transfer)
            config.save()

            cheque_type = DocumentType(name='Cheque')
            cheque_type.save()

            sequence = create_sequence(
                'Cash/Bank Sequence',
                'cash_bank.receipt',
                company)
            sequence_convertion = create_sequence(
                'Cash/Bank Convertion',
                'cash_bank.convertion',
                company)

            config.convertion_seq = sequence_convertion
            config.save()

            cash = create_cash_bank(
                company, 'Main Cashier', 'cash',
                journal, account_cash, sequence
                )
            self.assertEqual(len(cash.receipt_types), 2)

            _, bank_account = create_bank_account(
                party_bank=self._create_party('Party Bank', None),
                party_owner=company.party)

            with self.assertRaises(SQLConstraintError):
                # Must be a diferent account
                bank = create_cash_bank(
                    company, 'Main Bank', 'bank',
                    journal, account_cash, sequence,
                    bank_account
                    )

            with self.assertRaises(RequiredValidationError):
                # Bank Account is required for type bank
                bank = create_cash_bank(
                    company, 'Main Bank', 'bank',
                    journal, account_revenue, sequence
                    )
            ReceiptType.delete(ReceiptType.search(
                [('cash_bank.type', '=', 'bank')]))
            CashBank.delete(CashBank.search(
                [('type', '=', 'bank')]))

            bank = create_cash_bank(
                company, 'Main Bank', 'bank',
                journal, account_revenue, sequence,
                bank_account
                )
            self.assertEqual(len(bank.receipt_types), 2)

            date = datetime.date.today()

            # Receipt Cash IN
            receipt = create_receipt(
                company, cash, 'in', date)

            receipt.cash = Decimal('100.0')
            receipt.save()

            self.assertEqual(receipt.state, 'draft')
            self.assertEqual(receipt.diff, Decimal('-100.0'))

            line = ReceiptLine(
                receipt=receipt,
                amount=Decimal('100.0'),
                account=account_revenue
                )

            receipt.lines = [line]
            receipt.save()

            self.assertEqual(len(receipt.lines), 1)
            self.assertEqual(receipt.diff, Decimal('0.0'))
            self.assertEqual(receipt.state, 'draft')
            self.assertEqual(receipt.move, None)

            Receipt.confirm([receipt])
            self.assertEqual(receipt.state, 'confirmed')
            self.assertEqual(receipt.move.state, 'draft')

            Receipt.cancel([receipt])
            self.assertEqual(receipt.state, 'cancel')
            self.assertEqual(receipt.move, None)

            Receipt.draft([receipt])
            self.assertEqual(receipt.state, 'draft')
            self.assertEqual(receipt.move, None)

            Receipt.confirm([receipt])
            self.assertEqual(receipt.state, 'confirmed')
            self.assertEqual(receipt.move.state, 'draft')

            Receipt.post([receipt])
            self.assertEqual(receipt.state, 'posted')
            self.assertEqual(receipt.move.state, 'posted')

            # Nothing change because 'posted' to 'cancel'
            # not in cls._transitions
            Receipt.cancel([receipt])
            self.assertEqual(receipt.state, 'posted')
            self.assertEqual(receipt.move.state, 'posted')

            # Receipt Cash OUT
            receipt = create_receipt(
                company, cash, 'out', date, party)

            receipt.cash = Decimal('100.0')
            receipt.save()

            line = ReceiptLine(
                receipt=receipt,
                amount=Decimal('100.0'),
                account=account_expense
                )

            receipt.lines = [line]
            receipt.save()

            Receipt.confirm([receipt])
            Receipt.post([receipt])

            # 'out' receipts can not create documents
            receipt = create_receipt(
                company, cash, 'out', date)
            receipt.cash = Decimal('10.0')
            receipt.save()
            with self.assertRaises(UserError):
                docs = [
                    self._get_document(
                        cheque_type, Decimal('20.0'), date, 'a'),
                    self._get_document(
                        cheque_type, Decimal('30.0'), date, 'b'),
                    self._get_document(
                        cheque_type, Decimal('40.0'), date, 'c'),
                    ]
                receipt.documents = docs
                receipt.save()
            Receipt.delete([receipt])
            Document.delete(Document.search([]))

            # Receipt IN with cash and documents

            receipt = create_receipt(
                company, cash, 'in', date)
            receipt.cash = Decimal('10.0')

            docs = [
                self._get_document(
                    cheque_type, Decimal('20.0'), date, 'abc'),
                self._get_document(
                    cheque_type, Decimal('30.0'), date, 'def'),
                self._get_document(
                    cheque_type, Decimal('40.0'), date, 'ghi'),
                ]
            receipt.documents = docs
            receipt.save()

            self.assertEqual(len(receipt.documents), 3)
            self.assertEqual(receipt.total_documents, Decimal('90.0'))
            self.assertEqual(receipt.total, Decimal('100.0'))
            self.assertEqual(receipt.diff, Decimal('-100.0'))

            self._verify_document('abc', receipt.id)
            self._verify_document('def', receipt.id)
            self._verify_document('ghi', receipt.id)

            line = ReceiptLine(
                receipt=receipt,
                amount=Decimal('100.0'),
                account=account_revenue
                )
            receipt.lines = [line]
            receipt.save()

            Receipt.confirm([receipt])

            with self.assertRaises(UserError):
                # Should be 'draft' state
                Receipt.delete([receipt])

            Receipt.cancel([receipt])
            Receipt.draft([receipt])
            Receipt.delete([receipt])

            docs = Docs.search([])  # cash_bank.document-cash_bank.receipt
            self.assertEqual(len(docs), 0)

            docs = Document.search([])  # cash_bank.document
            self.assertEqual(len(docs), 3)
            self._verify_document('abc', None)
            self._verify_document('def', None)
            self._verify_document('ghi', None)

            ########################
            # Documents domain tests
            ########################

            receipt_1 = create_receipt(
                company, cash, 'in', date)
            receipt_1.cash = Decimal('10.0')
            receipt_1.lines = [
                ReceiptLine(
                    amount=Decimal('100.0'),
                    account=account_expense
                )]
            receipt_1.documents = Document.search([])
            receipt_1.save()
            self._verify_document('abc', receipt_1.id)
            self._verify_document('def', receipt_1.id)
            self._verify_document('ghi', receipt_1.id)

            # if documents are detached then returns to a previous receipt
            receipt_1.documents = [Document.search([])[0]]
            receipt_1.save()
            self._verify_document('abc', receipt_1.id)
            self._verify_document('def', None)
            self._verify_document('ghi', None)
            Receipt.delete([receipt_1])

            # Ensure that if docs belongs to a receipt 'in'
            # no other receipt 'in' can add them
            receipt_1 = create_receipt(
                company, cash, 'in', date)
            receipt_1.cash = Decimal('10.0')
            receipt_1.lines = [
                ReceiptLine(
                    amount=Decimal('100.0'),
                    account=account_expense
                )]
            receipt_1.documents = Document.search([])
            receipt_1.save()
            self._validate_domain_in(Receipt, Document, receipt_1)

            # Same if receipt_1 is confirmed
            Receipt.confirm([receipt_1])
            self._validate_domain_in(Receipt, Document, receipt_1)

            # Same if receipt_1 is canceled
            Receipt.cancel([receipt_1])
            self.assertEqual(receipt_1.state, 'cancel')
            self.assertEqual(receipt_1.move, None)
            self._validate_domain_in(Receipt, Document, receipt_1)

            # Same if receipt_1 is draft
            Receipt.draft([receipt_1])
            self._validate_domain_in(Receipt, Document, receipt_1)

            # Same if receipt_1 is confirmed
            Receipt.confirm([receipt_1])
            self._validate_domain_in(Receipt, Document, receipt_1)

            # Same if receipt_1 is posted
            Receipt.post([receipt_1])
            self.assertEqual(receipt_1.state, 'posted')
            self.assertEqual(receipt_1.move.state, 'posted')
            self._validate_domain_in(Receipt, Document, receipt_1)

            last_docs_receipt_id = receipt_1.id

            # Group of Receipts
            receipt_grp_1 = create_receipt(
                company, cash, 'in', date)
            receipt_grp_1.cash = Decimal('10.0')
            receipt_grp_2 = create_receipt(
                company, cash, 'in', date)
            receipt_grp_2.cash = Decimal('20.0')

            Receipt.save([receipt_grp_1, receipt_grp_2])
            self.assertEqual(receipt_grp_1.cash, Decimal('10.0'))
            self.assertEqual(receipt_grp_2.cash, Decimal('20.0'))

            receipt_grp_1.cash = Decimal('15.0')
            receipt_grp_2.cash = Decimal('25.0')

            Receipt.save([receipt_grp_1, receipt_grp_2])
            self.assertEqual(receipt_grp_1.cash, Decimal('15.0'))
            self.assertEqual(receipt_grp_2.cash, Decimal('25.0'))

            #####################################
            # Transfer (specially with documents)
            #####################################

            cash_2 = create_cash_bank(
                company, 'Cashier 2', 'cash',
                journal, account_expense, sequence
                )

            with self.assertRaises(UserError):
                # Raise document domain error because
                # documents are not in cash_2
                transfer = Transfer(
                    company=company,
                    date=date,
                    cash_bank_from=cash_2,
                    type_from=cash_2.receipt_types[1],  # out
                    cash_bank_to=cash,
                    type_to=cash.receipt_types[0],  # in
                    documents=Document.search([])
                    )
                transfer.save()

            transfer = Transfer(
                company=company,
                date=date,
                cash=Decimal('10.0'),
                cash_bank_from=cash,
                type_from=cash.receipt_types[1],  # out
                cash_bank_to=cash_2,
                type_to=cash_2.receipt_types[0],  # in
                documents=Document.search([])
                )
            transfer.save()
            self.assertEqual(transfer.total_documents, Decimal('90.0'))
            self.assertEqual(transfer.total, Decimal('100.0'))
            self.assertEqual(transfer.state, 'draft')
            self.assertEqual(transfer.receipt_from, None)
            self.assertEqual(transfer.receipt_to, None)

            Transfer.confirm([transfer])
            self.assertEqual(transfer.state, 'confirmed')
            self.assertEqual(transfer.receipt_from.transfer, transfer)
            self.assertEqual(transfer.receipt_to.transfer, transfer)
            self.assertEqual(transfer.receipt_from.state, 'confirmed')
            self.assertEqual(transfer.receipt_to.state, 'confirmed')

            self._verify_document('abc', transfer.receipt_to.id)
            self._verify_document('def', transfer.receipt_to.id)
            self._verify_document('ghi', transfer.receipt_to.id)

            Transfer.cancel([transfer])
            self.assertEqual(transfer.state, 'cancel')
            self.assertEqual(transfer.receipt_from.transfer, transfer)
            self.assertEqual(transfer.receipt_to.transfer, transfer)
            self.assertEqual(transfer.receipt_from.state, 'cancel')
            self.assertEqual(transfer.receipt_to.state, 'cancel')

            self._verify_document('abc', transfer.receipt_to.id)
            self._verify_document('def', transfer.receipt_to.id)
            self._verify_document('ghi', transfer.receipt_to.id)

            Transfer.draft([transfer])
            self.assertEqual(transfer.state, 'draft')
            self.assertEqual(transfer.receipt_from, None)
            self.assertEqual(transfer.receipt_to, None)

            # Documents go back to the origin receipt
            self._verify_document('abc', last_docs_receipt_id)
            self._verify_document('def', last_docs_receipt_id)
            self._verify_document('ghi', last_docs_receipt_id)

            Transfer.confirm([transfer])
            self.assertEqual(transfer.state, 'confirmed')
            self.assertEqual(transfer.receipt_from.transfer, transfer)
            self.assertEqual(transfer.receipt_to.transfer, transfer)
            self.assertEqual(transfer.receipt_from.state, 'confirmed')
            self.assertEqual(transfer.receipt_to.state, 'confirmed')

            self._verify_document('abc', transfer.receipt_to.id)
            self._verify_document('def', transfer.receipt_to.id)
            self._verify_document('ghi', transfer.receipt_to.id)

            Transfer.post([transfer])
            self.assertEqual(transfer.state, 'posted')
            self.assertEqual(transfer.receipt_from.transfer, transfer)
            self.assertEqual(transfer.receipt_to.transfer, transfer)
            self.assertEqual(transfer.receipt_from.state, 'posted')
            self.assertEqual(transfer.receipt_to.state, 'posted')

            self._verify_document('abc', transfer.receipt_to.id)
            self._verify_document('def', transfer.receipt_to.id)
            self._verify_document('ghi', transfer.receipt_to.id)

            # Convertions

            with self.assertRaises(UserError):
                # Documents are in cash_2
                convertion = Convertion(
                    cash_bank=cash,
                    date=date,
                    documents=Document.search([])
                    )
                convertion.save()

            convertion = Convertion(
                cash_bank=cash_2,
                date=date,
                documents=Document.search([])
                )
            convertion.save()

            Convertion.confirm([convertion])
            self.assertEqual(convertion.state, 'confirmed')
            docs = Document.search([])
            for doc in docs:
                self.assertEqual(doc.convertion.id, convertion.id)

            Convertion.cancel([convertion])
            self.assertEqual(convertion.state, 'cancel')
            docs = Document.search([])
            for doc in docs:
                self.assertEqual(doc.convertion.id, convertion.id)

            Convertion.draft([convertion])
            self.assertEqual(convertion.state, 'draft')
            docs = Document.search([])
            for doc in docs:
                self.assertEqual(doc.convertion, None)

    def _validate_domain_in(self, Receipt, Document, receipt_1):
        self._verify_document('abc', receipt_1.id)
        self._verify_document('def', receipt_1.id)
        self._verify_document('ghi', receipt_1.id)

        receipt_2 = create_receipt(
            receipt_1.company, receipt_1.cash_bank, 'in', receipt_1.date)
        receipt_2.cash = Decimal('10.0')
        receipt_2.save()
        with self.assertRaises(UserError):
            receipt_2.documents = Document.search([])
            receipt_2.save()
        receipt_2.documents = None
        receipt_2.save()
        Receipt.delete([receipt_2])

        # Verify docs again
        self._verify_document('abc', receipt_1.id)
        self._verify_document('def', receipt_1.id)
        self._verify_document('ghi', receipt_1.id)

    def _verify_document(self, reference, receipt_id):
        pool = Pool()
        Document = pool.get('cash_bank.document')
        Docs = pool.get('cash_bank.document-cash_bank.receipt')

        doc = Document.search([
            ('reference', '=', reference)
            ])[0]

        docs = Docs.search(
            [('document', '=', doc.id)], order=[('id', 'ASC')])

        if receipt_id is None:
            self.assertEqual(doc.last_receipt, receipt_id)
            self.assertEqual(docs, [])
        else:
            self.assertEqual(doc.last_receipt.id, receipt_id)
            self.assertEqual(receipt_id, docs[-1].receipt.id)

    def _get_document(self, type_, amount, date, reference):
        Document = Pool().get('cash_bank.document')
        doc = Document(
            type=type_,
            amount=amount,
            date=date,
            reference=reference
            )
        return doc

    @classmethod
    def _create_party(cls, name, account):
        pool = Pool()
        Party = pool.get('party.party')
        Address = pool.get('party.address')
        addr = Address(
            name=name,
            )
        party = Party(
            name=name,
            account_receivable=account,
            addresses=[addr],
            )
        party.save()
        return party

def create_fiscalyear(company):
    pool = Pool()
    FiscalYear = pool.get('account.fiscalyear')
    InvoiceSequence = pool.get(
        'account.fiscalyear.invoice_sequence')

    invoice_seq = create_sequence(
        'Invoice Sequence', 'account.invoice', company.id, True)

    seq = InvoiceSequence()
    seq.company = company
    seq.out_invoice_sequence = invoice_seq
    seq.out_credit_note_sequence = invoice_seq
    seq.in_invoice_sequence = invoice_seq
    seq.in_credit_note_sequence = invoice_seq

    fy = get_fiscalyear(company)
    fy.invoice_sequences = [seq]
    fy.save()
    FiscalYear.create_period([fy])

def create_journal(company, fs_id):
    pool = Pool()
    ModelData = pool.get('ir.model.data')
    Journal = pool.get('account.journal')
    journal = Journal(ModelData.get_id('account', fs_id))
    return journal

def create_receipt_types(name, sequence):
    pool = Pool()
    ReceiptType = pool.get('cash_bank.receipt_type')
    types = ['in', 'out']

    res = []
    for t in types:
        rt = ReceiptType(
            name=name + ' ' + t,
            type=t,
            sequence=sequence,
        )
        res.append(rt)
    return res

def create_sequence(name, code, company, is_strict=False):
    pool = Pool()
    if is_strict:
        Sequence = pool.get('ir.sequence.strict')
    else:
        Sequence = pool.get('ir.sequence')
    seq = Sequence(
        name=name,
        code=code,
        company=company,
        type='incremental'
        )
    seq.save()
    return seq

def create_bank_account(party_bank, party_owner):
    pool = Pool()
    Bank = pool.get('bank')
    Account = pool.get('bank.account')
    Number = pool.get('bank.account.number')
    PartyAccount = pool.get('bank.account-party.party')

    bank = Bank(
        party=party_bank,
        )
    bank.save()

    account = Account(
        bank=bank,
        numbers=[Number(
            type='other',
            number='12345678'
            )]
        )
    account.save()

    party_account = PartyAccount(
        owner=party_owner,
        account=account
        )
    party_account.save()

    return bank, account

def create_cash_bank(
        company, name, type_, journal, account,
        receipt_sequence, bank_account=None):
    CashBank = Pool().get('cash_bank.cash_bank')

    cash = CashBank(
        company=company,
        name=name,
        type=type_,
        journal_cash_bank=journal,
        account=account,
        bank_account=bank_account,
        receipt_types=create_receipt_types(name, receipt_sequence)
        )
    cash.save()
    return cash

def create_receipt(
        company, cash_bank, receipt_type, date, party=None):
    pool = Pool()
    Receipt = pool.get('cash_bank.receipt')
    ReceiptType = pool.get('cash_bank.receipt_type')

    type_ = ReceiptType.search([
        ('cash_bank', '=', cash_bank.id),
        ('type', '=', receipt_type)])[0]

    receipt = Receipt(
        company=company,
        cash_bank=cash_bank,
        type=type_,
        date=date,
        party=party,
        )
    return receipt

def suite():
    suite = trytond.tests.test_tryton.suite()
    suite.addTests(unittest.TestLoader().loadTestsFromTestCase(
        CashBankTestCase))
    return suite


# 滑动窗口最大值
# 给定一个数组 nums，有一个大小为 k 的滑动窗口从数组的最左侧移动到数组的最右侧。你只可以看到在滑动窗口 k 内的数字。滑动窗口每次只向右移动一位。
#
# 返回滑动窗口最大值。
#
# 示例:
#
# 输入: nums = [1,3,-1,-3,5,3,6,7], 和 k = 3
# 输出: [3,3,5,5,6,7]
# 解释:
#
# 滑动窗口的位置                最大值
# ---------------               -----
# [1  3  -1] -3  5  3  6  7       3
# 1 [3  -1  -3] 5  3  6  7       3
# 1  3 [-1  -3  5] 3  6  7       5
# 1  3  -1 [-3  5  3] 6  7       5
# 1  3  -1  -3 [5  3  6] 7       6
# 1  3  -1  -3  5 [3  6  7]      7

from typing import List

# 双端队列(deque)
class Solution:
    def maxSlidingWindow(self, nums: List[int], k: int) -> List[int]:
        if not nums:
            return []
        window, res = [], []
        for i, x in enumerate(nums):
            if i >= k and window[0] <= i - k:
                window.pop(0)
            while window and nums[window[-1]] <= x:
                window.pop()
            window.append(i)
            if i >= k - 1:
                res.append(nums[window[0]])
        return res

if __name__ == '__main__':
    soul = Solution()
    print(soul.maxSlidingWindow([1, 3, -1, -3, 5, 3, 6, 7], 3))


import sys

import os
import tornado.ioloop
import tornado.web
import tornado.template
from model import ModelManager
from handlers import ShowListHandler, EpisodeHandler, UpdateStateHandler, AddedEpisodesHandler, NextEpisodesHandler, StartedEpisodesHandler, RefreshHandler
from core import configurator

if __name__ == '__main__':
    sys.path.insert(0, os.path.join(os.path.dirname(__file__)))

    configurator.init('settings.json')

    ModelManager.refresh()

    class Application(tornado.web.Application):

        def __init__(self):
            root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
            handlers = [
                # GET
                (r"/shows", ShowListHandler),
                (r"/episodes/([^/]*)", EpisodeHandler),
                (r"/started", StartedEpisodesHandler),
                (r"/added", AddedEpisodesHandler),
                (r"/next", NextEpisodesHandler),
                (r"/refresh", RefreshHandler),

                # PUT
                (r"/update", UpdateStateHandler),

                # STATIC
                (r"/content/(.*)", tornado.web.StaticFileHandler,
                 {"path": configurator.get()['settings']['search_path']}),
                (r"/(.*)", tornado.web.StaticFileHandler, {"path": root, "default_filename": "app/index.html"}),
            ]

            settings = {
                'debug': True,
                'template_path': os.path.join(os.path.dirname(__file__), 'templates'),
            }
            tornado.web.Application.__init__(self, handlers, **settings)

    application = Application()

    application.listen(8888)
    print("Starting tornado loop ...")
    tornado.ioloop.IOLoop.instance().start()