# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Installs packages using pip and tests them for version compatibility.

Conceptually, it runs:
  $ pip install <packages> 2>out.txt && \\
    (pip check >out.txt; \\
     pip freeze >requirements.txt)
and returns the contents of "out.txt" and "requirements.txt".

See https://pip.pypa.io/en/stable/user_guide/.
"""

import datetime
import enum
import json
import logging
import os.path
import shlex
import subprocess
import tempfile
import urllib.request

from typing import Any, List, Mapping, Optional

PYPI_URL = 'https://pypi.org/pypi/'

class PipError(Exception):
    """A pip command failed in an unexpected way."""

    def __init__(self, command: List[str], returncode: int, stderr_path: str):
        super(PipError, self).__init__(
            'pip command ({command}) failed with error [{returncode}], '
            'logs at: {stderr_path}'.format(
                command=command, returncode=returncode,
                stderr_path=stderr_path))
        self.command = command
        self.returncode = returncode
        self.stderr_path = stderr_path

    @property
    def command_string(self) -> str:
        return ' '.join(shlex.quote(c) for c in self.command)

@enum.unique
class PipCheckResultType(enum.Enum):
    """The combined results of "pip install" and "pip check".

    SUCCESS: Indicates that "pip install <packages> && pip check" completed
        successfully.
    INSTALL_ERROR: Indicates that "pip install <packages>" failed.
    CHECK_WARNING: Indicates that "pip check" completed with a non-zero exit
        code.
    """
    SUCCESS = 'success'
    INSTALL_ERROR = 'install_error'
    CHECK_WARNING = 'check_warning'

class PipCheckResult:
    """The results of "pip install <packages> && pip check".

    Attributes:
        packages: The list of packages that were installed
            e.g. ['tensorflow', 'numpy'].
        result_type: The result of "pip install <packages> && pip check".
        result_text: The text output of "pip install && pip check". For
            example: "Could not find a version that satisfies the requirement
            tensorflow." or "numpy has requirement six<3, but you have six
            4.0.1."
            Will be None if "pip install <packages> && pip check" completed
            without error.
        dependency_info: The text output of "pip list" i.e. the packages that
            were installed as a product of the "pip install <packages>"
            command. Dict which stores the dependency versions and latest
            release time, together with the timestamp that run this check.

            e.g. "dependency_info": {
                    "astroid": {
                        "installed_version": "1.6.5",
                        "latest_version": "1.6.5",
                        "current_time": "2018-06-14T13:46:04.180159",
                        "latest_version_time": "2018-06-06T15:08:26",
                        "is_latest": true
                    },
                    ...
                }
    """

    def __init__(self,
                 packages: List[str],
                 result_type: PipCheckResultType,
                 result_text: Optional[str] = None,
                 dependency_info: Optional[Mapping[str, Any]] = None):
        """Initializer for PipCheckResult/

        Args:
            packages: The list of packages that were installed
                e.g. ['tensorflow', 'numpy'].
            result_type: The result of "pip install <packages> && pip check".
            result_text: The text output of "pip install && pip check".
            dependency_info: The text output of "pip list" i.e. the packages
                that were installed as a product of the "pip install <packages"
                command. And also the latest release date get by querying
                Pypi json API and the timestamp when running this check.
        """

        self._packages = packages
        self._result_type = result_type
        self._result_text = result_text
        self._dependency_info = dependency_info

    def __eq__(self, other):
        return (isinstance(other, PipCheckResult) and
                self.packages == other.packages and
                self.result_type == other.result_type and
                self.result_text == other.result_text and
                self.dependency_info == other.dependency_info)

    def __repr__(self):
        return ('PipCheckResult(packages={!r}, result_type={!r}, ' +
                'result_text={!r}, dependency_info={!r})').format(
            self.packages,
            self.result_type,
            self.result_text,
            self.dependency_info)

    def with_extra_attrs(self, dependency_info: Optional[str] = None):
        """Return a new PipCheckResult with extra attributes."""
        return PipCheckResult(self.packages,
                              self.result_type,
                              self.result_text,
                              dependency_info=dependency_info)

    @property
    def packages(self) -> List[str]:
        return self._packages

    @property
    def result_type(self) -> PipCheckResultType:
        return self._result_type

    @property
    def result_text(self) -> Optional[str]:
        return self._result_text

    @property
    def dependency_info(self) -> Optional[Mapping[str, Any]]:
        return self._dependency_info

class _OneshotPipCheck():
    """Execute a single pip version compatibility check.

    Conceptually, it runs:
      $ pip install <packages> 2>out.txt && \\
        (pip check >out.txt; \\
         pip freeze >requirements.txt)
    and returns the contents of "out.txt" and "requirements.txt".

    See https://pip.pypa.io/en/stable/user_guide/.
    """

    def __init__(self,
                 pip_command: List[str],
                 packages: List[str],
                 tmp_path: str,
                 clean: bool):
        """Initializes _OneshotPipCheck with the arguments needed to run pip.

        Args:
            pip_command: The arguments to use when invoking pip e.g.
                ['python3', '-m', 'pip'].
            packages: The packages to check for compatibility e.g.
                ['numpy', 'tensorflow'].
            tmp_path: The file system path to use for temporary files
                e.g."/tmp".
            clean: If True then all previously installed packages will be
                uninstalled before installing "packages".
        """
        self._pip_command = pip_command
        self._packages = packages
        self._tmp_path = tmp_path
        self._output_directory = None
        self._clean = clean

    @staticmethod
    def _run_command(command, stdout_path, stderr_path, raise_on_failure=True):
        with open(stdout_path, 'w') as stdout_file, open(stderr_path,
                                                         'w') as stderr_file:
            completed_command = subprocess.run(
                command, stdout=stdout_file, stderr=stderr_file)

        logging.debug('Running %s [returncode=%s]', command,
                      completed_command.returncode)
        if completed_command.returncode and raise_on_failure:
            raise PipError(command, completed_command.returncode, stderr_path)

        return completed_command.returncode

    @staticmethod
    def _call_pypi_json_api(pkg_name, pkg_version):
        pypi_pkg_url = PYPI_URL + '{}/{}/json'.format(pkg_name, pkg_version)

        try:
            r = urllib.request.Request(pypi_pkg_url)

            with urllib.request.urlopen(r) as f:
                result = json.loads(f.read().decode('utf-8'))
        except urllib.error.HTTPError:
            logging.error('Package {} with version {} not found in Pypi'.
                          format(pkg_name, pkg_version))
            return None
        return result

    def _build_command(self, subcommands):
        return self._pip_command + subcommands

    def _freeze(self, requirements_file_path):
        command = self._build_command(['freeze'])
        std_err_path = os.path.join(self._output_directory, 'freeze-error.txt')
        self._run_command(command, requirements_file_path, std_err_path)

    def _uninstall(self, requirements_file_path):
        std_out_path = os.path.join(self._output_directory,
                                    'uninstall-out.txt')
        std_err_path = os.path.join(self._output_directory,
                                    'uninstall-error.txt')
        command = self._build_command(
            ['uninstall', '--yes', '-r', requirements_file_path])
        self._run_command(command, std_out_path, std_err_path)

    def _read(self, path):
        with open(path) as f:
            return f.read()

    def _install(self):
        std_out_path = os.path.join(self._output_directory,
                                    'install-out.txt')
        std_err_path = os.path.join(self._output_directory,
                                    'install-error.txt')
        command = self._build_command(['install', '-U'] + self._packages)
        returncode = self._run_command(
            command, std_out_path, std_err_path, raise_on_failure=False)
        if returncode:
            return PipCheckResult(self._packages,
                                  PipCheckResultType.INSTALL_ERROR,
                                  self._read(std_err_path))
        return PipCheckResult(self._packages, PipCheckResultType.SUCCESS)

    def _check(self):
        std_out_path = os.path.join(self._output_directory, 'check-out.txt')
        std_err_path = os.path.join(self._output_directory, 'check-error.txt')
        command = self._build_command(['check'])
        returncode = self._run_command(
            command, std_out_path, std_err_path, raise_on_failure=False)
        if returncode:
            return PipCheckResult(self._packages,
                                  PipCheckResultType.CHECK_WARNING,
                                  self._read(std_out_path))
        return PipCheckResult(self._packages, PipCheckResultType.SUCCESS)

    def _list(self):
        """Use pypi json api to get the release date of the latest version."""
        std_out_path = os.path.join(self._output_directory, 'list-out.txt')
        std_err_path = os.path.join(self._output_directory, 'list-error.txt')

        pkg_version_date = {}

        # Get the package installed version and latest version
        command = self._build_command(['list', '--format=json'])
        self._run_command(
            command, std_out_path, std_err_path, raise_on_failure=False)

        pip_list_result = json.loads(self._read(std_out_path))

        std_out_path = os.path.join(
            self._output_directory, 'list-outdate-out.txt')
        std_err_path = os.path.join(
            self._output_directory, 'list-outdate-error.txt')

        command = self._build_command(['list', '-o', '--format=json'])
        self._run_command(
            command, std_out_path, std_err_path, raise_on_failure=False)

        pip_list_latest_result = json.loads(self._read(std_out_path))

        # Get the outdated packages and latest versions
        outdated_pkgs = {}

        for pkg in pip_list_latest_result:
            pkg_name = pkg.get('name')
            latest_version = pkg.get('latest_version')
            outdated_pkgs[pkg_name] = latest_version

        for pkg in pip_list_result:
            pkg_name = pkg.get('name')
            installed_version = pkg.get('version')
            latest_version = installed_version

            is_latest = True
            if pkg_name in outdated_pkgs:
                latest_version = outdated_pkgs.get(pkg_name)
                # For py2, pip list -o returns all the packages but not just
                # the outdated pkgs.
                if latest_version != installed_version:
                    is_latest = False

            # Get the package latest version release date
            result = self._call_pypi_json_api(pkg_name, latest_version)

            # For each release versions, first item is wheel file,
            # second is tar.gz file, we use the time of the wheel file.
            installed_version_time = None
            latest_version_time = None
            if result is not None:
                if 'releases' in result:
                    latest_release = result.get('releases').get(
                        latest_version)
                    installed_release = result.get('releases').get(
                        installed_version)
                if latest_release:
                    latest_version_time = latest_release[0].get('upload_time')
                if installed_release:
                    installed_version_time = installed_release[0].get(
                        'upload_time')

            pkg_info = {
                'installed_version': installed_version,
                'installed_version_time': installed_version_time,
                'latest_version': latest_version,
                'current_time': datetime.datetime.now().isoformat(),
                'latest_version_time': latest_version_time,
                'is_latest': is_latest,
            }

            pkg_version_date[pkg_name] = pkg_info

        return pkg_version_date

    def run(self):
        """Run the version compatibility check."""
        self._output_directory = tempfile.mkdtemp(dir=self._tmp_path)
        if self._clean:
            requirements_old_file_path = os.path.join(self._output_directory,
                                                      'requirements-old.txt')
            self._freeze(requirements_old_file_path)
            if os.path.getsize(requirements_old_file_path) > 0:
                self._uninstall(requirements_old_file_path)
        install_result = self._install()

        dependency_info = None
        if install_result.result_type != PipCheckResultType.INSTALL_ERROR:
            dependency_info = self._list()

        if install_result.result_type == PipCheckResultType.SUCCESS:
            install_result = self._check()

        return install_result.with_extra_attrs(
            dependency_info=dependency_info)

def check(pip_command: List[str],
          packages: List[str],
          tmp_path: str = None,
          clean: bool = False) -> PipCheckResultType:
    """Runs a version compatibility check using the given packages.

    Conceptually, it runs:
      $ pip install <packages> 2>out.txt && \\
        (pip check >out.txt; \\
         pip freeze >requirements.txt)
    and returns the contents of "out.txt" and "requirements.txt".

    See https://pip.pypa.io/en/stable/user_guide/.

    Args:
        pip_command: The arguments to use when invoking pip e.g.
            ['python3', '-m', 'pip'].
        packages: The packages to check for compatibility e.g.
            ['numpy', 'tensorflow'].
        tmp_path: The file system path to use for temporary files e.g. "/tmp".
        clean: If True then all previously installed packages will be
            uninstalled before installing "packages".
    """
    return _OneshotPipCheck(pip_command, packages, tmp_path, clean).run()


import tensorflow as tf
from frankModel import FrankNet  # The model we are gonna use to train
from logReader import Reader
from sklearn.model_selection import train_test_split
import time
import numpy as np
import os
print("Observed TF Version: ",tf.__version__)
print("Observed Numpy Version: ",np.__version__)

#! Training Configuration
EPOCHS = 1000000 #EPOCHS
INIT_LR = 1e-3   #LEARNING RATE
BS = 8          #Batch Size
GPU_COUNT = 1    # Change this value if you are using multiple GPUs
MULTI_GPU = True #Change this to enable multi-GPU

#! Log Interpretation
STORAGE_LOCATION = "trained_models/behavioral_cloning"

#! Global training data storage
# TODO: This should be optimized?
observation = []
linear = []
angular = []

def load_data():
    global observation, linear, angular
    reader = Reader('train.log')
    observation, linear, angular = reader.read()
    observation = np.array(observation)
    linear = np.array(linear)
    angular = np.array(angular)
    print('Observation Length: ', len(observation))
    print('Linear Length: ', len(linear))
    print('Angular Length: ', len(angular))
    # exit()
    return

# -----------------------------------------------------------------------------
# Define custom loss functions for regression in Keras
# -----------------------------------------------------------------------------
# root mean squared error (rmse) for regression
def rmse(y_true, y_pred):
    from tensorflow.keras import backend
    return backend.sqrt(backend.mean(backend.square(y_pred - y_true), axis=-1))

# mean squared error (mse) for regression
def mse(y_true, y_pred):
    from tensorflow.keras import backend
    return backend.mean(backend.square(y_pred - y_true), axis=-1)

# coefficient of determination (R^2) for regression
def r_square(y_true, y_pred):
    from tensorflow.keras import backend
    SS_res = K.sum(K.square(y_true - y_pred))
    SS_tot = K.sum(K.square(y_true - K.mean(y_true)))
    return (1 - SS_res/(SS_tot + K.epsilon()))

#!================================================================
# 0. Create directories
try:
    os.makedirs("trainedModel")
except FileExistsError:
    print("Directory already exists!")
except OSError:
    print("Create folder for trained model failed. Please check system permissions.")
    exit()

# 1. Load all the datas
load_data()
print('Load all complete')

# 2. Split training and testing
observation_train, observation_valid, linear_train, linear_valid, angular_train, angular_valid = train_test_split(
    observation, linear, angular, test_size=0.2, shuffle=True)

# 3. Build the model using strategy
if MULTI_GPU:
    print("Currently using multiple GPUs")
    if tf.config.list_physical_devices('gpu'):
        strategy = tf.distribute.MirroredStrategy()
    else:
        print("Selected MultiGPU but only observe single GPU. Cancel Multi GPU training!")
        strategy = tf.distribute.get_strategy()
else:
    print("Currently using single GPUs")
    strategy = tf.distribute.get_strategy()

with strategy.scope():
    model = FrankNet.build(200, 150)

# 4. Define the loss function and weight
losses = {
    "Linear": "mse",
    "Angular": "mse"
}
lossWeights = {"Linear": 0.001, "Angular": 10.0}

# 5. Select optimizer
opt = tf.keras.optimizers.Adam(lr=INIT_LR, decay=INIT_LR / EPOCHS)

# 6. Select loss metrics
metrics_list = ["mse", rmse, r_square]

# 8. Compile model and plot to see
model.compile(optimizer=opt, loss=losses, loss_weights=lossWeights,
              metrics=metrics_list)

# 9. Setup tensorboard
tensorboard = tf.keras.callbacks.TensorBoard(
    log_dir='logs/{}'.format(time.ctime()))

# 10. checkpoint
#? Keep track of the best validation loss model
filepath1 = "trainedModel/FrankNetBest_Validation.h5"
checkpoint1 = tf.keras.callbacks.ModelCheckpoint(
    filepath1, monitor='val_loss', verbose=1, save_best_only=True, mode='min')

#? Keep track of the best loss model
filepath2 = "trainedModel/FrankNetBest_Loss.h5"
checkpoint2 = tf.keras.callbacks.ModelCheckpoint(
    filepath2, monitor='loss', verbose=1, save_best_only=True, mode='min')

callbacks_list = [checkpoint1, checkpoint2, tensorboard]

# 11. GO!
history = model.fit(observation_train,
                    {"Linear": linear_train,
                        "Angular": angular_train}, validation_data=(observation_valid, {
                            "Linear": linear_valid, "Angular": angular_valid}),
                    epochs=EPOCHS, callbacks=callbacks_list, verbose=0)

model.save('trainedModel/FrankNet.h5')


import env as env_package
import parser.ast.ast_expression as ast_expression
import parser.ast.ast_type as ast_type
import parser.ast.ast_variable_declaration as ast_variable_declaration
import parser.ast.statement.ast_block as ast_block
import parser.ast.statement.ast_for as ast_for

def link_names(ast):
  '''Link all the names in an AST
  Expects an ASTRoot as input.
  '''
  if not ast.class_or_interface:
    return

  ast = ast.class_or_interface
  env = ast.environment

  this_type = ast_type.ASTType.from_str(str(ast.name), is_primitive=False)
  this_type.definition = ast

  check_forward_field_init(ast.fields, env)
  for f in ast.fields:
    substitute_type = this_type
    # Static fields cannot use "this", so we give the None type for it.
    if f.is_static:
      substitute_type = None
    _link_name_in_expr_or_stmt(f.children[3], env, substitute_type)

  for m in ast.methods:
    if m.body is None:
      continue
    block = m.body
    substitute_type = this_type
    # Static methods cannot use "this", so we give the None type instead.
    if m.is_static:
      substitute_type = None

    for ex in block.children:
      _link_name_in_expr_or_stmt(ex, block.environment, substitute_type)

def check_forward_field_init(fields, env):
  '''Checks to make sure no fields do a forward declaration
  JLS 8.3.2.3
  TODO(songandrew): Do static/non-static checks here
  '''
  if len(fields) == 0:
    return

  for ix, f in enumerate(fields):
    # Check if any identifiers are fields of this class.  If they are,
    # check to see if they are declared after the current field.
    idens = get_all_field_identifiers(f)
    for id in idens:
      # Just use the first part of the name since this is only an error for
      # short names.
      name = id.parts[0]
      tmp, enclosing_type = env.lookup_field(name)
      if tmp and tmp in fields and fields.index(tmp) >= ix:
        # A field existed that was declared in the enclosing class, and
        # its declaration appears after the current declaration or is itself
        # the current declaration.
        raise NameLinkingError(
            'Forward initialization of field {0} with {1}.'.format(
                f.identifier, name))

def get_all_field_identifiers(f):
  '''Gets all identifiers that appear in a list initialization expression'''
  if f.expression is None:
    return []
  return _get_all_identifiers(f.expression)

def _get_all_identifiers(expr):
  '''Helper to get all identifiers from an expression or statement node'''
  acc = []
  if isinstance(expr, ast_expression.ASTMethodInvocation):
    # Only check the left side and the arguments.
    acc.extend(_get_all_identifiers(expr.left))
    for arg in expr.arguments:
      acc.extend(_get_all_identifiers(arg))
  elif isinstance(expr, ast_expression.ASTFieldAccess):
    # We check only the left side.
    acc.extend(_get_all_identifiers(expr.left))
  elif isinstance(expr, ast_expression.ASTIdentifiers):
    # Really the base case -- if we are an ASTIdentifiers node, then add
    # ourself to the list.
    acc.append(expr)
  elif isinstance(expr, ast_expression.ASTAssignment):
    # We are allowed to use a field declared after us if we are assigning
    # to it on the LHS of a method.
    if not isinstance(expr.left, ast_expression.ASTIdentifiers):
      acc.extend(_get_all_identifiers(expr.left))

    # Always get things from the RHS of an assignment.
    acc.extend(_get_all_identifiers(expr.right))
  else:
    for ex in expr.expressions:
      acc.extend(_get_all_identifiers(ex))

  return acc

def _link_name_in_expr_or_stmt(expr, env, this_type):
  if expr is None:
    return

  if isinstance(expr, ast_expression.ASTMethodInvocation):
    # We check everything except the right part of a method invocation,
    # since that will depend on the type of the left part.  That is, for the
    # expression (a.b).c(foo, bar), we will link a.b, foo, and bar.
    if expr.right is None:
      # The method name will be on the left hand side.
      if isinstance(expr.left, ast_expression.ASTIdentifiers):
        if len(expr.left.parts) == 1:
          # In this case the method should be called on "this", so we can set
          # the first definition and let it be handled by the method invocation
          # rule.
          expr.left.first_definition = ('', this_type)
        else:
          # A qualified name can just be done in a recursive call.
          _link_name_in_expr_or_stmt(expr.left, env, this_type)
      else:
        # Something more complex.  Just recurse.
        _link_name_in_expr_or_stmt(expr.left, env, this_type)
    else:
      # If there's something on the right, then just recuse on the left side
      # since the method name will not be there.
      _link_name_in_expr_or_stmt(expr.left, env, this_type)

    # Always check the arguments
    for arg in expr.arguments:
      _link_name_in_expr_or_stmt(arg, env, this_type)

  elif isinstance(expr, ast_expression.ASTFieldAccess):
    # We check only the left side since the "right" field access will depend
    # on the type of the left expression.
    _link_name_in_expr_or_stmt(expr.left, env, this_type)
  elif isinstance(expr, ast_expression.ASTIdentifiers):
    is_static = this_type is None
    defn, name = find_first_definition(expr, env, is_static)
    expr.first_definition = (name, defn)
  elif isinstance(expr, ast_block.ASTBlock) or isinstance(expr, ast_for.ASTFor):
    # ASTBlock and ASTFor need to use their containing environment.
    _link_containing_expressions(expr, expr.environment, this_type)
  elif isinstance(expr, ast_expression.ASTThis):
    # ASTThis only needs a pointer to the current class.
    expr.expr_type = this_type
  else:
    # If it's not one of the special cases, just recuse on all the expressions
    # and statements.
    _link_containing_expressions(expr, env, this_type)

def _link_containing_expressions(expr, env, this_type):
  for ex in expr.expressions:
    _link_name_in_expr_or_stmt(ex, env, this_type)

def find_first_definition(ast_idens, env, is_static):
  '''Looks up the first definiton for an ast_identifiers node
  TODO(songandrew): Comment what "first definition" means here.
  Returns: an AST node of the definition'''

  # Suppose we are resolving 'a.b.c'.
  full_name = str(ast_idens)

  if len(ast_idens.children) == 1:
    # The ast contains a simple name. Check if it is a local variable first and
    # then resort to a field lookup.
    try:
      ret = env.lookup_local(full_name)
      if ret:
        return ret, full_name
    except env_package.EnvironmentError:
      # Not in a block environment since identifiers can be in a field
      # definition.
      pass

    ret, enclosing_type = env.lookup_field(full_name)
    if ret is None:
      raise NameLinkingError('No definition found for simple name {0}'.format(
        full_name))
    # There is no implicit this for static fields so the first definition can
    # not be static.
    if ret.is_static:
      raise NameLinkingError('Static lookup in non-static context')
    if is_static:
      raise NameLinkingError(
        'Using implicit this for {0} in static method'.format(full_name))
    return ret, full_name

  # Name is qualified, i.e. contains a dot character.
  parts = list(ast_idens.children)
  # Try to resolve 'a' to a local variable or formal parameter.
  try:
    local_var = env.lookup_local(parts[0])
    if local_var:
      return local_var, parts[0]
  except env_package.EnvironmentError:
    # Not in a block environment: identifiers can be in a method definition,
    # field definition or class/interface.
    pass

  # Try to resolve 'a' to a field variable of the enclosing class.
  field, enclosing_type = env.lookup_field(parts[0])
  if field:
    if is_static and not field.is_static:
      raise NameLinkingError('Static lookup in non-static context')
    return field, parts[0]

  # Try all prefixes of 'a.b.c' until the shortest matches to a type.
  ast_class_or_interface = None
  for type_index, p in enumerate(parts[:-1]):
    possible_type_name = '.'.join(parts[:type_index + 1])
    ast_class_or_interface = env.lookup_type(possible_type_name)
    if ast_class_or_interface:
      break

  if ast_class_or_interface is None:
    raise NameLinkingError(
        'Could not resolve a prefix of {0} as a type'.format(full_name))
  return ast_class_or_interface, '.'.join(parts[:type_index + 1])

class NameLinkingError(Exception):
  def __init__(self, msg):
    self.msg = msg


import subprocess
import os
import shutil
from urllib.parse import urlparse
import feedparser
import requests
import newspaper
from ffmpy import FFmpeg

feed = feedparser.parse('http://feeds.feedburner.com/iolandachannel')
chapters = []

for entry in feed['entries']:
    title = entry['title']
    print(f'Parsing: {title}')
    news_link = entry['link']
    article = newspaper.Article(news_link)
    article.download()
    article.parse()
    media = article.top_img
    if not media:
        continue
    images = []
    response = requests.get(media, stream=True)
    filename = os.path.basename(urlparse(media).path)
    filename = f'images/{filename}'
    with open(filename, 'wb') as out_file:
        shutil.copyfileobj(response.raw, out_file)
    images.append(filename)

    from gtts import gTTS
    text = article.text
    tts = gTTS(text=text, lang='pt', slow=False)
    tts.save("article.mp3")
    inputs = {
        'article.mp3': None,
    }
    for image in images:
        inputs[image] = '-loop 1 -r 1'
    ff = FFmpeg(inputs=inputs, outputs={'article.avi': '-y -acodec copy -shortest -qscale 5'})
    print(ff.cmd)
    ff.run()
    command = f'youtube-upload article.avi --title "{title}"'
    command += f' --description "{text}\n\n'
    command += f'LINK PARA A NOTÍCIA ORIGINAL: {news_link}"'
    subprocess.call(command, shell=True)
    print(f'Article parsed: {title}')
    import sys; sys.exit()

print('That\'s all folks!')


import ivadomed.mixup as imed_mixup
import torch
import pytest

@pytest.mark.parametrize("debugging", [False, True])
@pytest.mark.parametrize("ofolder", ["test", "mixup_test"])
def test_mixup(debugging, ofolder):
    inp = [[[[0 for i in range(40)] for i in range(40)]]]
    targ = [[[[0 for i in range(40)] for i in range(40)]]]
    for i in range(10):
        for j in range(10):
            targ[0][0][i][j] = 1
    inp = torch.tensor(inp).float()
    targ = torch.tensor(targ).float()
    # just testing if mixup function run
    out = imed_mixup.mixup(inp, targ, alpha=0.5, debugging=debugging, ofolder=ofolder)


from collections import Counter
from functools import lru_cache
from collections import defaultdict
# s1 = "xabcdex"
# s2 = 'xcaedbx'

# s1 = "abcdefghijklmn"
# s2 = "efghijklmndacb"

s1 = 'great'
s2 = 'rgeat'

s1 = 'abcd'
s2 = 'cabd'

def matchScramble(s1, s2):
  if s1 == s2:
    return True

  if len(s1) != len(s2) or Counter(s1) != Counter(s2):
    return False

  d = defaultdict(int)
  for c in s1:
    d[c] += 1
  for c in s2:
    d[c] -= 1

  if any(d.values()):
    return False

  for p in range(1, len(s1)):
    # print('P ', p)
    # print('Straight')
    # print(s1[:p], s1[p:])
    # print(s2[:p], s2[p:])
    # print('Cross')
    # print(s1[:p], s1[p:])
    # print(s2[:len(s2)-p], s2[-p:])

    if matchScramble(s1[:p], s2[:p]) and matchScramble(s1[p:], s2[p:]):
      return True
    if matchScramble(s1[:p], s2[-p:]) and matchScramble(s1[p:], s2[:len(s2)-p]):
      return True

  return False

print(matchScramble(s1, s2))
# print(Counter(s1) == Counter(s2))

# test s1 and s2 starting with 'a'

# WRONG for
# 'abcd'
# 'cabd'
# def matchScramble2(s1, s2):
#   if s1 == s2:
#     return True
#   if Counter(s1) != Counter(s2):
#     return False

#   # Fix s1 try to split s2 at every point
#   c = s1[0]
#   for p in [p for p, letter in enumerate(s2) if letter == c]:
#     print('P: ', p, '*'*20)
#     print('Cross')
#     print(s1[:len(s2)-p], s1[len(s2)-p:])
#     print(s2[:p], s2[p:])
#     print('Straight')
#     print(s1[:p+1], s1[p+1:])
#     print(s2[:p+1], s2[p+1:])

#     if p == 0:
#       if matchScramble(s1[1:], s2[1:]):
#         return True
#     elif p == len(s2)-1:
#       if matchScramble(s1[1:], s2[:-1]):
#         return True
#     elif matchScramble(s2[p:], s1[:len(s2)-p]) and matchScramble(s2[:p], s1[len(s2)-p:]):
#       return True
#     elif matchScramble(s1[:p+1], s2[:p+1]) and matchScramble(s1[p+1:], s2[p+1:]):
#       return True
#   return False

# print(matchScramble(s1, s2))
# d = dict()
# for c in s1:
#   d[c] = set(c)

# def getScrambles(s):
#   if s in d:
#     return d[s]

#   d[s] = set()
#   for div in range(1, len(s)):
#     left_set = getScrambles(s[:div])
#     right_set = getScrambles(s[div:])
#     for x1 in left_set:
#       for x2 in right_set:
#         d[s].add(x1+x2)
#         d[s].add(x2+x1)
#   return d[s]

# x = getScrambles(s1)
# print(sorted(list(x)))

# print(s2 in x)


# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging

from horizon import tables
from horizon import tabs
from horizon import workflows

from saharadashboard.api.client import client as saharaclient

from saharadashboard.clusters.tables import ClustersTable
import saharadashboard.clusters.tabs as _tabs
import saharadashboard.clusters.workflows.create as create_flow
import saharadashboard.clusters.workflows.scale as scale_flow

LOG = logging.getLogger(__name__)

class ClustersView(tables.DataTableView):
    table_class = ClustersTable
    template_name = 'clusters/clusters.html'

    def get_data(self):
        sahara = saharaclient(self.request)
        clusters = sahara.clusters.list()
        return clusters

class ClusterDetailsView(tabs.TabView):
    tab_group_class = _tabs.ClusterDetailsTabs
    template_name = 'clusters/details.html'

    def get_context_data(self, **kwargs):
        context = super(ClusterDetailsView, self)\
            .get_context_data(**kwargs)
        return context

    def get_data(self):
        pass

class CreateClusterView(workflows.WorkflowView):
    workflow_class = create_flow.CreateCluster
    success_url = \
        "horizon:sahara:clusters:create-cluster"
    classes = ("ajax-modal")
    template_name = "clusters/create.html"

class ConfigureClusterView(workflows.WorkflowView):
    workflow_class = create_flow.ConfigureCluster
    success_url = "horizon:sahara:clusters"
    template_name = "clusters/configure.html"

class ScaleClusterView(workflows.WorkflowView):
    workflow_class = scale_flow.ScaleCluster
    success_url = "horizon:sahara:clusters"
    classes = ("ajax-modal")
    template_name = "clusters/scale.html"

    def get_context_data(self, **kwargs):
        context = super(ScaleClusterView, self)\
            .get_context_data(**kwargs)

        context["cluster_id"] = kwargs["cluster_id"]
        return context

    def get_object(self, *args, **kwargs):
        if not hasattr(self, "_object"):
            template_id = self.kwargs['cluster_id']
            sahara = saharaclient(self.request)
            template = sahara.cluster_templates.get(template_id)
            self._object = template
        return self._object

    def get_initial(self):
        initial = super(ScaleClusterView, self).get_initial()
        initial.update({'cluster_id': self.kwargs['cluster_id']})
        return initial


from rest_framework import serializers
from snippets.models import Snippet, LANGUAGE_CHOICES, STYLE_CHOICES
from django.contrib.auth.models import User

#'''by serializer'''
#class SnippetSerializer(serializers.Serializer):
#    id = serializers.IntegerField(read_only=True)
#    title = serializers.CharField(require=False, allow_blank=True, max_length=100)
#    code = serializers.CharField(style={'base_template': 'textarea.html'})
#    linenos = serializers.BooleanField(required=False)
#    language = serializers.ChoiceField(choices=LANGUAGE_CHOICES, default="friendly")
#
#    def create(self,validated_data):
#
#        # create and return a new "Snippet" instance, given the validated data.
#        return Snippet.Objects.create(**validated_data)
#
#    def update(self, instance, validated_data):
#        # update and return an existing 'Snippet' instance, given the validated data.
#
#        instance.title = validated_data.get('title' instance.title)
#        inatance.code = validated_data.get('code', instance.code)
#        instance.linenos = validated_data.get('linenos', instance.linenos)
#        instance.language = validated_data.get('language', instance.language)
#        instance style = validated_data.get('style', instance.style)
#        instance.save()
#        return insatance

'''by ModelSerializer'''
class SnippetSerializer(serializers.HyperlinkedModelSerializer):

    owner = serializers.ReadOnlyField(source='owner.username')
    highlight = serializers.HyperlinkedIdentityField(view_name='snippet-highlight',format='html')

    class Meta:
        model = Snippet
        fields = ('url','id','title','highlight','owner','code','linenos','language','style')

#        id = IntegerField(label='ID', read_only=True)
#        title = CharField(allow_blank=True, max_length=100, required=False)
#        code = CharField(style={'base_template': 'textarea.html'})
#        linenos = BooleanField(required=False)
#        language = ChoiceField(choices=[('Clipper', 'FoxPro'), ('Cucumber', 'Gherkin'), ('RobotFramework', 'RobotFramework'), ('abap', 'ABAP'), ('ada', 'Ada')...
#        style = ChoiceField(choices=[('autumn', 'autumn'), ('borland', 'borland'), ('bw', 'bw'), ('colorful', 'colorful')...

'''make for userlist'''
class UserSerializer(serializers.HyperlinkedModelSerializer):
    snippets = serializers.HyperlinkedRelatedField(many=True,view_name='snippet-detail', read_only=True)

    class Meta:
        model =User
        fields=('url','id', 'username', 'snippets')


#! /usr/bin/env python
# -*- mode: python; coding: utf-8 -*-
# Copyright 2016 the Tectonic Project.
# Licensed under the MIT License.

"""Create a Zip file containing all of the resources from a TeXLive
installation. Usefulness not guaranteed unless the installation was created
with the "texlive-builder/builder.sh" script.

"""
from __future__ import absolute_import, division, print_function, unicode_literals

import argparse, hashlib, io, os.path, struct, sys, zipfile
from six import PY2, reraise

ignored_base_names = set([
    'README',
    'ls-R',
])

ignored_tex_paths = set([
    'tex/generic/tex-ini-files/pdftexconfig.tex',
    'tex/luatex/hyph-utf8/etex.src',
])

def warn(fmt, *args):
    if len(args):
        msg = fmt % args
    else:
        msg = str(fmt)
    print('warning:', msg, file=sys.stderr)

def die(fmt, *args):
    if len(args):
        msg = fmt % args
    else:
        msg = str(fmt)
    raise SystemExit('error: ' + msg)

def make_arg_parser():
    p = argparse.ArgumentParser(
        description = __doc__,
    )
    p.add_argument(
        'install_dir',
        metavar = 'DIR',
        help = 'The path of the TeXLive installation directory.'
    )
    p.add_argument(
        'dest_path',
        metavar = 'PATH',
        help = 'The name of the Zip file to create.'
    )
    return p

class Bundler(object):
    def __init__(self, args, zip):
        self.args = args
        self.zip = zip
        self.item_shas = {}
        self.final_hexdigest = None

    def add_file(self, full_path):
        base = os.path.basename(full_path)

        if base in self.item_shas:
            warn('"%s" duplicates a file basename; skipping', full_path)
            return

        with io.open(full_path, 'rb') as f:
            # zipfile has no way to stream data into the archive (lame), so we
            # read the whole file into memory.
            contents = f.read()

        self.zip.writestr(base, contents)
        s = hashlib.sha256()
        s.update(contents)
        self.item_shas[base] = s.digest()

    def consider_file(self, full_path, tex_path, base_name):
        """Consider adding the specified TeXLive file to the installation tree. This
        is where all the nasty hairy logic will accumulate that enables us to
        come out with a nice pretty tarball in the end.

        """
        if base_name in ignored_base_names:
            return

        if tex_path in ignored_tex_paths:
            return

        if base_name.endswith('.log'):
            return

        if base_name.endswith('.fmt'):
            return

        self.add_file(full_path)

    def _walk_onerr(self, oserror):
        warn('error navigating installation tree: %s', oserror)

    def go(self):
        # Add a couple of version files from the builder.

        p = os.path.join(self.args.install_dir, 'SVNREV')
        if os.path.exists(p):
            self.add_file(p)
        else:
            warn('expected but did not see the file %s', p)

        p = os.path.join(self.args.install_dir, 'GITHASH')
        if os.path.exists(p):
            self.add_file(p)
        else:
            warn('expected but did not see the file %s', p)

        # Add the main tree.

        p = os.path.join(self.args.install_dir, 'texmf-dist')
        n = len(p) + 1

        for dirpath, dirnames, filenames in os.walk(p, onerror=self._walk_onerr):
            for fn in filenames:
                full = os.path.join(dirpath, fn)
                tex = full[n:]
                self.consider_file(full, tex, fn)

        # Compute a hash of it all.

        s = hashlib.sha256()
        s.update(struct.pack('>I', len(self.item_shas)))
        s.update(b'\0')

        for name in sorted(self.item_shas.keys()):
            if PY2:
                s.update(name)
            else:
                s.update(name.encode('utf8'))
            s.update(b'\0')
            s.update(self.item_shas[name])

        self.final_hexdigest = s.hexdigest()
        self.zip.writestr('SHA256SUM', self.final_hexdigest)

def main(argv):
    args = make_arg_parser().parse_args(argv[1:])

    try:
        with zipfile.ZipFile(args.dest_path, 'w', zipfile.ZIP_DEFLATED, True) as zip:
            b = Bundler(args, zip)
            b.go()
            print(b.final_hexdigest)
    except Exception as e:
        e1, e2, e3 = sys.exc_info()
        try:
            os.unlink(args.dest_path)
        except:
            pass
        reraise(e1, e2, e3)

if __name__ == '__main__':
    sys.exit(main(sys.argv))


from flask import Blueprint, render_template, request, redirect

publications = Blueprint('publications', __name__, template_folder='templates')

def get_publications(fname):
    fname = "web/static/data/publications/" + fname + ".txt"
    f = open(fname, "r", encoding="utf-8")
    papers = []
    while True:
        line = f.readline()
        if not line: break
        line = line.rstrip()
        info = line.split("()")
        papers.append({"title":info[0], "content":info[1]})
    f.close()
    return papers

@publications.route("/publications")
def publications_list():
    t = request.args.get('type')
    if(t == "DJ"):
        fname = "Domestic Journal"
    elif(t == "DC"):
        fname = "Domestic Conference"
    elif(t == "IJ"):
        fname = "International Journal"
    elif(t == "IC"):
        fname = "International Conference"
    else:
        fname = "Domestic Journal"
    papers = get_publications(fname)
    return render_template("publications_list.html", publications=papers, type=fname)


# conv.py file
import time
startTime = time.time()
import torch
import torch.nn as nn
import torchvision.transforms as transforms
from torch.autograd import Variable
import numpy as np
import random
import cv2
#import nn

#what is "input" in the "forward" method
#am i just outputting 1 hidden layer at a time?
#4. Part A: "By running forward(input) the script will perform the forward propagation pass on the network previously built using sigmoid nonlinearities."

class NeuralNetwork():

    def __init__(self, arrayZ):
        #3,4,1:
        #(self, size of the input layer, size of the hidden layers, size of the output layer)
        #super(NeuralNetwork, self).__init__()
        #need to init the "netwrok" and theta(layer) <--(weights)
        #(3,4) matrix for first hidden layer the i guess (4,1) for the next

        #arrayZ = arrayZ

        ThetaDic = {}
        keys = []
        weights = []

        numHidLayers = len(arrayZ) -2 # of hidden layers
        for i in range(0, numHidLayers+1):
            keys.append(i)

        print("This is the keys list:", keys)

        #Populating the dictionary below!!

        for i in range(0, numHidLayers+1): #runs through the h1, h2, h3
            numElements = arrayZ[i]
            mean = 0
            SD = 1/(np.sqrt(numElements))

        for k in range(0, numElements+1):
            for i in range(0, numHidLayers+1):
                q = np.random.normal(mean, SD, ((arrayZ[i]+1),(arrayZ[i+1]))) #<--- use numpy thing where i can specify a mean = 0 and a SD
                #print("this is q:", q)
                x = torch.Tensor(q)
                #print("this is x: ", x)
                weights.append(x)

        self.ThetaDic = dict(zip(keys, weights))

        print("this is ThetaDic: ",self.ThetaDic)

    def getLayer(self, layer): #takes "layer number" to indicate which layer

        return self.ThetaDic[layer] #should return a 2D tensor of weights ....for one layer at a time

    #the forward script will perform the forward propagation pass on the "network"  previously built using sigmoid nonlinearities.
    def forward(self, input): #feedforward pass single vector

        z = []
        print("this is O.G. input",input)

        def sigmoid(x, Derivative=False):
            if not Derivative:
                return 1 / (1 + np.exp (-x))
            else:
                out = sigmoid(x)
                return out * (1 - out)

        #numExamples = input.shape[0]
        #print("numExamples:", numExamples)
        numLayers = len(self.ThetaDic)
        #print("haha",numLayers)

        #input = torch.t(input)
        #biases = torch.ones(1,numExamples+1)
        #print(biases)
        #input = torch.t(input)

        for d in range(0, numLayers):
            #self.ThetaDic[d] = torch.t(self.ThetaDic[d])
            #input = torch.t(input)
            #input = torch.t(input)

        #layerInput = []

            if input.type() == "torch.DoubleTensor":

                numExamples = (torch.t(input)).shape[0]
                print("this is numExamples",numExamples)
                biases = torch.ones(1,numExamples)
                print(biases)

                biases = biases.type(torch.DoubleTensor)
                self.ThetaDic[d] = self.ThetaDic[d].type(torch.DoubleTensor)
                print("theta is notT: ", self.ThetaDic[d])

                x=torch.cat([biases, input],0) # X
                print("this is x",x)

                #print("theta is T: ", self.ThetaDic[d])

                ThetaDic = torch.t(self.ThetaDic[d])
                print("theta is T: ", ThetaDic)

                z = torch.mm(ThetaDic,x)
                print("here is z nignog: ",z)
                zSig = sigmoid(z)
                print("here is zSig: ", zSig)
                input = z

            else:
                print("kkkk",input)
                numExamples = (input).numpy()
                print("this is numExamplesssssssssssss s",numExamples)
                numExamples =  numExamples.size
                print("this is numExamplesssssssssssss s",numExamples)
                biases = torch.ones(1,numExamples-1)
                print("BIASES",biases)
                print("input = ", input)

                # if input.shape == biases.shape:
                #     print("Looks good to me!")
                # else:
                #     input = torch.t(input)

                x = torch.cat([biases, input],0)
                print("this is x",x)
                ThetaDic = torch.t(self.ThetaDic[d])
                print("theta is T: ", ThetaDic)
                #x = [[x]
                z = torch.mm(ThetaDic,x)
                zSig = sigmoid(z)
                print("here is zSig: ", zSig)
                input = z

        return zSig

            #y = y.type(torch.Tensor)

        #     for index in range(numLayers):

        #         if index ==0:

        #             layerInput.append(torch.mm(self.ThetaDic[0],y))

        #         else:

        #             layerInput = torch.mm(torch.cat(self.ThetaDic[0], ([biases, self.layerOutput[-1]],0)))
        #     return self.layerOutput[-1].T

        # else:
        #     y=torch.cat([biases, input],0)

        #     for index in range(numLayers):

        #         if index ==0:

        #             layerInput = torch.dot(self.ThetaDic[0],y)

        #         else:

        #             layerInput = self.weights[index].dot(np.vstack(np.ones([1,numExamples],[self._layerOutput[-1]])))

        #     self._layerInput.append(layerInput)
        #     self._layerOutput.append(sigmoid(layerInput))

        # return self._layerOutput[-1].T

   # providing input , run it through the network, return result..?

    # so this needs to take a either a list (make it a tensor) or a tensor/matrix? and multiply it and the weights (specified theta) to get y. then get a z somehow
#matrix mult
#add 1ones
#if input 1 D
    #if (input == 1):

# # in the 2xn (2 cross n) and 1xn, "n" is the number of inputs into the gate--
# #AND gate will take n=2 inputs while the NOT gate will get just n=1
# #so n number

#         return output #? 1xn

#     def forward(self, input): #feedforward pass transposed design matrix

#     	return output #? 1xn

#     #if __name__ == "__main__":
#   	#main()

#     	class Shark:
#     def __init__(self, name):
#         self.name = name

#     def swim(self):
#         print(self.name + " is swimming.")

#     def be_awesome(self):
#         print(self.name + " is being awesome.")

# def main():
#     sammy = Shark("Sammy")
#     sammy.be_awesome()
#     stevie = Shark("Stevie")
#     stevie.swim()

# if __name__ == "__main__":
#   main()


#!/usr/bin/env python3

import argparse
import os
import sys
import yaml
import sqlite3
import json
import re

class CravatFilter ():
    def __init__ (self, dbpath=None, filterpath=None, filtername=None,
            filterstring=None, filter=None, mode='sub'):
        self.mode = mode
        if self.mode == 'main':
            self.stdout = True
        else:
            self.stdout = False
        self.dbpath = None
        self.filterpath = None
        self.cmd = None
        self.level = None
        self.filter = None
        self.filtertable = 'filter'
        self.savefiltername = None
        self.filtername = None
        self.filterstring = None
        self.conn = None
        self.cursor = None

        if dbpath != None:
            self.dbpath = dbpath
            if mode == 'sub':
                self.connect_db()

        if filter != None:
            self.filter = filter
        else:
            if filterstring != None:
                self.filterstring = filterstring
            elif filtername != None:
                self.filtername = filtername
            elif filterpath != None:
                self.filterpath = filterpath
            if mode == 'sub':
                self.loadfilter()

    def run (self, cmd=None, args=None, dbpath=None, filter=None):
        if args != None:
            self.parse_args(args)
        if cmd != None:
            self.cmd = cmd

        if dbpath != None:
            self.dbpath = dbpath
            self.connect_db()
        elif self.dbpath != None and self.cursor == None:
            self.connect_db()

        # Loads filter.
        if filter != None:
            self.filter = filter
        elif (self.filtername != None or self.filterpath != None or
            self.filterstring != None) and self.filter == None:
            self.loadfilter()

        ret = None
        if self.cursor != None and self.filter != None:
            if self.cmd == 'uidpipe':
                ret = self.run_level_based_func(self.getuiditerator)
            elif self.cmd == 'count':
                ret = self.run_level_based_func(self.getcount)
            elif self.cmd == 'rows':
                ret = self.run_level_based_func(self.getrows)
            elif self.cmd == 'pipe':
                ret = self.run_level_based_func(self.getiterator)
        elif self.cursor != None and self.cmd == 'list':
            ret = self.listfilter()

        # Saves filter.
        if self.filter != None:
            if self.cmd == 'save' or self.savefiltername != None:
                ret = self.savefilter()

        return ret

    def run_level_based_func (self, cmd):
        ret = {}
        if self.level != None:
            ret[self.level] = cmd(level=self.level)
        else:
            levels = ['variant', 'gene']
            ret = {}
            for level in levels:
                ret_onelevel = cmd(level=level)
                ret[level] = ret_onelevel
        return ret

    def parse_args (self, args):
        parser = argparse.ArgumentParser()
        parser.add_argument('-d',
            dest='dbpath',
            required=True,
            help='Path of a result database file (.sqlite)')
        parser.add_argument('-f',
            dest='filterpath',
            help='Path of a filtering criteria file')
        parser.add_argument('-F',
            dest='filtername',
            help='Name of the filter to apply (saved in the database)')
        parser.add_argument('--filterstring',
            dest='filterstring',
            default=None,
            help='Filter in JSON')
        parser.add_argument('-l',
            dest='level',
            default=None,
            choices=['variant', 'gene'],
            help='Analysis level to filter')
        parser.add_argument('-s',
            dest='savefiltername',
            help='Name to save the filter as (in the database)')
        if self.mode == 'main':
            parser.add_argument('command',
            choices=['uidpipe', 'count', 'rows', 'pipe', 'save', 'list'],
            help='Command')

        parsed_args = parser.parse_args(args)
        self.dbpath = parsed_args.dbpath
        self.filterpath = parsed_args.filterpath
        self.level = parsed_args.level
        if self.mode == 'main':
            self.cmd = parsed_args.command
        self.savefiltername = parsed_args.savefiltername
        self.filtername = parsed_args.filtername
        self.filterstring = parsed_args.filterstring

    def connect_db (self, dbpath=None):
        if dbpath != None:
            self.dbpath = dbpath

        self.conn = sqlite3.connect(self.dbpath)
        self.cursor = self.conn.cursor()
        self.conn.create_function('regexp', 2, regexp)

    def close_db (self):
        self.cursor.close()
        self.conn.close()

    def create_filtertable (self):
        if self.cursor == None:
            return
        sql = 'create table ' + self.filtertable + ' (name text, criteria text)'
        self.cursor.execute(sql)
        self.conn.commit()

    def filtertable_exists (self):
        sql = 'select name from sqlite_master where ' +\
            'type="table" and name="' + self.filtertable + '"'
        self.cursor.execute(sql)
        row = self.cursor.fetchone()
        if row == None:
            ret = False
        else:
            ret = True
        return ret

    def loadfilter (self, filterpath=None, filtername=None,
            filterstring=None, filter=None):
        if filterpath != None:
            self.filtername = filtername
        if filtername != None:
            self.filterpath = filterpath
        if filterstring != None:
            self.filterstring = filterstring
        if filter != None:
            self.filter = filter

        if self.filterstring != None:
            self.filterstring = self.filterstring.replace("'", '"')
            self.filter = json.loads(self.filterstring)
        elif self.filtername != None and self.filtertable_exists():
            self.cursor.execute('select criteria from ' + self.filtertable +
                ' where name="' + self.filtername + '"')
            criteria = self.cursor.fetchone()
            if criteria != None:
                self.filter = json.loads(criteria[0])
        elif self.filterpath != None and os.path.exists(self.filterpath):
            with open(self.filterpath) as f:
                self.filter = yaml.load(f)

    def delete_filtered_uid_table (self):
        self.cursor.execute('pragma synchronous=0')
        q = 'drop table if exists variant_filtered'
        self.cursor.execute(q)
        q = 'drop table if exists gene_filtered'
        self.cursor.execute(q)
        self.conn.commit()
        self.cursor.execute('pragma synchronous=2')

    def getwhere (self, level):
        if self.filter == None:
            return ''
        if level not in self.filter:
            return ''
        criteria = self.filter[level]
        columns = criteria.keys()
        where = ''
        for column in columns:
            op_val = criteria[column]
            where += column + ' ' + op_val + ' and '
        if where != '':
            where = ' where ' + where
            where = where.rstrip(' and ')
        return where

    def getvariantcount (self):
        return self.getcount('variant')

    def getgenecount (self):
        return self.getcount('gene')

    def getcount (self, level='variant'):
        level = 'variant'
        self.make_filtered_uid_table()
        ftable = level + '_filtered'
        q = 'select count(*) from ' + ftable
        self.cursor.execute(q)
        n = self.cursor.fetchone()[0]
        if self.stdout == True:
            print('#' + level)
            print(str(n))
        return n

    def getvariantrows (self):
        return self.getrows('variant')

    def getgenerows (self):
        return self.getrows('gene')

    def getrows (self, level='variant'):
        where = self.getwhere(level)
        q = 'select *  from ' + level + where
        self.cursor.execute(q)

        ret = [list(v) for v in self.cursor.fetchall()]

        if self.stdout == True:
            print('#' + level)
            for row in ret:
                print('\t'.join([str(v) for v in row]))

        return ret

    def get_gene_row (self, hugo):
        q = 'select * from gene where base__hugo="' + hugo + '"'
        self.cursor.execute(q)
        row = self.cursor.fetchone()
        return row

    def getvariantiterator (self):
        return self.getiterator('variant')

    def getgeneiterator (self):
        return self.getiterator('gene')

    def getiterator (self, level='variant'):
        where = self.getwhere(level)
        sql = 'select * from ' + level + where
        self.cursor.execute(sql)
        it = self.cursor.fetchall()
        return it

    def get_filtered_iterator (self, level='variant'):
        if level == 'variant':
            kcol = 'base__uid'
            ftable = 'variant_filtered'
        elif level == 'gene':
            kcol = 'base__hugo'
            ftable = 'gene_filtered'
        elif level == 'sample':
            kcol = 'base__uid'
            ftable = 'variant_filtered'
        elif level == 'mapping':
            kcol = 'base__uid'
            ftable = 'variant_filtered'
        table = level
        if level in ['variant', 'gene', 'sample', 'mapping']:
            sql = 'select t.* from ' + table + ' as t inner join ' + ftable +\
                ' as f on t.' + kcol + '=f.' + kcol
        self.cursor.execute(sql)
        it = self.cursor.fetchall()
        return it

    def make_filtered_uid_table (self):
        self.cursor.execute('pragma synchronous=0')
        level = 'variant'
        vtable = level
        vftable = level + '_filtered'
        q = 'drop table if exists ' + vftable
        self.cursor.execute(q)
        where = self.getwhere(level)
        q = 'create table ' + vftable +\
            ' as select base__uid from ' + level + where
        self.cursor.execute(q)
        self.cursor.execute('pragma synchronous=2')

    def make_filtered_hugo_table (self):
        self.cursor.execute('pragma synchronous=0')
        level = 'gene'
        vtable = 'variant'
        vftable = vtable + '_filtered'
        gftable = level + '_filtered'
        q = 'drop table if exists ' + gftable
        self.cursor.execute(q)
        q = 'create table ' + gftable +\
            ' as select distinct v.base__hugo from ' + vtable + ' as v'\
            ' inner join ' + vftable + ' as vf on vf.base__uid=v.base__uid'
        self.cursor.execute(q)
        self.cursor.execute('pragma synchronous=2')

    def savefilter (self, name=None):
        if self.cursor == None or self.filter == None:
            return

        if name == None:
            if self.savefiltername != None:
                name = self.savefiltername
            else:
                name = 'default'

        # Creates filter save table if not exists.
        self.cursor.execute('select name from sqlite_master where ' +
            'type="table" and name="' + self.filtertable + '"')
        ret = self.cursor.fetchone()
        if ret == None:
            self.cursor.execute('create table ' + self.filtertable +
                ' (name text unique, criteria text)')

        # Saves the filter.
        filterstr = json.dumps(self.filter)
        sql = 'insert or replace into ' + self.filtertable +\
            ' values ("' + name + '", \'' + filterstr + '\')'
        self.cursor.execute(sql)
        self.conn.commit()

    def listfilter (self, name=None):
        if name == None:
            if self.savefiltername != None:
                name = self.savefiltername
            else:
                name = 'default'

        # Creates filter save table if not exists.
        self.cursor.execute('select name from sqlite_master where ' +
            'type="table" and name="' + self.filtertable + '"')
        ret = self.cursor.fetchone()
        if ret == None:
            self.cursor.execute('create table ' + self.filtertable +
                ' (name text, criteria text)')

        sql = 'select name, criteria from ' + self.filtertable
        self.cursor.execute(sql)
        ret = {}
        for row in self.cursor.fetchall():
            name = row[0]
            criteria = json.loads(row[1])
            ret[name] = criteria
            if self.stdout:
                print('#' + name)
                for level in criteria:
                    print('    ' + level + ':')
                    for column in criteria[level]:
                        print('        ' + column + ': ' +
                              criteria[level][column])

        return ret

    def addvariantfilter (self, column, condition):
        self.addfilter(column, condition, 'variant')

    def addgenefilter (self, column, condition):
        self.addcriterion(column, condition, 'gene')

    def addfilter (self, column, condition, level='variant'):
        if self.filter == None:
            self.filter = {}
        if level not in self.filter:
            self.filter[level] = {}
        self.filter[level][column] = condition

    def removevariantfilter (self, column):
        self.removefilter(column, 'variant')

    def removegenefilter (self, column):
        self.removefilter(column, 'gene')

    def removefilter (self, column, level='variant'):
        if self.filter == None:
            return
        if level in self.filter and column in self.filter[level]:
            del self.filter[level][column]

    def table_exists (self, table):
        sql = 'select name from sqlite_master where type="table" and ' +\
            'name="' + table + '"'
        self.cursor.execute(sql)
        if self.cursor.fetchone() == None:
            return False
        else:
            return True

    def get_variant_iterator_filtered_uids_cols (self, cols):
        q = 'select ' + ','.join(cols) + ' from variant as v ' +\
            'inner join variant_filtered as f on v.base__uid=f.base__uid'
        self.cursor.execute(q)
        for row in self.cursor.fetchall():
            d = {}
            for i in range(len(row)):
                d[cols[i].split('__')[1]] = row[i]
            yield d

    def get_result_levels (self):
        q = 'select name from sqlite_master where type="table" and ' +\
            'name like "%_header"'
        self.cursor.execute(q)
        table_names = []
        for row in self.cursor.fetchall():
            table_names.append(row[0].replace('_header', ''))
        return table_names

def regexp (y, x, search=re.search):
    if x is None:
        return 0
    return 1 if search(y, x) else 0

def main ():
    cv = CravatFilter(mode='main')
    cv.run(args=sys.argv[1:])

def test ():
    cf = CravatFilter(dbpath='d:\\git\\cravat-newarch\\tmp\\job\\in1000.sqlite',
                      mode='sub',
                      filterstring='{"variant": {"thousandgenomes__af": ">0.1"}}')
    print(cf.getcount())
    cf.make_filtered_uid_table()
    for row in cf.get_filtered_iterator(level='variant'):
        print(row)
    for row in cf.get_filtered_iterator(level='gene'):
        print(row)

if __name__ == '__main__':
    #main()
    test()


import asyncio
import hashlib
import os
import tempfile
import time

from devtools_testutils import AzureTestCase

from azure.confidentialledger import (
    LedgerUserRole,
    TransactionState,
)

from .constants import NETWORK_CERTIFICATE, USER_CERTIFICATE
from .testcase_async import AsyncConfidentialLedgerTestCase

CONFIDENTIAL_LEDGER_URL = "https://fake-confidential-ledger.azure.com"

class AsyncConfidentialLedgerClientTestMixin:
    class AsyncBaseTest(AsyncConfidentialLedgerTestCase):
        def setUp(self):
            super().setUp()

            self.confidential_ledger_url = self.set_value_to_scrub(
                "CONFIDENTIAL_LEDGER_URL", CONFIDENTIAL_LEDGER_URL
            )

            with tempfile.NamedTemporaryFile(
                "w", suffix=".pem", delete=False
            ) as tls_cert_file:
                tls_cert_file.write(NETWORK_CERTIFICATE)
                self.network_certificate_path = tls_cert_file.name

            with tempfile.NamedTemporaryFile(
                "w", suffix=".pem", delete=False
            ) as user_cert_file:
                user_cert_file.write(
                    self.set_value_to_scrub(
                        "CONFIDENTIAL_LEDGER_USER_CERTIFICATE", USER_CERTIFICATE
                    )
                )
                self.user_certificate_path = user_cert_file.name

        def tearDown(self):
            os.remove(self.user_certificate_path)
            os.remove(self.network_certificate_path)

            # Since tearDown cannot be async
            task = asyncio.ensure_future(self.client.close())
            while not task.done:
                time.sleep(0.5)

            return super().tearDown()

        @AzureTestCase.await_prepared_test
        async def test_append_entry_flow(self):
            entry_contents = "Test entry from Python SDK"
            append_result = await self.client.append_to_ledger(
                entry_contents=entry_contents
            )
            self.assertTrue(append_result.transaction_id)
            self.assertTrue(append_result.sub_ledger_id)

            # Test unpacking
            append_result_sub_ledger_id, append_result_transaction_id = append_result

            await self.client.wait_until_durable(
                transaction_id=append_result_transaction_id
            )

            transaction_status = await self.client.get_transaction_status(
                transaction_id=append_result_transaction_id
            )
            self.assertIsNotNone(transaction_status)
            self.assertIs(transaction_status.state, TransactionState.COMMITTED)
            self.assertEqual(
                transaction_status.transaction_id, append_result_transaction_id
            )

            receipt = await self.client.get_transaction_receipt(
                transaction_id=append_result_transaction_id
            )
            self.assertEqual(receipt.transaction_id, append_result_transaction_id)
            self.assertTrue(receipt.contents)

            latest_entry = await self.client.get_ledger_entry()
            # The transaction ids may not be equal in the unfortunate edge case where a governance
            # operation occurs after the ledger append (e.g. because a node was restarted). Then,
            # the latest id will be higher.
            self.assertGreaterEqual(
                latest_entry.transaction_id, append_result_transaction_id
            )
            self.assertEqual(latest_entry.contents, entry_contents)
            self.assertEqual(latest_entry.sub_ledger_id, append_result_sub_ledger_id)

            await self.client.append_to_ledger(
                "Test entry 2 from Python SDK", wait_for_commit=True
            )

            latest_entry = await self.client.get_ledger_entry()
            self.assertNotEqual(
                latest_entry.transaction_id, append_result_transaction_id
            )
            self.assertNotEqual(latest_entry.contents, entry_contents)
            self.assertEqual(latest_entry.sub_ledger_id, append_result_sub_ledger_id)

            original_entry = await self.client.get_ledger_entry(
                transaction_id=append_result_transaction_id
            )
            self.assertEqual(
                original_entry.transaction_id, append_result_transaction_id
            )
            self.assertEqual(original_entry.contents, entry_contents)
            self.assertEqual(original_entry.sub_ledger_id, append_result_sub_ledger_id)

        @AzureTestCase.await_prepared_test
        async def test_append_entry_flow_with_sub_ledger_id(self):
            sub_ledger_id = "132"
            entry_contents = "Test sub-ledger entry from Python SDK"
            append_result = await self.client.append_to_ledger(
                entry_contents=entry_contents, sub_ledger_id=sub_ledger_id
            )
            self.assertTrue(append_result.transaction_id)
            self.assertEqual(append_result.sub_ledger_id, sub_ledger_id)

            # Test unpacking
            append_result_sub_ledger_id, append_result_transaction_id = append_result

            await self.client.wait_until_durable(
                transaction_id=append_result_transaction_id
            )

            transaction_status = await self.client.get_transaction_status(
                transaction_id=append_result_transaction_id
            )
            self.assertIsNotNone(transaction_status)
            self.assertIs(transaction_status.state, TransactionState.COMMITTED)
            self.assertEqual(
                transaction_status.transaction_id, append_result_transaction_id
            )

            receipt = await self.client.get_transaction_receipt(
                transaction_id=append_result_transaction_id
            )
            self.assertEqual(receipt.transaction_id, append_result_transaction_id)
            self.assertTrue(receipt.contents)

            latest_entry = await self.client.get_ledger_entry(
                sub_ledger_id=sub_ledger_id
            )
            # The transaction ids may not be equal in the unfortunate edge case where a governance
            # operation occurs after the ledger append (e.g. because a node was restarted). Then,
            # the latest id will be higher.
            self.assertGreaterEqual(
                latest_entry.transaction_id, append_result_transaction_id
            )
            self.assertEqual(latest_entry.contents, entry_contents)
            self.assertEqual(latest_entry.sub_ledger_id, append_result_sub_ledger_id)

            await self.client.append_to_ledger(
                "Test sub-ledger entry 2 from Python SDK",
                sub_ledger_id=sub_ledger_id,
                wait_for_commit=True,
            )

            latest_entry = await self.client.get_ledger_entry(
                sub_ledger_id=sub_ledger_id
            )
            self.assertNotEqual(
                latest_entry.transaction_id, append_result_transaction_id
            )
            self.assertNotEqual(latest_entry.contents, entry_contents)
            self.assertEqual(latest_entry.sub_ledger_id, sub_ledger_id)

            original_entry = await self.client.get_ledger_entry(
                transaction_id=append_result_transaction_id, sub_ledger_id=sub_ledger_id
            )
            self.assertEqual(
                original_entry.transaction_id, append_result_transaction_id
            )
            self.assertEqual(original_entry.contents, entry_contents)
            self.assertEqual(original_entry.sub_ledger_id, append_result_sub_ledger_id)

        @AzureTestCase.await_prepared_test
        async def test_range_query(self):
            modulus = 5
            num_messages_sent = 201  # Should result in 2 pages.

            messages = {m: [] for m in range(modulus)}
            for i in range(num_messages_sent):
                message = "message-{0}".format(i)
                kwargs = (
                    {} if modulus == 0 else {"sub_ledger_id": "{0}".format(i % modulus)}
                )
                append_result = await self.client.append_to_ledger(
                    entry_contents=message, **kwargs
                )

                messages[i % modulus].append(
                    (append_result.transaction_id, message, kwargs)
                )

            num_matched = 0
            for i in range(modulus):
                query_result = self.client.get_ledger_entries(
                    from_transaction_id=messages[i][0][0], **messages[i][0][2]
                )
                index = 0
                async for historical_entry in query_result:
                    self.assertEqual(
                        historical_entry.transaction_id, messages[i][index][0]
                    )
                    self.assertEqual(historical_entry.contents, messages[i][index][1])
                    index += 1
                    num_matched += 1

            # Due to replication delay, it's possible not all messages are matched.
            self.assertGreaterEqual(num_matched, 0.9 * num_messages_sent)

        @AzureTestCase.await_prepared_test
        async def test_user_management(self):
            user_id = "0" * 36  # AAD Object Ids have length 36
            user = await self.client.create_or_update_user(
                user_id, LedgerUserRole.CONTRIBUTOR
            )
            self.assertEqual(user.id, user_id)
            self.assertEqual(user.role, LedgerUserRole.CONTRIBUTOR)

            user = await self.client.get_user(user_id)
            self.assertEqual(user.id, user_id)
            self.assertEqual(user.role, LedgerUserRole.CONTRIBUTOR)

            await self.client.delete_user(user_id)

            user = await self.client.create_or_update_user(
                user_id, LedgerUserRole.READER
            )
            self.assertEqual(user.id, user_id)
            self.assertEqual(user.role, LedgerUserRole.READER)

            user = await self.client.get_user(user_id)
            self.assertEqual(user.id, user_id)
            self.assertEqual(user.role, LedgerUserRole.READER)

            await self.client.delete_user(user_id)

        @AzureTestCase.await_prepared_test
        async def test_verification_methods(self):
            consortium = await self.client.get_consortium()
            self.assertEqual(len(consortium.members), 1)
            for member in consortium.members:
                self.assertTrue(member.certificate)
                self.assertTrue(member.id)

            constitution = await self.client.get_constitution()
            self.assertTrue(constitution.contents)
            self.assertTrue(constitution.digest)
            self.assertEqual(
                constitution.digest.lower(),
                hashlib.sha256(constitution.contents.encode()).hexdigest().lower(),
            )

            ledger_enclaves = await self.client.get_enclave_quotes()
            self.assertEqual(len(ledger_enclaves.quotes), 3)
            self.assertIn(ledger_enclaves.source_node, ledger_enclaves.quotes)
            for node_id, quote in ledger_enclaves.quotes.items():
                self.assertEqual(node_id, quote.node_id)
                self.assertTrue(quote.node_id)
                self.assertTrue(quote.mrenclave)
                self.assertTrue(quote.raw_quote)
                self.assertTrue(quote.version)


from datetime import datetime
from common import routines
from common import parsers
import sqlite3
import json
import time
import re
import os

class AthletesDB():
    def __init__(self, db_name):
        self.db_name = db_name
        self.db = None

    def __del__(self):
        self.disconnect()

    def connect(self):
        if self.db is not None:
            return
        self.db = sqlite3.connect(self.db_name)

    def disconnect(self):
        if self.db is None:
            return
        self.db.close()

    def generate_id(self):
        if self.db is None:
            return
        max_id = -1
        try:
            for athlete in self.db.execute('SELECT * FROM athlete').fetchall():
                if athlete[0] > max_id:
                    max_id = athlete[0]
        except Exception as e:
            print(e)
        count_id = max_id
        count_id += 1
        return count_id

    def find_athlete(self, find_hash=None):
        try:
            self.db.execute('SELECT * FROM athlete').fetchall()
        except:
            return []
        if find_hash is None:
            return self.db.execute('SELECT * FROM athlete').fetchall()
        params_str = ''
        for key in find_hash.keys():
            if params_str == '':
                params_str += ' '
            else:
                ' and '
            params_str += key + ' = "' + find_hash[key] + '"'
        if params_str == '':
            return self.db.execute('SELECT * FROM athlete').fetchall()
        return self.db.execute('SELECT * FROM athlete WHERE' + params_str).fetchall()

    def find_trainings(self, athlete_id, date=None, type=None, activity=None):
        training_list = self.db.execute('SELECT * FROM training WHERE athlete_id=%s' % athlete_id).fetchall()

        if date is not None:
            date_trainings = []
            for trainings in training_list:
                day_training = datetime.fromtimestamp(trainings[0])
                if day_training.date() == date.date():
                    date_trainings.append(trainings)
            return date_trainings
        return training_list

    def find_activity(self, training_id, date=None, type=None, activity=None):
        training_list = self.db.execute(
            'SELECT filetype,activity,data FROM activities  WHERE training_id=%s AND filetype=\"fit\" LIMIT 1' % training_id).fetchall()
        if date is not None:
            date_trainings = []
            for trainings in training_list:
                day_training = datetime.fromtimestamp(trainings[0])
                if day_training.date() == date.date():
                    date_trainings.append(trainings)
            return date_trainings
        return training_list

    def update_athlete_info(self, athlete_name, athlete_path):
        if self.db is None:
            return
        athlete_path = routines.process_path(athlete_path)
        athlete_dict = {
            'born_day': -1,
            'weight': -1,
            'height': -1,
            'id': -1,
            'sex': -1,
            'last_update': -1
        }
        athlete_born_datetime = -1

        athlete_dict = routines.open_yaml(athlete_path + 'data.yaml', athlete_dict)

        if athlete_dict['born_day'] != -1:
            athlete_dict['born_day'].strip()
            date_list = athlete_dict['born_day'].split(' ')
            if len(date_list) > 2:
                athlete_dict['born_day'] = -1
            else:
                date = date_list[0]
                mask = ''
                if len(date_list) > 1:
                    mask = date_list[1]
                else:
                    mask = 'd.m.y'
                date_list = date.split('.')
                mask_list = mask.split('.')
                if len(date_list) != len(mask_list):
                    athlete_dict['born_day'] = -1
                else:
                    date_dict = dict()
                    for i in range(0, len(mask_list)):
                        date_dict[mask_list[i].lower()] = int(date_list[i])
                    try:
                        athlete_born_datetime = datetime(year=date_dict['y'], month=date_dict['m'], day=date_dict['d'])
                    except:
                        athlete_dict['born_day'] = -1

        if athlete_dict['id'] == -1:
            athlete_dict['id'] = self.generate_id()

        if os.path.getmtime(athlete_path) < athlete_dict['last_update']:
            for key in athlete_dict.keys():
                if athlete_dict[key] == -1 and key != 'last_update':
                    routines.save_yaml(athlete_path + 'data.yaml', athlete_dict)
                    break
            try:
                if self.db.execute('SELECT * FROM athlete WHERE ID = %d' % athlete_dict['id']).fetchone() is not None:
                    return
            except:
                pass

        athlete_dict['last_update'] = time.mktime(datetime.now().timetuple())
        routines.save_yaml(athlete_path + 'data.yaml', athlete_dict)
        try:
            self.db.execute('create table if not exists ' + 'athlete' + \
                            ' (id integer PRIMARY KEY, name string, sex string, born_day string, weight integer, height integer)')
            # self.db.commit()
        except Exception as e:
            print(e)

        try:
            self.db.execute('create table if not exists ' + 'training' + \
                            ' (training_id integer PRIMARY KEY, athlete_id integer, date string, ' +
                            'FOREIGN KEY(athlete_id) REFERENCES athlete(id))')
            # self.db.commit()
        except Exception as e:
            print(e)

        try:
            self.db.execute('create table if not exists ' + 'activities' + \
                            ' (id integer PRIMARY KEY, training_id integer, activity integer, data string, filetype string, ' +
                            'FOREIGN KEY(training_id) REFERENCES training(training_id))')
            # self.db.commit()
        except Exception as e:
            print(e)

        athlete_id = self.db.execute('SELECT * FROM athlete WHERE id = %d' % athlete_dict['id']).fetchone()
        # print(athlete_id)
        if athlete_id is None:
            self.db.execute('INSERT INTO ' + 'athlete' + '(name, sex, born_day, weight, height) '
                                                         'VALUES("%s", "%s", "%s", %d, %d)' % (
                                athlete_name, athlete_dict['sex'], \
                                str(athlete_born_datetime), athlete_dict['weight'], \
                                athlete_dict['height']))
            self.db.commit()

            athlete_dict['id'] = self.db.execute(
                'SELECT ID FROM athlete WHERE name = "{}" and sex = "{}" and born_day="{}" and weight={} and height={}'. \
                    format(athlete_name, athlete_dict['sex'], \
                           str(athlete_born_datetime), athlete_dict['weight'], \
                           athlete_dict['height'])).fetchone()[0]
            routines.save_yaml(athlete_path + 'data.yaml', athlete_dict)
        #self.db.execute('create table if not exists ' + 'training_' + str(athlete_dict['id']) + ' (Date string PRIMARY KEY, Type string, Activity string, Training string)')
        for activity_dir in os.listdir(athlete_path):
            if not os.path.isdir(athlete_path + activity_dir):
                continue
            activity_path = routines.process_path(athlete_path + activity_dir)
            for training in os.listdir(activity_path):
                if training.endswith('fit'):
                    time_line = parsers.fit_parser(activity_path + training)
                    if time_line is None:
                        continue
                    #print('SELECT * FROM training  WHERE date = {} and athlete_id = {}'.format(min(time_line.keys()), athlete_dict['id']))
                    if self.db.execute('SELECT * FROM training  WHERE date = %s and athlete_id = %s' % (
                            min(time_line.keys()), athlete_dict['id'])).fetchone() is None:
                        self.db.execute('INSERT INTO training (athlete_id, date) '
                                        'VALUES(%d, %s)' % (athlete_dict['id'], min(time_line.keys())))
                        self.db.commit()
                    training_id = \
                        self.db.execute('SELECT training_id FROM training  WHERE date = %s and athlete_id = %s' %
                                        (min(time_line.keys()), athlete_dict['id'])).fetchone()[0]
                    self.db.execute('INSERT INTO activities (activity, training_id,data,filetype) '
                                    'VALUES("%s", %s,"%s","fit")' % (activity_dir, training_id, re.compile('"').sub('@',
                                                                                json.dumps(
                                                                                parsers.change_starting_point(
                                                                                time_line)))))

        self.db.commit()


# -*- coding: utf-8 -*-
import argparse
import os
from utils.io import exit_with_error

args = None
this_dir = os.path.abspath(os.path.dirname(__file__))

def validate_version(version):
    try:
        return tuple(int(p) for p in version.split('.'))
    except ValueError:
        exit_with_error('Invalid version number: {}', version)

def main():
    validate_version(args.next_dev_version)
    # this line pretends to be a hotfix in the uat branch

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Perform a full release waterfall.')
    parser.add_argument('next_dev_version', help='The next version number to be assigned to the develop branch.', type=str)
    args = parser.parse_args()
    main()


import argparse, cv2, os, shutil

parser = argparse.ArgumentParser()

parser.add_argument('--raw_data', type=str, help='Videos')

args = parser.parse_args()

print('Raw data: %s' % args.raw_data)

file_paths = []

for root, dirs, files in os.walk(args.raw_data):
    for video in files:
        if '.MP4' in video:
            file_paths.append(os.path.join(root, file))

return file_paths