"""
This test extends the bmuf_metrics_aggregation_test and tests multiple learners in the distributed training.
"""
import pytest
import cntk
import numpy as np
import sys, os
import argparse
import re
import platform
sys.path.append(os.path.dirname(__file__))
cntk.cntk_py.set_fixed_random_seed(1)
from distributed_learner_test import mpiexec_execute
from bmuf_metrics_aggregation_test import get_minibatch

feat_dim = 5
label_dim = 3
cell_dim = 5
seq_len = 20
num_batches = 101
progress_freq =10

class SingleDataParallelTrainer():
    def __init__(self, frame_mode=False):
        self.create_model(frame_mode)
        self.create_trainer()

    def create_model(self, frame_mode=False):
        if frame_mode:
            self.feat = cntk.input_variable(shape=(feat_dim,))
            self.label = cntk.input_variable((label_dim,))

            net = cntk.layers.Sequential([cntk.layers.Dense(cell_dim), cntk.layers.Dense(label_dim)])
            self.output = net(self.feat)
        else:
            #sequence mode
            self.feat = cntk.sequence.input_variable(shape=(feat_dim,))
            self.label = cntk.sequence.input_variable((label_dim,))

            net = cntk.layers.Sequential([cntk.layers.Recurrence(cntk.layers.LSTM(shape=label_dim, cell_shape=(cell_dim,)))])
            self.output = net(self.feat)

        self.ce = cntk.cross_entropy_with_softmax(self.output, self.label)
        self.err = cntk.classification_error(self.output, self.label)

    def create_trainer(self):
        try:
            lr_per_sample = cntk.learning_parameter_schedule_per_sample(0.007)
            learner = cntk.data_parallel_distributed_learner(cntk.sgd(self.output.parameters, lr_per_sample))

            comm_rank = cntk.distributed.Communicator.rank()
            self.trainer = cntk.Trainer(self.output, (self.ce, self.err), [learner], [cntk.logging.ProgressPrinter(freq=progress_freq, tag="Training", rank=comm_rank)])
        except RuntimeError:
            self.trainer = None
        return

class TwoDataParallelTrainer(SingleDataParallelTrainer):
    def __init__(self, frame_mode=False):
        SingleDataParallelTrainer.__init__(self, frame_mode)

    def create_trainer(self):
        try:
            lr_per_sample = cntk.learning_parameter_schedule_per_sample(0.007)
            p = self.output.parameters
            # Three of four parameters are learned by first data_parallel_distributed_learner.
            learner1 = cntk.data_parallel_distributed_learner(cntk.sgd([p[0],p[1],p[2]], lr_per_sample))

            # New API to mark which learner is to use for metric aggregaion.
            learner1.set_as_metric_aggregator()

            # The last parameter is learned by another data_parallel_distributed_learner.
            learner2 = cntk.data_parallel_distributed_learner(cntk.sgd([p[3]], lr_per_sample))

            comm_rank = cntk.distributed.Communicator.rank()
            self.trainer = cntk.Trainer(self.output, (self.ce, self.err), [learner1, learner2], [cntk.logging.ProgressPrinter(freq=progress_freq, tag="Training", rank=comm_rank)])
        except RuntimeError:
            self.trainer = None
        return

class MultiLearnerTrainer(SingleDataParallelTrainer):
    def __init__(self, frame_mode=False):
        SingleDataParallelTrainer.__init__(self, frame_mode)

    def create_trainer(self):
        try:
            p = self.output.parameters
            # Three of four parameters are learned by block_momentum_distributed_learner.
            bmd_learner = cntk.block_momentum_distributed_learner(cntk.momentum_sgd([p[0],p[1],p[2]], cntk.learning_parameter_schedule(0.0001), cntk.momentum_as_time_constant_schedule(1000)),
                                                                    block_size=1000, block_learning_rate=0.01, block_momentum_as_time_constant=1000)

            # New API to mark which learner is to use for metric aggregaion.
            bmd_learner.set_as_metric_aggregator()

            # The last parameter is learned by the data_parallel_distributed_learner.
            momentum_schedule = cntk.momentum_schedule_per_sample(0.9990913221888589)
            lr_per_sample = cntk.learning_parameter_schedule_per_sample(0.007)
            dpd_learner = cntk.data_parallel_distributed_learner(cntk.momentum_sgd([p[3]], lr_per_sample, momentum_schedule, True))

            comm_rank = cntk.distributed.Communicator.rank()
            self.trainer = cntk.Trainer(self.output, (self.ce, self.err), [bmd_learner, dpd_learner], [cntk.logging.ProgressPrinter(freq=progress_freq, tag="Training", rank=comm_rank)])
        except RuntimeError:
            self.trainer = None
        return

def mpi_worker_multi_learner(trainer, working_dir, checkpoint_dir, mb_source):
    comm_rank = cntk.distributed.Communicator.rank()
    np.random.seed(comm_rank)

    num_paritions = cntk.Communicator.num_workers();
    partition_index = cntk.Communicator.rank();
    checkpoint_performed = False
    for i, data in enumerate(get_minibatch(trainer, working_dir, mb_source, num_paritions, partition_index)):
        trainer.trainer.train_minibatch(data)
        if i % 50 == 0:
            trainer.trainer.summarize_training_progress()
            if not checkpoint_performed and not checkpoint_dir == "":
                trainer.trainer.save_checkpoint(checkpoint_dir)
                trainer.trainer.restore_from_checkpoint(checkpoint_dir)
                checkpoint_performed = True

def get_loss_perepoch_perworker(log_line, num_workers):
    # [0]Finished Epoch[1]: [Training] loss = 1.663636 * 10, metric = 52.40% * 10 0.890s ( 11.2 samples/s);
    regex_pattern = r"\[(?P<worker_rank>\d)\].*? Epoch\[(?P<epoch>\d+)\].*? loss = (?P<loss>\d+\.\d+) \* (?P<samples>\d+).*? metric = (?P<metric>\d+\.\d+)"
    loss_perepoch_perworker = {i:{} for i in range(num_workers)}
    for match in re.finditer(regex_pattern, log_line):
        rank = int(match.groupdict()["worker_rank"])
        epoch = int(match.groupdict()["epoch"])
        loss = match.groupdict()["loss"]
        metric = match.groupdict()["metric"]
        samples = int(match.groupdict()["samples"])
        loss_perepoch_perworker[rank].update({epoch:(loss, metric, samples)})
    return loss_perepoch_perworker

MB_SOURCES = ["ctf_frame"]
@pytest.mark.parametrize("mb_source", MB_SOURCES)
def test_single_data_parallel_learner_vs_two_data_parallel_learners(tmpdir, device_id, mb_source):
    if platform.system() == 'Linux':
        pytest.skip('test only runs on Windows due to mpiexec -l option')

    launch_args = []
    launch_args += ["--outputdir", str(tmpdir)]
    launch_args += ["--mb_source", mb_source]
    launch_args += ["--trainer_type", "single"]

    num_workers = 1 # use a single worker.
    ret_str = mpiexec_execute(__file__, ['-n', str(num_workers), '-l'], launch_args)
    print(ret_str)
    loss_perepoch_perworker = get_loss_perepoch_perworker(ret_str, num_workers)

    loss_per_worker = loss_perepoch_perworker.values()
    single_learner_loss_per_worker_epochsort = []
    for epoch_losses in loss_per_worker:
        single_learner_loss_per_worker_epochsort.append([epoch_losses[i] for i in sorted(epoch_losses)])

    launch_args = []
    launch_args += ["--outputdir", str(tmpdir)]
    launch_args += ["--mb_source", mb_source]
    launch_args += ["--trainer_type", "two"]

    num_workers = 2 # now run in distributed workers.
    ret_str = mpiexec_execute(__file__, ['-n', str(num_workers), '-l'], launch_args)
    print(ret_str)
    loss_perepoch_perworker = get_loss_perepoch_perworker(ret_str, num_workers)

    loss_per_worker = loss_perepoch_perworker.values()
    multi_learner_loss_per_worker_epochsort = []
    for epoch_losses in loss_per_worker:
        multi_learner_loss_per_worker_epochsort.append([epoch_losses[i] for i in sorted(epoch_losses)])

    assert all([single_learner_loss_per_worker_epochsort[0] == i for i in multi_learner_loss_per_worker_epochsort])

MB_SOURCES = ["ctf_frame"]
@pytest.mark.parametrize("mb_source", MB_SOURCES)
def test_multi_learner_bmuf_correct_metrics_averaging(tmpdir, device_id, mb_source):
    if platform.system() == 'Linux':
        pytest.skip('test only runs on Windows due to mpiexec -l option')

    num_workers = 2
    # check whether trainer can be initialized or not
    bmuf = MultiLearnerTrainer()
    if not bmuf.trainer:
        pytest.skip('BMUF not available on this build')

    launch_args = []
    launch_args += ["--outputdir", str(tmpdir)]
    launch_args += ["--mb_source", mb_source]
    launch_args += ["--trainer_type", "multi"]

    ret_str = mpiexec_execute(__file__, ['-n', str(num_workers), '-l'], launch_args)
    print(ret_str)
    loss_perepoch_perworker = get_loss_perepoch_perworker(ret_str, num_workers)

    num_epochs_per_worker = list(map(len,loss_perepoch_perworker.values()))

    #assert that data exists
    assert len(num_epochs_per_worker) != 0

    #assert that number of epochs isn't zero for 1st worker.
    assert num_epochs_per_worker[0] != 0

    # assert all workers have same number of epochs
    assert min(num_epochs_per_worker) == max(num_epochs_per_worker)

    # assert all workers have same loss and metric values
    loss_per_worker = loss_perepoch_perworker.values()
    loss_per_worker_epochsort = []
    for epoch_losses in loss_per_worker:
        loss_per_worker_epochsort.append([epoch_losses[i] for i in sorted(epoch_losses)])

    assert all([loss_per_worker_epochsort[0] == i for i in loss_per_worker_epochsort])

    # Do the same test with checkpoint and compare the results.
    launch_args += ["--checkpointdir", str(tmpdir.join('checkpoint'))]

    ret_str = mpiexec_execute(__file__, ['-n', str(num_workers), '-l'], launch_args)
    print(ret_str)

    loss_perepoch_perworker = get_loss_perepoch_perworker(ret_str, num_workers)

    num_epochs_per_worker = list(map(len,loss_perepoch_perworker.values()))

    #assert that data exists
    assert len(num_epochs_per_worker) != 0

    #assert that number of epochs isn't zero for 1st worker.
    assert num_epochs_per_worker[0] != 0

    # assert all workers have same number of epochs
    assert min(num_epochs_per_worker) == max(num_epochs_per_worker)

    # assert all workers have same loss and metric values
    loss_per_worker = loss_perepoch_perworker.values()
    multi_learner_loss_per_worker_epochsort = []
    for epoch_losses in loss_per_worker:
        multi_learner_loss_per_worker_epochsort.append([epoch_losses[i] for i in sorted(epoch_losses)])

    # Compare no checkpoint loss, matric, and num samples, to checkpoint loss values.
    for i in multi_learner_loss_per_worker_epochsort:
        for n in range(3):
            for m in range(3):
                assert np.allclose(float(loss_per_worker_epochsort[0][n][m]), float(i[n][m]))

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-outputdir', '--outputdir')
    parser.add_argument('-checkpointdir', '--checkpointdir')
    parser.add_argument('-mb_source', '--mb_source')
    parser.add_argument("-trainer_type","--trainer_type")
    args = vars(parser.parse_args())

    frame_mode = (args["mb_source"] == "ctf_frame")

    if args["trainer_type"] == "multi":
        trainer = MultiLearnerTrainer(frame_mode)
        if args["checkpointdir"]:

            mpi_worker_multi_learner(trainer, args["outputdir"], args["checkpointdir"], args["mb_source"])
        else:
            mpi_worker_multi_learner(trainer, args["outputdir"], "", args["mb_source"])

    elif args["trainer_type"] == "two":
        trainer = TwoDataParallelTrainer(frame_mode)
        mpi_worker_multi_learner(trainer, args["outputdir"], "", args["mb_source"])

    elif args["trainer_type"] == "single":
        print("Coming to a single learner")
        trainer = SingleDataParallelTrainer(frame_mode)
        mpi_worker_multi_learner(trainer, args["outputdir"], "", args["mb_source"])

    cntk.distributed.Communicator.finalize()


import argparse
import os
from enum import Enum

from autouri import URIBase

from .arg_tool import update_parsers_defaults_with_conf
from .backward_compatibility import PARAM_KEY_NAME_CHANGE
from .caper_workflow_opts import CaperWorkflowOpts
from .cromwell import Cromwell
from .cromwell_backend import (
    CALL_CACHING_DUP_STRAT_COPY,
    CALL_CACHING_DUP_STRAT_REFERENCE,
    LOCAL_HASH_STRAT_FILE,
    LOCAL_HASH_STRAT_PATH,
    LOCAL_HASH_STRAT_PATH_MTIME,
    CromwellBackendAws,
    CromwellBackendBase,
    CromwellBackendCommon,
    CromwellBackendDatabase,
    CromwellBackendGcp,
    CromwellBackendLocal,
    CromwellBackendSlurm,
)
from .cromwell_rest_api import CromwellRestAPI
from .hpc import LsfWrapper, PbsWrapper, SgeWrapper, SlurmWrapper
from .resource_analysis import ResourceAnalysis
from .server_heartbeat import ServerHeartbeat

DEFAULT_CAPER_CONF = '~/.caper/default.conf'
DEFAULT_LIST_FORMAT = 'id,status,name,str_label,user,parent,submission'
DEFAULT_OUT_DIR = '.'
DEFAULT_CROMWELL_STDOUT = './cromwell.out'

class ResourceAnalysisReductionMethod(Enum):
    sum = sum
    max = max
    min = min
    none = None

def get_defaults(conf_file=None):
    """Wrapper for `get_parser_and_defaults()`.
    Use this function to get default values updated with `conf_file`.

    Args:
        conf_file:
            `DEFAULT_CAPER_CONF` will be used if it is None.

    Returns updated defaults only.
    """
    _, conf_dict = get_parser_and_defaults(conf_file=conf_file)
    return conf_dict

def get_parser_and_defaults(conf_file=None):
    """Creates a main parser and make a subparser for each subcommand.
    There are many parent parsers defined here.
    Each subparser will take a certain combination of these parent parsers
    to share the same parameter arguments between subcommands.
    e.g. subcommand run and server share the same --cromwell argument, which
    is defined in a parent parser "parent_runner".

    Finally each subparser's default is updated with values defined in conf_file.

    Args:
        conf_file:
            If defined, this will be used instead of partially parsing command line
            arguments to find conf_file (-c).
            `DEFAULT_CAPER_CONF` will be used if it is None.
    Returns:
        parser:
            ArgumentParser object with all arguments defined for each sub-
            command (subparser).
        conf_dict:
            Dict of key/value pairs parsed from conf_file.
            Such value is converted into a correct type guessed from
            defaults of arguments defined in ArgumentParser object.
    """
    parser = argparse.ArgumentParser(
        description='Caper (Cromwell-assisted Pipeline ExecutioneR)'
    )
    parser.add_argument('-v', '--version', action='store_true', help='Show version')

    subparser = parser.add_subparsers(dest='action')

    parent_init = argparse.ArgumentParser(add_help=False)
    parent_init.add_argument('platform', help='Platform to initialize Caper for.')

    # all
    parent_all = argparse.ArgumentParser(add_help=False)
    parent_all.add_argument(
        '-c', '--conf', help='Specify config file', default=DEFAULT_CAPER_CONF
    )
    parent_all.add_argument(
        '-D', '--debug', action='store_true', help='Prints all logs >= DEBUG level'
    )
    parent_all.add_argument(
        '--gcp-service-account-key-json',
        help='Secret key JSON file for Google Cloud Platform service account. '
        'This service account should have enough permission to '
        'Storage for client functions and '
        'Storage/Compute Engine/Genomics API/Life Sciences API '
        'for server/runner functions.',
    )

    group_loc = parent_all.add_argument_group(
        title='cache directories for localization'
    )
    group_loc.add_argument(
        '--local-loc-dir',
        '--tmp-dir',
        help='Temporary directory to store Cromwell\'s intermediate backend files. '
        'These files include backend.conf, workflow_opts.json, imports.zip. and '
        'localized input JSON files due to deepcopying (recursive localization). '
        'Cromwell\'s MySQL/PostgreSQL DB password can be exposed on backend.conf '
        'on this directory. Therefore, DO NOT USE /tmp HERE. This directory is '
        'also used for storing cached files for local/slurm/sge/pbs/lsf backends.',
    )
    group_loc.add_argument(
        '--gcp-loc-dir',
        '--tmp-gcs-bucket',
        help='Temporary directory to store cached files for gcp backend. '
        'e.g. gs://my-bucket/caper-cache-dir. ',
    )
    group_loc.add_argument(
        '--aws-loc-dir',
        '--tmp-s3-bucket',
        help='Temporary directory to store cached files for aws backend. '
        'e.g. s3://my-bucket/caper-cache-dir. ',
    )

    # run, server, submit
    parent_backend = argparse.ArgumentParser(add_help=False)
    parent_backend.add_argument('-b', '--backend', help='Backend to run a workflow')
    parent_backend.add_argument(
        '--dry-run',
        action='store_true',
        help='Caper localizes remote files and validates WDL '
        'but does not run/submit a pipeline.',
    )

    # run, server
    parent_runner = argparse.ArgumentParser(add_help=False)

    parent_runner = parent_runner.add_argument_group(title='Cromwell logging arguments')
    parent_runner.add_argument(
        '--cromwell-stdout',
        default=DEFAULT_CROMWELL_STDOUT,
        help='Local file to write STDOUT of Cromwell Java process to. '
        'This is for Cromwell (not for Caper\'s logging system). '
        'If this file already exists then Caper will make a new file suffixed with '
        'incremented index. e.g. cromwell.out.1 ',
    )
    group_db = parent_runner.add_argument_group(
        title='General DB settings (for both file DB and MySQL DB)'
    )
    group_db.add_argument(
        '--db',
        default=CromwellBackendDatabase.DB_FILE,
        help='Cromwell metadata database type',
    )
    group_db.add_argument(
        '--db-timeout',
        type=int,
        default=CromwellBackendDatabase.DEFAULT_DB_TIMEOUT_MS,
        help='Milliseconds to wait for DB connection.',
    )

    group_file_db = parent_runner.add_argument_group(
        title='HyperSQL file DB arguments (unstable, not recommended)'
    )
    group_file_db.add_argument(
        '--file-db',
        '-d',
        help='Default DB file for Cromwell\'s built-in HyperSQL database.',
    )

    group_mysql = parent_runner.add_argument_group(title='MySQL DB arguments')
    group_mysql.add_argument(
        '--mysql-db-ip',
        default=CromwellBackendDatabase.DEFAULT_MYSQL_DB_IP,
        help='MySQL Database IP address (e.g. localhost)',
    )
    group_mysql.add_argument(
        '--mysql-db-port',
        type=int,
        default=CromwellBackendDatabase.DEFAULT_MYSQL_DB_PORT,
        help='MySQL Database TCP/IP port (e.g. 3306)',
    )
    group_mysql.add_argument(
        '--mysql-db-user',
        default=CromwellBackendDatabase.DEFAULT_MYSQL_DB_USER,
        help='MySQL DB username',
    )
    group_mysql.add_argument(
        '--mysql-db-password',
        default=CromwellBackendDatabase.DEFAULT_MYSQL_DB_PASSWORD,
        help='MySQL DB password',
    )
    group_mysql.add_argument(
        '--mysql-db-name',
        default=CromwellBackendDatabase.DEFAULT_MYSQL_DB_NAME,
        help='MySQL DB name for Cromwell',
    )

    group_postgresql = parent_runner.add_argument_group(title='PostgreSQL DB arguments')
    group_postgresql.add_argument(
        '--postgresql-db-ip',
        default=CromwellBackendDatabase.DEFAULT_POSTGRESQL_DB_IP,
        help='PostgreSQL DB IP address (e.g. localhost)',
    )
    group_postgresql.add_argument(
        '--postgresql-db-port',
        type=int,
        default=CromwellBackendDatabase.DEFAULT_POSTGRESQL_DB_PORT,
        help='PostgreSQL DB TCP/IP port (e.g. 5432)',
    )
    group_postgresql.add_argument(
        '--postgresql-db-user',
        default=CromwellBackendDatabase.DEFAULT_POSTGRESQL_DB_USER,
        help='PostgreSQL DB username',
    )
    group_postgresql.add_argument(
        '--postgresql-db-password',
        default=CromwellBackendDatabase.DEFAULT_POSTGRESQL_DB_PASSWORD,
        help='PostgreSQL DB password',
    )
    group_postgresql.add_argument(
        '--postgresql-db-name',
        default=CromwellBackendDatabase.DEFAULT_POSTGRESQL_DB_NAME,
        help='PostgreSQL DB name for Cromwell',
    )

    group_cromwell = parent_runner.add_argument_group(title='Cromwell settings')
    group_cromwell.add_argument(
        '--cromwell',
        default=Cromwell.DEFAULT_CROMWELL,
        help='Path or URL for Cromwell JAR file',
    )
    group_cromwell.add_argument(
        '--max-concurrent-tasks',
        type=int,
        default=CromwellBackendBase.DEFAULT_CONCURRENT_JOB_LIMIT,
        help='Number of concurrent tasks. '
        '"config.concurrent-job-limit" in Cromwell backend configuration '
        'for each backend',
    )
    group_cromwell.add_argument(
        '--max-concurrent-workflows',
        type=int,
        default=CromwellBackendCommon.DEFAULT_MAX_CONCURRENT_WORKFLOWS,
        help='Number of concurrent workflows. '
        '"system.max-concurrent-workflows" in backend configuration',
    )
    group_cromwell.add_argument(
        '--memory-retry-error-keys',
        default=','.join(CromwellBackendCommon.DEFAULT_MEMORY_RETRY_ERROR_KEYS),
        help='(CURRENTLY NOT WORKING) '
        'If an error caught by these comma-separated keys occurs, '
        'then increase memory by --memory-retry-multiplier '
        'for retrials controlled by --max-retries. '
        'See https://cromwell.readthedocs.io/en/develop/cromwell_features/RetryWithMoreMemory/ '
        'for details. ',
    )
    group_cromwell.add_argument(
        '--disable-call-caching',
        action='store_true',
        help='Disable Cromwell\'s call caching, which re-uses outputs from '
        'previous workflows',
    )
    group_cromwell.add_argument(
        '--backend-file',
        help='Custom Cromwell backend configuration file to override all',
    )
    group_cromwell.add_argument(
        '--soft-glob-output',
        action='store_true',
        help='Use soft-linking when globbing outputs for a filesystem that '
        'does not allow hard-linking. e.g. beeGFS. '
        'This flag does not work with backends based on a Docker container. '
        'i.e. gcp and aws. Also, '
        'it does not work with local backends (local/slurm/sge/pbs/lsf) '
        'with --. However, it works fine with --singularity.',
    )
    group_cromwell.add_argument(
        '--local-hash-strat',
        default=CromwellBackendLocal.DEFAULT_LOCAL_HASH_STRAT,
        choices=[
            LOCAL_HASH_STRAT_FILE,
            LOCAL_HASH_STRAT_PATH,
            LOCAL_HASH_STRAT_PATH_MTIME,
        ],
        help='File hashing strategy for call caching. '
        'For local backends (local/slurm/sge/pbs/lsf) only. '
        'file: use md5sum hash (slow), path: use path only, '
        'path+modtime (default): use path + mtime.',
    )

    group_local = parent_runner.add_argument_group(title='local backend arguments')
    group_local.add_argument(
        '--local-out-dir',
        '--out-dir',
        default=DEFAULT_OUT_DIR,
        help='Output directory path for local backend. '
        'Cloud backends (gcp, aws) use different output directories. '
        'For gcp, define --gcp-out-dir. '
        'For aws, define --aws-out-dir.',
    )
    group_local.add_argument(
        '--slurm-resource-param',
        help='SLURM resource parameters to be passed to sbatch. '
        'You can customize this to fit your cluster\'s configuration. '
        'You can use WDL syntax in ${} notation with Cromwell\'s built-in resource '
        'variables. See documentation for details. ',
        default=CromwellBackendSlurm.DEFAULT_SLURM_RESOURCE_PARAM,
    )

    group_gc_all = parent_backend.add_argument_group(
        title='GCP backend arguments for server/runner/client'
    )
    group_gc = parent_runner.add_argument_group(
        title='GCP backend arguments for server/runner'
    )
    group_gc.add_argument('--gcp-prj', help='GC project')
    group_gc_all.add_argument(
        '--use-google-cloud-life-sciences',
        action='store_true',
        help='Use Google Cloud Life Sciences API (v2beta) instead of '
        'deprecated Genomics API (v2alpha1).'
        'Life Sciences API requires only one region specified with'
        'gcp-region. gcp-zones will be ignored since it is for Genomics API.'
        'See https://cloud.google.com/life-sciences/docs/concepts/locations '
        'for supported regions.',
    )
    group_gc.add_argument(
        '--gcp-region',
        default=CromwellBackendGcp.DEFAULT_REGION,
        help='GCP region for Google Cloud Life Sciences API. '
        'This is used only when --use-google-cloud-life-sciences is defined.',
    )
    group_gc_all.add_argument(
        '--gcp-zones',
        help='Comma-separated GCP zones used for Genomics API. '
        '(e.g. us-west1-b,us-central1-b). '
        'If you use --use-google-cloud-life-sciences then '
        'define --gcp-region instead.',
    )
    group_gc.add_argument(
        '--gcp-call-caching-dup-strat',
        default=CromwellBackendGcp.DEFAULT_CALL_CACHING_DUP_STRAT,
        choices=[CALL_CACHING_DUP_STRAT_REFERENCE, CALL_CACHING_DUP_STRAT_COPY],
        help='Duplication strategy for call-cached outputs for GCP backend: '
        'copy: make a copy, reference: refer to old output in metadata.json.',
    )
    group_gc.add_argument(
        '--gcp-out-dir',
        '--out-gcs-bucket',
        help='Output directory path for GCP backend. ' 'e.g. gs://my-bucket/my-output.',
    )

    group_aws = parent_runner.add_argument_group(title='AWS backend arguments')
    group_aws.add_argument('--aws-batch-arn', help='ARN for AWS Batch')
    group_aws.add_argument('--aws-region', help='AWS region (e.g. us-west-1)')
    group_aws.add_argument(
        '--aws-out-dir',
        '--out-s3-bucket',
        help='Output path on S3 bucket for AWS backend. '
        'e.g. s3://my-bucket/my-output.',
    )
    group_aws.add_argument(
        '--aws-call-caching-dup-strat',
        default=CromwellBackendAws.DEFAULT_CALL_CACHING_DUP_STRAT,
        choices=[CALL_CACHING_DUP_STRAT_REFERENCE, CALL_CACHING_DUP_STRAT_COPY],
        help='Duplication strategy for call-cached outputs for AWS backend: '
        'copy: make a copy, reference: refer to old output in metadata.json.',
    )

    # run, submit
    parent_submit = argparse.ArgumentParser(add_help=False)

    parent_submit.add_argument(
        'wdl',
        help='Path, URL or URI for WDL script '
        'Example: /scratch/my.wdl, gs://some/where/our.wdl, '
        'http://hello.com/world/your.wdl',
    )
    parent_submit.add_argument('-i', '--inputs', help='Workflow inputs JSON file')
    parent_submit.add_argument('-o', '--options', help='Workflow options JSON file')
    parent_submit.add_argument('-l', '--labels', help='Workflow labels JSON file')
    parent_submit.add_argument(
        '-p', '--imports', help='Zip file of imported subworkflows'
    )
    parent_submit.add_argument(
        '-s',
        '--str-label',
        help='Caper\'s special label for a workflow '
        'This label will be added to a workflow labels JSON file '
        'as a value for a key "caper-label". '
        'DO NOT USE IRREGULAR CHARACTERS. USE LETTERS, NUMBERS, '
        'DASHES AND UNDERSCORES ONLY (^[A-Za-z0-9\\-\\_]+$) '
        'since this label is used to compose a path for '
        'workflow\'s temporary/cache directory (.caper_tmp/label/timestamp/)',
    )
    parent_submit.add_argument(
        '--hold',
        action='store_true',
        help='Put a hold on a workflow when submitted to a Cromwell server.',
    )
    parent_submit.add_argument(
        '--use-gsutil-for-s3',
        action='store_true',
        help='Use gsutil CLI for direct trasnfer between S3 and GCS buckets. '
        'Otherwise, such file transfer will stream through local machine. '
        'Make sure that gsutil is installed on your system and it has access to '
        'credentials for AWS (e.g. ~/.boto or ~/.aws/credentials).',
    )
    parent_submit.add_argument(
        '--no-deepcopy',
        action='store_true',
        help='(IMPORTANT) --deepcopy has been deprecated. '
        'Deepcopying is now activated by default. '
        'This flag disables deepcopy for '
        'JSON (.json), TSV (.tsv) and CSV (.csv) '
        'files specified in an input JSON file (--inputs). '
        'Find all path/URI string values in an input JSON file '
        'and make copies of files on a local/remote storage '
        'for a target backend. Make sure that you have installed '
        'gsutil for GCS and aws for S3.',
    )
    parent_submit.add_argument(
        '--ignore-womtool',
        action='store_true',
        help='Ignore warnings from womtool.jar.',
    )
    parent_submit.add_argument(
        '--womtool',
        default=Cromwell.DEFAULT_WOMTOOL,
        help='Path or URL for Cromwell\'s womtool JAR file',
    )
    parent_submit.add_argument(
        '--java-heap-womtool',
        default=Cromwell.DEFAULT_JAVA_HEAP_WOMTOOL,
        help='Java heap size for Womtool (java -Xmx)',
    )
    parent_submit.add_argument(
        '--max-retries',
        type=int,
        default=CaperWorkflowOpts.DEFAULT_MAX_RETRIES,
        help='Number of retries for failing tasks. '
        'equivalent to "maxRetries" in workflow options JSON file.',
    )
    parent_submit.add_argument(
        '--memory-retry-multiplier',
        default=CaperWorkflowOpts.DEFAULT_MEMORY_RETRY_MULTIPLIER,
        help='(CURRENTLY NOT WORKING) '
        'If an error caught by --memory-retry-error-keys occurs, '
        'then increase memory by this '
        'for retrials controlled by --max-retries. '
        'See https://cromwell.readthedocs.io/en/develop/cromwell_features/RetryWithMoreMemory/ '
        'for details.',
    )
    parent_submit.add_argument(
        '--gcp-monitoring-script',
        default=CaperWorkflowOpts.DEFAULT_GCP_MONITORING_SCRIPT,
        help='Monitoring script for gcp backend only. '
        'Caper defaults to use its own monitoring script which works fine '
        'with subcommand gcp_profile. '
        'To make your script work with gcp_profile, '
        'make this script generate a TSV with a header in the first row. '
        'The first column of such TSV will be ignored since it is usually timestamp. '
        'Check monitoring_script in '
        'https://cromwell.readthedocs.io/en/stable/wf_options/Google/'
        '#google-pipelines-api-workflow-options '
        'for details.',
    )
    group_dep = parent_submit.add_argument_group(
        title='dependency resolver for all backends',
        description='Cloud-based backends (gc and aws) will only use Docker '
        'so that "--docker URI_FOR_DOCKER_IMG" must be specified '
        'in the command line argument or as a comment "#CAPER '
        'docker URI_FOR_DOCKER_IMG" or value for "workflow.meta.default_docker"'
        'in a WDL file',
    )
    group_dep.add_argument(
        '--docker',
        nargs='?',
        const='',
        default=None,
        help='URI for Docker image (e.g. ubuntu:latest). '
        'This can also be used as a flag to use Docker image URI '
        'defined in your WDL file as a comment ("#CAPER docker") or '
        'as "workflow.meta.default_docker" in WDL.',
    )
    group_dep_local = parent_submit.add_argument_group(
        title='dependency resolver for local backend',
        description='Singularity is for local backend only. Other backends '
        '(gcp and aws) will use Docker only. '
        'Local backend defaults to not use any container-based methods. '
        'Use "--singularity" or "--docker" to use containers',
    )
    group_dep_local.add_argument(
        '--singularity',
        nargs='?',
        const='',
        default=None,
        help='URI or path for Singularity image '
        '(e.g. ~/.singularity/ubuntu-latest.simg, '
        'docker://ubuntu:latest, shub://vsoch/hello-world). '
        'This can also be used as a flag to use Singularity image URI '
        'defined in your WDL file as a comment ("#CAPER singularity") or '
        'as "workflow.meta.default_singularity" in WDL.',
    )
    group_dep_local.add_argument(
        '--conda',
        nargs='?',
        const='',
        default=None,
        help='Default Conda environment\'s name. '
        'If defined each task in WDL will be called with conda run -n ENV_NAME.'
        'This can also be used as a flag to use Conda environment '
        'defined in your WDL file under "workflow.meta.default_conda".',
    )
    group_hpc_submit = parent_submit.add_argument_group(
        title='Parameters for "caper hpc submit" command only',
    )
    group_hpc_submit.add_argument(
        '--leader-job-name',
        help='Leader job name for a submitted workflow.'
        'This name will be appended to the prefix "CAPER_LEADER_" and then '
        'submitted to HPC. Such prefix is used to identify Caper leader jobs.',
    )
    group_hpc_submit.add_argument(
        '--slurm-leader-job-resource-param',
        help='Resource parameters to submit a Caper leader job to SLURM. '
        'Make sure to quote if you use it in the command line arguments.',
        default=' '.join(SlurmWrapper.DEFAULT_LEADER_JOB_RESOURCE_PARAM),
    )
    group_hpc_submit.add_argument(
        '--sge-leader-job-resource-param',
        help='Resource parameters to submit a Caper leader job to SGE'
        'Make sure to quote if you use it in the command line arguments.',
        default=' '.join(SgeWrapper.DEFAULT_LEADER_JOB_RESOURCE_PARAM),
    )
    group_hpc_submit.add_argument(
        '--pbs-leader-job-resource-param',
        help='Resource parameters to submit a Caper leader job to PBS'
        'Make sure to quote if you use it in the command line arguments.',
        default=' '.join(PbsWrapper.DEFAULT_LEADER_JOB_RESOURCE_PARAM),
    )
    group_hpc_submit.add_argument(
        '--lsf-leader-job-resource-param',
        help='Resource parameters to submit a Caper leader job to LSF'
        'Make sure to quote if you use it in the command line arguments.',
        default=' '.join(LsfWrapper.DEFAULT_LEADER_JOB_RESOURCE_PARAM),
    )

    group_slurm = parent_submit.add_argument_group('SLURM arguments')
    group_slurm.add_argument('--slurm-partition', help='SLURM partition')
    group_slurm.add_argument('--slurm-account', help='SLURM account')
    group_slurm.add_argument(
        '--slurm-extra-param', help='SLURM extra parameters to be passed to sbatch. '
    )

    group_sge = parent_submit.add_argument_group('SGE arguments')
    group_sge.add_argument(
        '--sge-pe', help='SGE parallel environment. Check with "qconf -spl"'
    )
    group_sge.add_argument('--sge-queue', help='SGE queue. Check with "qconf -sql"')
    group_sge.add_argument(
        '--sge-extra-param', help='SGE extra parameters. Must be double-quoted'
    )

    group_pbs = parent_submit.add_argument_group('PBS arguments')
    group_pbs.add_argument('--pbs-queue', help='PBS queue')
    group_pbs.add_argument(
        '--pbs-extra-param', help='PBS extra parameters. Must be double-quoted'
    )

    group_lsf = parent_submit.add_argument_group('LSF arguments')
    group_lsf.add_argument('--lsf-queue', help='LSF queue')
    group_lsf.add_argument(
        '--lsf-extra-param', help='LSF extra parameters. Must be double-quoted'
    )

    # server
    parent_server = argparse.ArgumentParser(add_help=False)
    parent_server.add_argument(
        '--java-heap-server',
        default=Cromwell.DEFAULT_JAVA_HEAP_CROMWELL_SERVER,
        help='Cromwell Java heap size for "server" mode (java -Xmx)',
    )
    parent_server.add_argument(
        '--disable-auto-write-metadata',
        action='store_true',
        help='Disable automatic retrieval/update/writing of metadata.json upon workflow/task status change.',
    )

    # run
    parent_run = argparse.ArgumentParser(add_help=False)
    parent_run.add_argument(
        '-m',
        '--metadata-output',
        help='An optional directory path to output metadata JSON file',
    )
    parent_run.add_argument(
        '--java-heap-run',
        default=Cromwell.DEFAULT_JAVA_HEAP_CROMWELL_RUN,
        help='Cromwell Java heap size for "run" mode (java -Xmx)',
    )

    # list, metadata, abort
    parent_search_wf = argparse.ArgumentParser(add_help=False)
    parent_search_wf.add_argument(
        'wf_id_or_label',
        nargs='*',
        help='List of workflow IDs to find matching workflows to '
        'commit a specified action (list, metadata and abort). '
        'Wildcards (* and ?) are allowed.',
    )

    # server, all client subcommands
    parent_server_client = argparse.ArgumentParser(add_help=False)
    parent_server_client.add_argument(
        '--port',
        type=int,
        default=Cromwell.DEFAULT_SERVER_PORT,
        help='Port for Caper server',
    )
    parent_server_client.add_argument(
        '--no-server-heartbeat',
        action='store_true',
        help='Disable server heartbeat file.',
    )
    parent_server_client.add_argument(
        '--server-heartbeat-file',
        default=ServerHeartbeat.DEFAULT_SERVER_HEARTBEAT_FILE,
        help='Heartbeat file for Caper clients to get IP and port of a server',
    )
    parent_server_client.add_argument(
        '--server-heartbeat-timeout',
        type=int,
        default=ServerHeartbeat.DEFAULT_HEARTBEAT_TIMEOUT_MS,
        help='Timeout for a heartbeat file in Milliseconds. '
        'A heartbeat file older than '
        'this interval will be ignored.',
    )

    parent_client = argparse.ArgumentParser(add_help=False)
    parent_client.add_argument(
        '--hostname',
        '--ip',
        default=CromwellRestAPI.DEFAULT_HOSTNAME,
        help='Hostname (or IP address) of Caper server.',
    )

    # list
    parent_list = argparse.ArgumentParser(add_help=False)
    parent_list.add_argument(
        '-f',
        '--format',
        default=DEFAULT_LIST_FORMAT,
        help='Comma-separated list of items to be shown for "list" '
        'subcommand. Any key name in workflow JSON from Cromwell '
        'server\'s response is allowed. '
        'Available keys are "id" (workflow ID), "status", "str_label", '
        '"name" (WDL/CWL name), "submission" (date/time), "start", '
        '"end" and "user". '
        '"str_label" is a special key for Caper. See help context '
        'of "--str-label" for details',
    )
    parent_list.add_argument(
        '--hide-result-before',
        help='Hide workflows submitted before this date/time. '
        'Use the same (or shorter) date/time format shown in '
        '"caper list". '
        'e.g. 2019-06-13, 2019-06-13T10:07',
    )
    parent_list.add_argument(
        '--show-subworkflow',
        action='store_true',
        help='Show subworkflows in "caper list". '
        'WARNING: If there are too many subworkflows, '
        'this can result in crash of Caper/Cromwell server ',
    )

    # troubleshoot/debug
    parent_troubleshoot = argparse.ArgumentParser(add_help=False)
    parent_troubleshoot.add_argument(
        '--show-completed-task',
        action='store_true',
        help='Show information about completed tasks.',
    )
    parent_troubleshoot.add_argument(
        '--show-stdout', action='store_true', help='Show STDOUT for failed tasks.'
    )

    # gcp_monitor
    parent_gcp_monitor = argparse.ArgumentParser(add_help=False)
    parent_gcp_monitor.add_argument(
        '--json-format',
        action='store_true',
        help='Prints out outputs in a JSON format.',
    )

    # gcp_res_analysis
    parent_gcp_res_analysis = argparse.ArgumentParser(add_help=False)
    parent_gcp_res_analysis.add_argument(
        '--in-file-vars-def-json',
        help='JSON file to define task name and input file variabless '
        'to be included in resource analysis. '
        'Key: task name, wild-cards (*, ?) are allowed. '
        'Value: list of input file var names. '
        'e.g. "atac.align*": ["fastqs_R1", "fastqs_R2"]. '
        'Once this file is defined, tasks not included in it will be ignored.',
    )
    parent_gcp_res_analysis.add_argument(
        '--reduce-in-file-vars',
        choices=[method.name for method in list(ResourceAnalysisReductionMethod)],
        default=ResourceAnalysisReductionMethod.sum.name,
        help='Reduce X matrix (resource data) into a vector. '
        'e.g. summing up all input file sizes. '
        'Reducing X will convert a multiple linear regression into a single linear regression. '
        'This is useful since single linear regression requires much less data '
        '(at least 2 for each task). '
        'Choose NONE to keep all input file variables '
        'without reduction in the analysis. '
        '2D Scatter plot (--plot-pdf) will not available for analysis without reduction. '
        'If NONE then make sure that number of datasets (per task) '
        '> number of input file variables in a task.',
    )
    parent_gcp_res_analysis.add_argument(
        '--target-resources',
        nargs='+',
        default=list(ResourceAnalysis.DEFAULT_TARGET_RESOURCES),
        help='Keys for resources in a JSON gcp_monitor outputs, '
        'which forms y vector for a linear problem. '
        'Analysis will be done separately for each key (resource metric). '
        'See help for gcp_monitor to find available resources. '
        'e.g. stats.max.disk, stats.mean.cpu_pct.',
    )
    parent_gcp_res_analysis.add_argument(
        '--plot-pdf',
        help='Local path for a 2D scatter plot PDF file. '
        'Scatter plot will not be available if --reduce-in-file-vars is none.',
    )

    # cleanup
    parent_cleanup = argparse.ArgumentParser(add_help=False)
    parent_cleanup.add_argument(
        '--delete',
        action='store_true',
        help='DELETE OUTPUTS. caper cleanup runs in a dry-run mode by default. ',
    )
    parent_cleanup.add_argument(
        '--num-threads',
        default=URIBase.DEFAULT_NUM_THREADS,
        type=int,
        help='Number of threads for cleaning up workflow\'s outputs. '
        'This is used for cloud backends only.',
    )

    # hpc abort
    parent_hpc_abort = argparse.ArgumentParser(add_help=False)
    parent_hpc_abort.add_argument(
        'job_ids',
        nargs='+',
        help='Job ID or list of job IDs to abort matching Caper leader jobs.',
    )

    # all subcommands
    p_init = subparser.add_parser(
        'init',
        help='Initialize Caper\'s configuration file. THIS WILL OVERWRITE ON '
        'A SPECIFIED(-c)/DEFAULT CONF FILE. e.g. ~/.caper/default.conf.',
        parents=[parent_all, parent_init],
    )
    p_run = subparser.add_parser(
        'run',
        help='Run a single workflow without server',
        parents=[parent_all, parent_submit, parent_run, parent_runner, parent_backend],
    )
    p_server = subparser.add_parser(
        'server',
        help='Run a Cromwell server',
        parents=[
            parent_all,
            parent_server_client,
            parent_server,
            parent_runner,
            parent_backend,
        ],
    )
    p_submit = subparser.add_parser(
        'submit',
        help='Submit a workflow to a Cromwell server',
        parents=[
            parent_all,
            parent_server_client,
            parent_client,
            parent_submit,
            parent_backend,
        ],
    )
    p_abort = subparser.add_parser(
        'abort',
        help='Abort running/pending workflows on a Cromwell server',
        parents=[parent_all, parent_server_client, parent_client, parent_search_wf],
    )
    p_unhold = subparser.add_parser(
        'unhold',
        help='Release hold of workflows on a Cromwell server',
        parents=[parent_all, parent_server_client, parent_client, parent_search_wf],
    )
    p_list = subparser.add_parser(
        'list',
        help='List running/pending workflows on a Cromwell server',
        parents=[
            parent_all,
            parent_server_client,
            parent_client,
            parent_search_wf,
            parent_list,
        ],
    )
    p_metadata = subparser.add_parser(
        'metadata',
        help='Retrieve metadata JSON for workflows from a Cromwell server',
        parents=[parent_all, parent_server_client, parent_client, parent_search_wf],
    )
    p_troubleshoot = subparser.add_parser(
        'troubleshoot',
        help='Troubleshoot workflow problems from metadata JSON file or '
        'workflow IDs',
        parents=[
            parent_all,
            parent_server_client,
            parent_client,
            parent_search_wf,
            parent_troubleshoot,
        ],
    )
    p_debug = subparser.add_parser(
        'debug',
        help='Identical to "troubleshoot"',
        parents=[
            parent_all,
            parent_server_client,
            parent_client,
            parent_search_wf,
            parent_troubleshoot,
        ],
    )

    p_hpc = subparser.add_parser(
        'hpc',
        help='Subcommand for HPCs',
        parents=[parent_all],
    )
    subparser_hpc = p_hpc.add_subparsers(dest='hpc_action')
    subparser_hpc.add_parser(
        'submit',
        help='Submit a single workflow to HPC.',
        parents=[parent_all, parent_submit, parent_run, parent_runner, parent_backend],
    )

    subparser_hpc.add_parser(
        'list',
        help='List all workflows submitted to HPC.',
        parents=[parent_all, parent_backend],
    )
    subparser_hpc.add_parser(
        'abort',
        help='Abort a workflow submitted to HPC.',
        parents=[parent_all, parent_backend, parent_hpc_abort],
    )

    p_gcp_monitor = subparser.add_parser(
        'gcp_monitor',
        help='Tabulate task\'s resource data collected on '
        'instances run on Google Cloud Compute. '
        'Use this for any workflows run with Caper>=1.2.0 on gcp backend. '
        'This is for gcp backend only.',
        parents=[
            parent_all,
            parent_server_client,
            parent_client,
            parent_search_wf,
            parent_gcp_monitor,
        ],
    )
    p_gcp_res_analysis = subparser.add_parser(
        'gcp_res_analysis',
        help='Linear resource analysis on monitoring data collected on '
        'instances run on Google Cloud Compute. This is for gcp backend only. '
        'Use this for any workflows run with Caper>=1.2.0 on gcp backend. '
        'Calculates coefficients/intercept for task\'s required resources '
        'based on input file size of a task. '
        'For each task it solves a linear regression problem of y=Xc + i1 + e where '
        'X is a matrix (m by n) of input file sizes and '
        'c is a coefficient vector (n by 1) and '
        'i is intercept and 1 is ones vector. '
        'y is a vector (m by 1) of resource taken and '
        'e is residual to be minimized. '
        'm is number of dataset and n is number of input file variables. '
        'Each resource metric will be solved separately. '
        'Refer to --target-resources for details about available resource metrics. '
        'Output will be a tuple of coefficient vector and intercept. ',
        parents=[
            parent_all,
            parent_server_client,
            parent_client,
            parent_search_wf,
            parent_gcp_res_analysis,
        ],
    )
    p_cleanup = subparser.add_parser(
        'cleanup',
        help='Cleanup outputs of workflows.',
        parents=[
            parent_all,
            parent_server_client,
            parent_client,
            parent_search_wf,
            parent_cleanup,
        ],
    )

    if conf_file is None:
        # partially parse args to get conf file from cmd line
        known_args, _ = parent_all.parse_known_args()
        conf_file = known_args.conf
    conf_file = os.path.expanduser(conf_file)

    subparsers = [
        p_init,
        p_run,
        p_server,
        p_submit,
        p_abort,
        p_unhold,
        p_list,
        p_metadata,
        p_troubleshoot,
        p_debug,
        p_gcp_monitor,
        p_gcp_res_analysis,
        p_cleanup,
    ]
    if os.path.exists(conf_file):
        conf_dict = update_parsers_defaults_with_conf(
            parsers=subparsers, conf_file=conf_file, conf_key_map=PARAM_KEY_NAME_CHANGE
        )
    else:
        conf_dict = None

    return parser, conf_dict


from .models import Pupil
from .serializers import PupilSerializer, RuSerializer, UzSerializer, EngSerializer
from django.http import Http404
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status, permissions
from django.utils.translation import ugettext_lazy as _

class PupilList(APIView):

    """
    List all pupils, or create a new pupil.
    """
    def get(self, request, format=None):
        try:
            pupils = Pupil.objects.all()
            serializer = PupilSerializer(pupils, many=True)
            return Response(serializer.data)
        except:
            return Response(
                {"detail": _("Pupil not found.")},
                status=status.HTTP_404_NOT_FOUND
            )

    def post(self, request, format=None):
        serializer = PupilSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class PupilDetail(APIView):
    """
    Retrieve, update or delete a pupil instance.
    """
    def get_object(self, pk):
        try:
            return Pupil.objects.get(pk=pk)
        except Pupil.DoesNotExist:
            raise Http404

    def get(self, request, pk, format=None):
        try:
            pupil = self.get_object(pk)
            serializer = PupilSerializer(pupil)
            return Response(serializer.data)
        except:
            return Response(
                {"detail": _("pupil not found.")},
                status=status.HTTP_404_NOT_FOUND
            )

    def put(self, request, pk, format=None):
        try:
            pupil = self.get_object(pk)
            serializer = PupilSerializer(pupil, data=request.data)
            if serializer.is_valid():
                serializer.save()
                return Response(serializer.data)
            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
        except:
            return Response(
                {"detail": _("Bad Request.")},
                status=status.HTTP_400_BAD_REQUEST
            )

    def delete(self, request, pk, format=None):
        try:
            pupil = self.get_object(pk)
            pupil.delete()
            return Response({"Data was deleted."},status=status.HTTP_204_NO_CONTENT)
        except:
            return Response(
                {"detail": _("Bad Request.")},
                status=status.HTTP_400_BAD_REQUEST
            )

class PupilLanguage(APIView):

    permission_classes = (permissions.AllowAny,)

    def get(self, request, language):
        try:
            pupils = Pupil.objects.all()
            if language=="uz":
                serializer = UzSerializer(pupils, many=True)
            elif language=="eng":
                serializer = EngSerializer(pupils, many=True)
            else:
                serializer = RuSerializer(pupils, many=True)

            return Response(serializer.data)
        except:
            return Response(
                {"detail": _("Pupil not found.")},
                status=status.HTTP_404_NOT_FOUND
            )


import requests
import re

def crawl_subdomains(domain):
  """Crawls all subdomains of a domain."""
  subdomains = []
  response = requests.get(f"https://{domain}")
  for link in response.text.split("/"):
    if re.match(r"^[a-z0-9-]+$", link):
      subdomains.append(link)
  return subdomains

if __name__ == "__main__":
  subdomains = crawl_subdomains("https://autolazy.vercel.app")
  print(subdomains)


from functools import reduce
from operator import mul
lista_numero = []
while True:
    quantidade = int(input("Informe um número (número zero para sair): "))
    if quantidade != 0:
        lista_numero.append(quantidade)
        #Soma da lista
        soma = sum(lista_numero)
        #multiplicação da lista
        mult = reduce(mul, lista_numero,1)
        #Maior valor de uma lista
        max_value = max(lista_numero)
        #Menor valor de uma lista
        min_value = min(lista_numero)
    elif quantidade == 0:
        break
for posicao, numero in enumerate(lista_numero, start=1):
    print(f"#{posicao} - {numero}")
print(f'Soma total:{soma} \n Multiplicação Total:{mult} \n Valor Maximo:{max_value} \n Valor Minimo:{min_value}')

Exercicio Enumerate and List/exercicio01.py


"""Users views."""

#DRF
from rest_framework import status
from rest_framework.response import Response
from rest_framework.views import APIView

from cride.users.serializers import (
    AccountVerficationSerializer,
    UserLoginSerializer,
    UserModelSerializer,
    UserSingupSerializer,)

class UserLoginAPIView(APIView):

    def post(self, request, *args, **kwargs):
        """Handle the login"""
        serializer = UserLoginSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        user, token = serializer.save()
        data ={
            'user':UserModelSerializer(user).data,
            "acces_token": token
        }

        return Response(data=data, status=status.HTTP_201_CREATED)

class UserSingupViewPIView(APIView):

    def post(self, request, *args, **kwargs):
        """Handle the login"""
        serializer = UserSingupSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        user = serializer.save()
        data =UserModelSerializer(user).data

        return Response(data=data, status=status.HTTP_201_CREATED)

class AccountVerficationAPIView(APIView):
    """Account verification API view"""

    def post(self, request, *args, **kwargs):
        """Handle HTTP POST request."""
        serializer = AccountVerficationSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        serializer.save()
        data ={"message": "Congragulation, now go share rides"}

        return Response(data=data, status=status.HTTP_200_OK)


# This Cog contains code from Arbitlog, which was made by Roadcrosser.
# https://github.com/Roadcrosser/archiver
import discord
import json
import os
import httplib2
import re
import datetime
import asyncio
import zipfile
import textwrap

from io import BytesIO

from discord.ext import commands
from discord.ext.commands import Cog
from helpers.checks import check_if_staff

class Arbitlog(Cog):
    def __init__(self, bot):
        self.bot = bot

    def textify_embed(self, embed, limit=40, padding=0, pad_first_line=True):
        text_proc = []
        title = ""
        if embed.title:
            title += embed.title
            if embed.url:
                title += " - "
        if embed.url:
            title += embed.url
        if not title and embed.author:
            title = embed.author.name
        if title:
            text_proc += [title, ""]
        if embed.description:
            text_proc += [embed.description, ""]
        if embed.thumbnail:
            text_proc += ["Thumbnail: " + embed.thumbnail.url, ""]
        for f in embed.fields:
            text_proc += [
                f.name
                + (
                    ":"
                    if not f.name.endswith(
                        ("!", ")", "}", "-", ":", ".", "?", "%", "$")
                    )
                    else ""
                ),
                *f.value.split("\n"),
                "",
            ]
        if embed.image:
            text_proc += ["Image: " + embed.image.url, ""]
        if embed.footer:
            text_proc += [embed.footer.text, ""]

        text_proc = [textwrap.wrap(t, width=limit) for t in text_proc]

        texts = []

        for tt in text_proc:
            if not tt:
                tt = [""]
            for t in tt:
                texts += [t + " " * (limit - len(t))]

        ret = " " * (padding * pad_first_line) + "╓─" + "─" * limit + "─╮"

        for t in texts[:-1]:
            ret += "\n" + " " * padding + "║ " + t + " │"

        ret += "\n" + " " * padding + "╙─" + "─" * limit + "─╯"

        return ret

    async def log_whole_channel(self, channel, zip_files=False):
        st = ""

        if zip_files:
            b = BytesIO()
            z = zipfile.ZipFile(b, "w", zipfile.ZIP_DEFLATED)
            zipped_count = 0

        async for m in channel.history(limit=None):
            blank_content = True
            ts = "{:%Y-%m-%d %H:%M} ".format(m.created_at)
            padding = len(ts) + len(m.author.name) + 2
            add = ts
            if m.type == discord.MessageType.default:
                add += "{0.author.name}: {0.clean_content}".format(m)
                if m.clean_content:
                    blank_content = False
            else:
                add += m.system_content

            for a in m.attachments:
                if not blank_content:
                    add += "\n"
                add += (
                    " " * (padding * (not blank_content)) + "Attachment: " + a.filename
                )
                if zip_files:
                    fn = "{}-{}-{}".format(m.id, a.id, a.filename)
                    async with self.bot.session.get(a.url) as r:
                        f = await r.read()

                    z.writestr(fn, f)
                    add += " (Saved as {})".format(fn)
                    zipped_count += 1

                blank_content = False

            for e in m.embeds:
                if e.type == "rich":
                    if not blank_content:
                        add += "\n"
                    add += self.textify_embed(
                        e, limit=40, padding=padding, pad_first_line=not blank_content
                    )
                    blank_content = False

            if m.reactions:
                if not blank_content:
                    add += "\n"
                add += " " * (padding * (not blank_content))
                add += " ".join(
                    ["[{} {}]".format(str(r.emoji), r.count) for r in m.reactions]
                )
                blank_content = False

            add += "\n"
            st = add + st

        ret = st
        if zip_files:
            if zipped_count:
                z.writestr("log.txt", st)
                b.seek(0)
                ret = (ret, b)
            else:
                ret = (ret, None)

        return ret

    @commands.guild_only()
    @commands.check(check_if_staff)
    @commands.command()
    async def arbitlog(self, ctx, *, args=""):
        channels = args.split()
        to_log = []
        for ch in channels:
            ch = ctx.guild.get_channel_or_thread(int(ch.strip("<#>")))
            if ch:
                if type(ch) == discord.CategoryChannel:
                    to_log += ch.channels
                else:
                    to_log += [ch]
        channels = to_log
        await ctx.channel.send(f"Logging: {' '.join([c.mention for c in channels])}")
        async with ctx.channel.typing():
            for ch in channels:
                out = await self.log_whole_channel(ch, zip_files=True)
                zipped_files = out[1]
                out = out[0]

                f = BytesIO()
                f.write(out.encode("utf-8"))
                f.seek(0)

                fn = "#{}-{}-{}".format(
                    ch.name, ch.id, int(ctx.message.created_at.timestamp())
                )

                reply = "Saved to disk as `{}.txt`.\n".format(fn)

                if zipped_files:
                    if not os.path.isdir("arbitlogs"):
                        os.mkdir("arbitlogs")

                    with open("arbitlogs/" + fn + ".zip", "wb+") as o:
                        o.write(zipped_files.getvalue())
                    reply += (
                        "Also saved files to disk as `{}.zip` ({:,.2f} MB).".format(
                            fn, int(len(zipped_files.getvalue()) / (1024 * 1024))
                        )
                    )

                with open("arbitlogs/" + fn + ".txt", "wb+") as o:
                    o.write(f.read())

                await ctx.channel.send(reply)

            return True

async def setup(bot):
    await bot.add_cog(Arbitlog(bot))


import logging
import os
import argparse
import sys
from string import Template
from c3.pythonscript import Pythonscript
from c3.utils import convert_notebook
from c3.create_operator import create_operator
from c3.templates import grid_wrapper_template, cos_grid_wrapper_template, gw_component_setup_code, dockerfile_template

def wrap_component(component_path,
                   component_description,
                   component_dependencies,
                   component_interface,
                   component_inputs,
                   component_process,
                   cos,
                   ):
    # get component name from path
    component_name = os.path.splitext(os.path.basename(component_path))[0]

    gw_template = cos_grid_wrapper_template if cos else grid_wrapper_template
    grid_wrapper_code = gw_template.substitute(
        component_name=component_name,
        component_description=component_description,
        component_dependencies=component_dependencies,
        component_inputs=component_inputs,
        component_interface=component_interface,
        component_process=component_process,
    )

    # Write edited code to file
    grid_wrapper_file = f'cgw_{component_name}.py' if cos else f'gw_{component_name}.py'
    grid_wrapper_file_path = os.path.join(os.path.dirname(component_path), grid_wrapper_file)
    # remove 'component_' from gw path
    grid_wrapper_file_path = grid_wrapper_file_path.replace('component_', '')
    with open(grid_wrapper_file_path, 'w') as f:
        f.write(grid_wrapper_code)

    logging.info(f'Saved wrapped component to {grid_wrapper_file_path}')

    return grid_wrapper_file_path

def get_component_elements(file_path):
    # get required elements from component code
    py = Pythonscript(file_path)
    # convert description into a string with a single line
    description = (py.get_description().replace('\n', ' ').replace('"', '\''))
    inputs = py.get_inputs()
    outputs = py.get_outputs()
    dependencies = py.get_requirements()

    # combine inputs and outputs
    interface_values = {}
    interface_values.update(inputs)
    interface_values.update(outputs)

    # combine dependencies list
    dependencies = '\n# '.join(dependencies)

    # generate interface code from inputs and outputs
    interface = ''
    type_to_func = {'String': '', 'Boolean': 'bool', 'Integer': 'int', 'Float': 'float'}
    for variable, d in interface_values.items():
        interface += f"# {d['description']}\n"
        interface += f"component_{variable} = {type_to_func[d['type']]}(os.getenv('{variable}', {d['default']}))\n"

    # generate kwargs for the subprocesses
    process_inputs = ', '.join([f'{i}=component_{i}' for i in inputs.keys()])
    # use log level from grid wrapper
    process_inputs = process_inputs.replace('component_log_level', 'log_level')

    return description, interface, process_inputs, dependencies

# Adding code
def edit_component_code(file_path):
    file_name = os.path.basename(file_path)
    if file_path.endswith('.ipynb'):
        logging.info('Convert notebook to python script')
        target_file = convert_notebook(file_path)
        file_path = target_file
        file_name = os.path.basename(file_path)
    else:
        # write edited code to different file
        target_file = os.path.join(os.path.dirname(file_path), 'component_' + file_name)

    target_file_name = os.path.basename(target_file)

    with open(file_path, 'r') as f:
        script = f.read()
    # Add code for logging and cli parameters to the beginning of the script
    script = gw_component_setup_code + script
    # replace old filename with new file name
    script = script.replace(file_name, target_file_name)
    with open(target_file, 'w') as f:
        f.write(script)

    if '__main__' not in script:
        logging.warning('No __main__ found in component code. Grid wrapper will import functions from component, '
                        'which can lead to unexpected behaviour without using __main__.')

    logging.info('Saved component python script in ' + target_file)

    return target_file

def apply_grid_wrapper(file_path, component_process, cos, *args, **kwargs):
    assert file_path.endswith('.py') or file_path.endswith('.ipynb'), \
        "Please provide a component file path to a python script or notebook."

    file_path = edit_component_code(file_path)

    description, interface, inputs, dependencies = get_component_elements(file_path)

    component_elements = dict(
        component_path=file_path,
        component_description=description,
        component_dependencies=dependencies,
        component_interface=interface,
        component_inputs=inputs,
        component_process=component_process
    )

    logging.debug('Wrap component with parameters:')
    for component, value in component_elements.items():
        logging.debug(component + ':\n' + str(value) + '\n')

    logging.info('Wrap component')
    grid_wrapper_file_path = wrap_component(cos=cos, **component_elements)
    return grid_wrapper_file_path, file_path

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('file_path', type=str,
                        help='Path to python script or notebook')
    parser.add_argument('additional_files', type=str, nargs='*',
                        help='List of paths to additional files to include in the container image')
    parser.add_argument('-p', '--component_process', type=str, required=True,
                        help='Name of the component sub process that is executed for each batch.')
    parser.add_argument('--cos', action=argparse.BooleanOptionalAction, default=False,
                        help='Creates a grid wrapper for processing COS files')
    parser.add_argument('-r', '--repository', type=str, default=None,
                        help='Container registry address, e.g. docker.io/<username>')
    parser.add_argument('-v', '--version', type=str, default=None,
                        help='Container image version. Auto-increases the version number if not provided (default 0.1)')
    parser.add_argument('-l', '--log_level', type=str, default='INFO')
    parser.add_argument('--dockerfile_template_path', type=str, default='',
                        help='Path to custom dockerfile template')
    parser.add_argument('--test_mode', action='store_true')
    args = parser.parse_args()

    # Init logging
    root = logging.getLogger()
    root.setLevel(args.log_level)
    handler = logging.StreamHandler(sys.stdout)
    formatter = logging.Formatter('%(levelname)s - %(message)s')
    handler.setFormatter(formatter)
    handler.setLevel(args.log_level)
    root.addHandler(handler)

    grid_wrapper_file_path, component_path = apply_grid_wrapper(**vars(args))

    if args.repository is not None:
        logging.info('Generate CLAIMED operator for grid wrapper')

        # Add component path and init file path to additional_files
        args.additional_files.append(component_path)

        # Update dockerfile template if specified
        if args.dockerfile_template_path != '':
            logging.info(f'Uses custom dockerfile template from {args.dockerfile_template_path}')
            with open(args.dockerfile_template_path, 'r') as f:
                _dockerfile_template = Template(f.read())
        else:
            _dockerfile_template = dockerfile_template

        create_operator(
            file_path=grid_wrapper_file_path,
            repository=args.repository,
            version=args.version,
            dockerfile_template=_dockerfile_template,
            additional_files=args.additional_files,
            log_level=args.log_level,
            test_mode=args.test_mode,
        )

        logging.info('Remove local component file')
        os.remove(component_path)

if __name__ == '__main__':
    main()


from openupgradelib import openupgrade

@openupgrade.migrate()
def migrate(env, version):
    if not openupgrade.column_exists(
        env.cr, "product_product", "commingled_policy"
    ) and openupgrade.column_exists(env.cr, "product_template", "commingled_policy"):

        openupgrade.logged_query(
            env.cr,
            """
            ALTER TABLE product_product
            ADD COLUMN commingled_policy varchar,
            ADD COLUMN commingled_prefer_homogenous boolean
            """,
        )

        openupgrade.logged_query(
            env.cr,
            """
            UPDATE
                product_product
            SET
                commingled_policy = product_template.commingled_policy,
                commingled_prefer_homogenous = product_template.commingled_prefer_homogenous
            FROM product_template
            WHERE
                product_template.id = product_product.product_tmpl_id
            """,
        )


from __future__ import print_function
import os.path
from time import sleep
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from selenium.webdriver.chrome.service import Service
from selenium import webdriver
from selenium.webdriver.common.by import By

# If modifying these scopes, delete the file token.json.
SCOPES = ['https://www.googleapis.com/auth/spreadsheets']

# The ID and range of a sample spreadsheet.

row = 2

SAMPLE_SPREADSHEET_ID = '1TrBjFW-bFRMBze4ShWvYtB9eFvEyvyW10J_Aoqfw4fQ' #Add the ID of the spreadsheet with you urls and then where you
SAMPLE_RANGE_NAME_SHEET1 = 'Sheet1!A2:A' # Where your URLS begin on the sheets

def main():
    service = get_google_service() # connects to your google sheets
    data = get_data(service) # getting the url from the sheets
    data = run_selenium(data)

    append_values(data, service)

def get_google_service():
    """Shows basic usage of the Sheets API.
    Prints values from a sample spreadsheet.
    """
    creds = None
    # The file token.json stores the user's access and refresh tokens, and is
    # created automatically when the authorization flow completes for the first
    # time.
    if os.path.exists('token.json'):
        creds = Credentials.from_authorized_user_file('token.json', SCOPES)
    # If there are no (valid) credentials available, let the user log in.
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(
                'credentials.json', SCOPES)
            creds = flow.run_local_server(port=0)
        # Save the credentials for the next run
        with open('token.json', 'w') as token:
            token.write(creds.to_json())

    return build('sheets', 'v4', credentials=creds)

def get_data(service):
    try:
         # Call the Sheets API
        sheet = service.spreadsheets()
        result = sheet.values().get(spreadsheetId=SAMPLE_SPREADSHEET_ID,
                                    range=SAMPLE_RANGE_NAME_SHEET1).execute()
        values = result.get('values', [])

        if not values:
            print('No data found.')
            return
        print(values)
        return values
    except HttpError as err:
        print(err)

def run_selenium(data):
    service = Service(executable_path="/path/to/chromedriver")
    driver = webdriver.Chrome(service=service)
    driver.get("https://wordcounter.net/website-word-count")
    input = driver.find_element(By.XPATH, '//*[@id="url"]')
    return_data = []
    for row in data:
        input = driver.find_element(By.XPATH, '//*[@id="url"]')
        input.send_keys(row[0])
        form = driver.find_element(By.XPATH, '/html/body/div[6]/div/form/div/div[2]/input')
        form.click()
        top_word = driver.find_element(By.XPATH, '/html/body/div[6]/div/div[2]/div[1]/div/table/tbody/tr[1]/td[1]')
        top_word_inner_html = top_word.get_attribute('innerHTML')
        top_word_count = driver.find_element(By.XPATH, '/html/body/div[6]/div/div[2]/div[1]/div/table/tbody/tr[1]/td[2]')
        top_word_count_inner_html = top_word_count.get_attribute("innerHTML")
        total_word_count = driver.find_element(By.XPATH, '//*[@id="word_count_block"]/span')
        total_word_count_inner_html = total_word_count.get_attribute('innerHTML')
        index_to_remove = total_word_count_inner_html.index('<')
        total_word_count_inner_html = total_word_count_inner_html[:index_to_remove]
        arr_to_append = [total_word_count_inner_html, top_word_inner_html, top_word_count_inner_html]
        return_data.append(arr_to_append)
    driver.close()
    return return_data

def append_values(values, service):
    range = f'Sheet1!B2'
    result = service.spreadsheets().values().update(
        spreadsheetId=SAMPLE_SPREADSHEET_ID,
        range=range,
        valueInputOption='RAW',
        body={'values': values}
    ).execute()
    # Return the result of the API call
    return result

if __name__ == '__main__':
    main()


#coding: utf-8
import time
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
'''
यहां हम फेसबुक सम्बन्धी अस्त्रों का उल्लेख करेंगे
'''
#from .astra import *
import astra
import karma

#Fire and Forget Astras, to be run with {'msg':'run','func':function_object,'args':(),'kwargs':{}}
def fb_login(browser,fbconfig,logger=astra.baselogger):
		fbusr=fbconfig.get("Facebook","fbusername")
		fbpwd=fbconfig.get("Facebook","fbpassword")
		# or you can use Chrome(executable_path="/usr/bin/chromedriver")
		logger.info("Trying to log into FB in browser...")
		try:
			browser.get("http://www.facebook.com")
			assert "Facebook" in browser.title
			elem = browser.find_element_by_id("email")
			elem.send_keys(fbusr)
			elem = browser.find_element_by_id("pass")
			elem.send_keys(fbpwd)
			elem.send_keys(Keys.RETURN)
			time.sleep(10)
			browser.get("http://facebook.com/profile.php")
			time.sleep(10)
			logger.info("Successfully logged into FB")
		except Exception as exception:
			logger.error("Could not log into FB.."+ repr(exception))

def fb_search(fbbrowser,searchstring,logger=astra.baselogger):
		logger.info("Searching FB for " + searchstring)
		searchbar=fbbrowser.find_element_by_name("q")
		searchbar.clear()
		searchbar.send_keys(searchstring)
		searchbar.send_keys(Keys.ENTER)
		time.sleep(10)

def fb_get_postlinks_from_timeline(fbbrowser,url="https://facebook.com",count=10,logger=astra.baselogger):
    fbbrowser.get(url)
    karma.wait()
    postlinks=[]

    while len(postlinks)<count:
        posts=[]
        postcontents=[]
        posts=posts+fbbrowser.find_elements_by_class_name("userContentWrapper")
        for post in posts:
            postcontents.append(BeautifulSoup(post.get_property("innerHTML")))
        for postcontent in postcontents:

            pc=list(postcontent.find_all("a",{"class":"_5pcq"}))
            if len(pc)>0:
                pc=pc[0]
            else:
                continue
            url="https://facebook.com"+pc.get("href")
            postlinks.append(url)
        logger.info("Got %s posts " %(len(postlinks)))
        postlinks=list(set(postlinks))
        if "https://facebook.com#" in postlinks:
            postlinks.pop(postlinks.index("https://facebook.com#"))
        karma.scroll_page(fbbrowser)
        karma.wait()
    return postlinks[:count]


from flask import Flask
from flask import redirect, url_for
from flask import render_template
from flask import request,session
import requests
from datetime import timedelta
from bs4 import BeautifulSoup

from bs4 import BeautifulSoup
import pandas as pd
import numpy as np

from time import sleep
from random import randint
from flask_table import Table, Col
from PIL import Image
import seaborn as sns
import matplotlib.pyplot as plt
from wordcloud import WordCloud, STOPWORDS
from werkzeug.utils import secure_filename
from werkzeug.datastructures import  FileStorage
import re
from PIL import Image
import gensim
from gensim import corpora

from nltk import FreqDist
import plotly
import plotly.graph_objs as go
from bokeh.io import output_file, show
from bokeh.plotting import figure
import json
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
import nltk
nltk.download('wordnet')
from nltk.sentiment.vader import SentimentIntensityAnalyzer
nltk.download('vader_lexicon')

cut_name = []
dates = []
rating = []
cust_review = []

source = requests.get('https://webscraper.netlify.com/').text
soup = BeautifulSoup(source, 'lxml')

app= Flask(__name__)
app.secret_key = 'hello'
app.permanent_session_lifetime = timedelta(seconds=2)

def get_profile_detail(user_handle):

    url = "https://www.flipkart.com/search?q={}".format(user_handle)

    page = requests.get(url)
    soup = BeautifulSoup(page.content,'html.parser')
    print(soup.prettify())
    final_link = []
    for a in soup.find_all('a',class_='_31qSD5', href=True):
       final_link.append("https://www.flipkart.com"+a['href'])

    price = soup.find_all("div",class_= "_1vC4OE _2rQ-NK")
    name = soup.find_all("div",class_= "col col-7-12")

    review_title = []
    price_pro = []
    for i in range(0,len(name)):
        review_title.append(name[i].get_text())

    review_title[:]= [name.lstrip("\n")for name in review_title]
    review_title[:]= [name.rstrip("\n")for name in review_title]

    for i in range(0,len(price)):
        price_pro.append(price[i].get_text())

    price_pro[:]= [price.lstrip("\n")for price in price_pro]
    price_pro[:]= [price.rstrip("\n")for price in price_pro]

    return review_title,price_pro,final_link

@app.route('/<user_handle>/')
def home(user_handle):
    review_title,price_pro,final_link = get_profile_detail(user_handle)
    amazon =pd.DataFrame({'price': price_pro, 'name': review_title, 'link': final_link})
    html = amazon.to_html()
    return html

def get_review_detail(url_handle,pd_handle):
    header={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'}
    cust_review = []
    html=home(pd_handle)
    dfs = pd.read_html(html)
    df = dfs[0]
    def num(url_handle):
        return f'{url_handle}'
    orange=int(num(url_handle))
    trng=df.iloc[orange,3]
    res = trng.split('srno', 1)[0]

    reep=str(res).replace('/p/', '/product-reviews/')
    reep
    urr=reep+'page='

    rev_num = requests.get(trng, headers=header)

    soup = BeautifulSoup(rev_num.text, 'html.parser')
    list_header=[]
    pa = soup.find_all('span', class_='_38sUEc')
    ss=str(pa).split('&amp', 1)[1]
    ss
    su=str(ss).split('Reviews', 1)[0]
    su
    l=su[15:]
    int_b = int(l.replace(',',''))
    print ("The integer value",int_b)
    srinivas=(int_b//10)+1
    srinivas
    pages = np.arange(1, srinivas)
    for items in header:
        try:
            list_header.append(items.get_text())
        except:
            continue

    for page in pages:

      page = requests.get(urr + str(page) + "&ref_=adv_nxt", headers=header)

      soup = BeautifulSoup(page.text, 'html.parser')

      review_div = soup.find_all('div', class_='col _390CkK _1gY8H-')

      sleep(randint(1,8))
      for container in review_div:
            # runtime
            review = container.find('div', class_='qwjRop').text
            mod_review= review.rstrip("READ MORE")
            cust_review.append(mod_review)
            dataFrame = pd.DataFrame(data = cust_review)
            dataFrame.to_csv('Geeks.csv')

    return cust_review

@app.route('/homme/<url_handle>/<pd_handle>/')
def homme(url_handle,pd_handle):
    cust_review=get_review_detail(url_handle,pd_handle)
    flipkart = pd.DataFrame({'Review': cust_review})
    Review_tbe=flipkart.to_html()
    return Review_tbe

@app.route('/download/<url_handle>/<pd_handle>/')
def download(url_handle,pd_handle):
    cust_review=get_review_detail(url_handle,pd_handle)
    flipkart = pd.DataFrame({'Review': cust_review})
    df= flipkart.to_html()
    return df

@app.route('/king')
def king():
    return redirect(url_for('home', user_handle='mobile'))

@app.route('/',methods=['POST','GET'])
def login():
    if request.method == 'POST':
        session.permanent = True
        user = request.form['nm']
        session['user']=user
        return redirect(url_for('user'))
    else:
        if 'user' in session:
            return redirect(url_for('user'))
        return render_template('login.html')

@app.route('/user',methods=['POST','GET'])
def user(user_handle='sumit'):
    if request.method == 'POST':
        session.permanent = True
        usr = request.form['url']
        session['user']=usr
        return redirect(url_for('home', user_handle= usr))

    if 'user' in session:
        user = session['user']
        return render_template("user.html", user=user)

    else:
        return redirect(url_for('login'))

@app.route('/logout.html/',methods=['POST','GET'])
def logout():
    if request.method == 'POST':
        session.permanent = True
        ussr = request.form['urll']
        usssr = request.form['urlll']
        session['user']=ussr
        return render_template('redirect.html',number=ussr,pd=usssr)
    else:
        return render_template('logout.html')

@app.route('/logout1.html/')
def upload():
    return render_template("logout 1.html")

@app.route('/success', methods = ['POST'])
def success():
    if request.method == 'POST':
        f = request.files['file']
        f.save(f.filename)
        return render_template("success.html", name = f.filename)

from textblob import TextBlob

def fetch_sentiment_using_textblob(text):
    sentiment = []
    for i in text:
        analysis = TextBlob(i)
        # set sentiment
        if analysis.sentiment.polarity >= 0:
            sentiment.append('positive')
        else:
            sentiment.append('negative')
    return sentiment

@app.route('/polarity')
def polarity():
    data1=pd.read_html('datalog.html')
    df=data1[0]
    final_data=pd.DataFrame(df)
    corpus = []
    for i in range(0,len(final_data)):
        review = re.sub('[^a-zA-Z]', ' ', final_data['Review'][i])
        review = review.lower()
        review = review.split()
        ps = WordNetLemmatizer()
        review = [ps.lemmatize(word) for word in review if not word in set(stopwords.words('english'))]
        review = ' '.join(review)
        corpus.append(review)
    review_list = final_data['Review'].tolist()
    final_data['sentiment']= fetch_sentiment_using_textblob(review_list)
    final_data['sentiment'].value_counts()
    pos=final_data.groupby('sentiment')
    pos_values=pos.get_group('positive')
    pos_values
    pos_values['Review']
    sid = SentimentIntensityAnalyzer()
    sentiment_summary = dict()
    # for readme in readmes:
    #     sentences = nltk.tokenize.sent_tokenize(readme)
    #     for sentence in sentences:
    #         sentiment_score = sid.polarity_scores(sentence)messages = pos_list
    messages = pos_values['Review']
    summary = {"positive":0,"neutral":0,"negative":0}
    for x in messages:
        ss = sid.polarity_scores(x)
        if ss["compound"] == 0.0:
            summary["neutral"] +=1
        elif ss["compound"] > 0.0:
            summary["positive"] +=1
        else:
            summary["negative"] +=1
    sen=[]

    sid = SentimentIntensityAnalyzer()
    sentiment_summary = dict()
    # for readme in readmes:
    #     sentences = nltk.tokenize.sent_tokenize(readme)
    #     for sentence in sentences:
    #         sentiment_score = sid.polarity_scores(sentence)messages = pos_list
    messages = pos_values['Review']
    summary = {"positive":0,"neutral":0,"negative":0}
    for x in messages:
        ss = sid.polarity_scores(x)
        if ss["compound"] == 0.0:
            sen.append('Neutral')
        elif ss["compound"] > 0.0:
            sen.append('Positive')
        else:
            sen.append('negative')
    pd.DataFrame(sen).count()

    pos_values.index

    positive=pd.concat([pos_values.drop('sentiment',axis=1),pd.DataFrame(sen,index=pos_values.index)],axis=1)

    pos.get_group('negative')

    sds=positive.rename(columns={0: "sentiment"})

    sumit=pd.concat([sds,pos.get_group('negative')])

    sns.countplot('sentiment',data=sumit)
    lp=sumit['sentiment'].value_counts()
    kp=pd.DataFrame(lp)
    kp.reset_index(inplace=True)
    kp['sentiment']
    output_file('bars.html')
    colors = ["#c9d9d3", "#718dbf", "#e84d60"]
    p=figure(x_range= list(kp['index']), plot_height=250, title="polarity",
           toolbar_location=None, tools="")
    p.vbar(
         x= list(kp['index']),
        top= list(kp['sentiment']),color=colors,
        width=0.9
    )

    p.xgrid.grid_line_color = None
    p.y_range.start = 0
    show(p)

    return sumit.to_html()

def par():
    data1=pd.read_html('datalog.html')
    df=data1[0]
    final_data=pd.DataFrame(df)
    corpus = []
    for i in range(0,len(final_data)):
        review = re.sub('[^a-zA-Z]', ' ', final_data['Review'][i])
        review = review.lower()
        review = review.split()
        ps = WordNetLemmatizer()
        review = [ps.lemmatize(word) for word in review if not word in set(stopwords.words('english'))]
        review = ' '.join(review)
        corpus.append(review)
    review_list = final_data['Review'].tolist()
    final_data['sentiment']= fetch_sentiment_using_textblob(review_list)
    final_data['sentiment'].value_counts()
    pos=final_data.groupby('sentiment')
    pos_values=pos.get_group('positive')
    pos_values
    pos_values['Review']
    sid = SentimentIntensityAnalyzer()
    sentiment_summary = dict()
    # for readme in readmes:
    #     sentences = nltk.tokenize.sent_tokenize(readme)
    #     for sentence in sentences:
    #         sentiment_score = sid.polarity_scores(sentence)messages = pos_list
    messages = pos_values['Review']
    summary = {"positive":0,"neutral":0,"negative":0}
    for x in messages:
        ss = sid.polarity_scores(x)
        if ss["compound"] == 0.0:
            summary["neutral"] +=1
        elif ss["compound"] > 0.0:
            summary["positive"] +=1
        else:
            summary["negative"] +=1
    sen=[]

    sid = SentimentIntensityAnalyzer()
    sentiment_summary = dict()
    # for readme in readmes:
    #     sentences = nltk.tokenize.sent_tokenize(readme)
    #     for sentence in sentences:
    #         sentiment_score = sid.polarity_scores(sentence)messages = pos_list
    messages = pos_values['Review']
    summary = {"positive":0,"neutral":0,"negative":0}
    for x in messages:
        ss = sid.polarity_scores(x)
        if ss["compound"] == 0.0:
            sen.append('Neutral')
        elif ss["compound"] > 0.0:
            sen.append('Positive')
        else:
            sen.append('negative')
    pd.DataFrame(sen).count()

    pos_values.index

    positive=pd.concat([pos_values.drop('sentiment',axis=1),pd.DataFrame(sen,index=pos_values.index)],axis=1)

    pos.get_group('negative')

    sds=positive.rename(columns={0: "sentiment"})

    sumit=pd.concat([sds,pos.get_group('negative')])
    return sumit
@app.route('/word')
def word():
    data=par()
    wc = WordCloud(background_color="black",
                   width = 800, height = 800)

    wc.generate(str(data['Review']))
    wc.to_file("wordcloud.png")

    filename = Image.open("wordcloud.png")
    filename.show()
    return '
see the wordcloud in the PNG FILE BELOW ðŸ‘‡
'

@app.route('/positive')
def positive():
    data=par()
    sent=data.groupby('sentiment')
    positive_values=sent.get_group('Positive')
    wc = WordCloud(background_color="black",
                   width = 800, height = 800)

    wc.generate(str(positive_values['Review']))
    wc.to_file("wordcloud.png")

    filename = Image.open("wordcloud.png")
    filename.show()
    return '
see the wordcloud in the PNG FILE BELOW ðŸ‘‡
'

@app.route('/negative')
def negative():
    data=par()
    sent=data.groupby('sentiment')
    positive_values=sent.get_group('negative')
    wc = WordCloud(background_color="black",
                   width = 800, height = 800)

    wc.generate(str(positive_values['Review']))
    wc.to_file("wordcloud.png")

    filename = Image.open("wordcloud.png")
    filename.show()
    return '
see the wordcloud in the PNG FILE BELOW ðŸ‘‡
'

if __name__ == '__main__':
    app.run(debug=True)

    app.run(host='0.0.0.0', port=5000)


#!/usr/bin/env python
# -*- coding:utf-8 -*-

from __future__ import print_function
import os, sys

if sys.version_info[:2] < (2, 7) or (3, 0) <= sys.version_info[:2] < (3, 4):
    raise RuntimeError("Python version 2.7 or >= 3.4 required.")

try:
    from setuptools import setup, find_packages
except ImportError:
    print('# Error: you don\'t have "setuptools" installed!')
    print('# Before install: $ pip install setuptools')
    print('# Or visit https://pypi.org/project/setuptools/')

def rd(filename):
    f = open(filename)
    r = f.read()
    f.close()
    return r

if sys.argv[-1] == "publish":
    os.system("python setup.py sdist upload")
    sys.exit()

setup(name='beautytext',
    version='0.1',
    description=('A tool to fits a text'),
    url='https://github.com/dbednarski/desafios/tree/master/strings',
    author='Daniel Bednarski',
    author_email='daniel.bednarski.ramos@gmail.com',
    license='GNU GPLv3.0',
    packages=['beautytext'],
    scripts=[os.path.join('scripts', f) for f in os.listdir('scripts')],
    long_description=rd('README.md'),
    long_description_content_type="text/markdown",
    install_requires=['numpy'],
    classifiers=[
        "Development Status :: 4 - Beta",
        "Intended Audience :: Developers",
        "Intended Audience :: Others",
        "License :: OSI Approved :: GNU General Public License v3 or later" + \
        " (GPLv3+)",
        "Operating System :: OS Independent",
        "Programming Language :: Python",
        "Programming Language :: Python :: 2",
        "Programming Language :: Python :: 3",
    ],
    keywords=['text', 'string', 'justify'],
    zip_safe=True
)


# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import csv
import logging

from cliff import lister

from goal_tools import governance
from goal_tools import sponsors

LOG = logging.getLogger(__name__)

class ContributionsReportBase(lister.Lister):
    "Base class for commands that report about contributions."

    def get_parser(self, prog_name):
        parser = super().get_parser(prog_name)
        parser.add_argument(
            '--role',
            default=[],
            action='append',
            help='filter to only include specific roles (may be repeated)',
        )
        sponsor_group = parser.add_mutually_exclusive_group()
        sponsor_group.add_argument(
            '--highlight-sponsors',
            default=False,
            action='store_true',
            help=('highlight sponsor organizations and '
                  'combine stats for others'),
        )
        sponsor_group.add_argument(
            '--only-sponsors',
            default=False,
            action='store_true',
            help=('show only stats for sponsor organizations'),
        )
        parser.add_argument(
            '--sponsor-level',
            default='all',
            choices=('all', 'platinum', 'gold'),
            help='limit sponsor highlights to a subset of sponsors',
        )
        parser.add_argument(
            '--ignore-team',
            default=[],
            action='append',
            help='do not show stats for the named team (may be repeated)',
        )
        parser.add_argument(
            '--only-team',
            default=[],
            action='append',
            help='only show stats for the named team (may be repeated)',
        )
        parser.add_argument(
            '--ignore-tag',
            dest='ignore_tag',
            default=[],
            action='append',
            help=('do not show stats for projects with the tag '
                  '(may be repeated)'),
        )
        parser.add_argument(
            '--only-tag',
            dest='only_tag',
            default=[],
            action='append',
            help=('only show stats for projects with the tag '
                  '(may be repeated)'),
        )
        parser.add_argument(
            '--ignore-single-vendor',
            action='append_const',
            const='team:single-vendor',
            dest='ignore_tag',
            help='do not show stats for single-vendor projects',
        )
        parser.add_argument(
            '--governance-project-list',
            default=governance.PROJECTS_LIST,
            help='location of governance project list',
        )
        parser.add_argument(
            'contribution_list',
            nargs='+',
            help='name(s) of files containing contribution details',
        )
        return parser

    def get_contributions(self, parsed_args):

        def rows():
            for filename in parsed_args.contribution_list:
                LOG.debug('reading %s', filename)
                with open(filename, 'r', encoding='utf-8') as f:
                    reader = csv.DictReader(f)
                    yield from reader

        data = rows()

        roles = parsed_args.role
        if roles:
            data = (d for d in data if d['Role'] in roles)

        ignore_teams = set(t.lower() for t in parsed_args.ignore_team)
        if ignore_teams:
            data = (d for d in data if d['Team'].lower() not in ignore_teams)

        only_teams = set(t.lower() for t in parsed_args.only_team)
        if only_teams:
            data = (d for d in data if d['Team'].lower() in only_teams)

        if parsed_args.only_sponsors:
            sponsor_map = sponsors.Sponsors(parsed_args.sponsor_level)

            data = (
                d
                for d in data
                if d['Organization'] in sponsor_map
            )

        ignore_tags = set(parsed_args.ignore_tag)
        if ignore_tags:
            team_data = governance.Governance(
                url=parsed_args.governance_project_list)

            data = (
                d
                for d in data
                if not team_data.get_repo_tags(d['Project']).intersection(
                    ignore_tags)
            )

        only_tags = set(parsed_args.only_tag)
        if only_tags:
            team_data = governance.Governance(
                url=parsed_args.governance_project_list)

            data = (
                d
                for d in data
                if only_tags.issubset(team_data.get_repo_tags(d['Project']))
            )

        if parsed_args.highlight_sponsors:
            sponsor_map = sponsors.Sponsors(parsed_args.sponsor_level)

            def filter_sponsors(row):
                row['Organization'] = sponsor_map[row['Organization']]
                return row

            data = (filter_sponsors(d) for d in data)

        return data