import abc
import time

import numpy as np
import scipy
import scipy.sparse as sp
import torch
from torch import nn
from torch.nn import functional as F

from cell import utils
from cell.graph_statistics import compute_graph_statistics

DEVICE = "cpu"
DTYPE = torch.float32

class Callback(abc.ABC):
    """Abstract Class to customize train loops.
    Attributes:
        invoke_every(int): The number of calls required to invoke the Callback once.
    """

    def __init__(self, invoke_every):
        self._training_stopped = False
        self.invoke_every = invoke_every

    def __call__(self, loss, model):
        if model.step % self.invoke_every == 0:
            self.invoke(loss, model)

    def stop_training(self):
        self._training_stopped = True

    @abc.abstractmethod
    def invoke(self):
        pass

class EdgeOverlapCriterion(Callback):
    """Tracks the edge overlap and stops the training if the limit is met.
    Attributes:
        invoke_every(int): The number of calls required to invoke the Callback once.
        edge_overlap_limit(float): Stops the training if the models edge overlap reaches this limit.
    """

    def __init__(self, invoke_every, edge_overlap_limit=1.0):
        super().__init__(invoke_every)
        self.edge_overlap_limit = edge_overlap_limit

    def invoke(self, loss, model):
        """Computes the edge overlap and prints the actual step, loss, edge overlap and total time.
        It also stops the training if the computed edge overlap reaches self.edge_overlap_limit.
        Args:
            loss(float): The latest loss value.
            model(Cell): The instance of the model being trained.
        """
        start = time.time()
        model.update_scores_matrix()
        sampled_graph = model.sample_graph()
        overlap = utils.edge_overlap(model.A_sparse, sampled_graph) / model.num_edges
        overlap_time = time.time() - start
        model.total_time += overlap_time

        step_str = f"{model.step:{model.step_str_len}d}"
        print(
            f"Step: {step_str}/{model.steps}",
            f"Loss: {loss:.5f}",
            f"Edge-Overlap: {overlap:.3f}",
            f"Total-Time: {int(model.total_time)}",
        )
        if overlap >= self.edge_overlap_limit:
            self.stop_training()

class LinkPredictionCriterion(Callback):
    """Evaluates the link prediction performance and stops the training if there is no improvement for several steps.

    It ensures that the model's score_matrix is set to the score_matrix yielding the best results so far.

    Attributes:
        invoke_every(int): The number of calls required to invoke the Callback once.
        edge_overlap_limit(float): Stops the training if the models edge overlap reaches this limit.
        val_ones(np.ndarray): Validation ones for link prediction.
        val_zeros(np.ndarray): Validation zeros for link prediction.
        max_patience(int): Maximal number of invokes without improvement of link prediction performance
            until the training is stopped.
    """

    def __init__(self, invoke_every, val_ones, val_zeros, max_patience):
        super().__init__(invoke_every)
        self.val_ones = val_ones
        self.val_zeros = val_zeros
        self.max_patience = max_patience

        self._patience = 0
        self._best_scores_matrix = None
        self._best_link_pred_score = 0.0

    def invoke(self, loss, model):
        """Evaluates the link prediction performance and prints the actual step, loss, edge overlap and total time.
        It also stops the training if there is no improvement for self.max_patience invokes.
        Args:
            loss(float): The latest loss value.
            model(Cell): The instance of the model being trained.
        """
        start = time.time()
        model.update_scores_matrix()
        roc_auc, avg_prec = utils.link_prediction_performance(
            model._scores_matrix, self.val_ones, self.val_zeros
        )

        link_pred_time = time.time() - start
        model.total_time += link_pred_time

        step_str = f"{model.step:{model.step_str_len}d}"
        print(
            f"Step: {step_str}/{model.steps}",
            f"Loss: {loss:.5f}",
            f"ROC-AUC Score: {roc_auc:.3f}",
            f"Average Precision: {avg_prec:.3f}",
            f"Total-Time: {int(model.total_time)}",
        )
        link_pred_score = roc_auc + avg_prec

        if link_pred_score > self._best_link_pred_score:
            self._best_link_pred_score = link_pred_score
            self._best_scores_matrix = model._scores_matrix.copy()
            self._patience = 0

        elif self._patience >= self.max_patience:
            self.stop_training()
        else:
            self._patience += 1
        model._scores_matrix = self._best_scores_matrix

class StatisticCollector(Callback):
    """Tracks graph statistics.
    Attributes:
        invoke_every(int): The number of calls required to invoke the Callback once.
        A(sp.csr_matrix): input graph.
        test_ones(np.ndarray): Test ones for link prediction.
        test_zeros(np.ndarray): Test zeros for link prediction.
        graphic_mode(str): x-axis for the graph plotting.
        n_samples(int): number of generated graphs for averaging
    """
    def __init__(self, invoke_every, A, test_ones, test_zeros, graphic_mode='overlap', n_samples=5):
        super().__init__(invoke_every)
        self.invoke_every = invoke_every
        self.test_ones = test_ones
        self.test_zeros = test_zeros
        self.n_samples = n_samples
        self.A = A
        self.graphic_mode = graphic_mode
        self.training_stat = []

    def invoke(self, loss, model):
        """Evaluates the link prediction performance and collect graph statistics.
        Args:
            loss(float): The latest loss value - not needed.
            model(Cell): The instance of the model being trained.
        """
        model.update_scores_matrix()
        roc_auc, avg_prec = utils.link_prediction_performance(scores_matrix=model._scores_matrix,
                                val_ones=self.test_ones,
                                val_zeros=self.test_zeros)

        generated_graphs = [model.sample_graph() for _ in range(self.n_samples)]
        current_overlaps = [utils.edge_overlap(self.A, gg) / model.num_edges for gg in generated_graphs]
        current_overlap = np.mean(current_overlaps)
        stats = [compute_graph_statistics(gg) for gg in generated_graphs]
        for i, x in enumerate(stats):
            x['ROC-AUC'] = roc_auc
            x['AVG-PREC'] = avg_prec
            x['EO'] = current_overlaps[i]
        if self.graphic_mode == 'overlap':
            self.training_stat.append({'overlap': current_overlap, 'stats': stats})
        else:
            self.training_stat.append({'iteration': model.step, 'stats': stats})

class G_cell(nn.Module):
    """Model to hold originally proposed low rank decomposition.
    Attributes:
        N(int) - number of vertices.
        H(int) - rank.
        gamma(float) - cofficient for initialization.
    """
    def __init__(self, N, H, gamma):
        super().__init__()
        self.W_down = nn.Parameter((
            (gamma * torch.randn(N, H, device=DEVICE, dtype=DTYPE))
                .clone()
                .detach()
                .requires_grad_()
        ))
        self.W_up = nn.Parameter((
            (gamma * torch.randn(H, N, device=DEVICE, dtype=DTYPE))
                .clone()
                .detach()
                .requires_grad_()
        ))

    def add_loss(self):
        return 0

    def forward(self):
        W = torch.mm(self.W_down, self.W_up)
        W -= W.max(dim=-1, keepdims=True)[0]
        return W

class G_svd(nn.Module):
    """Model to hold low rank SVD decomposition.
    Attributes:
        N(int) - number of vertices.
        H(int) - rank.
        gamma(float) - cofficient for initialization.
    """
    def __init__(self, N, H, gamma):
        super().__init__()
        self.N = N
        self.H = H

        self.U = nn.Parameter((
            (gamma * torch.randn(N, H, device=DEVICE, dtype=DTYPE))
                .clone()
                .detach()
                .requires_grad_()
        ))

        self.V = nn.Parameter((
            (gamma * torch.randn(H, N, device=DEVICE, dtype=DTYPE))
                .clone()
                .detach()
                .requires_grad_()
        ))

        self.sigma = nn.Parameter((
            (gamma * torch.randn(H, device=DEVICE, dtype=DTYPE))
                .clone()
                .detach()
                .requires_grad_()
        ))

    def add_loss(self):
        alpha = 1e-4
        return alpha / self.H ** 2 * (torch.norm(self.U.T @ self.U - torch.eye(self.H, self.H), p='fro') ** 2 + \
                                      torch.norm(self.V @ self.V.T - torch.eye(self.H, self.H), p='fro') ** 2)

    def forward(self):
        W = self.U * (self.sigma[None, :]) ** 2
        W = torch.mm(W, self.V)
        W -= W.max(dim=-1, keepdims=True)[0]
        return W

class G_fc(nn.Module):
    """Model to hold nonlinear low rank decomposition.
    Attributes:
        N(int) - number of vertices.
        H(int) - rank.
        gamma(float) - cofficient for initialization.
    """
    def __init__(self, N, H, gamma):
        super().__init__()
        self.N = N
        self.W_down1 = nn.Linear(N, 20 * H, bias=False)
        self.W_down2 = nn.Linear(20 * H, 4 * H, bias=False)
        self.W_down3 = nn.Linear(4 * H, H, bias=False)
        self.W_up1 = nn.Linear(H, N, bias=False)

        self._init_weights()

    def _init_weights(self):
        nn.init.xavier_uniform(self.W_down1.weight)
        nn.init.xavier_uniform(self.W_down2.weight)
        nn.init.xavier_uniform(self.W_down3.weight)
        nn.init.xavier_uniform(self.W_up1.weight)

    def add_loss(self):
        return 0

    def forward(self):
        vs = torch.eye(self.N, self.N)
        W_down = self.W_down3(F.relu(
            self.W_down2(
                F.relu(
                    self.W_down1(vs)
                ))))
        W = self.W_up1(W_down)

        W -= W.max(dim=-1, keepdims=True)[0]

        return W

class Cell(object):
    """Implements the Cross Entropy Low-rank Logits graph generative model.

    Attributes:
        A(torch.tensor): The adjacency matrix representing the target graph.
        A_sparse(sp.csr.csr_matrix): The sparse representation of A.
        H(int): The maximum rank of W.
        loss_fn(function): The loss function minimized during the training process.
        callbacks(list): A list containing instances of classes derived from Callback.
        step(int): Keeps track of the actual training step.
        num_edges(int): The total number of edges in A.
    """

    def __init__(self, A, H, loss_fn=None, g_type='cell', callbacks=[]):
        self.num_edges = A.sum() / 2
        self.A_sparse = A
        self.A = torch.tensor(A.toarray())
        self.step = 1
        self.callbacks = callbacks
        self._optimizer = None

        N = A.shape[0]
        gamma = np.sqrt(2 / (N + H))

        if g_type == 'cell':
            self.g = G_cell(N, H, gamma)
        elif g_type == 'fc':
            self.g = G_fc(N, H, gamma)
        elif g_type == 'svd':
            self.g = G_svd(N, H, gamma)
        else:
            raise NameError

        if loss_fn == 'local_cell':
            self.loss_fn = self.local_loss
            self.mask = self._compute_mask_for_local_loss(self.A_sparse)
        elif loss_fn is not None:
            self.loss_fn = loss_fn
        else:
            self.loss_fn = self.built_in_loss_fn

        self.total_time = 0
        self.scores_matrix_needs_update = True

    def __call__(self):
        """Computes the learned random walk transition matrix.

        Returns:
            (np.array): The learned random walk transition matrix of shape(N,N)
        """
        return torch.nn.functional.softmax(self.get_W(), dim=-1).detach().numpy()

    def get_W(self):
        """Computes the logits of the learned random walk transition matrix.

        Returns:
            W(torch.tensor): Logits of the learned random walk transition matrix of shape(N,N)
        """
        W = self.g()

        return W

    def built_in_loss_fn(self, W, A, num_edges):
        """Computes the weighted cross-entropy loss in logits with weight matrix.
        Args:
            W(torch.tensor): Logits of learnable (low rank) transition matrix.
            A(torch.tensor): The adjaceny matrix representing the target graph.
            num_edges(int): The total number of edges of the target graph.

        Returns:
            (torch.tensor): Loss at logits.
        """
        d = torch.log(torch.exp(W).sum(dim=-1, keepdims=True))
        loss = 0.5 * torch.sum(A * (d - W)) / num_edges
        return loss

    def _compute_mask_for_local_loss(self, A_sparse):
        """
        Computes mask (indicator) for local_loss
        Args:
            A_sparse (scipy.sparse.csr_matrix) : adjacency matrix in compressed sparse row format

        Returns:
            mask (torch.tensor): resulting mask tensor
        """
        if not isinstance(A_sparse, scipy.sparse.csr.csr_matrix):
            A_sparse = scipy.sparse.csr_matrix(A_sparse)
        dists = sp.csgraph.shortest_path(csgraph=A_sparse, directed=False)
        return torch.tensor([dists[i, :] <= i for i in range(dists.shape[0])], dtype=int)
        #return torch.tensor(dists <= 4, dtype=int)

    def local_loss(self, W, A, num_edges):
        """Computes the LOCAL weighted cross-entropy loss in logits with weight matrix.
        Args:
            W(torch.tensor): Logits of learnable (low rank) transition matrix.
            A(torch.tensor): The adjaceny matrix representing the target graph.
            num_edges(int): The total number of edges of the target graph.

        Returns:
            (torch.tensor): Loss at logits.
        """
        d = torch.log(torch.exp(W).sum(dim=-1, keepdims=True))
        return 0.5 * (torch.sum(A * (d - W)) / num_edges + torch.sum(self.mask * A * (d - W)) / num_edges)

    def _closure(self):
        W = self.get_W()
        loss = self.loss_fn(W=W, A=self.A, num_edges=self.num_edges) + self.g.add_loss()
        self._optimizer.zero_grad()
        loss.backward()
        return loss

    def _train_step(self):
        """Performs and times one optimization step."""
        time_start = time.time()
        loss = self._optimizer.step(self._closure)
        time_end = time.time()
        return loss.item(), (time_end - time_start)

    def train(self, steps, optimizer_fn, optimizer_args, EO_criterion=None):
        """Starts the train loop.
        """
        self._optimizer = optimizer_fn(self.g.parameters(), **optimizer_args)
        self.steps = steps
        self.step_str_len = len(str(steps))
        self.scores_matrix_needs_update = True
        stop = False
        for self.step in range(self.step, steps + self.step):
            loss, time = self._train_step()
            self.total_time += time
            for callback in self.callbacks:
                callback(loss=loss, model=self)
                stop = stop or callback._training_stopped
            if stop:
                break

    def update_scores_matrix(self):
        """Updates the score matrix according to W."""
        self._scores_matrix = utils.scores_matrix_from_transition_matrix(
            transition_matrix=self(), symmetric=True
        )
        self.scores_matrix_needs_update = False

    def sample_graph(self):
        """Samples a graph from the learned parameters W.

        Edges are sampled independently from the score maxtrix.

        Returns:
            sampled_graph(sp.csr.csr_matrix): A synthetic graph generated by the model.
        """
        if self.scores_matrix_needs_update:
            self.update_scores_matrix()

        sampled_graph = utils.graph_from_scores(self._scores_matrix, self.num_edges)
        return sampled_graph


from sklearn import datasets, feature_extraction, metrics
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import Pipeline
from sklearn.neural_network import MLPClassifier
from pylab import *

print('2.1)')
newsgroups = datasets.fetch_20newsgroups()

train_20news = datasets.fetch_20newsgroups(subset='train', shuffle=True, random_state=42)
test_20news = datasets.fetch_20newsgroups(subset='test', shuffle=True, random_state=42)

print('What is the role of the \'categories\' parameter?')
print('The categories parameter is used to load a subset of the dataset based on the given \n'
      'collection of string inputs. This collection can for example be an array of strings. \n'
      'If categories is none (not specified), then the full dataset is loaded.')

print('\n\n2.2)')
print('CountVectorizer:')
print('Converts a collection of text documents into a matrix of token counts.\n')
print('TfidTransformer:')
print('Converts a count matrix into a normalized tf (term-frequency) or \n'
      'tf-idf (term-frequency times inverse document frequency) representation.\n')
print('TfidVectorizer:')
print('Converts a collection of raw documents into a matrix of tf-idf features.')
print('fit:')
print('Learn a vocabulary dictionary of all tokens in the raw documents. This is a \n'
      'function of CountVectorizer, TfidTransformer, and TfidVectorizer.\n')
print('fit_transform')
print('Learn the vocabulary dictionary and return term-document matrix. This function is a \n'
      'function of CountVectorizer, TfidTransformer, and TfidVectorizer. This function is \n'
      'also equivalent to calling fit followed by transform, but more efficiently implemented.\n')
print('transform:')
print('Transform documents to document-term matrix. This function is a \n'
      'function of CountVectorizer, TfidTransformer, and TfidVectorizer.\n')

vectorizer = feature_extraction.text.TfidfVectorizer()
train_vectors = vectorizer.fit_transform(train_20news.data)
test_vectors = vectorizer.transform(test_20news.data)

print('train_vectors:\n{}\n'.format(train_vectors))
print('test_vectors:\n{}'.format(test_vectors))

print('\n\n2.3)')
num_trees = 50
n = int(train_vectors.shape[0] / 2)
y1 = zeros(n, int)
y2 = zeros(n, int)+1
y = hstack((y1, y2))
RF = RandomForestClassifier(n_estimators=num_trees)

# training
RF.fit(train_vectors, y)

# testing
pred = RF.predict(test_vectors)

n_test = int(pred.size / 2)
y1_test = zeros(n_test, int)
y2_test = zeros(n_test, int)+1
y_test = hstack((y1_test, y2_test))

print('Accuracy-score: {}'.format(metrics.accuracy_score(y_test, pred)))
print('Precision-score: {}'.format(metrics.precision_score(y_test, pred)))
print('Recall-score: {}'.format(metrics.recall_score(y_test, pred)))
print('F1-score: {}'.format(metrics.f1_score(y_test, pred)))
print('Confusion matrix:\n{}'.format(metrics.confusion_matrix(y_test, pred)))

print('\n\n2.4)')
vectorizer = feature_extraction.text.TfidfVectorizer()
RF_pipe = RandomForestClassifier(n_estimators=num_trees)
n = int(train_vectors.shape[0] / 2)
y1 = zeros(n, int)
y2 = zeros(n, int)+1
y = hstack((y1, y2))
pipeline_training = Pipeline([('tfid_vectorizer', vectorizer), ('RF', RF_pipe)])

# training
pipeline_training.fit(train_20news.data, y)

# testing
pipeline_testing = pipeline_training.predict(test_20news.data)

# calculating scores
n_test = int(pipeline_testing.size / 2)
y1_test = zeros(n_test, int)
y2_test = zeros(n_test, int)+1
y_test = hstack((y1_test, y2_test))
print('Accuracy-score: {}'.format(metrics.accuracy_score(y_test, pipeline_testing)))
print('Precision-score: {}'.format(metrics.precision_score(y_test, pipeline_testing)))
print('Recall-score: {}'.format(metrics.recall_score(y_test, pipeline_testing)))
print('F1-score: {}'.format(metrics.f1_score(y_test, pipeline_testing)))
print('Confusion matrix:\n{}'.format(metrics.confusion_matrix(y_test, pipeline_testing)))

print('\n\n2.5)')
vectorizer = feature_extraction.text.TfidfVectorizer()
MLP = MLPClassifier(hidden_layer_sizes=(10, 20, 10), max_iter=10)
n = int(train_vectors.shape[0] / 2)
y1 = zeros(n, int)
y2 = zeros(n, int)+1
y = hstack((y1, y2))
MLP_pipeline = Pipeline([('tfid_vectorizer', vectorizer), ('MLP', MLP)])

# training
MLP_pipeline.fit(train_20news.data, y)

# testing
MLP_testing = MLP_pipeline.predict(test_20news.data)

# calculating scores
n_test = int(pipeline_testing.size / 2)
y1_test = zeros(n_test, int)
y2_test = zeros(n_test, int)+1
y_test = hstack((y1_test, y2_test))
print('Accuracy-score: {}'.format(metrics.accuracy_score(y_test, MLP_testing)))
print('Precision-score: {}'.format(metrics.precision_score(y_test, MLP_testing)))
print('Recall-score: {}'.format(metrics.recall_score(y_test, MLP_testing)))
print('F1-score: {}'.format(metrics.f1_score(y_test, MLP_testing)))
print('Confusion matrix:\n{}'.format(metrics.confusion_matrix(y_test, MLP_testing)))

Machine Learning/Intro-to-Machine-Learning/Python/2 Intro to SKLearn/random_forest.py


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
copywrite = """# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
# this file except in compliance with the License. A copy of the License is
# located at
#
# http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.
"""

import argparse
import configparser
import logging

from awacs.aws import Allow, Statement, Principal, PolicyDocument
from awacs.sts import AssumeRole
from troposphere import Template, Ref, Output
from troposphere.iam import Role
from troposphere.codebuild import Artifacts, Environment, Source, Project

logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)

def build_project(template=Template(), section=None, project_name=None, raw_env=None,
                  service_role: str = None) -> Template:
    template.set_version('2010-09-09')
    artifacts = Artifacts(Type='NO_ARTIFACTS')
    env_list = list()

    try:
        logging.debug(f'raw_env is {raw_env}')
        env = raw_env.split(' ')
    except AttributeError:
        env = config.get(section, 'env').split(' ')
        logging.debug(f'Section is {section}')

    for i in env:
        k, v = i.split("=")
        env_list.append({"Name": k, "Value": v})

    environment = Environment(
        ComputeType=config.get(section, 'compute_type'),
        Image=str(config.get(section, 'image')),
        Type=str(config.get(section, 'env_type')),
        PrivilegedMode=True,
        EnvironmentVariables=env_list,
    )

    source = Source(
        Location=config.get(section, 'source_location'),
        Type=config.get(section, 'source_type'),
        GitCloneDepth=config.get(section, 'source_clonedepth'),
        BuildSpec=config.get(section, 'buildspec'),
        ReportBuildStatus=True
    )

    project_id = project = Project(
        project_name,
        Artifacts=artifacts,
        Environment=environment,
        Name=project_name,
        ServiceRole=Ref(service_role),
        Source=source,
        SourceVersion=config.get(section, 'source_version'),
        BadgeEnabled=True,
        DependsOn=service_role,
    )
    template.add_resource(project)
    template.add_output([Output(f"CodeBuildProject{project_name}", Value=Ref(project_id))])

def build_role(template=Template(), section="CFNRole", project_name: str = None, **kwargs) -> Ref:
    """ Build a role with a CodeBuild managed policy. """
    template.set_version('2010-09-09')
    assert project_name
    project_name += 'Role'

    # NOTE: By default CodeBuild manages the policies for this role.  If you delete a CFN stack and try to recreate the project
    # or make changes to it when the Codebuild managed Policy still exists, you'll see an error in the UI:
    # `The policy is attached to 0 entities but it must be attached to a single role`. (CFN fails with fail to update)
    # Orphaned policies created by CodeBuild will have CodeBuildBasePolicy prepended to them; search for policies with this
    # name and no role and delete to clear the error.
    # TODO: Get a CloudFormation feature request to turn this off for project creation- let CFN manage the policy.
    role_id = template.add_resource(
        Role(
            project_name,
            Path='/',
            AssumeRolePolicyDocument=PolicyDocument(
                Statement=[
                    Statement(
                        Effect=Allow,
                        Action=[AssumeRole],
                        Principal=Principal("Service", ["codebuild.amazonaws.com"])
                    )
                ]
            )
        )
    )

    template.add_output([Output(project_name, Value=Ref(role_id))])
    return Ref(role_id)

def main(**kwargs):
    """ Create the CFN template and either write to screen or update/create boto3. """
    codebuild = Template()

    for job in config.sections():
        if 'CodeBuild:' in job:
            job_title = job.split(':')[1]
            service_role = build_role(template=codebuild, project_name=job_title).to_dict()
            # Pull the env out of the section, and use the snippet for the other values.
            if 'snippet' in config[job]:
                build_project(template=codebuild, project_name=job_title, section=config.get(job, 'snippet'),
                              service_role=service_role['Ref'], raw_env=config.get(job, 'env'))
            else:
                build_project(template=codebuild, project_name=job_title, section=job, service_role=service_role['Ref'])

    with(open(args.output_dir + "/codebuild_test_projects.yml", 'w')) as fh:
        fh.write(codebuild.to_yaml())

    if args.dry_run:
        logging.debug('Dry Run: wrote cfn file, but not calling AWS.')
    else:
        print('Boto functionality goes here.')

if __name__ == '__main__':
    # Parse  options
    parser = argparse.ArgumentParser(description='Creates AWS CodeBuild Project CloudFormation files ' +
                                                 'based on a simple config')
    parser.add_argument('--config', type=str, default="codebuild.config", help='The config filename to create the '
                                                                               'CodeBuild projects')
    parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='Output CFN to stdout; Do not call AWS')
    parser.add_argument('--output-dir', dest='output_dir', default='cfn', help="Directory to write CFN files")
    args = parser.parse_args()

    config = configparser.RawConfigParser()
    logging.debug(f'Try to load config file {args.config}')
    config.read(args.config)
    assert config.get('CFNRole', 'account_number')
    main(args=args, config=config)


# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""
import csv

def loadcsv(inputcsv):
    rows = []
    with open(inputcsv, newline='') as csvfile:
        sreader = csv.reader(csvfile, delimiter=',', quotechar='"')
        headerrow = next(sreader)
        for row in sreader:
            if row:
                rows.append(row)
    return headerrow, rows

def coldata(inputcsv, theset):
    headerrow, rows = loadcsv(inputcsv)
    return [r[theset] for r in rows]

import matplotlib.pyplot as plt
plt.figure(1)

inputcsv = 'Pagano_3lay_cyl_bend_THIN_T10_conv_Stress_errors_default.CSV'
x = coldata(inputcsv, 0)
y = coldata(inputcsv, 3)
plt.loglog(x, y, color = "black", marker = "x", markersize = 12, label = "DCE")

inputcsv = 'Pagano_3lay_cyl_bend_THIN_MST10_conv_Stress_errors_extraptrend.CSV'
x = coldata(inputcsv, 0)
y = coldata(inputcsv, 3)
plt.loglog(x, y, color = "black", marker = "o", markersize = 12, label = "TBE")

inputcsv = 'Pagano_3lay_cyl_bend_THIN_MST10_conv_Stress_errors_extrapmean.CSV'
x = coldata(inputcsv, 0)
y = coldata(inputcsv, 3)
plt.loglog(x, y, color = "black", marker = "d", markersize = 12, label = "MSOE")

plt.xlabel('Element size')
plt.ylabel('Normalized error')
plt.axis([0.004, 0.1, 0.0001, 1.0])
plt.grid(True)
legend = plt.legend(loc='best', shadow=False, fontsize='x-large')
plt.show()


from PyQt5 import QtCore, uic
from PyQt5.QtWidgets import QApplication, QLabel, \
    QPushButton, QAction, QSpinBox

def formulas():

    cash = qtd_cash.value()

    if cash == 0:
        pouco = 0
        razoavel = 0
        adequado = 0

    elif (0<cash)and(cash<=30):
        pouco = 1
        razoavel = 0
        adequado = 0

    elif (cash > 30)and(cash<50):
        pouco = (50-cash)/(50-30)
        razoavel = (cash-30)/(50-30)
        adequado = 0

    elif cash >= 70:
        pouco = 0
        razoavel = 0
        adequado = 1

    elif cash == 50:
        pouco = 0
        adequado = 0
        razoavel = 1

    elif (cash > 50)and(cash < 70):
        razoavel = (70-cash)/(70-50)
        adequado = (cash-50)/(70-50)
        pouco = 0

    people = quantidade_pessoa.value()

    if people == 0:
        insuficiente = 0
        satisfatorio = 0

    elif people >= 70:
        insuficiente = 0
        satisfatorio = 1

    elif (people > 0)and(people <= 30):
        insuficiente = 1
        satisfatorio = 0

    elif (people > 30)and(people < 70):
        insuficiente = (70-people)/(70-30)
        satisfatorio = (people - 30)/(70-30)

    regras(pouco, razoavel, adequado, insuficiente, satisfatorio)

def regras(pouco, razoavel, adequado, insuficiente, satisfatorio):

    if (pouco >= insuficiente) and (pouco >= satisfatorio):
        alto = pouco

    else:
        if (insuficiente <= satisfatorio):
            alto = insuficiente

        else:
            alto = satisfatorio

    if razoavel <= satisfatorio:
        medio = razoavel
    else:
        medio = satisfatorio

    if adequado <= satisfatorio:
        baixo = adequado
    else:
        baixo = satisfatorio

    soma = 0
    divisor = 0

    if baixo > 0:
        soma += 60 * baixo
        divisor += 3*baixo

    if medio > 0:
        soma += 150 * medio
        divisor += 3*medio

    if alto > 0:
        soma += 240 * alto
        divisor += 3*alto

    risco_final = soma/divisor
    show(risco_final)

def show(risco_final):
    if (risco_final <= 35):
        resultado.setText('Baixo : ' + str(risco_final))
        resultado.setVisible(True)
    elif(risco_final > 35)and(risco_final <= 65):
        resultado.setText('Médio : ' + str(risco_final))
        resultado.setVisible(True)
    elif(risco_final > 65):
        resultado.setText('Alto : ' + str(risco_final))
        resultado.setVisible(True)

if __name__ == "__main__":
    QtCore.QCoreApplication.setAttribute(QtCore.Qt.AA_ShareOpenGLContexts)
    app = QApplication([])

    window = uic.loadUi(r"C:\Users\Aender\Desktop\CC\Inteligencia artificial\ia-cc-2020-2-AendersonSouza\2_logica_fuzzy\interface.ui")

    dinheiro = window.findChild(QLabel, 'dinheiro')
    qtd_cash = window.findChild(QSpinBox, 'qtd_dinheiro')

    pessoa = window.findChild(QLabel, 'pessoa')
    quantidade_pessoa = window.findChild(QSpinBox, 'qtd_pessoa')

    risco = window.findChild(QLabel, 'risco')
    resultado= window.findChild(QLabel, 'resultado')
    resultado.setVisible(False)

    Calcular = window.findChild(QPushButton, 'calcular')
    Calcular.clicked.connect(formulas)

    window.show()

    app.exec_()


# encoding: utf8
from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        (u'series', '0003_seriesrelease_tags'),
    ]

    operations = [
        migrations.AlterField(
            model_name='seriessubtitle',
            name='file',
            field=models.FileField(help_text=u'A f\xe1jln\xe9v a release szab\xe1lyok alapj\xe1n lesz t\xe1rolva.', upload_to=u'subtitles', null=True, verbose_name=u'Feliratf\xe1jl', blank=True),
        ),
        migrations.AlterField(
            model_name='seriessubtitle',
            name='filename',
            field=models.CharField(help_text=u'Kiz\xe1r\xf3lag akkor t\xf6ltsd ki ha a filen\xe9v elt\xe9r a release-ben megadott formai jel\xf6l\xe9st\u0151l (pl. dupla r\xe9sz, elt\xe9r\u0151 filen\xe9v, stb.)', max_length=255, verbose_name=u'Egyedi f\xe1jln\xe9v', blank=True),
        ),
        migrations.AlterField(
            model_name='seriessubtitle',
            name='double',
            field=models.BooleanField(default=False, verbose_name=u'Dupla r\xe9sz'),
        ),
    ]


import tensorflow as tf

from ..ops import optimizer as opt
from ..ops import layer_collection as lc
from ..ops import sampler as sp
from ..network.registry import get_model
from .base_model import BaseModel
from . import MODE_CLASSIFICATION, MODE_REGRESSION, MODE_IRD

class KFACModel(BaseModel):
    def __init__(self, config, input_dim, n_data):
        super().__init__(config)
        self.layer_collection = lc.LayerCollection()
        self.input_dim = input_dim
        self.n_data = n_data
        self.cov_update_op = None
        self.inv_update_op = None

        self.build_model()
        self.init_optim()
        self.init_saver()

    def build_model(self):
        self.inputs = tf.placeholder(tf.float32, [None] + self.input_dim,
                name="inputs")
        self.aux_inputs = tf.placeholder(tf.float32, [None] + self.input_dim,
                name="aux_inputs")
        self.is_training = tf.placeholder(tf.bool, name="is_training")
        self.n_particles = tf.placeholder(tf.int32, name="n_particles")

        net = get_model(self.config.model_name)

        self.sampler = sp.Sampler(self.config, self.n_data, self.n_particles)
        outputs, aux_outputs, l2_loss, mode = net(self.inputs, self.aux_inputs,
                              self.sampler, self.is_training,
                              self.config.batch_norm, self.layer_collection,
                              self.n_particles)

        self.aux_outputs = aux_outputs
        self.outputs = outputs
        self.mode = mode

        if mode == MODE_REGRESSION:
            self.targets = tf.placeholder(tf.float32, [None])
            targets_ = tf.tile(self.targets, [self.n_particles])
            self.loss = tf.losses.mean_squared_error(targets_,
                    tf.squeeze(outputs, 1))
            self.acc = self.loss
        elif mode == MODE_IRD:
            self.targets = None
            self.aux_lse = tf.reduce_logsumexp(self.aux_outputs, name="aux_lse")
            self.loss = -tf.reduce_mean(tf.subtract(self.outputs, self.aux_lse, name="loss"))
            self.acc = self.loss
        elif mode == MODE_CLASSIFICATION:
            self.targets = tf.placeholder(tf.int64, [None])
            targets_ = tf.tile(self.targets, [self.n_particles])
            logits_ = tf.reduce_mean(
                tf.reshape(outputs, [self.n_particles, -1, tf.shape(outputs)[-1]]
                    ), 0)
            self.acc = tf.reduce_mean(tf.cast(tf.equal(
                self.targets, tf.argmax(logits_, axis=1)), dtype=tf.float32))
            self.loss = tf.reduce_mean(
                tf.nn.sparse_softmax_cross_entropy_with_logits(
                    labels=targets_, logits=outputs))
        else:
            raise TypeError("Illegal mode: {}".format(mode))

        coeff = self.config.kl / (self.n_data * self.config.eta)
        self.total_loss = self.loss + coeff * l2_loss

    def init_optim(self):
        self.optim = opt.KFACOptimizer(
            var_list=self.trainable_variables,
            learning_rate=self.config.learning_rate,
            cov_ema_decay=self.config.cov_ema_decay,
            damping=self.config.damping,
            layer_collection=self.layer_collection,
            norm_constraint=tf.train.exponential_decay(self.config.kl_clip,
                                                       self.global_step_tensor,
                                                       390, 0.95,
                                                       staircase=True),
            momentum=self.config.momentum)

        self.cov_update_op = self.optim.cov_update_op
        self.inv_update_op = self.optim.inv_update_op

        with tf.control_dependencies([self.inv_update_op]):
            self.var_update_op = self.sampler.update(
                    self.layer_collection.get_blocks())

        update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
        with tf.control_dependencies(update_ops):
            self.train_op = self.optim.minimize(self.total_loss,
                    global_step=self.global_step_tensor)


"""
The module handles execution and monitoring of workflows. It enables executing a workflow multiple times and in
parallel. It monitors execution times and handles any error that might occur in the process. At the end it generates a
report which contains summary of the workflow and process of execution.

All this is implemented in EOExecutor class.

Credits:
Copyright (c) 2017-2022 Matej Aleksandrov, Matej Batič, Grega Milčinski, Domagoj Korais, Matic Lubej (Sinergise)
Copyright (c) 2017-2022 Žiga Lukšič, Devis Peressutti, Tomislav Slijepčević, Nejc Vesel, Jovan Višnjić (Sinergise)
Copyright (c) 2017-2022 Anže Zupanc (Sinergise)
Copyright (c) 2017-2019 Blaž Sovdat, Andrej Burja (Sinergise)

This source code is licensed under the MIT license found in the LICENSE
file in the root directory of this source tree.
"""
import concurrent.futures
import datetime as dt
import logging
import multiprocessing
import threading
import warnings
from dataclasses import dataclass
from enum import Enum
from logging import Filter, Handler, Logger
from typing import Callable, Dict, Iterable, List, Optional, Sequence, Tuple, TypeVar, cast

import fs
from fs.base import FS
from tqdm.auto import tqdm

from .eonode import EONode
from .eoworkflow import EOWorkflow, WorkflowResults
from .exceptions import EORuntimeWarning
from .utils.fs import get_base_filesystem_and_path, get_full_path
from .utils.logging import LogFileFilter

LOGGER = logging.getLogger(__name__)
MULTIPROCESSING_LOCK = None

# pylint: disable=invalid-name
_InputType = TypeVar("_InputType")
_OutputType = TypeVar("_OutputType")
_HandlerFactoryType = Callable[[str], Handler]

class _ProcessingType(Enum):
    """Type of EOExecutor processing"""

    SINGLE_PROCESS = "single process"
    MULTIPROCESSING = "multiprocessing"
    MULTITHREADING = "multithreading"
    RAY = "ray"

@dataclass(frozen=True)
class _ProcessingData:
    """Data to be used in EOExecutor processing. This will be passed to a process pool, so everything has to be
    serializable with pickle."""

    workflow: EOWorkflow
    workflow_kwargs: Dict[EONode, Dict[str, object]]
    log_path: Optional[str]
    filter_logs_by_thread: bool
    logs_filter: Optional[Filter]
    logs_handler_factory: _HandlerFactoryType

class EOExecutor:
    """Simultaneously executes a workflow with different input arguments. In the process it monitors execution and
    handles errors. It can also save logs and create a html report about each execution.
    """

    REPORT_FILENAME = "report.html"
    STATS_START_TIME = "start_time"
    STATS_END_TIME = "end_time"

    def __init__(
        self,
        workflow: EOWorkflow,
        execution_kwargs: Sequence[Dict[EONode, Dict[str, object]]],
        *,
        execution_names: Optional[List[str]] = None,
        save_logs: bool = False,
        logs_folder: str = ".",
        filesystem: Optional[FS] = None,
        logs_filter: Optional[Filter] = None,
        logs_handler_factory: _HandlerFactoryType = logging.FileHandler,
    ):
        """
        :param workflow: A prepared instance of EOWorkflow class
        :param execution_kwargs: A list of dictionaries where each dictionary represents execution inputs for the
            workflow. `EOExecutor` will execute the workflow for each of the given dictionaries in the list. The
            content of such dictionary will be used as `input_kwargs` parameter in `EOWorkflow.execution` method.
            Check `EOWorkflow.execution` for definition of a dictionary structure.
        :param execution_names: A list of execution names, which will be shown in execution report
        :param save_logs: Flag used to specify if execution log files should be saved locally on disk
        :param logs_folder: A folder where logs and execution report should be saved. If `filesystem` parameter is
            defined the folder path should be relative to the filesystem.
        :param filesystem: A filesystem object for saving logs and a report.
        :param logs_filter: An instance of a custom filter object that will filter certain logs from being written into
            logs. It works only if save_logs parameter is set to True.
        :param logs_handler_factory: A callable class or function that takes logging path as its only input parameter
            and creates an instance of logging handler object
        """
        self.workflow = workflow
        self.execution_kwargs = self._parse_and_validate_execution_kwargs(execution_kwargs)
        self.execution_names = self._parse_execution_names(execution_names, self.execution_kwargs)
        self.save_logs = save_logs
        self.filesystem, self.logs_folder = self._parse_logs_filesystem(filesystem, logs_folder)
        self.logs_filter = logs_filter
        self.logs_handler_factory = logs_handler_factory

        self.start_time: Optional[dt.datetime] = None
        self.report_folder: Optional[str] = None
        self.general_stats: Dict[str, object] = {}
        self.execution_results: List[WorkflowResults] = []

    @staticmethod
    def _parse_and_validate_execution_kwargs(
        execution_kwargs: Sequence[Dict[EONode, Dict[str, object]]]
    ) -> List[Dict[EONode, Dict[str, object]]]:
        """Parses and validates execution arguments provided by user and raises an error if something is wrong."""
        if not isinstance(execution_kwargs, (list, tuple)):
            raise ValueError("Parameter 'execution_kwargs' should be a list.")

        for input_kwargs in execution_kwargs:
            EOWorkflow.validate_input_kwargs(input_kwargs)

        return [input_kwargs or {} for input_kwargs in execution_kwargs]

    @staticmethod
    def _parse_execution_names(execution_names: Optional[List[str]], execution_kwargs: Sequence) -> List[str]:
        """Parses a list of execution names."""
        if execution_names is None:
            return [str(num) for num in range(1, len(execution_kwargs) + 1)]

        if not isinstance(execution_names, (list, tuple)) or len(execution_names) != len(execution_kwargs):
            raise ValueError(
                "Parameter 'execution_names' has to be a list of the same size as the list of execution arguments."
            )
        return execution_names

    @staticmethod
    def _parse_logs_filesystem(filesystem: Optional[FS], logs_folder: str) -> Tuple[FS, str]:
        """Ensures a filesystem and a file path relative to it."""
        if filesystem is None:
            return get_base_filesystem_and_path(logs_folder)
        return filesystem, logs_folder

    def run(self, workers: int = 1, multiprocess: bool = True) -> List[WorkflowResults]:
        """Runs the executor with n workers.

        :param workers: Maximum number of workflows which will be executed in parallel. Default value is `1` which will
            execute workflows consecutively. If set to `None` the number of workers will be the number of processors
            of the system.
        :param multiprocess: If `True` it will use `concurrent.futures.ProcessPoolExecutor` which will distribute
            workflow executions among multiple processors. If `False` it will use
            `concurrent.futures.ThreadPoolExecutor` which will distribute workflow among multiple threads.
            However, even when `multiprocess=False`, tasks from workflow could still be using multiple processors.
            This parameter is used especially because certain task cannot run with
            `concurrent.futures.ProcessPoolExecutor`.
            In case of `workers=1` this parameter is ignored and workflows will be executed consecutively.
        :return: A list of EOWorkflow results
        """
        self.start_time = dt.datetime.now()
        self.report_folder = fs.path.combine(
            self.logs_folder, f'eoexecution-report-{self.start_time.strftime("%Y_%m_%d-%H_%M_%S")}'
        )
        if self.save_logs:
            self.filesystem.makedirs(self.report_folder, recreate=True)

        log_paths = self.get_log_paths(full_path=True) if self.save_logs else [None] * len(self.execution_kwargs)

        filter_logs_by_thread = not multiprocess and workers > 1
        processing_type = self._get_processing_type(workers, multiprocess)
        processing_args = [
            _ProcessingData(
                workflow=self.workflow,
                workflow_kwargs=workflow_kwargs,
                log_path=log_path,
                filter_logs_by_thread=filter_logs_by_thread,
                logs_filter=self.logs_filter,
                logs_handler_factory=self.logs_handler_factory,
            )
            for workflow_kwargs, log_path in zip(self.execution_kwargs, log_paths)
        ]

        full_execution_results = self._run_execution(processing_args, workers, processing_type)

        self.execution_results = [results.drop_outputs() for results in full_execution_results]
        self.general_stats = self._prepare_general_stats(workers, processing_type)

        return full_execution_results

    @staticmethod
    def _get_processing_type(workers: int, multiprocess: bool) -> _ProcessingType:
        """Decides processing type according to parameters."""
        if workers == 1:
            return _ProcessingType.SINGLE_PROCESS
        if multiprocess:
            return _ProcessingType.MULTIPROCESSING
        return _ProcessingType.MULTITHREADING

    def _run_execution(
        self, processing_args: List[_ProcessingData], workers: int, processing_type: _ProcessingType
    ) -> List[WorkflowResults]:
        """Runs the execution for each item of processing_args list."""
        if processing_type is _ProcessingType.SINGLE_PROCESS:
            return list(tqdm(map(self._execute_workflow, processing_args), total=len(processing_args)))

        if processing_type is _ProcessingType.MULTITHREADING:
            with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as thread_executor:
                return submit_and_monitor_execution(thread_executor, self._execute_workflow, processing_args)

        # pylint: disable=global-statement
        global MULTIPROCESSING_LOCK
        try:
            MULTIPROCESSING_LOCK = multiprocessing.Manager().Lock()
            with concurrent.futures.ProcessPoolExecutor(max_workers=workers) as process_executor:
                return submit_and_monitor_execution(process_executor, self._execute_workflow, processing_args)
        finally:
            MULTIPROCESSING_LOCK = None

    @classmethod
    def _try_add_logging(
        cls,
        log_path: Optional[str],
        filter_logs_by_thread: bool,
        logs_filter: Optional[Filter],
        logs_handler_factory: _HandlerFactoryType,
    ) -> Tuple[Optional[Logger], Optional[Handler]]:
        """Adds a handler to a logger and returns them both. In case this fails it shows a warning."""
        if log_path:
            try:
                logger = logging.getLogger()
                logger.setLevel(logging.DEBUG)
                handler = cls._build_log_handler(log_path, filter_logs_by_thread, logs_filter, logs_handler_factory)
                logger.addHandler(handler)
                return logger, handler
            except BaseException as exception:
                warnings.warn(f"Failed to start logging with exception: {repr(exception)}", category=EORuntimeWarning)

        return None, None

    @classmethod
    def _try_remove_logging(cls, log_path: Optional[str], logger: Optional[Logger], handler: Optional[Handler]):
        """Removes a handler from a logger in case that handler exists."""
        if log_path and logger and handler:
            try:
                handler.close()
                logger.removeHandler(handler)
            except BaseException as exception:
                warnings.warn(f"Failed to end logging with exception: {repr(exception)}", category=EORuntimeWarning)

    @classmethod
    def _execute_workflow(cls, data: _ProcessingData) -> WorkflowResults:
        """Handles a single execution of a workflow."""
        logger, handler = cls._try_add_logging(
            data.log_path, data.filter_logs_by_thread, data.logs_filter, data.logs_handler_factory
        )

        results = data.workflow.execute(data.workflow_kwargs, raise_errors=False)

        cls._try_remove_logging(data.log_path, logger, handler)
        return results

    @staticmethod
    def _build_log_handler(
        log_path: str,
        filter_logs_by_thread: bool,
        logs_filter: Optional[Filter],
        logs_handler_factory: _HandlerFactoryType,
    ) -> Handler:
        """Provides object which handles logs."""
        handler = logs_handler_factory(log_path)

        if not handler.formatter:
            formatter = logging.Formatter("%(asctime)s %(name)-12s %(levelname)-8s %(message)s")
            handler.setFormatter(formatter)

        if filter_logs_by_thread:
            handler.addFilter(LogFileFilter(threading.currentThread().getName()))

        if logs_filter:
            handler.addFilter(logs_filter)

        return handler

    def _prepare_general_stats(self, workers: int, processing_type: _ProcessingType) -> Dict[str, object]:
        """Prepares a dictionary with a general statistics about executions."""
        failed_count = sum(results.workflow_failed() for results in self.execution_results)
        return {
            self.STATS_START_TIME: self.start_time,
            self.STATS_END_TIME: dt.datetime.now(),
            "finished": len(self.execution_results) - failed_count,
            "failed": failed_count,
            "processing_type": processing_type.value,
            "workers": workers,
        }

    def get_successful_executions(self) -> List[int]:
        """Returns a list of IDs of successful executions. The IDs are integers from interval
        `[0, len(execution_kwargs) - 1]`, sorted in increasing order.

        :return: List of successful execution IDs
        """
        return [idx for idx, results in enumerate(self.execution_results) if not results.workflow_failed()]

    def get_failed_executions(self) -> List[int]:
        """Returns a list of IDs of failed executions. The IDs are integers from interval
        `[0, len(execution_kwargs) - 1]`, sorted in increasing order.

        :return: List of failed execution IDs
        """
        return [idx for idx, results in enumerate(self.execution_results) if results.workflow_failed()]

    def get_report_path(self, full_path: bool = True) -> str:
        """Returns the filename and file path of the report.

        :param full_path: A flag to specify if it should return full absolute paths or paths relative to the
            filesystem object.
        :return: Report filename
        """
        if self.report_folder is None:
            raise RuntimeError("Executor has to be run before the report path is created.")
        report_path = fs.path.combine(self.report_folder, self.REPORT_FILENAME)
        if full_path:
            return get_full_path(self.filesystem, report_path)
        return report_path

    def make_report(self, include_logs: bool = True):
        """Makes a html report and saves it into the same folder where logs are stored.

        :param include_logs: If `True` log files will be loaded into the report file. If `False` they will be just
            referenced with a link to a log file. In case of a very large number of executions it is recommended that
            this parameter is set to `False` to avoid compiling a too large report file.
        """
        # pylint: disable=import-outside-toplevel,raise-missing-from
        try:
            from eolearn.visualization.eoexecutor import EOExecutorVisualization
        except ImportError:
            raise RuntimeError(
                "Subpackage eo-learn-visualization has to be installed in order to create EOExecutor reports."
            )

        return EOExecutorVisualization(self).make_report(include_logs=include_logs)

    def get_log_paths(self, full_path: bool = True) -> List[str]:
        """Returns a list of file paths containing logs.

        :param full_path: A flag to specify if it should return full absolute paths or paths relative to the
            filesystem object.
        :return: A list of paths to log files.
        """
        if self.report_folder is None:
            raise RuntimeError("Executor has to be run before log paths are created.")
        log_paths = [fs.path.combine(self.report_folder, f"eoexecution-{name}.log") for name in self.execution_names]
        if full_path:
            return [get_full_path(self.filesystem, path) for path in log_paths]
        return log_paths

    def read_logs(self) -> List[Optional[str]]:
        """Loads the content of log files if logs have been saved."""
        if not self.save_logs:
            return [None] * len(self.execution_kwargs)

        log_paths = self.get_log_paths(full_path=False)
        with concurrent.futures.ThreadPoolExecutor() as executor:
            return list(executor.map(self._read_log_file, log_paths))

    def _read_log_file(self, log_path: str) -> str:
        """Read a content of a log file."""
        try:
            with self.filesystem.open(log_path, "r") as file_handle:
                return file_handle.read()
        except BaseException as exception:
            warnings.warn(f"Failed to load logs with exception: {repr(exception)}", category=EORuntimeWarning)
            return "Failed to load logs"

def submit_and_monitor_execution(
    executor: concurrent.futures.Executor,
    function: Callable[[_InputType], _OutputType],
    execution_params: Iterable[_InputType],
) -> List[_OutputType]:
    """Performs the execution parallelization and monitors the process using a progress bar.

    :param executor: An object that performs parallelization.
    :param function: A function to be parallelized.
    :param execution_params: Each element in a sequence are parameters for a single call of `function`.
    :return: A list of results in the same order as input parameters given by `executor_params`.
    """
    futures = [executor.submit(function, params) for params in execution_params]
    future_order = {future: i for i, future in enumerate(futures)}

    results: List[Optional[_OutputType]] = [None] * len(futures)
    with tqdm(total=len(futures)) as pbar:
        for future in concurrent.futures.as_completed(futures):
            results[future_order[future]] = future.result()
            pbar.update(1)

    return cast(List[_OutputType], results)

def execute_with_mp_lock(execution_function: Callable, *args, **kwargs) -> object:
    """A helper utility function that executes a given function with multiprocessing lock if the process is being
    executed in a multiprocessing mode.

    :param execution_function: A function
    :param args: Function's positional arguments
    :param kwargs: Function's keyword arguments
    :return: Function's results
    """
    if multiprocessing.current_process().name == "MainProcess" or MULTIPROCESSING_LOCK is None:
        return execution_function(*args, **kwargs)

    # pylint: disable=not-context-manager
    with MULTIPROCESSING_LOCK:
        return execution_function(*args, **kwargs)


import os
import requests
from flask import Flask, jsonify, render_template, request
from flask_socketio import SocketIO, emit

app = Flask(__name__)
app.config["SECRET_KEY"] = os.getenv("SECRET_KEY")
socketio = SocketIO(app)

@app.route("/")
def index():
    return render_template("index.html")

@socketio.on("submit vote")#when the event "submit votes" (in the js file), take date and save in var called selection, then braodcast to everyone (any other websockets that are listening.) the vote.
def vote(data):
    selection = data["selection"]
    emit("announce vote", {"selection": selection}, broadcast=True)

if __name__ == '__main__':
    socketio.run(app)


import numpy as np
import matplotlib.pyplot as plt
import colorsys as colors

def get_color_order(i,max_num):
    #take a blue to red scale from 0 to max_num
    # uses HSV system, H_red = 0, H_green = 1/3 H_blue=2/3
    return colors.hsv_to_rgb(2.*float(i)/(float(max_num)*3.),1.,1.)

def get_color_from_cmap(i,max_num):
    pass

def flex_color_plot_vs_x(xvals,yvals,zvals,ax=None,
                         xwidth=None,
                         normalize=False,log=False,
                         save_name=None,**kw):
    """
    Display a color figure for something like a tracked DAC sweep.
    xvals should be a single vector with values for the primary sweep.
    yvals and zvals should be a list of arrays with the sweep points and measured values.
    """
    # create a figure and set of axes
    if ax == None:
        fig = plt.figure(figsize=(12,7))
        ax = fig.add_subplot(111)

    # calculate coordinates for corners of color blocks
    # x coordinates
    if xwidth is None:
        xvals = np.array(xvals)
        xvertices = np.zeros(np.array(xvals.shape)+1)
        xvertices[1:-1] = (xvals[:-1]+xvals[1:])/2.
        xvertices[0] = xvals[0] - (xvals[1]-xvals[0])/2
        xvertices[-1] = xvals[-1] + (xvals[-1]-xvals[-2])/2
    else:
        xvertices = []
        for xval in xvals:
            xvertices.append(xval+np.array([-0.5,0.5])*xwidth)
    # y coordinates
    yvertices = []
    for xx in range(len(xvals)):
        yvertices.append(np.zeros(np.array(yvals[xx].shape)+1))
        yvertices[xx][1:-1] = (yvals[xx][:-1]+yvals[xx][1:])/2.
        yvertices[xx][0] = yvals[xx][0] - (yvals[xx][1]-yvals[xx][0])/2
        yvertices[xx][-1] = yvals[xx][-1] + (yvals[xx][-1]-yvals[xx][-2])/2

    # various plot options
    # define colormap
    cmap = plt.get_cmap(kw.pop('cmap', 'CMRmap'))
    clim = kw.pop('clim', [None, None])
    # normalized plot
    if normalize:
        for xx in range(len(xvals)):
            zvals[xx] /= np.mean(zvals[xx])
    # logarithmic plot
    if log:
        for xx in range(len(xvals)):
            zvals[xx] = np.log(zvals[xx])/np.log(10)

    alpha = kw.pop('alpha',1)

    # add blocks to plot
    # hold = kw.pop('hold',False)
    colormap = []
    for xx in range(len(xvals)):
        tempzvals = np.array([np.append(zvals[xx],np.array(0)),np.append(zvals[xx],np.array(0))]).transpose()
        if xwidth is None:
            colormap.append(ax.pcolor(xvertices[xx:xx+2],
                                      yvertices[xx],
                                      tempzvals,
                                      cmap=cmap, vmin=clim[0], vmax=clim[1],
                                      alpha=alpha))
        else:
            colormap.append(ax.pcolor(xvertices[xx],yvertices[xx],tempzvals,cmap=cmap,alpha=alpha))
        #XX, YY = np.meshgrid(xvertices[xx:xx+2],yvertices[xx])
        #ax1.pcolor(XX,YY,tempzvals,cmap=cmap)

    return {'fig':ax.figure,'ax':ax,'cmap':colormap}

def flex_colormesh_plot_vs_xy(xvals,yvals,zvals,ax=None,
                         normalize=False,log=False,
                         save_name=None,**kw):
    """
    Add a rectangular block to a color plot using pcolormesh.
    xvals and yvals should be single vectors with values for the two sweep points.
    zvals should be a list of arrays with the measured values with shape (len(yvals), len(xvals)).
    """
    # create a figure and set of axes
    if ax == None:
        fig = plt.figure(figsize=(12,7))
        ax = fig.add_subplot(111)

    # convert xvals and yvals to single dimension arrays
    xvals = np.squeeze(np.array(xvals))
    yvals = np.squeeze(np.array(yvals))

    # calculate coordinates for corners of color blocks
    # x coordinates
    xvertices = np.zeros(np.array(xvals.shape)+1)
    xvertices[1:-1] = (xvals[:-1]+xvals[1:])/2.
    xvertices[0] = xvals[0] - (xvals[1]-xvals[0])/2
    xvertices[-1] = xvals[-1] + (xvals[-1]-xvals[-2])/2
    # y coordinates
    yvertices = np.zeros(np.array(yvals.shape)+1)
    yvertices[1:-1] = (yvals[:-1]+yvals[1:])/2.
    yvertices[0] = yvals[0] - (yvals[1]-yvals[0])/2
    yvertices[-1] = yvals[-1] + (yvals[-1]-yvals[-2])/2

    xgrid, ygrid = np.meshgrid(xvertices, yvertices)

    # various plot options
    # define colormap
    cmap = plt.get_cmap(kw.pop('cmap', 'CMRmap'))
    clim = kw.pop('clim', [None, None])
    # normalized plot
    if normalize:
        zvals /= np.mean(zvals, axis=0)
    # logarithmic plot
    if log:
        for xx in range(len(xvals)):
            zvals[xx] = np.log(zvals[xx])/np.log(10)

    alpha = kw.pop('alpha',1)

    # add blocks to plot
    # hold = kw.pop('hold',False)
    do_transpose = kw.pop('transpose', False)
    if do_transpose:
        colormap = ax.pcolormesh(ygrid.transpose(),
                      xgrid.transpose(),
                      zvals.transpose(),
                      cmap=cmap, vmin=clim[0], vmax=clim[1])
    else:
        colormap = ax.pcolormesh(xgrid, ygrid, zvals, cmap=cmap,
                      vmin=clim[0], vmax=clim[1])

    return {'fig':ax.figure,'ax':ax,'cmap':colormap}


from setuptools import find_packages
from setuptools import setup
from setuptools.command.test import test
import os

def read_file(name):
    with open(os.path.join(os.path.dirname(__file__), name)) as f:
        return f.read()

version = '1.0'
shortdesc = 'Provide parts of a web application as tiles.'
longdesc = '\n\n'.join([read_file(name) for name in [
    'README.rst',
    'CHANGES.rst',
    'LICENSE.rst'
]])

class Test(test):

    def run_tests(self):
        from cone.tile import tests
        tests.run_tests()

setup(
    name='cone.tile',
    version=version,
    description=shortdesc,
    long_description=longdesc,
    classifiers=[
        'Environment :: Web Environment',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
    ],
    keywords='',
    author='Robert Niederreiter',
    author_email='rnix@squarewave.at',
    url='https://github.com/bluedynamics/cone.tile',
    license='Simplified BSD',
    packages=find_packages('src'),
    package_dir={'': 'src'},
    namespace_packages=['cone'],
    include_package_data=True,
    zip_safe=False,
    install_requires=[
        'setuptools',
        'zope.component',
        'zope.deprecation',
        'zope.exceptions',
        'pyramid>=1.5',
        'pyramid_chameleon',
    ],
    extras_require=dict(test=['zope.testrunner']),
    tests_require=['zope.testrunner'],
    cmdclass=dict(test=Test)
)


from __future__ import print_function

import sys
from operator import add
from csv import reader
from pyspark import SparkContext

#task 5

if __name__ == "__main__":
 if len(sys.argv) != 2:
  print("Usage: wordcount <file>", file=sys.stderr)
  exit(-1)
 sc = SparkContext()
 lines = sc.textFile(sys.argv[1], 1)
 line = lines.mapPartitions(lambda x:reader(x)).map(lambda x:((x[14],x[16]),1))

 line = line.reduceByKey(add)

 line = line.sortBy(lambda x: x[1],False)
 top = line.take(1)
 output = sc.parallelize(top)

 line = output.map (lambda x:'%s, %s\t%s' % (x[0][0],x[0][1],x[1]))
 line.saveAsTextFile("task5.out")

 sc.stop()


import cv2
# trích suất ảnh từ camera
cap = cv2.VideoCapture(0)
# load model haarcascade
detect = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')

while (True):
    ret, img = cap.read()
    # chuyển về ảnh xám
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    # nhận diện khuôn mặt
    face = detect.detectMultiScale(gray, 1.3, 5)
    for (x, y, w, h) in face:
        # vẽ 1 đường bao quanh khuôn mặt
        cv2.rectangle(img, (x, y), (x + w, y + h), (255, 255, 255), 2)
    # hiển thị ảnh nhận diện
    cv2.imshow('frame', img)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()


#!/usr/bin/env python3

import numpy as np
import matplotlib.pyplot as plt

from fastslow import *

if __name__ == '__main__':

    nph, nth = 800, 400
    Q = 1
    k = 90
    a = 1

    f, ffast, fslow = fastslow(
        Q=Q, k=k, a=a, nph=nph, nth=nth, ph0=0, th0=np.pi/2)

    fax = f[int(nth/2), :]
    fslowax = fslow[int(nth/2), :]
    ph = np.linspace(0, 2*np.pi, nph, endpoint=False)

    n0 = int(nph/2)

    fig, ax = plt.subplots(figsize=(5, 2))

    ax.plot(
        ph[0:n0],
        np.fft.fftshift(np.real(fax))[0:n0],
        'b')

    ax.plot(
        ph[0:n0],
        np.real(np.fft.fftshift(np.real(fslowax))[0:n0]),
        'g')

    ax.set_xlim(0, ph[n0 - 1]);
    ax.set_xticks([ph[0], ph[int(n0/2)], ph[n0 - 1]])
    ax.set_xticklabels([r'$0$', r'$\pi/2$', r'$\pi$'])
    ax.legend([r'$\psi$', r'$\psi_{slow}$'])

    fig.subplots_adjust(0.07, 0.12, 0.99, 0.99)
    fig.savefig('axial.eps')
    fig.show()

    fig, ax = plt.subplots(figsize=(5, 2))

    ax.plot(
        np.arange(n0),
        np.real(np.fft.fft(np.fft.fftshift(np.real(fax)))[0:n0])/nph,
        'b', zorder=9)

    ax.plot(
        np.arange(n0),
        np.real(np.fft.fft(np.fft.fftshift(np.real(fslowax)))[0:n0])/nph,
        'g', zorder=10)

    ax.add_line(plt.Line2D([k, k], ax.get_ylim(), 1, '--', 'k'))

    ax.set_xlim(0, n0 - 1)
    ax.legend([r'$\mathcal{F}(\psi)$', r'$\mathcal{F}(\psi_{slow})$'])

    fig.subplots_adjust(0.09, 0.12, 0.99, 0.99)
    fig.savefig('axialdft.eps')
    fig.show()


import requests

from bs4 import BeautifulSoup

user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Safari/537.36'

headers = {'User-Agent': user_agent}

url = 'https://www.ptt.cc/bbs/movie/M.1578413203.A.1DB.html'

### 宣告cookie
ss = requests.session()
ss.cookies['over18'] = '1'

res = ss.get(url, headers=headers)
c = ss.cookies.get_dict()

soup = BeautifulSoup(res.text, 'html.parser')

print(soup)


from base_page import BasePage
from locators import SignUpPageLocators
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

class SignUpPage(BasePage):
    """Sign Up page action methods come here."""

    def verify_apple_button(self):
        """Verifies Continue with Apple button is displayed on the page."""
        WebDriverWait(self.driver, 15).until(
            EC.presence_of_element_located((By.ID, "appleid-signin"))
        )
        apple_id_button = self.driver.find_element(*SignUpPageLocators.APPLE_SIGN_UP_BUTTON)
        apple_id_button.is_displayed()

    def continue_with_invalid_email(self):
        """Verifies Invalid Email error message is displayed on the page."""
        continue_button = self.driver.find_element(*SignUpPageLocators.CONTINUE_BUTTON)
        continue_button.click()
        error_message = self.driver.find_element(*SignUpPageLocators.INVALID_EMAIL_ERROR_MESSAGE)
        error_message.is_displayed()


import pyrealsense2 as rs
import numpy as np
import cv2

# Start streaming

# Create a pipeline
pipeline = rs.pipeline()
config = rs.config()
config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30)
config.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 30)

profile = pipeline.start(config)
# Getting the depth sensor's depth scale (see rs-align example for explanation)
# depth_sensor = profile.get_device().first_depth_sensor()
# depth_scale = depth_sensor.get_depth_scale()
# print("Depth Scale is: " , depth_scale)

depth_sensor = profile.get_device().first_depth_sensor()

laser_pwr = depth_sensor.get_option(rs.option.laser_power)
print("laser power = ", laser_pwr)
# print(depth_sensor.get_option_range(rs.option.min_distance))
# print("laser power range = " , laser_range.min , "~", laser_range.max)
set_laser = 8
# depth_sensor.set_option(rs.option.min_distance, 0.25)
depth_scale = depth_sensor.get_depth_scale()
print(depth_sensor.get_depth_scale())
title = 'mouse event'

try:
    while True:

        # Wait for a coherent pair of frames: depth and color
        frames = pipeline.wait_for_frames()
        depth_frame = frames.get_depth_frame()
        color_frame = frames.get_color_frame()
        if not depth_frame or not color_frame:
            continue

        # Convert images to numpy arrays
        depth_image = np.asanyarray(depth_frame.get_data())
        color_image = np.asanyarray(color_frame.get_data())
        # depth_image_3d = np.dstack((depth_image,depth_image,depth_image))
        # print(np.shape(depth_image))
        # print(np.shape(color_image))
        depth_colormap = cv2.applyColorMap(cv2.convertScaleAbs(depth_image, alpha=0.03), cv2.COLORMAP_JET)
        # depth_colormap_3d = np.dstack((depth_colormap,depth_colormap,depth_colormap))
        # depth_colormap = cv2.cvtColor(np.float32(depth_colormap))
                # cv2.imshow(title, color_frame)
        # cv2.imshow(title,color_image)
        cv2.imshow(title,color_image)

       #
        def onMouse(event, x, y, flags, params):
            if event == cv2.EVENT_RBUTTONDOWN:
                depth = depth_image[y,x].astype(float)
                print(y,x)
                distance = depth * depth_scale
                print ("Distance (m): ", distance)

        cv2.setMouseCallback(title,onMouse)

        key  = cv2.waitKey(10)
        if key == 27:
            cv2.imwrite('C:/Users/jyj98/Desktop/Realsense/color_sensor.jpg',color_image)
            cv2.imwrite('C:/Users/jyj98/Desktop/Realsense/depth_sensor.jpg',depth_colormap)
            break
finally:
    cv2.destroyAllWindows()


import torch
import torchvision.utils as vutils
import numpy as np
import torchvision.models as models
from torchvision import datasets
from tensorboardX import SummaryWriter
from torch.autograd import Variable

writer=SummaryWriter()

for n_iter in range(100,120):

    dummy_s1=torch.rand(1)
    dummy_s2=torch.rand(1)

    # data groupping by 'slash'
    writer.add_scalar('data/scalar1', dummy_s1[0], n_iter)
    writer.add_scalar('data/scalar2', dummy_s2[0], n_iter)

    writer.add_scalars('data/scalar_group', {'xsinx': n_iter* np.sin(n_iter),
                                             'xcosx': n_iter*np.cos(n_iter),
                                             'arctanx': np.arctan(n_iter)}, n_iter)
    dummy_img=torch.rand(32,3,64,64) # output from network
    if n_iter % 10 == 0:
        x=vutils.make_grid(dummy_img, normalize=True, scale_each=True)
        writer.add_image('Image', x, n_iter)

        writer.add_text('Text', 'text logged at step:'+str(n_iter), n_iter)
        writer.add_text('Text', ' new text logged at step:' + str(n_iter), n_iter)

        # needs tensorboard 0.4RC or later
        writer.add_pr_curve('xoxo', np.random.randint(2, size=100), np.random.rand(100), n_iter)

# export scalar data to JSON for external processing
writer.export_scalars_to_json('../logs/all_scalars.json')
writer.close()


import argparse
from PIL import Image
import cv2
from face.yolo import YOLO
from detect_open_mouth import Detect_mouth_opening
from emotion_pred import emotion
from Gender_detection import gender
from eyeglass_detector import glasses

#####################################################################
def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--model', type=str, default='model-weights/YOLO_Face.h5',
                        help='path to model weights file')
    parser.add_argument('--anchors', type=str, default='yolo_anchors.txt',
                        help='path to anchor definitions')
    parser.add_argument('--classes', type=str, default='face_classes.txt',
                        help='path to class definitions')
    parser.add_argument('--score', type=float, default=0.5,
                        help='the score threshold')
    parser.add_argument('--iou', type=float, default=0.45,
                        help='the iou threshold')
    parser.add_argument('--img-size', type=list, action='store',
                        default=(416, 416), help='input image size')
    args = parser.parse_args()
    return args

args = get_args()
data=input('Enter Image Name:')
photo=cv2.imread(data)
i=Image.open(data)
yolo=YOLO(args)
faces=yolo.detect_image(i)
print(faces,type(faces))
for y1,x1,y2,x2 in faces:
    sex,gender_prob=gender(photo,x1,y1,x2,y2)
    pro="He"if sex=="man" else "she"
    looks,prob=emotion(photo,x1,y1,x2,y2)
    print("Gender :",sex,"      ",gender_prob)
    print(pro,"looks",looks,"      ",prob)
    print("Mouth :",Detect_mouth_opening(photo[y1:y2,x1:x2],x1,y1,x2,y2))
    print("Glasses :",glasses(photo.copy(),x1,y1,x2,y2))
    cv2.imshow('output',photo[y1:y2,x1:x2])
    full=cv2.rectangle(photo,(x1,y1),(x2,y2),(255,255,255),1)
    cv2.waitKey(0)
cv2.imshow('Full',full)
cv2.waitKey(0)


from sklearn.linear_model import LinearRegression
import pytest
x = [[0], [1], [2], [3], [4], [5], [6], [7], [8], [9]]

y = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
model = LinearRegression()

model.fit(x, y)

linear_test_set = [([[10]], 12), ([[11]], 13)]

@pytest.mark.parametrize(("ip", "op"), linear_test_set)
def test_code(ip, op):
    assert int(model.predict(ip)[0]) == op


from guizero import *
import matplotlib as plt
from random import randint
import string
import numpy as np
import matplotlib.pyplot as plt
import os

coordinateX = []
coordinateY = []

def grafici():
    f = open("dati.txt", 'r')
    for riga in f:
        valori = str(riga)
        valori = valori.strip('\n')
        valori = valori.split(',')
        valori = list(valori)
        output.append(valori)
        coordinateX.append(int(valori[0]))
        coordinateY.append(int(valori[1]))

    f.close()

    coordinateX.sort()
    coordinateY.sort()

    plt.title('Grafico')
    plt.title("Grafico a dispersione", color='#007FFF')
    plt.ylabel('Asse Y')
    plt.xlabel('Asse X')
    plt.scatter(coordinateX,coordinateY,marker='*')
    plt.savefig("immagine_grafico.jpg")

def Scrittore():
    f = open('dati.txt', 'w')
    coppia = ""
    for x in range(100):
        for y in range(1):
            coppia = coppia + str(randint(1,100)) + "," + str(randint(1,100))
        coppia = coppia + "\n"
    f.write(coppia)
    f.close()

def creafile():
    if box_di_testo.value:
        Scrittore()
        nome=box_di_testo.value
        grafici()
        os.rename('dati.txt',nome)
        succes= Text(interfaccia, text=("Il file è stato generato"), font=("helvetica",40), bg="green")

    else:
        errore= Text(interfaccia, text=("Inserisci un nome!"), font=("helvetica",40), bg="red")

def mostragrafico():
    foto_grafico= Picture(interfaccia, image='immagine_grafico.jpg')
    succes2= Text(interfaccia, text=("Il grafico è stato generato"), font=("helvetica",40), bg="green")

testo_titolo="Dai un nome al file con le coppie:"
interfaccia = App(title="Generatore file di cordinate", width=1100, height=700, bg="#c0c0c0" )
inserisci_nome = Text(interfaccia, text=testo_titolo, font="helvetica", size=30)
box_di_testo= TextBox(interfaccia, width=60)
bottone_crea_file = PushButton(interfaccia, text=("Crea file"), command=creafile, width=10, height=2)
bottone_crea_file.bg="#c0c0c0"
bottone_crea_grafico = PushButton(interfaccia, text=("Crea grafico"), command=mostragrafico, width=10, height=2)
bottone_crea_grafico.bg="#c0c0c0"
output = TextBox(interfaccia, width=60, height=10, multiline=True)
interfaccia.display()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models

# Create your models here.

class Task(models.Model):
    title = models.CharField(
        max_length=1000,
        blank=False
    )
    candidate_profile = models.CharField(
        max_length=5000,
        blank=False
    )
    DIFFICULTY_CHOICES = (
        ('EG', 'Easy'),
        ('NM', 'Normal'),
        ('HD', 'Hard'),
        ('CH', 'Challenging'),
    )
    model = models.CharField(
        max_length=500,
        blank=False,
        default="model_name"
    )
    difficulty = models.CharField(
        max_length=2,
        choices=DIFFICULTY_CHOICES,
        default="EG"
    )
    description = models.CharField(
        max_length=5000,
        blank=False
    )
    direction = models.CharField(
        max_length=1000,
        blank=True,
        null=True
    )
    solving_time = models.PositiveIntegerField(
        blank=False,
        default=120
    )
    expiration_period = models.PositiveIntegerField(
        blank=False,
        default=7
    )
    full_marks = models.PositiveIntegerField(
        blank=False,
        null=False,
        default=0
    )
    is_active = models.BooleanField(
        default=True
    )
    is_private = models.BooleanField(
        default=False
    )
    file_required = models.BooleanField(
        default=True
    )
    check = models.IntegerField(
        blank=False,
        null=False,
        default=3
    )

    def __unicode__(self):
        return "{} with id {}".format(self.title, self.id)

    def __str__(self):
        return "{} with id {}".format(self.title, self.id)

class DefaultJobTask(models.Model):
    job_title = models.CharField(
        max_length=100,
        blank=False
    )
    experience = models.CharField(
        max_length=100,
        blank=True,
        null=True
    )
    task = models.ForeignKey(
        Task,
        on_delete=models.PROTECT
    )

class TaskTags(models.Model):
    task = models.ForeignKey(
        Task,
        on_delete=models.PROTECT
    )
    tag = models.CharField(
        max_length=100,
        blank=False
    )

class TaskFiles(models.Model):
    task = models.ForeignKey(
        Task,
        on_delete=models.PROTECT
    )
    file_url = models.CharField(
        max_length=100,
        blank=False
    )
    file_name = models.CharField(
        max_length=100,
        blank=False
    )

class Question(models.Model):
    question = models.CharField(
        max_length=1000,
        blank=True,
        null=True
    )
    task = models.ForeignKey(
        Task,
        on_delete=models.PROTECT
    )
    QUESTION_TYPE_CHOICES = (
        ('FB', 'Fill In The Blanks'),
        ('MC', 'Multiple Choices'),
        ('UA', 'Upload Answers'),
        ('PR', 'Paragraphs'),
        ('MF', 'Multiple Fill In The Blanks'),
    )
    question_type = models.CharField(
        max_length=2,
        choices=QUESTION_TYPE_CHOICES,
        default="FB"
    )
    weight = models.PositiveIntegerField(
        null=True,
        blank=True,
    )
    cell = models.CharField(
        max_length=10,
        blank=True,
        null=True,
    )

    def __unicode__(self):
        return self.question

    def is_null(self):
        if not self.question:
            return True
        return False

    def is_objective(self):
        if self.question_type == 'FB' or self.question_type == 'MC' or self.question_type == 'MF':
            return True
        return False

class InnerQuestion(models.Model):
    main_question = models.ForeignKey(
        Question,
        on_delete=models.CASCADE
    )
    question = models.CharField(
        max_length=100,
        blank=False
    )
    weight = models.PositiveIntegerField(
        null=True,
    )
    cell = models.CharField(
        max_length=10,
    )

    def __str__(self):
        return self.question

class QuestionOption(models.Model):
    question = models.ForeignKey(
        Question,
        on_delete=models.CASCADE
    )
    option = models.CharField(
        max_length=100,
        blank=False
    )
    point = models.PositiveIntegerField(
        default=0
    )

    def __str__(self):
        return self.option

class TaskInputSet(models.Model):
    task = models.ForeignKey(
        Task,
        on_delete=models.CASCADE
    )
    is_active = models.BooleanField(default=True)
    is_demo = models.BooleanField(default=False)

    def __unicode__(self):
        return str(self.id)

class TaskInput(models.Model):
    task_input_set = models.ForeignKey(
        TaskInputSet,
        on_delete=models.CASCADE
    )
    input = models.CharField(
        max_length=100,
        blank=False
    )
    input_value = models.DecimalField(
        max_digits=20,
        blank=False,
        decimal_places=2
    )

class QuestionAnswer(models.Model):
    task_input_set = models.ForeignKey(
        TaskInputSet,
        on_delete=models.CASCADE
    )
    question = models.ForeignKey(
        Question,
        on_delete=models.CASCADE
    )
    answer = models.DecimalField(
        max_digits=20,
        blank=False,
        decimal_places=2
    )

class InnerQuestionAnswer(models.Model):
    task_input_set = models.ForeignKey(
        TaskInputSet,
        on_delete=models.CASCADE
    )
    inner_question = models.ForeignKey(
        InnerQuestion,
        on_delete=models.CASCADE
    )
    answer = models.DecimalField(
        max_digits=20,
        blank=False,
        decimal_places=2
    )


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
RUNALL = False

"""
Read in train and test as Pandas DataFrames
"""
df_train = pd.read_csv("train.csv")
df_test = pd.read_csv("test.csv")

# randomly sample a portion of df_train
# df_train = df_train.sample(n=10000)

#store gap values
Y_train = df_train.gap.values
#row where testing examples start
test_idx = df_train.shape[0]
#delete 'Id' column
df_test = df_test.drop(['Id'], axis=1)
#delete 'gap' column
df_train = df_train.drop(['gap'], axis=1)

#DataFrame with all train and test examples so we can more easily apply feature engineering on
df_all = pd.concat((df_train, df_test), axis=0)
df_all.head()

# adding features
from __future__ import print_function
from rdkit import Chem
from rdkit.Chem import Descriptors
from rdkit.Chem import rdMolDescriptors

# Feature engineering
# store smiles
smiles = df_all.smiles

start = time.time()

carbons = []
carbons = np.vstack(df_all.smiles.apply(lambda x: x.count('c')))
df_all['carbons'] = carbons

end = time.time()
print("how long", (end - start))

double = []
double = np.vstack(df_all.smiles.apply(lambda x: x.count('=')))
df_all['double'] = double

end = time.time()
print("how long", (end - start))

single = []
single = np.vstack(df_all.smiles.apply(lambda x: x.count('-')))
df_all['single'] = single

end = time.time()
print("how long", (end - start))

nitrogen = []
nitrogen = np.vstack(df_all.smiles.apply(lambda x: x.count('n')))
df_all['nitrogen'] = nitrogen

end = time.time()
print("how long", (end - start))

oxygen = []
oxygen = np.vstack(df_all.smiles.apply(lambda x: x.count('o')))
df_all['oxygen'] = oxygen

end = time.time()
print("how long", (end - start))

ccccc = []
ccccc = np.vstack(df_all.smiles.apply(lambda x: x.count('ccccc')))
df_all['ccccc'] = ccccc

end = time.time()
print("how long", (end - start))

ccc = []
ccc = np.vstack(df_all.smiles.apply(lambda x: x.count('ccc')))
df_all['ccc'] = ccc

end = time.time()
print("how long", (end - start))

Si = []
Si = np.vstack(df_all.smiles.apply(lambda x: x.count('Si')))
df_all['Si'] = Si

end = time.time()
print("how long", (end - start))

se = []
se = np.vstack(df_all.smiles.astype(str).apply(lambda x: x.count('se')))
df_all['se'] = se

end = time.time()
print("how long", (end - start))

mols = df_all.smiles.astype(str).apply(lambda x: Chem.MolFromSmiles(x))
df_all['mols'] = mols

atoms = np.vstack(df_all.mols.apply(lambda x: x.GetNumAtoms()))
df_all['atoms'] = atoms
bonds = np.vstack(df_all.mols.apply(lambda x: x.GetNumBonds()))
df_all['bonds'] = bonds

# aromaticity
aro = []
aro = np.vstack(df_all.mols.apply(lambda x: sum(int(x.GetAtomWithIdx(i).GetIsAromatic()) for i in range(x.GetNumAtoms()))))
df_all['aro'] = aro

# adding new features

# TPSA BAAAAAD
# tpsa = np.vstack(df_all.smiles.apply(lambda x: Descriptors.TPSA(Chem.MolFromSmiles(x))))
# df_all['tpsa'] = pd.DataFrame(tpsa)

# sp3 hybridization
sp3 = np.vstack(df_all.mols.apply(lambda x: rdMolDescriptors.CalcFractionCSP3(x)))
df_all['sp3'] = sp3

#Drop the 'smiles' column
df_all = df_all.drop(['smiles'], axis=1)
df_all = df_all.drop(['mols'], axis=1)
vals = df_all.values
X_train = vals[:test_idx]
X_test = vals[test_idx:]
print("Train features:", X_train.shape)
print("Train gap:", Y_train.shape)
print("Test features:", X_test.shape)

hd = np.vstack(mols.apply(lambda x: Descriptors.NumHDonors(x)))
df_all['hd'] = hd

ha = np.vstack(mols.apply(lambda x: Descriptors.NumHAcceptors(x)))
df_all['ha'] = ha

aliphcarbo = np.vstack(mols.apply(lambda x: Descriptors.NumAliphaticCarbocycles(x)))
df_all['aliphcarbo'] = aliphcarbo

aliphhetero = np.vstack(mols.apply(lambda x: Descriptors.NumAliphaticHeterocycles(x)))
df_all['aliphhetero'] = aliphhetero

aliphrings = np.vstack(mols.apply(lambda x: Descriptors.NumAliphaticRings(x)))
df_all['aliphrings'] = aliphrings

arocarbos = np.vstack(mols.apply(lambda x: Descriptors.NumAromaticCarbocycles(x)))
df_all['arocarbos'] = arocarbos

aroheteros = np.vstack(mols.apply(lambda x: Descriptors.NumAromaticHeterocycles(x)))
df_all['aroheteros'] = aroheteros

arorings = np.vstack(mols.apply(lambda x: Descriptors.NumAromaticRings(x)))
df_all['arorings'] = arorings

heteros = np.vstack(mols.apply(lambda x: Descriptors.NumHeteroatoms(x)))
df_all['heteros'] = heteros

radelecs = np.vstack(mols.apply(lambda x: Descriptors.NumRadicalElectrons(x)))
df_all['radelecs'] = radelecs

rotbonds = np.vstack(mols.apply(lambda x: Descriptors.NumRotatableBonds(x)))
df_all['rotbonds'] = rotbonds

satcarbos = np.vstack(mols.apply(lambda x: Descriptors.NumSaturatedCarbocycles(x)))
df_all['satcarbos'] = satcarbos

satheteros = np.vstack(mols.apply(lambda x: Descriptors.NumSaturatedHeterocycles(x)))
df_all['satheteros'] = satheteros

satrings = np.vstack(mols.apply(lambda x: Descriptors.NumSaturatedRings(x)))
df_all['satrings'] = satrings

valelecs = np.vstack(mols.apply(lambda x: Descriptors.NumValenceElectrons(x)))
df_all['valelecs'] = valelecs

from sklearn import linear_model
from sklearn.linear_model import LinearRegression
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor

# Models

# fn that returns trained linear function to make predictions
def train_and_test_linear_model(X_train, Y_train):
    LR = LinearRegression()
    LR.fit(X_train, Y_train)
    # return the prediction we've trained
    return LR.predict

# RIDGE REG
def train_and_test_ridge_model(alpha):
    def helper(X_train, Y_train):
        reg = linear_model.Ridge(alpha=alpha)
        reg.fit(X_train, Y_train)
        return reg.predict
    return helper

# LASSO REG
def train_and_test_lasso_model(alpha):
    def helper(X_train, Y_train):
        reg = linear_model.Lasso(alpha=alpha)
        reg.fit(X_train, Y_train)
        return reg.predict
    return helper

# ELASTICNET REG
def train_and_test_elastic_net_model(alpha):
    def helper(X_train, Y_train):
        reg = linear_model.ElasticNet(alpha=alpha)
        reg.fit(X_train, Y_train)
        return reg.predict
    return helper

# SVM REG
def train_and_test_svm_model():
    def helper(X_train, Y_train):
        reg = SVR(kernel='rbf', C=1.0, epsilon=0.2)
        reg.fit(X_train, Y_train)
        return reg.predict
    return helper

# RF pred
# fn that returns trained Random Forest to make predictions
def train_and_test_RF_model(X_train, Y_train):
    RF = RandomForestRegressor(n_estimators=20)
    RF.fit(X_train, Y_train)
    # return the prediction we've trained
    return RF.predict

from training_and_validation import select_best_model

if RUNALL:
    select_best_model(X_train, Y_train, 4, train_and_test_linear_model)

if RUNALL:
    select_best_model(X_train, Y_train, 2, train_and_test_ridge_model(0.1))

if RUNALL:
    select_best_model(X_train, Y_train, 4, train_and_test_lasso_model(0.1))

if RUNALL:
    select_best_model(X_train, Y_train, 4, train_and_test_elastic_net_model(0.1))

if RUNALL:
    select_best_model(X_train, Y_train, 4, train_and_test_svm_model())

if RUNALL:
    select_best_model(X_train, Y_train, 2, train_and_test_RF_model)

def write_to_file(filename, predictions):
    with open(filename, "w") as f:
        f.write("Id,Prediction\n")
        for i,p in enumerate(predictions):
            f.write(str(i+1) + "," + str(p) + "\n")

# write a file for ridge regression
rid_reg = linear_model.Ridge(alpha=0.1)
rid_reg.fit(X_train, Y_train)
rid_reg_pred = rid_reg.predict(X_test)
write_to_file("rid_reg_correct_more_features.csv", rid_reg_pred)
print("starting RF")
# write a file for RF
rf = RandomForestRegressor(n_estimators=20)
rf.fit(X_train, Y_train)
rf_pred = rf.predict(X_test)
write_to_file("rf_correct_more_features.csv", rf_pred)

from sklearn.cluster import KMeans
from mpl_toolkits.mplot3d import Axes3D

X_red = PCA_red(X_train)

estimators = [('k_means_15', KMeans(n_clusters=15)),
              ('k_means_iris_10', KMeans(n_clusters=10)),
              ('k_means_iris_5', KMeans(n_clusters=5))]

fignum = 1
titles = ['15 clusters', '10 clusters', '5 clusters']
for name, est in estimators:
    fig = plt.figure(fignum, figsize=(8, 6))
    ax = Axes3D(fig, rect=[0, 0, .95, 1], elev=48, azim=134)
    est.fit(X_red)
    labels = est.labels_
    ax.scatter(X_red[:, 1], X_red[:, 0], X_red[:, 2],
               c=labels.astype(np.float), edgecolor='k')

    ax.w_xaxis.set_ticklabels([])
    ax.w_yaxis.set_ticklabels([])
    ax.w_zaxis.set_ticklabels([])
    ax.set_xlabel('PC2')
    ax.set_ylabel('PC1')
    ax.set_zlabel('PC3')
    ax.set_title(titles[fignum - 1])
    ax.dist = 12
    fignum = fignum + 1

# Plot the ground truth
fig = plt.figure(fignum, figsize=(8, 6))
ax = Axes3D(fig, rect=[0, 0, .95, 1], elev=48, azim=134)

#labels = Y_train*10-10
#y = labels.astype(np.float)

ax.scatter(X_red[:, 1], X_red[:, 0], X_red[:, 2], c=Y_train, edgecolor='k')

ax.w_xaxis.set_ticklabels([])
ax.w_yaxis.set_ticklabels([])
ax.w_zaxis.set_ticklabels([])
ax.set_xlabel('PCA 2')
ax.set_ylabel('PCA 1')
ax.set_zlabel('PCA 3')
ax.set_title('Real Y Train')
ax.dist = 12

plt.show()

# PCA theory and learning:
from sklearn.preprocessing import StandardScaler

# standardize data to mean=0, var=1
X_std = StandardScaler().fit_transform(X_train)

# compute covariance matrix
cov_mat = np.cov(X_std.T)

#eigendecomposition - make real
eig_vals, eig_vecs = np.linalg.eig(cov_mat)
eig_vals = np.real(eig_vals)
eig_vecs = np.real(eig_vecs)

# Singular Vector Decomposition
u, s, v = np.linalg.svd(X_std.T)

# select principal components
eig_pairs = [(np.abs(eig_vals[i]), eig_vecs[:,i]) for i in range(len(eig_vals))]

eig_pairs.sort(key=lambda x: x[0])
eig_pairs.reverse()
eig_vals_sorted = [x[0] for x in eig_pairs]

idx = [i for i in range(len(eig_pairs)) ]

plt.figure()
plt.scatter(idx, eig_vals_sorted)
plt.show()

import numpy as np
from sklearn.model_selection import KFold

# Validation

# to compute RSM error
def compute_error(pred, actual):
    num_pts = float(pred.shape[0])
    # RSM error
    return np.sqrt(np.sum(np.square(np.subtract(pred, actual))) / num_pts)

# estimate model performance using validation sets
def select_best_model(X, Y, num_validation_sets, model_fn):
    # error accumulator
    error = 0
    # set up the kfold operation
    kf = KFold(n_splits=num_validation_sets)
    kf.get_n_splits(X)
    for train_index, validation_index in kf.split(X):
        print("next validation set")
        # get train and validate data
        X_train, X_validation = X[train_index], X[validation_index]
        Y_train, Y_validation = Y[train_index], Y[validation_index]
        # train, should return a function to make predictions
        print("training")
        trained_fn = model_fn(X_train, Y_train)
        # get error
        pred = trained_fn(X_validation)
        print("got a prediction!")
        error += compute_error(pred, Y_validation)
    return error / float(num_validation_sets)