import datetime

from django.conf import settings
from django.http import HttpResponseForbidden
from django.shortcuts import render, redirect
from django.template import Template, Context
from django.utils.timezone import now

from spoilr.core.api.cache import memoized_cache
from spoilr.core.api.events import dispatch, HuntEvent
from spoilr.hints.models import HintSubmission
from spoilr.core.models import PuzzleAccess, HuntSetting

from hunt.app.core.assets.resolvers import create_round_url_resolver
from hunt.app.core.assets.rewriter import rewrite_relative_paths
from hunt.app.core.constants import ROUND_RD0_URL
from hunt.data_loader.round import get_round_data_text
from hunt.deploy.util import is_it_hunt, are_hunt_hints_released, is_hunt_complete

from .common import should_show_solutions, require_puzzle_access, get_puzzle_context, xframe_sameorigin_if_post

@require_puzzle_access(allow_rd0_access=False, require_access=True, skip_cache=True)
@xframe_sameorigin_if_post
def hints_view(request, embed):
    hints_unlocked = are_hints_available_for_puzzle(request.puzzle)
    time_left = None
    at_quota = False
    if hints_unlocked:
        time_left, at_quota = are_hints_available_for_team(request.team, request.puzzle, request.puzzle_access)

    hints_available = hints_unlocked and not time_left and not at_quota

    if request.method == 'POST':
        if not hints_available:
            return redirect(request.get_full_path())
        return hints_post_view(request, request.team, request.puzzle)

    round_url_resolver = create_round_url_resolver(request.puzzle.round.url, 'round')
    common_style = get_round_data_text(request.puzzle.round.url, 'round_common.css')

    hints = HintSubmission.objects.filter(team=request.team, puzzle=request.puzzle)
    context = get_puzzle_context(request.team, request.puzzle)
    context['hints'] = hints
    context['solved'] = request.puzzle_access.solved
    context['email'] = request.team.team_email
    context['hints_available'] = hints_available
    context['time_left'] = time_left
    context['at_quota'] = at_quota

    context_obj = Context(context)
    context['round_common_style'] = rewrite_relative_paths(
        Template(common_style).render(context_obj), round_url_resolver) if common_style else None

    return render(request, 'puzzle/hints_embed.tmpl' if embed else 'puzzle/hints.tmpl', context)

def hints_post_view(request, team, puzzle):
    assert not is_hunt_complete(), 'Can no longer submit hint requests'

    question = request.POST['question'].strip()
    email = request.POST['email']

    if len(question):
        hint, created = (HintSubmission.objects
            .update_or_create(team=team, puzzle=puzzle, question__iexact=question, defaults={
                'question': question,
                'email': email,
            }))
        if created:
            dispatch(
                HuntEvent.HINT_REQUESTED, team=team, puzzle=puzzle, hint=hint, object_id=puzzle.url,
                message=f'{team.name} asked “{question}” about {puzzle}')
    return redirect(request.get_full_path())

def are_hints_available_for_puzzle(puzzle):
    if not is_it_hunt():
        return False
    if puzzle.round.url == ROUND_RD0_URL:
        return False
    if are_hunt_hints_released() or puzzle.puzzledata.hints_force_released:
        return True
    return puzzle.puzzledata.hints_release_delay != None

def are_hints_available_for_team(team, puzzle, puzzle_access):
    at_quota = False
    time_left = None
    solved_puzzle_ids = PuzzleAccess.objects.filter(team=team, solved=True).values_list('puzzle_id', flat=True)
    open_hints_count = HintSubmission.objects.filter(team=team, resolved_time__isnull=True).exclude(puzzle__in=solved_puzzle_ids).count()
    quota = settings.HUNT_OPEN_HINTS_EXPANDED_QUOTA if are_hunt_hints_released() else settings.HUNT_OPEN_HINTS_QUOTA
    if open_hints_count >= quota:
        at_quota = True

    if not are_hunt_hints_released():
        available_time = now() - puzzle_access.timestamp
        if puzzle.puzzledata.hints_release_delay and puzzle.puzzledata.hints_release_delay > available_time:
            time_left = puzzle.puzzledata.hints_release_delay - available_time
    return time_left, at_quota


import torch
import torch.nn as nn
from torch.autograd import Function
from torchsummary import summary

def weight_init(model):

    classname = model.__class__.__name__
    if classname.find('Conv') != -1:
        nn.init.xavier_normal_(model.weight.data)
        model.bias.data.fill_(0)

class VectorQuantization(Function):

    @staticmethod
    def forward(ctx, inputs, codebook):
        """

        Parameters
        ----------
        ctx : context. It is used to store arbitrary data that can be retrieved
              during the backward pass.
        inputs : Tensor
            Encoded image vector(z_e_x).
        codebook : Tensor
            Embeddings.

        Returns
        -------
        idx : quantized indices / latents.

        """

        with torch.no_grad():

            input_size = inputs.size()
            emb_size = codebook.size(1)
            flat_input = inputs.view(-1, emb_size)

            codebook_sq = torch.sum(codebook ** 2, dim = 1)
            inputs_sq = torch.sum(inputs ** 2, dim = 1, keepdim = True)
            #torch.addmm: result = beta * input + alpha * (mat1[i] @ mat2[i])
            l2_dis = torch.addmm(input = codebook_sq + inputs_sq,
                                 mat1 = flat_input, mat2 = codebook.t(),
                                 alpha = 2.0, beta = 1.0)
            _, idx_flatten = torch.min(l2_dis, dim = 1)
            idx = idx_flatten(*input_size[:-1])
            ctx.mark_non_differentiable(idx)

            return idx

    @staticmethod
    def backward(ctx, grad_outputs):

        raise RuntimeError('Trying to call backward on graph containing `Vector Quantization`'
                           'which is non-differentiable. Use VQStraightThrough instead.')

class VQStraightThrough(Function):

    @staticmethod
    def forward(ctx, inputs, codebook):

        idx = VQ(inputs, codebook)
        flat_idx = idx.view(-1)
        ctx.save_for_backward(flat_idx, codebook)
        ctx.mark_non_differentiable(flat_idx)
        codes_flatten = torch.index_select(input = codebook, dim = 0, index = flat_idx)
        codes = codes_flatten.view_as(inputs)

        return (codes, flat_idx)

    @staticmethod
    def backward(ctx, grad_outputs, grad_indices):

        grad_inputs, grad_codebook = None, None

        if ctx.needs_input_grad[0]:

            grad_inputs = grad_outputs.clone()

        if ctx.needs_input_grad[1]:

            idx, codebook = ctx.saved_tensors
            emb_size = codebook.size(1)
            flat_grad_output = (grad_outputs.contiguous().view(-1, emb_size))
            grad_codebook = torch.zeros_like(codebook)
            grad_codebook.index_add_(0, idx, flat_grad_output)

        return (grad_inputs, grad_codebook)

VQ = VectorQuantization.apply
VQ_ST =  VQStraightThrough.apply

class VQEmbedding(nn.Module):

    def __init__(self, K: int, D: int):

        """

        Parameters
        ----------
        K : int
            Total number of embeddings in codebook.
        D : int
            Dimensions of every embedding in the codebook.

        """

        super().__init__()
        self.vq_embs = nn.Embedding(K, D)
        self.vq_embs.weight.data.uniform_(-1.0 / K, 1.0 / K)

    def forward(self, z_e_x):

        z_e_x = z_e_x.permute(0,2,3,1).contiguous()
        latents = VQ(z_e_x, self.vq_embs.weight)
        return latents

    def straight_through_forward(self, z_e_x):

        zex = z_e_x.permute(0,2,3,1).contiguous()
        z_q_x, idx = VQ_ST(zex, self.vq_embs.weight.detach())
        z_q_x = z_q_x.permute(0,3,1,2).contiguous()
        flat_zqx_tilde = torch.index_select(self.vq_embs.weight, dim = 0, index = idx)
        zqx_tilde = flat_zqx_tilde.view_as(zex)
        zqx_tilde = zqx_tilde.permute(0,3,1,2).contiguous()
        return z_q_x, zqx_tilde

class ResidualBlock(nn.Module):

    def __init__(self, channels):

        super().__init__()
        self.resblock = nn.Sequential(nn.ReLU(inplace = True),
                                      nn.Conv2d(channels, channels, kernel_size = 3, stride = 1, padding = 1),
                                      nn.BatchNorm2d(channels),
                                      nn.ReLU(inplace = True),
                                      nn.Conv2d(channels, channels, 1),
                                      nn.BatchNorm2d(channels)
                                      )

    def forward(self, x):

        return self.resblock(x) + x

class VQVAE(nn.Module):

    def __init__(self, in_c, out_c, K = 512):

        super().__init__()
        self.encoder = nn.Sequential(nn.Conv2d(in_c, out_c, kernel_size = 4, stride = 2, padding = 1),
                                     nn.BatchNorm2d(out_c),
                                     nn.ReLU(inplace = True),
                                     nn.Conv2d(out_c, out_c, kernel_size = 4, stride = 2, padding = 1),
                                     ResidualBlock(out_c),
                                     ResidualBlock(out_c)
                                     )
        self.codebook = VQEmbedding(K, D = out_c)

        self.decoder = nn.Sequential(ResidualBlock(out_c),
                                     ResidualBlock(out_c),
                                     nn.ReLU(inplace = True),
                                     nn.ConvTranspose2d(out_c, out_c, kernel_size = 4, stride = 2, padding = 1),
                                     nn.BatchNorm2d(out_c),
                                     nn.ReLU(inplace = True),
                                     nn.ConvTranspose2d(out_c, in_c, kernel_size = 4, stride = 2, padding = 1),
                                     nn.Tanh()
                                     )

        self.apply(weight_init)

    def encode(self, x):

        z_e_x = self.encoder(x)
        latents = self.codebook(z_e_x)
        return latents

    def decode(self, latents):

        z_q_x = self.codebook.vq_embs(latents).permute(0,3,1,2)
        x_tilde = self.decoder(z_q_x)
        return x_tilde

    def forward(self, x):

        z_e_x = self.encoder(x)
        z_q_x_st, z_q_x = self.codebook.straight_through_forward(z_e_x)
        x_tilde = self.decoder(z_q_x_st)
        return z_e_x, z_q_x, x_tilde

if __name__ == '__main__':

    vqvae = VQVAE(3, 256).to(torch.device('cuda'))
    print(summary(vqvae, (3,32,32)))


import logging
import glob
import os
import itertools

import sbol2
import sbol3
from sbol_utilities.conversion import convert2to3

from sbol_utilities.helper_functions import GENETIC_DESIGN_FILE_TYPES

def convert_package_sbol2_files(package: str) -> dict[str, str]:
    """Find all SBOL2 import files in a package directory and convert them to SBOL3 sorted n-triples

    :param package: path of package to search
    :return: dictionary mapping paths of SBOL2 inputs to SBOL3 outputs
    """
    mappings = {}

    # import SBOL2
    for file in itertools.chain(
        *(glob.glob(os.path.join(package, f'*{ext}')) for ext in GENETIC_DESIGN_FILE_TYPES['SBOL2'])
    ):
        print(f'Attempting to convert SBOL2 file {file}')
        file3 = os.path.splitext(file)[0] + '.nt'  # make an SBOL3 version of the file name
        doc2 = sbol2.Document()
        doc2.read(file)  # confirm that it is, in fact, an SBOL2 file
        doc3 = convert2to3(file)
        # check if it's valid before writing
        report = doc3.validate()
        if len(report.errors) > 0:
            logging.warning('Conversion failed: SBOL3 file has errors')
            for issue in report.errors:
                logging.warning(issue)
            continue

        # If there was a previous instance of the file, merge in all non-replaced objects
        if os.path.isfile(file3):
            merge_doc = sbol3.Document()
            merge_doc.read(file3)
            # figure out which ones to copy
            non_replaced = set(o.identity for o in merge_doc.objects) - set(o.identity for o in doc3.objects)
            for o in merge_doc.objects:
                if o.identity in non_replaced:
                    o.copy(doc3)

        print(f'Writing converted SBOL3 file to {file3}')
        doc3.write(file3, sbol3.SORTED_NTRIPLES)
        # record the conversion for later use
        mappings[file] = file3

    return mappings


#!/usr/bin/env python3

import argparse
import subprocess

def get_argument_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        description="Uses hunspell to spell check the given file while also taking a file with allowed words.")

    parser.add_argument("file", help="The file to spell check.")
    parser.add_argument("-a", "--allowed",
                        help="Allow the words in the file. Each word should be on a separate line.")

    return parser

def main() -> None:
    DEFAULT_WORDLIST = "wordlist.txt"

    args = get_argument_parser().parse_args()

    wordlist = args.allowed if args.allowed else DEFAULT_WORDLIST
    command = ["bash", "-c",
               "cat {} | hunspell -l -t -d en_GB,hu_HU -i utf-8 | grep -v -x -F -f {}".format(
                   args.file,
                   wordlist)]

    subprocess.run(command)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
# coding:utf-8
import argparse
import re
import os
from qiime2 import Artifact
from pandas import *
# import pdb
# argument:
p = argparse.ArgumentParser(description="This script is used to plot phylogentic tree of top abundant otus")
p.add_argument('-i', '--input', dest='otu', metavar='<path>', default=False,
               help='table.qza')
p.add_argument('-m', '--metadata', dest='map', metavar='<path>', default=False,
               help='Specify metadata with sample id at first column')
p.add_argument('-c', '--category', dest='group', metavar='<str>', default=False,
               help='Column name of group in mapping file, according to this group, the missing samples and zero features will be removed')
p.add_argument('-r', '--repseqs', dest='rep', metavar='<path>', default=False,
               help='representative seqs of features (.qza)')
p.add_argument('-t', '--taxonomy', dest='taxon', metavar='<path>', default=False,
               help='table.qza')
p.add_argument('-f', '--filter', dest='filter', metavar='<flag:taxon1,taxon2>', default=False,
               help='Taxon to be filtered. use "keep:taxon1,taxon2" to keep noly taxon1, taxon2. or use "exclude:taxon1,taxon2" to filter taxon1, taxon2.')
p.add_argument('--min-abundance', dest='mina', metavar='int', default=1,
               help='pass this to filter out the otu table and representative sequences less than the specified value')
p.add_argument('-o', '--out', dest='out', metavar='<directory>',
               default='./', help='Specify the output directory')
p.add_argument('--raw', metavar='<directory>',
               default=None, help='output csv tables before filter')
p.add_argument('--tsv', metavar='<path>',
               default=None, help='output otu table with taxonomy at last column')
p.add_argument('--consensus', metavar='<path>',
               default=None, help='output otu table with Consensus Lineage at last column')
p.add_argument('--even', metavar='<path>',
               default=None, help='output even otu table with taxonomy at last column')
p.add_argument('--evenconsensus', metavar='<path>',
               default=None, help='output even otu table with Consensus Lineage at last column')
p.add_argument('--fmap', metavar='<path>',
               default=None, help='output mapping file after filter')
p.add_argument('--odepth', metavar='<path>',
               default=None, help='Output depth used in even tabale calculation')
opt = p.parse_args()
if not os.path.exists(opt.out):
    os.makedirs(opt.out)
opt.out = opt.out + '/'

def iterin(a, b):
    return list(map(lambda x: x in list(b), a))

def iterfind(a, b):
    out = []
    for i in a:
        found = False
        for j in b:
            if re.search(j, i, flags=re.IGNORECASE):
                found = True
                break
        out.append(found)
    return out

def clean_tax(tax):
    for ambi in ['uncultured', 'Ambiguous', 'unidentified', 'unassigned', 'unclassified']:
        tax = re.sub(';[^;]*' + ambi + '.*', '', tax, flags=re.I)
    tax = re.sub('; ', ';', tax)
    tax = re.sub(r'[\-| ]+', '_', tax)
    tax = re.sub(';$', '', tax)
    return(tax)

def ch_col(df, index, val):
    columns = list(df.columns)
    columns[index] = val
    df.columns = columns
    return df

# load otu table from qiime2 Artifact
if opt.otu.endswith('.qza'):
    otu = Artifact.load(opt.otu)
    otu = otu.view(DataFrame)
    taxon = Artifact.load(opt.taxon)
    taxon = taxon.view(DataFrame)
# otherwise load otu table from tsv table
else:
    skip_rows = []
    with open(opt.otu) as f:
        for num, line in enumerate(f):
            if len(line.split('\t')) < 2:
                skip_rows.append(num)
    otu = read_csv(opt.otu, sep='\t', skiprows=skip_rows, index_col=0)
    otu_t = otu.iloc[:, -1]
    taxon = DataFrame({
        'Taxon': otu_t.copy(),
        'Confidence': [0.9] * otu_t.shape[0]
    }, index=otu_t.index.copy())
    taxon.index.set_names("Feature ID", inplace=True)
    otu.drop(otu.columns[-1], axis=1, inplace=True)
    otu = otu.T
    otu.columns.set_names(None, inplace=True)

if opt.raw:
    otu.T.to_csv(os.path.join(opt.raw, "otu_tale_before_filter.csv"))
    taxon.to_csv(os.path.join(opt.raw, "taxonomy_before_filter.csv"))

# load rep seqs from qiime2 Artifact
if opt.rep.endswith('.qza'):
    rep = Artifact.load(opt.rep)
# otherwise import rep seqs from fasta
else:
    rep = Artifact.import_data('FeatureData[Sequence]', opt.rep)
rep = rep.view(Series)

if opt.map and opt.group:
    # load mapping file
    mapd = read_csv(opt.map, sep='\t')
    # filter mapping file
    mapd = mapd.loc[mapd[opt.group].notna(), :]
    mapd = mapd.loc[:, mapd.notna().sum() > 1]
    mapd.to_csv(
        opt.fmap if opt.fmap else os.path.join(opt.out, 'mapping_file.txt'),
        sep='\t', index=False
    )
    # filter otu table
    otu = otu.loc[iterin(otu.index, mapd.iloc[:, 0]), :]

otu = otu.loc[:, otu.sum() >= int(opt.mina)]
otu_set1 = otu.columns

# pdb.set_trace()

# filter taxa
if opt.filter:
    flag, ftaxa = opt.filter.split(':')
    ftaxa = ftaxa.split(',')
    if flag == "keep":
        taxon = taxon.loc[iterfind(taxon.iloc[:, 0], ftaxa), :]
    else:
        taxon = taxon.loc[[not b for b in iterfind(taxon.iloc[:, 0], ftaxa)], :]

# clean taxa
taxon.iloc[:, 0] = [clean_tax(t) for t in taxon.iloc[:, 0]]

otu_set2 = taxon.index

# calculate inner set
inner_otuset = [o for o in otu_set1 if o in otu_set2]

# loc is much more faster
# otu = otu.iloc[:, iterin(otu.columns, inner_otuset)]
otu = otu.loc[:, inner_otuset]
# taxon = taxon.iloc[iterin(taxon.index, inner_otuset), :]
taxon = taxon.loc[inner_otuset, :]
# rep = rep.iloc[iterin(rep.index, inner_otuset)]
rep = rep.loc[inner_otuset]
# save data
if opt.tsv or opt.consensus:
    df = otu.T
    df = df.join(taxon)
    df.drop(df.columns[-1], axis=1, inplace=True)
    df.index.set_names("#OTU ID", inplace=True)
    df = ch_col(df, -1, "taxonomy")
    # print(df)
    df.to_csv(opt.tsv, sep='\t')
    if opt.consensus:
        df = ch_col(df, -1, "Consensus Lineage")
        df.to_csv(opt.consensus, sep='\t')

otu_qza = Artifact.import_data("FeatureTable[Frequency]", otu)
otu_qza.save(opt.out + 'table.qza')

if opt.even or opt.evenconsensus or opt.odepth:
    from qiime2.plugins import feature_table
    depth = int(otu.T.sum().min())
    # if depth < 5000:
    #     print("Min depth among samples is too small({}); reset sample depth to: 5000".format(depth))
    #     depth = 5000
    if opt.odepth:
        with open(opt.odepth, 'w') as f:
            f.write(str(depth))
    print("sample depth is: {}".format(depth))
    if opt.even or opt.evenconsensus:
        rarefy_result = feature_table.methods.rarefy(table=otu_qza, sampling_depth=depth)
        df = rarefy_result.rarefied_table.view(DataFrame).T
        df = df.join(taxon)
        df.drop(df.columns[-1], axis=1, inplace=True)
        df.index.set_names("#OTU ID", inplace=True)
        if opt.even:
            df = ch_col(df, -1, "taxonomy")
            df.to_csv(opt.even, sep='\t')
        if opt.evenconsensus:
            df = ch_col(df, -1, "Consensus Lineage")
            df.to_csv(opt.evenconsensus, sep='\t')

rep = Artifact.import_data("FeatureData[Sequence]", rep)
rep.save(opt.out + 'rep-seqs.qza')
taxon = Artifact.import_data("FeatureData[Taxonomy]", taxon)
taxon.save(opt.out + 'taxonomy.qza')


import os
import socket

from pydantic import AmqpDsn, BaseSettings, PositiveInt
from settings.base import BaseConfig
from settings.sender import NotificationsEnricherAmqpSender
from settings.services import AuthApiSettings, TemplateApiSettings

class BaseConsumerSettings(BaseSettings):
    run: bool = True
    consumer_tag: str
    url: AmqpDsn
    queue_name: str
    exchange_name: str
    routing_key: str
    prefetch_count: PositiveInt = 5
    timeout: PositiveInt = 5

class NotificationsEnricherConfig(BaseConfig):
    # клиенты
    auth_api_client: AuthApiSettings
    notifications_enricher_amqp_sender: NotificationsEnricherAmqpSender
    # консьюмеры
    notifications_enricher_consumer: BaseConsumerSettings

class EmailConfig(BaseConfig):
    # клиенты
    template_api_client: TemplateApiSettings
    # консьюмеры
    email_consumer: BaseConsumerSettings

class ConsumersConfig(NotificationsEnricherConfig, EmailConfig):
    pass

CONSUMER_TAG = (
    os.getenv("TASK_CONSUMER_TAG", default=None) or socket.gethostname()
)

NOTIFICATIONS_ENRICHER_CONSUMER = {
    "url": os.getenv(
        "NOTIFICATIONS_ENRICHER_CONSUMER_AMQP_URL",
        default="amqp://user:pass@rabbitmq:5672/test",
    ),
    "queue_name": os.getenv(
        "NOTIFICATIONS_ENRICHER_CONSUMER_QUEUE",
        default="notifications_api.created_delivery",
    ),
    "exchange_name": os.getenv(
        "NOTIFICATIONS_ENRICHER_CONSUMER_EXCHANGE",
        default="notifications_api.created_delivery",
    ),
    "routing_key": os.getenv(
        "NOTIFICATIONS_ENRICHER_CONSUMER_ROUTING_KEY",
        default="event.created",
    ),
    "consumer_tag": CONSUMER_TAG,
}

EMAIL_CONSUMER = {
    "url": os.getenv(
        "EMAIL_CONSUMER_AMQP_URL",
        default="amqp://user:pass@rabbitmq:5672/test",
    ),
    "queue_name": os.getenv(
        "EMAIL_CONSUMER_QUEUE",
        default="notifications_sender.send_delivery",
    ),
    "exchange_name": os.getenv(
        "EMAIL_CONSUMER_EXCHANGE",
        default="notifications_sender.send_delivery",
    ),
    "routing_key": os.getenv(
        "EMAIL_CONSUMER_ROUTING_KEY",
        default="event.ready",
    ),
    "consumer_tag": CONSUMER_TAG,
}


"""
This file is part of the TheLMA (THe Laboratory Management Application) project.
See LICENSE.txt for licensing, CONTRIBUTORS.txt for contributor information.

Experiment metadata table.
"""
from datetime import datetime

from sqlalchemy import Column
from sqlalchemy import DateTime
from sqlalchemy import ForeignKey
from sqlalchemy import Integer
from sqlalchemy import String
from sqlalchemy import Table

__docformat__ = "reStructuredText en"
__all__ = ['create_table']

def create_table(metadata, subproject_tbl, experiment_metadata_type_tbl):
    "Table factory."
    tbl = Table('experiment_metadata', metadata,
                Column('experiment_metadata_id', Integer, primary_key=True),
                Column('label', String, nullable=False, unique=True),
                Column('subproject_id', Integer,
                       ForeignKey(subproject_tbl.c.subproject_id),
                       nullable=False),
                Column('number_replicates', Integer, nullable=False),
                Column('creation_date', DateTime(timezone=True),
                       default=datetime.now, nullable=False),
                Column('experiment_metadata_type_id', String(10),
                       ForeignKey(experiment_metadata_type_tbl.c.
                                            experiment_metadata_type_id),
                       nullable=False),
                Column('ticket_number', Integer, nullable=False, unique=True),
                )
    return tbl


import argparse
import os

def get_parser():

    parser = argparse.ArgumentParser()

    parser.add_argument('--save_dir', type=str, default='path/to/save/models',
                        help='path where checkpoints will be saved')

    parser.add_argument('--model_name', type=str, default='model',
                        help='save_dir/project_name/model_name will be the path where logs and checkpoints are stored')

    parser.add_argument('--suff', type=str, default='',
                        help='the id of the dictionary to load for training')

    parser.add_argument('--image_model', type=str, default='resnet50', choices=['resnet18', 'resnet50', 'resnet101',
                                                                                'resnet152', 'inception_v3'])

    parser.add_argument('--recipe1m_dir', type=str, default='path/to/recipe1m',
                        help='directory where recipe1m dataset is extracted')

    parser.add_argument('--aux_data_dir', type=str, default='../data',
                        help='path to other necessary data files (eg. vocabularies)')

    parser.add_argument('--crop_size', type=int, default=224, help='size for randomly or center cropping images')

    parser.add_argument('--image_size', type=int, default=256, help='size to rescale images')

    parser.add_argument('--log_step', type=int , default=10, help='step size for printing log info')

    parser.add_argument('--learning_rate', type=float, default=0.001,
                        help='base learning rate')

    parser.add_argument('--scale_learning_rate_cnn', type=float, default=0.01,
                        help='lr multiplier for cnn weights')

    parser.add_argument('--lr_decay_rate', type=float, default=0.99,
                        help='learning rate decay factor')

    parser.add_argument('--lr_decay_every', type=int, default=2,
                        help='frequency of learning rate decay (default is every epoch)')

    parser.add_argument('--weight_decay', type=float, default=0.)

    parser.add_argument('--embed_size', type=int, default=512,
                        help='hidden size for all projections')

    parser.add_argument('--n_att_units', type=int, default=8,
                        help='number of attention heads in the unit decoder')

    parser.add_argument('--n_att_portions', type=int, default=8,
                        help='number of attention heads in the portion decoder')

    parser.add_argument('--transf_layers_units', type=int, default=2,
                        help='number of transformer layers in the unit decoder')

    parser.add_argument('--dim_feedforward', type=int, default=256,
                        help='dimension of inner linear layer of transformer decoder last linear projection')

    parser.add_argument('--transf_layers_portions', type=int, default=6,
                        help='number of transformer layers in the portion decoder')

    parser.add_argument('--num_epochs', type=int, default=30,
                        help='maximum number of epochs')

    parser.add_argument('--batch_size', type=int, default=128)

    parser.add_argument('--ckpt_epoch', type=int, default=1)

    parser.add_argument('--num_workers', type=int, default=8)

    parser.add_argument('--dropout_encoder', type=float, default=0.3,
                        help='dropout ratio for the image and ingredient encoders')

    parser.add_argument('--dropout_decoder_r', type=float, default=0.3,
                        help='dropout ratio in the instruction decoder')

    parser.add_argument('--dropout_decoder_i', type=float, default=0.3,
                        help='dropout ratio in the ingredient decoder')

    parser.add_argument('--finetune_after', type=int, default=-1,
                        help='epoch to start training cnn. -1 is never, 0 is from the beginning')

    parser.add_argument('--patience', type=int, default=50,
                        help='maximum number of epochs to allow before early stopping')

    parser.add_argument('--maxnumingrs', type=int, default=10,
                        help='maximum number of ingredients')

    parser.add_argument('--maxnumims', type=int, default=5,
                        help='maximum number of images per sample')

    parser.add_argument('--es_metric', type=str, default='loss', choices=['loss', 'accuracy'],
                        help='early stopping metric to track')

    parser.add_argument('--eval_split', type=str, default='val')

    parser.add_argument('--numgens', type=int, default=3)

    parser.add_argument('--resume', dest='resume', action='store_true',
                        help='resume training from the checkpoint in model_name')
    parser.set_defaults(resume=False)

    parser.add_argument('--nodecay_lr', dest='decay_lr', action='store_false',
                        help='disables learning rate decay')
    parser.set_defaults(decay_lr=True)

    parser.add_argument('--load_jpeg', dest='use_lmdb', action='store_false',
                        help='if used, images are loaded from jpg files instead of lmdb')
    parser.set_defaults(use_lmdb=True)

    parser.add_argument('--image_encoder_type', type=str, default='bottom-up',
                        help='could be resnet or bottom-up or resnet_features')  # resnet, bottom-up

    parser.add_argument('--ckpt_name', type=str, default='',
                        help='the name of the checkpoint used for sample.py to load')

    parser.add_argument('--splits', nargs='+', type=str, default=["train", "val"],
                        help='splits used either [train, val] or [test]')

    parser.add_argument('--saving_epochs', nargs='+', type=float, default=[2, 5, 10, 15, 25],
                        help='epochs @ where model is saved.')

    parser.add_argument('--model_type', type=str, default='unit',
                        help='specifies what type of model is trained? 1-stream (unit), or 2-stream(joint), or jointwithcalorie')

    parser.add_argument('--loss_lambda', nargs='+', type=float, default=[1.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.01, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], help='loss tradeoff')

    parser.add_argument('--no_train_base_wcalorie', dest='train_base_wcalorie', action='store_false',
                        help='sets to train the base net with calorie loss')
    parser.set_defaults(train_base_wcalorie=True)

    parser.add_argument('--scaled_output', dest='scaled_output', action='store_true',
                        help='sets to train the base net with calorie loss')
    parser.set_defaults(scaled_output=False)

    parser.add_argument('--no_pre_load', dest='pre_load', action='store_false',
                        help='pre load bottom-up image features or not')
    parser.set_defaults(pre_load=True)

    parser.add_argument('--no_image_to_calorie_stream', dest='image_to_calorie_stream', action='store_false',
                        help='whether there is an image to calorie stream or not')
    parser.set_defaults(image_to_calorie_stream=True)

    parser.add_argument('--img_pad_size', type=int, default=50,
                        help='for bottom-up features up to 50 for images fixed 49')

    parser.add_argument('--no_warmup', dest='warmup', action='store_false',
                        help='optimizer starts with warm-up or not')
    parser.set_defaults(warmup=True)

    parser.add_argument('--warmup_epochs', type=int, default=4)

    parser.add_argument('--no_cnn_gradients', dest='keep_cnn_gradients', action='store_false',
                        help='train resnet or not when using cnn instead of bottom-up features')
    parser.set_defaults(keep_cnn_gradients=True)

    parser.add_argument('--just_store_cnn_features', dest='store_cnn_feat', action='store_true',
                        help='store cnn features')
    parser.set_defaults(store_cnn_feat=False)

    parser.add_argument('--features_dir', type=str, default='data',
                        help='path to stored cnn features')

    parser.add_argument('--calorie_range', nargs='+', type=int, default=[100, 1900],
                        help='Range of calorie for recipes.')

    parser.add_argument('--subsequent_calorie', dest='subsequent_calorie', action='store_true',
                        help='compute recipe calorie from predicted individual ingredient calories')
    parser.set_defaults(subsequent_calorie=False)

    parser.add_argument('--question_answer_path', type=str, default='/data/ajelodar/mcan-vqa-master/datasets/classes_dataset/',
                        help='path to question-answer json files')

    parser.add_argument('--calorie_dataset_path', type=str, default='/data/ajelodar/dish_classification/data/calorie_dataset',
                        help='path to calorie and dish dataset')

    parser.add_argument('--just_plot_error', dest='just_plot_error', action='store_true',
                        help='if set in sample.py just plots error without running the evaluation again')
    parser.set_defaults(just_plot_error=False)

    parser.add_argument('--use_distrib_loss', dest='use_distrib_loss', action='store_true',
                        help='use kl-divergence loss to make portion output of ingredients similar')
    parser.set_defaults(use_distrib_loss=False)

    parser.add_argument('--temp', default=4.0, type=float, help='temperature for kl loss scaling')

    parser.add_argument('--just_input_image', dest='just_input_image', action='store_true',
                        help='if using only image input')
    parser.set_defaults(just_input_image=False)

    parser.add_argument('--just_input_ingr', dest='just_input_ingr', action='store_true',
                        help='if using only ingredient input')
    parser.set_defaults(just_input_ingr=False)

    parser.add_argument('--remove_highest_errors', dest='remove_highest_errors', action='store_true',
                        help='if true removes N number of ingredients with highest error')
    parser.set_defaults(remove_highest_errors=False)

    parser.add_argument('--N_error', default=0, type=int, help='N number of ingredients')

    parser.add_argument('--pick_subset_ingrs', dest='pick_subset_ingrs', action='store_true',
                        help='if true removes only use the ingredients specified')
    parser.set_defaults(pick_subset_ingrs=False)

    parser.add_argument('--ingr_subset', nargs='+', type=str, default=["egg"],
                        help='the subset of ingredients specified to use for training.')

    parser.add_argument('--ingr_list_errors', type=str, default='data/ingr_list_errors.json',
                        help='path to a file which contains list of ingredients & their errors')

    parser.add_argument('--just_eval_subset_ingrs', action='store_true',
                        help='evaluate the model on a subset of the ingrdeints (not train time)')
    parser.set_defaults(just_eval_subset_ingrs=False)

    parser.add_argument('--pre_last_layer_size', default=128, type=int, help='pre last layer size')

    parser.add_argument('--portion_uses_ingr_only', action='store_true',
                        help='portion uses ingredient only or uses both ingr-image')
    parser.set_defaults(portion_uses_ingr_only=False)

    parser.add_argument('--do_calorie_scale', action='store_true',
                        help='calorie scaling is available or not')
    parser.set_defaults(do_calorie_scale=False)

    parser.add_argument('--do_total_calorie', action='store_true',
                        help='compute total calorie loss for simple decoder')
    parser.set_defaults(do_total_calorie=False)

    parser.add_argument('--do_prior_calorie', action='store_true',
                        help='compute prior mean calorie loss for indiv ingredient')
    parser.set_defaults(do_prior_calorie=False)

    parser.add_argument('--cnn_reduced', action='store_true',
                        help='CNN reduced')
    parser.set_defaults(cnn_reduced=False)

    parser.add_argument('--use_bert', action='store_true',
                        help='using bert embedding')
    parser.set_defaults(use_bert=False)

    parser.add_argument('--activation', type=str, default='relu', help='default activation function is relu')

    parser.add_argument('--dataset', type=str, default='recipe1m', help='dataset choice (recipe1m, menumatch)')

    parser.add_argument('--cross_valid_idx', default=0, type=int, help='cross validation iteration')

    parser.add_argument('--calorie_class_width', default=100, type=int, help='width for calorie classes')

    parser.add_argument('--model_output', type=str, default='all', choices=['all', 'just_calorie', 'just_ingr_calories', 'calories_and_calorie', 'portions_and_calorie'])

    parser.add_argument('--use_predicted', action='store_true',
                        help='using predicted ingredients or not')
    parser.set_defaults(use_predicted=False)

    parser.add_argument('--images_folder', type=str, default='/home/rpal/ahmad/personalized_unit_recognition/data/test_images',
                        help='path to a folder of images')

    parser.add_argument('--evaluate_given_images', action='store_true',
                        help='evaluate images in the test_images')
    parser.set_defaults(evaluate_given_images=False)

    args = parser.parse_args()

    return args


import csv
import collections
import os
import pathlib

RANKING_COLUMN_NAME = 'NAME'
RANKING_COLUMN_COUNT = 'COUNT'
RANKING_CSV_FILE_PATH = 'ranking.csv'

class CsvModel:
    def __init__(self, csv_file):
        self.csv_file = csv_file

        # csv ファイルが存在しないなら空のファイルを作る
        if not os.path.exists(self.csv_file):
            pathlib.Path(csv_file).touch()

class RankingModel(CsvModel):

    def __init__(self, csv_file=None, *args, **kargs):
        if not csv_file:
            csv_file = self.get_csv_file_path()
            super().__init__(csv_file)
            self.column = [RANKING_COLUMN_NAME, RANKING_COLUMN_COUNT]
            self.data = collections.defaultdict(int)
            self.load_data()

    # csv ファイルのパスを返す関数
    def get_csv_file_path(self):
        csv_file_path = None

        # settings.py にパス情報が書いてあるならここで templates フォルダのパスを取得
        try:
            import settings
            if settings.CSV_FILE_PATH:
                csv_file_path = settings.CSV_FILE_PATH
        except ImportError:
            pass

        # 書いていないなら RANKING_CSV_FILE_PATH のパスを取得
        if not csv_file_path:
            csv_file_path = RANKING_CSV_FILE_PATH
        return csv_file_path

    # データの読み込み
    def load_data(self):
        with open(self.csv_file, 'r+') as csv_file:
            reader = csv.DictReader(csv_file)
            for row in reader:
                self.data[row[RANKING_COLUMN_NAME]] = int(
                    row[RANKING_COLUMN_COUNT])
        return self.data

    # 一番人気な店の取得
    def get_most_popular(self, not_list=None):

        # リストが None ならリストの空配列を取得
        if not_list is None:
            not_list = []

        # データがない = csv に何も書き込まれていないときは None を返す
        if not self.data:
            return None

        # 辞書を値でソート(返り値はリスト)
        sorted_data = sorted(self.data, key=self.data.get, reverse=True)

        # not_list にない店を取得して返す
        for name in sorted_data:
            if name in not_list:
                continue
            return name

    # ユーザーの好きな店の Count をインクリメントする関数
    def increment(self, name):
        self.data[name.title()] += 1
        self.save()

    # csv ファイルに値を保存する関数
    def save(self):
        with open(self.csv_file, 'w+') as csv_file:
            writer = csv.DictWriter(csv_file, fieldnames=self.column)
            writer.writeheader()

            for name, count in self.data.items():
                writer.writerow({
                    RANKING_COLUMN_NAME: name,
                    RANKING_COLUMN_COUNT: count
                })


'''Código para as páginas e ações da área logada do usúario'''

from flask import Blueprint, render_template, request, flash, redirect, url_for
from flask_login import login_required, current_user
from .models import Lista, Aluno, Trabalho, NotaLista, NotaTrabalho
from . import db

pags = Blueprint('pags', __name__)

@pags.route('/', methods=['GET', 'POST'])
def home():
    '''Função para a página inicial'''
    return render_template("home.html", user=current_user)

@pags.route('/areaLogada', methods=['GET', 'POST'])
@login_required
def areaLogada():
    '''Função para a área logada'''
    nome_usuario = current_user.nome

    if request.method == 'POST':
        if request.form['botao'] == 'Adicionar lista':
            return redirect(url_for('pags.addLista'))

        elif request.form['botao'] == 'Adicionar nota da lista':
            return redirect(url_for('pags.addNotaLista'))

        elif request.form['botao'] == 'Adicionar nota do trabalho':
            return redirect(url_for('pags.addNotaTrab'))

        elif request.form['botao'] == 'Adicionar aluno':
            return redirect(url_for('pags.addAluno'))

        else:
            return redirect(url_for('pags.calcSituacao'))

    return render_template("area_logada.html", user=current_user, nome_usuario=nome_usuario)

@pags.route('/areaLogada/addLista', methods=['GET', 'POST'])
@login_required
def addLista():
    '''Função para adicionar novas listas'''

    if request.method == 'POST':
        lista = request.form.get('lista').title()

        verif_lista = Lista.query.filter(Lista.lista==lista).first() #verifica se a lista já existe

        if verif_lista != None:
            flash('Essa tarefa já existe.', category='error')
        elif len(lista) < 3: #verifica o tamanho do nome da lista
            flash('O nome da lista deve ter mais de 2 caracteres', category='error')
        else:
            nova_lista = Lista(lista=lista) #cria nova lista
            db.session.add(nova_lista)
            db.session.commit() #adiciona nova lista à base de dados
            flash('Lista criada!', category='success')

    return render_template("add_lista.html", user=current_user)

@pags.route('/areaLogada/addAluno', methods=['GET', 'POST'])
@login_required
def addAluno():
    '''Função para adicionar novos alunos'''

    if request.method == 'POST':
        aluno = request.form.get('aluno').title()

        verif_aluno = Aluno.query.filter(Aluno.aluno==aluno).first() #verifica se o aluno já existe

        if verif_aluno != None:
            flash('Essa aluno já existe.', category='error')
        elif len(aluno) < 3: #verifica o tamanho do nome do aluno
            flash('O nome da lista deve ter mais de 2 caracteres', category='error')
        else:
            novo_aluno = Aluno(aluno=aluno) #cria novo aluno
            db.session.add(novo_aluno)
            db.session.commit() #adiciona novo aluno à base de dados
            flash('Aluno adicionado!', category='success')

    return render_template("add_aluno.html", user=current_user)

@pags.route('/areaLogada/addNotaLista', methods=['GET', 'POST'])
@login_required
def addNotaLista():
    '''Função para adicionar nota a uma lista'''

    if request.method == 'POST':
        aluno = request.form.get('aluno')
        lista = request.form.get('lista')
        nota = request.form.get('nota')

        id_aluno = Aluno.query.filter(Aluno.aluno==aluno).first().id #procura o id do aluno na base de dados
        id_lista = Lista.query.filter(Lista.lista==lista).first().id #procura o id da lista na base de dados

        if NotaLista.query.filter(NotaLista.id_aluno==id_aluno, NotaLista.id_lista==id_lista).first() != None: #verifica se a lista já tem nota
            flash('Esse aluno já possui nota nessa tarefa.', category='error')
        else:
            nova_nota = NotaLista(id_aluno=id_aluno, id_lista=id_lista, nota_lista=nota) #cria nova nota
            db.session.add(nova_nota)
            db.session.commit() #adiciona nova nota à base de dados
            flash('Nota adicionada!', category='success')

    busca = Lista.query.all() #busca todas as colunas da tabela Lista
    listas = []
    for lista in busca:
        listas.append(lista.lista) #adiciona na lista todas as linhas da coluna lista

    busca = Aluno.query.all() #busca todas as colunas da tabela Aluno
    alunos = []
    for aluno in busca:
        alunos.append(aluno.aluno) #adiciona na lista todas as linhas da coluna aluno

    return render_template("add_nota_lista.html", user=current_user, listas=listas, alunos=alunos)

@pags.route('/areaLogada/addNotaTrab', methods=['GET', 'POST'])
@login_required
def addNotaTrab():
    '''Função para adicionar nota a um trabalho'''

    if request.method == 'POST':
        aluno = request.form.get('aluno')
        trabalho = request.form.get('trabalho')
        nota = request.form.get('nota')

        id_aluno = Aluno.query.filter(Aluno.aluno==aluno).first().id #procura o id do aluno no banco de dados

        id_trabalho = Trabalho.query.filter(Trabalho.trabalho==trabalho).first() #verifica se o trabalho já existe
        if id_trabalho == None: #caso não exista
            novo_trabalho = Trabalho(trabalho=trabalho) #cria um novo trabalho
            db.session.add(novo_trabalho)
            db.session.commit() #adiciona o novo trabalho à base de dados
            id_trabalho = Trabalho.query.filter(Trabalho.trabalho==trabalho).first().id
        else: #caso já exista
            id_trabalho = id_trabalho.id #armazena na variável o id do trabalho

        if NotaTrabalho.query.filter(NotaTrabalho.id_aluno==id_aluno, NotaTrabalho.id_trabalho==id_trabalho).first() != None: #verifica se a nota já existe
            flash('Esse aluno já possui nota nesse trabalho.', category='error')
        else:
            nova_nota = NotaTrabalho(id_aluno=id_aluno, id_trabalho=id_trabalho, nota_trabalho=nota) #cria nova nota
            db.session.add(nova_nota)
            db.session.commit() #adiciona noca nota à base de dados
            flash('Nota adicionada!', category='success')

    busca= Aluno.query.all() #busca todas as colunas da tabela Aluno
    alunos = []
    for aluno in busca:
        alunos.append(aluno.aluno) #adiciona na lista todas as linhas da coluna aluno

    return render_template("add_nota_trab.html", user=current_user, alunos=alunos)

@pags.route('/areaLogada/calcSituacao', methods=['GET', 'POST'])
@login_required
def calcSituacao():
    '''Função para calcular situação do aluno e sua média'''

    if request.method == 'POST':
        aluno = request.form.get('aluno')

        id_aluno = Aluno.query.filter(Aluno.aluno==aluno).first().id #procura o id do aluno no banco de dados

        if NotaTrabalho.query.filter(NotaTrabalho.id_aluno==id_aluno).first() == None: #verifica se existe algum trabalho com nota
            flash('Não é possível calcular situção, adicione a nota de pelo menos um trabalho.', category='error')
            return render_template("calc_situacao.html", user=current_user)

        parte_trabalhos = 0
        quant = 0
        for nota in NotaTrabalho.query.filter(NotaTrabalho.id_aluno==id_aluno): #cada nota de trabalho é somada
            parte_trabalhos += nota.nota_trabalho
            quant += 1
        parte_trabalhos = (parte_trabalhos/quant)*0.8 #é feita a média das notas e o resultado multiplicado por 0.8

        if NotaLista.query.filter(NotaLista.id_aluno==id_aluno).first() == None: #verifica se existe alguma lista com nota
            flash('Não é possível calcular situção, adicione a nota de pelo menos uma lista.', category='error')
            return render_template("calc_situacao.html", user=current_user)

        parte_listas = 0
        quant = 0
        for nota in NotaLista.query.filter(NotaLista.id_aluno==id_aluno): #cada nota de lista é somada
            parte_listas += nota.nota_lista
            quant += 1
        parte_listas = (parte_listas/quant)*0.2 #é feita a média das notas e o resultado multiplicado por 0.8

        media = parte_trabalhos + parte_listas #os valores obtidos para as listas e trabalhos são somados
        media = round(media, 1) #o valor da média é arredondado para 1 casa decimal

        if media >= 7: #caso a média seja maior ou igual a 7
            situacao = 'Aprovado'
        elif media >= 3: #caso a média seja maior ou igual a 3
            situacao = 'Prova final'
        else: #caso a média seja menor que 3
            situacao = 'Reprovado'

        exibir_situacao = True #variável para permitir a exibição do texto com a situção do aluno

    else: #caso a página seja acessada através do método 'GET'
        media = None
        aluno = None
        situacao = None
        exibir_situacao = False

    busca= Aluno.query.all() #busca todas as colunas da tabela Aluno
    alunos = []
    for busca_aluno in busca:
        alunos.append(busca_aluno.aluno) #adiciona na lista todas as linhas da coluna aluno

    return render_template("calc_situacao.html",
                            user=current_user,
                            media=media,
                            nome_aluno=aluno,
                            situacao=situacao,
                            exibir_situacao=exibir_situacao,
                            alunos=alunos)


# -*- coding: utf-8 -*-
"""
This preprocessor removes lines in code cells that have been marked as `folded`
by the codefolding extension
"""

from nbconvert.preprocessors import Preprocessor

class CodeFoldingPreprocessor(Preprocessor):
    """
    :mod:`nbconvert` Preprocessor for the code_folding nbextension.

    Folds codecells as displayed in the notebook.

    The preprocessor is installed by default. To enable codefolding with
    NbConvert, you need to set the configuration parameter
    `NbConvertApp.codefolding=True`.
    This can be done either in the `jupyter_nbconvert_config.py` file::

        c.NbConvertApp.codefolding = True

    or using a command line parameter when calling NbConvert::

        $ jupyter nbconvert --to html --NbConvertApp.codefolding=True mynotebook.ipynb

    """  # noqa: E501

    fold_mark = u'↔'

    def fold_cell(self, cell, folded):
        """
        Remove folded lines and add a '<->' at the parent line
        """
        lines = cell.splitlines(True)

        if folded[0] == 0 and (lines[0][0] == '#' or lines[0][0] == '%'):
            # fold whole cell when first line is a comment or magic
            return lines[0].rstrip('\n') + self.fold_mark + '\n'
        fold_indent = 0
        fold = False
        fcell = ""
        for i, l in enumerate(lines):
            # fold indent level
            indent = len(l) - len(l.lstrip(' '))
            if indent <= fold_indent:
                fold = False
                fold_indent = 0
            if i in folded:
                fold = True
                fold_indent = indent
                fcell += l.rstrip('\n') + self.fold_mark + '\n'
            if fold is False:
                fcell += l
        return fcell

    def preprocess_cell(self, cell, resources, index):
        """
        Read cell metadata and remove lines marked as `folded`.
        Requires configuration parameter 'NbConvertApp.codefolding = True'

        Parameters
        ----------
        cell : NotebookNode cell
            Notebook cell being processed
        resources : dictionary
            Additional resources used in the conversion process.  Allows
            preprocessors to pass variables into the Jinja engine.
        index : int
            Index of the cell being processed (see base.py)
        """
        dofolding = self.config.NbConvertApp.get('codefolding', False) is True
        if hasattr(cell, 'source') and cell.cell_type == 'code' and dofolding:
            if hasattr(cell['metadata'], 'code_folding'):
                folded = cell['metadata']['code_folding']
                if len(folded) > 0:
                    cell.source = self.fold_cell(cell.source, folded)
        return cell, resources


import json
from unittest.mock import Mock, patch

import pytest
from traitlets.config import Config, Configurable

from jupyterlab.extensions import PyPIExtensionManager, ReadOnlyExtensionManager
from jupyterlab.extensions.manager import ExtensionManager, ExtensionPackage

from . import fake_client_factory

@pytest.mark.parametrize(
    "version, expected",
    (
        ("1", "1"),
        ("1.0", "1.0"),
        ("1.0.0", "1.0.0"),
        ("1.0.0a52", "1.0.0-alpha.52"),
        ("1.0.0b3", "1.0.0-beta.3"),
        ("1.0.0rc22", "1.0.0-rc.22"),
        ("1.0.0rc23.post2", "1.0.0-rc.23"),
        ("1.0.0rc24.dev2", "1.0.0-rc.24"),
        ("1.0.0rc25.post4.dev2", "1.0.0-rc.25"),
    ),
)
def test_ExtensionManager_get_semver_version(version, expected):
    assert ExtensionManager.get_semver_version(version) == expected

async def test_ExtensionManager_list_extensions_installed(monkeypatch):
    extension1 = ExtensionPackage("extension1", "Extension 1 description", "", "prebuilt")

    async def mock_installed(*args, **kwargs):
        return {"extension1": extension1}

    monkeypatch.setattr(ReadOnlyExtensionManager, "_get_installed_extensions", mock_installed)

    manager = ReadOnlyExtensionManager()

    extensions = await manager.list_extensions()

    assert extensions == ([extension1], 1)

async def test_ExtensionManager_list_extensions_query(monkeypatch):
    extension1 = ExtensionPackage("extension1", "Extension 1 description", "", "prebuilt")
    extension2 = ExtensionPackage("extension2", "Extension 2 description", "", "prebuilt")

    async def mock_list(*args, **kwargs):
        return {"extension1": extension1, "extension2": extension2}, None

    monkeypatch.setattr(ReadOnlyExtensionManager, "list_packages", mock_list)

    manager = ReadOnlyExtensionManager()

    extensions = await manager.list_extensions("ext")

    assert extensions == ([extension1, extension2], 1)

@patch("tornado.httpclient.AsyncHTTPClient", new_callable=fake_client_factory)
async def test_ExtensionManager_list_extensions_query_allow(mock_client, monkeypatch):
    extension1 = ExtensionPackage("extension1", "Extension 1 description", "", "prebuilt")
    extension2 = ExtensionPackage("extension2", "Extension 2 description", "", "prebuilt")

    mock_client.body = json.dumps({"allowed_extensions": [{"name": "extension1"}]}).encode()

    async def mock_list(*args, **kwargs):
        return {"extension1": extension1, "extension2": extension2}, None

    monkeypatch.setattr(ReadOnlyExtensionManager, "list_packages", mock_list)

    manager = ReadOnlyExtensionManager(
        ext_options={"allowed_extensions_uris": {"http://dummy-allowed-extension"}},
    )

    extensions = await manager.list_extensions("ext")

    assert extensions == ([extension1], 1)

@patch("tornado.httpclient.AsyncHTTPClient", new_callable=fake_client_factory)
async def test_ExtensionManager_list_extensions_query_block(mock_client, monkeypatch):
    extension1 = ExtensionPackage("extension1", "Extension 1 description", "", "prebuilt")
    extension2 = ExtensionPackage("extension2", "Extension 2 description", "", "prebuilt")

    mock_client.body = json.dumps({"blocked_extensions": [{"name": "extension1"}]}).encode()

    async def mock_list(*args, **kwargs):
        return {"extension1": extension1, "extension2": extension2}, None

    monkeypatch.setattr(ReadOnlyExtensionManager, "list_packages", mock_list)

    manager = ReadOnlyExtensionManager(
        ext_options={"blocked_extensions_uris": {"http://dummy-blocked-extension"}}
    )

    extensions = await manager.list_extensions("ext")

    assert extensions == ([extension2], 1)

@patch("tornado.httpclient.AsyncHTTPClient", new_callable=fake_client_factory)
async def test_ExtensionManager_list_extensions_query_allow_block(mock_client, monkeypatch):
    extension1 = ExtensionPackage("extension1", "Extension 1 description", "", "prebuilt")
    extension2 = ExtensionPackage("extension2", "Extension 2 description", "", "prebuilt")

    mock_client.body = json.dumps(
        {
            "allowed_extensions": [{"name": "extension1"}],
            "blocked_extensions": [{"name": "extension1"}],
        }
    ).encode()

    async def mock_list(*args, **kwargs):
        return {"extension1": extension1, "extension2": extension2}, None

    monkeypatch.setattr(ReadOnlyExtensionManager, "list_packages", mock_list)

    manager = ReadOnlyExtensionManager(
        ext_options={
            "allowed_extensions_uris": {"http://dummy-allowed-extension"},
            "blocked_extensions_uris": {"http://dummy-blocked-extension"},
        }
    )

    extensions = await manager.list_extensions("ext")

    assert extensions == ([extension1], 1)

async def test_ExtensionManager_install():
    manager = ReadOnlyExtensionManager()

    result = await manager.install("extension1")

    assert result.status == "error"
    assert result.message == "Extension installation not supported."

async def test_ExtensionManager_uninstall():
    manager = ReadOnlyExtensionManager()

    result = await manager.uninstall("extension1")

    assert result.status == "error"
    assert result.message == "Extension removal not supported."

@patch("jupyterlab.extensions.pypi.xmlrpc.client")
async def test_PyPiExtensionManager_list_extensions_query(mocked_rpcclient):
    extension1 = ExtensionPackage(
        name="jupyterlab-git",
        description="A JupyterLab extension for version control using git",
        homepage_url="https://github.com/jupyterlab/jupyterlab-git",
        pkg_type="prebuilt",
        latest_version="0.37.1",
        author="Jupyter Development Team",
        license="BSD-3-Clause",
        package_manager_url="https://pypi.org/project/jupyterlab-git/",
    )
    extension2 = ExtensionPackage(
        name="jupyterlab-github",
        description="JupyterLab viewer for GitHub repositories",
        homepage_url="https://github.com/jupyterlab/jupyterlab-github/blob/main/README.md",
        pkg_type="prebuilt",
        latest_version="3.0.1",
        author="Ian Rose",
        license="BSD-3-Clause",
        bug_tracker_url="https://github.com/jupyterlab/jupyterlab-github/issues",
        package_manager_url="https://pypi.org/project/jupyterlab-github/",
        repository_url="https://github.com/jupyterlab/jupyterlab-github",
    )

    proxy = Mock(
        browse=Mock(
            return_value=[
                ["jupyterlab-git", "0.33.0"],
                ["jupyterlab-git", "0.34.0"],
                ["jupyterlab-git", "0.34.1"],
                ["jupyterlab-git", "0.37.0"],
                ["jupyterlab-git", "0.37.1"],
                ["jupyterlab-github", "3.0.0"],
                ["jupyterlab-github", "3.0.1"],
            ]
        ),
    )
    mocked_rpcclient.ServerProxy = Mock(return_value=proxy)

    manager = PyPIExtensionManager()

    async def mock_pkg_metadata(n, l, b):  # noqa
        return (
            {
                "name": "jupyterlab-git",
                "version": "0.37.1",
                "stable_version": None,
                "bugtrack_url": None,
                "package_url": "https://pypi.org/project/jupyterlab-git/",
                "release_url": "https://pypi.org/project/jupyterlab-git/0.37.1/",
                "docs_url": None,
                "home_page": "https://github.com/jupyterlab/jupyterlab-git",
                "download_url": "",
                "project_url": "",
                "project_urls": {},
                "author": "Jupyter Development Team",
                "author_email": "",
                "maintainer": "",
                "maintainer_email": "",
                "summary": "A JupyterLab extension for version control using git",
                "license": "BSD-3-Clause",
                "keywords": "Jupyter,JupyterLab,JupyterLab3,jupyterlab-extension,Git",
                "platform": "Linux",
                "classifiers": [
                    "Framework :: Jupyter",
                    "Framework :: Jupyter :: JupyterLab",
                    "Framework :: Jupyter :: JupyterLab :: 3",
                    "Framework :: Jupyter :: JupyterLab :: Extensions",
                    "Framework :: Jupyter :: JupyterLab :: Extensions :: Prebuilt",
                    "Intended Audience :: Developers",
                    "Intended Audience :: Science/Research",
                    "License :: OSI Approved :: BSD License",
                    "Programming Language :: Python",
                    "Programming Language :: Python :: 3",
                    "Programming Language :: Python :: 3.10",
                    "Programming Language :: Python :: 3.6",
                    "Programming Language :: Python :: 3.7",
                    "Programming Language :: Python :: 3.8",
                    "Programming Language :: Python :: 3.9",
                ],
                "requires": [],
                "requires_dist": [
                    "jupyter-server",
                    "nbdime (~=3.1)",
                    "nbformat",
                    "packaging",
                    "pexpect",
                    "black ; extra == 'dev'",
                    "coverage ; extra == 'dev'",
                    "jupyter-packaging (~=0.7.9) ; extra == 'dev'",
                    "jupyterlab (~=3.0) ; extra == 'dev'",
                    "pre-commit ; extra == 'dev'",
                    "pytest ; extra == 'dev'",
                    "pytest-asyncio ; extra == 'dev'",
                    "pytest-cov ; extra == 'dev'",
                    "pytest-tornasync ; extra == 'dev'",
                    "black ; extra == 'tests'",
                    "coverage ; extra == 'tests'",
                    "jupyter-packaging (~=0.7.9) ; extra == 'tests'",
                    "jupyterlab (~=3.0) ; extra == 'tests'",
                    "pre-commit ; extra == 'tests'",
                    "pytest ; extra == 'tests'",
                    "pytest-asyncio ; extra == 'tests'",
                    "pytest-cov ; extra == 'tests'",
                    "pytest-tornasync ; extra == 'tests'",
                    "hybridcontents ; extra == 'tests'",
                    "jupytext ; extra == 'tests'",
                ],
                "provides": [],
                "provides_dist": [],
                "obsoletes": [],
                "obsoletes_dist": [],
                "requires_python": "<4,>=3.6",
                "requires_external": [],
                "_pypi_ordering": 55,
                "downloads": {"last_day": -1, "last_week": -1, "last_month": -1},
                "cheesecake_code_kwalitee_id": None,
                "cheesecake_documentation_id": None,
                "cheesecake_installability_id": None,
            }
            if n == "jupyterlab-git"
            else {
                "name": "jupyterlab-github",
                "version": "3.0.1",
                "stable_version": None,
                "bugtrack_url": None,
                "package_url": "https://pypi.org/project/jupyterlab-github/",
                "release_url": "https://pypi.org/project/jupyterlab-github/3.0.1/",
                "docs_url": None,
                "home_page": "",
                "download_url": "",
                "project_url": "",
                "project_urls": {
                    "Homepage": "https://github.com/jupyterlab/jupyterlab-github/blob/main/README.md",
                    "Bug Tracker": "https://github.com/jupyterlab/jupyterlab-github/issues",
                    "Source Code": "https://github.com/jupyterlab/jupyterlab-github",
                },
                "author": "Ian Rose",
                "author_email": "jupyter@googlegroups.com",
                "maintainer": "",
                "maintainer_email": "",
                "summary": "JupyterLab viewer for GitHub repositories",
                "license": "BSD-3-Clause",
                "keywords": "Jupyter,JupyterLab,JupyterLab3",
                "platform": "Linux",
                "classifiers": [
                    "Framework :: Jupyter",
                    "Framework :: Jupyter :: JupyterLab",
                    "Framework :: Jupyter :: JupyterLab :: 3",
                    "Framework :: Jupyter :: JupyterLab :: Extensions",
                    "Framework :: Jupyter :: JupyterLab :: Extensions :: Prebuilt",
                    "License :: OSI Approved :: BSD License",
                    "Programming Language :: Python",
                    "Programming Language :: Python :: 3",
                    "Programming Language :: Python :: 3.6",
                    "Programming Language :: Python :: 3.7",
                    "Programming Language :: Python :: 3.8",
                    "Programming Language :: Python :: 3.9",
                ],
                "requires": [],
                "requires_dist": ["jupyterlab (~=3.0)"],
                "provides": [],
                "provides_dist": [],
                "obsoletes": [],
                "obsoletes_dist": [],
                "requires_python": ">=3.6",
                "requires_external": [],
                "_pypi_ordering": 12,
                "downloads": {"last_day": -1, "last_week": -1, "last_month": -1},
                "cheesecake_code_kwalitee_id": None,
                "cheesecake_documentation_id": None,
                "cheesecake_installability_id": None,
            }
        )

    manager._fetch_package_metadata = mock_pkg_metadata

    extensions = await manager.list_extensions("git")

    assert extensions == ([extension1, extension2], 1)

async def test_PyPiExtensionManager_custom_server_url():
    BASE_URL = "https://mylocal.pypi.server/pypi"  # noqa

    parent = Configurable(config=Config({"PyPIExtensionManager": {"base_url": BASE_URL}}))

    manager = PyPIExtensionManager(parent=parent)

    assert manager.base_url == BASE_URL


import re
import unittest
from datetime import datetime

from ...constants import *  # NOQA
from . import (
    ArchiverTestCaseBase,
    RemoteArchiverTestCaseBase,
    ArchiverTestCaseBinaryBase,
    RK_ENCRYPTION,
    src_dir,
    BORG_EXES,
)

class ArchiverTestCase(ArchiverTestCaseBase):
    def test_prune_repository(self):
        self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
        self.cmd(f"--repo={self.repository_location}", "create", "test1", src_dir)
        self.cmd(f"--repo={self.repository_location}", "create", "test2", src_dir)
        # these are not really a checkpoints, but they look like some:
        self.cmd(f"--repo={self.repository_location}", "create", "test3.checkpoint", src_dir)
        self.cmd(f"--repo={self.repository_location}", "create", "test3.checkpoint.1", src_dir)
        self.cmd(f"--repo={self.repository_location}", "create", "test4.checkpoint", src_dir)
        output = self.cmd(f"--repo={self.repository_location}", "prune", "--list", "--dry-run", "--keep-daily=1")
        assert re.search(r"Would prune:\s+test1", output)
        # must keep the latest non-checkpoint archive:
        assert re.search(r"Keeping archive \(rule: daily #1\):\s+test2", output)
        # must keep the latest checkpoint archive:
        assert re.search(r"Keeping checkpoint archive:\s+test4.checkpoint", output)
        output = self.cmd(f"--repo={self.repository_location}", "rlist", "--consider-checkpoints")
        self.assert_in("test1", output)
        self.assert_in("test2", output)
        self.assert_in("test3.checkpoint", output)
        self.assert_in("test3.checkpoint.1", output)
        self.assert_in("test4.checkpoint", output)
        self.cmd(f"--repo={self.repository_location}", "prune", "--keep-daily=1")
        output = self.cmd(f"--repo={self.repository_location}", "rlist", "--consider-checkpoints")
        self.assert_not_in("test1", output)
        # the latest non-checkpoint archive must be still there:
        self.assert_in("test2", output)
        # only the latest checkpoint archive must still be there:
        self.assert_not_in("test3.checkpoint", output)
        self.assert_not_in("test3.checkpoint.1", output)
        self.assert_in("test4.checkpoint", output)
        # now we supersede the latest checkpoint by a successful backup:
        self.cmd(f"--repo={self.repository_location}", "create", "test5", src_dir)
        self.cmd(f"--repo={self.repository_location}", "prune", "--keep-daily=2")
        output = self.cmd(f"--repo={self.repository_location}", "rlist", "--consider-checkpoints")
        # all checkpoints should be gone now:
        self.assert_not_in("checkpoint", output)
        # the latest archive must be still there
        self.assert_in("test5", output)

    def _create_archive_ts(self, name, y, m, d, H=0, M=0, S=0):
        self.cmd(
            f"--repo={self.repository_location}",
            "create",
            "--timestamp",
            datetime(y, m, d, H, M, S, 0).strftime(ISO_FORMAT_NO_USECS),  # naive == local time / local tz
            name,
            src_dir,
        )

    # This test must match docs/misc/prune-example.txt
    def test_prune_repository_example(self):
        self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
        # Archives that will be kept, per the example
        # Oldest archive
        self._create_archive_ts("test01", 2015, 1, 1)
        # 6 monthly archives
        self._create_archive_ts("test02", 2015, 6, 30)
        self._create_archive_ts("test03", 2015, 7, 31)
        self._create_archive_ts("test04", 2015, 8, 31)
        self._create_archive_ts("test05", 2015, 9, 30)
        self._create_archive_ts("test06", 2015, 10, 31)
        self._create_archive_ts("test07", 2015, 11, 30)
        # 14 daily archives
        self._create_archive_ts("test08", 2015, 12, 17)
        self._create_archive_ts("test09", 2015, 12, 18)
        self._create_archive_ts("test10", 2015, 12, 20)
        self._create_archive_ts("test11", 2015, 12, 21)
        self._create_archive_ts("test12", 2015, 12, 22)
        self._create_archive_ts("test13", 2015, 12, 23)
        self._create_archive_ts("test14", 2015, 12, 24)
        self._create_archive_ts("test15", 2015, 12, 25)
        self._create_archive_ts("test16", 2015, 12, 26)
        self._create_archive_ts("test17", 2015, 12, 27)
        self._create_archive_ts("test18", 2015, 12, 28)
        self._create_archive_ts("test19", 2015, 12, 29)
        self._create_archive_ts("test20", 2015, 12, 30)
        self._create_archive_ts("test21", 2015, 12, 31)
        # Additional archives that would be pruned
        # The second backup of the year
        self._create_archive_ts("test22", 2015, 1, 2)
        # The next older monthly backup
        self._create_archive_ts("test23", 2015, 5, 31)
        # The next older daily backup
        self._create_archive_ts("test24", 2015, 12, 16)
        output = self.cmd(
            f"--repo={self.repository_location}",
            "prune",
            "--list",
            "--dry-run",
            "--keep-daily=14",
            "--keep-monthly=6",
            "--keep-yearly=1",
        )
        # Prune second backup of the year
        assert re.search(r"Would prune:\s+test22", output)
        # Prune next older monthly and daily backups
        assert re.search(r"Would prune:\s+test23", output)
        assert re.search(r"Would prune:\s+test24", output)
        # Must keep the other 21 backups
        # Yearly is kept as oldest archive
        assert re.search(r"Keeping archive \(rule: yearly\[oldest\] #1\):\s+test01", output)
        for i in range(1, 7):
            assert re.search(r"Keeping archive \(rule: monthly #" + str(i) + r"\):\s+test" + ("%02d" % (8 - i)), output)
        for i in range(1, 15):
            assert re.search(r"Keeping archive \(rule: daily #" + str(i) + r"\):\s+test" + ("%02d" % (22 - i)), output)
        output = self.cmd(f"--repo={self.repository_location}", "rlist")
        # Nothing pruned after dry run
        for i in range(1, 25):
            self.assert_in("test%02d" % i, output)
        self.cmd(
            f"--repo={self.repository_location}", "prune", "--keep-daily=14", "--keep-monthly=6", "--keep-yearly=1"
        )
        output = self.cmd(f"--repo={self.repository_location}", "rlist")
        # All matching backups plus oldest kept
        for i in range(1, 22):
            self.assert_in("test%02d" % i, output)
        # Other backups have been pruned
        for i in range(22, 25):
            self.assert_not_in("test%02d" % i, output)

    # With an initial and daily backup, prune daily until oldest is replaced by a monthly backup
    def test_prune_retain_and_expire_oldest(self):
        self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
        # Initial backup
        self._create_archive_ts("original_archive", 2020, 9, 1, 11, 15)
        # Archive and prune daily for 30 days
        for i in range(1, 31):
            self._create_archive_ts("september%02d" % i, 2020, 9, i, 12)
            self.cmd(f"--repo={self.repository_location}", "prune", "--keep-daily=7", "--keep-monthly=1")
        # Archive and prune 6 days into the next month
        for i in range(1, 7):
            self._create_archive_ts("october%02d" % i, 2020, 10, i, 12)
            self.cmd(f"--repo={self.repository_location}", "prune", "--keep-daily=7", "--keep-monthly=1")
        # Oldest backup is still retained
        output = self.cmd(
            f"--repo={self.repository_location}", "prune", "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1"
        )
        assert re.search(r"Keeping archive \(rule: monthly\[oldest\] #1" + r"\):\s+original_archive", output)
        # Archive one more day and prune.
        self._create_archive_ts("october07", 2020, 10, 7, 12)
        self.cmd(f"--repo={self.repository_location}", "prune", "--keep-daily=7", "--keep-monthly=1")
        # Last day of previous month is retained as monthly, and oldest is expired.
        output = self.cmd(
            f"--repo={self.repository_location}", "prune", "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1"
        )
        assert re.search(r"Keeping archive \(rule: monthly #1\):\s+september30", output)
        self.assert_not_in("original_archive", output)

    def test_prune_repository_save_space(self):
        self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
        self.cmd(f"--repo={self.repository_location}", "create", "test1", src_dir)
        self.cmd(f"--repo={self.repository_location}", "create", "test2", src_dir)
        output = self.cmd(f"--repo={self.repository_location}", "prune", "--list", "--dry-run", "--keep-daily=1")
        assert re.search(r"Keeping archive \(rule: daily #1\):\s+test2", output)
        assert re.search(r"Would prune:\s+test1", output)
        output = self.cmd(f"--repo={self.repository_location}", "rlist")
        self.assert_in("test1", output)
        self.assert_in("test2", output)
        self.cmd(f"--repo={self.repository_location}", "prune", "--save-space", "--keep-daily=1")
        output = self.cmd(f"--repo={self.repository_location}", "rlist")
        self.assert_not_in("test1", output)
        self.assert_in("test2", output)

    def test_prune_repository_prefix(self):
        self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
        self.cmd(f"--repo={self.repository_location}", "create", "foo-2015-08-12-10:00", src_dir)
        self.cmd(f"--repo={self.repository_location}", "create", "foo-2015-08-12-20:00", src_dir)
        self.cmd(f"--repo={self.repository_location}", "create", "bar-2015-08-12-10:00", src_dir)
        self.cmd(f"--repo={self.repository_location}", "create", "bar-2015-08-12-20:00", src_dir)
        output = self.cmd(
            f"--repo={self.repository_location}",
            "prune",
            "--list",
            "--dry-run",
            "--keep-daily=1",
            "--match-archives=sh:foo-*",
        )
        assert re.search(r"Keeping archive \(rule: daily #1\):\s+foo-2015-08-12-20:00", output)
        assert re.search(r"Would prune:\s+foo-2015-08-12-10:00", output)
        output = self.cmd(f"--repo={self.repository_location}", "rlist")
        self.assert_in("foo-2015-08-12-10:00", output)
        self.assert_in("foo-2015-08-12-20:00", output)
        self.assert_in("bar-2015-08-12-10:00", output)
        self.assert_in("bar-2015-08-12-20:00", output)
        self.cmd(f"--repo={self.repository_location}", "prune", "--keep-daily=1", "--match-archives=sh:foo-*")
        output = self.cmd(f"--repo={self.repository_location}", "rlist")
        self.assert_not_in("foo-2015-08-12-10:00", output)
        self.assert_in("foo-2015-08-12-20:00", output)
        self.assert_in("bar-2015-08-12-10:00", output)
        self.assert_in("bar-2015-08-12-20:00", output)

    def test_prune_repository_glob(self):
        self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
        self.cmd(f"--repo={self.repository_location}", "create", "2015-08-12-10:00-foo", src_dir)
        self.cmd(f"--repo={self.repository_location}", "create", "2015-08-12-20:00-foo", src_dir)
        self.cmd(f"--repo={self.repository_location}", "create", "2015-08-12-10:00-bar", src_dir)
        self.cmd(f"--repo={self.repository_location}", "create", "2015-08-12-20:00-bar", src_dir)
        output = self.cmd(
            f"--repo={self.repository_location}",
            "prune",
            "--list",
            "--dry-run",
            "--keep-daily=1",
            "--match-archives=sh:2015-*-foo",
        )
        assert re.search(r"Keeping archive \(rule: daily #1\):\s+2015-08-12-20:00-foo", output)
        assert re.search(r"Would prune:\s+2015-08-12-10:00-foo", output)
        output = self.cmd(f"--repo={self.repository_location}", "rlist")
        self.assert_in("2015-08-12-10:00-foo", output)
        self.assert_in("2015-08-12-20:00-foo", output)
        self.assert_in("2015-08-12-10:00-bar", output)
        self.assert_in("2015-08-12-20:00-bar", output)
        self.cmd(f"--repo={self.repository_location}", "prune", "--keep-daily=1", "--match-archives=sh:2015-*-foo")
        output = self.cmd(f"--repo={self.repository_location}", "rlist")
        self.assert_not_in("2015-08-12-10:00-foo", output)
        self.assert_in("2015-08-12-20:00-foo", output)
        self.assert_in("2015-08-12-10:00-bar", output)
        self.assert_in("2015-08-12-20:00-bar", output)

class RemoteArchiverTestCase(RemoteArchiverTestCaseBase, ArchiverTestCase):
    """run the same tests, but with a remote repository"""

@unittest.skipUnless("binary" in BORG_EXES, "no borg.exe available")
class ArchiverTestCaseBinary(ArchiverTestCaseBinaryBase, ArchiverTestCase):
    """runs the same tests, but via the borg binary"""


# import the necessary packages
from imutils import face_utils
import numpy as np
import argparse
import imutils
import dlib
import cv2
import math
from numpy.linalg import norm
import sys
import copy
import os

def start(swap, image, video):
    # construct the argument parser and parse the arguments
    # ap = argparse.ArgumentParser()
    # ap.add_argument("-p", "--shape-predictor", required=True,
    # 	help="path to facial landmark predictor")
    # ap.add_argument("-i", "--image", required=True,
    # 	help="path to input image")
    # args = vars(ap.parse_args())

    # initialize dlib's face detector (HOG-based) and then create
    # the facial landmark predictor
    cam = cv2.VideoCapture(video)
    width = int(cam.get(3))  # float
    height = int(cam.get(4))
    print(width, height)
    vidWriter = cv2.VideoWriter("./video_output_data1.mp4",cv2.VideoWriter_fourcc(*'mp4v'), int(cam.get(5)), (width, height))
    detector = dlib.get_frontal_face_detector()
    predictor = dlib.shape_predictor('shape_predictor_68_face_landmarks.dat')
    # swap = True
    face_swap_file = image
    # load the input image, resize it, and convert it to grayscale
    # images = ['Scarlett.jpg', 'Rambo.jpg']
    # imageA = cv2.imread('Scarlett.jpg')

            # cv2.circle(imageB, (x, y), 1, (0, 0, 255), -1)
    currentframe = 0
    while (True):
        # reading from frame
        ret, frame = cam.read()
        if ret:
            imageA = frame
            grayA = cv2.cvtColor(imageA, cv2.COLOR_BGR2GRAY)
            rectsA = detector(grayA, 1)
            if (swap and len(rectsA) < 2) or (not swap and len(rectsA) == 0):
                vidWriter.write(frame)
                print("Can't find enough faces in frame")
                continue
            PA = []
            xPointsA = []
            yPointsA = []
            shapeA = None
            # for (i, rect) in enumerate(rectsA):
                # determine the facial landmarks for the face region, then
                # convert the facial landmark (x, y)-coordinates to a NumPy
                # array
            if swap:
                shapeA = predictor(grayA, rectsA[0])
            else:
                shapeA = predictor(grayA, rectsA[0])
            shapeA = face_utils.shape_to_np(shapeA)
            print(len(shapeA))
            for (x, y) in shapeA:
                PA.append([x, y, 1])
                xPointsA.append(x)
                yPointsA.append(y)
                # convert dlib's rectangle to a OpenCV-style bounding box
                # [i.e., (x, y, w, h)], then draw the face bounding box
                # (x, y, w, h) = face_utils.rect_to_bb(rect)
                # cv2.rectangle(imageA, (x, y), (x + w, y + h), (0, 255, 0), 2)
                # # show the face number
                # cv2.putText(imageA, "Face #{}".format(i + 1), (x - 10, y - 10),
                # 	cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
                # loop over the (x, y)-coordinates for the facial landmarks
                # and draw them on the image
                # for (x, y) in shapeA:
                # cv2.circle(imageA, (x, y), 1, (0, 0, 255), -1)
            if swap:
                imageB = frame
            else:
                imageB = cv2.imread(face_swap_file)

            # detect faces in the grayscale image

            grayB = cv2.cvtColor(imageB, cv2.COLOR_BGR2GRAY)
            # detect faces in the grayscale image
            # rectsB = detector(grayB, 1)

            PB = []
            shapeB = None
            xPointsB = []
            yPointsB = []

            # loop over the face detections
            # for (i, rect) in enumerate(rectsB):
                # determine the facial landmarks for the face region, then
                # convert the facial landmark (x, y)-coordinates to a NumPy
                # array
            if swap:
                shapeB = predictor(grayB, rectsA[1])
            else:
                rectsB = detector(grayB, 1)
                shapeB = predictor(grayB, rectsB[0])
            shapeB = face_utils.shape_to_np(shapeB)
            # convert dlib's rectangle to a OpenCV-style bounding box
            # [i.e., (x, y, w, h)], then draw the face bounding box
            # (x, y, w, h) = face_utils.rect_to_bb(rect)
            # cv2.rectangle(imageB, (x, y), (x + w, y + h), (0, 255, 0), 2)
            # # show the face number
            # cv2.putText(imageB, "Face #{}".format(i + 1), (x - 10, y - 10),
            # 	cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
            # loop over the (x, y)-coordinates for the facial landmarks
            # and draw them on the image
            for (x, y) in shapeB:
                PB.append([x, y, 1])
                xPointsB.append(x)
                yPointsB.append(y)
            weights_x, weights_y = find_weights(PA, shapeA, xPointsB, yPointsB)
            # print(weights)
            mask_warped_img, warped_img = warp_face(imageA, imageB, shapeA, shapeB, weights_x, weights_y)
            r = cv2.boundingRect(mask_warped_img)
            center = ((r[0] + int(r[2] / 2), r[1] + int(r[3] / 2)))
            output = cv2.seamlessClone(warped_img.copy(), imageA, mask_warped_img, center, cv2.NORMAL_CLONE)
            if swap:
                weights_x, weights_y = find_weights(PB, shapeB, xPointsA, yPointsA)
                # print(weights)
                mask_warped_img, warped_img = warp_face(imageB, imageA, shapeB, shapeA, weights_x, weights_y)
                r = cv2.boundingRect(mask_warped_img)
                center = ((r[0] + int(r[2] / 2), r[1] + int(r[3] / 2)))
                output = cv2.seamlessClone(warped_img.copy(), output, mask_warped_img, center, cv2.NORMAL_CLONE)
                # imageA[rects[0].top() - 40:rects[0].bottom() + 40, rects[0].left() - 40:rects[0].right() + 40, :] = output
                # cv2.imshow("warped_img", warped_img)
                # cv2.imshow("mask_warped", mask_warped_img)
                # cv2.imshow("mask", mask)
            vidWriter.write(output)
            cv2.imwrite('output_data1_' + str(currentframe) + '.jpg', output)
            # cv2.waitKey(0)

            # increasing counter so that it will
            # show how many frames are created
            currentframe += 1
        else:
            break
    vidWriter.release()
    # Release all space and windows once done
    cam.release()
    cv2.destroyAllWindows()
    # loop over the face detections

    # a1 = weights[-1]
    # ay = weights[-2]
    # ax = weights[-3]
    # weights = weights[:len(weights)-3]
    # warpedA = []
    # for i in range(grayB.shape[0]):
    # 	for j in range(grayB.shape[1]):
    # 		sum = 0
    # 		for ind, weight in enumerate(weights):
    # 			# print(shapeB[ind])
    # 			r = norm(shapeB[ind] - [i, j], 1) + np.exp(10 ** -7)
    # 			u = (r ** 2) * (math.log(r ** 2))
    # 			sum = sum + weight*u
    # 		warpedA.append(a1 + ax*i + ay*j + sum)
    #
    # # print(warpedA)
    #
    # new = np.zeros(grayB.shape)
    #
    # warpedA = np.reshape(warpedA, (333,500,2))
    # print(warpedA.shape)
    # print(new.shape)
    #
    # for i in range(new.shape[0]):
    # 	for j in range(new.shape[1]):
    # 		if 0 <= int(warpedA[i][j][1]) < grayA.shape[1] and 0 <= int(warpedA[i][j][0]) < grayA.shape[0]:
    # 			print(warpedA[i][j])
    # 			new[i][j] = grayA[int(warpedA[i][j][0])][int(warpedA[i][j][1])]

    # show the output image with the face detections + facial landmarks
    # cv2.imshow("OutputA", imageA)

def warp_face(imageA, imageB, shapeA, shapeB, weights_x, weights_y):
    w, h = imageB.shape[:2]
    mask = mask_from_points((w, h), shapeB)
    # def warp_tps(img_source,img_target,points1,points2,weights_x,weights_y,mask):
    xy1_min = np.float32([min(shapeA[:, 0]), min(shapeA[:, 1])])
    xy1_max = np.float32([max(shapeA[:, 0]), max(shapeA[:, 1])])
    xy2_min = np.float32([min(shapeB[:, 0]), min(shapeB[:, 1])])
    xy2_max = np.float32([max(shapeB[:, 0]), max(shapeB[:, 1])])
    x = np.arange(xy1_min[0], xy1_max[0]).astype(int)
    y = np.arange(xy1_min[1], xy1_max[1]).astype(int)
    X, Y = np.mgrid[x[0]:x[-1] + 1, y[0]:y[-1] + 1]
    # X,Y = np.mgrid[0:src_shape[2],0:src_shape[3]]
    pts_src = np.vstack((X.ravel(), Y.ravel()))
    xy = pts_src.T
    u = np.zeros_like(xy[:, 0])
    v = np.zeros_like(xy[:, 0])
    # print(u.shape)
    # print(v.shape)
    for i in range(xy.shape[0]):
        u[i] = fxy(xy[i, :], shapeA, weights_x)
    u[u < xy2_min[0]] = xy2_min[0]
    u[u > xy2_max[0]] = xy2_max[0]
    for j in range(xy.shape[0]):
        v[j] = fxy(xy[j, :], shapeA, weights_y)
    v[v < xy2_min[1]] = xy2_min[1]
    v[v > xy2_max[1]] = xy2_max[1]
    #     print(u.shape)
    #     print(img_source.shape)
    warped_img = imageA.copy()
    mask_warped_img = np.zeros_like(warped_img[:, :, 0])
    for a in range(u.shape[0]):
        #     for b in range(v.shape[0]):
        #     warped_img[xy[a,1],xy[a,0],:] = warped_src_face[v[a],u[a],:]
        if mask[v[a], u[a]] > 0:
            warped_img[xy[a, 1], xy[a, 0], :] = imageB[v[a], u[a], :]
            mask_warped_img[xy[a, 1], xy[a, 0]] = 255
        # plt.imshow(warped_img)
        # plt.show()
        # return warped_img, mask_warped_img
    return mask_warped_img, warped_img

def find_weights(PA, shapeA, xPointsB, yPointsB):
    K = np.zeros([len(shapeA), len(shapeA)])
    for i in range(len(shapeA)):
        for j in range(len(shapeA)):
            r = norm(shapeA[i] - shapeA[j], 2) + np.exp(10 ** -7)
            K[i][j] = (r ** 2) * (math.log(r ** 2))
    mat = np.vstack([np.hstack((K, PA)), np.hstack([np.transpose(PA), np.zeros([3, 3])])])
    lam = np.exp(10 ** -7)
    V = []
    for v in xPointsB:
        V.append(v)
    V.append(0)
    V.append(0)
    V.append(0)
    # print('K', K)
    weights_x = np.matmul(np.linalg.inv(mat + lam * np.identity(len(shapeA) + 3)), V)
    V = []
    for v in yPointsB:
        V.append(v)
    V.append(0)
    V.append(0)
    V.append(0)
    weights_y = np.matmul(np.linalg.inv(mat + lam * np.identity(len(shapeA) + 3)), V)
    return weights_x, weights_y

def U(r):
    return (r**2)*(math.log(r**2))

def fxy(pt1,pts2,weights):
    K = np.zeros([pts2.shape[0],1])
    for i in range(pts2.shape[0]):
        K[i] = U(np.linalg.norm((pts2[i]-pt1),2) + np.exp(10**-7))
    f = weights[-1] + weights[-3]*pt1[0] +weights[-2]*pt1[1]+np.matmul(K.T,weights[0:-3])
    return f

def mask_from_points(size, points,erode_flag=1):
    radius = 10  # kernel size
    kernel = np.ones((radius, radius), np.uint8)

    mask = np.zeros(size, np.uint8)
    cv2.fillConvexPoly(mask, cv2.convexHull(points), 255)
    if erode_flag:
        mask = cv2.erode(mask, kernel,iterations=1)

    return mask

if __name__ == '__main__':
    start()


import argparse
import os
import cv2
import numpy as np

def flip(img_path):
    dirname = os.path.dirname(img_path)
    basename = os.path.basename(img_path)
    filename, ext = os.path.splitext(basename)
    modified_basename = filename + '_flipped' + ext
    im = cv2.imread(img_path)
    cv2.imwrite(os.path.join(dirname, modified_basename), im[::-1, :, :])

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Flip Traversability Map")
    parser.add_argument("--img_path", type=str, help="Path to load map from", default="")
    args = parser.parse_args()
    flip(args.img_path)


import datetime
import re
from tabulate import tabulate

def find_project(projects_db):
    found_projects = {}

    while True:
        search_date = input(f"Please enter the search date for project's campaign (DD/MM/YYYY): ").strip()

        if (search_date.lower() == "exit"):
            return search_date

        if not (re.search('^(0[1-9]|[12][0-9]|3[01])\/(0[1-9]|1[0,1,2])\/(19|20)\d{2}$', search_date)):
            print("Invalid input, start date must be date in this format (DD/MM/YYYY), please try again or type exit to return to main menu")
            continue
        search_lst = search_date.split('/')
        search_date_var = datetime.datetime(int(search_lst[-1]), int(search_lst[1]), int(search_lst[0]))
        break

    for email, proj_lst in projects_db.items():
        for proj in proj_lst:
            start_date_var = datetime.datetime.strptime(proj["Start_Date"], "%d/%m/%Y")
            end_date_var = datetime.datetime.strptime(proj["End_Date"], "%d/%m/%Y")

            if search_date_var >= start_date_var and search_date_var <= end_date_var:
                found_projects[email] = found_projects.get(email, [])
                found_projects[email].append(proj)
            else:
                continue

    if len(found_projects) == 0:
        print("No projects found with that date")
        return
    else:
        headers = ["Email", "Projects"]
        print(tabulate(found_projects.items(), headers=headers))
        return


import torch
import torch.nn as nn
import torch.nn.functional as F
from torch import Tensor
from torch.nn import Module
from torch.nn.common_types import (_size_any_t, _size_any_opt_t, _size_2_t,
                                   _size_2_opt_t)
from typing import Optional

class _MaxPoolNd(Module):
  __constants__ = [
      'kernel_size', 'stride', 'padding', 'dilation', 'return_indices',
      'ceil_mode', 'B'
  ]
  return_indices: bool
  ceil_mode: bool
  B: int

  def __init__(self,
               kernel_size: _size_any_t,
               stride: Optional[_size_any_t] = None,
               padding: _size_any_t = 0,
               dilation: _size_any_t = 1,
               return_indices: bool = False,
               ceil_mode: bool = False,
               B: int = 1) -> None:
    super(_MaxPoolNd, self).__init__()
    self.kernel_size = kernel_size
    self.stride = stride if (stride is not None) else kernel_size
    self.padding = padding
    self.dilation = dilation
    self.return_indices = return_indices
    self.ceil_mode = ceil_mode
    self.B = B

  def extra_repr(self) -> str:
    return ('kernel_size={kernel_size}, stride={stride}, padding={padding} '
            ', dilation={dilation}, ceil_mode={ceil_mode}, B={B}'.format(
                **self.__dict__))

class MaxPool2d(_MaxPoolNd):
  r""" Input format: [N B C H W]
  B: Training array size (number of batches/jobs).
  N: Batch size.
  C: Number of channels.
  H: Height.
  W: Width.
  """

  kernel_size: _size_2_t
  stride: _size_2_t
  padding: _size_2_t
  dilation: _size_2_t

  def forward(self, input: Tensor) -> Tensor:
    N, B, C, H, W = input.size()
    res = F.max_pool2d(input.view(N * B, C, H, W), self.kernel_size,
                       self.stride, self.padding, self.dilation, self.ceil_mode,
                       self.return_indices)
    if self.return_indices:
      y, indices = res
    else:
      y = res
    NxB, C, H, W = y.size()
    y = y.view(N, B, C, H, W)
    return (y, indices.view(N, B, C, H, W)) if self.return_indices else y

class _AdaptiveAvgPoolNd(Module):
  __constants__ = ['output_size', 'B']

  def __init__(self, output_size: _size_any_opt_t, B: int = 1) -> None:
    super(_AdaptiveAvgPoolNd, self).__init__()
    self.output_size = output_size
    self.B = B

  def extra_repr(self) -> str:
    return 'output_size={}, B={}'.format(self.output_size, self.B)

class AdaptiveAvgPool2d(_AdaptiveAvgPoolNd):
  r""" Input format: [N B C H W]
  B: Training array size (number of batches/jobs).
  N: Batch size.
  C: Number of channels.
  H: Height.
  W: Width.
  """

  output_size: _size_2_opt_t

  def forward(self, input: Tensor) -> Tensor:
    N, B, C, H, W = input.size()
    output = F.adaptive_avg_pool2d(input.view(N * B, C, H, W), self.output_size)
    NxB, C, H, W = output.size()
    return output.view(N, B, C, H, W)

class _AvgPoolNd(Module):
  __constants__ = [
      'kernel_size', 'stride', 'padding', 'ceil_mode', 'count_include_pad', 'B'
  ]

  def extra_repr(self) -> str:
    return 'kernel_size={}, stride={}, padding={}, B={}'.format(
        self.kernel_size, self.stride, self.padding, self.B)

class AvgPool2d(_AvgPoolNd):
  r"""Applies a 2D average pooling over an input signal composed of several input
    planes.
    The additional argument: "B" is added to the constructor to support fused
    inputs, Otherwise, the usage is identical to those described here:
        https://pytorch.org/docs/stable/generated/torch.nn.AvgPool2d.html

    Shape:
        - Input: :math:`(N, B, C, H_{in}, W_{in})` or :math:`(C, H_{in}, W_{in})`.
        - Output: :math:`(N, B, C, H_{out}, W_{out})` or :math:`(C, H_{out}, W_{out})`, where
          .. math::
              H_{out} = \left\lfloor\frac{H_{in}  + 2 \times \text{padding}[0] -
                \text{kernel\_size}[0]}{\text{stride}[0]} + 1\right\rfloor
          .. math::
              W_{out} = \left\lfloor\frac{W_{in}  + 2 \times \text{padding}[1] -
                \text{kernel\_size}[1]}{\text{stride}[1]} + 1\right\rfloor
    Examples::
        >>> # pool of square window of size=3, stride=2, B=2
        >>> m = nn.AvgPool2d(3, stride=2, B=2)
        >>> # pool of non-square window
        >>> m = nn.AvgPool2d((3, 2), stride=(2, 1, B=2))
        >>> input = torch.randn(20, 2, 16, 50, 32)
        >>> output = m(input)
    """
  __constants__ = [
      'kernel_size', 'stride', 'padding', 'ceil_mode', 'count_include_pad',
      'divisor_override', 'B'
  ]

  kernel_size: _size_2_t
  stride: _size_2_t
  padding: _size_2_t
  ceil_mode: bool
  count_include_pad: bool

  def __init__(self,
               kernel_size: _size_2_t,
               stride: Optional[_size_2_t] = None,
               padding: _size_2_t = 0,
               ceil_mode: bool = False,
               count_include_pad: bool = True,
               divisor_override: Optional[int] = None,
               B: Optional[int] = 0) -> None:
    super(AvgPool2d, self).__init__()
    self.kernel_size = kernel_size
    self.stride = stride if (stride is not None) else kernel_size
    self.padding = padding
    self.ceil_mode = ceil_mode
    self.count_include_pad = count_include_pad
    self.divisor_override = divisor_override
    self.B = B

  def forward(self, input: Tensor) -> Tensor:
    N, B, C, H, W = input.size()
    input = input.view(N * B, C, H, W)
    out = F.avg_pool2d(input, self.kernel_size, self.stride, self.padding,
                       self.ceil_mode, self.count_include_pad,
                       self.divisor_override)
    _, _, H_out, W_out = out.size()
    return out.view(N, B, C, H_out, W_out)