#!/usr/bin/env python
"""This implements MoE layers for modeling."""

from dataclasses import dataclass
import math
from typing import Optional, Tuple
import torch
import torch.nn as nn
from transformers import BertLayer

try:
    from transformers.modeling_bert import (
        BertAttention,
        BertIntermediate,
        BertOutput,
        BertEmbeddings,
        BertPooler,
        BertPreTrainedModel,
        BertSelfOutput,
    )
    from transformers.modeling_utils import (
        apply_chunking_to_forward,
        find_pruneable_heads_and_indices,
        prune_linear_layer,
    )
except:
    from transformers.models.bert.modeling_bert import (
        BertAttention,
        BertIntermediate,
        BertOutput,
        BertEmbeddings,
        BertPooler,
        BertPreTrainedModel,
        BertSelfOutput,
    )
    from transformers.modeling_utils import (
        apply_chunking_to_forward,
        find_pruneable_heads_and_indices,
        prune_linear_layer,
    )
from transformers.activations import ACT2FN
import logging

logger = logging.getLogger(__name__)

moe_type_to_func = {
    "mod2": lambda layer_idx: (layer_idx + 1) % 2 == 0,
    "mod3": lambda layer_idx: (layer_idx + 1) % 3 == 0,
    "mod4": lambda layer_idx: (layer_idx + 1) % 4 == 0,
    "mod6": lambda layer_idx: (layer_idx + 1) % 6 == 0,
    "mod12": lambda layer_idx: (layer_idx + 1) % 12 == 0,
    "ge6mod3": lambda layer_idx: (layer_idx + 1) >= 6 and (layer_idx + 1) % 3 == 0,
    "le7mod3": lambda layer_idx: layer_idx <= 6 and (layer_idx % 3 == 0),
    "late6": lambda layer_idx: layer_idx >= 6,
    "late3": lambda layer_idx: layer_idx >= 9,
}

# class BertIntermediate(nn.Module):
#     def __init__(self, config):
#         super().__init__()
#         self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
#         if isinstance(config.hidden_act, str):
#             self.intermediate_act_fn = ACT2FN[config.hidden_act]
#         else:
#             self.intermediate_act_fn = config.hidden_act

#     def forward(self, hidden_states):
#         hidden_states = self.dense(hidden_states)
#         hidden_states = self.intermediate_act_fn(hidden_states)
#         return hidden_states

# class BertOutput(nn.Module):
#     def __init__(self, config):
#         super().__init__()
#         self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
#         self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
#         self.dropout = nn.Dropout(config.hidden_dropout_prob)

#     def forward(self, hidden_states, input_tensor):
#         hidden_states = self.dense(hidden_states)
#         hidden_states = self.dropout(hidden_states)
#         hidden_states = self.LayerNorm(hidden_states + input_tensor)
#         return hidden_states

def slice_for_expert(all_outputs, expert_idx):
    """Selects the expert for forwarding."""
    num_experts, bsz, seq_len, dim = all_outputs.shape
    offset = torch.arange(0, bsz).to(expert_idx.device)
    slice_indices = expert_idx * bsz + offset
    # slice_indices = torch.Tensor([bsz * idx + bs_idx for bs_idx, idx in enumerate(expert_idx)]).type(torch.int64)
    all_outputs = all_outputs.view(bsz * num_experts, seq_len, dim)
    selected_outputs = all_outputs[slice_indices, :, :]
    return selected_outputs

def select_fwd(x, expert_idx, interm_modules, output_modules):
   """Forwards inputs with different experts in the given batch."""

   # Forwards all experts and slices the target expert outputs.
   # Tensor of shape [num_expert, bsz, seq_len, dim].
   # all_interm_outputs = torch.stack([
   #     mm_layer(x.clone()) for mm_layer in interm_modules], dim=0)
   # interm_outputs = slice_for_expert(all_interm_outputs, expert_idx)
   interm_outputs = single_select_fwd(x, expert_idx, interm_modules)

   # Tensor of shape [num_expert, bsz, seq_len, dim].
   # all_final_outputs = torch.stack([
   #     out_layer(interm_outputs.clone(), x.clone())
   #     for out_layer in output_modules], dim=0)
   # final_outputs = slice_for_expert(all_final_outputs, expert_idx)
   final_outputs = single_select_fwd(interm_outputs, expert_idx, output_modules, y=x)

   return final_outputs

def single_select_fwd(x, expert_idx, output_modules, y=None):
    """Forwards inputs with different experts in the given batch."""
    if y is None:
        all_outputs = torch.stack([
            mm_layer(x.clone()) for mm_layer in output_modules], dim=0)
    else:
        all_outputs = torch.stack([
            mm_layer(x.clone(), y.clone()) for mm_layer in output_modules], dim=0)
    final_outputs = slice_for_expert(all_outputs, expert_idx)
    return final_outputs

class MoELayer(nn.Module):
    def __init__(self, config, n_expert=1):
        super().__init__()
        self.interm_layers = nn.ModuleList(
            [BertIntermediate(config) for _ in range(n_expert)]
        )
        self.output_layers = nn.ModuleList(
            [BertOutput(config) for _ in range(n_expert)]
        )

    def forward(self, input_tensor, expert_idx):
        if isinstance(expert_idx, int):
            intermediate_outputs = self.interm_layers[expert_idx](input_tensor)
            hidden_states = self.output_layers[expert_idx](intermediate_outputs, input_tensor)
        elif isinstance(expert_idx, torch.Tensor):
            assert expert_idx.dtype == torch.int64, "only int64 is supported for slicing experts."
            hidden_states = select_fwd(input_tensor, expert_idx, self.interm_layers, self.output_layers)
        else:
            raise ValueError("Unknown expert_idx type", type(expert_idx))
        return hidden_states

def select_expert(bsz, num_expert, expert_id=None):
    if expert_id is None:
        return torch.randint(low=0, high=num_expert, size=(bsz,)).tolist()
    return expert_id

def _expert_forward(fwd_func_list, input_tensor, expert_idx):
    if isinstance(expert_idx, int):
        outputs = fwd_func_list[expert_idx](input_tensor)
    elif isinstance(expert_idx, torch.Tensor):
        assert expert_idx.dtype == torch.int64, "only int64 is supported for slicing experts."
        outputs = single_select_fwd(input_tensor, expert_idx, fwd_func_list)
    else:
        raise ValueError("Unknown expert_idx type", type(expert_idx))
    return outputs

class MoEBertSelfAttention(nn.Module):
    def __init__(self, config, n_expert=1):
        super().__init__()
        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
            raise ValueError(
                f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
                f"heads ({config.num_attention_heads})"
            )

        self.num_attention_heads = config.num_attention_heads
        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
        self.all_head_size = self.num_attention_heads * self.attention_head_size

        self.moe_query= nn.ModuleList(
            [nn.Linear(config.hidden_size, self.all_head_size) for _ in range(n_expert)])
        self.moe_key= nn.ModuleList(
            [nn.Linear(config.hidden_size, self.all_head_size) for _ in range(n_expert)])
        self.moe_value= nn.ModuleList(
            [nn.Linear(config.hidden_size, self.all_head_size) for _ in range(n_expert)])

        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)
        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
        if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
            self.max_position_embeddings = config.max_position_embeddings
            self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size)

        self.is_decoder = config.is_decoder

    def transpose_for_scores(self, x):
        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
        x = x.view(*new_x_shape)
        return x.permute(0, 2, 1, 3)

    def forward(
        self,
        hidden_states,
        expert_idx,
        attention_mask=None,
        head_mask=None,
        encoder_hidden_states=None,
        encoder_attention_mask=None,
        past_key_value=None,
        output_attentions=False,
    ):
        mixed_query_layer = _expert_forward(self.moe_query, hidden_states, expert_idx)

        # If this is instantiated as a cross-attention module, the keys
        # and values come from an encoder; the attention mask needs to be
        # such that the encoder's padding tokens are not attended to.
        is_cross_attention = encoder_hidden_states is not None

        if is_cross_attention and past_key_value is not None:
            # reuse k,v, cross_attentions
            key_layer = past_key_value[0]
            value_layer = past_key_value[1]
            attention_mask = encoder_attention_mask
        elif is_cross_attention:
            key_layer = self.transpose_for_scores(
                _expert_forward(self.moe_key, encoder_hidden_states, expert_idx))
            value_layer = self.transpose_for_scores(
                _expert_forward(self.moe_value, encoder_hidden_states, expert_idx))
            attention_mask = encoder_attention_mask
        elif past_key_value is not None:
            key_layer = self.transpose_for_scores(
                _expert_forward(self.moe_key, hidden_states, expert_idx))
            value_layer = self.transpose_for_scores(
                _expert_forward(self.moe_value, hidden_states, expert_idx))
            key_layer = torch.cat([past_key_value[0], key_layer], dim=2)
            value_layer = torch.cat([past_key_value[1], value_layer], dim=2)
        else:
            key_layer = self.transpose_for_scores(
                _expert_forward(self.moe_key, hidden_states, expert_idx))
            value_layer = self.transpose_for_scores(
                _expert_forward(self.moe_value, hidden_states, expert_idx))

        query_layer = self.transpose_for_scores(mixed_query_layer)

        if self.is_decoder:
            # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states.
            # Further calls to cross_attention layer can then reuse all cross-attention
            # key/value_states (first "if" case)
            # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of
            # all previous decoder key/value_states. Further calls to uni-directional self-attention
            # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
            # if encoder bi-directional self-attention `past_key_value` is always `None`
            past_key_value = (key_layer, value_layer)

        # Take the dot product between "query" and "key" to get the raw attention scores.
        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))

        if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
            seq_length = hidden_states.size()[1]
            position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1)
            position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1)
            distance = position_ids_l - position_ids_r
            positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1)
            positional_embedding = positional_embedding.to(dtype=query_layer.dtype)  # fp16 compatibility

            if self.position_embedding_type == "relative_key":
                relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
                attention_scores = attention_scores + relative_position_scores
            elif self.position_embedding_type == "relative_key_query":
                relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
                relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
                attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key

        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
        if attention_mask is not None:
            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
            attention_scores = attention_scores + attention_mask

        # Normalize the attention scores to probabilities.
        attention_probs = nn.Softmax(dim=-1)(attention_scores)

        # This is actually dropping out entire tokens to attend to, which might
        # seem a bit unusual, but is taken from the original Transformer paper.
        attention_probs = self.dropout(attention_probs)

        # Mask heads if we want to
        if head_mask is not None:
            attention_probs = attention_probs * head_mask

        context_layer = torch.matmul(attention_probs, value_layer)

        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
        context_layer = context_layer.view(*new_context_layer_shape)

        outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)

        if self.is_decoder:
            outputs = outputs + (past_key_value,)
        return outputs

class MoEBertSelfOutput(nn.Module):
    def __init__(self, config, n_expert=1):
        super().__init__()
        self.moe_dense = nn.ModuleList(
            [nn.Linear(config.hidden_size, config.hidden_size) for _ in range(n_expert)])

        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

    def forward(self, hidden_states, input_tensor, expert_idx):
        if isinstance(expert_idx, int):
            hidden_states = self.moe_dense[expert_idx](hidden_states)
        elif isinstance(expert_idx, torch.Tensor):
            assert expert_idx.dtype == torch.int64, "only int64 is supported for slicing experts."
            hidden_states = single_select_fwd(hidden_states, expert_idx, self.moe_dense)
        else:
            raise ValueError("Unknown expert_idx type", type(expert_idx))
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states

class MoEBertAttention(nn.Module):
    def __init__(self, config, n_expert=1):
        super().__init__()
        self.self = MoEBertSelfAttention(config, n_expert=n_expert)
        self.output = MoEBertSelfOutput(config, n_expert=n_expert)
        self.pruned_heads = set()

    def prune_heads(self, heads):
        if len(heads) == 0:
            return
        heads, index = find_pruneable_heads_and_indices(
            heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads
        )

        # Prune linear layers
        self.self.query = prune_linear_layer(self.self.query, index)
        self.self.key = prune_linear_layer(self.self.key, index)
        self.self.value = prune_linear_layer(self.self.value, index)
        self.output.dense = prune_linear_layer(self.output.dense, index, dim=1)

        # Update hyper params and store pruned heads
        self.self.num_attention_heads = self.self.num_attention_heads - len(heads)
        self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads
        self.pruned_heads = self.pruned_heads.union(heads)

    def forward(
        self,
        hidden_states,
        expert_idx,
        attention_mask=None,
        head_mask=None,
        encoder_hidden_states=None,
        encoder_attention_mask=None,
        past_key_value=None,
        output_attentions=False,
    ):
        self_outputs = self.self(
            hidden_states,
            expert_idx,
            attention_mask,
            head_mask,
            encoder_hidden_states,
            encoder_attention_mask,
            past_key_value,
            output_attentions,
        )
        attention_output = self.output(self_outputs[0], hidden_states, expert_idx)
        outputs = (attention_output,) + self_outputs[1:]  # add attentions if we output them
        return outputs

class MoEBertLayer(nn.Module):
    def __init__(self, config, use_attn_moe=False, use_fwd_moe=True, is_tok_moe=False):
        super().__init__()
        self.chunk_size_feed_forward = config.chunk_size_feed_forward  # 3.0.2 does not have this yet, set to 0
        self.seq_len_dim = 1
        self.num_expert = config.num_expert
        self.use_attn_moe = use_attn_moe
        self.use_fwd_moe = use_fwd_moe
        self.is_tok_moe = is_tok_moe
        # if self.num_expert <= 1:
        #     raise ValueError("num_expert is required >1")

        if self.use_attn_moe and self.is_tok_moe:
            raise ValueError("Currently, is_tok_moe and use_attn_moe can not be set at the same time.")

        if config.is_decoder and self.is_tok_moe:
            raise ValueError("Currently, decoder does not support token-level MoE.")

        if self.use_attn_moe:
            self.attention = MoEBertAttention(config, n_expert=self.num_expert)
        else:
            self.attention = BertAttention(config)
        self.is_decoder = config.is_decoder
        self.add_cross_attention = config.add_cross_attention  # 3.0.2 does not have this yet, set to False
        if self.add_cross_attention:
            if not self.is_decoder:
                raise ValueError(f"{self} should be used as a decoder model if cross attention is added")
            self.crossattention = BertAttention(config)

        if self.use_fwd_moe:
            self.moe_layer = MoELayer(config, n_expert=self.num_expert)
        else:
            self.output = BertOutput(config)
            self.intermediate = BertIntermediate(config)

    def forward(
        self,
        hidden_states,
        attention_mask=None,
        head_mask=None,
        encoder_hidden_states=None,
        encoder_attention_mask=None,
        past_key_value=None,
        output_attentions=False,
        expert_id=None,
    ):
        # decoder uni-directional self-attention cached key/values tuple is at positions 1,2
        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
        batch_size, seq_len, hidden_dim = hidden_states.shape

        attn_inputs = {
            "hidden_states": hidden_states,
            "attention_mask": attention_mask,
            "head_mask": head_mask,
            "output_attentions": output_attentions,
            #"past_key_value": self_attn_past_key_value,
        }

        if self.use_attn_moe:
            attn_inputs["expert_idx"] = expert_id
        self_attention_outputs = self.attention(**attn_inputs)
        attention_output = self_attention_outputs[0]

        # if decoder, the last output is tuple of self-attn cache
        if self.is_decoder:
            outputs = self_attention_outputs[1:-1]
            present_key_value = self_attention_outputs[-1]
        else:
            outputs = self_attention_outputs[1:]  # add self attentions if we output attention weights

        cross_attn_present_key_value = None
        if self.is_decoder and encoder_hidden_states is not None:
            if not hasattr(self, "crossattention"):
                raise ValueError(
                    f"If `encoder_hidden_states` are passed, {self} has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`"
                )

            # cross_attn cached key/values tuple is at positions 3,4 of past_key_value tuple
            cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
            cross_attention_outputs = self.crossattention(
                attention_output,
                attention_mask,
                head_mask,
                encoder_hidden_states,
                encoder_attention_mask,
                cross_attn_past_key_value,
                output_attentions,
            )
            attention_output = cross_attention_outputs[0]
            outputs = outputs + cross_attention_outputs[1:-1]  # add cross attentions if we output attention weights

            # add cross-attn cache to positions 3,4 of present_key_value tuple
            cross_attn_present_key_value = cross_attention_outputs[-1]
            present_key_value = present_key_value + cross_attn_present_key_value

        if self.is_tok_moe:
            # Reshapes inputs for token level MoE.
            attention_output = attention_output.view(batch_size * seq_len, 1, hidden_dim)
            self.expert_id = select_expert(batch_size * seq_len, self.num_expert, expert_id=expert_id)
        else:
            self.expert_id = select_expert(batch_size, self.num_expert, expert_id=expert_id)

        layer_output = apply_chunking_to_forward(
            self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output
        )
        # KM: for transformers 3.0.2, this function arguments have different ordering
        # layer_output = apply_chunking_to_forward(
        #     self.chunk_size_feed_forward, self.seq_len_dim, self.feed_forward_chunk, attention_output
        # )

        if self.is_tok_moe:
            # Reshapes the outupt back to the input shape.
            layer_output = layer_output.view(batch_size, seq_len, hidden_dim)

        outputs = (layer_output,) + outputs

        # if decoder, return the attn key/values as the last output
        if self.is_decoder:
            outputs = outputs + (present_key_value,)

        return outputs

    def feed_forward_chunk(self, attention_output):
        if self.use_fwd_moe:
            layer_output = self.moe_layer(attention_output, self.expert_id)
        else:
            intermediate_output = self.intermediate(attention_output)
            layer_output = self.output(intermediate_output, attention_output)
        return layer_output

def sample_gumbel(shape, epsilon=1e-8):
    """Samples from gumbel distribution."""
    gk = torch.rand(shape)
    return -torch.log(-torch.log(gk + epsilon) + epsilon)

def gumbel_softmax(logits, tau=1.0, hard=True):
    y = logits + sample_gumbel(logits.shape).to(logits.device)
    y = torch.nn.functional.softmax(y/tau, dim=-1)
    if hard:
        max_y, _ = torch.max(y, dim=-1)
        y = (max_y.view(-1, 1) - y).detach() + y
    return y

def onehot_max(logits):
    _, max_ind = torch.max(logits, dim=-1)
    y = torch.nn.functional.one_hot(max_ind, num_classes=logits.size(-1))
    return y

class ExpertGating(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.hidden_dim = config.hidden_size
        self.num_expert = config.num_expert
        self.fwd_func = nn.Linear(self.hidden_dim, self.num_expert)
        self.fwd_func.weight.data.normal_(mean=0.0, std=0.02)

    def forward(self, x, tau=1.0, mask=None):
        logits = self.fwd_func(x)
        if mask is not None:
            logits += torch.log(mask)

        if self.training:
            # one_hot = gumbel_softmax(logits, tau=tau, hard=True)
            one_hot = torch.nn.functional.gumbel_softmax(logits, tau=tau, hard=True)
        else:
            one_hot = onehot_max(logits)

        _, max_ind = torch.max(one_hot, dim=-1)
        weight = torch.sum(one_hot, dim=-1, keepdim=True)

        p_y = nn.functional.softmax(logits, dim=-1)
        log_p_y = nn.functional.log_softmax(logits, dim=-1)
        entropic_loss = torch.sum(p_y * log_p_y, dim=-1).mean()

        return weight, max_ind, entropic_loss

class MoEBertEncoder(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        moe_type = config.moe_type.split(":")[0]
        self.use_infer_expert = config.use_infer_expert
        self.per_layer_gating = config.per_layer_gating

        # TODO(chenghao): Makes this configurable later.
        self.per_layer_gate_input = True
        is_moe_layer = moe_type_to_func[moe_type]
        logger.info(f"Using {moe_type} for MoEBERT")
        logger.info(f"Using {config.moe_type}")
        self.is_tok_moe = False
        if "tok" in config.moe_type.split(":")[1]:
            self.is_tok_moe = True
            logger.info("Using token-level moe")
        else:
            logger.info("Using sentence-level moe")

        if "attn" in config.moe_type.split(":")[1]:
            use_attn_moe = True
            logger.info("Using moe for attention layers")
        else:
            use_attn_moe = False
        if "fwd" in config.moe_type.split(":")[1]:
            use_fwd_moe = True
            logger.info("Using moe for forward layers")
        else:
            use_fwd_moe = False

        self.layer = nn.ModuleList([
            MoEBertLayer(config, use_attn_moe=use_attn_moe, use_fwd_moe=use_fwd_moe, is_tok_moe=self.is_tok_moe)
            if is_moe_layer(layer_id) else BertLayer(config) for layer_id in range(config.num_hidden_layers)])
        self.num_gate_layer = sum([
            1.0
            if is_moe_layer(layer_id) else 0.0 for layer_id in range(config.num_hidden_layers)])
        if self.use_infer_expert:
            logger.info("Using gating for selecting expert")
            if self.per_layer_gating:
                logger.info("Using per layer gating")
                self.expert_gate = nn.ModuleList([ExpertGating(config) if is_moe_layer(layer_id) else None for layer_id in range(config.num_hidden_layers)])
            else:
                logger.info("Using single layer for all gating")
                if self.per_layer_gate_input:
                    logger.info("Each layer will select different gate")
                else:
                    logger.info("All layers will select the same gate")
                self.expert_gate = ExpertGating(config)
        else:
            logger.info("Using predefined expert")
            self.expert_gate = None
        self.gradient_checkpointing = False

    def forward(
        self,
        hidden_states,
        attention_mask=None,
        head_mask=None,
        encoder_hidden_states=None,
        encoder_attention_mask=None,
        past_key_values=None,
        use_cache=None,
        output_attentions=False,
        output_hidden_states=False,
        return_dict=True,
        expert_id=None,
        expert_id_offset=0,
    ):
        all_hidden_states = () if output_hidden_states else None
        all_self_attentions = () if output_attentions else None
        all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None

        if self.expert_gate is not None and expert_id is not None:
            logger.warning("When expert_id is specified, the expert_gate is not used")

        if not self.use_infer_expert:
            if expert_id is None:
                raise ValueError("expert_id is required when use_infer_expert=False")
            if self.is_tok_moe and expert_id is not None:
                raise ValueError("Input expert_id is not allowed for token-level MoE without infer expert.")

        batch_size, seq_len, hidden_dim = hidden_states.shape
        next_decoder_cache = () if use_cache else None
        expert_weight = None
        total_entropy_loss = 0.0
        for i, layer_module in enumerate(self.layer):
            if output_hidden_states:
                all_hidden_states = all_hidden_states + (hidden_states,)

            layer_head_mask = head_mask[i] if head_mask is not None else None
            past_key_value = past_key_values[i] if past_key_values is not None else None

            is_moe_layer = isinstance(layer_module, MoEBertLayer)

            if self.expert_gate is not None and is_moe_layer:
                if self.is_tok_moe:
                    # For token level moe, reshapes the input into the shape of
                    # [batch_size x seq_len, hidden_dim].
                    gate_input = hidden_states.view(-1, hidden_dim)
                else:
                    # TODO(chenghao): makes the sentinel token configurable.
                    gate_input = hidden_states[:, 0, :]
                if self.per_layer_gating:
                    expert_weight, local_expert_id, entro_loss = self.expert_gate[i](gate_input)
                    total_entropy_loss += entro_loss
                elif self.per_layer_gate_input or expert_weight is None:
                    expert_weight, local_expert_id, entro_loss = self.expert_gate(gate_input)
                    total_entropy_loss += entro_loss
                else:
                    expert_weight = expert_weight.clone()
                    local_expert_id = local_expert_id.clone()
            else:
                local_expert_id = expert_id
                expert_weight = None

            if self.gradient_checkpointing and self.training:

                if use_cache:
                    logger.warning(
                        "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
                    )
                    use_cache = False

                def create_custom_forward(module):
                    def custom_forward(*inputs):
                        return module(*inputs, past_key_value, output_attentions)

                    return custom_forward

                if is_moe_layer:
                    layer_outputs = torch.utils.checkpoint.checkpoint(
                        create_custom_forward(layer_module),
                        hidden_states,
                        attention_mask,
                        layer_head_mask,
                        encoder_hidden_states,
                        encoder_attention_mask,
                        expert_id=local_expert_id,
                    )
                else:
                    layer_outputs = torch.utils.checkpoint.checkpoint(
                        create_custom_forward(layer_module),
                        hidden_states,
                        attention_mask,
                        layer_head_mask,
                        encoder_hidden_states,
                        encoder_attention_mask,
                    )
            else:
                if is_moe_layer:
                    layer_outputs = layer_module(
                        hidden_states,
                        attention_mask,
                        layer_head_mask,
                        encoder_hidden_states,
                        encoder_attention_mask,
                        past_key_value,
                        output_attentions,
                        expert_id=local_expert_id,
                    )
                else:
                    layer_outputs = layer_module(
                        hidden_states,
                        attention_mask,
                        layer_head_mask,
                        encoder_hidden_states,
                        encoder_attention_mask,
                        past_key_value,  # for 3.0.2 compatibility, comment this out
                        output_attentions,
                    )

            hidden_states = layer_outputs[0]
            # if expert_weight is not None:
            #     if self.is_tok_moe:
            #         hidden_states *= expert_weight.view(batch_size, seq_len, 1)
            #     else:
            #         hidden_states *= expert_weight.view(batch_size, 1, 1)
            if use_cache:
                next_decoder_cache += (layer_outputs[-1],)
            if output_attentions:
                all_self_attentions = all_self_attentions + (layer_outputs[1],)
                if self.config.add_cross_attention:
                    all_cross_attentions = all_cross_attentions + (layer_outputs[2],)

        if output_hidden_states:
            all_hidden_states = all_hidden_states + (hidden_states,)

        total_entropy_loss /= self.num_gate_layer

        if not return_dict:
            return tuple(
                v
                for v in [
                    hidden_states,
                    next_decoder_cache,
                    all_hidden_states,
                    all_self_attentions,
                    all_cross_attentions,
                ]
                if v is not None
            )
        return (hidden_states, next_decoder_cache, all_hidden_states, all_self_attentions, all_cross_attentions)

class MoEBertModel(BertPreTrainedModel):
    """

    The model can behave as an encoder (with only self-attention) as well
    as a decoder, in which case a layer of cross-attention is added between
    the self-attention layers, following the architecture described in `Attention is all you need`_ by Ashish Vaswani,
    Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the
    :obj:`is_decoder` argument of the configuration set to :obj:`True`; an
    :obj:`encoder_hidden_states` is expected as an input to the forward pass.

    .. _`Attention is all you need`:
        https://arxiv.org/abs/1706.03762

    """

    def __init__(self, config):
        super().__init__(config)
        self.config = config

        self.embeddings = BertEmbeddings(config)
        self.encoder = MoEBertEncoder(config)
        self.pooler = BertPooler(config)

        self.init_weights()

    def get_input_embeddings(self):
        return self.embeddings.word_embeddings

    def set_input_embeddings(self, value):
        self.embeddings.word_embeddings = value

    def _prune_heads(self, heads_to_prune):
        """ Prunes heads of the model.
            heads_to_prune: dict of {layer_num: list of heads to prune in this layer}
            See base class PreTrainedModel
        """
        for layer, heads in heads_to_prune.items():
            self.encoder.layer[layer].attention.prune_heads(heads)

    def forward(
        self,
        input_ids=None,
        attention_mask=None,
        token_type_ids=None,
        position_ids=None,
        head_mask=None,
        inputs_embeds=None,
        encoder_hidden_states=None,
        encoder_attention_mask=None,
        output_attentions=None,
        output_hidden_states=None,
        expert_id=None,
        expert_id_offset=0,
    ):
        r"""
    Return:
        :obj:`tuple(torch.FloatTensor)` comprising various elements depending on the configuration (:class:`~transformers.BertConfig`) and inputs:
        last_hidden_state (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        pooler_output (:obj:`torch.FloatTensor`: of shape :obj:`(batch_size, hidden_size)`):
            Last layer hidden-state of the first token of the sequence (classification token)
            further processed by a Linear layer and a Tanh activation function. The Linear
            layer weights are trained from the next sentence prediction (classification)
            objective during pre-training.

            This output is usually *not* a good summary
            of the semantic content of the input, you're often better with averaging or pooling
            the sequence of hidden-states for the whole input sequence.
        hidden_states (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_hidden_states=True`` is passed or when ``config.output_hidden_states=True``):
            Tuple of :obj:`torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
            of shape :obj:`(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (:obj:`tuple(torch.FloatTensor)`, `optional`, returned when ``output_attentions=True`` is passed or when ``config.output_attentions=True``):
            Tuple of :obj:`torch.FloatTensor` (one for each layer) of shape
            :obj:`(batch_size, num_heads, sequence_length, sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
        """
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )

        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
        elif input_ids is not None:
            input_shape = input_ids.size()
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.size()[:-1]
        else:
            raise ValueError("You have to specify either input_ids or inputs_embeds")

        device = input_ids.device if input_ids is not None else inputs_embeds.device

        if attention_mask is None:
            attention_mask = torch.ones(input_shape, device=device)
        if token_type_ids is None:
            token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device)

        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
        # ourselves in which case we just need to make it broadcastable to all heads.
        extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(attention_mask, input_shape, device)

        # If a 2D ou 3D attention mask is provided for the cross-attention
        # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length]
        if self.config.is_decoder and encoder_hidden_states is not None:
            encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
            if encoder_attention_mask is None:
                encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
            encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
        else:
            encoder_extended_attention_mask = None

        # Prepare head mask if needed
        # 1.0 in head_mask indicate we keep the head
        # attention_probs has shape bsz x n_heads x N x N
        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
        head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)

        embedding_output = self.embeddings(
            input_ids=input_ids, position_ids=position_ids, token_type_ids=token_type_ids, inputs_embeds=inputs_embeds
        )

        encoder_outputs = self.encoder(
            embedding_output,
            attention_mask=extended_attention_mask,
            head_mask=head_mask,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_extended_attention_mask,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            expert_id=expert_id,
            expert_id_offset=expert_id_offset,
        )
        sequence_output = encoder_outputs[0]
        pooled_output = self.pooler(sequence_output)

        outputs = (sequence_output, pooled_output,) + encoder_outputs[
            1:
        ] + (embedding_output,)  # add hidden_states and attentions if they are here
        return outputs  # sequence_output, pooled_output, (hidden_states), (attentions)


from django.core.urlresolvers import reverse
from test_plus.test import TestCase, settings

class TestHomeView(TestCase):

    def test_login_required(self):
        home_url = 'home'

        response = self.get(home_url)
        reversed_url = reverse(home_url)
        login_url = reverse(settings.LOGIN_URL)
        expected_url = "{0}?next={1}".format(login_url, reversed_url)
        self.assertRedirects(response, expected_url)


import cv2 as cv
import copy as copy
import sys
sys.path.append('FacialDetection')
from helperFunctions import *
from ImageManager import ImageManager

image = cv.imread('human\people2.jpg')
image2 = cv.imread('human\people2.jpg')
# image = resizeMinTo(image,800)
# print("DEBUG ", image[2])

imgMngr = ImageManager(image)

# eyes_rect = imgMngr.findPairsOfEyes(1.1, 10)

# print(f'Number of eyes found = {len(eyes_rect)}')

# for pair in eyes_rect:
#     pair[0].draw(image, (0,255,0), thickness= 2)
#     pair[1].draw(image, (0,255,0), thickness= 2)

# faces_rect = imgMngr.findFacesUsingPairOfEyes(eyes_rect, 1.1, 10)
faces_rect = imgMngr.findFaces(1.1, 10)

for face in faces_rect:
    face.draw(image, (0,0,255), thickness= 2)

image2Gray = cv.cvtColor(image2, cv.COLOR_BGR2GRAY)
faces_rect_default = imgMngr.haarcascade_face.detectMultiScale(image2Gray, 1.1, 10)

for (x,y,w,h) in faces_rect_default:
    cv.rectangle(image2, (x,y), (x + w, y + h), (0,255,0), 2)

# og = cv.resize(og, dimensions, interpolation=cv.INTER_AREA)
cv.imshow('My method', image)
cv.imshow('Default method', image2)

cv.waitKey(0)


import logging
import scrapy
import os
from scrapy import Spider
from vandenborre.items import Manual

logger = logging.getLogger(__name__)

class VandenBorreSpider(Spider):
    name = "vandenborre.be"
    start_urls = [
        "https://www.vandenborre.be/sitemap/nl/default/",
        "https://www.vandenborre.be/sitemap/fr/default/",
        ]

    def parse(self, response):
        for section in response.css('.border-btm-grey-sm.margin-btm-10-sm'):
            if not section:
                continue
            for goto in section.css('.listicon.clearfix.margin-btm-60-md.margin-btm-20-sm.margin-top-20-sm.js-content.hide-sm .nolist.flex.flex-wrap .col-xs-12.col-md-3 a::attr(href)').extract():
                goto = response.urljoin(goto)
                yield scrapy.Request(url=goto, callback=self.get_each_brand_li)

    def get_each_brand_li(self, response):
        for link in response.css('.border-btm-grey-sm.margin-btm-10-sm'):
            if not link:
                continue
            for goto in link.css('.listicon.clearfix.margin-btm-60-md.js-content.margin-btm-20-sm.margin-top-20-sm.hide-sm .nolist.flex.flex-wrap .col-xs-12.col-md-3 a::attr(href)').extract():
                goto = response.urljoin(goto)
                yield scrapy.Request(url=goto, callback=self.get_product_data)

    def get_product_data(self, response):
        pdf_files = response.css('.js-specs-doc.clearfix .doc-link.flex div>a::attr(href)').extract()
        if len(pdf_files) == 0:
            return
        for pdf in pdf_files:
            pdf = pdf.replace(' ', '%20') # removing spaces from the pdf link
            manual = Manual()
            bread_crums = response.css('.breadcrumbs.js-breadcrumbs.hidden-print.w-100 .row .col-xs-12 ul>li')
            manual["product"] = bread_crums[-2].css('.js-ellipsis *::text').extract_first()
            brand = response.css('.product-detail-title.no-margin-btm a *::text').extract_first()
            manual["brand"] = brand.strip()
            thumb = response.css('.image-gallery-main-image.js-image-gallery-main-image.js-image-zoom-image.image-gallery-main-image-0::attr(data-large-image)').extract_first()
            manual["thumb"] = response.urljoin(thumb)
            # nested_fonts = response.css('.product-detail-title.no-margin-btm::text').extract_first()
            # model = nested_fonts.css('font ::text').extract_first()
            model = response.css('.product-detail-title.no-margin-btm::text').extract_first()
            manual["model"] = model
            # manual["model"] = model.strip()
            manual["source"] = self.name
            manual["file_urls"] = [response.urljoin(pdf)]
            manual["url"] = response.urljoin('')
            manual["type"] = response.css('.js-specs-doc.clearfix .doc-link.flex div>a *::text').extract_first()
            print(manual)
            yield manual


import time
from django.core.management.base import BaseCommand
from ...fetcher import *

class Command(BaseCommand):
    help = "Fetches new commits from the repository under test every 10 minutes"

    def handle(self, *args, **options):
        while True:
            fetch_main_branch_revisions()
            fetch_staging_revisions()
            print("Sleeping for 10 mins...")
            time.sleep(10 * 60)


import logging

from run_match import play_matches, TEST_AGENTS, NUM_PROCS, NUM_ROUNDS, TIME_LIMIT
from argparse import Namespace
from isolation import Agent
from my_standard_player import AlphaBetaPlayer

logger = logging.getLogger(__name__)

args = Namespace()
setattr(args, 'debug', False)
setattr(args, 'fair_matches', True)
setattr(args, 'processes', NUM_PROCS)
setattr(args, 'rounds', NUM_ROUNDS)
setattr(args, 'time_limit', TIME_LIMIT)

if __name__ == "__main__":

    test_agent = Agent(AlphaBetaPlayer, "Alpha Beta Agent")
    custom_agent = TEST_AGENTS["SELF"]
    wins, num_games = play_matches(custom_agent, test_agent, args)

    logger.info("Your agent won {:.1f}% of matches against {}".format(
       100. * wins / num_games, test_agent.name))
    print("Your agent won {:.1f}% of matches against {}".format(
       100. * wins / num_games, test_agent.name))
    print()


# -*- encoding: utf-8 -*-
from odoo import api, fields, models, _, tools, release
from odoo.exceptions import UserError
from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT, DEFAULT_SERVER_DATE_FORMAT
import odoo.addons.decimal_precision as dp
import logging
_logger = logging.getLogger(__name__)

class AccountMoveLine(models.Model):
    _inherit = 'account.move.line'

    @api.depends('quantity','product_uom_id','product_id')
    def _compute_cce_quantity(self):
        for rec in self:
            if not rec.display_type and rec.product_id and rec.product_id.sat_factor_conversion:
                if rec.product_id.uom_id != rec.product_uom_id:
                    rec.cce_quantity = rec.product_uom_id._compute_quantity(rec.quantity, rec.product_id.uom_id) * rec.product_id.sat_factor_conversion

                else:
                    rec.cce_quantity = rec.quantity * rec.product_id.sat_factor_conversion
            else:
                rec.cce_quantity = 0

    cce_complemento = fields.Selection(related="move_id.cfdi_complemento")
    cce_quantity = fields.Float(string='Cant. CCE', digits='Product Unit of Measure',
                               compute="_compute_cce_quantity", store=True)
    cce_sat_arancel_id = fields.Many2one('sat.arancel', string="Fracción Arancelaria",
                                        related="product_id.sat_arancel_id", store=True, readonly=True)
    cce_uom = fields.Selection(related="cce_sat_arancel_id.unidad_de_medida",
                              store=True)

class AccountMove(models.Model):
    _inherit = 'account.move'

    @api.depends('invoice_line_ids', 'state','invoice_date')
    def _get_comercio_exterior_total_usd(self):
        currency_usd = self.env['res.currency'].search([('name','=','USD')], limit=1)
        for rec in self:
            totalusd = 0.0
            if rec.move_type == 'out_invoice' and rec.cfdi_complemento=='comercio_exterior':
                for line in self.invoice_line_ids:
                    totalusd += rec.currency_id._convert((line.price_unit * line.quantity), currency_usd, rec.company_id, rec.invoice_date or fields.Date.today())
                    #totalusd += round(rec.currency_id.with_context(date=rec.invoice_date).compute(line.price_unit * line.quantity, currency_usd),2)
            rec.cfdi_comercio_exterior_total_usd = totalusd

    cfdi_complemento = fields.Selection(selection_add=[('comercio_exterior', 'Comercio Exterior')],
                                        ondelete={'comercio_exterior': 'set default'})
    cfdi_motivo_traslado =fields.Selection([('01','Envío de mercancias facturadas con anterioridad'),
                                            ('02','Reubicación de mercancías propias'),
                                            ('03','Envío de mercancías objeto de contrato de consignación'),
                                            ('04','Envío de mercancías para posterior enajenación'),
                                            ('05','Envío de mercancías propiedad de terceros'),
                                            ('99','Otros')],
                                           readonly=True, states={'draft': [('readonly', False)]}, copy=False,
                                         string="Motivo de Traslado")
    cfdi_certificado_origen = fields.Selection([('0','[0] No Funge como certificado de origen'),
                                                ('1','[1] Funge como certificado de origen')],
                                               readonly=True, states={'draft': [('readonly', False)]}, copy=False,
                                               string="Certificado Origen")
    cfdi_num_certificado_origen = fields.Char(string="Número Certificado Origen",
                                             readonly=True, states={'draft': [('readonly', False)]}, copy=False)
    #cfdi_num_exportador_confiable = fields.Char(string="Número Exportador Confiable",
    #                                           readonly=True, states={'draft': [('readonly', False)]}, copy=False)
    cfdi_comercio_exterior_notas = fields.Text(string="Observaciones (Complemento Comercio Exterior)",
                                              readonly=True, states={'draft': [('readonly', False)]}, copy=False)
    cfdi_comercio_exterior_total_usd = fields.Float(string="Total USD", digits=(16, 2), compute="_get_comercio_exterior_total_usd",
                                                   readonly=True, store=True)
    cfdi_comercio_exterior_propietario_id  = fields.Many2one('res.partner', string="Propietario",
                                                            readonly=True, states={'draft': [('readonly', False)]}, copy=False)

    @api.onchange('partner_id')
    def _onchange_partner_id(self):
        res = super(AccountMove, self)._onchange_partner_id()
        if self.partner_id.parent_id:
            if self.partner_id.parent_id.cfdi_comercio_exterior_notas:
                self.cfdi_comercio_exterior_notas = self.partner_id.parent_id.cfdi_comercio_exterior_notas
            else:
                if self.partner_id.cfdi_comercio_exterior_notas:
                    self.cfdi_comercio_exterior_notas = self.partner_id.cfdi_comercio_exterior_notas
        else:
            if self.partner_id.cfdi_comercio_exterior_notas:
                self.cfdi_comercio_exterior_notas = self.partner_id.cfdi_comercio_exterior_notas

    def _get_einvoice_complement_dict(self, xcomprobante):
        if self.cfdi_complemento != 'comercio_exterior':
            return super(AccountMove, self)._get_einvoice_complement_dict(xcomprobante)

        if 'transport_document_cfdi' in self._fields and self.transport_document_cfdi:
            raise UserError(_('Advertencia !!! Por el momento no se soporta el Complemento de Comercio Exterior con CFDI de Traslado'))

        comprobante = xcomprobante.copy()

        currency_usd = self.env['res.currency'].search([('name','=','USD')], limit=1)
        rate = currency_usd.with_context(date=self.invoice_date).rate
        rate = rate != 0 and 1.0/rate or 0.0
        if rate == 1.0:
            rate = 1
        else:
            rate = '%.4f' % rate or 1

        if not (rate and self.cfdi_comercio_exterior_total_usd):
            raise UserError(_('Error !!! No ha definido la información relacionada al Tipo de Cambio a usar en Complemento de Comercio Exterior y/o el Total en USD no está definido'))
        complemento = {'cce11:ComercioExterior':
                        {
                        'xmlns:cce11'       : "http://www.sat.gob.mx/ComercioExterior11",
                        'xsi:schemaLocation': "http://www.sat.gob.mx/ComercioExterior11 http://www.sat.gob.mx/sitio_internet/cfd/ComercioExterior11/ComercioExterior11.xsd",
                        'xmlns:xsi'         : "http://www.w3.org/2001/XMLSchema-instance",
                        'Version'           : '1.1',
                        'MotivoTraslado'    : self.cfdi_motivo_traslado,
                        'TipoOperacion'     : '2',
                        'ClaveDePedimento'  : 'A1',
                        'CertificadoOrigen' : self.cfdi_certificado_origen,
                        'NumCertificadoOrigen'  : self.cfdi_num_certificado_origen,
                        'NumeroExportadorConfiable' : self.company_id.cfdi_num_exportador_confiable or '',
                        'Incoterm'          : self.invoice_incoterm_id.code,
                        'Subdivision'       : '0', #self.cfdi_comercio_exterior_subdivision,
                        'Observaciones'     : self.cfdi_comercio_exterior_notas or '.',
                        'TipoCambioUSD'     : rate,
                        'TotalUSD'          : self.cfdi_comercio_exterior_total_usd and "{0:.2f}".format(self.cfdi_comercio_exterior_total_usd) or '',
                        }
                    }

        diccionario = complemento['cce11:ComercioExterior'].copy()
        for key, value in diccionario.items():
            if key not in ('CertificadoOrigen','TotalUSD') and not bool(value):
                complemento['cce11:ComercioExterior'].pop(key)
        if not complemento['cce11:ComercioExterior']['CertificadoOrigen'] and complemento['cce11:ComercioExterior']['NumCertificadoOrigen']:
            raise UserError(_('El campo "Número Certificado Origen" debe estar vacío si "Certificado Origen" está establecido como: "No funge como certificado de origen"'))

        # ------- ------- EMISOR ------- -------
        complemento['cce11:ComercioExterior'].update({'cce11:Emisor': {}})

        address_invoice_parent = self.address_issued_id or False
        if not address_invoice_parent:
            raise UserError(_('Error !!! No se encontró la Dirección de emisión'))

        if address_invoice_parent.persona_fisica and not address_invoice_parent.curp:
            raise UserError(_('Error !!! El Emisor está marcado como Persona Física pero no tiene definido el CURP'))

        if not (address_invoice_parent.street and \
                address_invoice_parent.zip_sat_id.code and \
                address_invoice_parent.state_id.code and \
                address_invoice_parent.country_id.sat_code
               ):
            raise UserError(_("Error !!! La dirección del Emisor parece estar incompleta o algún valor no tiene relacionado el Código SAT correspondiente,"
                              "por favor revise que tenga los siguientes datos en la dirección:\n"
                              "- Calle\n"
                              "- Código Postal\n"
                              "- Estado\n"
                              "- País"
                             ))

        _logger.info("\n########## address_invoice_parent: %s" % address_invoice_parent)
        _logger.info("\n########## address_invoice_parent.curp: %s" % address_invoice_parent.curp)
        if address_invoice_parent.curp:
            complemento['cce11:ComercioExterior']['cce11:Emisor'].update({'Curp':address_invoice_parent.curp})

        complemento['cce11:ComercioExterior']['cce11:Emisor'].update({'cce11:Domicilio': {
            'Calle'     : address_invoice_parent.street,
            'CodigoPostal' : address_invoice_parent.zip_sat_id.code,
            'Estado'    : address_invoice_parent.state_id.code,
            'Pais'      : address_invoice_parent.country_id.sat_code
            }})

        if address_invoice_parent.street_number:
            complemento['cce11:ComercioExterior']['cce11:Emisor']['cce11:Domicilio'].update({'NumeroExterior': address_invoice_parent.street_number})

        if address_invoice_parent.street_number2:
            complemento['cce11:ComercioExterior']['cce11:Emisor']['cce11:Domicilio'].update({'NumeroInterior': address_invoice_parent.street_number2})

        if address_invoice_parent.colonia_sat_id.code:
            complemento['cce11:ComercioExterior']['cce11:Emisor']['cce11:Domicilio'].update({'Colonia': address_invoice_parent.colonia_sat_id.code})

        if address_invoice_parent.locality_sat_id.code:
            complemento['cce11:ComercioExterior']['cce11:Emisor']['cce11:Domicilio'].update({'Localidad': address_invoice_parent.locality_sat_id.code})

        if address_invoice_parent.township_sat_id.code:
            complemento['cce11:ComercioExterior']['cce11:Emisor']['cce11:Domicilio'].update({'Municipio': address_invoice_parent.township_sat_id.code})

        # ------- ------- PROPIETARIO -------
        # ------- SOLO SE AÑADE SI EL TIPO DE COMPROBANTE ES T - TRASLADO - Aun no se soporta
        if ('transport_document_cfdi' in self._fields and self.transport_document_cfdi) and not self.cfdi_comercio_exterior_propietario_id:
            raise UserError(_('Error !!! El tipo de Comprobante es de Traslado y exige que se defina el Propietario de la Mercancía'))
        elif 'transport_document_cfdi' in self._fields and self.transport_document_cfdi:
            if not (self.cfdi_comercio_exterior_propietario_id.num_reg_trib and self.cfdi_comercio_exterior_propietario_id.country_id.sat_code):
                raise UserError(_('Error !!! El Propietario no tiene definido el Registro Tributario o el País, no es posible generar el CFDI sin esa información'))
            complemento['cce11:ComercioExterior'].update({'cce11:Propietario': {
                'NumRegIdTrib'     : self.cfdi_comercio_exterior_propietario_id.num_reg_trib,
                'ResidenciaFiscal' : self.cfdi_comercio_exterior_propietario_id.country_id.sat_code,
            }})

        # ------- ------- RECEPTOR ------- -------
        complemento['cce11:ComercioExterior'].update({'cce11:Receptor': {}})
        partner = self.partner_id.commercial_partner_id

        if partner.country_id.code != 'MX':
            if not partner.num_reg_trib:
                raise UserError(_('Error !!! El cliente es extranjero pero no tiene definido el Registro Tributario, no es posible generar el CFDI sin esa información'))

        if not (partner.street and partner.zip and partner.state_id and partner.country_id.sat_code):
            raise UserError(_("Error !!! La dirección del Receptor parece estar incompleta o algún valor no tiene relacionado el Código SAT correspondiente,"
                              "por favor revise que tenga los siguientes datos en la dirección:\n"
                              "- Calle\n"
                              "- Código Postal / PO BOX\n"
                              "- Estado\n"
                              "- País"
                             ))

        if partner.country_id.code in ('CAN','USA','MX') and not partner.state_id.code:
            raise UserError('Para los Receptores de países México / Canadá / Estados Unidos es indispensable que cree el estado esté asociado en el Catálogo de Estados del SAT, por favor revise Configuración => Catálogos SAT CFDI => Catálogo Códigos de Estados')

        complemento['cce11:ComercioExterior']['cce11:Receptor'].update({'cce11:Domicilio': {
            'Calle'     : partner.street,
            'CodigoPostal' : partner.zip or partner.zip_sat_id.code,
            'Estado'    : (partner.state_id.code and partner.state_id.code) or partner.state_id.name,
            'Pais'      : partner.country_id.sat_code,
            }})

        if not complemento['cce11:ComercioExterior']['cce11:Receptor']['cce11:Domicilio']['CodigoPostal']:
            complemento['cce11:ComercioExterior']['cce11:Receptor']['cce11:Domicilio'].pop('CodigoPostal')

        comprobante['cfdi:Comprobante']['cfdi:Receptor'].update({'NumRegIdTrib'     : partner.num_reg_trib,
                                                                 'ResidenciaFiscal' : partner.country_id.sat_code})

        if partner.street_number:
            complemento['cce11:ComercioExterior']['cce11:Receptor']['cce11:Domicilio'].update({'NumeroExterior': partner.street_number})

        if partner.street_number2:
            complemento['cce11:ComercioExterior']['cce11:Receptor']['cce11:Domicilio'].update({'NumeroInterior': partner.street_number2})

        # ------- ------- DESTINATARIO ------- -------
        if partner != self.partner_shipping_id and self.partner_shipping_id:
            complemento['cce11:ComercioExterior'].update({'cce11:Destinatario': {}})
            partner = self.partner_shipping_id

            if not partner.num_reg_trib:
                raise UserError(_('Error !!! El Destinatario no tiene definido el Registro Tributario, no es posible generar el CFDI sin esa información'))

            complemento['cce11:ComercioExterior']['cce11:Destinatario'].update({'NumRegIdTrib': partner.num_reg_trib})

            if not (partner.street and partner.state_id.code and partner.country_id.sat_code):
                raise UserError(_("Error !!! La dirección del Receptor parece estar incompleta o algún valor no tiene relacionado el Código SAT correspondiente,"
                                  "por favor revise que tenga los siguientes datos en la dirección:"
                                  "- Calle"
                                  "- Estado"
                                  "- País"
                                 ))

            complemento['cce11:ComercioExterior']['cce11:Destinatario'].update({'cce11:Domicilio': {
                'Calle'     : partner.street,
                'CodigoPostal' : partner.zip_sat_id.code,
                'Estado'    : partner.state_id.code,
                'Pais'      : partner.country_id.sat_code
                }})

            if partner.street_number:
                complemento['cce11:ComercioExterior']['cce11:Destinatario']['cce11:Domicilio'].update({'NumeroExterior': partner.street_number})

            if partner.street_number2:
                complemento['cce11:ComercioExterior']['cce11:Destinatario']['cce11:Domicilio'].update({'NumeroInterior': partner.street_number2})

            if partner.colonia_sat_id.code:
                complemento['cce11:ComercioExterior']['cce11:Destinatario']['cce11:Domicilio'].update({'Colonia': partner.colonia_sat_id.code or partner.street2})

            if partner.locality_sat_id.code:
                complemento['cce11:ComercioExterior']['cce11:Destinatario']['cce11:Domicilio'].update({'Localidad': partner.locality_sat_id.code})

            if partner.township_sat_id.code:
                complemento['cce11:ComercioExterior']['cce11:Destinatario']['cce11:Domicilio'].update({'Municipio': partner.township_sat_id.code or partner.city})

        # ----- ----- MERCANCIAS ----- -----
        complemento['cce11:ComercioExterior'].update({'cce11:Mercancias': []})
        group_to_ce_mercancia = {}
        for line in self.invoice_line_ids.filtered(lambda w: not w.display_type):
            if not line.cce_quantity:
                raise UserError(_("Error !!!\n\nNo definió la Cantidad de Producto Aduana para el producto %s, no es posible generar el CFDI...") % line.product_id.name)
            if not line.product_id.sat_arancel_id:
                raise UserError(_("Error !!!\n\nEl producto %s no tiene definida la Fracción Arancelaria, no es posible generar el CFDI...") % line.product_id.name)

            ### Añadiendo el No de Indentificacion
            product_code = ""
            if line.product_id.no_identity_type:
                if line.product_id.no_identity_type != 'none':
                    if line.product_id.no_identity_type == 'default_code':
                        product_code = line.product_id.default_code
                    elif line.product_id.no_identity_type == 'barcode':
                        product_code = line.product_id.barcode
                    else:
                        product_code = line.product_id.no_identity_other

            if not product_code:
                product_code = line.product_id.default_code
                if not product_code:
                    raise UserError(_("Error !!!\n\nEl producto %s no tiene definido el Identificador (Referencia), no es posible generar el CFDI...") % line.product_id.name)

            cantidad_aduana = 0.0
            valor_dolares = 0.0
            if line.product_id in group_to_ce_mercancia:
                vals_line_prev = group_to_ce_mercancia[line.product_id]
                cantidad_aduana = vals_line_prev['CantidadAduana']
                valor_dolares = vals_line_prev['ValorDolares']
                conversion_usd = self.currency_id.with_context(date=self.invoice_date).compute(line.price_unit * line.quantity, currency_usd)
                group_to_ce_mercancia[line.product_id].update({
                         'CantidadAduana'       : line.cce_quantity+cantidad_aduana or 0.0,
                         'ValorDolares'  : conversion_usd + valor_dolares,
                          })
            else:
                group_to_ce_mercancia.update({
                    line.product_id: {'NoIdentificacion'     : product_code,
                         'FraccionArancelaria'  : line.product_id.sat_arancel_id.code or '',
                         'CantidadAduana'       : line.cce_quantity or 0.0,
                         'UnidadAduana'         : line.product_id.sat_arancel_id.unidad_de_medida or '',
                         'ValorUnitarioAduana'  : self.currency_id.with_context(date=self.invoice_date).compute((line.price_unit * line.quantity) / line.cce_quantity, currency_usd),
                         'ValorDolares'         : self.currency_id.with_context(date=self.invoice_date).compute(line.price_unit * line.quantity, currency_usd),
                        }
                    })
            # concepto = {'cce11:Mercancia':
            #             {'NoIdentificacion'     : product_code,
            #              'FraccionArancelaria'  : line.product_id.sat_arancel_id.code or '',
            #              'CantidadAduana'       : "%.2f" % line.cce_quantity+cantidad_aduana or 0.0,
            #              'UnidadAduana'         : line.product_id.sat_arancel_id.unidad_de_medida or '',
            #              'ValorUnitarioAduana'  : self.currency_id.with_context(date=self.invoice_date).compute((line.price_unit * line.quantity) / line.cce_quantity, currency_usd),
            #              'ValorDolares'         : self.currency_id.with_context(date=self.invoice_date).compute(line.price_unit * line.quantity, currency_usd)+valor_dolares,
            #             }
            #            }
            # if line.product_id.sat_marca:
            #     concepto['cce11:Mercancia'].update({'cce11:DescripcionesEspecificas':{
            #         'xmlns' : 'http://www.sat.gob.mx/ComercioExterior',
            #         'Marca' : line.product_id.sat_marca}})
            #     if line.product_id.sat_modelo:
            #         concepto['cce11:Mercancia']['cce11:DescripcionesEspecificas'].update({'Modelo':line.product_id.sat_modelo})
            #         if line.product_id.sat_submodelo:
            #             concepto['cce11:Mercancia']['cce11:DescripcionesEspecificas'].update({'SubModelo':line.product_id.sat_submodelo})

            # complemento['cce11:ComercioExterior']['cce11:Mercancias'].append(concepto)

        for groupline in group_to_ce_mercancia:
            line_mercancia = group_to_ce_mercancia[groupline]
            cantidad_aduana_final = line_mercancia['CantidadAduana']
            valor_dolares_final =  line_mercancia['ValorDolares']
            valor_unitario_aduana_final = float(valor_dolares_final) / float(cantidad_aduana_final)
            line_mercancia.update({'CantidadAduana':"%.2f" % cantidad_aduana_final,
                                    'ValorDolares': round(valor_dolares_final,2),
                                    'ValorUnitarioAduana': round(valor_unitario_aduana_final,2),
                                    })
            product_br = groupline
            concepto = {'cce11:Mercancia':
                        line_mercancia
                       }
            if product_br.sat_marca:
                concepto['cce11:Mercancia'].update({'cce11:DescripcionesEspecificas':{
                    'xmlns' : 'http://www.sat.gob.mx/ComercioExterior',
                    'Marca' : product_br.sat_marca}})
                if product_br.sat_modelo:
                    concepto['cce11:Mercancia']['cce11:DescripcionesEspecificas'].update({'Modelo':product_br.sat_modelo})
                    if product_br.sat_submodelo:
                        concepto['cce11:Mercancia']['cce11:DescripcionesEspecificas'].update({'SubModelo':product_br.sat_submodelo})

            complemento['cce11:ComercioExterior']['cce11:Mercancias'].append(concepto)

        comprobante['cfdi:Comprobante'].update({'cfdi:Complemento':complemento})
        #print "================================================="
        #print "comprobante: \n",comprobante
        #print "================================================="
        return comprobante


# -*- coding: utf-8 -*-
"""

@author: TavoGLC
From:

-Mining biological databases: IntAct-

"""
####################################################################
#                   Importing packages and libraries
####################################################################

import re
import csv
import time
import numpy as np

from bs4 import BeautifulSoup
from selenium import webdriver

####################################################################
#                            Data Directory
####################################################################

GlobalDir='Global data directory'
DataDir=GlobalDir+'\\'+'Data'

TargetListDir=GlobalDir+'\\'+'TargetList.csv'

####################################################################
#                   Data saving functions
####################################################################

#Generates a directory to save a file, name will be the Uniprot identifier
def GenCSVDir(ParentDirectory,Name):

    return ParentDirectory+'\\'+Name+'.csv'

#Save the interactions of a unique protein in a csv file
def SaveCSVFile(Data,FileDirectory):

    cData=Data
    cFile=FileDirectory

    with open(cFile,'w') as output:

        LocalWriter=csv.writer(output)
        LocalWriter.writerow(cData)

####################################################################
#                   Importing packages and libraries
####################################################################

#Makes an edit to merge the uniprot and EBI identifiers
def EBIEdit(EbiID):

    cID=EbiID

    try:

        NewEBIID=cID[0:6]+'-'+cID[7:len(cID)]

    except IndexError:

        NewEBIID='none'

    return NewEBIID

#Find the number of pages in the IntAct Table
def FindNumberOfPages(ParsedHTML):

    cPage=ParsedHTML
    #Selection of the numeration in the header table
    cStrings=cPage.find_all('span','ui-paginator-page','ui-corner-all')

    try:

        #It will work for a table with 10 or less result pages
        cVal=int(str(cStrings[-1])[-8:-7])

        if cVal==0:

            nPages=int(str(cStrings[-1])[-9:-7])

        else:

            nPages=cVal

    except IndexError:

        nPages=0

    return nPages

#Returns the interactions of a protein
def GetDataTable(ParsedHTML,localTarget):

    cParsed=ParsedHTML
    #Selects the text entries in the results table
    cSelection=cParsed.select('.ui-dt-c')
    Container=[]

    for line in cSelection:

        #Takes the text inside the table
        Container.append(line.text.strip())

    kLines=len(Container)
    IntContainer=[]

    EBIID='none'

    #Takes the text inside the table
    for line in Container:

        if re.match(localTarget+'(.*)',line):

            EBIID=line

        else:

            pass

    #Takes the text inside the table
    editID=EBIEdit(EBIID)

    #Takes the text inside the table
    if editID=='none':

        pass

    else:

        IntContainer.append(editID)

    for k in range(kLines):

        #Looks for the EBI identifier number
        if re.match('(.*)EBI-(.*)',Container[k]):

            #Selects only proteins expressed in humans
            if Container[k+1]=='human (9606)':

                IntContainer.append(Container[k])

            else:

                pass

        else:

            pass

    return IntContainer

#IntAct scraping function
def MakeEntryCall(UniprotID):

    localTarget=UniprotID

    #Browser mnipulation
    localDriver=webdriver.Chrome(executable_path='ChromeDriverPath')
    localDriver.get("https://www.ebi.ac.uk/intact/")
    time.sleep(5)

    localDriver.find_element_by_id('queryTxt').send_keys(localTarget)
    time.sleep(5)

    localDriver.find_element_by_id('quickSearchBtn').click()
    time.sleep(5)

    localDriver.find_element_by_link_text('Interactors').click()
    time.sleep(5)

    #Parsing results data
    CurrentParsed = BeautifulSoup(localDriver.page_source, 'html.parser')
    cPages=FindNumberOfPages(CurrentParsed)
    cData=GetDataTable(CurrentParsed,localTarget)

    if cPages==0:

        time.sleep(5)
        localDriver.close()

    else:

        #Loop over the number of results
        for k in range(cPages-1):

            #
            try:

                #Global xpath of the next element
                localDriver.find_element_by_xpath('/html/body/div[2]/div/div[2]/form[1]/div/div/div[2]/div[2]/div/div[1]/div[3]/table/thead/tr[1]/th/span[4]/span').click()
                LParsed=BeautifulSoup(localDriver.page_source, 'html.parser')
                lData=GetDataTable(LParsed,localTarget)
                cData.extend(lData)
                time.sleep(5)

            except Exception:

                break

        localDriver.close()

    return cData

####################################################################
#                   Importing packages and libraries
####################################################################

#Loading Uniprot identifiers
targetList=np.genfromtxt(TargetListDir,dtype="|U6")

#Looping through the identifiers adn saving the data
for target in targetList:

    data=MakeEntryCall(target)
    cDir=GenCSVDir(DataDir,target)
    SaveCSVFile(data,cDir)


import json, os, string, random, time, pickle, gc
from PIL import Image
import numpy as np
import torch
import torch.nn as nn
import torch.utils.data as data
import torchvision.transforms as transforms

from pycocotools.coco import COCO

class CocoObject(data.Dataset):
    def __init__(self, ann_dir, image_dir, split = 'train', transform = None):
        self.ann_dir = ann_dir
        self.image_dir = image_dir
        self.split = split
        self.transform = transform

        if self.split == 'train':
            ann_path = os.path.join(self.ann_dir, "instances_train2014.json")
            gender_file = "train_0.data"
        else:
            gender_file = "val_0.data"
            ann_path = os.path.join(self.ann_dir, "instances_val2014.json")
        self.cocoAPI = COCO(ann_path)
        self.data = json.load(open(ann_path))
        self.image_ids = [elem['id'] for elem in self.data['images']]
        print(type(self.image_ids[0]))
        if self.split == 'val':
            self.image_ids = self.image_ids[:10000]
        elif self.split == 'test':
            self.image_ids = self.image_ids[10000:]

        self.image_path_map = {elem['id']: elem['file_name'] for elem in self.data['images']}

        genders = pickle.load(open(gender_file))
        imageid2gender = {}
        for gender_image in genders:
            if gender_image['gender'][0] == 1:
                gender_label = 'man'
            else:
                gender_label = 'woman'
            imageid2gender[gender_image["image_id"]] = gender_label
        #80 objects
        id2object = dict()
        object2id = dict()
        self.person_id = -1
        for idx, elem in enumerate(self.data['categories']):
            if elem['name'] == 'person':
                self.person_id = idx
                print("person index: " + str(idx))
                id2object[idx] = "man"
                object2id["man"] = idx
                continue
            id2object[idx] = elem['name']
            object2id[elem['name']] = idx
        id2object[80] = "woman"
        object2id['woman'] = 80
        assert self.person_id != -1
        #generate one-hot encoding objects annotation for every image
        #self.object_ann = np.zeros((len(self.image_ids), 81))
        self.id2object = id2object
        self.object2id = object2id
        self.id2labels = {}
        self.new_image_ids = []
        for idx, image_id in enumerate(self.image_ids):
            ann_ids = self.cocoAPI.getAnnIds(imgIds = image_id)
            anns = self.cocoAPI.loadAnns(ids = ann_ids)
            category_ids = [elem['category_id'] for elem in anns]
            category_names = [elem['name'] for elem in self.cocoAPI.loadCats(ids=category_ids)]
            if 'person' in category_names and image_id not in imageid2gender:
                continue
            else:
                self.new_image_ids.append(image_id)
            #encoding_ids = [object2id[name] for name in category_names]
            #for encoding_id in encoding_ids:
                #self.object_ann[idx, encoding_id] = 1
        print("image ids len: " + str(len(self.image_ids)))
        print("new image ids len: " + str(len(self.new_image_ids)))
        gender_count = {}
        gender_count["man"] = 0
        gender_count["woman"] = 0
        self.object_ann = np.zeros((len(self.new_image_ids), 81))
        for idx, image_id in enumerate(self.new_image_ids):
            ann_ids = self.cocoAPI.getAnnIds(imgIds = image_id)
            anns = self.cocoAPI.loadAnns(ids = ann_ids)
            category_ids = [elem['category_id'] for elem in anns]
            category_names = [elem['name'] for elem in self.cocoAPI.loadCats(ids=category_ids)]
            encoding_ids = []
            self.id2labels[image_id] = []
            for name in category_names:
                if 'person' == name:
                    gender_count[imageid2gender[image_id]] = gender_count[imageid2gender[image_id]] + 1
                    encoding_ids.append(object2id[imageid2gender[image_id]])
                    self.id2labels[image_id].append(imageid2gender[image_id])
                else:
                    encoding_ids.append(object2id[name])
                    self.id2labels[image_id].append(name)
            for encoding_id in encoding_ids:
                self.object_ann[idx, encoding_id] = 1
        print(gender_count)

    def __getitem__(self, index):
        image_id = self.new_image_ids[index]
        image_file_name = self.image_path_map[image_id]
        if self.split == 'train':
            image_path = os.path.join(self.image_dir,"train2014", image_file_name)
        else:
            image_path = os.path.join(self.image_dir,"val2014", image_file_name)

        img = Image.open(image_path).convert('RGB')

        if self.transform is not None:
            img = self.transform(img)

        return img, torch.Tensor(self.object_ann[index]), image_id

    def getObjectWeights(self):
        return (self.object_ann == 0).sum(axis = 0) / (1e-8 + self.object_ann.sum(axis = 0))

    def __len__(self):
        return len(self.new_image_ids)


''' Standup.py
File that contains all functions related to standup.

'''

import datetime
import data
import message

from error import InputError, AccessError

def standup_start(token, channel_id, length):

	# Check token is valid.
    try:
        data.token_to_user_id(token)
    except:
        raise AccessError(description='Invalid token')

    # Check channel ID is valid.
    try:
        channel_index = data.resolve_channel_id_index(channel_id)
    except LookupError:
        raise InputError(description='Invalid Channel ID')

	# Check that there is not already an active standup in this channel.
    if standup_active(token, channel_id)['is_active'] is True:
        raise InputError(description='Standup Already Active')

    current_time = datetime.datetime.now()
    finish_time = (current_time + datetime.timedelta(seconds=length)).timestamp()

	# Edit data structure to store standup details.
    standup = data.data['channels'][channel_index]['standup']
    standup['time_finish'] = finish_time
    standup['creator'] = token

    return  {'time_finish': finish_time}

# Close a standup and clear standup data.
def close_standup(channel_index, channel_id):
    # Collate all messages sent.
    standup = data.data['channels'][channel_index]['standup']

    standup_str = ''

    for message_info in standup['messages']: # pragma: no cover
	    user_index = data.resolve_user_id_index(message_info['u_id'])
	    # Use handle rather than first name as handle is unique.
	    handle = data.data['users'][user_index]['handle']
	    # Add a message in format 'hayden: I ate a catfish'.
	    standup_str += handle + ': '+ message_info['message'] + '\n'

    # Send out collated messages (unless no messages were sent during standup).
    if standup_str != '': # pragma: no cover
        try:
            send_time = standup['time_finish']
            message.message_sendlater(standup['creator'], channel_id, standup_str, send_time)
        except InputError:
            message.message_send(standup['creator'], channel_id, standup_str)

    # Clear data
    standup['time_finish'] = None
    standup['messages'] = []
    standup['creator'] = None

def standup_active(token, channel_id):

	# Check token is valid.
    try:
        data.token_to_user_id(token)
    except:
        raise AccessError(description="Invalid token")

    # Check channel ID is valid.
    try:
        channel_index = data.resolve_channel_id_index(channel_id)
    except LookupError:
        raise InputError(description='Invalid Channel ID')

    current_time = datetime.datetime.now().timestamp()

    active = False

    standup = data.data['channels'][channel_index]['standup']
    # Check if any data for a standup has been stored
    if standup['time_finish'] is not None:
	    if standup['time_finish'] < current_time:
		    # Close the standup:
		    # ie. send out final message and erase all data.
		    close_standup(channel_index, channel_id)
	    else:
		    active = True

    return {
	    'is_active': active,
        'time_finish':  standup['time_finish']
    }

def standup_send(token, channel_id, message):

    # Check token is valid.
    try:
        u_id = data.token_to_user_id(token)
    except:
        raise AccessError(description="Invalid token")

    # Check channel ID is valid.
    try:
        channel_index = data.resolve_channel_id_index(channel_id)
    except LookupError:
        raise InputError(description='Invalid Channel ID')

    # Check that message is valid.
    if len(message) > 1000:
	    raise InputError(description='Message Too Long')

    # Check that standup is active in this channel.
    if standup_active(token, channel_id)['is_active'] is False:
	    raise InputError(description='Standup Not Active')

    # Check that user is authorised to send a message in channel.
    permissions = [1, 2]
    if data.resolve_permissions(channel_id, u_id) not in permissions:
	    raise AccessError(description='User Not Authorised')

    # Store the message in standup buffer.
    standup = data.data['channels'][channel_index]['standup']
    standup['messages'].append({
	    'u_id': u_id,
	    'message': message
    })

    return {}


from googletrans import Translator
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters

def start_func(update, context):
    update.message.reply_text("""سلام \nپیام انگلیسی یا فارسی خود را ارسال کنید.\n/about""")
    print('Start REQ from {}'.format(update.message.from_user.first_name))

def trans_func(update, context):
    user_text = update.message.text
    lang = Translator().detect(user_text).lang

    if lang == 'fa':
        transed = Translator().translate(user_text, dest='en').text
        update.message.reply_text(transed)
    elif lang == 'en':
        transed = Translator().translate(user_text, dest='fa').text
        update.message.reply_text(transed)
    else:
        transed = Translator().translate(user_text, dest='fa').text
        update.message.reply_text(transed)
def about_func(update, context):
	update.message.reply_text("coder @six_6_six\nchannel @freezed_cash\ngithub github.com/mohsenfn")

updater = Updater("token here", use_context=True)
updater.dispatcher.add_handler(CommandHandler("about", about_func))
updater.dispatcher.add_handler(CommandHandler("start", start_func))
updater.dispatcher.add_handler(MessageHandler(Filters.text, trans_func))
print("Running ...")
updater.start_polling()
updater.idle()
# done


import re
from lark import Lark, InlineTransformer
from typing import NamedTuple

class Symbol(NamedTuple):
    value: str

grammar = Lark(
    r"""
start : "implemente-aqui!"
""")

class LispyTransformer(InlineTransformer):
    CHARS = {
        "altmode": "\x1b",
        "backnext": "\x1f",
        "backspace": "\b",
        "call": "SUB",
        "linefeed": "\n",
        "page": "\f",
        "return": "\r",
        "rubout": "\xc7",
        "space": " ",
        "tab": "\t",
    }


import ssl
import os
import pandas as pd
import glob
import csv
import re
import numpy as np
import datetime as dt
from pandas.core.indexes.base import Index

codelist=['9983','7751','5201','2871','9613','6752','4507','2809','8411','5332','3407','2296']
path="stock_data\\"
df1 = pd.read_csv(path + "stockopencode.csv",encoding="cp932")

df1['date'] = pd.to_datetime(df1['date'])
df5 = df1[df1['date']>dt.datetime(2018,7,1)]
for code in codelist:
    print(code)
    if not code in df1.columns:
        continue
    colist = []
    df2 = df5[code]
    df2 = df2.reset_index(drop = True)
    length = len(df2)
    length = length - 1
    for s in df1.columns[3::]:
        df3 = df1.loc[0:length,s]

        df4 = pd.concat([df2,df3],axis=1)
        df4.columns = ['1','2']
        df4 = df4[(df4["1"] != 0 ) & (df4['2'] != 0) ]
        corr = df4.corr()
        colist.append(corr.loc['1','2'])
    colist_sort=sorted(colist,reverse=True)
    print(colist_sort[0:5])
    indexlist = []
    for i in colist_sort[0:5]:
        indexnum = colist.index(i)
        indexlist.append(indexnum+3)
    print(df1.columns[indexlist])


import math
import pyaudio

import numpy as np
import scipy.integrate as integrate
import matplotlib.pyplot as plt
import seaborn as sns

from math import e as e
from math import pi as pi

p = pyaudio.PyAudio()

plt.ion()
fig, (ax1, ax2) = plt.subplots(2)
fig.suptitle('Vertically stacked subplots')

# freqs = [523, 440, 349]
freqs = [1, 4, 15]

def func_aperiodic(x):
    falloff = e ** (-2 * pi * (x ** 2))
    out = sum([math.cos(2 * pi * f * x) for f in freqs])

    return out

def func_periodic(x):
    out = sum([math.cos(f * x) for f in freqs])

    return out

def fourier_transform(freq):
    """ integral(x(t) * e^(-i2(pi)ft)) dx/dt """

    fourier = lambda t: func_aperiodic(t) * (e ** (-2j * pi * freq * t))
    fourier_full = lambda t: fourier(t).real + fourier(t).imag

    out = integrate.quad(fourier_full, -np.inf, np.inf)[0]

    return out

def fourier_series(x, num_iter=1):
    a_0 = (1 / pi) * integrate.quad(func_periodic, -pi, pi)[0]
    coef_a = lambda x: func_periodic(x) * math.cos(n * x)
    coef_b = lambda x: func_periodic(x) * math.sin(n * x)
    a_n = lambda n: (1 / pi) * integrate.quad(coef_a, -pi, pi)[0]
    b_n = lambda n: (1 / pi) * integrate.quad(coef_b, -pi, pi)[0]

    out = a_0 / 2
    for n in range(num_iter):
        out += (a_n(n) * math.cos(n * x)) + (b_n(n) * math.sin(n * x))

    return out

def plot_func(function, span, ax, n=None):
    volume = 0.5     # range [0.0, 1.0]
    fs = 44100       # sampling rate, Hz, must be integer
    duration = 1.0   # in seconds, may be float
    f = 440.0        # sine frequency, Hz, may be float

    x = np.linspace(-(duration / 2), -(duration / 2), fs)
    y = np.vectorize(function)(x, n) if n else np.vectorize(function)(x)

    samples = (y).astype(np.float32)

    print(samples)
    print(len(samples))

    # for paFloat32 sample values must be in range [-1.0, 1.0]
    stream = p.open(format=pyaudio.paFloat32,
                                    channels=1,
                                    rate=fs,
                                    output=True)

    # play. May repeat with different volume values (if done interactively)
    stream.write(volume*samples)

    stream.stop_stream()
    stream.close()

    return x, y

def play_sound():
    volume = 0.5     # range [0.0, 1.0]
    fs = 44100       # sampling rate, Hz, must be integer
    duration = 1.0   # in seconds, may be float
    f = 440.0        # sine frequency, Hz, may be float

    # generate samples, note conversion to float32 array
    samples = (np.sin(2*np.pi*np.arange(fs*duration)*f/fs)).astype(np.float32)

    print(samples)
    print(len(samples))

    # for paFloat32 sample values must be in range [-1.0, 1.0]
    stream = p.open(format=pyaudio.paFloat32,
                                    channels=1,
                                    rate=fs,
                                    output=True)

    # play. May repeat with different volume values (if done interactively)
    stream.write(volume*samples)

    stream.stop_stream()
    stream.close()

if __name__=="__main__":
    play_sound()

    x, y = plot_func(func_periodic, 50, ax1)
    sns.lineplot(x=x, y=y, ax=ax1)

    print("asdf")
    plt.show()
    plt.pause(10)

    x, y = plot_func(fourier_series, 50, ax2, 20)
    sns.lineplot(x=x, y=y, ax=ax2)

    # num_iter = 0
    # while num_iter < 20:
        # x, y = plot_func(fourier_series, 50, ax2, num_iter)
        # print(y)

        # for line in ax2.lines:
            # line.set_ydata(y)
            # fig.canvas.draw()
            # fig.canvas.flush_events()

        # plt.pause(1)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import os
from os import path
import speech_recognition as sr
from gtts import gTTS

def rec():
    r = sr.Recognizer()
    with sr.Microphone() as source:
        #r.adjust_for_ambient_noise(source)
        print('Say something!')
        audio = r.listen(source)

    try:
        sttTXT_org = r.recognize_google(audio, language = 'en-US')
        print("Google Speech Recognition thinks you said: " + sttTXT_org)
        return sttTXT_org
    except sr.UnknownValueError:
        print("Google Speech Recognition could not understand audio")
        return 0
    except sr.RequestError as e:
        print('Could not request results from Google Speech Recognition service; {0}'.format(e))
        return 0

def t2speech(txt):
    tts =  gTTS(text=txt,lang='en-US').save('{}.mp3'.format(txt))


# SF-482
import json
import pandas as pd

# from xcalar.solutions.singleton import Singleton_Controller
from xcalar.solutions.tools.connector import XShell_Connector
from colorama import Style
from tabulate import tabulate
from pygments import highlight
from pygments.lexers import JsonLexer
from pygments.formatters import TerminalFormatter
from .action_confirm import confirm

def dispatcher(line):
    help = {
        'show_all':
            f"""
Usage:

DATASET
    {Style.NORMAL}dataset {Style.DIM}-list{Style.RESET_ALL}                                                     {Style.BRIGHT}- lists all datasets in xd.{Style.RESET_ALL}
    {Style.NORMAL}dataset {Style.DIM}-show      <dataset> <#start : #rows>{Style.RESET_ALL}                     {Style.BRIGHT}- shows top 10 rows/[x:y rows] in the dataset.{Style.RESET_ALL}
    {Style.NORMAL}dataset {Style.DIM}-describe  <dataset>{Style.RESET_ALL}                                      {Style.BRIGHT}- shows information of a dataset, including number of records, etc.{Style.RESET_ALL}
    {Style.NORMAL}dataset {Style.DIM}-drop      <dataset>{Style.RESET_ALL}                                      {Style.BRIGHT}- deletes the dataset.{Style.RESET_ALL}
""",
        'dataset_list':
            f"""
Usage:
    dataset
    {Style.NORMAL}dataset {Style.DIM}-list{Style.RESET_ALL}                       {Style.BRIGHT}- lists all datasets in xd.{Style.RESET_ALL}
        """,
        'dataset_show':
            f"""
Usage:
    dataset
    {Style.NORMAL}dataset {Style.DIM}-show <dataset>{Style.RESET_ALL} <#start : #rows>            {Style.BRIGHT}- shows top 10 rows/[x:y rows] in the dataset.{Style.RESET_ALL}
        """,
        'dataset_describe':
            f"""
Usage:
    dataset
    {Style.NORMAL}dataset {Style.DIM}-describe <dataset>{Style.RESET_ALL}         {Style.BRIGHT}- shows information of a dataset, including number of records, etc.{Style.RESET_ALL}

        """,
        'dataset_drop':
            f"""
Usage:
    dataset
        {Style.NORMAL}dataset {Style.DIM}-drop <dataset>{Style.RESET_ALL}             {Style.BRIGHT}- deletes the dataset.{Style.RESET_ALL}
        """,
    }

    def show_dataset(dataset_name, start, num_rows):
        dataset = controller.orchestrator.dispatcher.xcalarClient.get_dataset(
            dataset_name)
        columns = list(map(lambda x: x['name'], dataset.get_info()['columns']))
        show_table_or_dataset(dataset, columns, start, num_rows)

    def show_table_or_dataset(table_or_dataset, columns, start, num_rows):
        data = []
        for row in table_or_dataset.records(
                start_row=start, num_rows=num_rows):
            r_data = []
            for col in columns:
                if col in row:
                    r_data.append(row[col])
                else:
                    r_data.append(None)
            data.append(tuple(r_data))
        print(
            tabulate(
                pd.DataFrame.from_records(data, columns=columns),
                showindex=range(start, start + len(data)),
                headers='keys',
                tablefmt='psql',
            ))

    tokens = line.split()

    if len(tokens) == 0:
        print(help['show_all'])
        return

    try:
        # singleton_controller = Singleton_Controller.getInstance()
        # controller = singleton_controller.get_controller()
        xshell_connector = XShell_Connector()
        controller = xshell_connector.get_controller()
        # dataset
        if tokens[0] == '-list':
            datasets = controller.orchestrator.dispatcher.xcalarClient.list_datasets(
            )
            names = list(map(lambda x: x.name, datasets))

            t = zip(names)
            print(tabulate(
                t,
                headers=['dataset'],
                tablefmt='psql',
            ))

        elif tokens[0] == '-show':
            if len(tokens) == 3 and tokens[1]:
                dataset_name = tokens[1]
                x = int(tokens[2].split(':')[0])
                y = int(tokens[2].split(':')[1])

                show_dataset(dataset_name, x, y)
            else:
                print(help['dataset_show'])

        elif tokens[0] == '-describe':
            if len(tokens) == 2 and tokens[1]:
                dataset = controller.orchestrator.dispatcher.xcalarClient.get_dataset(
                    tokens[1])
                info = dataset.get_info()

                json_object = json.loads(json.dumps(info))
                json_str = json.dumps(json_object, indent=4, sort_keys=True)
                print(highlight(json_str, JsonLexer(), TerminalFormatter()))

            else:
                print(help['dataset_describe'])

        elif tokens[0] == '-drop':
            if len(tokens) == 2 and tokens[1]:
                if confirm() is None:
                    return
                pass
            else:
                print(help['dataset_drop'])

        else:
            print(help['show_all'])

    except AttributeError as ex:
        print(f'{Style.DIM}Warning: {Style.RESET_ALL} {ex}')
    except KeyboardInterrupt:
        print(f'{Style.DIM}Warning: {Style.RESET_ALL} Ctrl-c detected.')
        return
    except Exception as e:
        print(f'{Style.DIM}Warning: {Style.RESET_ALL} {e}')

def load_ipython_extension(ipython, *args):
    def dataset_completers(self, event):
        return [
            '-list',
            '-show',
            '-describe',
            '-drop',
        ]

    ipython.register_magic_function(dispatcher, 'line', magic_name='dataset')
    ipython.set_hook('complete_command', dataset_completers, str_key='dataset')
    ipython.set_hook(
        'complete_command', dataset_completers, str_key='%dataset')

def unload_ipython_extension(ipython):
    pass


import requests

KEY = ''

gbookurl="https://www.googleapis.com/books/v1/volumes?q=isbn:%s&key=" + KEY

def getData(urlRequest):
    r = requests.get(url = urlRequest)
    try:
        json = r.json()
        #descricao = r.json()["items"][0]["volumeInfo"]["description"]
        if ("error" in json):
            print (json["error"]["errors"][0]["reason"])
        else:
            webReaderLink = json["items"][0]["accessInfo"]["webReaderLink"]
            accessViewStatus = json["items"][0]["accessInfo"]["accessViewStatus"]
        if (accessViewStatus != "NONE"):
            return webReaderLink
    except KeyboardInterrupt:
        exit()
    except:
        pass
        #print (urlRequest)

fsuccs = open("succ.csv", "w")
ferrors = open("error.csv", "w")
base = open("livros.csv")
isbns = []
errors = []
succs = []

for livro in base:
    if livro:
        stripped = livro.strip("\n").split(",")
        current_isbn = stripped[-1]
        if current_isbn and current_isbn not in isbns:
            isbns.append(current_isbn)
            print (current_isbn)
            urlRequest = gbookurl % current_isbn
            webview = getData(urlRequest)
            if (webview):
                line = stripped[0] + "," + current_isbn + "," + webview + "\n"
                succs.append(line)
                print (line)
            else:
                line = stripped[0] + "," + current_isbn + "," + urlRequest + "\n"
                errors.append(line)

fsuccs.writelines(succs)
ferrors.writelines(errors)
fsuccs.close()
ferrors.close()

#print (descricao)
#print (webview)