# Author: Pyeong Eun Kim
# Date: Dec.01.2020
# Title: Graph-based Deep Generative Model version 3 (single batch only)
# Affiliation: JLK Genome Research Centre

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.distributions.categorical import Categorical
import dgl
from functools import partial

class ScaffoldGNN(nn.Module):
    def __init__(self, num_auxiliary, num_prop_rounds=[3,3,2,2,2,2],
                 num_node_features=[8,4,3,2], num_edge_features=[4,5],
                 node_emb_dim=128, edge_emb_dim=128, edge_embedding_per_round=True):
        super(ScaffoldGNN, self).__init__()

        # set number of propagation of each propagating functions
        self.num_prop_rounds = {"whole":None, "scaffold":None,
                                "addnode":None, "addedge":None,
                                "selectnode":None, "selectisomer":None}
        for n, key in enumerate(self.num_prop_rounds.keys()):
            self.num_prop_rounds[key] = num_prop_rounds[n]

        # set all arguments to be shared
        self.num_auxiliary = num_auxiliary
        self.num_node_types = num_node_features[0]
        self.num_edge_types = num_edge_features[0]
        self.node_emb_dim = node_emb_dim
        self.edge_emb_dim = edge_emb_dim

        self.init_node_embedding_atom = nn.Embedding(num_node_features[0], node_emb_dim)
        self.init_node_embedding_chiral = nn.Embedding(num_node_features[1], node_emb_dim)
        self.init_node_embedding_formal_charge = nn.Embedding(num_node_features[2], node_emb_dim)
        self.init_node_embedding_aromatic = nn.Embedding(num_node_features[3], node_emb_dim)

        self.init_edge_embedding_bond = nn.Embedding(num_edge_features[0], edge_emb_dim)
        self.init_edge_embedding_stereo = nn.Embedding(num_edge_features[1], edge_emb_dim)

        """ENCODING PART"""
        # whole graph part
        self.whole_graph_prop = ConditionalProp(self.num_prop_rounds["whole"],
                                                num_auxiliary, node_emb_dim, edge_emb_dim) # node_emb_dim
        self.whole_graph_readout = GraphReadOut(node_embed_dim=node_emb_dim, graph_embed_dim=2*node_emb_dim)  # node_emb_dim*2
        self.fc_mu = nn.Linear(node_emb_dim*2, node_emb_dim)
        self.fc_var = nn.Linear(node_emb_dim*2, node_emb_dim)

        """DECODING PART"""
        # scaffold graph part
        self.scaffold_graph_prop = ConditionalProp(self.num_prop_rounds["scaffold"],
                                                   num_auxiliary, node_emb_dim, edge_emb_dim)

        """DECODING ACTION DECISION"""
        # add node
        # we will concat auxiliary with latent vector when propagate scaffold graph
        num_auxiliary_decode = num_auxiliary + node_emb_dim
        self.add_nodes_prop = ConditionalProp(self.num_prop_rounds["addnode"],
                                              num_auxiliary_decode,
                                              node_emb_dim, edge_emb_dim)
        self.add_nodes_readout = GraphReadOut(node_embed_dim=node_emb_dim, graph_embed_dim=node_emb_dim)
        self.add_nodes_mlp = MLP(3, node_emb_dim + num_auxiliary_decode, node_emb_dim, self.num_node_types + 1, True)
        #self.add_nodes_softmax = nn.Softmax(dim=1)

        # init node
        self.init_nodes_mlp1 = nn.Embedding(self.num_node_types, node_emb_dim)
        self.init_nodes_mlp2 = nn.Linear(node_emb_dim*2, node_emb_dim)
        self.init_nodes_readout = GraphReadOut(node_embed_dim=node_emb_dim, graph_embed_dim=node_emb_dim)

        # add edge
        self.add_edges_prop = ConditionalProp(self.num_prop_rounds["addedge"],
                                              num_auxiliary_decode,
                                              node_emb_dim, edge_emb_dim)
        self.add_edges_readout = GraphReadOut(node_embed_dim=node_emb_dim, graph_embed_dim=node_emb_dim)
        # in this case we did not use num_edge_types + 1
        # instead we used num_edge_type because we are not considering adding aromatic bonds
        # thus, the final logit would correspond to [single, double, triple, STOP]
        self.add_edges_mlp = MLP(3, node_emb_dim + num_auxiliary_decode, node_emb_dim, self.num_edge_types, True)
        #self.add_edges_softmax = nn.Softmax(dim=1)

        # init edge
        # use self.num_edge_types -1 because it corresponds to [single, double, triple]
        # from which aromatic is excluded.
        self.init_edges_mlp1 = nn.Embedding(self.num_edge_types - 1, node_emb_dim)
        self.init_edges_mlp2 = nn.Linear(node_emb_dim*2, node_emb_dim)
        self.init_edges_readout = GraphReadOut(node_embed_dim=node_emb_dim, graph_embed_dim=node_emb_dim)

        # select dest node
        self.select_nodes_prop = ConditionalProp(self.num_prop_rounds["selectnode"],
                                                 num_auxiliary_decode,
                                                 node_emb_dim, edge_emb_dim)
        self.select_nodes_mlp = nn.Linear(node_emb_dim + node_emb_dim + num_auxiliary_decode, 1)

    def prepare_for_traininig(self):
        self.node_probability = []
        self.edge_probability = []
        self.dest_probability = []

    def whole_graph_encode(self, g_whole, conditions):
        """
        conditions: real conditions such as MW, logP, etc...
        """
        g_whole.ndata["node_features"] = torch.stack([
                                                self.init_node_embedding_atom(g_whole.ndata["atom_type"]),
                                                self.init_node_embedding_chiral(g_whole.ndata["chirality_type"]),
                                                self.init_node_embedding_formal_charge(g_whole.ndata["charge_type"]),
                                                self.init_node_embedding_aromatic(g_whole.ndata["aromatic_type"]),
                                                ], dim=1).sum(1)
        g_whole.edata["edge_features"] = torch.stack([
                                                self.init_edge_embedding_bond(g_whole.edata["bond_type"]),
                                                self.init_edge_embedding_stereo(g_whole.edata["stereo_type"])
                                                ], dim=1).sum(1)
        g_whole = self.whole_graph_prop(g_whole, conditions)
        # print("################after whole graph:", g_whole.edata["edge_features"].size())
        readout = self.whole_graph_readout(g_whole)
        mu = self.fc_mu(readout)
        logvar = self.fc_var(readout)
        std = torch.exp(0.5 * logvar)
        eps = torch.randn_like(std)
        z = eps * std + mu

        return z, logvar, mu

    def scaffold_propagate(self, g_scaffold, conditions):
        g_scaffold.ndata["node_features"] = torch.stack([
                                                self.init_node_embedding_atom(g_scaffold.ndata["atom_type"]),
                                                self.init_node_embedding_chiral(g_scaffold.ndata["chirality_type"]),
                                                self.init_node_embedding_formal_charge(g_scaffold.ndata["charge_type"]),
                                                self.init_node_embedding_aromatic(g_scaffold.ndata["aromatic_type"]),
                                                ], dim=1).sum(1)
        g_scaffold.edata["edge_features"] = torch.stack([
                                                self.init_edge_embedding_bond(g_scaffold.edata["bond_type"]),
                                                self.init_edge_embedding_stereo(g_scaffold.edata["stereo_type"])
                                                ], dim=1).sum(1)
        g_scaffold = self.scaffold_graph_prop(g_scaffold, conditions)

        return g_scaffold

    def add_nodes_encode(self, g_scaffold, conditions, training=True):
        g_scaffold = self.add_nodes_prop(g_scaffold, conditions)
        readout = self.add_nodes_readout(g_scaffold)
        readout = torch.cat([readout, conditions])
        readout = self.add_nodes_mlp(readout)
        #readout = self.add_nodes_softmax(readout)
        if not training:
            # instead of sampling max, we sampled from distribution
            dist = F.softmax(readout, dim=0)
            dist = Categorical(dist)
            return dist.sample()
        else:
            return readout

    def init_nodes_encode(self, g_scaffold, categorical_node_feat):
        node_feat = self.init_nodes_mlp1(categorical_node_feat) #Gxembed
        readout = self.init_nodes_readout(g_scaffold)   # Gxembed
        readout = torch.cat([readout, node_feat])
        node_feat = self.init_nodes_mlp2(readout)
        return node_feat

    def add_edges_encode(self, g_scaffold, conditions, training=True):
        g_scaffold = self.add_edges_prop(g_scaffold, conditions)
        readout = self.add_edges_readout(g_scaffold)
        readout = torch.cat([readout, conditions])
        readout = self.add_edges_mlp(readout)
        #readout = self.add_edges_softmax(readout)
        if not training:
            # instead of sampling max, we sampled from distribution
            dist = F.softmax(readout, dim=0)
            dist = Categorical(dist)
            return dist.sample()
        else:
            return readout

    def init_edges_encode(self, g_scaffold, categorical_edge_feat):
        edge_feat = self.init_edges_mlp1(categorical_edge_feat) #Gxembed
        readout = self.init_edges_readout(g_scaffold)   # Gxembed
        readout = torch.cat([readout, edge_feat])
        edge_feat = self.init_edges_mlp2(readout)
        return edge_feat

    def select_nodes_encode(self, g_scaffold, conditions):
        src = g_scaffold.number_of_nodes() - 1
        possible_dest = range(src)
        g_scaffold = self.select_nodes_prop(g_scaffold, conditions)
        possible_dest_emb = g_scaffold.nodes[possible_dest].data["node_features"]
        src_emb_expand = g_scaffold.nodes[src].data["node_features"].expand(src, -1)
        condition_expand = conditions.expand(src, -1)
        # print(possible_dest_emb.size(), src_emb_expand.size(), condition_expand.size())
        logits = self.select_nodes_mlp(torch.cat([possible_dest_emb,
                                                  src_emb_expand,
                                                  condition_expand], dim=1).float()).squeeze(1)
        # print(logits.size())
        return logits

    def forward(self, g_whole, g_scaffold, conditions, actions):
        self.prepare_for_traininig()
        z, logvar, mu = self.whole_graph_encode(g_whole, conditions)
        g_scaffold = self.scaffold_propagate(g_scaffold, conditions)
        # print("after scaffold:",g_scaffold.edata["edge_features"].size())
        # concat conditions with latent vector for decoding process
        conditions = torch.cat([z, conditions])
        # print("conditions", conditions.size())
        add_node = True
        index = 0
        while add_node:
            # add node but which node?
            node_dist = self.add_nodes_encode(g_scaffold, conditions)  # if stop add_node = False (setting from dataset)
            self.node_probability.append(node_dist)
            if actions[index][0] == self.num_node_types:
                add_node = False
                continue
            else:
                g_scaffold.add_nodes(1)
                num_nodes = g_scaffold.number_of_nodes()
                # get node feature
                init_node = actions[index][0]
                new_node_feat = self.init_nodes_encode(g_scaffold, init_node)
                # print(new_node_feat.size(), g_scaffold.nodes[num_nodes - 1].data['node_features'].size())
                g_scaffold.nodes[num_nodes - 1].data['node_features'] = new_node_feat.unsqueeze(0)
            edge_index = 0
            add_edge = True
            #print(actions[index], actions)
            while add_edge and edge_index < len(actions[index][1]):
                edge_add_action = self.add_edges_encode(g_scaffold, conditions)
                self.edge_probability.append(edge_add_action)
                # print("%%%%%%%%%%%",actions[index], edge_index)
                if actions[index][1][edge_index] == self.num_edge_types - 1:
                    add_edge = False
                    continue
                else:
                    init_edge = actions[index][1][edge_index]  # edge types
                    # print("after scaffold:",g_scaffold.edata["edge_features"].size())
                    select_node = self.select_nodes_encode(g_scaffold, conditions)
                    # print("###############after scaffold:",g_scaffold.edata["edge_features"].size())
                    self.dest_probability.append(select_node)
                    src, dest = num_nodes - 1, actions[index][2][edge_index]  # edge dest
                    src_list = [src, dest]
                    dest_list = [dest, src]
                    g_scaffold.add_edges(src_list, dest_list)
                    edge_emb = self.init_edges_encode(g_scaffold, init_edge)
                    # print("###edge###",g_scaffold.edges[src_list, dest_list].data['edge_features'].size(),torch.stack([edge_emb, edge_emb], dim=0).size())
                    g_scaffold.edges[src_list, dest_list].data['edge_features'] = torch.stack([edge_emb, edge_emb], dim=0)
                edge_index += 1
            index += 1

        return z, logvar, mu, self.node_probability, self.edge_probability, self.dest_probability

    def sample(self):
        """
        # TO DO: implement sampling for validation
        """
        return None

class ConditionalProp(nn.Module):
    def __init__(self, num_prop_rounds, num_auxiliary,
                 node_emb_dim=128, edge_emb_dim=128):
        super(ConditionalProp, self).__init__()

        # number of propagation
        self.num_prop_rounds = num_prop_rounds
        # print("auxiliary number:", num_auxiliary)
        # make propagation layers corresponding to num rounds
        self.prop_layers = nn.ModuleList()
        for round in range(num_prop_rounds):
            self.prop_layers.append(SingleProp(num_auxiliary, node_emb_dim, edge_emb_dim))

    def forward(self, g, auxiliary):
        # condition should be list of conditions
        # node categorical to embed
        """
        # if feed g of whole molecule and scaffold, "node_features, and edge_features"
        # should be already embedded by shared embedding layer
        """
        # print("**********edge feature")
        # print(g.edata["edge_features"].size())
        original_edge = g.edata["edge_features"]
        g.edata["edge_features"] = torch.cat((g.edata["edge_features"], auxiliary.expand(g.num_edges(),-1)), -1)
        for round in range(self.num_prop_rounds):
            g = self.prop_layers[round](g, auxiliary)
        g.edata["edge_features"] = original_edge
        return g

class MLP(nn.Module):
    """ MultilayerPerceptron (modified from https://github.com/weihua916/powerful-gnns/blob/master/models/mlp.py)
    Args:
        num_layers (int): number of layers in the neural networks. If num_layers=1, this reduces to linear model.
        input_dim (int): dimensionality of input features
        hidden_dim (int): dimensionality of hidden units at ALL layers
        output_dim (int): number of classes for prediction
        batch_norm (Bool): batch normalisation of after MLP (default to True)
    """
    def __init__(self, num_layers, input_dim, hidden_dim, output_dim, batch_norm=True):
        super(MLP, self).__init__()

        self.linear_or_not = True #default is linear model
        self.num_layers = num_layers

        if num_layers < 1:
            raise ValueError("number of layers should be positive!")
        elif num_layers == 1:
            #Linear model
            self.linear = nn.Linear(input_dim, output_dim)
        else:
            # Multi-layer model
            self.linear_or_not = False
            self.linears = torch.nn.ModuleList()
            self.batch_norms = torch.nn.ModuleList()

            self.linears.append(nn.Linear(input_dim, hidden_dim))
            for layer in range(num_layers - 2):
                self.linears.append(nn.Linear(hidden_dim, hidden_dim))
            self.linears.append(nn.Linear(hidden_dim, output_dim))

            for layer in range(num_layers - 1):
                if batch_norm:
                    self.batch_norms.append(nn.BatchNorm1d(hidden_dim))
                else:
                    self.batch_norms.append(nn.Identity())
    def forward(self, x):
        if self.linear_or_not:
            #If linear model
            return self.linear(x)
        else:
            #If MLP
            h = x.float()
            for layer in range(self.num_layers - 1):
                # print(layer, h.size(), self.linears[layer])
                ## print(self.linears[layer](h))
                if len(h.size()) > 2:
                    h = F.relu(self.linears[layer](h))
                else:

                    h = F.relu(self.linears[layer](h))
            return self.linears[self.num_layers - 1](h)

class SingleProp(nn.Module):
    def __init__(self, num_auxiliary, node_emb_dim,
                 edge_emb_dim=None, num_mlp_layer=2,
                 activation_dim=None, hidden_dim=None,
                 batch_norm=True):
        super(SingleProp, self).__init__()
        # num_auxiliary can be number of conditions or dimension of latent vector
        # eg) 4 for conditions such as [logP_whole, mw_whole, logP_scaffold, mw_whole]
        # eg) 128 for latent vectors such as [0.1, 0.3, ..., 0.9, -1.1]
        self.num_auxiliary = num_auxiliary

        self.node_emb_dim = node_emb_dim

        if not edge_emb_dim:
            edge_emb_dim = node_emb_dim
        if not hidden_dim:
            # same as GIN paper
            hidden_dim = 2 * (node_emb_dim*2 + edge_emb_dim + self.num_auxiliary)

        # if custom activation dim exists apply; otherwise use default
        if activation_dim:
            self.activation_dim = activation_dim
        else:
            # same as paper <https://arxiv.org/abs/1905.13639> indicated
            self.activation_dim = node_emb_dim

        # input being [hv, hu, xuv]
        # dimension [node_hidden_dim, node_hidden_dim, edge_hidden_dim]

        self.message_func = MLP(num_layers=num_mlp_layer,
                                input_dim=node_emb_dim*2 + edge_emb_dim + self.num_auxiliary,
                                hidden_dim=hidden_dim,
                                output_dim=self.activation_dim,
                                batch_norm=False)
        self.reduce_func = partial(self.get_activation_node)
        self.node_update_func = nn.GRUCell(self.activation_dim, node_emb_dim)

        # initialise model parameters (Xaiver Uniform)
        #self.initialise_parameters()

    def initialise_parameters(self):
        """Reinitialise model parameters."""

        nn.init.xavier_uniform_(self.node_update_func.weight.data)
        nn.init.xavier_uniform_(self.message_func)
        if self.edge_embedding_per_round:
            nn.init.xavier_uniform_(self.edge_embedding.weight.data)

    def concat_msg(self, edges):
        """
            Concatenate features from source nodes with edge features and aggregate them in destination nodes.
            For an edge u->v, return concat([h_u, x_uv]).
            v is the destination node which would get the aggregated signals.
        """
        return {'m': torch.cat([edges.src['node_features'],  # feature of neighbor nodes
                                edges.data['edge_features']], # feature of edges connecting to neighbor nodes
                                dim=1)}

    def get_activation_node(self, nodes):
        """
            Concatenate features of each node with features of its neighboring nodes and edges connecting the node.
            Condition is already concatenated within the edge features.
            Then, we use MLP to calculate weighted sum of all messages, which is called activation of nodes.
        """
        node_old = nodes.data['node_features']
        m = nodes.mailbox['m'] # collect messages assigned by msg which is fed to update_all function.
        message = torch.cat([node_old.unsqueeze(1).expand(-1, m.size(1), -1), m], dim=2)
        # print("message_size()--------------------")
        # print(message.size(), m.size(), node_old.size())
        # print("--------------------")
        node_activation = self.message_func(message).sum(1)
        return {'a': node_activation}

    def forward(self, g, auxiliary):

        #edge_condition = torch.cat((g.edata["edge_features"], auxiliary.expand(g.num_edges(),-1)), -1)
        #g.edata["edge_features"] = self.message_func(edge_condition).sum(1)
        #g.edata["edge_features"] = torch.cat((g.edata["edge_features"], auxiliary.expand(g.num_edges(),-1)), -1)
        # print("feat size()--------------------")
        # print(g.edata["edge_features"].size(), auxiliary)
        # print("--------------------")
        # get activation of each nodes ("a")
        g.update_all(message_func=self.concat_msg,
                     reduce_func=self.get_activation_node)

        # update node features based on the activation of nodes
        g.ndata['node_features'] = self.node_update_func(g.ndata['a'], g.ndata['node_features'])

        return g

class GraphReadOut(nn.Module):
    def __init__(self, node_embed_dim, graph_embed_dim=None, activation="sigmoid", mode='mean'):
        super(GraphReadOut, self).__init__()

        assert mode in ['max', 'mean', 'sum'], \
            "Expect mode to be 'max' or 'mean' or 'sum', got {}".format(mode)
        assert activation in ['sigmoid', 'relu', 'tanh'], \
            "Expect activation to be 'sigmoid' or 'relu' or 'tanh', got {}".format(activation)
        self.mode = mode
        # Setting from the paper
        if graph_embed_dim:
            self.graph_embed_dim = graph_embed_dim
        else:
            self.graph_embed_dim = 2 * node_embed_dim

        if activation == "sigmoid":
            act = nn.Sigmoid()
        elif activation == "relu":
            act = nn.ReLU()
        elif activation == "tanh":
            act = nn.Tanh()
        # Embed graphs
        self.node_gating = nn.Sequential(
            nn.Linear(node_embed_dim, 1),
            act
        )
        self.node_to_graph = nn.Linear(node_embed_dim,
                                       self.graph_embed_dim)

    def forward(self, g):
        node_embeds = g.ndata['node_features']
        graph_embeds = self.node_gating(node_embeds) * self.node_to_graph(node_embeds)
        with g.local_scope():
            g.ndata['graph_features'] = graph_embeds
            if self.mode == 'max':
                graph_feats = dgl.max_nodes(g, 'graph_features')
            elif self.mode == 'mean':
                graph_feats = dgl.mean_nodes(g, 'graph_features')
            elif self.mode == 'sum':
                graph_feats = dgl.sum_nodes(g, 'graph_features')
        if graph_feats.size(0) == 1:
            return graph_feats.squeeze(0)  # if batch == 1 delete batch dimension
        else:
            return graph_feats


#! /usr/bin/env python
#coding=utf-8
from django.conf.urls import patterns, url
from albums.views import albumdetailview
urlpatterns = patterns('albums.views',
    url(r'^add/$', 'add'),
    url(r'^$', 'indexview'),
    url(r'^(?P<album_id>\d+)/$', albumdetailview),
    url(r'^genre/(?P<genre_name>.+)/$', 'genre'),
    url(r'^search/(?P<key_word>.+)/$', 'search'),
    url(r'^search/$', 'search'),
    )


from bs4 import BeautifulSoup
import re
import requests
import socket

# Connect to the challenge endpoint and read in the data
server_address = ('C01-target.allyourbases.co', 8147)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(server_address)
data: bytes = sock.recv(10000)
data: str = data.decode('utf-8')

# Extract the encrypted key
pattern = r"Protected credential: (.+)"
result = re.search(pattern, data)
key = result.group(1)

# Send Vigenere ciphered key to be auto solved
response = requests.get("https://www.guballa.de/vigenere-solver")
soup = BeautifulSoup(response.text, "html.parser")
form = soup.find("form")
url = form.get("action")
request_token = [
    inp for inp in form.find_all("input")
    if inp.get("name") == "REQUEST_TOKEN"
][0].get("value")
cookie = response.headers.get("Set-Cookie").split(' ')[0]

response = requests.post(
    url,
    data={
        "REQUEST_TOKEN": request_token,
        "cipher": key,
        "variant": "vigenere",
        "lang": "en",
        "key_len": "3-30",
        "break": "Break Cipher"
    },
    headers={"cookie": cookie}
)

# Parse response to retrieve solution
soup = BeautifulSoup(response.text, "html.parser")
div = soup.find(id="vig_clear")
protected_key: str = div.find("textarea").contents[0]
pattern = r"THEPROTECTEDKEYIS(.+)"
result = re.search(pattern, protected_key)
protected_key: str = result.group(1)

# Send plaintext key to challenge endpoint
sock.send(protected_key.encode("utf-8") + b"\n")
data: bytes = sock.recv(1024)
sock.send(b"\n")
data: bytes = sock.recv(1024)
data: str = data.decode("utf-8")

# Extract the flag
pattern = r"Flag\[\w+\]"
result = re.search(pattern, data)
flag = result.group(0)
print(f"[*] Got the flag: {flag}")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# File              : referential.py
# Author            : Chi Han, Jiayuan Mao
# Email             : haanchi@gmail.com, maojiayuan@gmail.com
# Date              : 23.10.2019
# Last Modified Date: 23.10.2019
# Last Modified By  : Chi Han, Jiayuan Mao
#
# This file is part of the VCML codebase
# Distributed under MIT license

import six
import sys
import torch
import numpy as np

from IPython.core import ultratb

import jacinle.random as random
from jacinle.utils.meter import GroupMeters

sys.excepthook = ultratb.FormattedTB(
    mode='Plain', color_scheme='Linux', call_pdb=True)

class Definition(object):
    annotation_attribute_names = ['color', 'material', 'shape', 'size']
    annotation_relation_names = ['behind', 'front', 'left', 'right']
    concepts = {
        'color': ['gray', 'red', 'blue', 'green',
                  'brown', 'purple', 'cyan', 'yellow'],
        'material': ['rubber', 'metal'],
        'shape': ['cube', 'sphere', 'cylinder'],
        'size': ['small', 'large']
    }
    concept2attribute = {
        v: k for k, vs in concepts.items() for v in vs
    }
    relational_concepts = {
        'spatial_relation': ['left', 'right', 'front', 'behind']
    }

    synonyms = {
        "thing": ["thing", "object"],
        "sphere": ["sphere", "ball"],
        "cube": ["cube", "block"],
        "cylinder": ["cylinder"],
        "large": ["large", "big"],
        "small": ["small", "tiny"],
        "metal": ["metallic", "metal", "shiny"],
        "rubber": ["rubber", "matte"],
    }

    word2lemma = {
        v: k for k, vs in synonyms.items() for v in vs
    }

def_ = Definition()

def get_desc(obj):
    names = [obj[k] for k in def_.annotation_attribute_names]
    for i, n in enumerate(names):
        if n in def_.synonyms:
            names[i] = random.choice_list(def_.synonyms[n])
    return names

def run_desc_obj(obj, desc):
    for d in desc:
        dd = def_.word2lemma.get(d, d)
        if dd != obj[def_.concept2attribute[dd]]:
            return False
    return True

def run_desc_pred(all_preds, desc):
    s = 10000
    for d in desc:
        s = np.fmin(s, all_preds[d])
    return s

def test(index, all_objs, all_preds, meter):
    obj = all_objs[index]
    nr_descriptors = random.randint(1, 3)
    desc = random.choice_list(get_desc(obj), size=nr_descriptors)
    if isinstance(desc, six.string_types):
        desc = [desc]

    filtered_objs = np.array([
        i for i, o in enumerate(all_objs)
        if not run_desc_obj(o, desc)], dtype=int)
    all_scores = run_desc_pred(all_preds, desc)
    rank = (all_scores[filtered_objs] > all_scores[index]).sum()

    meter.update('r@01', rank <= 1)
    meter.update('r@02', rank <= 2)
    meter.update('r@03', rank <= 3)
    meter.update('r@04', rank <= 4)
    meter.update('r@05', rank <= 5)

def get_preds(model, ref_dataset, logger):
    device = model.device
    args = model.args
    concepts = model.tools.concepts

    preds = {}

    def add_preds(data_batch):
        batch = {
            'batch_size': len(data_batch),
            'image': torch.Tensor(
                [data['image'] for data in data_batch]
            ).to(device),
            'objects': torch.Tensor(
                np.concatenate([data['objects'] for data in data_batch])
            ).to(device),
            'object_length': torch.LongTensor(
                [data['object_length'] for data in data_batch]),
            'image_id': [data['image_id'] for data in data_batch],
        }
        features = model.process_objects(batch)
        concept_embedding = model.embedding.get_all_concept_embeddings()
        for image_id, one_feature in zip(batch['image_id'], features):
            logits = model.embedding.logit_fn(
                one_feature, concept_embedding
            ).transpose(0, 1).detach().cpu().numpy()
            preds[image_id] = dict(zip(
                concepts, logits
            ))

    with torch.no_grad():
        pbar = logger.tqdm(ref_dataset.indexes)
        data_batch = []

        for image_index in pbar:
            data_batch.append(ref_dataset[image_index])
            if len(data_batch) == args.batch_size:
                add_preds(data_batch)
                data_batch.clear()
        if len(data_batch) != 0:
            add_preds(data_batch)
        data_batch.clear()

    return preds

def ref_epoch(coach, prepare_fn, recording, ref_dataset):
    prepare_fn()
    model = coach.model
    recording.reset()

    preds = get_preds(model, ref_dataset, coach.logger)

    meter = GroupMeters()

    for image_id, pred in coach.logger.tqdm(preds.items(), mininterval=0.5):
        scene = ref_dataset.sceneGraphs[image_id]
        for j in range(len(scene['objects'])):
            test(j, list(scene['objects'].values()), pred, meter)

    recording.record(meter.avg)


__author__ = 'sail'
import pandas as pd
from functools import reduce
import numpy as np
import matplotlib.pyplot as plt

def analyse():
    # 该函数用于将一个dataFrame合并为一条记录
    # 入参x为一个dataFrame，调用tt对每个Series分别进行合并
    def t(x):
        return x.apply(tt)

    # 该函数用于对一个Series进行合并,入参x为一个Series
    def tt(x):
        # 当x为标题时用reduce调用lcs函数求从字符串头开始的最长公共子串
        if x.name == '标题':
            return reduce(lcs, x)
        # 当x为下列内容是取最后一项
        elif x.name in {'日期', '粉丝数'}:
            return x[-1]
        # 其他情况时对将x中所有数据合并为一组求和，并返回该组求和结果
        else:
            return x.groupby(lambda x: 1).sum().values[0]

    def lcs(x, y):
        for i in range(min(len(y), len(x))):
            if x[i] != y[i]:
                break
        else:
            i += 1
        return x[:i]

    df = pd.ExcelFile(r"resource/wxgzhdata.xlsx").parse(u"3月")
    df.insert(0, '标题', df['文章标题'])
    df = df.fillna(method="pad").dropna().set_index(['文章标题']).groupby(lambda x: x[:2]).apply(lambda x: x.apply(
        lambda y: {1: lambda z: reduce(lcs, z), 2: lambda z: z[-1], None: lambda z: z.groupby(type).sum().values[0]}
        [{'标题': 1, '日期': 2, '粉丝数': 2}.get(y.name)](y))).set_index(['标题'])
    # 下面为上句的另一种表达方式
    # 用fillna填充nan
    # df=df.fillna(method="pad")
    # 将开头未能被pad填充的nan去除
    # df=df.dropna()
    # 将标题设为index以方便分组
    # df=df.set_index(['文章标题'])
    # 以标题的前两个字为基准分组
    # df=df.groupby(lambda x: x[:2])
    # 利用函数t对每个groupBy进行合并
    # df=df.apply(t)
    # 将各系列内文章标题的相同部分作为系列名称
    # df=df.set_index(['标题'])

    # 从dataframe中取得之后计算需要的Series
    fields = [df.xs('总阅读人数', axis=1), df.xs('初次打开阅读人数', axis=1), df.xs('分享次数', axis=1), df.xs('每日增粉人数', axis=1),
              df.xs('粉丝数', axis=1)]
    to_percent = lambda x: (x * 100).round(1)
    # 各字段计算公式
    expressions = [('初次打开率', lambda f: f[1] / f[4]), ('分享率', lambda f: f[2] / f[0]), ('分享拉粉率', lambda f: f[3] / f[2]),
                   ('增粉速率', lambda f: f[3] / f[4]), ( '阅读涨粉率', lambda f: f[3] / f[0]),
                   ('传播涨粉率', lambda f: f[3] / (f[0] - f[1])), ('二次传播率', lambda f: (f[0] - f[1]) / f[0])]
    # 记录现有字段数量，用于下面计算新字段插入位置以及删除旧字段。
    old_field_count = len(df.columns)
    # 计算各字段的值，处理为percent后，依次插入datagrame中
    for i in enumerate(expressions, start=old_field_count):
        df.insert(i[0], i[1][0], (i[1][1](fields)).apply(to_percent))
    # 转制后将原始字段去除
    df = df.T[old_field_count:]

    df.plot(kind='barh', figsize=(16, 9))

    from pylab import mpl
    mpl.rcParams['font.sans-serif'] = ['SimHei']  # 指定默认字体
    mpl.rcParams['axes.unicode_minus'] = False  # 解决保存图像是负号'-'显示为方块的问题
    plt.show()


from decimal import Decimal

from django.test import TestCase

from order.models import Order, OrderProduct, Parcel
from shop.models import Product, Address, Category

class OrderModelTest(TestCase):
    def setUp(self):
        category = Category.objects.create(name='Books', slug='books')
        self.product = Product.objects.create(category=category,
                                name='Django 3',
                                slug='django_3',
                                description='practical web application development',
                                price=69.99,
                                available=True,
                                quantity_available=13)
        self.product_2 = Product.objects.create(category=category,
                                name='Peak',
                                slug='peak',
                                description='Anyone who wants to get better at anything should read Peak',
                                price=49.99,
                                available=True,
                                quantity_available=4)
        Parcel.objects.create(street='Krotka', street_number='3', ZIP_code='08-116', town='Seroczyn', country='Poland', name='SER1', longitude='54.21212', latitude='31.21424')

        self.order = Order.objects.create(first_name='Alek', last_name='Wiedenski', street='Krotka', street_number='3', ZIP_code='08-116', town='Seroczyn', country='Poland')

        self.order_product = OrderProduct.objects.create(order=self.order, product=self.product, quantity=3)
        self.order_product_2 = OrderProduct.objects.create(order=self.order, product=self.product_2, quantity=2)

    def test_Order(self):
        self.assertEqual(self.order.first_name, 'Alek')

    def test_Order_Product(self):
        self.assertEqual(self.order_product.quantity, 3)

    def test_Order_Product_total_price(self):
        self.assertEqual(self.order_product.total_price(), self.product.price * self.order_product.quantity)

    def test_Order_total_price(self):
        self.assertEqual(self.order.total_price(), Decimal(str((self.product.price * self.order_product.quantity) + (self.product_2.price * self.order_product_2.quantity))))

    def test_Parcel(self):
        parcels = Parcel.objects.count()

        self.assertEqual(parcels, 1)


from django.db import models
from django.utils import timezone
from . import producer as benchmark_producer
from . import models as benchmark_models
from . import consumers as benchmark_consumers

def on_new_benchmark(
    sender: models.Model,
    instance: benchmark_models.Benchmark,
    **kwargs
) -> None:
    """Publish a new benchmark to the event bus and created a benchmark
    progress record.

    Args:
        instance: The benchmark to publish.
    """
    benchmark_models.BenchmarkProgress.objects.create(
        benchmark=instance
    ).save()
    benchmark_producer.new_benchmark(instance)

def on_benchmark_progress_update(
    sender: models.Model,
    instance: benchmark_models.BenchmarkProgress,
    **kwargs
) -> None:
    status_choices = benchmark_models.BenchmarkProgress.StatusChoices

    # Update the benchmark time logs.
    loggable_statues = (
        status_choices.PROVISIONING,
        status_choices.SCHEDULING,
        status_choices.COMPLETED,
    )
    if instance.status not in loggable_statues:
        pass
    if instance.status == status_choices.PROVISIONING:
        instance.benchmark.started_on = timezone.now()
    elif instance.status == status_choices.SCHEDULING:
        instance.benchmark.scheduled_on = timezone.now()
    elif instance.status == status_choices.COMPLETED:
        instance.benchmark.completed_on = timezone.now()
    instance.benchmark.save()

    # Send notification with new status.
    benchmark_consumers.BenchmarkProgressConsumer.send_status_update(
        site_id=instance.benchmark.site_id,
        benchmark_id=instance.benchmark_id,
        status=instance.get_status_display(),
        num_servers=instance.benchmark.num_servers,
        num_requests=instance.benchmark.num_requests,
        completed_requests=instance.benchmark.completed_requests,
        failed_requests=instance.benchmark.failed_requests,
        created_on=instance.benchmark.created_on,
        scheduled_on=instance.benchmark.scheduled_on,
        min_time=instance.benchmark.min_time,
        mean_time=instance.benchmark.mean_time,
        max_time=instance.benchmark.max_time,
    )

benchmark_models.NEW_BENCHMARK.connect(
    on_new_benchmark,
    benchmark_models.Benchmark
)

benchmark_models.UPDATED_BENCHMARK_PROGRESS.connect(
    on_benchmark_progress_update,
    benchmark_models.BenchmarkProgress
)


import os
from PIL import Image
from torch.utils.data import Dataset

class myDataset(Dataset):
    def __init__(self, data_dir, transform):
        # data_dir：图片保存路径
        # transform：初始化方法
        self.data_dir = data_dir
        self.transform = transform
        # 训练集图片为png格式，若为jpg格式，可对应修改为x.endswith("jpg")
        self.img_names = [name for name in list(filter(lambda x: x.endswith(".png"), os.listdir(self.data_dir)))]

    def __getitem__(self, index):
        path_img = os.path.join(self.data_dir, self.img_names[index])
        img = Image.open(path_img).convert('RGB')

        if self.transform is not None:
            img = self.transform(img)

        return img

    def __len__(self):
        if len(self.img_names) == 0:
            raise Exception("\ndata_dir:{} is a empty dir! Please checkout your path to images_wgan_gp!".format(self.data_dir))
        return len(self.img_names)


# K-means implementation for assignment 2
# William Pourmajidi

# Importing Required library for plotting numbers
import matplotlib.pyplot as plt
import seaborn as sns;
sns.set()  # for plot styling
# Importing Required library for working with arrays (key,value)
from sklearn.datasets.samples_generator import make_blobs

# Importing Required library for K-means
from sklearn.cluster import KMeans

X = np.array([[8, 4], [5, 4], [2, 4], [2, 6], [2, 8], [8, 6]])

kmeans = KMeans(n_clusters=3, random_state=0).fit(X)
y_kmeans = kmeans.predict(X)
print(X)
print(y_kmeans)
plt.scatter(X[:, 0], X[:, 1]);
plt.show()
plt.scatter(X[:, 0], X[:, 1], c=y_kmeans, s=50, cmap='viridis')
plt.show()

centers = kmeans.cluster_centers_
plt.scatter(centers[:, 0], centers[:, 1], c='black', s=200, alpha=0.5);
plt.show()


import os
import torch
import torch_geometric as pyg
import numpy as np
import networkx as nx
from utils import load_pickle

class GraphDataset(torch.utils.data.Dataset):
    """
    Graph Dataset. Collects NetworkX graph from a pre-defined folder and
    transforms them to Pytorch Geometric (pyg.data.Data()) instances.
    """
    def __init__(self, path, graph_type = 'zoning'):
        self.graph_path = os.path.join(path, 'graph_in' if 'zoning' in graph_type else 'graph_out')
        self.struct_path = os.path.join(path, 'struct_in')
        self.full_path = os.path.join(path, 'full_out')

        # include graph transformations if you like
        # self.graph_transform = graph_transform

    def __getitem__(self, index):

        # get access graph (name is index)
        graph_nx = load_pickle(os.path.join(self.graph_path, f'{index}.pickle'))

        # add images as graph attributes
        struct = np.load(os.path.join(self.struct_path, f'{index}.npy'))
        full = np.load(os.path.join(self.full_path, f'{index}.npy'))
        graph_nx.graph['struct'] = struct[np.newaxis, ...]
        graph_nx.graph['full'] = full[np.newaxis, ...]

        # transform networkx graph to pytorch geometric graph
        graph_pyg = pyg.utils.from_networkx(graph_nx)

        # transform graph if you like
        # graph_pyg = self.graph_transform(graph_pyg)

        return graph_pyg

    def __len__(self):
        return len(os.listdir(self.graph_path))


import logging
import pandas as pd
from datetime import datetime
import talib as ta
import plotly.graph_objects as go
from plotly.subplots import make_subplots
from .base_strategy import BaseStrategy
import indicators as mta
from order import Order, OrderType, OrderSide, OrderStatus
from utils import get_line_coffs, find_uptrend_line, find_downtrend_line, get_y_on_line

bot_logger = logging.getLogger("bot_logger")

class BreakStrategy(BaseStrategy):
    """
    - EMA HeikinAshi strategy
    """

    def __init__(self, name, params, tfs):
        super().__init__(name, params, tfs)
        self.tf = self.tfs["tf"]
        self.state = None
        self.trend = None
        self.min_zz_ratio = 0.01 * self.params["min_zz_pct"]
        self.temp = []

    def attach(self, tfs_chart):
        self.tfs_chart = tfs_chart
        self.init_indicators()

    def init_indicators(self):
        # calculate HA candelstick
        chart = self.tfs_chart[self.tf]
        self.ma_vol = ta.SMA(chart["Volume"], self.params["ma_vol"])
        self.zz_points = mta.zigzag(chart, self.min_zz_ratio)
        self.init_main_zigzag()
        self.start_trading_time = chart.iloc[-1]["Open time"]

    def init_main_zigzag(self):
        self.main_zz_idx = []
        if len(self.zz_points) > 0:
            self.main_zz_idx.append(0)
        else:
            return
        last_main_zz_idx = 0
        while last_main_zz_idx + 3 < len(self.zz_points):
            last_main_zz_type = self.zz_points[last_main_zz_idx].ptype
            if last_main_zz_type == mta.POINT_TYPE.PEAK_POINT:
                if (
                    self.zz_points[last_main_zz_idx + 2].pline.high < self.zz_points[last_main_zz_idx].pline.high
                    and self.zz_points[last_main_zz_idx + 3].pline.low < self.zz_points[last_main_zz_idx + 1].pline.low
                ):
                    last_main_zz_idx += 2
                else:
                    last_main_zz_idx += 1
                    self.main_zz_idx.append(last_main_zz_idx)
            else:
                if (
                    self.zz_points[last_main_zz_idx + 2].pline.low > self.zz_points[last_main_zz_idx].pline.low
                    and self.zz_points[last_main_zz_idx + 3].pline.high > self.zz_points[last_main_zz_idx + 1].pline.low
                ):
                    last_main_zz_idx += 2
                else:
                    last_main_zz_idx += 1
                    self.main_zz_idx.append(last_main_zz_idx)

    def update_main_zigzag(self):
        last_main_zz_idx = self.main_zz_idx[-1]
        while last_main_zz_idx + 3 < len(self.zz_points):
            last_main_zz_type = self.zz_points[last_main_zz_idx].ptype
            if last_main_zz_type == mta.POINT_TYPE.PEAK_POINT:
                if (
                    self.zz_points[last_main_zz_idx + 2].pline.high < self.zz_points[last_main_zz_idx].pline.high
                    and self.zz_points[last_main_zz_idx + 3].pline.low < self.zz_points[last_main_zz_idx + 1].pline.low
                ):
                    last_main_zz_idx += 2
                else:
                    last_main_zz_idx += 1
                    self.main_zz_idx.append(last_main_zz_idx)
            else:
                if (
                    self.zz_points[last_main_zz_idx + 2].pline.low > self.zz_points[last_main_zz_idx].pline.low
                    and self.zz_points[last_main_zz_idx + 3].pline.high
                    > self.zz_points[last_main_zz_idx + 1].pline.high
                ):
                    last_main_zz_idx += 2
                else:
                    last_main_zz_idx += 1
                    self.main_zz_idx.append(last_main_zz_idx)

        last_main_zz_type = self.zz_points[last_main_zz_idx].ptype
        if last_main_zz_idx + 2 < len(self.zz_points):
            if last_main_zz_type == mta.POINT_TYPE.POKE_POINT:
                if self.zz_points[last_main_zz_idx + 2].pline.low < self.zz_points[last_main_zz_idx].pline.low:
                    last_main_zz_idx += 1
                    self.main_zz_idx.append(last_main_zz_idx)
            else:
                if self.zz_points[last_main_zz_idx + 2].pline.high > self.zz_points[last_main_zz_idx].pline.high:
                    last_main_zz_idx += 1
                    self.main_zz_idx.append(last_main_zz_idx)

        last_main_zz_type = self.zz_points[last_main_zz_idx].ptype
        if last_main_zz_idx + 1 < len(self.zz_points):
            last_kline = self.tfs_chart[self.tf].iloc[-1]
            if last_main_zz_type == mta.POINT_TYPE.POKE_POINT:
                if last_kline["Low"] < self.zz_points[last_main_zz_idx].pline.low:
                    last_main_zz_idx += 1
                    self.main_zz_idx.append(last_main_zz_idx)
            else:
                if last_kline["High"] > self.zz_points[last_main_zz_idx].pline.high:
                    last_main_zz_idx += 1
                    self.main_zz_idx.append(last_main_zz_idx)

    def update_indicators(self, tf):
        if tf != self.tf:
            return
        chart = self.tfs_chart[self.tf]
        self.ma_vol.loc[len(self.ma_vol)] = ta.stream.SMA(chart["Volume"], self.params["ma_vol"])
        last_Zz_pidx = self.zz_points[-1].pidx
        mta.zigzag_stream(chart, self.min_zz_ratio, self.zz_points)
        last_main_idx = self.main_zz_idx[-1]
        self.update_main_zigzag()
        if last_main_idx != self.main_zz_idx[-1]:
            self.temp.append((self.zz_points[self.main_zz_idx[-1]].pidx, len(chart) - 1))
            self.adjust_sl()

    def check_required_params(self):
        return all(
            [
                key in self.params.keys()
                for key in [
                    "ma_vol",
                    "vol_ratio_ma",
                    "kline_body_ratio",
                    "sl_fix_mode",
                ]
            ]
        )

    def is_params_valid(self):
        if not self.check_required_params():
            bot_logger.info("   [-] Missing required params")
            return False
        return True

    def update(self, tf):
        # update when new kline arrive
        super().update(tf)
        # check order signal
        self.check_close_signal()
        self.check_signal()

    def close_opening_orders(self):
        super().close_opening_orders(self.tfs_chart[self.tf].iloc[-1])

    def check_signal(self):
        chart = self.tfs_chart[self.tf]
        last_kline = chart.iloc[-1]
        if last_kline["Volume"] < self.params["vol_ratio_ma"] * self.ma_vol.iloc[-1]:
            return
        idx = 1
        while idx < len(self.zz_points):
            zz_point_1 = self.zz_points[-idx]
            zz_point_2 = self.zz_points[-idx - 1]
            if zz_point_2.ptype == mta.POINT_TYPE.POKE_POINT:
                change = (zz_point_1.pline.high - zz_point_2.pline.low) / zz_point_2.pline.low
            else:
                change = (zz_point_2.pline.high - zz_point_1.pline.low) / zz_point_2.pline.high
            if change > self.params["zz_dev"] * self.min_zz_ratio:
                break
            idx += 1

        n_df = chart[self.zz_points[-idx].pidx : -1]
        if len(n_df) < self.params["min_num_cuml"]:
            return
        n_last_poke_points = []
        n_last_peak_points = []
        for i, kline in n_df.iterrows():
            n_last_poke_points.append((i, kline["Low"]))
            n_last_peak_points.append((i, kline["High"]))
        kline_body_pct = n_df[["Open", "Close"]].max(axis=1) - n_df[["Open", "Close"]].min(axis=1)
        mean_kline_body = kline_body_pct.mean()
        if abs(last_kline["Close"] - last_kline["Open"]) < self.params["kline_body_ratio"] * mean_kline_body:
            return
        self.up_trend_line = find_uptrend_line(n_last_poke_points)
        self.down_trend_line = find_downtrend_line(n_last_peak_points)

        self.up_pct = (self.up_trend_line[1][1] - self.up_trend_line[0][1]) / self.up_trend_line[0][1]
        self.down_pct = (self.down_trend_line[1][1] - self.down_trend_line[0][1]) / self.down_trend_line[0][1]
        delta_end = abs(self.down_trend_line[1][1] - self.up_trend_line[1][1]) / self.up_trend_line[1][1]
        if delta_end > self.params["zz_dev"] * self.min_zz_ratio:
            return
        if last_kline["Close"] > last_kline["Open"]:
            # green kkline
            if (last_kline["High"] - last_kline["Close"]) > 0.5 * (last_kline["High"] - last_kline["Low"]):
                return
            y_down_pct = get_y_on_line(self.down_trend_line, self.down_trend_line[1][0] + 1)
            if last_kline["Close"] > y_down_pct and last_kline["Close"] > ta.stream.SMA(chart["Close"], 200):
                sl = self.up_trend_line[1][1]
                order = Order(
                    OrderType.MARKET,
                    OrderSide.BUY,
                    last_kline["Close"],
                    tp=None,
                    sl=sl,
                    status=OrderStatus.FILLED,
                )
                order["FILL_TIME"] = last_kline["Open time"]
                order["strategy"] = self.name
                order["description"] = self.description
                order["desc"] = {
                    "up_trend_line": self.up_trend_line,
                    "down_trend_line": self.down_trend_line,
                }
                order = self.trader.fix_order(order, self.params["sl_fix_mode"], self.max_sl_pct)
                if order:
                    self.trader.create_trade(order, self.volume)
                    self.orders_opening.append(order)
        else:
            # red kkline
            if (last_kline["Close"] - last_kline["Low"]) > 0.5 * (last_kline["High"] - last_kline["Low"]):
                return
            y_up_pct = get_y_on_line(self.up_trend_line, self.up_trend_line[1][0] + 1)
            if last_kline["Close"] < y_up_pct and last_kline["Close"] < ta.stream.SMA(chart["Close"], 200):
                sl = self.down_trend_line[1][1]
                order = Order(
                    OrderType.MARKET,
                    OrderSide.SELL,
                    last_kline["Close"],
                    tp=None,
                    sl=sl,
                    status=OrderStatus.FILLED,
                )
                order["FILL_TIME"] = last_kline["Open time"]
                order["strategy"] = self.name
                order["description"] = self.description
                order["desc"] = {
                    "up_trend_line": self.up_trend_line,
                    "down_trend_line": self.down_trend_line,
                }
                order = self.trader.fix_order(order, self.params["sl_fix_mode"], self.max_sl_pct)
                if order:
                    self.trader.create_trade(order, self.volume)
                    self.orders_opening.append(order)

    def check_close_signal(self):
        self.check_close_reverse()
        chart = self.tfs_chart[self.tf]
        last_kline = chart.iloc[-1]
        if last_kline["Volume"] < self.params["vol_ratio_ma"] * self.ma_vol.iloc[-1]:
            return
        if last_kline["Close"] < last_kline["Open"]:
            # red kline, check close buy orders
            if (last_kline["High"] - last_kline["Close"]) < 0.75 * (last_kline["High"] - last_kline["Low"]):
                return
            for i in range(len(self.orders_opening) - 1, -1, -1):
                order = self.orders_opening[i]
                if order.side == OrderSide.SELL:
                    continue
                desc = order["desc"]
                up_trend_line = desc["up_trend_line"]
                down_trend_line = desc["down_trend_line"]
                y_up = get_y_on_line(up_trend_line, len(chart) - 1)
                if last_kline["Close"] < y_up:
                    order["desc"]["stop_idx"] = len(chart) - 1
                    order["desc"]["y"] = y_up
                    order.close(last_kline)
                    self.trader.close_trade(order)
                    if order.is_closed():
                        self.orders_closed.append(order)
                    del self.orders_opening[i]
        else:
            # green kline, check close sell orders
            if (last_kline["Close"] - last_kline["Low"]) < 0.75 * (last_kline["High"] - last_kline["Low"]):
                return
            for i in range(len(self.orders_opening) - 1, -1, -1):
                order = self.orders_opening[i]
                if order.side == OrderSide.BUY:
                    continue
                desc = order["desc"]
                up_trend_line = desc["up_trend_line"]
                down_trend_line = desc["down_trend_line"]
                y_down = get_y_on_line(down_trend_line, len(chart) - 1)
                if last_kline["Close"] > y_down:
                    order["desc"]["stop_idx"] = len(chart) - 1
                    order["desc"]["y"] = y_down
                    order.close(last_kline)
                    self.trader.close_trade(order)
                    if order.is_closed():
                        self.orders_closed.append(order)
                    del self.orders_opening[i]

    def check_close_reverse(self):
        # check close reverse order (buy order has uptrend downward)
        if len(self.main_zz_idx) < 3:
            return
        chart = self.tfs_chart[self.tf]
        last_kline = chart.iloc[-1]
        if self.zz_points[self.main_zz_idx[-1]].ptype == mta.POINT_TYPE.PEAK_POINT:
            if self.zz_points[self.main_zz_idx[-1]].pline.high < self.zz_points[self.main_zz_idx[-3]].pline.high:
                for i in range(len(self.orders_opening) - 1, -1, -1):
                    order = self.orders_opening[i]
                    if order.side == OrderSide.SELL:
                        continue
                    desc = order["desc"]
                    up_trend_line = desc["up_trend_line"]
                    if up_trend_line[0][1] * 1.005 >= up_trend_line[1][1]:
                        order["desc"]["stop_idx"] = len(chart) - 1
                        order["desc"]["y"] = last_kline["Close"]
                        order.close(last_kline)
                        self.trader.close_trade(order)
                        if order.is_closed():
                            self.orders_closed.append(order)
                        del self.orders_opening[i]
        else:
            if self.zz_points[self.main_zz_idx[-1]].pline.low > self.zz_points[self.main_zz_idx[-3]].pline.low:
                for i in range(len(self.orders_opening) - 1, -1, -1):
                    order = self.orders_opening[i]
                    if order.side == OrderSide.BUY:
                        continue
                    desc = order["desc"]
                    down_trend_line = desc["down_trend_line"]
                    if down_trend_line[0][1] <= down_trend_line[1][1] * 1.005:
                        order["desc"]["stop_idx"] = len(chart) - 1
                        order["desc"]["y"] = last_kline["Close"]
                        order.close(last_kline)
                        self.trader.close_trade(order)
                        if order.is_closed():
                            self.orders_closed.append(order)
                        del self.orders_opening[i]

    def adjust_sl(self):
        last_main_zz = self.zz_points[self.main_zz_idx[-1]]
        if last_main_zz.ptype == mta.POINT_TYPE.PEAK_POINT:
            for i in range(len(self.orders_opening) - 1, -1, -1):
                order = self.orders_opening[i]
                if order.side == OrderSide.BUY:
                    continue
                if not order.has_sl() or order.sl > last_main_zz.pline.high:
                    order.adjust_sl(last_main_zz.pline.high)
                    self.trader.adjust_sl(order, last_main_zz.pline.high)
        else:
            for i in range(len(self.orders_opening) - 1, -1, -1):
                order = self.orders_opening[i]
                if order.side == OrderSide.SELL:
                    continue
                if not order.has_sl() or order.sl < last_main_zz.pline.low:
                    order.adjust_sl(last_main_zz.pline.low)
                    self.trader.adjust_sl(order, last_main_zz.pline.low)

    def plot_orders(self):
        fig = make_subplots(2, 1, vertical_spacing=0.02, shared_xaxes=True, row_heights=[0.8, 0.2])
        fig.update_layout(
            xaxis_rangeslider_visible=False,
            xaxis2_rangeslider_visible=False,
            yaxis=dict(showgrid=False),
            yaxis2=dict(showgrid=False),
            xaxis=dict(showgrid=False),
            xaxis2=dict(showgrid=False),
            plot_bgcolor="rgb(19, 23, 34)",
            paper_bgcolor="rgb(121,125,127)",
            font=dict(color="rgb(247,249,249)"),
        )
        df = self.tfs_chart[self.tf]
        dt2idx = dict(zip(df["Open time"], list(range(len(df)))))
        tmp_ot = df["Open time"]
        df["Open time"] = list(range(len(df)))
        super().plot_orders(fig, self.tf, 1, 1, dt2idx=dt2idx)

        fig.add_trace(
            go.Scatter(
                x=[df.iloc[ad.pidx]["Open time"] for ad in self.zz_points],
                y=[ad.pline.low if ad.ptype == mta.POINT_TYPE.POKE_POINT else ad.pline.high for ad in self.zz_points],
                mode="lines",
                line=dict(dash="dash"),
                marker_color=[
                    "rgba(255, 255, 0, 1)" if ad.ptype == mta.POINT_TYPE.POKE_POINT else "rgba(0, 255, 0, 1)"
                    for ad in self.zz_points
                ],
                name="ZigZag Point",
                hoverinfo="skip",
            ),
            row=1,
            col=1,
        )

        fig.add_trace(
            go.Scatter(
                x=df["Open time"],
                y=ta.SMA(df["Close"], 200),
                mode="lines",
                line=dict(color="orange"),
                name="SMA_200",
                hoverinfo="skip",
            ),
            row=1,
            col=1,
        )

        main_zz_points = [self.zz_points[i] for i in self.main_zz_idx]
        fig.add_trace(
            go.Scatter(
                x=[df.iloc[ad.pidx]["Open time"] for ad in main_zz_points],
                y=[ad.pline.low if ad.ptype == mta.POINT_TYPE.POKE_POINT else ad.pline.high for ad in main_zz_points],
                mode="lines",
                name="Main ZigZag Point",
                hoverinfo="skip",
            ),
            row=1,
            col=1,
        )
        for x, y in self.temp:
            fig.add_shape(
                type="line",
                x0=df.iloc[x]["Open time"],
                y0=df.iloc[x]["High"],
                x1=df.iloc[y]["Open time"],
                y1=df.iloc[x]["High"],
                line=dict(color="red"),
                row=1,
                col=1,
            )

        for order in self.orders_closed:
            desc = order["desc"]
            up_trend = desc["up_trend_line"]
            down_trend = desc["down_trend_line"]
            fig.add_trace(
                go.Scatter(
                    x=[
                        df.iloc[up_trend[0][0]]["Open time"],
                        df.iloc[up_trend[1][0]]["Open time"],
                        df.iloc[down_trend[1][0]]["Open time"],
                        df.iloc[down_trend[0][0]]["Open time"],
                        df.iloc[up_trend[0][0]]["Open time"],
                    ],
                    y=[up_trend[0][1], up_trend[1][1], down_trend[1][1], down_trend[0][1], up_trend[0][1]],
                    mode="lines",
                    line=dict(color="rgba(156, 39, 176, 0.8)", width=2),
                    fill="toself",
                    fillcolor="rgba(128, 0, 128, 0.2)",
                    hoverinfo="skip",
                    showlegend=False,
                )
            )

        colors = [
            "rgb(242, 54, 69)" if kline["Open"] > kline["Close"] else "rgb(8, 153, 129)" for i, kline in df.iterrows()
        ]
        fig.add_trace(
            go.Bar(x=list(range(len(df))), y=df["Volume"], marker_color=colors, marker_line_width=0, name="Volume"),
            row=2,
            col=1,
        )
        fig.add_trace(
            go.Scatter(
                x=df["Open time"],
                y=self.ma_vol,
                mode="lines",
                line=dict(color="rgba(0, 255, 0, 1)"),
                name="MA_VOL_{}".format(self.params["ma_vol"]),
            ),
            row=2,
            col=1,
        )
        fig.update_xaxes(showspikes=True, spikesnap="data")
        fig.update_yaxes(showspikes=True, spikesnap="data")
        fig.update_layout(hovermode="x", spikedistance=-1)
        fig.update_layout(hoverlabel=dict(bgcolor="white", font_size=16))
        fig.update_layout(
            title={
                "text": "Breakout Strategy({}) (tf {})".format(self.trader.symbol_name, self.tf),
                "x": 0.5,
                "xanchor": "center",
            }
        )
        df["Open time"] = tmp_ot
        return fig


import json
import os

from requests_oauthlib import OAuth2Session
from requests_oauthlib.compliance_fixes.facebook import facebook_compliance_fix

from planevent import settings
from planevent.accounts import (
    models,
    tasks,
    auth,
)

os.environ['OAUTHLIB_RELAX_TOKEN_SCOPE'] = 'True'

def _get_oauth2_callback_url(request, provider):
    return request.route_url('oauth2_callback', provider=provider)

def login_oauth(request, provider):
    provider_settings = settings.OAUTH[provider]

    oauth = OAuth2Session(
        provider_settings.client_id,
        redirect_uri=_get_oauth2_callback_url(request, provider),
        scope=provider_settings.scope,
    )

    authorization_url, state = oauth.authorization_url(
        provider_settings.authorize_url,
    )

    return authorization_url

def process_oauth_callback(request, provider):
    provider_settings = settings.OAUTH[provider]

    oauth = OAuth2Session(
        provider_settings.client_id,
        redirect_uri=_get_oauth2_callback_url(request, provider),
        state=request.params['state'],
        scope=provider_settings.scope,
    )

    if provider == 'facebook':
        oauth = facebook_compliance_fix(oauth)

    token = oauth.fetch_token(
        provider_settings.access_token_url,
        authorization_response=request.url,
        client_secret=provider_settings.secret_key,
    )

    oauth.token = token

    response = oauth.get(provider_settings.user_info_url)
    provider_user = json.loads(response.content.decode('utf8'))

    account, is_new = process_oauth_user(provider, provider_user)

    auth._login(request, account)

    if is_new:
        return request.route_url('home') + '#/userProfile/firstLogging'
    else:
        return request.route_url('home')

def process_oauth_user(provider, provider_user):

    def fill_user_fields(account, provider_user, provider_settings):
        def fill_user_field(field):
            provider_field = provider_settings.mapper.get(field) or field
            if not getattr(account, field):
                setattr(account, field, provider_user.get(provider_field))

        fill_user_field('name')
        fill_user_field('first_name')
        fill_user_field('last_name')
        fill_user_field('gender')
        fill_user_field('link')

    account = models.Account.get_by_provider(provider, provider_user['id'])

    is_new = False
    if not account:
        email = provider_user.get('email')
        if email:
            account = models.Account.get_by_email(email)

    if not account:
        account = models.Account.create(email=provider_user.get('email'))
        is_new = True

    account.credentials.origin_id = provider_user.get('id'),
    account.credentials.provider = provider,

    provider_settings = settings.OAUTH[provider]

    fill_user_fields(account, provider_user, provider_settings)

    if is_new:
        tasks.send_welcome_email(account)

    return account, is_new


# !/usr/bin/env python
# -*- coding: utf-8 -*-
import sys
import os
import re

__author__ = 'JrD'
__time__ = '2019.03.07'

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET

def xmlparser(path=None):
    cwd = os.getcwd()
    absolute_path = os.path.join(cwd, path)
    if os.path.exists(absolute_path):
        filepath = (cwd+'/'+path.split('/')[-1].split('.')[0]+'.csv')
        if os.path.splitext(absolute_path)[-1].lower() == '.xml':
            with open('%s'%(filepath),'w') as write:
                try:
                    tree = ET.ElementTree(file=absolute_path)
                except ET.ParseError:
                    with open(absolute_path, 'a+') as file:
                        file.write('\n</nmaprun>')
                    tree = ET.ElementTree(file=absolute_path)
                root = tree.getroot()
                print('Nmap command: ' + root.attrib['args'])
                print('Start time: ' + root.attrib['startstr'])
                ADDR,PORT,STATE,SERVICE,PRODUCT,VERSION = 'ADDR','PORT','STATE','SERVICE','PRODUCT','VERSION'
                column_name = ('%-20s %-7s %-10s %-15s %-20s %-30s' %(ADDR,PORT,STATE,SERVICE,PRODUCT,VERSION))
                write.write('''%s,%s,%s,%s,%s,%s'''%(ADDR,PORT,STATE,SERVICE,PRODUCT,VERSION))
                print(column_name)
                for host in root.iter('host'):
                    addr = host[1].attrib['addr']
                    for ports in host[3][1:]:
                        port, state, service, product, version = '','','','',''
                        try:
                            port = ports.attrib['portid']
                            state = ports[0].attrib['state']
                            service = ports[1].attrib['name']
                            product = ports[1].attrib['product']
                            version = ports[1].attrib['version']
                        except KeyError:
                            pass
                        print('%-20s %-7s %-10s %-15s %-20s %-30s'% (addr,port,state,service,product,version))
                        write.write('''\n%s,%s,%s,%s,%s,%s''' % (addr,port,state,service,product,version))

            print('Finished: %s' % filepath)
        else:
            print('[ERROR] File is not XML: %s' % path)
    else:
        print('[ERROR] File not found: %s' % path)

if __name__ == '__main__':
    print('''
 | \ | |  _ __ ___     __ _   _ __   \ \/ /  _ __     __ _   ___    ___   _ __
 |  \| | | r'_ ` _ \   / _` | | '_ \  \  /  | r'_ \  / _` | / __|  / _ \ | '__|
 | |\  | | | | | | | | (_| | | |_) |  /  \  | |_) | | (_| | \__ \ |  __/ | |
 |_| \_| |_| |_| |_|  \__,_| | .__/  /_/\_\ | .__/   \__,_| |___/  \___| |_|
                             |_|            |_|                                ''')
    if len(sys.argv) == 2:
        xmlpath = sys.argv[1]
        xmlparser(path=xmlpath)
    else:
        print('''[usage]:python NmapXpaser.py xxxxxx.xml''')
        sys.exit()


# Data Representation - Assignment 3
# Author: Jamie Tohall

# Write a program that retrieves the dataset for the "exchequer account (historical series)"
# from the CSO, and stores it into a file called "cso.json"

import requests # Import relevant modules
import json

url = "https://ws.cso.ie/public/api.restful/PxStat.Data.Cube_API.ReadDataset/FIQ02/JSON-stat/2.0/en"
# url for the exchequer account (historical series) dataset, JSON format

def getalldata():
    response = requests.get(url)
    return response.json()

if __name__ == "__main__":

    with open("cso.json", "w") as fp:
        print(json.dumps(getalldata()),file=fp)


# -*- coding: utf-8 -*-

"""
    Actions for CDNRequest rules
"""

import json

from collections import OrderedDict
from datetime import datetime, timedelta
from time import mktime

from django.forms.models import model_to_dict

from ...engine.actions import rule_action, BaseActions
from ...engine.fields import FIELD_TEXT
from ....models import History, Proof, Ticket, User, ReportItem
from ....utils.cache import redis_lock, RedisHandler
from ....tasks import enqueue_in, helpers

class DefaultCDNRequestActions(BaseActions):
    """
        This class implements actions for CDN requests
    """

    providers = {"cloudflare": {"email": "abuseteam@cloudflare.com"}}

    cache_expirations_days = 15
    redis_queue = "cdnrequest:{}:request"

    def __init__(self, report, domain_to_request):
        """
        """
        self.report = report
        self.domain_to_request = domain_to_request

    @rule_action(params=[{"fieldType": FIELD_TEXT, "name": "provider"}])
    def do_cdn_request(self, provider):
        """
            Make request to supported CDN providers
        """
        ticket, expiration = self._retrieve_from_cache(provider)

        # if already resolved, just attach report
        if expiration and datetime.now() < expiration:
            self.report.defendant = ticket.defendant
            self.report.service = ticket.service
            self.report.save()
            if all((self.report.defendant, self.report.service)):
                # rexecute report workflow
                self._rexecute_report_workflow(ticket)
        else:  # else, send request to provider
            ticket = self._send_request(provider)

        self.report.ticket = ticket
        self.report.status = "Attached"
        self.report.save(update_fields=["ticket", "status"])

        History.log_ticket_action(
            ticket=ticket, action="attach_report", new_ticket=False, report=self.report
        )

    @redis_lock("cdnrequest:lock")
    def _retrieve_from_cache(self, provider):
        """
            Check if there is already a pending/resolved similar request
        """
        expiration = ticket = None
        entries = RedisHandler.ldump(self.redis_queue.format(provider))

        for entry in entries:
            entry = json.loads(entry, object_pairs_hook=OrderedDict)
            if entry["domain"] == self.domain_to_request:
                ticket = Ticket.get(id=entry["request_ticket_id"])
                expiration = datetime.fromtimestamp(entry["expiration"])
                break

        return ticket, expiration

    @redis_lock("cdnrequest:lock")
    def _send_request(self, provider):
        """
            Send email request to CDN Provider
        """
        ticket = helpers.create_ticket(self.report, denied_by=None, attach_new=False)

        ticket.treatedBy = User.objects.get(username="abuse.robot")
        ticket.save()

        self._send_email_request(provider, ticket)
        self._update_cache(ticket, provider)

        return ticket

    def _send_email_request(self, provider, ticket):

        try:
            email = self.providers[provider]["email"]
        except KeyError:
            raise Exception("Unsupported CDN provider")

        proof_content = (
            self.report.reportItemRelatedReport.filter(itemType="FQDN").last().rawItem
        )

        Proof.create(ticket=ticket, content=proof_content)

        helpers.send_email(ticket, [email], "{}_ip_request".format(provider), lang="EN")

        ticket.save()

    def _update_cache(self, ticket, provider):

        entries = RedisHandler.ldump(self.redis_queue.format(provider))

        # Clear old entries
        for entry in entries:
            entry_json = json.loads(entry)
            if entry_json["domain"] == self.domain_to_request:
                RedisHandler.lrem(self.redis_queue.format(provider), entry)

        # Push task
        RedisHandler.rpush(
            self.redis_queue.format(provider),
            json.dumps(
                {
                    "domain": self.domain_to_request,
                    "request_ticket_id": ticket.id,
                    "expiration": self._get_expiration(),
                }
            ),
        )

    def _get_expiration(self):

        exp = datetime.now() + timedelta(days=self.cache_expirations_days)
        return int(mktime(exp.timetuple()))

    def _rexecute_report_workflow(self, ticket):

        # attach resolved URLs
        self.report.attach_url_matching_domain(self.domain_to_request)

        # attach resolved IP address
        item = (
            ticket.reportTicket.first()
            .reportItemRelatedReport.filter(itemType="IP")
            .last()
        )
        item = model_to_dict(item)
        item.pop("id")
        item["report"] = self.report
        ReportItem.create(**item)

        # schedule job
        enqueue_in(
            timedelta(seconds=10),
            "report.validate_with_defendant",
            report_id=self.report.id,
            timeout=3600,
        )


from datetime import datetime
from random import sample

from django.conf import settings
from django.db.models.signals import post_save
from django.dispatch import receiver
from rest_framework.authtoken.models import Token
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import IsAuthenticated

from questdata.serializers import FeedbackSerializer
from users.models import *
from rest_framework.response import Response

from django.shortcuts import render
from django.core.mail import send_mail
from django.contrib.auth import get_user_model
from django.utils.http import urlsafe_base64_encode, urlsafe_base64_decode
from django.contrib.sites.shortcuts import get_current_site
from django.utils.encoding import force_bytes, force_text
from .tokens import account_activation_token
from django.template.loader import render_to_string
# Create your views here.
from users.serializers import UserMetaSerializer
from journeys.serializers import *

from notifications.expo import send_push_message
User = get_user_model()

@receiver(post_save, sender=settings.AUTH_USER_MODEL)
def create_token(sender, instance=None, created=False, **kwargs):
    if created:
        Token.objects.create(user=instance)

@api_view(['POST'])
@permission_classes([])
def register(request):
    # Does not check if request is formatted correctly
    em = request.data["email"]
    um = request.data["username"]

    # If the username and email have not been used already
    if User.objects.filter(username=um).count() == 0 and User.objects.filter(email=em).count() == 0:

        # Store user's data into database
        user = User.objects.create_user(request.data["username"],
                                    request.data["email"],
                                    request.data["password"])
        user.is_active = False
        user.save()

        UserMeta(user=user, firstname=request.data["firstname"],
            lastname=request.data["lastname"],
            age=request.data["age"], sex=request.data["sex"]).save()

        current_site = get_current_site(request)

        mail_subject = 'Activate your Thrive account'

        # Confirmation message
        # message = render_to_string('email_template.html', {
        #                 'user': user,
        #                 'domain': current_site.domain,
        #                 'uid': urlsafe_base64_encode(force_bytes(user.pk)),
        #                 'token': account_activation_token.make_token(user),
        #             })

        # Quick fix to skip confirmation email
        activate(
            None,
            urlsafe_base64_encode(force_bytes(user.pk)),
            account_activation_token.make_token(user)
        )

        # The new user's email
        to_email = em

        # Send confirmation email to user. Does not check whether the email was
        # sent or not. Note that it does not pass an authorization username nor
        # a password to actually authorize the sending of the confirmation email
        #
        # send_mail(mail_subject, message, 'bell woods email @ ...', [to_email])
        return Response({"response" : 'Please confirm your email address to complete the registration'})
    else:

        # This user already exists, response should be more specific
        return Response({"response" : 'Unsuccessful'})

# Confirm user on data base, a user cannot login without this
def activate(request, uidb64, token):
    User = get_user_model()
    try:
        uid = force_text(urlsafe_base64_decode(uidb64))
        user = User.objects.get(pk=uid)
    except(TypeError, ValueError, OverflowError, User.DoesNotExist):
        user = None

    # Checks that activation token
    if user is not None and account_activation_token.check_token(user, token):
        user.is_active = True
        user.save()
        return render(request, 'confirmed.html')
    else:
        return render(request, 'timout.html')

@api_view(['GET'])
@permission_classes((IsAuthenticated,))
def get_preferred_notification_time(request):
    return Response(
        {"time": UserMeta.objects.get(user=request.user).pref_alert_time})

@api_view(['POST'])
@permission_classes([])
def send_buddy_request(request):
    f = request.data["from"]
    t = request.data["to"]
    fu = User.objects.get(username=f)
    for friend in fu.buddy.all():
        if friend.username == t:
            return Response({"response" : 'Already buddy!'})
    if User.objects.filter(username=t).count() > 0:
        tu = User.objects.get(username=t)
        if tu != fu:
            if Buddy_Request.objects.filter(from_user=fu, to_user=tu):
                return Response({"response" : 'Buddy request already sent!'})

            elif Buddy_Request.objects.filter(from_user=tu, to_user=fu):
                buddy_request = Buddy_Request.objects.get(from_user=tu, to_user=fu)
                fu.buddy.add(tu)
                tu.buddy.add(fu)
                buddy_request.delete()
                return Response({"response" : 'Buddies!'})

            else:
                buddy_request = Buddy_Request(from_user=fu, to_user=tu)
                buddy_request.save()
                if tu.expo_token != "" and tu.expo_token is not None:
                    send_push_message(tu.expo_token, "You got a buddy request from " + fu.username + "!")
                return Response({"response" : 'Buddy request sent!'})
        else:
            return Response({"response" : 'Same User'})
    else:
        return Response({"response" : 'No such user'})

@api_view(['POST'])
@permission_classes([])
def accept_buddy_request(request):
    f = request.data["from"]
    t = request.data["to"]
    fu = User.objects.get(username=f)
    tu = User.objects.get(username=t)
    buddy_request = Buddy_Request.objects.get(from_user=fu, to_user=tu)
    fu.buddy.add(tu)
    tu.buddy.add(fu)
    buddy_request.delete()
    if Buddy_Request.objects.filter(from_user=tu, to_user=fu):
        buddy_request = Buddy_Request.objects.get(from_user=tu, to_user=fu)
        buddy_request.delete()
    return Response({"response" : 'Buddy Request Accepted!'})

@api_view(['POST'])
@permission_classes([])
def reject_buddy_request(request):
    f = request.data["from"]
    t = request.data["to"]
    fu = User.objects.get(username=f)
    tu = User.objects.get(username=t)
    buddy_request = Buddy_Request.objects.get(from_user=fu, to_user=tu)
    buddy_request.delete()
    return Response({"response" : 'Buddy Request Rejected!'})

@api_view(['POST'])
@permission_classes([])
def fetch_buddy_request(request):
    u = request.data["user"]
    user = User.objects.get(username=u)

    brs = Buddy_Request.objects.filter(to_user=user)
    buddy_requests = []

    for br in brs:
        buddy_requests.append(br.from_user.username)

    return Response({"requests": buddy_requests})

@api_view(['POST'])
@permission_classes([])
def fetch_buddy(request):
    username = request.data["user"]

    user = User.objects.get(username=username)
    possible = user.buddy.all()
    buddies = []
    for bud in possible:
        buddies.append(bud.username)

    return Response({"requests": buddies})

@api_view(['POST'])
@permission_classes((IsAuthenticated,))
def set_preferred_notification_time(request):
    meta = UserMeta.objects.get(user=request.user)
    meta.pref_alert_time = datetime.strptime(request.data["time"],
                                             "%H:%M:%S").time()
    meta.save()
    return Response({"time": meta.pref_alert_time})

@api_view(['GET'])
@permission_classes((IsAuthenticated,))
def get_user_meta(request):
    meta = UserMeta.objects.get(user=request.user)

    serializer = UserMetaSerializer(instance=meta)
    return Response(serializer.data)

# @api_view(['GET'])
# @permission_classes((IsAuthenticated,))
# def set_user_meta(request):
#     meta = UserMeta.objects.get(user=request.user)
#
#     serializer = UserMetaSerializer(instance=meta)
#     return Response(serializer.data)

@api_view(['GET'])
@permission_classes((IsAuthenticated,))
def is_admin(request):
    # return Response({"status": request.user.is_staff})
    return Response({"status": request.user.is_staff})

@api_view(['POST', 'DELETE'])
@permission_classes((IsAuthenticated,))
def active_journeys(request, jid):
    # username = request.user.username
    user = request.user
    if request.method == "POST":
        user.active_journeys.add(Journey.objects.get(id=jid))
        return Response({"response": 'Success'})
    elif request.method == "DELETE":
        user.active_journeys.remove(Journey.objects.get(id=jid))
        return Response({"response": 'Success'})

@api_view(['GET'])
@permission_classes((IsAuthenticated,))
def get_active_journeys(request):
    user = request.user
    if request.method == "GET":
        lst = user.active_journeys.all()
        jsonObj = []
        for j in lst:
            data = JourneySerializer(instance=j).data
            quests = []
            for i in range(1, len(data["quests"])+1):
                jq = JourneyQuests.objects.get(journey=j, order=i)
                quests.append(jq.quest)
            Qserializer = QuestSerializer(quests, many=True)
            data['quests'] = Qserializer.data
            jsonObj.append(data)
        return Response(jsonObj)

@api_view(['GET'])
@permission_classes([IsAuthenticated])
def get_random_non_buddy(request):
    """Searches for a random user that is not a buddy of the requesting user
    and responds the username for that user.

    The purpose of this is to allow for the random matching of accountability
    buddies without giving any user access to all user data.
    """
    related_users = [request.user] + list(request.user.buddy.all())
    non_buddy_usernames = User.objects.exclude(username__in=related_users) \
                                      .values_list('username', flat=True)

    if not non_buddy_usernames.exists():
        # There are no users who are not buddies with this user
        return Response({"details": "No random buddy match."})

    # Take a random username from this list, as it is non-empty
    random_match = sample(list(non_buddy_usernames), 1).pop()
    return Response({"username": random_match})

@api_view(['POST'])
@permission_classes([IsAuthenticated])
def set_expo_token(request):
    user = request.user

    user.expo_token = request.data["expo_token"]
    user.save()
    response = {
            "details": "Expo token added to table!"
        }

    return Response(response, status=200)


import argparse
from pathlib import Path
import shutil

def parse_path(path_string: str) -> Path:
    path_string = Path(path_string).resolve()
    return path_string

if __name__ == '__main__':
    save_help = 'File path to store the augmented training data and the '\
                'original validation and test data'
    parser = argparse.ArgumentParser()
    parser.add_argument("augmented_data_fp", type=parse_path,
                        help='File path to the augmented training data')
    parser.add_argument("test_val_folder_fp", type=parse_path,
                        help='File path to the folder containing the validation and test data')
    parser.add_argument("save_folder", type=parse_path, help=save_help)
    args = parser.parse_args()

    save_folder = args.save_folder
    save_folder.mkdir(parents=True, exist_ok=True)
    new_train_fp = Path(save_folder, 'train.json')
    shutil.copy(args.augmented_data_fp, new_train_fp)
    val_fp = Path(args.test_val_folder_fp, 'val.json')
    new_val_fp = Path(save_folder, 'val.json')
    shutil.copy(val_fp, new_val_fp)
    test_fp = Path(args.test_val_folder_fp, 'test.json')
    new_test_fp = Path(save_folder, 'test.json')
    shutil.copy(test_fp, new_test_fp)


import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
import dash_table
import pandas as pd
import dash_bootstrap_components as dbc
from app import app
import numpy as np
import plotly.graph_objs as go
import dash_html_components as html
from dash.dependencies import Input, Output, State
import plotly.figure_factory as ff

dft = pd.read_csv('./timesData.csv')
nb_data = 50
n_comp = 8

dfs = dft[dft.year==2016].iloc[:nb_data,:]
dfs = dfs.dropna()
dfs.world_rank = [each.replace('=','') for each in dfs.world_rank]
# dfs.world_rank = pd.to_numeric(dft.world_rank, errors='coerce')
# dfs.income = pd.to_numeric(dft.income, errors='coerce')
# dfs.international = pd.to_numeric(dft.international, errors='coerce')
# dfs.total_score = pd.to_numeric(dft.total_score, errors='coerce')
# dfs.num_students = [str(each).replace(',','') for each in dft.num_students]
# dfs.international_students = [each.replace('%','') for each in dft.international_students]
# dfs.female_male_ratio = [str(each).split() for each in dft.female_male_ratio]
# dfs.female_male_ratio = [round((float(each[0]) / float(each[2])),2)*100 for each in dft.female_male_ratio]
# dfs.female_male_ratio = pd.to_numeric(dft.female_male_ratio, errors='coerce')

dfs.research = pd.to_numeric(dft.research, errors="coerce")
dfs.world_rank  = pd.to_numeric(dft.world_rank, errors="coerce")
TirParGroup = dfs.groupby('university_name')
# ParGroup = dftableau.groupby('country')
y1=TirParGroup['research'].agg(np.mean).sort_values(ascending=False)
y2=TirParGroup['world_rank'].agg(np.mean)

trace1 = {
  'x': y1.index.get_level_values(0).tolist(),

  'y': y1,
  'name': 'Research',
  'type': 'bar'
}
trace2 = {
  'x':  y1.index.get_level_values(0).tolist(),

  'y': y2,
  'name': 'World Rank',
  'type': 'bar'
}
data = [trace1, trace2]

layout_graph = {
  'xaxis': {'title': 'Université'},
  'barmode': 'relative',
  'title': 'Correlation',

}
fig = go.Figure( data = data, layout = layout_graph)
fig.update_layout(
    plot_bgcolor='#181B1E' ,
    paper_bgcolor='#181B1E',
    font_color='#DCDCDC',

)

markdown_text = '''
* Réaliser une veille sur la librairie Dash.
* Faire une analyse du jeu de données correspondant au classement des 50 meilleures universités en 2016.
* Réaliser une Analyse en Composantes Principales (vous pourrez vous appuyer sur la librairie Scikit-Learn) https://scikit-learn.org/stable/modules/generated/sklearn.decomposition.PCA.html
* Mettre en place un Dashbord Dash multi-pages permettant de répondre à la question initiale : De toutes les universités du monde, quelles sont les meilleures ? La première page de votre Dashbord mettra en évidence l'analyse des données des 50 meilleures universités de l'année 2016 (avant L'ACP).

#### 1ère page

* Dans cette première page se trouvera notamment une table des données des 50 meilleurs universités de l'année 2016 avec un bouton de téléchargement permettant de télécharger un tableau .csv des données.
* Plusieurs graphiques mettant en évidences des corrélations entre certains critères (par exemple : qualité de la recherche en fonction du rand mondial du classement des universités etc..)
#### 2eme page

Cette page permettra d'afficher les résultats issus de l'ACP. On pourra ainsi y trouver des graphiques ainsi que des paragraphes de textes mettant en évidence des variables explicatives (par exemple la valeur propre, la proportion, le cumulé, les composantes principales (CP), les scores, les distances).

Vous mettrez en ligne votre Dashboard Dash sur le serveur Cloud Heroku

'''

modal = html.Div(
    [
        dbc.Button(style={'margin-bottom':'5vh','margin-left':'2vw','backgroundColor':'#181B1E'}, children=["Détail du brief"], id="open-xl"),
        dbc.Modal(
            [
                dbc.ModalHeader(""),
                dbc.ModalBody(style={}, children=[
                    html.H5(style={'text-align': 'center',}, children=['Contexte du projet']),
                    html.H5(style={'text-align': 'center',}, children=['Classement des universités']),
                    html.P(style={}, children=[
                        "Le classement des universités est une pratique difficile, politique et controversée. Il existe des centaines de systèmes de classement universitaires nationaux et internationaux différents, dont beaucoup sont en désaccord les uns avec les autres.Le Times Higher Education World University Ranking est largement considéré comme l'une des mesures universitaires les plus influentes et les plus largement observées. Fondée au Royaume-Uni en 2010, elle a été critiquée pour sa commercialisation et pour avoir 'affaibli' les établissements non anglophones. "
                    ]),
                    html.H5(style={'text-align': 'center',}, children=['Analyse en Composantes Principales']),
                    html.P(style={}, children=[
                        " Pour vous aider dans votre analyse du jeux de données, vous réaliserez une Analyse en Composantes Principales.Cette analyse permettra de répondre à certaines questions du type : quelles ressemblances peut-il y avoir d'une université à une autre. Quelles ressemblances existent-il entre différents critères d'évaluation des universités ? Vous pourrez ainsi définir quand est-ce que 2 universités se ressemblent et quand est-ce qu'elles se ressemblent du point de vue de l'ensemble des colonnes (c'est-à-dire des critères d'évaluation du Times Higher Education World University Ranking).Est-il possible de faire un bilan des ressemblances ? ( Vous chercherez ici à faire une typologie, une partition des universités, c'est-à-dire à construire des groupes d'universités homogènes du point de vue de l'ensemble des variables. A l'intérieur d'un groupe, les individus se ressemblent et d'un groupe à l'autre ils sont différents."
                    ]),
                    html.H5(style={'text-align': 'center',}, children=['Analyse en Composantes Principales']),
                    html.P(style={}, children=[
                        dcc.Markdown(children=markdown_text),

                    ]),

                ]),
                dbc.ModalFooter(
                    dbc.Button(style={'backgroundColor':'#181B1E'},children=["Fermer"], id="close-xl", className="ml-auto")
                ),
            ],
            id="modal-xl",
            size="xl",
        ),
    ]
)

# Modal 2 Corrélation
colonnes = ['world_rank', 'total_score', 'research', 'teaching', ]
data_pca = dfs[colonnes]
lay1_fig2 = ff.create_scatterplotmatrix(data_pca, diag='histogram',
                                  colormap='Viridis',
                                  colormap_type='cat',
                                  height=500, width=1000)

nav_menu = dbc.Nav(
    [
        dbc.NavLink(style={'color':'#DCDCDC'},children=["Home "], active=True, href="/"),
        dbc.NavLink(style={'color':'#DCDCDC'},children=["Page 2"], href="/acp"),

    ]
)

layout = html.Div(style={'backgroundColor':'#212529', 'padding-top':'2vh', 'padding-botton':'4vh'}, children=[

  nav_menu,

  html.H1(style={'text-align': 'center', 'color':'#DCDCDC', 'margin-top':'2vh', 'margin-bottom':'3vh'} , children=[ "Dashboard"]),
  html.Section(style={'height':'70vh', 'display':'flex', 'justify-content':'space-around'}, children=[
      html.Article(style={ 'backgroundColor':'#181B1E', 'height':'70vh','width':'66vw', 'padding-top':'1.2vh','border-radius':'15px'}, children=[
           dcc.Graph(

        figure=fig
    ),
      ]),
      html.Article(style={'backgroundColor':'#212529', 'height':'70vh', 'width':'30vw'}, children=[
        html.Div(style={'backgroundColor':'#181B1E', 'height':'33vh', 'width':'30vw','border-radius':'15px'}, children=[]),
        html.Div(style={'backgroundColor':'#181B1E', 'margin-top':'4vh','height':'33vh', 'width':'30vw','border-radius':'15px'}, children=[])
      ]),
  ]

  ),
  html.H2(style={'text-align': 'center','color':'#DCDCDC', 'margin-top':'5vh','margin-bottom':'5vh'},children=["Les 50 meilleurs universités de l'année 2016 "]),
  html.Section(style={'display':'flex', 'margin-bottom':'5vh','justify-content':'space-around'},children=[
      html.Article(style={'backgroundColor':'#181B1E', 'height':'75vh','width':'25vw','border-radius':'15px'},children=[]),
      html.Article(style={'backgroundColor':'#181B1E','margin-left':'2vw', 'height':'75vh', 'width':'70vw','border-radius':'15px'},children=[

          dash_table.DataTable(
            id='table',
          export_format='csv',
            columns=[{"name": i, "id": i} for i in dfs.columns],
            data=dfs.to_dict('records'),
            page_size=15,
            style_table={'height': '71vh', 'overflowY': 'auto', 'width':'69vw'},

            style_header={'backgroundColor': '#151819', 'color':'white'},
            style_cell={
                'backgroundColor': '#1C1F23',
                'color': '#DCDCDC'
            },),

      ])
  ]),
  html.Br(),
  html.Br(),
  html.Section([

  ]),
  modal,

])

def toggle_modal(n1, n2, is_open):
    if n1 or n2:
        return not is_open
    return is_open

app.callback(
    Output("modal-xl", "is_open"),
    [Input("open-xl", "n_clicks"), Input("close-xl", "n_clicks")],
    [State("modal-xl", "is_open")],
)(toggle_modal)


"""model definition"""
"""trained not GAN"""

from Net.ProCCaps import CapsNet_MR
# ------------
from torch import Tensor, device
from torch.utils.data import DataLoader
from torchvision import transforms
from Zhang_github.data_imagenet import  ValImageFolder
import utils_code.util_zhang as util_zhang
from utils_code.utils import *
import os
from PIL import Image, ImageFile
import numpy as np
import shutil
from tqdm import tqdm
ImageFile.LOAD_TRUNCATED_IMAGES = True
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"]="6"

additional_appendix = "ProCCaps"
routes = (32, 9, 9)
size = 224
root = 'test_dataset'
if os.path.exists(root):
  os.mkdir(root)
class RGB2LAB(object):
    def __init__(self):
        super(RGB2LAB,self).__init__()
    def __call__(self,img):
        (tens_rs_ab, tens_rs_l, tens_rxs_ab)= util_zhang.preprocess_img(img, HW=(size,size), resample=3)
        return tens_rs_ab, tens_rs_l, tens_rxs_ab
original_transform = transforms.Compose([RGB2LAB()])

"""Dataset to use"""
# val_dir_name, results_folder = '/media/TBData2/Datasets/COCO_stuff/val', "COCO_stuff"
val_dir_name, results_folder = '/media/TBData2/Datasets/ImageNet10k/Test', "Image10k_tiny"
# val_dir_name, results_folder = '/media/TBData2/Datasets/Places205/Test', "Places205"

"""batch_size"""
batch_size = 32
"""Init and load dataset"""
ImageDataset = ValImageFolder(val_dir_name, transform=original_transform)
dataloaders = DataLoader(ImageDataset, batch_size=batch_size, shuffle=False)

"""if you want to keep original dimension (enlarge the output of the model), you need orig_dim = True"""
orig_dim = False
extention, model_folder = 'png', "PL"
n_cuda, epoch = 0, str(65)
CUDA = "cuda:"+str(n_cuda)
device = device(CUDA)

"""Init model: the SSL models are in dataParallel box, the PL models are without dataParallel"""
model = CapsNet_MR(128, num_routes=routes)
depth = 'Q'
print(f"*The depth is {depth} ! Be aware of that*")

"""Checkpoint locations"""
model_path, appendix = os.path.join('models/WACV_model/checkpoint_ProCCaps_ImageNet_65.pth.tar'), "_wacv"  #<-PL_IN*

"""Resume pretrained model"""
model, epoch_start = resume_model(model_path, model, map_location=CUDA)
model = model.to(device)

"""Create folder for the coloured images"""
if not os.path.exists(os.path.join(root,results_folder)):
    os.mkdir(os.path.join(root,results_folder))
mother_folder = os.path.join(root,results_folder,epoch+"_"+model_folder+"_"+appendix)
mother_folder_orig_clone = mother_folder + '_orig_clone'
if os.path.exists(mother_folder):
    shutil.rmtree(mother_folder, ignore_errors=True)
if os.path.exists(mother_folder_orig_clone):
    shutil.rmtree(mother_folder_orig_clone, ignore_errors=True)
os.mkdir(mother_folder)
os.mkdir(mother_folder_orig_clone)

psnr_sum, psnr_sum_self = [], []
model.train()
for batch_id, (image_batch,index, target, name_path) in enumerate(tqdm(dataloaders)):
    ##extract the greyscale image
        img_ab_rs = image_batch[0].to(device)
        img_l_rs = image_batch[1].to(device)
        img_l_rs_v = img_l_rs.to(device)
        ##infer the colourisation channels
        if model_folder == 'SSL': img_ab_pred,_ = model(img_l_rs_v)
        elif model_folder == 'PL': img_ab_pred = model(img_l_rs_v, depth=depth)
        ##print check
        if batch_id == 0: print(img_ab_pred[0,:,:,:])
        ##for loop for image recomposition
        for j in range(len(img_l_rs_v)):
            if orig_dim:
                tens_orig = util_zhang.original_l(ImageDataset, index[j])
                L = tens_orig.unsqueeze(0).to(device)
                HW = tens_orig.shape[2:]
            else:
                L =  img_l_rs_v
                HW = img_ab_pred.shape[2:]

            img_rgb = util_zhang.postprocess_tens(L, img_ab_pred, j, mode='bilinear',HW_request=HW) #HW_request=img_ab_pred.shape[2:])#
            img_rgb = (img_rgb*255).astype(np.uint8)
            img_rgb_orig = util_zhang.postprocess_tens(L, img_ab_rs, j, mode='bilinear',HW_request=HW) #HW_request=img_ab_pred.shape[2:])#HW)
            img_rgb_orig = (img_rgb_orig * 255).astype(np.uint8)
            ##PSNR
            psnr_sum.append(psnr(img_rgb_orig, img_rgb))
            #Save output
            pil_image = Image.fromarray(img_rgb, 'RGB')
            if isinstance(target[j],Tensor):
                target_j = str(target[j].item())
            else:
                target_j = target[j]

            dstfolder = os.path.join(mother_folder, target_j)
            if not os.path.exists(dstfolder):
                os.mkdir(dstfolder)
                os.mkdir(os.path.join(mother_folder_orig_clone, target_j))
            pil_image.save(os.path.join(dstfolder, name_path[j].replace(extention,'png')))
            Image.fromarray(img_rgb_orig, 'RGB').save(os.path.join(mother_folder_orig_clone,
                                                                   target_j,
                                                                  name_path[j].replace(extention,'png')))
            del pil_image,img_rgb_orig,img_rgb,dstfolder#, tens_orig
        del image_batch,target,name_path
print(np.mean(psnr_sum))