import numpy as np
import getopt, sys
import pdb
import random
import math
import matplotlib.pyplot as plt
import matplotlib.lines as lines

verbose  = False
debug   = False
data = []

def read_data(path):
    data = []
    with open(path) as tsp:
        for line in tsp:
            token = line.split()
            d = {'n': int(token[0]), 'x': float(token[1]), 'y': float(token[2])}
            data.append(d)
    return data

def draw(nodes):
    fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(10, 10))
    axes.set_title('Nearest Neighbor')
    axes.set_xlabel('X')
    axes.set_ylabel('Y')
    for i in range(len(nodes)):
        node = data[nodes[i]]
        axes.scatter(node['x'], node['y'], color='black', s=100)
    for i in range(nodes.__len__() - 1):
        node1 = data[nodes[i]]
        node2 = data[nodes[i + 1]]
        axes.add_line(
            lines.Line2D([node1['x'], node2['x']],
                         [node1['y'], node2['y']]))
        axes.text((node1['x'] + node2['x']) / 2,
                  (node1['y'] + node2['y']) / 2,
                  i + 1, fontsize=15)
    plt.show()

def rank_selection(items, probs, n):
    if n == 0:
        return np.array([])

    arg_sort = np.argsort(probs)
    sorted_items = []
    for index in arg_sort:
        sorted_items.append(items[index])
    N = len(items)
    rank = np.arange(start=1, stop=N + 1)
    prob_new = 2 * rank / (N * (N + 1))

    rnds = np.random.random(size=n)
    inds = np.zeros(n, dtype=np.int)
    cum_sum = np.cumsum(prob_new)
    for i, rnd in enumerate(rnds):
        inds[i] = np.argmax(cum_sum >= rnd)
    return np.array(sorted_items)[inds]

def chromosome_generator(genes_num, chromosome_num):
    chromosomes = []
    for i in range(chromosome_num):
        c = np.random.permutation(194).tolist()
        chromosomes.append(c)
    return chromosomes

def evaluator(chromosome):
    global data
    distance = 0
    for i in range(len(chromosome)):
        num1        = chromosome[i]
        num2        = chromosome[(i + 1) % len(chromosome)]
        a1          = data[num1]
        a2          = data[num2]
        dx          = a1['x'] - a2['x']
        dy          = a1['y'] - a2['y']
        distance    = distance + math.sqrt(dx * dx + dy * dy)
    return distance

def roulette_wheel_selection(items, probs, n):
    rnds = np.random.random(size=n)
    inds = np.zeros(n, dtype=np.int)
    cum_sum = np.cumsum(probs)
    for i, rnd in enumerate(rnds):
        inds[i] = np.argmax(cum_sum >= rnd)
    return items[inds]

def cross_over(parent1, parent2, parameters=None):
    parent1 = parent1
    parent2 = parent2
    n = len(parent1)
    start_substr = random.randint(0, n - 2)
    end_substr = random.randint(start_substr + 1, n - 1)
    child1 = parent1.copy()
    child2 = parent2.copy()
    j = end_substr
    i = end_substr
    while j != start_substr:
        if i == 194:
            print('end', end_substr)
            print('here')
        if not parent1[i] in child2[start_substr:end_substr]:
            child2[j] = parent1[i]
            j = (j + 1) % n
        i = (i + 1) % n
        if i == 194:
            print('here')
    j = end_substr
    i = end_substr
    while j != start_substr:
        if not parent2[i] in child1[start_substr:end_substr]:
            child1[j] = parent2[i]
            j = (j + 1) % n
        i = (i + 1) % n
    return child1, child2

def mutation(chro, parameters={'prob': 0.05}):
    for i in range(len(chro)):
        if np.random.random() <= parameters['prob']:
            swap_idx = np.random.randint(0, len(chro))
            chro[i], chro[swap_idx] = chro[swap_idx], chro[i]
        return chro

def reverse_mutation(chromosome, parameters={'prob': 0.15}):
    if np.random.random() <= parameters['prob']:
        chr_length = len(chromosome)

        # Two random points is selected to change values of interval
        ind_1 = np.random.randint(chr_length - 1)
        ind_2 = np.random.randint(ind_1, chr_length)

        while ind_1 < ind_2:
            chromosome[ind_1], chromosome[ind_2] = chromosome[ind_2], chromosome[ind_1]
            ind_1 += 1
            ind_2 -= 1
    return chromosome

def q_tournament_selection(items, probs, q, n):

    if n == 0:
        return np.array([])

    else:
        items, probs = warning_data_type_check_selection_algorithms(items, probs)
        index = np.arange(len(items))
        np.random.shuffle(index)
        items = items[index]
        probs = probs[index]

        selected_items = []
        len_items = len(items)

        for i in range(n):
            indexes = np.random.choice(np.arange(len_items), q, replace=False)
            selected_items.append(items[indexes[np.argmax(probs[indexes])]])
    return np.array(selected_items)

def generate_child(parents):
    children = []
    random.shuffle(parents)
    for i in range(0, len(parents) - 1, 2):
        chromosome1, chromosome2 = cross_over(parents[i], parents[i + 1])
        chromosome1 = reverse_mutation(chromosome1)
        chromosome2 = reverse_mutation(chromosome2)
        children += [chromosome1.tolist(), chromosome2.tolist()]
    return children

def population_selection(population, probs):
    pairs = zip(probs, population)
    s_pairs = sorted(pairs)
    tuples = zip(*s_pairs)
    probs, population = [list(t) for t in tuples]
    return np.asarray(population[10:30]), np.matrix(probs[10:30]).T

def main():
    global verbose, debug, data
    opts, args = getopt.getopt(sys.argv[1:], "vd")
    for o, a in opts:
        if o == "-v":
            verbose = True
        elif o == "-d":
            debug = True
    data = read_data('tsp_data.txt')
    if verbose:
        print('data is')
        print(data)
        print("================")
        input()
    population = chromosome_generator(194, 20)
    if verbose:
        print('chromosomes are')
        print(population)
        print("================")
        input()
    probs = []
    for chromosome in population:
        #draw(chromosome)
        prob = evaluator(chromosome)
        probs.append(prob)
        #print(prob)
    fitness = 1 / np.asarray(probs)
    sum = np.sum(fitness)
    print('sum', sum)
    probs = fitness / sum
    probs = probs.tolist()
    print(probs)
    if verbose:
        print("prob is")
        print(probs)
        print("================")
    population = np.asarray(population)
    distance = []
    for i in range(5000):
        print(i)
        parents = roulette_wheel_selection(population, probs, 10)
        if verbose:
            print('selected parents are')
            print(parents)
            print("================")
            input()
        children = generate_child(parents)
        if verbose:
            print('children are')
            print(children)
            print(len(children))
            print("================")
            input()
        population = population.tolist()
        for child in children:
            population.append(child)
        probs = []
        for chromosome in population:
            prob = evaluator(chromosome)
            probs.append(prob)
        distance = probs.copy()
        fitness = 1 / np.asarray(probs)
        sum = np.sum(fitness)
        probs = fitness / sum
        probs = probs.tolist()
        if debug:
            pdb.set_trace()
        population, probs = population_selection(population, probs)
        if verbose:
            print('new populatin is:')
            print(population)
            print("================")
            print(len(population))
            input()
        if verbose:
            print('prob is')
            print(probs)
            print("================")
            print(len(probs))
            input()
    print('min distance found is')
    print(np.min(distance))

if __name__ == "__main__":
    main()


from django.http import JsonResponse

from ..Parsers import Parsers
from ..models import Caso, Paciente, FormaContagio, Sintoma
from datetime import datetime
import json

class CasoController:
    p = Parsers()
    def listar(self):
        casos = Caso.objects.all()
        casos_dict = self.p.casosToDict(casos)
        return JsonResponse(casos_dict, safe=False, json_dumps_params={'ensure_ascii': False})

    def cadastrar(self, request):
        data = json.loads(request.body)
        paciente = Paciente.objects.get(id=data['paciente']['id'])
        formaContagio = FormaContagio.objects.get(id=data['formaContagio']['id'])
        dataInicioSintomas = datetime.strptime(data['dataInicioSintomas'], '%Y-%m-%d')
        dataRelato = datetime.now()
        caso = Caso(paciente=paciente, formaContagio=formaContagio, dataInicioSintomas=dataInicioSintomas,
                    dataRelato=dataRelato)
        caso.save()
        caso.sintomas.set([Sintoma.objects.get(id=sintoma['id']) for sintoma in data['sintomas']])
        return JsonResponse(self.p.casoToDict(caso))

    def listarPorId(self, id):
        caso = self.p.casoToDict(Caso.objects.get(id=id))
        return JsonResponse(caso, json_dumps_params={'ensure_ascii': False})

    def atualizar(self, request, id):
        data = json.loads(request.body)
        caso = Caso.objects.get(id=id)
        caso.paciente = Paciente.objects.get(id=data['paciente']['id'])
        caso.formaContagio = FormaContagio.objects.get(id=data['formaContagio']['id'])
        caso.dataInicioSintomas = datetime.strptime(data['dataInicioSintomas'], '%Y-%m-%d')
        try:
            caso.dataFimSintomas = datetime.strptime(data['dataFimSintomas'], '%Y-%m-%d')
        except:
            pass
        caso.sintomas.set([Sintoma.objects.get(id=id) for id in data['sintomas']])
        caso.save()
        return JsonResponse(self.p.casoToDict(caso))

    def deletar(self, id):
        caso = Caso.objects.get(id=id)
        caso_dict = self.p.casoToDict(caso)
        caso.delete()
        return JsonResponse(caso_dict)

    def listarPorBairro(self, bairro_id):
        casos = self.p.casosToDict(Caso.objects.filter(paciente__bairro__id=bairro_id))
        return JsonResponse(casos, safe=False, json_dumps_params={'ensure_ascii': False})

    def listarPorLocal(self, local_id):
        locais = self.p.casosToDict(Caso.objects.filter(paciente__locaisVisitados__id=local_id))
        return JsonResponse(locais, safe=False, json_dumps_params={'ensure_ascii': False})


import os
import pyudev
import weakref

from enum import Enum
from struct import Struct

__all__ = ["Keyboard", "Mouse"]

class HidProtocol(Enum):
    KEYBOARD = 1
    MOUSE = 2

class Device():
    def __init__(self, gadget, protocol):
        self.gadget = gadget
        self.function = gadget.create_function(protocol, self.packet.size,
                                               self.HID_DESCRIPTOR)
        self.char_dev = None

    class GadgetUnboundError(Exception):
        def __init__(self):
            super().__init__("The UsbGadget for this device is not bound to anything")

    def _io_func(func):
        def func_wrapper(self, *args, **kwargs):
            if self.char_dev == None or self.char_dev.closed:
                try:
                    self.char_dev = self.gadget.find_hidg_device(
                        self.function)
                except pyudev.DeviceNotFoundByNumberError:
                    raise Device.GadgetUnboundError()

            return func(self, *args, **kwargs)
        return func_wrapper

class Keyboard(Device):
    HID_DESCRIPTOR = bytes([
        0x05, 0x01, # USAGE_PAGE (Generic Desktop)
        0x09, 0x06, # USAGE (Keyboard)
        0xa1, 0x01, # COLLECTION (Application)
        0x05, 0x07, #   USAGE_PAGE (Keyboard)
        0x19, 0xe0, #   USAGE_MINIMUM (KB Leftcontrol)
        0x29, 0xe7, #   USAGE_MAXIMUM (KB Right GUI)
        0x15, 0x00, #   LOGICAL_MINIMUM (0)
        0x25, 0x01, #   LOGICAL_MAXIMUM (1)
        0x75, 0x01, #   REPORT_SIZE (1)
        0x95, 0x08, #   REPORT_COUNT (8)
        0x81, 0x02, #   INPUT (Var)
        0x95, 0x01, #   REPORT_COUNT (1)
        0x75, 0x08, #   REPORT_SIZE (8)
        0x81, 0x03, #   INPUT (Cnst, Var)
        0x95, 0x05, #   REPORT_COUNT (5)
        0x75, 0x01, #   REPORT_SIZE (1)
        0x05, 0x08, #   USAGE_PAGE (LED)
        0x19, 0x01, #   USAGE_MINIMUM (1)
        0x29, 0x05, #   USAGE_MAXIMUM (5)
        0x91, 0x02, #   OUTPUT (Var)
        0x95, 0x01, #   REPORT_COUNT (1)
        0x75, 0x03, #   REPORT_SIZE (3)
        0x91, 0x03, #   OUTPUT(Cnst, Var)
        0x95, 0x06, #   REPORT_COUNT (6)
        0x75, 0x08, #   REPORT_SIZE (8)
        0x15, 0x00, #   LOGICAL_MINIMUM (0)
        0x25, 0x65, #   LOGICAL_MAXIMUM (101)
        0x05, 0x07, #   USAGE_PAGE (Keyboard)
        0x19, 0x00, #   USAGE_MINIMUM (None)
        0x29, 0x65, #   USAGE_MAXIMUM (KB Application)
        0x81, 0x00, #   INPUT ()
        0xc0        # END_COLLECTION
    ])

    class Modifier(Enum):
        LEFT_CTRL   = 0x01
        LEFT_SHIFT  = 0x02
        LEFT_ALT    = 0x04
        LEFT_META   = 0x08
        RIGHT_CTRL  = 0x10
        RIGHT_SHIFT = 0x20
        RIGHT_ALT   = 0x40
        RIGHT_META  = 0x80

    MODIFIER_MASK = 0xFF
    packet = Struct('cx6s')

    def __init__(self, gadget):
        super().__init__(gadget, HidProtocol.KEYBOARD.value)
        self.__pressed_keys = []
        self.__modifier_mask = 0

    @property
    def pressed_keys(self):
        return self.__pressed_keys

    @property
    def modifier_mask(self):
        return self.__modifier_mask

    @Device._io_func
    def set_pressed(self, modifier_mask=0, keys=[]):
        assert not modifier_mask & ~self.MODIFIER_MASK
        assert len(keys) <= 6

        self.char_dev.write(self.packet.pack(bytes([modifier_mask]),
                                             bytes(keys)))
        self.__pressed_keys = keys
        self.__modifier_mask = modifier_mask

class Mouse(Device):
    HID_DESCRIPTOR = bytes([
        0x05, 0x01, # USAGE_PAGE (Generic Desktop)
        0x09, 0x02, # USAGE (Mouse)
        0xa1, 0x01, # COLLECTION (Application)
        0x09, 0x01, #   USAGE (Pointer)
        0xa1, 0x00, #   COLLECTION (Physical)
        0x05, 0x09, #     USAGE_PAGE (Button)
        0x19, 0x01, #     USAGE_MINIMUM (Button 1)
        0x29, 0x03, #     USAGE_MAXIMUM (Button 3)
        0x15, 0x00, #     LOGICAL_MINIMUM (0)
        0x25, 0x01, #     LOGICAL_MAXIMUM (1)
        0x95, 0x03, #     REPORT_COUNT (3)
        0x75, 0x01, #     REPORT_SIZE (1)
        0x81, 0x02, #     INPUT (Data,Var,Abs)
        0x95, 0x01, #     REPORT_COUNT (1)
        0x75, 0x05, #     REPORT_SIZE (5)
        0x81, 0x03, #     INPUT (Cnst,Var,Abs)
        0x05, 0x01, #     USAGE_PAGE (Generic Desktop)
        0x09, 0x30, #     USAGE (X)
        0x09, 0x31, #     USAGE (Y)
        0x15, 0x81, #     LOGICAL_MINIMUM (-127)
        0x25, 0x7f, #     LOGICAL_MAXIMUM (127)
        0x75, 0x08, #     REPORT_SIZE (8)
        0x95, 0x02, #     REPORT_COUNT (2)
        0x81, 0x06, #     INPUT (Data,Var,Rel)
        0xc0,       #   END_COLLECTION
        0xc0        # END_COLLECTION
    ])

    """
    Layout:
      Button bitmask (1 byte)
      X-translation (1 byte)
      Y-translation (1 byte)
    """
    packet = Struct('ccc')

    def Button(Enum):
        LEFT   = (1 << 0)
        RIGHT  = (1 << 1)
        MIDDLE = (1 << 2)

    BUTTON_MASK = 0x7

    def __init__(self, gadget):
        super().__init__(gadget, HidProtocol.MOUSE.value)
        self.__btn_mask = 0

    @property
    def btn_mask(self):
        return self.__btn_mask

    @Device._io_func
    def set_pressed(self, btn_mask=0):
        assert not btn_mask & ~self.BUTTON_MASK

        self.char_dev.write(self.packet.pack(bytes([btn_mask]),
                                             bytes([0]), bytes([0])))
        self.__btn_mask = btn_mask

    @Device._io_func
    def move(self, x, y):
        self.char_dev.write(self.packet.pack(bytes([self.__btn_mask]),
                                             bytes([x]), bytes([y])))


import argparse
import re
import os
import json
import numpy as np
import pickle as pkl

import torchtext
from torchtext.vocab import GloVe

"""
For getting word embeddings from glove
Cite: https://github.com/ruotianluo/zsl-gcn-pth/blob/c65888fa98df0e5148806d8493b1581f74df4294/src/tools/obtain_word_embedding.py
"""

url = {'glove': 'http://nlp.stanford.edu/data/glove.6B.zip'}

data_dir = 'data/'
feat_len = 300

def embed_text_file(text_file, word_vectors, get_vector, save_file):
    with open(text_file) as fp:
        text_list = json.load(fp)

    all_feats = []

    has = 0
    cnt_missed = 0
    missed_list = []
    for i in range(len(text_list)):
        class_name = text_list[i].lower()
        if i % 500 == 0:
            print('%d / %d : %s' % (i, len(text_list), class_name))
        feat = np.zeros(feat_len)

        options = class_name.split(',')
        cnt_word = 0
        for j in range(len(options)):
            now_feat = get_embedding(options[j].strip(), word_vectors, get_vector)
            if np.abs(now_feat.sum()) > 0:
                cnt_word += 1
                feat += now_feat
        if cnt_word > 0:
            feat = feat / cnt_word

        if np.abs(feat.sum()) == 0:
            print('cannot find word ' + class_name)
            cnt_missed = cnt_missed + 1
            missed_list.append(class_name)
        else:
            has += 1
            # feat = feat / (np.linalg.norm(feat) + 1e-6)

        all_feats.append(feat)

    all_feats = np.array(all_feats)

    for each in missed_list:
        print(each)
    print('does not have semantic embedding: ', cnt_missed, 'has: ', has)

    if not os.path.exists(os.path.dirname(save_file)):
        os.makedirs(os.path.dirname(save_file))
        print('## Make Directory: %s' % save_file)
    with open(save_file, 'wb') as fp:
        pkl.dump(all_feats, fp)
    print('save to : %s' % save_file)

def get_embedding(entity_str, word_vectors, get_vector):
    try:
        feat = get_vector(word_vectors, entity_str)
        return feat
    except:
        feat = np.zeros(feat_len)

    str_set = list(filter(None, re.split("[ \-_]+", entity_str)))

    cnt_word = 0
    for i in range(len(str_set)):
        temp_str = str_set[i]
        try:
            now_feat = get_vector(word_vectors, temp_str)
            feat = feat + now_feat
            cnt_word = cnt_word + 1
        except:
            continue

    if cnt_word > 0:
        feat = feat / cnt_word
    return feat

def get_glove_dict(txt_dir):
    print('load glove word embedding')
    txt_file = os.path.join(txt_dir, 'glove.6B.300d.txt')
    word_dict = {}
    with open(txt_file) as fp:
        for line in fp:
            feat = np.zeros(feat_len)
            words = line.split()
            assert len(words) - 1 == feat_len
            for i in range(feat_len):
                feat[i] = float(words[i+1])
            word_dict[words[0]] = feat
    print('loaded to dict!')
    return word_dict

def glove_google(word_vectors, word):
    return word_vectors[word]

def get_vector(word_vectors, word):
    if word in word_vectors.stoi:
        return word_vectors[word].numpy()
    else:
        raise NotImplementedError

if __name__ == '__main__':
    text_file = 'data/graph_struct/labels.json'

    model_path = ''#add model path here later
    save_file = os.path.join(data_dir, 'word_embedding_model', 'labels.pkl')
    if not os.path.exists(save_file):
        word_vectors = GloVe('6B')
        # get_vector = glove_google

    if not os.path.exists(save_file):
        print('obtain semantic word embedding', save_file)
        embed_text_file(text_file, word_vectors, get_vector, save_file)
    else:
        print('Embedding existed :', save_file, 'Skip!!!')


# -*- coding:utf-8 -*-
from flask import Flask,request
from tools.log import Logger
from algrithm import get_curve_points,get_multi_curve_points
from werkzeug import secure_filename
import json
log = Logger('./logs/server.log',level='info')

app = Flask(__name__)
app.config.from_object('setting')

@app.route('/get_curve_points',methods=['POST','GET'])
def curve_gcn():
	return curve_point(request)

def curve_point(request):
	if request.method == 'POST':
		log.logger.info('get single region post info')
		data = str(request.data, encoding='utf-8')
		json_data = json.loads(data)
		log.logger.debug(json_data)
		poly = get_curve_points(json_data['img_path'],json_data['region'],int(app.config['CONTEXT_SCALE']))
		result = {
			'poly':poly.tolist() #numpy array can't convert to json
		}
		log.logger.debug(result)
		return json.dumps(result,indent=4,ensure_ascii=False)
	return 'index html'

@app.route('/multi_regions',methods=['POST','GET'])
def multi_regions():
	return multi_curve_points()

def multi_curve_points():
	if request.method == 'POST':
		log.logger.info('get multiple regions post info')
		data = str(request.data, encoding='utf-8')
		json_data = json.loads(data)
		log.logger.debug(json_data[0]['img_path'])
		result = get_multi_curve_points(json_data,app.config)
		# result = {
		# 	'poly':polys.tolist() #numpy array can't convert to json
		# }
		# log.logger.debug(result)
		# return json.dumps(json_data,indent=4,ensure_ascii=False)
		return json.dumps(result,indent=4,ensure_ascii=False)
	return 'multi... html'

if __name__ == '__main__':
	app.run(host='0.0.0.0', debug=False, port=5101 )


import datetime
import mimetypes
import os
import os.path

from .common import get_content_hash
from .content_handlers import handler_factory

mimetypes.init()

class PercsDocument(object):

    def __init__(self, collection, person, filepath):
        self.collection = collection
        self.person = person
        self._fullpath = filepath
        self.filedir, self.filename = os.path.split(filepath)
        self.date = datetime.datetime.utcnow()
        self.content_type = unicode(mimetypes.guess_type(filepath)[0])
        self.content_length = os.stat(filepath).st_size
        self.load_content()

    def load_content(self):
        if hasattr(self, "_content"):
            return self._content

        handler = handler_factory(self.content_type)
        self._content = handler.get_content(self._fullpath)
        self._page_content = handler.split_pages(self._content)

    @property
    def id(self):
        return self.file_hash

    @property
    def content(self):
        return self._content

    @property
    def file_hash(self):
        result = getattr(self, '_file_hash', None)
        if not result:
            result = get_content_hash(open(self._fullpath, 'rb').read())
            self._file_hash = result
        return result

    def __getitem__(self, name):
        return getattr(self, name, None)

    def get_pages(self):
        if self._page_content is None:
            raise StopIteration
        for page_num, page_content in enumerate(self._page_content, 1):
            yield PercsPage(self, page_num, page_content)

class PercsPage(object):
    """ Provides a PercsDocument interface for one of the given document's
    pages.
    """
    def __init__(self, document, page_num, content):
        self._doc = document
        self.page = page_num
        self.content = content
        self.id = u"{0}-{1}".format(
            self._doc.file_hash,
            self.page
        )
        self.page_hash = get_content_hash(self.content)

    def __getitem__(self, name):
        if hasattr(self, name):
            return getattr(self, name)
        return self._doc[name]


import numpy as np

import torch
import torch.nn.functional as F
from torch.autograd import Variable, grad

from foolbox.adversarial import Adversarial

def cw_loss(logits, y):
    one_hot = torch.zeros_like(logits)
    one_hot.scatter_(1, y.view(-1, 1), 1)
    correct_logit = (logits * one_hot).sum(1)
    worst_wrong_logit = logits[one_hot == 0].view(one_hot.size(0), -1).max(1)[0]

    adv_margin = correct_logit - worst_wrong_logit
    return -F.relu(adv_margin + 50).mean()

def tensor_clamp(x, min_x, max_x):
    x[x < min_x] = min_x[x < min_x]
    x[x > max_x] = max_x[x > max_x]
    return x

def manual_fgs(model, x, y, eps=0.1, clamp=True):
    model.zero_grad()
    x.requires_grad = True
    pred = model(x)
    loss = cw_loss(pred, y)

    g = torch.sign(grad(loss, x)[0])
    adv_x = x + g * eps
    if clamp:
        adv_x.clamp_(0, 1)
    return adv_x, model(adv_x).argmax(1), pred.argmax(1)

def manual_pgd(model, x, y, stepsize=0.01, eps=0.1, iters=100, rand_start=True, clamp=True):
    model.zero_grad()
    x_adv = torch.zeros_like(x)
    x_adv.copy_(x)

    x_min = (x.detach() - eps)
    x_max = (x.detach() + eps)
    if clamp:
        x_min.clamp_(0, 1)
        x_max.clamp_(0, 1)

    if rand_start:
        rand = torch.zeros_like(x)
        rand.uniform_(-eps, eps)
        x_adv = x_adv + rand
    x_adv.requires_grad = True
    for i in range(iters):
        model.zero_grad()
        if x_adv.grad:
            x_adv.grad.zero_()
        pred = model(x_adv)
        # loss = F.cross_entropy(pred, y)
        loss = cw_loss(pred, y)
        g = torch.sign(grad(loss, x_adv)[0])
        x_adv = x_adv + g * stepsize
        x_adv = tensor_clamp(x_adv, x_min, x_max)
    return x_adv, model(x_adv).argmax(1), model(x).argmax(1)

def perform_attack(attack, model, input_adv, cuda=True, **attack_kwargs):
    adversarial_np = attack(input_adv, **attack_kwargs)
    image = torch.Tensor(input_adv.original_image)
    label = input_adv.original_class

    if adversarial_np is None:
        # Attack failed.
        return adversarial_np, 0.0, 0.0, 0.0
    else:
        # Check if attack was successful.
        adversarial = Variable(torch.Tensor(adversarial_np))
        if cuda:
            adversarial = adversarial.cuda()
            image = image.cuda()

        pred = lambda x: F.softmax(model.forward(x.unsqueeze(0)), dim=1).max(1)[1].data[0]

        # Compute adversarial MSE.
        adv_mse = torch.pow(adversarial - Variable(image), 2).mean().item()
        adv_inf = torch.max(torch.abs(adversarial - image))
        adv_inf = adv_inf.item()

        if pred(adversarial).item() != label and adv_inf > 0.0:
            success = 1.0
        else:
            success = 0.0

        return adversarial_np, success, adv_mse, adv_inf

def batch_attack(attack, model, criterion, x, y, attack_config={}, distance=None):
    adv_ex = []
    adv_targets = []
    ret_success = []
    ret_adv_mse = []
    ret_adv_inf = []

    for i in range(x.shape[0]):
        is_cuda = x.is_cuda
        input_adv = Adversarial(model, criterion, x[i].cpu().numpy(), y[i], distance=distance)
        adv, success, adv_mse, adv_inf = perform_attack(attack, model._model, input_adv, cuda=is_cuda, **attack_config)
        if adv is not None:
            adv_targets.append(y[i])
            adv_ex.append(adv)
        ret_success.append(success)
        ret_adv_mse.append(adv_mse)
        ret_adv_inf.append(adv_inf)
    return torch.Tensor(np.array(adv_ex)), torch.LongTensor(np.array(adv_targets)), torch.Tensor(
        ret_success), torch.Tensor(ret_adv_mse), torch.Tensor(ret_adv_inf)


# Typically run using
# python launch.py -c configs_clevr/019_NIPS18/008_I3D_NL_localize_imagenetPretrained_32f_8SR.yaml -t vis  # noQA

from __future__ import division
from __future__ import absolute_import
from __future__ import unicode_literals
from __future__ import print_function

import os.path as osp
import matplotlib; matplotlib.use('Agg')  # noQA
from mpl_toolkits.mplot3d import Axes3D  # noQA  # Needed for 3D projections
import numpy as np
import logging
import argparse
import sys
import cPickle as pkl
import cv2
import json
import subprocess
from moviepy.editor import (
    VideoFileClip, clips_array, CompositeVideoClip, TextClip,
    ImageSequenceClip, concatenate_videoclips, ImageClip)
import seaborn as sns
import matplotlib.pyplot as plt
from tqdm import tqdm

from utils.eval import evaluate_result, get_labels_and_prob
from core.config import config as cfg
from core.config import (
    cfg_from_file, cfg_from_list, assert_and_infer_cfg, print_cfg)

FORMAT = '[%(levelname)s: %(filename)s: %(lineno)4d]: %(message)s'
logging.basicConfig(level=logging.INFO, format=FORMAT, stream=sys.stdout)
logger = logging.getLogger(__name__)
sns.set_style("whitegrid")

GRID_SIZE = 3
OUT_WD = 320
OUT_HT = 240
MOVIEPY_FONT = 'Utopia'
PLT_EXT = '.pdf'

# For dark mode
# PLT_EXT = '.svg'
# plt.style.use('dark_background')

def chunkify(lst, n):
    return [lst[i::n] for i in xrange(n)]

def flatten_list(l):
    return [item for sublist in l for item in sublist]

def add_snitch(frame, loc, color=(255, 215, 0)):
    cv2.circle(frame,
               (int((loc[0] + GRID_SIZE) * OUT_WD / (2 * GRID_SIZE)),
                int((GRID_SIZE - loc[1]) * OUT_HT / (2 * GRID_SIZE))),
               5, color, 3)

def mkdir_p(dpath):
    subprocess.call('mkdir -p {}'.format(dpath), shell=True)

def location_predictions_to_map(prob, video_meta):
    """ Probability values for each location. """
    # This is how the matrix classes are defined finally
    # 30, 31, 32, 33, 34, 35
    # ...
    # 0, 1, 2, 3, 4, 5
    prob = np.flipud(prob.reshape((2 * GRID_SIZE, -1, 1)) * 256.0).astype(
        np.uint8)
    # Lighter version of the prob map
    prob_cmap = 0.7 * cv2.applyColorMap(prob, cv2.COLORMAP_BONE)
    prob_cmap = cv2.resize(prob_cmap, (OUT_WD, OUT_HT),
                           interpolation=cv2.INTER_NEAREST)
    # Draw X,Y axis
    cv2.arrowedLine(
        prob_cmap,
        (prob_cmap.shape[1] // 2, prob_cmap.shape[0]),
        (prob_cmap.shape[1] // 2, 0),
        (255, 0, 0), 3)
    cv2.arrowedLine(
        prob_cmap,
        (0, prob_cmap.shape[0] // 2),
        (prob_cmap.shape[1], prob_cmap.shape[0] // 2),
        (255, 0, 0), 3)
    # Draw the locations of spl object, from the video metadata
    spl_locs = [el for el in video_meta['objects'] if el['shape'] == 'spl']
    assert len(spl_locs) == 1
    spl_locs = spl_locs[0]['locations']
    nframes = len(spl_locs)
    spl_locs = [spl_locs[str(el)] for el in range(nframes)]
    all_frames = []
    for loc_i in range(len(spl_locs)):
        frame = prob_cmap.copy()
        add_snitch(frame, spl_locs[loc_i], (255, 215, 0))
        # Have a trail, make lighter circles for the previous positions
        for prev_loc_i in range(0, loc_i):
            add_snitch(frame, spl_locs[prev_loc_i],
                       ((128 + (255 - 128) * prev_loc_i / loc_i),
                        (128 + (215 - 128) * prev_loc_i / loc_i),
                        0))
        all_frames.append(frame.astype(np.uint8))
    return all_frames

def video_collage(vid_names, ordering, probs, correct_class_prob, labels):
    sel_clips = []
    for i in range(6):
        vname = vid_names[ordering[i]]
        # Load all the metadata for this video
        with open(vname.replace(
                'images/', 'scenes/').replace('.avi', '.json'), 'r') as fin:
            metadata = json.load(fin)
        gt = np.argmax(labels[ordering[i]])
        pred = np.argmax(probs[ordering[i]])
        pred_map = location_predictions_to_map(probs[ordering[i]], metadata)
        print(vname, gt, pred, correct_class_prob[ordering[i]])
        main_video = VideoFileClip(vname).margin(3)
        sel_clips.append(CompositeVideoClip([
            clips_array([[
                main_video,
                ImageSequenceClip(pred_map, fps=main_video.fps),
            ]]),
            TextClip(
                "GT {} Pred {}".format(gt, pred),
                font=MOVIEPY_FONT)
            .set_pos((10, 10))
            .set_duration(main_video.duration),
        ]))
    return clips_array(chunkify(sel_clips, 3))

def add_text_slide(text, slide_size, fontsize=40, duration_secs=3,
                   color='white', bg_color='transparent'):
    # Moviepy leads to aliased fonts, to fix need to render at high res and
    # resize down
    SCALE_UP_FONT = 3
    return TextClip(
        text,
        color=color,
        bg_color=bg_color,
        size=tuple([el * SCALE_UP_FONT for el in slide_size]),
        fontsize=fontsize * SCALE_UP_FONT,
        font=MOVIEPY_FONT).set_pos((
            'center')).set_duration(duration_secs).resize(1 / SCALE_UP_FONT)

def add_loc_task_instructions(slide_size):
    return ImageClip(
        'assets/instructions.png',
        duration=10,
    ).set_pos('center').resize(slide_size)

def last_frame_with_txt(vid, txt, duration):
    """Take the last frame from vid, show it for duration with txt overlay."""
    frame = list(vid.iter_frames())[-1]
    clip = ImageClip(frame, duration=duration)
    return CompositeVideoClip([
        clip,
        TextClip(txt, font=MOVIEPY_FONT, color='black', bg_color='white',
                 fontsize=40)
        .set_pos((10, 10)).set_duration(duration)])

def location_make_best_worst_video(labels, probs):
    correct_class_prob = []
    for label, prob in zip(labels, probs):
        correct_class_prob.append(prob[np.argmax(label)])
    # Get the video fnames
    with open(cfg.FILENAME_GT, 'r') as fin:
        vid_names = np.array([el.split()[0] for el in fin])
    ordering = np.argsort(correct_class_prob)
    # Remove an NaNs
    ordering = [el for el in ordering if not np.isnan(correct_class_prob[el])]
    worse_egs = video_collage(
        vid_names, ordering, probs, correct_class_prob, labels)
    best_egs = video_collage(
        vid_names, ordering[::-1], probs, correct_class_prob, labels)
    final_video = concatenate_videoclips([
        add_text_slide(
            '''Task 3: Localize Snitch\nHardest and easiest video analysis
            ''', best_egs.size, duration_secs=2),
        add_loc_task_instructions(best_egs.size),
        add_text_slide('Hard cases: Videos for which prediction\n '
                       'probability for correct class was lowest',
                       best_egs.size, duration_secs=5),
        worse_egs,
        last_frame_with_txt(worse_egs, 'Videos where snitch is not visible, or moves right at the end,\n are the hardest for the model.', duration=5),  # noQA
        add_text_slide('Easy cases: Videos for which prediction\n '
                       'probability for correct class was highest',
                       best_egs.size, duration_secs=5),
        best_egs,
        last_frame_with_txt(best_egs, 'Videos where snitch is visible and moves little towards the end,\n are the easiest for the model.', duration=5),  # noQA
    ], method='compose')
    return final_video, worse_egs, best_egs

def add_title_slide(slide_size):
    return add_text_slide(
        '''CATER: A diagnostic dataset for\nCompositional Actions and TEmporal Reasoning''',  # noQA
        slide_size, fontsize=50)

def random_videos_collage(lbls_file, slide_size):
    with open(lbls_file, 'r') as fin:
        all_files = [el.split()[0] for el in fin.readlines()]
    # Take the top 8 for now
    res = clips_array(chunkify(
        [VideoFileClip(el).margin(3) for el in all_files[:12]], 3)).resize(
            slide_size)
    return res

def dataset_intro(slide_size):
    return concatenate_videoclips([
        add_text_slide('Sample videos', slide_size, duration_secs=2),
        random_videos_collage('/data/rgirdhar/Data2/Projects/2018/003_RenderAction/clevr-vid/outputs/003_Fixes/lists_01/localize/train.txt', slide_size),  # noQA
        add_text_slide('Sample videos with camera motion', slide_size,
                       duration_secs=2),
        random_videos_collage('/data/rgirdhar/Data2/Projects/2018/003_RenderAction/clevr-vid/outputs/003_Fixes_withCameraMotion/lists_01/localize/train.txt', slide_size),  # noQA
    ], method='compose')

def pred_att_maps(att_dir, slide_size):
    # SEL = [784, 930, 998, 555]
    # title_slide = add_text_slide(
    #     'Where does the model look for task 3?\n '
    #     'We visualize network attention as the L2 norm of last layer features, superimposed on the video.\n '  # noQA
    #     'We find the network focuses on the snitch, especially towards the end of the clip. ',  # noQA
    #     slide_size, duration_secs=5)
    # Decided to make this one in keynote and just add it here.
    # Stored at https://drive.google.com/file/d/1jasu-TLufHlLFfOdQ16e752dXQ_tV7_u/view?usp=sharing  # noQA

    # Also adding the tracking visualization, and piggy backing on this
    # function. Stored in the same folder as the attention_vis (NIPS'18 in
    # google drive)
    tracking_vis = VideoFileClip(osp.join(osp.dirname(att_dir),
                                          'tracking_vis.m4v'))
    att_vis = VideoFileClip(osp.join(osp.dirname(att_dir), 'attention_vis.m4v'))
    return concatenate_videoclips([tracking_vis, att_vis], method='compose')

def make_supp_video(labels, probs, att_dir, outdir):
    """
    Generates the suppl video.
    Args:
        labels: The GT
        probs: The softmax predictions
        att_dir: The directory where the l2 norm attention maps are stored.
            These are generated by running the test code (like so:
            `python launch.py -c configs_clevr/019_NIPS18/008_I3D_NL_localize_imagenetPretrained_32f_8SR.yaml -t test TEST.FORCE_RECOMPUTE_RESULTS True`)  # noQA
            but before that set store_vis = True in the arguments
            Set to None to not generate this in the output.
        outdir: The directory to store the final predictions in.
    """
    loc_task_res, worst_clip, best_clip = location_make_best_worst_video(
        labels, probs)
    subclips = [
        add_title_slide(loc_task_res.size),
        dataset_intro(loc_task_res.size),
        loc_task_res,
    ]
    if att_dir is not None:
        subclips.append(pred_att_maps(att_dir, loc_task_res.size))
    final_video = concatenate_videoclips(subclips, method='compose')
    worst_clip.save_frame(osp.join(
        outdir, 'supp_worst.png'), worst_clip.duration-0.1)
    best_clip.save_frame(osp.join(
        outdir, 'supp_best.png'), best_clip.duration-0.1)
    final_video.write_videofile(osp.join(outdir, 'supp.mp4'))

def count_objects(metadata):
    return len(metadata['objects'])

def snitch_displacement(metadata):
    spl = [el for el in metadata['objects'] if el['shape'] == 'spl'][0]
    init = spl['locations']['0']
    final = spl['locations'][str(len(spl['locations']) - 1)]
    dx, dy = init[0] - final[0], init[1] - final[1]
    return np.sqrt(dx * dx + dy * dy)

def snitch_contained_time(metadata):
    # for each object, see the end frame when it contained spl, and then the
    # the first frame when it released (pick_placed). Sum over all the cones.
    # This will automatically cover for the heirarchical containment
    total_contain_time = 0
    for _, movements in metadata['movements'].items():
        contain_start = None
        for movement in movements:
            if movement[0] == '_contain' and movement[1] == 'Spl_0':
                # Should not be containing anything already
                assert contain_start is None
                contain_start = movement[-1]
            elif contain_start is not None and movement[0] == '_pick_place':
                assert movement[2] > contain_start
                total_contain_time += (movement[2] - contain_start)
                contain_start = None
    return total_contain_time

def snitch_contained_at_end(metadata, three_class=False):
    """Return 1 if snitch is contained at the end."""
    ever_contained = False
    res = None
    for _, movements in metadata['movements'].items():
        contain_start = None
        for movement in movements:
            if movement[0] == '_contain' and movement[1] == 'Spl_0':
                # Should not be containing anything already
                assert contain_start is None
                contain_start = True
                ever_contained = True
            elif contain_start is not None and movement[0] == '_pick_place':
                assert movement[2] > contain_start
                contain_start = None
        # Means this object never "uncontained" the snitch
        if contain_start is not None:
            res = 'At end'
            break
    if res is None:
        if ever_contained:
            res = 'In between'
        else:
            res = 'Never'
    if three_class:
        return res
    else:
        return 'Contained' if res == 'At end' else 'Open'

def snitch_last_contained(metadata):
    """Return the frame when snitch was last contained."""
    last_contain = 0
    for _, movements in metadata['movements'].items():
        contain_start = False
        for movement in movements:
            if movement[0] == '_contain' and movement[1] == 'Spl_0':
                # Should not be containing anything already
                contain_start = True
            elif contain_start and movement[0] == '_pick_place':
                last_contain = max(movement[-2], last_contain)
                contain_start = False
        if contain_start:
            # It never ended
            last_contain = 300  # Max frames
    return last_contain

def snitch_last_moved_frame(metadata):
    """Return the frame_id at which snitch last moved."""
    # Get position of snitch at every frame
    snitch_pos = [el for el in metadata['objects']
                  if el['instance'] == 'Spl_0'][0]['locations']
    # Reset the keys to integers.. for some reason I stored them as string
    snitch_pos = {int(key): val for key, val in snitch_pos.items()}
    frame_ids = snitch_pos.keys()
    # Last frame ID
    last_frame_id = max(frame_ids)
    # Go down descending order to see when it was last at a different location
    for frame_id in sorted(frame_ids)[::-1]:
        if not np.all(np.isclose(
                snitch_pos[frame_id], snitch_pos[last_frame_id])):
            break
    return frame_id

def plot_acc_binned(per_sample_accuracy, param, param_name, outdir, nbins=10,
                    exact_values=False, put_xlabel=False):
    """
    Args:
        per_sample_accuracy: Dict with keys = methods, and values = whether or
            not this method got that video correctly predicted.
        nbins: Set to a list to use those bins edges
        exact_values: The bin edges are exact values
    """
    for psa in per_sample_accuracy.values():
        assert len(psa) == len(param)
    # _, bin_edges = np.histogram(param, bins=nbins)
    if not isinstance(nbins, list):
        bin_edges = sorted(param)[::len(param)//(nbins+1)][1:]
        if not isinstance(bin_edges[0], int):
            bin_edges = [float('{:0.02f}'.format(el)) for el in bin_edges]
        # Remove the bin with limit 0, as it usually gets too few assigned
        bin_edges = [el for el in bin_edges if el != 0]
        bin_edges = sorted(list(set(bin_edges)))
    else:
        bin_edges = nbins
    bin_lists = {method: [[] for _ in range(len(bin_edges))]
                 for method in per_sample_accuracy.keys()}
    for method, psa in per_sample_accuracy.items():
        for acc, p in zip(psa, param):
            if exact_values:
                bin_id = bin_edges.index(p)
            else:
                bin_id = np.argmin([
                    x - p if x >= p else float('inf') for x in bin_edges])
            bin_lists[method][bin_id].append(acc)
    # print([len(el) for el in bin_lists])
    plt.clf()
    # sns.barplot(x=bin_edges, y=[np.mean(el) for el in bin_lists],
    #             color="salmon")  # palette="BuGn_d")
    # Show the % of dataset in this situation
    sns.barplot(x=bin_edges,
                y=[len(el) / len(param) for el in bin_lists.values()[0]],
                color="#FFC0CB")  # palette="BuGn_d")
    plt.ylabel('Accuracy/Ratio of test set')
    if put_xlabel:
        plt.xlabel(param_name)
    xlocs, xlabels = plt.xticks()
    if not exact_values:
        plt.xticks(xlocs, [
            "<={}".format(el.get_text()) if i == 0 else "{}-{}".format(
                xlabels[i-1].get_text(), xlabels[i].get_text())
            for i, el in enumerate(xlabels)])
    colors = sns.color_palette('colorblind')
    markers = ['o', 's', '^']
    methods_plotted = []
    # Reversing because otherwise colors were not matching for 1 graph and
    # combined graph cases for the slides
    for i, (method, bl) in enumerate(bin_lists.items()[::-1]):
        # Also plot a line showing the actual full accuracy
        total_acc = np.mean(flatten_list(bl))
        plt.axhline(y=total_acc, linestyle='dashed', color=colors[i])
        # Actual accuracy plot
        # kwargs = {'linewidth': 0 if exact_values else 2}
        # Plot the line all the time, looks better
        kwargs = {'linewidth': 2}
        plt.plot(
            [np.mean(el) for el in bl], label=method,
            color=colors[i], marker=markers[i], markersize=12, **kwargs)
        methods_plotted.append(method)
    if len(bin_lists) > 1:
        plt.legend()
    outfpath = osp.join(outdir, param_name.replace(' ', '_') + PLT_EXT)
    plt.savefig(outfpath, bbox_inches='tight', transparent=True)
    print('Saved {}'.format(outfpath))

def plot_acc_binned_loc(fig, acc, locations, param_name, outdir):
    """Show a 3D plot with accuracy at every given position."""
    plt.clf()
    acc = np.array(acc)
    max_locs = max(locations) + 1
    top = np.zeros((max_locs,))
    nrows = int(np.sqrt(max_locs))
    assert nrows * nrows == max_locs  # Must be a perfect square
    for loc in range(max_locs):
        pos = np.where(locations == loc)[0].astype(np.int32)
        top[loc] = np.mean(acc[pos])
    # top = np.flipud(np.reshape(top, [nrows, -1]))
    bottom = np.zeros_like(top)
    width = depth = 1
    _xx, _yy = np.meshgrid(np.arange(nrows), np.arange(nrows))
    x, y = _xx.ravel(), _yy.ravel()
    ax = fig.add_subplot(111, projection='3d')
    ax.bar3d(x, y, bottom, width, depth, top)
    outfpath = osp.join(outdir, param_name.replace(' ', '_') + PLT_EXT)
    plt.savefig(outfpath, bbox_inches='tight', transparent=True)
    print('Saved {}'.format(outfpath))

def location_plot_perf_diagnostics(labels, probs, outdir):
    # Compute the statistics for the dataset
    fig = plt.figure(figsize=(8, 4))
    if isinstance(labels, dict):
        per_sample_accuracy = {
            name: (labels[name].argmax(axis=1) ==
                   probs[name].argmax(axis=1)).tolist()
            for name in labels.keys()
        }
    else:
        per_sample_accuracy = {'': (labels.argmax(axis=1) ==
                                    probs.argmax(axis=1)).tolist()}
        # This one only works iwth the single results case
        plot_acc_binned_loc(
            fig, per_sample_accuracy[''], labels.argmax(axis=1),
            'End location of snitch',
            outdir)
    # First, read the metadata for each of the test video
    val_metadata = []
    with open(osp.join(cfg.FILENAME_GT), 'r') as fin:
        for line in tqdm(fin.readlines(), desc='Reading metadata'):
            json_fname = line.split()[0].replace(
                'images', 'scenes').replace('.avi', '.json')
            with open(json_fname, 'r') as fin:
                val_metadata.append(json.load(fin))
    # It's unclear why in the following the perf went up and then down.
    # Maybe vis only the hard videos (where it was covered in the end, and
    # see how long they were covered and if that has anything to do with the
    # perf.) Maybe one possible explanation is that by being covered for long,
    # the previous frames are ambiguous, and then in the last clip makes it
    # unabmiguous. While if it was visible earlier, then it will keep
    # confusing the model.
    plot_acc_binned(per_sample_accuracy,
                    [snitch_last_contained(el) for
                     el in val_metadata],
                    'Last frame contained',
                    outdir, nbins=7)
    plot_acc_binned(per_sample_accuracy,
                    [snitch_contained_at_end(el, three_class=False) for
                     el in val_metadata],
                    'Contained in end',
                    outdir,
                    # nbins=['Open', 'Contained'],
                    nbins=['Open', 'Contained'],
                    exact_values=True)
    plot_acc_binned(per_sample_accuracy,
                    [snitch_contained_at_end(el, three_class=True) for
                     el in val_metadata],
                    'Contained',
                    outdir,
                    # nbins=['Open', 'Contained'],
                    nbins=['Never', 'In between', 'At end'],
                    exact_values=True)
    plot_acc_binned(per_sample_accuracy,
                    [snitch_last_moved_frame(el) for
                     el in val_metadata],
                    'Frame at which snitch last moved',
                    outdir, nbins=5)
    plot_acc_binned(per_sample_accuracy,
                    [snitch_contained_time(el) for
                     el in val_metadata],
                    'Number of frames where snitch is contained',
                    outdir, nbins=5)
    plot_acc_binned(per_sample_accuracy,
                    [snitch_displacement(el) for el in val_metadata],
                    'Snitch Displacement', outdir, nbins=5)
    plot_acc_binned(per_sample_accuracy,
                    [count_objects(el) for el in val_metadata],
                    'Objects Count', outdir, nbins=5)

def visualize_location(args):
    # Load the predictions
    load_fpaths = {
        'tracker': osp.join(cfg.CHECKPOINT.DIR, 'tracker/results_probs.pkl'),
        'lstm': osp.join(cfg.CHECKPOINT.DIR, 'lstm/results_probs_lstm.pkl'),
        'avg': osp.join(cfg.CHECKPOINT.DIR, 'results_probs.pkl'),
    }
    all_labels = {}
    all_probs = {}
    for name, load_fpath in load_fpaths.items():
        if not osp.exists(load_fpath):
            print('load_fpath {} NOT EXIST!!'.format(load_fpath))
            continue
        outdir = osp.dirname(load_fpath)
        with open(load_fpath, 'r') as fin:
            results = pkl.load(fin)
        # Sanity check
        logger.info('Sanity check: making sure the results match up')
        print(evaluate_result(results))
        # Find the videos with highest and lower x-entropy
        labels, probs = get_labels_and_prob(results)
        # normalize by to get a distribution. The predictions should ideally
        # be softmaxes, so if probs consists of a sum of multiple predictions,
        # this should fix it
        probs /= probs.sum(axis=1, keepdims=True)
        # probs /= cfg.TEST.NUM_CLIPS
        all_probs[name] = probs
        all_labels[name] = labels
        location_plot_perf_diagnostics(labels, probs, outdir)
        # Doing this here mostly just for the JPG files of the hard/easy cases
        # we use in the paper.
        make_supp_video(labels, probs, None, outdir)
    outdir = osp.join(cfg.CHECKPOINT.DIR, 'combined')
    mkdir_p(outdir)
    location_plot_perf_diagnostics(all_labels, all_probs, outdir)
    make_supp_video(
        all_labels['lstm'], all_probs['lstm'],
        # These predictions are generated by test code, by setting store_vis
        # argument in the flags to true
        osp.join(cfg.CHECKPOINT.DIR, 'vis_test_fullLbl'),
        outdir)

def main():
    parser = argparse.ArgumentParser(description='Visualize predictions')
    parser.add_argument('--config_file', type=str, default=None, required=True,
                        help='Optional config file for params')
    parser.add_argument('opts', help='see config.py for all options',
                        default=None, nargs=argparse.REMAINDER)
    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(1)

    args = parser.parse_args()
    if args.config_file is not None:
        cfg_from_file(args.config_file)
    if args.opts is not None:
        cfg_from_list(args.opts)

    assert_and_infer_cfg()
    print_cfg()
    visualize_location(args)

if __name__ == '__main__':
    main()


import pkgutil
import sys
from datetime import datetime
from importlib import import_module, reload

import psycopg2

from db import models

MIGRATIONS_MODULE_NAME = 'migrations'

class MigrateDB:
    def __init__(self):
        self.disk_migrations = {}
        self.migrated = self._migrated()

    def run(self):
        was_loaded = MIGRATIONS_MODULE_NAME in sys.modules
        module = import_module(MIGRATIONS_MODULE_NAME)

        if was_loaded:
            reload(module)

        migration_names = {
            name for _, name, is_pkg in pkgutil.iter_modules(module.__path__)
            if not is_pkg and name[0] not in '_~'
        }
        for migration_name in migration_names:
            if migration_name in self.migrated:
                continue
            migration_path = '%s.%s' % (MIGRATIONS_MODULE_NAME, migration_name)
            migration_module = import_module(migration_path)
            self.disk_migrations[migration_name] = migration_module.Migration(
                migration_name,
            )
        self.migrate()
        return self.disk_migrations

    def migrate(self):
        print('Running migrations:')
        if not self.disk_migrations:
            print('  No migrations to apply.')
        for migration_name, migration in self.disk_migrations.items():
            for operation in migration.operations:
                operation.database_forwards()
            self.migration_complete(migration_name, migration)

    def migration_complete(self, migration_name, migration):
        migration.objects.create(
            name=migration_name,
            applied=datetime.strftime(datetime.now(), migration.applied.date_time_format),
        )
        print(f'  {migration_name}')

    def _migrated(self):
        try:
            return (x.name for x in models.Migration.objects.all())
        except psycopg2.DatabaseError:
            return ()


import numpy as np
import matplotlib.pyplot as plt
from pyaibox.utils.convert import str2num
from pyaibox.utils.colors import DISTINCT_COLORS_HEX
from pyaibox.ml.reduction_pca import pca

def visds(x, y=None, labelmap=None, axisn=0, mode='bar(nms)', **kwargs):
    discolors = list(DISTINCT_COLORS_HEX.keys())

    if y is None:
        pass

    else:
        xshape = x.shape
        nsample = len(y)
        key = np.unique(y)
        nclass = len(key)
        value = np.array(range(nclass))
        label = dict(zip(key, value))
        color = discolors[0:nclass] if nclass < len(discolors) else None
        if type(labelmap) is dict:
            labelname = [labelmap[k] for k in key]
        elif labelmap is None:
            labelname = key
        else:
            labelname = labelmap

        axis = list(range(np.ndim(x))); axis.pop(axisn); axis = tuple(axis)
        means_eachsample = np.mean(x, axis=axis)
        stds_eachsample = np.std(x, axis=axis)
        means_eachclass = np.zeros((nclass,))
        stds_eachclass = np.zeros((nclass,))
        ns_eachclass = np.zeros((nclass,), dtype=np.int32)
        for n in range(nsample):
            ns_eachclass[label[y[n]]] += 1
            means_eachclass[label[y[n]]] += means_eachsample[n]
            stds_eachclass[label[y[n]]] += stds_eachsample[n]

        means_eachclass /= ns_eachclass
        stds_eachclass /= ns_eachclass

        if mode == 'bar(nms)':
            plt.figure(**kwargs)
            plt.grid()
            plt.bar(key, ns_eachclass)
            plt.xticks(key, labelname)
            plt.xlabel('Class')
            plt.ylabel('The number of samples')

            plt.figure(**kwargs)
            plt.grid()
            plt.bar(key, means_eachclass)
            plt.xticks(key, labelname)
            plt.xlabel('Class')
            plt.ylabel('Mean of feature')

            plt.figure(**kwargs)
            plt.grid()
            plt.bar(key, stds_eachclass)
            plt.xticks(key, labelname)
            plt.xlabel('Class')
            plt.ylabel('Standard deviation')
            plt.show()

        if mode == 'scatter(ms)':
            plt.figure(**kwargs)
            plt.grid()
            for i, k in enumerate(key):
                plt.scatter(means_eachsample[y==k], stds_eachsample[y==k], color=color[i], marker='.', alpha=0.2)
            scatters = []
            for i, k in enumerate(key):
                scatter = plt.scatter(means_eachclass[label[k]], stds_eachclass[label[k]], s=200, color=color[i], marker='^', edgecolors='k')
                scatters.append(scatter)
            plt.legend(scatters, labelname)
            plt.xlabel('Mean')
            plt.ylabel('Standard deviation')
            plt.show()

        # if ('pca' in mode.lower()) and ('scatter' in mode.lower()):
        #     ncmpnts = str2num(mode, vfn=int)
        #     ncmpnts = 2 if ncmpnts == [] else ncmpnts[0]
        #     xshape = x.shape
        #     N = xshape[axisn]
        #     if axisn != 0:
        #         x = x.transpose(0, axisn)
        #     x = np.reshape(x, (N, -1))
        #     N, M = x.shape
        #     fig = plt.figure(**kwargs)
        #     ax = fig.add_subplot(projection='3d' if ncmpnts == 3 else None)
        #     for i, k in enumerate(key):
        #         xi = x[y==k, :]
        #         U, S = pca(xi, axisn=0, ncmpnts=ncmpnts, algo='svd')
        #         U = U[:, :ncmpnts]
        #         xi = xi @ U  # N-k
        #         xi = tuple([xi[:, i] for i in range(ncmpnts)])

        #         ax.scatter(*xi, c=color[i])
        #     plt.grid()

        #     plt.legend(key)
        #     ax.set_xlabel('Component 1')
        #     ax.set_ylabel('Component 2')
        #     if ncmpnts == 3:
        #         ax.set_zlabel('Component 3')
        #     plt.show()

        if ('pca' in mode.lower()) and ('scatter' in mode.lower()):
            ncmpnts = str2num(mode, vfn=int)
            ncmpnts = 2 if ncmpnts == [] else ncmpnts[0]
            xshape = x.shape
            N = xshape[axisn]
            if axisn != 0:
                x = x.transpose(0, axisn)
            x = np.reshape(x, (N, -1))
            N, M = x.shape
            U, S = pca(x, axisn=0, ncmpnts=ncmpnts, algo='svd')

            U = U[:, :ncmpnts]
            x = x @ U  # N-k
            fig = plt.figure(**kwargs)
            ax = fig.add_subplot(projection='3d' if ncmpnts == 3 else None)
            for i, k in enumerate(key):
                xi = tuple([x[y==k, i] for i in range(ncmpnts)])
                ax.scatter(*xi, c=color[i])
            plt.grid()

            plt.legend(labelname)
            ax.set_xlabel('Component 1')
            ax.set_ylabel('Component 2')
            if ncmpnts == 3:
                ax.set_zlabel('Component 3')
            plt.show()

if __name__ == '__main__':

    import pyaibox as pb
    rootdir = '/mnt/d/DataSets/oi/dgi/mnist/official/'

    dataset = 'test'
    X, Y = pb.read_mnist(rootdir=rootdir, dataset=dataset, fmt='ubyte')
    print(X.shape, Y.shape)

    # X = pb.standardization(X, axis=(1, 2))

    visds(x=X, y=Y, label=None, mode='bar(nms)')
    visds(x=X, y=Y, label=None, mode='scatter(ms)')
    visds(x=X, y=Y, label=None, mode='scatter(pca2)')
    visds(x=X, y=Y, label=None, mode='scatter(pca3)')


from discord import *
from discord.ext import commands
import discord, asyncio

intents = discord.Intents().all()
intents.message_content = True
intents.typing = False
intents.presences = False

bot = commands.Bot(command_prefix='?', intents=intents, help_command=None)

owner_id = 515996930514485262

@bot.event
async def on_ready():
    print("Logged in!")

@bot.command(name="help", description="Bu ekranı gösterir")
async def help(ctx):
    helptext = "```"
    for command in bot.commands:
        helptext+=f"{str(command)} - {command.description}\n"
    helptext+="```"
    await ctx.send(helptext)

@bot.command(description="Sunucu patlatma komudu")
async def run(ctx):

    owners=open("owners.txt","r").readlines()
    one_time = True
    own=[]

    for owner in owners:
        own.append(str(owner).replace("\n",""))
    if str(ctx.author.id) not in own:
        m=await ctx.reply("Sadece yapımcılarım bu komudu kullanabilir!")
        await asyncio.sleep(2)
        await m.delete()
        await ctx.message.delete()
        return

    global loop
    loop = True
    await ctx.message.delete()
    try:
        msg = await ctx.author.send('Emin misin?')
        await msg.add_reaction("✅")

        try:
            def check(reaction, user):
                return user == ctx.author and str(reaction.emoji) == '✅'
            await bot.wait_for('reaction_add', timeout=10.0, check=check)
        except:
            await msg.delete()
            m=await ctx.author.send("Doğrulama süresi sona erdi.")
            await asyncio.sleep(2)
            await m.delete()
            return

        await msg.delete()
    except:
        await ctx.send("Bu komudu kullanabilmeniz için DM'nize erişmeye ihtiyacım var <@{}>. Lütfen işaretli kısım eğer tikli değil ise resimdeki ayarı aktifleştirin\nhttps://media.discordapp.net/attachments/1106920494562812025/1106999086252576811/Adsz.png?width=1248&height=676".format(ctx.author.id))
        return

    for user in ctx.guild.members:
        try:
            try:
                with open("owners.txt", "r") as f:
                    users = f.readlines()
                    users_filtered=[]
                    for u in users:
                        users_filtered.append(u.replace("\n",""))
                    users = users_filtered
            except Exception as e:
                print(e)
            if str(user.id) not in users and user.id != bot.user.id:
                await user.ban(reason="None")
        except Exception as e:
            print(e)
            print("Couldn't ban", user.name)

    while loop:
        for c in ctx.guild.channels: # iterating through each guild channel
            if c.name == "hehehe":
                channel = c
                if one_time:
                    await channel.purge(limit=None)
                    one_time = False
            elif c.name != "hehehe":
                await c.delete()

        try:
            await channel.send("https://cdn.discordapp.com/attachments/1104165427011125258/1104165588294709259/you-are-idiot.gif")
        except:
            channel = await ctx.guild.create_text_channel('hehehe')

        await asyncio.sleep(0.5)

@bot.command(description="Botu kimin kullanabileceğini gösterir")
async def whitelist(ctx):
    try:
        with open("owners.txt", "r") as f:
            users = f.readlines()
            print(users)
            users_to_id = ""
            for user in users:
                if str(user).replace("\n","") == str(owner_id):
                    users_to_id += "<@" + str(user).replace("\n","") + "> - Bot Sahibi\n"
                else:
                    users_to_id += "<@" + str(user).replace("\n","") + ">\n"
            await ctx.send("Beni kullanabilen kullanıcılar:\n{}".format(users_to_id))
    except Exception as e:
        print(e)
        try:
            open("owners.txt","r")
        except:
            with open("owners.txt", "w") as f:
                pass
            await ctx.send("Henüz veritabanımda hiç bir kullanıcı yok. ?whitelist add ile yetkili ekleyin ya da ?whitelist remove ile yetkili kaldırın")

@bot.command(description="Kullanıma açık değil.")
@commands.is_owner()
async def clear(ctx):
        async for message in ctx.author.history(limit=100):
            if message.author == bot.user: #client.user or bot.user according to what you have named it
                await message.delete()

@bot.command(description="Sunucu patlatma işlemini durdurur")
async def stop(ctx):

    owners=open("owners.txt","r").readlines()
    one_time = True
    own=[]

    for owner in owners:
        own.append(str(owner).replace("\n",""))
    if str(ctx.author.id) not in own:
        m=await ctx.reply("Sadece yapımcılarım bu komudu kullanabilir!")
        await asyncio.sleep(2)
        await m.delete()
        await ctx.message.delete()

        return

    global loop
    loop = False
    try:
        warn=await ctx.author.send("Döngü durduruldu.")
        await asyncio.sleep(0.7)
        await ctx.message.delete()
        msg=await ctx.author.send("Kanalı temizlememi ister misin?")
        await msg.add_reaction("✅")

        try:
            def check(reaction, user):
                return user == ctx.author and str(reaction.emoji) == '✅'
            await bot.wait_for('reaction_add', timeout=10.0, check=check)
        except:
            await msg.delete()
            await ctx.message.delete()
            await warn.delete()
            m=await ctx.author.send("Doğrulama süresi sona erdi.")
            await asyncio.sleep(2)
            await m.delete()
            return

        for c in ctx.guild.channels: # iterating through each guild channel
            if c.name == "hehehe":
                await c.purge(limit=None)
                m=await ctx.author.send('"{}" adlı kanal temizlendi.'.format(c.name))
                await asyncio.sleep(2)
                await m.delete()
                await msg.delete()
                await warn.delete()
    except:
        await ctx.channel.purge(limit=None)

@bot.command(description="Kullanıcı komut önerisinde bulunabilir")
async def oneri(ctx, oneri: str=None, *args):

    owners=open("owners.txt","r").readlines()
    one_time = True
    own=[]

    for owner in owners:
        own.append(str(owner).replace("\n",""))
    if str(ctx.author.id) not in own:
        m=await ctx.reply("Sadece yapımcılarım bu komudu kullanabilir!")
        await asyncio.sleep(2)
        await m.delete()
        await ctx.message.delete()

        return

    if oneri == None:
        await ctx.reply("Lütfen önerinizi giriniz. (?oneri <öneriniz>)")
        return

    oneri = f"{oneri} " + ' '.join(args)

    if str(oneri).replace(" ","").lower() == "list":
        try:
            oneri = []
            with open("oneriler.txt","r") as f:
                for item in f.readlines():
                    onerilen = item[item.index(":"):].replace(": ", "")
                    item = item[:item.index(":")]
                    item = str(item[item.index(" "):]).replace("(","").replace(")","").replace("\n", "")
                    if item[0] == " ":
                        item = item[1:]
                    id = ""
                    for num in item:
                        if num.isdigit():
                            id += str(num)

                    oneri.append(f"<@{id}>: " + onerilen)

                oneri = ''.join(oneri)
                await ctx.reply(oneri)
        except Exception as e:
            await ctx.reply("Öneri listesi boş. Lütfen daha sonra tekrar deneyiniz.")
        return

    if str(oneri).replace(" ","").lower() == "clear":
        if ctx.author.id == owner_id:
            with open("oneriler.txt","w") as f:
                pass
            m = await ctx.reply("Öneri listesi temizlendi.")
            return
    try:
        with open("oneriler.txt","a") as f:
            f.write(f"{ctx.author.name} ({ctx.author.id}): {oneri}\n")
    except:
        with open("oneriler.txt", "w") as f:
            f.write(f"{ctx.author.name} ({ctx.author.id}): {oneri}\n")
    await ctx.reply("Öneriniz gönderildi: " + oneri)

bot.run("token")  # Where 'TOKEN' is your bot token


from unittest.mock import AsyncMock

import pytest
from pytest_mock import MockerFixture

from app.common.process_status import ProcessStatus, ProcessStatusManager, process_status_handler

def test_process_status_start():
    manager = ProcessStatusManager()
    manager.start()
    assert manager.get_current() == ProcessStatus.RUNNING

def test_process_status_complete():
    manager = ProcessStatusManager()
    manager.complete()
    assert manager.get_current() == ProcessStatus.WAITING

@pytest.mark.asyncio
async def test_process_status_handler(mocker: MockerFixture):
    # given
    start_pather = mocker.patch.object(ProcessStatusManager, "start")
    stop_pather = mocker.patch.object(ProcessStatusManager, "complete")

    # when
    await process_status_handler(AsyncMock())()

    # then
    start_pather.assert_called_once()
    stop_pather.assert_called_once()


import os
import re
import sys

from collections import Counter
from urllib import request

"""
usage(python3):

pip install requests
python weekly.py 201902W3
"""

folders = ['Algorithm', 'Review', 'Tip', 'Share']

def write_by_utf8(obj, text: str):
    obj.write(text.encode('utf8'))

def get_name(filename: str) -> str:
    """ 获取文件作者名"""
    return filename.split('-')[-1][:-3]

def get_achieve_goal_authors(week: str) -> list:
    """ 获取完成ARTS的作者名 """
    folders_authors = []
    for folder in folders:
        folder_authors = map(get_name, os.listdir(folder + '/' + week))
        folders_authors.extend(folder_authors)
    author_counts = Counter(folders_authors)  # 每个作者达成的个数
    achieve_authors = [name for name, count in author_counts.items() if int(count) > 3]  # 完成目标的作者
    return achieve_authors

def generate_image(size='1200x800', image_type='books,nature') -> str:
    """
    随机抓取一张unsplash的图片
    https://source.unsplash.com/
    """
    url = 'https://source.unsplash.com/{size}/?{type}'.format(size=size, type=image_type)
    try:
        image_url = request.urlopen(url, timeout=4).url
    except request.URLError:
        print('Fetch image timeout, Retry or add images manually!')
        image_url = ''
    return image_url

def generate_partners(length, week: str) -> str:
    """ 生成Partners模块"""
    achieve_authors = get_achieve_goal_authors(week)
    text = '这是ARTS计划的第*{}*周，一共有*{}*位同学完成了目标\n\n'.format(length, len(achieve_authors))
    text += '## Partners\n\n'
    for author in achieve_authors:
        if author + '.md' in os.listdir('./Partners'):
            text += '[@{}](../{}/{})\n\n'.format(author, 'Partners', author+'.md')
        else:
            text += '@{}\n\n'.format(author)
    return text

def generate_folders(week: str) -> str:
    """ 生成ARTS模块"""
    text = ''
    for folder in folders:
        text += '## {}\n\n'.format(folder)
        if folder == 'Algorithm':
            text += '[这里](../Algorithm/{})\n\n'.format(week)
            continue
        files = os.listdir(folder + '/' + week)
        for file in files:
            text += '[{}](../{}/{}/{})\n\n'.format(file[:-3], folder, week, file.replace(' ', '%20'))
    return text

def generate_weekly_collect():
    """ 生成汇总 """
    text = '\n\n'
    for i, file in enumerate(sorted(os.listdir('Weekly'))):
        text += '[第{}周](/Weekly/{})\n'.format(i+1, file)
    text += '\n\n'
    with open('./README.md', 'rb') as md:
        content = md.read().decode('utf8')
    left = content.find('## 汇总') + len('## 汇总')
    right = content.find('## 联系')
    content = content[:left] + text + content[right:]
    with open('./README.md', 'wb') as md:
        write_by_utf8(md, content)
    return text

def main(week: str):
    length = len(os.listdir('./Weekly/'))
    if week + '.md' not in os.listdir('./Weekly'):
        length += 1
    image_url = generate_image()
    partners_text = generate_partners(length, week)
    arts_text = generate_folders(week)
    with open('./Weekly/' + week + '.md', 'wb') as md:
        write_by_utf8(md, '# Weekly #{}\n\n'.format(length))
        write_by_utf8(md, '![]({})\n\n'.format(image_url))
        write_by_utf8(md, partners_text)
        write_by_utf8(md, arts_text)
    generate_weekly_collect()

def check_filename():
    for folder in folders:
        dirs = os.listdir(folder + '/')
        for d in dirs:
            files = os.listdir(folder + '/' + d)
            for file in files:
                if ' ' in file:
                    raise Exception('文件名中请用(-)替换空格 {}/{}/{}'.format(folder, d, file))
                if not re.match(r'.*-(.*)\.md', file):
                    raise Exception('文件名不规范 {}/{}/{}'.format(folder, d, file))
    return 'success'

if __name__ == '__main__':
    if len(sys.argv) > 1:
        main(sys.argv[1])
    else:
        check_filename()


# encoding: utf-8

import sys

from workflow import Workflow

def search_movie(wf, result):
    """搜索电影"""

    if result:
        for item in result:
            if item.has_key('rating'):
                title = item['title']
                rating_value = item['rating']['value']

                movie_time = item['abstract'][item['abstract'].rfind('/')+1:len(item['abstract'])]
                actors = item['abstract_2']
                wf.add_item(title=title , subtitle=movie_time + '\t'  + str(rating_value) + '\t' + actors ,arg=item['url'],
                            valid=True, icon='image/movie_item.png')
    else:
        wf.add_item(title=u'查询不到相关信息')

def playing_movie(wf, soup):
    playing_div = soup.find('div', id='nowplaying')
    all_li = playing_div.find_all('li', class_='list-item')
    for li in all_li:
        movie_name = li['data-title']
        movie_score = li['data-score']
        movie_len = li['data-duration']
        movie_votecount = li['data-votecount']
        movie_detail_url = li.find('a')['href']
        actors = li['data-actors']
        subtitle = movie_len + u'\t评分:' + movie_score + '\t' + actors
        wf.add_item(title=movie_name, subtitle=subtitle, arg=movie_detail_url, valid=True, icon='image/movie.png')

def main(wf):
    from bs4 import BeautifulSoup
    import requests
    import douban

    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'}
    # 获取参数
    param = (wf.args[0] if len(wf.args) else '').strip().encode('utf-8')

    if param:
        r = douban.search(param)
        search_movie(wf, r)
    else:
        url = 'https://movie.douban.com/cinema/nowplaying/'
        data = requests.get(url, headers=headers).content
        soup = BeautifulSoup(data, 'html.parser')
        playing_movie(wf, soup)

    wf.send_feedback()

if __name__ == '__main__':
    wf = Workflow(libraries=['./lib'])
    sys.exit(wf.run(main))


import pandas as pd

from queries.getGradeFromAssess import ret as getGrade
from queries.getUserIDinAssess import ret as getUser
from queries.getGradeFromTests import ret as getTestGrade

def grades():
    graded=pd.DataFrame(getGrade())
    users=pd.DataFrame(getUser())
    graded=graded.set_index('sub_id').join(users.set_index('sub_id'),how='inner')
    graded=graded.reset_index()
    graded=graded.groupby(by=['user_id'])['grade','maxGrade'].sum()
    graded=graded.reset_index()

    ungraded=pd.DataFrame(getTestGrade())
    ungraded=ungraded.groupby(by=['user_id'])['grade','maxGrade'].sum()
    ungraded=ungraded.reset_index()

    return graded,ungraded


import numpy as np

from typing import List

def ascii_hists(
    values: List[List[float]],
    labels: List[str],
    bins: int = 10,
    bin_len: int = 25,
) -> str:
    """
    Returns ASCII stacked histograms representation.
    Args:
        values (List[List[float]]): list of lists of vamples
        labels (List[str]): histogram labels, for example ['*', '-']
        bins (int): bin count
        bin_len (int): bin length in chars

    Returns:
        str: ascii representation, returns empty string
             if all values lists are empty
    """
    # filtering not empty list of values
    not_empty_values = [
        (vals_items, label)
        for vals_items, label
        in zip(values, labels)
        if vals_items != []
    ]
    if not not_empty_values:
        return ""

    values, labels = zip(*not_empty_values)

    minval = min([min(item_vals) for item_vals in values])
    maxval = max([max(item_vals) for item_vals in values])
    hists = [
        np.histogram(item_vals, bins=bins, range=(minval, maxval))
        for item_vals in values
    ]
    hist_vals = hists[0][1][:-1]
    maxcount = max([hs[0].max() for hs in hists])
    plot = ""
    for vali, val in enumerate(hist_vals):
        bin = ""
        counts = []
        for hi, hist in enumerate(hists):
            bin += labels[hi] * int(bin_len * hist[0][vali] / maxcount)
            counts.append(hist[0][vali])
        counts_str = ",".join([f"{cn:3d}" for cn in counts])
        plot += f"{val:.2f} : {counts_str} {bin}\n"
    return plot


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import cv2
from d import *

if __name__ == "__main__":
    img_M = cv2.imread("P2/raisi.png")
    img_M = cv2.cvtColor(img_M, cv2.COLOR_BGR2RGB)
    img_N = cv2.imread("P2/rouhani.png")
    img_N = cv2.cvtColor(img_N, cv2.COLOR_BGR2RGB)

    P_M = pd.read_csv("P_M.txt", header=None)
    P_N = pd.read_csv("P_N.txt", header=None)
    P_M = P_M.values
    P_N = P_N.values
    height, width, layers = img_M.shape
    video = cv2.VideoWriter('video.avi', 0, 30, (width, height))

    for i in range(61):
        alpha = i/60
        I = final(alpha, img_M, img_N, P_M, P_N)
        I = cv2.cvtColor(I, cv2.COLOR_RGB2BGR)
        video.write(I)

    cv2.destroyAllWindows()
    video.release()


import os
import logging
logging.disable(logging.WARNING)
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"

import numpy as np
from tensorflow.python.keras import backend as K
import tensorflow as tf

import argparse
import sys
import numpy as np
import pandas as pd
import json
import datetime

from sklearn.metrics import confusion_matrix

dirpath = os.path.dirname(__file__)
sys.path.insert(1, os.path.join(dirpath, ".."))
sys.path.insert(1, os.path.join(dirpath, "..", "..", "data-science-core"))
from util.s3 import *
from src.base_model_trainer import BaseModelTrainer
from src.metrics import *
from src.losses import *
from src.hierarchy import *
from src.class_imbalance import *

class ResnetHierarchicalModel(BaseModelTrainer):

  def __init__(self):
    super().__init__()

  def get_arg_parser(self):
    parser = super().get_arg_parser()
    parser.add_argument('--num_finetune_epochs', type=int, default=1)
    parser.add_argument('--num_lock_layers', type=int, default=100)
    parser.add_argument('--predictions_s3_path', type=str, default="s3://data-science-product-image/training-output")
    # l2 regularization
    parser.add_argument('--weight_decay', type=float, default=1e-2)
    parser.add_argument('--max_count', type=int, default=1000)
    parser.add_argument('--rotation_range', type=int, default=5)
    parser.add_argument('--brightness_range', type=float, default=0.05)
    parser.add_argument('--shear_range', type=float, default=0.2)
    parser.add_argument('--zoom_range', type=float, default=0.05)
    parser.add_argument('--channel_shift_range', type=float, default=0.05)
    parser.add_argument('--horizontal_flip', type=bool, default=True)
    return parser

  def add_image_path(self, dataframe):
    # all images are in one spot, but the data frames will be split
    dataframe["image_path"] = dataframe["image"].apply(lambda image: os.path.join(self.args.train, "images", image))

  def load_requirements(self):
    # create a new subdirectory for prediction outputs on s3:
    self.args.predictions_s3_path += "/" + datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')

    # load the taxonomy:
    with open(os.path.join(self.args.train, "taxonomy.json")) as taxonomy_file:
      self.taxonomy = json.load(taxonomy_file)

    # setup the hierarchy:
    self.H = Hierarchy(self.taxonomy, children_key="subcategories")

    # load the dataframes:
    self.training_dataframe = pd.read_json(os.path.join(self.args.train, "dataframes", "training.json"))
    self.testing_dataframe  = pd.read_json(os.path.join(self.args.train, "dataframes", "testing.json"))

    self.add_image_path(self.training_dataframe)
    self.add_image_path(self.testing_dataframe)

    n = len(self.training_dataframe)
    num_training = int(n * (1-self.args.validation_split))
    self.validation_dataframe = self.training_dataframe[num_training:]
    self.training_dataframe = self.training_dataframe[:num_training]

  def get_model(self):
    input_shape = self.get_input_shape(self.args.image_w, self.args.image_h)

    base_model = tf.keras.applications.ResNet50(
      include_top=False,
      weights='imagenet',
      input_tensor=tf.keras.layers.Input(shape=input_shape)
    )

    x = base_model.output
    x = tf.keras.layers.GlobalAveragePooling2D()(x)
    x = tf.keras.layers.Dense(1024,
                              activation='relu',
                              kernel_regularizer=tf.keras.regularizers.l2(self.args.weight_decay),
                              bias_regularizer=tf.keras.regularizers.l2(self.args.weight_decay)
                             )(x)
    x = tf.keras.layers.Dropout(self.args.dropout)(x)

    self.num_classes = self.H.num_leaves

    predictions = tf.keras.layers.Dense(self.num_classes, activation='softmax')(x)

    model = tf.keras.models.Model(inputs=base_model.input, outputs=predictions)

    for idx, layer in enumerate(base_model.layers):
      if idx >= self.args.num_lock_layers:
        break
      layer.trainable = False

    model.summary()

    return model

  def create_data_generators(self):
    idg = tf.keras.preprocessing.image.ImageDataGenerator(
      preprocessing_function=tf.keras.applications.resnet50.preprocess_input,
      rotation_range=self.args.rotation_range,
      brightness_range=[1-self.args.brightness_range, 1+self.args.brightness_range],
      shear_range=self.args.shear_range,
      zoom_range=self.args.zoom_range,
      channel_shift_range=self.args.channel_shift_range,
      horizontal_flip=self.args.horizontal_flip
    )

    y_col = [str(x) for x in list(range(0, self.num_classes))]

    self.training_set = idg.flow_from_dataframe(
      self.training_dataframe,
      x_col="image_path",
      y_col=y_col,
      class_mode="raw",
      target_size=(self.args.image_w, self.args.image_h),
      batch_size=self.args.batch_size
    )

    self.validation_set = idg.flow_from_dataframe(
      self.validation_dataframe,
      x_col="image_path",
      y_col=y_col,
      class_mode="raw",
      target_size=(self.args.image_w, self.args.image_h),
      batch_size=self.args.batch_size
    )

    self.testing_set = idg.flow_from_dataframe(
      self.testing_dataframe,
      x_col="image_path",
      y_col=y_col,
      class_mode="raw",
      target_size=(self.args.image_w, self.args.image_h),
      batch_size=self.args.batch_size, shuffle=False)

  def train(self):
    mc_filename='mdl_best_wts.hdf5'
    mc_filepath=os.path.join(self.args.sm_model_dir, mc_filename)

    category_loss = tf.keras.losses.CategoricalCrossentropy()

    hierarchy_metric = HierarchyMetric(self.H)
    hierarchy_loss = hierarchical_loss(self.H)

    optimizer = tf.keras.optimizers.Adam(learning_rate=self.args.learning_rate)
    metrics = ["accuracy", hierarchy_metric]

    early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', mode='max', verbose=1, patience=2, min_delta=0.02)
    model_checkpoint=tf.keras.callbacks.ModelCheckpoint(mc_filepath, save_best_only=True, monitor='val_accuracy', mode='max')

    # first run it on categorical cross entropy
    self.model.compile(optimizer=optimizer, loss=category_loss, metrics=metrics)
    self.model.fit(self.training_set, validation_data=self.validation_set, epochs=self.args.num_epochs, callbacks=[early_stop, model_checkpoint])

    self.write_predictions('pred-pre-finetune.npy')

    # fine tune with hierarchical loss
    self.model.compile(optimizer=optimizer, loss=hierarchy_loss, metrics=metrics)
    self.model.fit(self.training_set, validation_data=self.validation_set, epochs=self.args.num_finetune_epochs, callbacks=[early_stop, model_checkpoint])

  def test(self):
    mc_filename='mdl_best_wts.hdf5'
    mc_filepath=os.path.join(self.args.sm_model_dir, mc_filename)

    self.model.load_weights(mc_filepath)

    print("Validation performance:")
    [validation_loss, validation_accuracy, validation_hierarchy, *_] = self.model.evaluate(self.validation_set) + [0]
    print("Validation-Loss: " + str(validation_loss) + ";")
    print("Validation-Accuracy: " + str(validation_accuracy) + ";")
    print("Validation-Hierarchy: " + str(validation_hierarchy) + ";")

    print("Testing performance:")
    [test_loss, test_accuracy, test_hierarchy, *_] = self.model.evaluate(self.testing_set) + [0]
    print("Test-Loss: " + str(test_loss) + ";")
    print("Test-Accuracy: " + str(test_accuracy) + ";")
    print("Test-Hierarchy: " + str(test_hierarchy) + ";")

    self.write_predictions('pred-post-finetune.npy')

  def save(self):
    self.model.save(os.path.join(self.args.sm_model_dir, '000000001'), save_format="tf")

  def write_predictions(self, pred_filename):
    self.testing_set.reset()
    probabilities = self.model.predict_generator(generator=self.testing_set)
    print("probabilities shape:", probabilities.shape)

    one_hot_true=[]

    for batch in self.testing_set:
      one_hot_true+=list(batch[1])
      if len(one_hot_true) >= len(probabilities):
        break
    one_hot_true=np.array(one_hot_true)
    print("one hot shape:", one_hot_true.shape)

    outputs = { "probabilities": probabilities, "one_hot_true": one_hot_true }

    if self.args.predictions_s3_path is not None:
      pred_filepath = os.path.join(self.args.sm_model_dir, pred_filename)
      np.save(pred_filepath, outputs)
      pred_s3_path = self.args.predictions_s3_path + "/" + pred_filename
      bucket, key = bucket_and_key_from_path(pred_s3_path)
      set_with_object_name(bucket, key, pred_filepath)

  def write_training_dataframe(self, training_df_filename):
    training_df_filepath = os.path.join(self.args.sm_model_dir, training_df_filename)
    self.training_dataframe.to_json(training_df_filepath)

if __name__ == "__main__":
  model = ResnetHierarchicalModel()
  model.train()
  model.test()
  if model.is_training_box:
    model.save()


import logging

#Setup for logging
logging.basicConfig(filename='app.log',
    format='%(asctime)s:%(levelname)s:%(message)s'
)

class PhoneEntry:
    '''
    Each contact is saved as a PhoneEntry in the PhoneBook where
    an entry has a name and a number
    '''
    def __init__(self, name, number):
        self.name = name
        self.number = number

    def display(self):
        print('{}={}'.format(self.name, self.number))

    @staticmethod
    def validateNumber(number):
        ans = False
        if(len(number) != 8):
            errorMsg = 'Number length should be equal to 8: ' + number
            logging.error(errorMsg)
        elif any(char not in '0123456789' for char in number):
            errorMsg = 'Numbers should only contain digits 0-9: ' + number
            logging.error(errorMsg)
        else:
            ans = True
        return ans

    @staticmethod
    def validateName(name):
        lowerCaseName = name.lower()
        ans = False
        if any(c not in 'abcdefghijklmnopqrstuvwxyz' for c in lowerCaseName):
            errorMsg = 'Name should only containg English alphabet: ' + lowerCaseName
            logging.error(errorMsg)
        else:
            ans = True
        return ans


import matplotlib.pyplot as plt
from sympy import *
def WBE(Z,N):
    "Returns a sympy version of Emperical Mass Formula"
   # Z=symbols('Z')
   # N=symbols('N')
    A=N+Z
    a1=15.49
    a2=17.23
    a3=.697
    a4=22.6
    equation=(a1*A-a2*A**(2.0/3.0)-a4*(N-Z)**2/A-a3*Z**2/A**(1/3.0))

    return equation

#def GetNeutronDrip(A,Z):
    #N=A-Z
Z=Symbol('Z')
N=Symbol('N')
Zs=list(range(1,120))
NeutronDrips=[]
for z in Zs:
    try:
        NeutronDrips.append(mpmath.findroot(lambda N: WBE(z,N),z*5))
    except:
        NeutronDrips.append(mpmath.findroot(lambda N: WBE(z,N),z*8))
    print(z)
NeutronMax=[]

Nn=list(range(100))
plt.plot(Zs,NeutronDrips)
plt.title("Neutron Drip Line")
plt.xlabel("Z Value")
plt.ylabel("N Value at Z Value")
WBs=[]
for i in Nn:
    WBs.append(WBE(8,i))
#plt.plot(Nn,WBs)
plt.show()
#print(WBE(Z,N))
#print(solve(WBE(Z,N),Z))
#print(mpmath.findroot(lambda N: WBE(8,N),8))
#mpmath.plot(lambda N: WBE(8,N),[0,20])
#mpmath.plot(lambda x: mpmath.exp(x)*mpmath.li(x), [1, 4])


from astropy.io import fits
from astropy.wcs import WCS
import numpy as np
from matplotlib import pyplot as plt
from typing import Tuple

def wcs_from_fits(filename: str, index: int = 0, extract_data: bool = False) -> WCS:
    """
    Extract WCS object (and data as arrays) from fits file.
    """
    w = None
    d = []
    with fits.open(filename) as hdul:
        w = WCS(hdul[index].header)
        if extract_data:
            for i in range(len(hdul)):
                d.append(np.array(hdul[i].data))
    if extract_data:
        return w, d
    return w

def figure_wcs(w: WCS, figsize: Tuple[int] = (10, 10)):
    """
    Create figure and axis with WCS projection.
    """
    fig = plt.figure(figsize=figsize)
    ax = fig.add_axes([0.1, 0.1, 0.8, 0.8], projection=w)
    ra = ax.coords[0]
    ra.set_format_unit('degree')
    return fig, ax

def show_fits(filename: str, fun: callable = None, data_index: int = 0,
              figsize: Tuple[int] = (10, 10), vmin=None, vmax=None, sparse: bool = False):
    """
    Show data from fits file with its WCS projection.
    """
    w, d = wcs_from_fits(filename, extract_data=True)
    if fun is not None:
        d = fun(d)
    fig, ax = figure_wcs(w, figsize=figsize)

    if not sparse:
        ax.imshow(d[data_index], cmap=plt.get_cmap('viridis'),
                  interpolation='nearest', vmin=vmin, vmax=vmax)
    else:
        ax.spy(d[data_index], cmap=plt.get_cmap('viridis'))
    return fig, ax