import cv2
import numpy as np
from math import pi, cos, sin

def conv1D(inSignal: np.ndarray, kernel1: np.ndarray) -> np.ndarray:
    """
    Convolve a 1-D array with a given kernel
    :param inSignal: 1-D array
    :param kernel1: 1-D array as a kernel
    :return: The convolved array
    """
    flip_kernel = kernel1[::-1]
    kernel_len = len(kernel1)
    signal_longer = inSignal

    # padding with zeros the inSignal arr:
    for i in range(kernel_len - 1):
        signal_longer = np.insert(signal_longer, 0, 0)
        signal_longer = np.append(signal_longer, 0)

    new_img = np.zeros(len(inSignal) + kernel_len - 1)
    for i in range(kernel_len + len(inSignal) - 1):
        new_img[i] = ((flip_kernel * signal_longer[i:kernel_len + i]).sum())

    return new_img

def conv2D(inImage: np.ndarray, kernel2: np.ndarray) -> np.ndarray:
    """
    Convolve a 2-D array with a given kernel
    :param inImage: 2D image
    :param kernel2: A kernel
    :return: The convolved image
    """
    flip_kernel = np.flipud(np.fliplr(kernel2))
    kernel_row = flip_kernel.shape[0]
    kernel_col = flip_kernel.shape[1]

    new_img = np.zeros_like(inImage)
    padded_img = padded_replicate(inImage, kernel_row, kernel_col)

    for x in range(inImage.shape[0]):
        for y in range(inImage.shape[1]):
            new_img[x, y] = (padded_img[x: x + kernel_col, y: y + kernel_row] * flip_kernel).sum()
            if flip_kernel.sum() != 0:
                new_img[x, y] /= flip_kernel.sum()

    return new_img

def padded_replicate(inImage: np.ndarray, kernel_row: int, kernel_col: int) -> np.ndarray:
    add_row = int((kernel_row - 1) / 2)
    add_col = int((kernel_col - 1) / 2)

    padded_img = np.zeros((inImage.shape[0] + kernel_row - 1, inImage.shape[1] + kernel_col - 1))
    padded_img[kernel_row - 2:-(kernel_row - 2), kernel_col - 2:-(kernel_col - 2)] = inImage

    up_row = padded_img[add_row]
    down_row = padded_img[-add_row - 1]
    padded_img_flip = np.flipud(np.fliplr(padded_img))
    left_col = padded_img_flip[add_col]
    right_col = padded_img_flip[-add_col - 1]

    for i in range(add_row):
        padded_img[i] = up_row
        padded_img[-i] = down_row
    for j in range(add_col):
        for row in range(padded_img.shape[0]):
            padded_img[row, j] = left_col[row]
            padded_img[row, -j] = right_col[-row]

    return padded_img

def convDerivative(inImage: np.ndarray) -> (np.ndarray, np.ndarray, np.ndarray, np.ndarray):
    """
    Calculate gradient of an image
    :param inImage: Grayscale image
    :return: (directions, magnitude, x_der, y_der)
    """
    kernel_x = np.array([[0, 0, 0], [1, 0, -1], [0, 0, 0]])
    kernel_y = np.array([[0, 1, 0], [0, 0, 0], [0, -1, 0]])

    # derivative by rows:
    Ix = cv2.filter2D(inImage, -1, kernel_x)  # , borderType=cv2.BORDER_REPLICATE

    # derivative by columns:
    Iy = cv2.filter2D(inImage, -1, kernel_y)

    eps = 0.0000000001
    magnitude = pow(Ix ** 2 + Iy ** 2, 0.5)
    direction = np.arctan(Iy / (Ix + eps))

    return direction, magnitude, Ix, Iy

# bonus:
def blurImage1(in_image: np.ndarray, kernel_size: np.ndarray) -> np.ndarray:
    """
    Blur an image using a Gaussian kernel
    :param in_image: Input image
    :param kernel_size: Kernel size
    :return: The Blurred image
    """
    size = kernel_size[0]
    sigma = 1
    x, y = np.mgrid[-size:size + 1, -size:size + 1]
    normal = 1 / (2.0 * np.pi * sigma ** 2)
    g = np.exp(-((x ** 2 + y ** 2) / (2.0 * sigma ** 2))) * normal
    in_image = cv2.filter2D(in_image, -1, g)
    return in_image

def blurImage2(in_image: np.ndarray, kernel_size: np.ndarray) -> np.ndarray:
    """
    Blur an image using a Gaussian kernel using OpenCV built-in functions
    :param in_image: Input image
    :param kernel_size: Kernel size
    :return: The Blurred image
    """
    gaussian_kernel = cv2.getGaussianKernel(kernel_size[0], sigma=0)
    out_img = cv2.filter2D(in_image, -1, gaussian_kernel)
    return out_img

def edgeDetectionSobel(img: np.ndarray, thresh: float = 0.7) -> (np.ndarray, np.ndarray):
    """
    Detects edges using the Sobel method
    :param img: Input image
    :param thresh: The minimum threshold for the edge response
    :return: opencv solution, my implementation
    """
    img = cv2.GaussianBlur(img, (3, 3), 0)

    Gx = np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]])
    Gy = np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]])
    thresh = thresh * 255

    smooth_x = cv2.filter2D(img, -1, Gx, borderType=cv2.BORDER_REPLICATE).astype(float)
    smooth_y = cv2.filter2D(img, -1, Gy, borderType=cv2.BORDER_REPLICATE).astype(float)

    my_sobel_image = np.sqrt(smooth_x ** 2 + smooth_y ** 2)

    sobel_x = cv2.Sobel(img, cv2.CV_64F, 1, 0, ksize=3)
    sobel_y = cv2.Sobel(img, cv2.CV_64F, 0, 1, ksize=3)
    cv_ans = np.sqrt(sobel_x ** 2 + sobel_y ** 2)

    change_by_thresh(my_sobel_image, thresh)
    change_by_thresh(cv_ans, thresh)

    return cv_ans, my_sobel_image

def change_by_thresh(img: np.ndarray, thresh: float = 0.7):
    for x in range(img.shape[0]):
        for y in range(img.shape[1]):
            if img[x, y] >= thresh:
                img[x, y] = 255
            else:
                img[x, y] = 0

def edgeDetectionZeroCrossingLOG(img: np.ndarray) -> np.ndarray:
    """
    Detecting edges using the "ZeroCrossingLOG" method
    :param img: Input image
    :return: :return: Edge matrix
    """

    # smooth the image with a Gaussian filter:
    img = img.astype(np.float)
    smooth_img = cv2.GaussianBlur(img, (3, 3), 0)

    # convolve the smoothed image with the Laplacian filter:
    kernel_laplacian = np.array([[0, 1, 0],
                                 [1, -4, 1],
                                 [0, 1, 0]])
    derive2_img = cv2.filter2D(smooth_img, -1, kernel=kernel_laplacian,
                               borderType=cv2.BORDER_REPLICATE)  # ddepth=cv2.CV_16S,

    # find the following patterns {-,0,+} or {-,+} :
    edge_img = np.zeros_like(derive2_img)
    row, col = img.shape
    for x in range(1, row - 1):
        for y in range(1, col - 1):
            wind = derive2_img[x - 1:x + 2, y - 1:y + 2]
            wind_max = wind.max()
            wind_min = wind.min()
            pixel = derive2_img[x, y]
            if pixel == 0.0 and wind_min <= 0.0 <= wind_max != wind_min:
                edge_img[x, y] = 255
            elif (wind_min <= 0.0 <= pixel != wind_min) or (pixel < 0.0 < wind_max != pixel):
                edge_img[x, y] = 255
            else:
                edge_img[x, y] = 0

    return edge_img

def edgeDetectionCanny(img: np.ndarray, thrs_1: float, thrs_2: float) -> (np.ndarray, np.ndarray):
    """
    Detecting edges usint "Canny Edge" method
    :param img: Input image
    :param thrs_1: T1
    :param thrs_2: T2
    :return: opencv solution, my implementation
    """
    # smooth the image with a Gaussian filter:
    smooth_img = cv2.GaussianBlur(img, (3, 3), 0)

    # Compute the partial derivatives Ix, Iy, and magnitude and direction of the gradient:
    directions, magnitude, Ix, Iy = convDerivative(smooth_img)
    angle = np.degrees(directions)
    magnitude = (magnitude / magnitude.max()) * 255

    # non maximum suppression:
    edge_img = np.zeros_like(magnitude)
    M, N = img.shape
    for i in range(1, M - 1):
        for j in range(1, N - 1):
            ni1 = ni2 = magnitude[i, j]
            # angle 0
            if (0 <= angle[i, j] < 22.5) or (157.5 <= angle[i, j] <= 180):
                ni1 = magnitude[i, j + 1]
                ni2 = magnitude[i, j - 1]
            # angle 45
            elif 22.5 <= angle[i, j] < 67.5:
                ni1 = magnitude[i + 1, j - 1]
                ni2 = magnitude[i - 1, j + 1]
            # angle 90
            elif 67.5 <= angle[i, j] < 112.5:
                ni1 = magnitude[i + 1, j]
                ni2 = magnitude[i - 1, j]
            # angle 135
            elif 112.5 <= angle[i, j] < 157.5:
                ni1 = magnitude[i - 1, j - 1]
                ni2 = magnitude[i + 1, j + 1]
            if (magnitude[i, j] >= ni1) and (magnitude[i, j] >= ni2):
                edge_img[i, j] = magnitude[i, j]
            else:
                edge_img[i, j] = 0

    # hysteresis:
    strong_edges = np.zeros_like(edge_img)
    for x in range(M):
        for y in range(N):
            if edge_img[x, y] > thrs_1:
                strong_edges[x, y] = edge_img[x, y]

    for x in range(M):
        for y in range(N):
            if edge_img[x, y] <= thrs_2:
                edge_img[x, y] = 0
            if thrs_2 < edge_img[x, y] <= thrs_1:
                if strong_edges[x - 1, y] == strong_edges[x + 1, y] == strong_edges[x, y - 1] == strong_edges[
                    x, y + 1] == strong_edges[x - 1, y - 1] == strong_edges[x + 1, y + 1] == strong_edges[
                    x + 1, y - 1] == strong_edges[x - 1, y + 1] == 0:
                    edge_img[x, y] = 0

    cv_ans = cv2.Canny(img, thrs_2, thrs_1)
    return cv_ans, edge_img

def houghCircle(img: np.ndarray, min_radius: float, max_radius: float) -> list:
    """
    Find Circles in an image using a Hough Transform algorithm extension
    :param img: Input image
    :param min_radius: Minimum circle radius
    :param max_radius: Maximum circle radius
    :return: A list containing the detected circles,
    [(x,y,radius),(x,y,radius),...]
    """

    canny_cv, canny_my = edgeDetectionCanny(img, 200, 100)
    edges = []

    for x in range(canny_cv.shape[0]):
        for y in range(canny_cv.shape[1]):
            if canny_cv[x, y] == 255:
                edges.append((x, y))

    thresh = 0.47  # at least 47% of the pixels of a circle must be detected
    steps = 100  # number of samples from each circle

    points = []
    for r in range(min_radius, max_radius + 1):
        for t in range(steps):
            alpha = 2 * pi * t / steps
            x = int(r * cos(alpha))
            y = int(r * sin(alpha))
            points.append((x, y, r))

    temp_circles = {}  # dict{circle center, radius: counter}
    for x, y in edges:  # iterate the pixels of the edges:
        for dx, dy, r in points:
            b = x - dx
            a = y - dy
            count = temp_circles.get((a, b, r))
            if count is None:
                count = 0
            temp_circles[(a, b, r)] = count + 1

    # now add the appropriate circles to the ans list:
    circles = []
    sorted_temp = sorted(temp_circles.items(), key=lambda i: -i[1])
    for circle, counter in sorted_temp:
        x, y, r = circle
        # once a circle has been selected, we reject all the circles whose center is inside that circle
        if counter / steps >= thresh and all((x - xc) ** 2 + (y - yc) ** 2 > rc ** 2 for xc, yc, rc in circles):
            circles.append((x, y, r))

    return circles


# gesture control python program for controlling certain functions in windows pc
# Code by BalaAppu
# Website: www.electronicshub.org

import serial                                      # add Serial library for serial communication
import pyautogui                                   # add pyautogui library for programmatically controlling the mouse and keyboard.

Arduino_Serial = serial.Serial('com3',9600)       # Initialize serial and Create Serial port object called Arduino_Serial

while 1:
    print("line received")
    incoming_data = str (Arduino_Serial.readline())
    print("line read")
    print(incoming_data)

    if 'next' in incoming_data:                    # if incoming data is 'next'
        pyautogui.hotkey('ctrl', 'pgdn')           # perform "ctrl+pgdn" operation which moves to the next tab
        print("h")

    if 'previous' in incoming_data:                # if incoming data is 'previous'
        pyautogui.hotkey('ctrl', 'pgup')           # perform "ctrl+pgup" operation which moves to the previous tab
        print("h")
    if 'down' in incoming_data:                    # if incoming data is 'down'
        #pyautogui.press('down')                   # performs "down arrow" operation which scrolls down the page
        pyautogui.press('r')
        print("h")
    if 'up' in incoming_data:                      # if incoming data is 'up'
        #pyautogui.press('up')
        # performs "up arrow" operation which scrolls up the page
        print("h")
        pyautogui.press('k')

    if 'change' in incoming_data:                  # if incoming data is 'change'
        pyautogui.keyDown('alt')                   # performs "alt+tab" operation which switches the tab
        pyautogui.press('tab')
        pyautogui.keyUp('alt')
        print("h")
    incoming_data = "";                            # clears the data


from django.contrib.auth.models import AbstractUser
from django.db import models

class CustomUser(AbstractUser):
    # Add additional fields here

    class Meta():
        permissions = (
            ('G1', 'Pode ver x'),
            ('G2', 'Pode ver y'),
            ('G3', 'Pode ver z')
        )

    def __str__(self):
        return self.email


import bcsv, binascii, csv, html, importlib, json, os, os.path, sys

in_path = sys.argv[1]
out_dir = sys.argv[2]
specs_file = sys.argv[3]
specs = importlib.import_module(specs_file.replace('.py', ''))

csv_dir = out_dir + '/csv'
json_dir = out_dir + '/json'
html_dir = out_dir + '/html'

if not os.path.exists(csv_dir):
	os.makedirs(csv_dir)
if not os.path.exists(json_dir):
	os.makedirs(json_dir)
if not os.path.exists(html_dir):
	os.makedirs(html_dir)

for filename, row_class in specs.lookup.items():
	bits = []
	bits.append('<!DOCTYPE html>')
	bits.append('')
	bits.append('')
	objs = []

	bits.append('
%s
' % filename)

	b = bcsv.File(row_class)
	with open('%s/%s' % (in_path, filename), 'rb') as f:
		b.load(f.read())

	csvw = csv.writer(open('%s/%s.csv' % (csv_dir, filename.replace('.bcsv', '')), 'w'))

	bits.append('
')
	bits.append('
')
	for field in row_class.fields():
		bits.append('%s	' % field)
	csvw.writerow(row_class.fields())
	bits.append('
')
	for row in b.rows:
		bits.append('
')
		obj = {}
		crow = []
		for field in row_class.fields():
			value = getattr(row, field)
			if isinstance(value, bytes):
				htmlValue = binascii.hexlify(value).decode('ascii')
				dataValue = htmlValue
			elif isinstance(value, tuple):
				htmlValue = "%s" % (html.escape(value[1], True), html.escape(value[0]))
				dataValue = value
			else:
				htmlValue = repr(value)
				dataValue = htmlValue
			bits.append('%s	' % htmlValue)
			crow.append(dataValue)
			obj[field] = dataValue
		bits.append('
')
		objs.append(obj)
		csvw.writerow(crow)
	bits.append('
')

	bits.append('')
	bits.append('')

	with open('%s/%s.html' % (html_dir, filename.replace('.bcsv', '')), 'w') as f:
		for line in bits:
			f.write(line)
			f.write('\n')
	with open('%s/%s.json' % (json_dir, filename.replace('.bcsv', '')), 'w') as f:
		json.dump(objs, f, indent=4)


import os
import re
from PIL import Image
import numpy as np
# from .common.log import logger
import torch
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms as transforms
from torchvision.datasets import Caltech256

class Caltech256AnomalyDataset(Caltech256):
        def __init__(self, root, inliners, outliners, transform=None, target_transform=None, download=False):
            super(Caltech256AnomalyDataset, self).__init__(root, transform, target_transform, download)

            def is_normal(i):
                return i in inliners
            def is_abnormal(i):
                return i in outliners

            self.index = []
            self.y = []
            for (i, c) in enumerate(self.categories):
                n = len(list(
                    filter(
                        lambda p: p.endswith('jpg'),
                        os.listdir(os.path.join(self.root, "256_ObjectCategories", c))
                    )
                ))
                self.index.extend(range(1, n + 1))
                self.y.extend(n * [i])

            self._imgs_path = []
            self._labels = []

            self._train_normal_imgs_path = []
            self._test_normal_imgs_path = []
            self._abnormal_imgs_path = []

            self._class_num = np.zeros((257, ))
            for item in self.y:
                self._class_num[item] += 1
            # print(np.argsort(self._class_num)[::-1])

            self._normal_samples_num, self._abnormal_samples_num = 0, 0

            for index in range(len(self.y)):
                img_path = os.path.join(self.root,
                                        "256_ObjectCategories",
                                        self.categories[self.y[index]],
                                        "{:03d}_{:04d}.jpg".format(self.y[index] + 1, self.index[index]))
                label = self.y[index]

                if is_normal(label):
                    self._imgs_path += [img_path]
                    if self.index[index] < self._class_num[self.y[index]] * 0.8:
                        self._train_normal_imgs_path += [img_path]
                    else:
                        self._test_normal_imgs_path += [img_path]
                    self._labels += [0]
                    self._normal_samples_num += 1
                if is_abnormal(label):
                    self._imgs_path += [img_path]
                    self._abnormal_imgs_path += [img_path]
                    self._labels += [1]
                    self._abnormal_samples_num += 1

        def __getitem__(self, index):
            img = Image.open(self._imgs_path[index])

            img_np = np.array(img)
            if len(img_np.shape) < 3:
                t = np.empty((3, ) + img_np.shape, dtype=img_np.dtype)
                t[0] = np.array(img_np)
                t[1] = np.array(img_np)
                t[2] = np.array(img_np)
                img = Image.fromarray(t.transpose(1, 2, 0))

            target = self._labels[index]

            if self.transform is not None:
                img = self.transform(img)

            if self.target_transform is not None:
                target = self.target_transform(target)

            return img, target

        def __len__(self):
            return len(self._imgs_path)

def gpnd_caltech256_dataloader(dataroot='/data/datasets/', inliners=[250, 144, 252, 22, 67], outliners=[256], train_batch_size=64, test_batch_size=64, image_size=32, num_workers=8):

    transform = transforms.Compose([
        transforms.Resize([image_size, image_size]),
        transforms.ToTensor(),
        # transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
        # transforms.Lambda(lambda img: img / 255.)
    ])
    dataset = Caltech256AnomalyDataset(os.path.join(dataroot, './Caltech-256/data/'), inliners, outliners, transform=transform)

    train_normal_imgs_path, test_normal_imgs_path = dataset._train_normal_imgs_path, dataset._test_normal_imgs_path
    abnormal_imgs_path = dataset._abnormal_imgs_path

    if len(test_normal_imgs_path) > len(abnormal_imgs_path):
        test_normal_imgs_path = test_normal_imgs_path[:len(abnormal_imgs_path)]
    else:
        abnormal_imgs_path = abnormal_imgs_path[:len(test_normal_imgs_path)]

    train_set = Caltech256AnomalyDataset(os.path.join(dataroot, './Caltech-256/data/'), inliners, outliners, transform=transform)
    train_set._imgs_path = train_normal_imgs_path
    train_set._labels = [0 for _ in train_normal_imgs_path]

    test_set = Caltech256AnomalyDataset(os.path.join(dataroot, './Caltech-256/data/'), inliners, outliners, transform=transform)
    test_set._imgs_path = test_normal_imgs_path + abnormal_imgs_path
    test_set._labels = [0 for _ in test_normal_imgs_path] + [1 for _ in abnormal_imgs_path]

    print('train normal sample {}, test normal sample {}, '
                'test abnormal sample {}'.format(len(train_normal_imgs_path), len(test_normal_imgs_path), len(abnormal_imgs_path)))

    train_loader = DataLoader(train_set, batch_size=train_batch_size, shuffle=True, num_workers=num_workers, pin_memory=True, drop_last=True)
    test_loader = DataLoader(test_set, batch_size=test_batch_size, shuffle=False, num_workers=num_workers, pin_memory=True, drop_last=True)

    return train_loader, test_loader


from util import Save, Load
from nn import CheckGrad, Train
from cnn import InitCNN, CNNForward, CNNBackward, CNNUpdate
from nn_plot import save_figure, evaluate_model
import numpy as np
import matplotlib.pyplot as plt

# Turn interactive plotting off
plt.ioff()

def cnn_with_args(eps=0.1,
                  momentum=0.5,
                  num_epochs=30,
                  filter_size=5,
                  num_filters_1=8,
                  num_filters_2=16,
                  batch_size=100):
    # Input-output dimensions.
    num_channels = 1
    num_outputs = 7

    # Initialize model.
    model = InitCNN(num_channels, filter_size, num_filters_1, num_filters_2, num_outputs)

    # Check gradient implementation.
    print('Checking gradients...')
    x = np.random.rand(10, 48, 48, 1) * 0.1
    CheckGrad(model, CNNForward, CNNBackward, 'W3', x)
    CheckGrad(model, CNNForward, CNNBackward, 'b3', x)
    CheckGrad(model, CNNForward, CNNBackward, 'W2', x)
    CheckGrad(model, CNNForward, CNNBackward, 'b2', x)
    CheckGrad(model, CNNForward, CNNBackward, 'W1', x)
    CheckGrad(model, CNNForward, CNNBackward, 'b1', x)

    # Train model.
    model, stats = Train(model, CNNForward, CNNBackward, CNNUpdate, eps, momentum, num_epochs, batch_size)

    # Construct file name
    fname = str(eps) + '_' + str(momentum) + '_' + str(num_epochs) + '_' + str(filter_size) + '_' + \
            str(num_filters_1) + '_' + str(num_filters_2) + '_' + str(batch_size)

    # Save the training statistics.
    Save('cnn_model/' + fname, model)
    Save('cnn_stats/' + fname, stats)

def plot_figures(stats_fname):
    stats = Load(stats_fname)
    train_ce_list = stats['train_ce']
    valid_ce_list = stats['valid_ce']
    train_acc_list = stats['train_acc']
    valid_acc_list = stats['valid_acc']
    figure_path = get_cnn_path(stats_fname)
    save_figure(train_ce_list, valid_ce_list, 'Cross Entropy', figure_path)
    save_figure(train_acc_list, valid_acc_list, 'Accuracy', figure_path)
    print('Figures saved to ' + figure_path)

def get_cnn_path(stats_fname):
    sep = stats_fname.find('/')
    figure_name = stats_fname[sep + 1:-4]
    return "cnn_figure/" + figure_name

def plot_first_filters(model_fname):
    model = Load(model_fname)
    W1 = model['W1']
    plt.clf()
    for i in xrange(W1.shape[3]):
        plt.subplot(1, W1.shape[3], i + 1)
        plt.imshow(W1[:, :, 0, i], cmap=plt.cm.gray)
    plt.savefig('visualization/cnn_first_layer_filters.png')
    print("cnn_first_layer_filters.png saved")

def cnn_main(*args):
    # Run CNN and save model, stats
    cnn_with_args(*args)
    # Get file name based on arguments
    fname = str(args[0]) + '_' + str(args[1]) + '_' + str(args[2]) + '_' + str(args[3]) + '_' + \
            str(args[4]) + '_' + str(args[5]) + '_' + str(args[6])
    # Plot and save figure based on model, stats
    plot_figures('cnn_stats/' + fname + '.npz')
    # Evaluate model and save results
    evaluate_model('cnn_model/' + fname + '.npz', 'cnn_result/' + fname, CNNForward, args[6])
    # Plot the first layer filters
    # plot_first_filters('cnn_model/' + fname + '.npz')

if __name__ == '__main__':
    cnn_main(0.1,  # eps
             0.0,  # momentum
             30,  # num_epochs
             5,  # filter_size
             8,  # num_filters_1
             16,  # num_filters_1
             100)  # batch_size


from pytsa.models import dquad_2sphere
from pytsa.sampler import setup_sampler
from scipy.stats import uniform, chi2, expon

# Example sampler setup file for double quadratic model.

# Initialize sampler object with the model (imported at the tope)
# and a name (for identification)
setup = setup_sampler.SamplerMethods(dquad_2sphere, "dquad_2sphere_example")

# Setup analysis hyper parameters
setup.set_analysis_params(tols=[1e-5, 1e-5])

# Setup field initial conditions; here we sample on the domain [-20, 20]
setup.set_field(0, 1, method=uniform(-20, 40))

# Setup dot field initial conditions; here we just use slow roll values
setup.set_dot_field(0, 1, method="sr")

# Setup parameter sampling; here we sample each mass parameter from an
# exponential distribution
setup.set_param(0, 1, method=expon(scale=0.01))

# We also sample the metric parameter, R. Here we sample from the chi2
# distribution.
# We chose this dist. since it is positive definite and the metric depends
# on R^2, so we don't over sample.
setup.set_param(2, method=chi2(3))

# Finally, build the sampler
setup.build_sampler()


import PySimpleGUI as sg
import download

sg.theme("DarkGrey12")

def tela_principal():
    layout = [
        [sg.Text("YOUTUBE DOWNLOAD")],
        [sg.Button("Baixar Músicas"),sg.Button("Baixar Vídeos")]
    ]
    return sg.Window("Youtube Download",layout,finalize=True)

def tela_musica():
    layout = [
        [sg.Text("Música Download")],
        [sg.Text("Link do vídeo:"),sg.InputText("",key="link_video")],
        [sg.Text("Diretorio da Pasta",key="text_diretorio"),sg.FolderBrowse(key="diretorio")],
        [sg.Button("Baixar Música"),sg.Button("Voltar")]
    ]
    return sg.Window("Música Download",layout,finalize=True)

def tela_video():
    layout = [
        [sg.Text("Vídeo Download")],
        [sg.Text("Link do vídeo:"),sg.InputText("",key="link_video1")],
        [sg.Text("Diretorio da Pasta",key="text_diretorio1"),sg.FolderBrowse(key="diretorio1")],
        [sg.Button("Baixar Vídeo"),sg.Button("Voltar")]
    ]
    return sg.Window("Vídeo Download",layout,finalize=True)

janela_principal = tela_principal()
janela_musica = None
janela_video = None

while True:
    janelas,eventos,valores = sg.read_all_windows()
    if eventos == sg.WINDOW_CLOSED:
        break

    if janelas == janela_principal and eventos == "Baixar Músicas":
        janela_principal.hide()
        janela_musica = tela_musica()

    if janelas == janela_musica and eventos == "Voltar":
        janela_musica.hide()
        janela_principal.un_hide()

    if janelas == janela_principal and eventos == "Baixar Vídeos":
        janela_principal.hide()
        janela_video = tela_video()

    if janelas == janela_video and eventos == "Voltar":
        janela_video.hide()
        janela_principal.un_hide()

    if janelas == janela_musica and eventos == "Baixar Música" and valores["diretorio"] == "":
        sg.Popup("Por Favor informe o local onde o arquivo vai ser salvo")

    if janelas == janela_musica and eventos == "Baixar Música" and valores["diretorio"] != "":
        link_video = valores["link_video"]
        pasta_arquivo = valores["diretorio"]
        sg.Popup("Vai aparecer que o youtube download não está respondendo. Isso é porque o download está sendo feito.Por favor click em OK e espere aparecer um Popup dizendo que o download está concluido")
        download.download_musica(link_video,pasta_arquivo)
        sg.Popup("Download Concluido")
        janela_musica["text_diretorio"].update("Por Favor informe o diretorio novamente")

    if janelas == janela_video and eventos == "Baixar Vídeo" and valores["diretorio1"] == "":
        sg.Popup("Por Favor informe o local onde o arquivo vai ser salvo")

    if janelas == janela_video and eventos == "Baixar Vídeo" and valores["diretorio1"] != "":
        link_video = valores["link_video1"]
        pasta_arquivo = valores["diretorio1"]
        sg.Popup("Vai aparecer que o youtube download não está respondendo. Isso é porque o download está sendo feito.Por favor click em OK e espere aparecer um Popup dizendo que o download está concluido")
        download.download_video(link_video,pasta_arquivo)
        sg.Popup("Download Concluido")
        janela_video["text_diretorio1"].update("Por Favor informe o diretorio novamente")


import random
import math
from pathlib import Path
from typing import Tuple, List, Optional

import torch
import torch.nn as nn
import numpy as np
import hydra
from omegaconf import DictConfig
import audiomentations as aud
import youtokentome as yttm

import core

def get_bpe_path(cfg, original_path: Optional[Path] = None):
    path = Path(cfg.common.files_path, cfg.bpe.model_path)
    return Path(original_path / path if original_path else hydra.utils.to_absolute_path(path))

def get_lengths_path(cfg, lengths_type: str, original_path: Optional[Path] = None):
    """
    :param lengths_type: either "waveform" or "utterance" for waveform and utterance lengths respectively
    :param original_path: Path object with original working directory. It is used when hydra.utils.to_absolute_path(path)
    is not available (in subprocess)
    """
    path = Path(cfg.common.files_path, cfg.lengths.lengths_filename.format(lengths_type))
    return Path(original_path / path if original_path else hydra.utils.to_absolute_path(path))

def fix_seeds(seed=1337):
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    np.random.seed(seed)
    random.seed(seed)
    torch.backends.cudnn.enabled = False
    torch.backends.cudnn.deterministic = True

def get_waveform_transforms(transforms: DictConfig, device=None):
    """
    get all necessary transforms from config
    :param transforms: transforms from config
    :param device: device if transforms need to be ported to device
    :return: transforms composed into aud.Compose
    """
    if transforms is None:
        return None
    if device is not None:
        return aud.Compose([
            hydra.utils.instantiate(transform).to(device)
            for transform in transforms
        ])
    else:
        return aud.Compose([
            hydra.utils.instantiate(transform)
            for transform in transforms
        ])

def pad_collate(batch: List[Tuple]) -> Tuple[torch.Tensor, torch.Tensor, List[torch.Tensor],
                                             torch.Tensor, List[int], torch.Tensor]:
    """
    Collator that transforms batch into tuple of lists and pads spectrograms and utterances
    :param batch: List of tuples of batch components
    :return: Tuple of tensors and lists
    """
    waveforms, spectrograms, sample_rates, utterances = zip(*batch)

    input_lengths = torch.tensor([math.ceil(spectrogram.shape[1] / 2) for spectrogram in spectrograms])
    target_lengths = torch.tensor([utterance.shape[0] for utterance in utterances])

    utterances = nn.utils.rnn.pad_sequence(utterances, True, 0)
    spectrograms = nn.utils.rnn.pad_sequence([s.transpose(0, 1) for s in spectrograms], True, 0)
    spectrograms = spectrograms.transpose(1, 2)

    return input_lengths, target_lengths, waveforms, spectrograms, sample_rates, utterances

def ctc_decode(seq: torch.Tensor, blank_idx: int):
    """
    Simple ctc decoding without beam search
    :param seq: torch tensor with tokens of shape (n_labels, len)
    :param blank_idx: index of blank character
    :return: torch tensor with ctc decoded tokens
    """
    seq = seq.argmax(dim=0)
    seq = torch.cat([seq[[0]], seq[1:][(seq[1:] - seq[:-1]).bool()]])  # Remove consecutive duplicates
    seq = seq[seq != blank_idx]  # Remove blanks
    return seq

def get_texts(utterances, logprobs, bpe: core.transforms.BPETransform, blank_idx) -> Tuple[List[str], List[str]]:
    """
    Get utterances and logprobs, ctc decode, remove padding, bpe decode and return texts
    :param utterances: bpe encoded utterances from dataset
    :param logprobs: output from network
    :param bpe: BPE encoder object for encoding (duh)
    :param blank_idx: index of ctc blank value
    :return: two lists of strings for true and predicted texts
    """
    true_tokens = [np.trim_zeros(utt.tolist()) for utt in utterances]
    pred_tokens = [np.trim_zeros(ctc_decode(prob, blank_idx).tolist()) for prob in logprobs]
    true_texts = [coded[0] if len(coded) > 0 else "" for coded in [bpe.bpe.decode(true) for true in true_tokens]]
    pred_texts = [coded[0] if len(coded) > 0 else "" for coded in [bpe.bpe.decode(pred) for pred in pred_tokens]]
    return true_texts, pred_texts


# 133. Clone Graph
# Medium
# https://leetcode.com/problems/clone-graph/

"""
# Definition for a Node.
class Node:
    def __init__(self, val = 0, neighbors = None):
        self.val = val
        self.neighbors = neighbors if neighbors is not None else []
"""

# Larry, https://www.youtube.com/watch?v=DcL9uvLtOAE
class Solution:
    def cloneGraph(self, root: 'Node') -> 'Node':
        if root is None:
            return None

        queue = collections.deque()
        seen = set()

        def maybe_enqueue(node):
            if node.val not in seen:
                seen.add(node.val)
                queue.append(node)

        dupe = {}
        def getter(node):
            if node.val not in dupe:
                dupe[node.val] = Node(node.val)
            return dupe[node.val]

        first = getter(root)
        maybe_enqueue(root)
        while len(queue) > 0:
            current = queue.popleft()
            current_dupe = getter(current)

            for neighbor in current.neighbors:
                current_dupe.neighbors.append(getter(neighbor))

            for neighbor in current.neighbors:
                maybe_enqueue(neighbor)

        return first
# 02/23/2022 17:04

# 7/25/2021
# Pathrise's solution in Graph Preworkshop
from collections import deque

class Solution:
    def cloneGraph(self, node: 'Node') -> 'Node':
        if not node: return None
        root = node
        clone = {}

        queue = deque([node])
        all_nodes = set([node])
        while queue:
            node = queue.popleft()
            for neighbor in node.neighbors:
                if neighbor not in all_nodes:
                    queue.append(neighbor)
                    all_nodes.add(neighbor)

        for node in all_nodes:
            clone[node] = Node(node.val)

        for node in all_nodes:
            for neighbor in node.neighbors:
                clone[node].neighbors.append(clone[neighbor])

        return clone[root]


# -*- coding: utf-8 -*-
__author__ = 'Yoon Ho Han, Kevin Zhang'
__email__ = 'yhhan@ucsd.edu, ktzhang@ucsd.edu'

from collections import deque

def ac3(csp, arcs=None):
    """Executes the AC3 or the MAC (p.218 of the textbook) algorithms.

    If the parameter 'arcs' is None, then this method executes AC3 - that is, it will check the arc consistency
    for all arcs in the CSP.  Otherwise, this method starts with only the arcs present in the 'arcs' parameter
    in the queue.

    Note that the current domain of each variable can be retrieved by 'variable.domains'.

    This method returns True if the arc consistency check succeeds, and False otherwise."""

    queue_arcs = deque(arcs if arcs is not None else csp.constraints.arcs())

    # loop through the arcs
    while (queue_arcs):
        arc = queue_arcs.popleft();
        # check if arc needs to be revised
        if (revise(csp, arc[0], arc[1])):
            # if no more in domain, arc consistency fails
            if (len(arc[0].domain) == 0):
                return False;
            else :
                # loop through all arcs with arc[0]
                for neighbor in csp.constraints[arc[0]].arcs():
                    # skip if neighbor is the same arc
                    if (neighbor[0] == arc[0] or neighbor[1] == arc[0]):
                        continue;
                    # add arc to queue if current arc has been revised
                    queue_arcs.append(neighbor);
    return True;

def revise(csp, xi, xj):
    revised = False;
    # loop through domain of xi
    for xVal in xi.domain:
        found = False;
        # loop through domain of xj
        for yVal in xj.domain:
            # check if domain for xj satisfies constraints
            satisfied = True;
            for constraint in csp.constraints[xi, xj]:
                if (not constraint.is_satisfied(xVal, yVal)):
                    # satisfied is false if value of xj doesn't satisfy any of the constraints
                    satisfied = False;
                    break;
            # if any value of domain for xj is consistent, don't have to check any other values
            if (satisfied):
                found = True;
                break;
        # none of the domain of xj satisfies constraint with xi
        if (not found):
            xi.domain.remove(xVal);
            revised = True;
    return revised;


import logging
import os

import config

def setup_logger(logger_name, log_file, level=logging.INFO):
    """
    Factory method for creating loggers
    :param logger_name:
    :param log_file:
    :param level:
    :return:
    """
    logger = logging.getLogger(logger_name)
    formatter = logging.Formatter(u'%(asctime)s %(message)s')
    fileHandler = logging.FileHandler(log_file, mode='a')
    fileHandler.setFormatter(formatter)
    streamHandler = logging.StreamHandler()
    streamHandler.setFormatter(formatter)

    logger.setLevel(level)
    logger.addHandler(fileHandler)
    # logger.addHandler(streamHandler)

    return logger

if not os.path.exists(config.data_dir):
    os.makedirs(config.data_dir)

error_log_path = os.path.join(config.data_dir, f'{config.bot_info["channel"]}_error-log.txt')
event_log_path = os.path.join(config.data_dir, f'{config.bot_info["channel"]}_event-log.txt')

error_logger = setup_logger('error_logger', error_log_path, level=logging.WARNING)
event_logger = setup_logger('event_logger', event_log_path, level=logging.INFO)


import cv2
import numpy as np

# CTR digit constants
DIGIT_WIDTH = 13
DIGIT_HEIGHT = 22
DIGIT_MULTIPLIER = 3
DIGIT_SIZE = (DIGIT_WIDTH, DIGIT_HEIGHT)
DIGIT_SIZE_HIGH = (DIGIT_WIDTH * DIGIT_MULTIPLIER, DIGIT_HEIGHT * DIGIT_MULTIPLIER)
KERNEL = np.ones((4, 4), np.uint8)

# Color constants
BLACK = 0
WHITE = 255
GRAY = 127

def detect_edges(img):
    ''' Detects edges from pictures of CTR digits.
        @param img: image of the CTR digit '''

    height, width = img.shape
    # Get the average color of the grayscale image
    color_mean = np.mean(img)
    color_mean *= 1.05

    # Loop every pixel
    for i in range(height):
        for j in range(width):
            # Select the darkest pixels as edges and color them black
            if img[i, j] >= color_mean:
                img[i, j] = WHITE
            else:
                img[i, j] = BLACK

    return img

def add_border(img):
    ''' Add a black border surrounding the digit, removing noise.
        @param img: image of the CTR digit '''

    height, width = img.shape

    # Calculating the mean distance of the border to the
    # black pixels surrounding the number
    left_mean = 0
    right_mean = 0
    # Number of valid black pixels detected
    right_row = 0
    left_row = 0

    # Looping every height
    # since we're starting with the right and left border
    for i in range(height):
        visited_left = False
        visited_right = False

        # Looping half of the width pixels to prevent
        # detecting black pixels in case of the image having holes
        # surrounding the edges of the number
        for j in range(width // 2):

            # If a black pixel is detected, j is its distance to the border
            # Lock the loop from detecting further pixels
            # by toogling the visited boolean
            if not visited_right and img[i, width - 1 - j] == BLACK:
                right_mean += j
                right_row += 1
                visited_right = True

            if not visited_left and img[i, j] == BLACK:
                left_mean += j
                left_row += 1
                visited_left = True

            # If both blacks were found, we can go to the next height
            if visited_left and visited_right:
                break

    # Same code as before, but now for the up and down sides
    up_mean = 0
    up_row = 0
    down_mean = 0
    down_row = 0
    for i in range(width):
        visited_up = False
        visited_down = False

        for j in range(height // 2):
            if not visited_down and img[height - 1 - j, i] == BLACK:
                down_mean += j
                down_row += 1
                visited_down = True

            if not visited_up and img[j, i] == BLACK:
                up_mean += j
                up_row += 1
                visited_up = True

            if visited_down and visited_up:
                break

    # We're safe to assume that the entire rectangle from the border
    # to the average value of the first black pixel is noise.
    # Thus, we remove it by making the entire rectangle be black.

    # Left and right sides
    for i in range(height):
        for j in range(max(right_mean // right_row, 1)):
            img[i, width - 1 - j] = BLACK
        for j in range(max(left_mean // left_row, 1)):
            img[i, j] = BLACK

    # Up and down sides
    for i in range(width):
        for j in range(max(down_mean // down_row, 1)):
            img[height - 1 - j, i] = BLACK
        for j in range(max(up_mean // up_row, 1)):
            img[j, i] = BLACK

    return img

def paint_colored_area(img, pixel, color_area, color_paint, diagonal=True):
    ''' Gets the colored area in a picture starting at pixel.
        @param img: image that you want to detect the area.
        @param pixel: tuple of the coordinates starting pixel.
        @param color_area: color of the areas that you want to find.
        @param color_paint: color that you want to paint the area.
        @param diagonal: set to True if you want to consider pixels in
        surrounding the diagonals as neighbors '''

    # Coordinates of the starting pixel
    (i, j) = pixel
    # Stack holds every single pixel that could have neighbors
    stack = []
    # Every pixel coordinates found in the area
    pixels = []

    # Paint the start pixel in order to avoid it being detected again
    img[i, j] = color_paint
    # Add start pixel to the stack, so you can start searching for neighbors
    stack.append((i, j))

    # While there are unverified pixels left
    while len(stack) > 0:
        # Get the next element to search for neighbors
        (i, j) = stack.pop(0)
        # Insert this pixel in the total
        # number of pixels in this area
        pixels.append((i, j))

        # Find more neighbors and store them in the stack
        if img[i - 1, j] == color_area:
            img[i - 1, j] = color_paint
            stack.append((i - 1, j))

        if img[i + 1, j] == color_area:
            img[i + 1, j] = color_paint
            stack.append((i + 1, j))

        if img[i, j - 1] == color_area:
            img[i, j - 1] = color_paint
            stack.append((i, j - 1))

        if img[i, j + 1] == color_area:
            img[i, j + 1] = color_paint
            stack.append((i, j + 1))

        if diagonal:
            if img[i + 1, j + 1] == color_area:
                img[i + 1, j + 1] = color_paint
                stack.append((i + 1, j + 1))

            if img[i - 1, j + 1] == color_area:
                img[i - 1, j + 1] = color_paint
                stack.append((i - 1, j + 1))

            if img[i + 1, j - 1] == color_area:
                img[i + 1, j - 1] = color_paint
                stack.append((i + 1, j - 1))

            if img[i - 1, j - 1] == color_area:
                img[i - 1, j - 1] = color_paint
                stack.append((i - 1, j - 1))

    return pixels

def detect_colored_areas(img, color_area, color_paint, color_rejected, size=150, diagonal=True):
    ''' Detects colored areas in a picture. This function doesn't check the pixels
        in the border of the picture.
        @param img: image that you want to extract the areas.
        @param color_area: color of the areas that you want to find.
        @param color_paint: color that you want to paint the area.
        @param color_rejected: color to paint every area that has color_area, but
        is less than your threshold size.
        @param size: minimum number of pixels required to be considered an area.
        @param diagonal: set to True if you want to consider pixels in
        surrounding the diagonals as neighbors '''

    height, width = img.shape
    # If the digit size is small, use a smaller threshold
    if height == DIGIT_HEIGHT:
        size = 20

    areas = []
    # Loop every single pixel in the image
    for i in range(1, height):
        for j in range(1, width):
            # If a good pixel was found
            if img[i, j] == color_area:
                # Detect the pixels in that area
                pixels = paint_colored_area(img, (i, j), color_area, color_paint, diagonal)

                # Once you have the area, if the size of it is less
                # than your threshold, then paint every pixel as rejected
                if len(pixels) < size:
                    for (pix_i, pix_j) in pixels:
                        img[pix_i, pix_j] = color_rejected
                else:
                    # Append pixels of that area to the total areas found
                    areas.append(pixels)

    return areas

def remove_noisy_areas(img):
    ''' Removes small white pixel areas from the picture.
        @param img: image of the CTR digit.
        @param size: size of the area. '''

    height, width = img.shape

    # Remove the smaller areas of the image. Default threshold is 150 pixels.
    # Areas too small usually are just noise, so removing helps cleaning up the image,
    # which helps predicting the right number.
    areas = detect_colored_areas(img, WHITE, GRAY, BLACK)
    for area in areas:
        paint_colored_area(img, area[0], GRAY, WHITE)

    # Dilate each white part of the picture. This helps witH glueing parts of the number
    # which are separated by a few black pixels, and also removes a bit of remaining noise.
    img = cv2.dilate(img, KERNEL, iterations=1)

    # If the height is the height of a digit that will be used for prediction
    if height != DIGIT_HEIGHT:

        # Adding a black border to the image
        for i in range(width):
            img[0, i] = BLACK
            img[height - 1, i] = BLACK
        for i in range(height):
            img[i, 0] = BLACK
            img[i, width - 1] = BLACK

        # Our dataset consists of CTR digits, which are unique and never changes.
        # However, depending on the capture card quality, the alligment of the Y axis
        # does change from video to video. So, in order to fix it and help getting a more
        # accurate prediction, I'm alligning the number with the top of the picture.

        # Variable to store the distance from the first WHITE pixel in the Y axis
        fix_height = 0
        # Looping every pixel, ignoring the border
        for i in range(1, height):
            found = False
            for j in range(1, width - 1):

                # Whenever you find a white pixel, the distance from where you start will be
                # y = the current height - 1, since we ignore the border
                if img[i, j] == WHITE:
                    fix_height = i - 1
                    found = True
                    break

            if found:
                break

        # If the distance from the first white pixel needs to be adjusted
        if fix_height > 0:

            # Move all pixels up, ignoring the border
            for i in range(1, height - fix_height):
                for j in range(1, width - 1):
                    img[i, j] = img[i + fix_height, j]

            # Erase the remaining pixels by painting them as BLACK
            for i in range(height - fix_height, height):
                for j in range(width):
                    img[i, j] = BLACK

    return img

def process_digit(digit):
    ''' Does all the pre-processing of a grayscale CTR digit.
        @param digit: grayscale image of a CTR digit '''

    digit = detect_edges(digit)
    digit = add_border(digit)
    digit = remove_noisy_areas(digit)

    return digit


import numpy as np
import scipy.io as sio
import datetime as dt
import io
import requests
import pandas as pd

def _get_timestamp(line):
    """ Get a datetime and epoch value from a standard DAT file line """
    timestamp = dt.datetime.strptime(' '.join(line.strip().split(' ')[1:3]), '%Y/%m/%d %H:%M:%S.%f')
    epoch = np.float64(timestamp.replace(tzinfo=dt.timezone.utc).timestamp()) # 'epoch' is unix time
    return timestamp, epoch

def _get_token(response):
    for key, value in response.cookies.items():
        if key.startswith('download_warning'):
            return value
    return None

def open_gdrive_file(blob_id):
    """ Read a file from Google Drive into memory. Returns an open (BytesIO) file-like object. """

    url = "https://docs.google.com/uc?export=download"
    session = requests.Session()
    response = session.get(url, params = { 'id' : blob_id }, stream = True,)
    token = _get_token(response)
    if token:
        params = { 'id' : blob_id, 'confirm' : token }
        response = session.get(url, params = params, stream = True)
    file_bytes = response.content
    return io.BytesIO(file_bytes)

def read_nav(blob_id, dive_number):
    f = open_gdrive_file(blob_id)
    mat = sio.loadmat(f, squeeze_me=True)
    nrows = len(mat['rnv']['t'].take(0))
    timestamp = []
    for i in range(nrows):
        timestamp.append(dt.datetime.utcfromtimestamp(mat['rnv']['t'].take(0)[i]))
    epoch = list(mat['rnv']['t'].take(0))
    dive_number = [dive_number] * nrows
    lat = list(mat['rnv']['lat'].take(0))
    lon = list(mat['rnv']['lon'].take(0))
    depth = list(mat['rnv']['pos'].take(0)[:,2])
    height = list(mat['rnv']['alt'].take(0))
    heading = list(mat['rnv']['pos'].take(0)[:,3])
    pitch = list(mat['rnv']['pos'].take(0)[:,4])
    roll = list(mat['rnv']['pos'].take(0)[:,5])

    # convert to dataframe
    return pd.DataFrame({'timestamp': timestamp, 'epoch': epoch, 'dive_number': dive_number,
                        'lat': lat, 'lon': lon, 'depth': depth, 'heading': heading,
                        'pitch': pitch, 'roll': roll, 'height': height})

def read_paros(blob_id, dive_number):
    paros_list = []
    f = open_gdrive_file(blob_id)
    for line in f:
        line = line.decode('utf-8').strip()
        if 'RAW' in line[0:3]:
            if 'P2=' in line:
                if len(line) == 57: # good lines from this instrument are length 57
                    timestamp, epoch = _get_timestamp(line)
                    a = line.split(' ')[3].split(',')[0].split('=')[1]
                    b = line.split(' ')[3].split(',')[1]
                    paros_list.append([timestamp, epoch, dive_number, np.float64(a), np.float64(b)])

    # convert to dataframe (tau and eta are the the pressure and temperature signal periods in microseconds)
    return pd.DataFrame(paros_list, columns=['timestamp', 'epoch', 'dive_number', 'tau', 'eta'])

def read_ustrain(blob_id, dive_number):
    ustrain_list = []
    f = open_gdrive_file(blob_id)
    for line in f:
        line = line.decode('utf-8').strip()
        if 'MSA3' in line[0:4]:
            if len(line.split(' ')) == 32: # good lines from this instrument will have 32 fields
                timestamp, epoch = _get_timestamp(line)
                ustrain_list.append([timestamp, epoch, dive_number] + list(map(np.float64, line.split(' ')[3:-1])))

    # convert to dataframe
    return pd.DataFrame(ustrain_list, columns=['timestamp','epoch','dive_number','a',
                                              'b','c','d','e','f','g','h','i','j','k',
                                              'l','m','n','o','p','q','r','s','t','u',
                                              'v','w','x','y','z','aa','bb'])

def read_sbe3(blob_id, dive_number):
    sbe3_list = []
    f = open_gdrive_file(blob_id)
    for line in f:
        try:
            line = line.decode('utf-8').strip()
        except:
            line = ''
        if 'SBE3' in line[0:4]:
            if len(line) == 58: # good lines from this instrument are length 58
                timestamp, epoch = _get_timestamp(line)
                counts_0 = np.int64(line.split(' ')[4])
                counts_1 = np.int64(line.split(' ')[6])
                if counts_0 > 500000 and counts_0 < 815000 and counts_1 > 450000 and counts_1 < 770000:
                    sbe3_list.append([timestamp, epoch, dive_number, counts_0, counts_1])

    # convert to dataframe
    return pd.DataFrame(sbe3_list, columns=['timestamp','epoch','dive_number','counts_0','counts_1'])

# ITS-90 equation
def _its90(f, g, h, i, j, f0):
    return 1/(g + h*(np.log(f0/f)) + i*((np.log(f0/f))**2) + j*((np.log(f0/f))**3)) - 273.15

# convert sbe3 counts to temperature
def convert_sbe3(counts, g, h, i, j, f0):
    Fxtal = 4.91548
    f = 512*Fxtal/(counts/1e6)
    T = _its90(f, g, h, i, j, f0)
    return T

# convert Paros values to temperature and pressure
def convert_paros(eta, tau, C1, C2, C3, D1, D2, T1, T2, T3, T4, T5, U0, Y1, Y2, Y3):

    U = eta - U0
    C = C1 + C2*U + C3*U**2
    D = D1 + D2*U
    T0 = T1 + T2*U + T3*U**2 + T4*U**3 + T5*U**4

    T = Y1*U + Y2*U**2 + Y3*U**3
    P = C*(1-T0**2/tau**2)*(1-D*(1-T0**2/tau**2))
    return T, P

# read Nortek ADV data
def read_nortek(blob_id, dive_number):
    nortek_list = []
    f = open_gdrive_file(blob_id)
    for line in f:
        try:
            line = line.decode('utf-8').strip()
        except:
            line = ''
        if 'VVD' in line[0:4]:
            if len(line.split(' ')) == 21: # good lines from this instrument will have 21 fields
                timestamp, epoch = _get_timestamp(line)
                nortek_list.append([timestamp, epoch, dive_number] +
                                    list(map(np.float64, line.split(' ')[8:11])) +
                                    list(map(np.float64, line.split(' ')[12:15])) +
                                    list(map(np.float64, line.split(' ')[16:19])))

    # convert to dataframe
    return pd.DataFrame(nortek_list, columns=['timestamp','epoch','dive_number','v0','v1','v2',
                                              'c0','c1','c2','a0','a1','a2'])


import blitterbike
import glob
import time
import random
from twisted.python import log

try:
	from PIL import Image
	from PIL import ImageOps
except ImportError:
	import Image
	import ImageOps

class BikeMode (blitterbike.BlitterBikeMode):

	def __init__(self):
		self.fullSpeed = 130.0;
		self.bootGif = blitterbike.BLITTER_BIKE_PATH  + "/gifs/bike.gif"
		self.gifPath = blitterbike.BLITTER_BIKE_PATH + "/gifs/bike/"
		self.im = None
		self.gifList = []
		self.gifList.append({"gif":"boo.gif", "step":12, "wait":100, "loops":[{"speed":0.05, "start":0, "end":0}, {"speed":1.0, "start":1, "end":26}]})
		self.gifList.append({"gif":"corgi.gif", "step":12, "wait":100, "loops":[{"speed":1.0, "start":0, "end":3}]})
		self.gifList.append({"gif":"mario.gif", "step":12, "wait":100, "loops":[{"speed":1.0, "start":0, "end":2}]})
		self.gifList.append({"gif":"megaman.gif", "step":12, "wait":300, "loops":[{"speed":0.02, "start":0, "end":6}, {"speed":0.65, "start":7, "end":10}, {"speed":1.0, "start":11, "end":29}]})
		self.gifList.append({"gif":"nyan.gif", "step":12, "wait":100, "loops":[{"speed":1.0, "start":0, "end":11}]})
		self.gifList.append({"gif":"pony_run.gif", "step":12, "wait":100, "loops":[{"speed":1.0, "start":0, "end":3}]})
		self.gifList.append({"gif":"rabbit.gif", "step":12, "wait":100, "loops":[{"speed":1.0, "start":0, "end":7}]})
		self.gifList.append({"gif":"samus.gif", "step":12, "wait":100, "loops":[{"speed":1.0, "start":0, "end":9}]})
		self.gifList.append({"gif":"sonic.gif", "step":6, "wait":200, "loops":[{"speed":0.02, "start":0, "end":7}, {"speed":0.5, "start":8, "end":15}, {"speed":0.75, "start":16, "end":19}, {"speed":1.0, "start":20, "end":23}]})
		self.gifList.append({"gif":"yoshi.gif", "step":12, "wait":100, "loops":[{"speed":1.0, "start":0, "end":9}]})

	def start(self):
		self.mirrorFlag = False
		self.flipFlag = False
		self.scratchFlag = False
		self.invertFlag = False
		self.updateFlag = False

		self.gifIndex = 0
		self.gif = None
		self.im = None
		self.frame = None
		self.lastFrame = None
		self.startIndex = 0
		self.lastTime = 0
		self.delay = 0
		self.newFlag = False
		self.loadingFlag = False
		self.current = None

		log.msg("BIKE MODE")
		random.shuffle(self.gifList, random.random)

		self.loadingFlag = True
		self.loadGif(self.gifList[self.gifIndex])
		self.loadingFlag = False

	def stop(self):
		pass

	def update(self, speed):

		result = None

		if self.updateFlag:
			self.im = None
			self.updateFlag = False
			self.loadGif(self.gifList[self.gifIndex])

		if not self.im == None:
			if self.newFlag and self.frame != None:
				self.newFlag = False
				result = self.frame

			else:

				currentTime = int(round(time.time() * 1000))
				elapsed = currentTime - self.lastTime

				if len(self.current["loops"]) > 1:
					if speed < self.current["loops"][0]["speed"]:
						if elapsed > self.current["wait"]:
							self.lastTime = currentTime
							self.nextFrame(speed)
							result = self.frame
					else:
						if elapsed >= (self.current["step"]/speed) * 1000:
							self.lastTime = currentTime
							self.nextFrame(speed)
							result = self.frame

				else:
					if speed > 0:
						currentTime = int(round(time.time() * 1000))
						elapsed = currentTime - self.lastTime

						if elapsed >= (self.current["step"]/speed) * 1000:
							self.lastTime = currentTime
							self.nextFrame(speed)
							result = self.frame

		if result != None:
			result = result.convert("RGB")

			if self.mirrorFlag:
				result = result.transpose(Image.FLIP_LEFT_RIGHT)

			if self.flipFlag:
				result = result.transpose(Image.FLIP_TOP_BOTTOM)

			if self.invertFlag:
				result = ImageOps.invert(result)

			result = result.getdata()

		return result

	def onButtonDown(self, button):
		if button == blitterbike.RIGHT_BUTTON:
			self.gifIndex += 1
			if self.gifIndex == len(self.gifList):
				self.gifIndex = 0

			self.updateFlag = True

		if button == blitterbike.LEFT_BUTTON:
			self.gifIndex -= 1
			if self.gifIndex == -1:
				self.gifIndex = len(self.gifList) - 1

			self.updateFlag = True

		if button == blitterbike.UP_BUTTON:
			self.gifIndex += 5
			if self.gifIndex >= len(self.gifList):
				self.gifIndex -= len(self.gifList)

			self.updateFlag = True

		if button == blitterbike.DOWN_BUTTON:
			self.gifIndex -= 5
			if self.gifIndex < 0:
				self.gifIndex += len(self.gifList)

			self.updateFlag = True

		if button == blitterbike.A_BUTTON:
			self.mirrorFlag = not self.mirrorFlag

		if button == blitterbike.E_BUTTON:
			self.flipFlag = not self.flipFlag

		if button == blitterbike.D_BUTTON:
			self.scratchFlag = True

		if button == blitterbike.H_BUTTON:
			self.invertFlag = not self.invertFlag

	def onButtonUp(self, button):
		pass

	def loadGif(self, info):

		self.current = info
		self.frame = None
		imagePath = "%s%s" % (self.gifPath, self.current["gif"])

		log.msg(imagePath)
		self.im = Image.open(imagePath)
		self.frame = Image.new("RGBA", (32, 32), (0,0,0))

		self.lastFrame = next = self.im.convert("RGBA")
		self.frame.paste(next, next.getbbox(), mask=next)
		self.lastTime = int(round(time.time() * 1000))
		self.newFlag = True

	def nextFrame(self, speed):
		if self.im != None:

			try:
				self.im.seek(self.im.tell() + 1)
			except EOFError:
				self.im.seek(0)
				self.frame = Image.new("RGBA", (32, 32), (0,0,0))

			percent = speed/self.fullSpeed

			if percent > 1.0:
				percent = 1.0

			index = 0
			while percent > self.current["loops"][index]["speed"]:
				index += 1

			start = self.current["loops"][index]["start"]
			end = self.current["loops"][index]["end"]

			while self.im.tell() > end:
				try:
					self.im.seek(self.im.tell() + 1)
				except EOFError:
					self.im.seek(0)
					self.frame = Image.new("RGBA", (32, 32), (0,0,0))

			while self.im.tell() < start:
				self.im.palette.dirty = 1
				self.im.palette.rawmode = "RGB"

				next = self.im.convert("RGBA")

				self.frame.paste(next, next.getbbox(), mask=next)
				self.im.seek(self.im.tell() + 1)

			self.im.palette.dirty = 1
			self.im.palette.rawmode = "RGB"

			next = self.im.convert("RGBA")

			self.frame.paste(next, next.getbbox(), mask=next)

			self.lastFrame = next

			try:
				self.delay = self.im.info['duration']
			except KeyError:
				self.delay = 100

			if self.delay < 20:
				self.delay = 100


import asyncio
from functools import wraps
from logging import getLogger
from time import perf_counter
from urllib.parse import urljoin

import httpx
from httpx import StatusCode, HTTPError

from api_composer import settings
from api_composer.hawk import HawkAuth

PAGE_SIZE = 1000
MAX_RESULTS = 10000
NUM_PAGES = MAX_RESULTS // PAGE_SIZE
TIMEOUT_SECS = 10

logger = getLogger(__name__)

def async_trace(func):
    @wraps(func)
    async def wrapper(*args, **kwargs):
        start_ts = perf_counter()

        ret = await func(*args, **kwargs)

        end_ts = perf_counter()
        logger.info(
            f'{func.__name__}() args={args} kwargs={kwargs} executed in {end_ts-start_ts:.3f}s',
        )
        return ret

    return wrapper

@async_trace
async def get_composed_companies(market_of_interest):
    dh_companies, ds_companies = await asyncio.gather(
        _get_data_hub_companies(NUM_PAGES),
        _get_data_science_companies(NUM_PAGES, market_of_interest),
    )

    ds_company_mapping = {
        company['datahub_company_id']: company
        for company in ds_companies
        if company['datahub_company_id']
    }

    filtered_dh_companies = [
        {
            'data_hub': company,
            'data_science': ds_company_mapping[company['id']],
        }
        for company in dh_companies
        if company['id'] in ds_company_mapping
    ]

    return {
        'results': filtered_dh_companies,
        'counts': {
            'data_hub_results_fetched': len(dh_companies),
            'data_science_results_fetched': len(ds_companies),
            'data_science_results_fetched_with_data_hub_ids': len(ds_company_mapping),
        },
    }

@async_trace
async def _get_data_hub_companies(num_pages):
    pages = await asyncio.gather(
        *(_get_data_hub_companies_page(page_num) for page_num in range(num_pages)),
    )
    return [
        result
        for page in pages
        for result in page['results']
    ]

@async_trace
async def _get_data_science_companies(num_pages, market_of_interest):
    pages = await asyncio.gather(
        *(
            _get_data_science_companies_page(page_num, market_of_interest)
            for page_num in range(num_pages)
        ),
    )
    return [
        result
        for page in pages
        for result in page['result']
    ]

@async_trace
async def _get_data_hub_companies_page(page_num):
    url = urljoin(str(settings.DATA_HUB_API_URL), '/v4/public/search/company')
    auth = HawkAuth(
        settings.DATA_HUB_API_HAWK_ID,
        str(settings.DATA_HUB_API_HAWK_KEY),
        verify_response=False,
    )
    response = await httpx.post(
        url,
        auth=auth,
        json={
            'original_query': '',
            'offset': PAGE_SIZE * page_num,
            'limit': PAGE_SIZE,
        },
        timeout=TIMEOUT_SECS,
    )
    response.raise_for_status()
    return response.json()

@async_trace
async def _get_data_science_companies_page(page_num, market_of_interest):
    url = urljoin(str(settings.DATA_SCIENCE_API_URL), '/api/v1/company/search/')
    auth = HawkAuth(
        settings.DATA_SCIENCE_API_HAWK_ID,
        str(settings.DATA_SCIENCE_API_HAWK_KEY),
        verify_response=False,
    )

    body = {
        'filters': {
            'market_of_interest': [market_of_interest]
        },
    } if market_of_interest else {}

    try:
        response = await httpx.post(
            url,
            auth=auth,
            params={
                'offset': PAGE_SIZE * page_num,
                'limit': PAGE_SIZE,
            },
            json=body,
            timeout=TIMEOUT_SECS,
        )
        response.raise_for_status()
    except HTTPError as exc:
        if exc.response and exc.response.status_code == StatusCode.NOT_FOUND:
            return {
                'result': [],
            }
        raise

    return response.json()


'''
Created on 15 Nov 2017

@author: ahmed.salem
'''

import sys

sys.dont_write_bytecode = True

from sklearn.metrics import classification_report, accuracy_score
import theano.tensor as T
import numpy as np
import lasagne
import theano
import argparse
from sklearn.metrics import precision_recall_fscore_support
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score

def iterate_minibatches(inputs, targets, batch_size, shuffle=True):
    assert len(inputs) == len(targets)
    if shuffle:
        indices = np.arange(len(inputs))
        np.random.shuffle(indices)

    start_idx = None
    for start_idx in range(0, len(inputs) - batch_size + 1, batch_size):
        if shuffle:
            excerpt = indices[start_idx:start_idx + batch_size]
        else:
            excerpt = slice(start_idx, start_idx + batch_size)
        yield inputs[excerpt], targets[excerpt]

    if start_idx is not None and start_idx + batch_size < len(inputs):
        excerpt = indices[start_idx + batch_size:] if shuffle else slice(start_idx + batch_size, len(inputs))
        yield inputs[excerpt], targets[excerpt]

def get_cnn_model(n_in, n_hidden, n_out):
    net = dict()
    #print(n_in)
    net['input'] = lasagne.layers.InputLayer(shape=(None, 3, 32, 32))

    net['conv1'] = lasagne.layers.Conv2DLayer(net['input'], num_filters=32, filter_size=(5, 5),
            pad='same',
            nonlinearity=lasagne.nonlinearities.rectify,
            W=lasagne.init.GlorotUniform(gain='relu'))

    net['maxPool1'] = lasagne.layers.MaxPool2DLayer(net['conv1'], pool_size=(2, 2))

    net['conv2'] = lasagne.layers.Conv2DLayer(
            net['maxPool1'], num_filters=32, filter_size=(5, 5),
            nonlinearity=lasagne.nonlinearities.rectify,
            W=lasagne.init.GlorotUniform(gain='relu'))
    net['maxPool2'] = lasagne.layers.MaxPool2DLayer(net['conv2'], pool_size=(2, 2))

    net['fc'] = lasagne.layers.DenseLayer(
        net['maxPool2'],
        num_units=n_hidden,
        nonlinearity=lasagne.nonlinearities.tanh)

    net['output'] = lasagne.layers.DenseLayer(
        net['fc'],
        num_units=n_out,
        nonlinearity=lasagne.nonlinearities.softmax)
    return net

def get_cnn2_model(n_in, n_hidden, n_out):
    net = dict()
    face = False
    if (face):
        net['input'] = lasagne.layers.InputLayer(shape=(None, 1, 62, 47))
    else:
        net['input'] = lasagne.layers.InputLayer(shape=(None, 1, 28, 28))

    net['conv1'] = lasagne.layers.Conv2DLayer(net['input'], num_filters=32, filter_size=(5, 5),
            pad='same',
            nonlinearity=lasagne.nonlinearities.rectify,
            W=lasagne.init.GlorotUniform(gain='relu'))

    net['maxPool1'] = lasagne.layers.MaxPool2DLayer(net['conv1'], pool_size=(2, 2))

    net['conv2'] = lasagne.layers.Conv2DLayer(
            net['maxPool1'], num_filters=32, filter_size=(5, 5),
            nonlinearity=lasagne.nonlinearities.rectify,
            W=lasagne.init.GlorotUniform(gain='relu'))
    net['maxPool2'] = lasagne.layers.MaxPool2DLayer(net['conv2'], pool_size=(2, 2))

    net['fc'] = lasagne.layers.DenseLayer(
        net['maxPool2'],
        num_units=n_hidden,
        nonlinearity=lasagne.nonlinearities.tanh)

    net['output'] = lasagne.layers.DenseLayer(
        net['fc'],
        num_units=n_out,
        nonlinearity=lasagne.nonlinearities.softmax)
    return net

def get_nn_model(n_in, n_hidden, n_out):
    net = dict()
    net['input'] = lasagne.layers.InputLayer((None, n_in))
    net['fc'] = lasagne.layers.DenseLayer(
        net['input'],
        num_units=n_hidden,
        nonlinearity=lasagne.nonlinearities.tanh)
    net['output'] = lasagne.layers.DenseLayer(
        net['fc'],
        num_units=n_out,
        nonlinearity=lasagne.nonlinearities.softmax)
    return net

def get_Dropnn_model(n_in, n_hidden, n_out,drop1,drop2):
    net = dict()
    net['input'] = lasagne.layers.InputLayer((None, n_in))

    net['inputDropOut'] = lasagne.layers.DropoutLayer(net['input'], p=drop1)

    net['fc'] = lasagne.layers.DenseLayer(
        net['inputDropOut'],
        num_units=n_hidden,
        nonlinearity=lasagne.nonlinearities.tanh)

    net['fcDropOut'] = lasagne.layers.DropoutLayer(net['fc'], p=drop2)

    net['output'] = lasagne.layers.DenseLayer(
        net['fcDropOut'],
        num_units=n_out,
        nonlinearity=lasagne.nonlinearities.softmax)
    return net
def get_Droppcnn_model(n_in, n_hidden, n_out,drop1,drop2,drop3):
    net = dict()
    net['input'] = lasagne.layers.InputLayer(shape=(None, 3, 32, 32))

    net['inputDropOut'] = lasagne.layers.DropoutLayer(net['input'], p=drop1)

    net['conv1'] = lasagne.layers.Conv2DLayer(net['inputDropOut'], num_filters=32, filter_size=(5, 5),
            pad='same',
            nonlinearity=lasagne.nonlinearities.rectify,
            W=lasagne.init.GlorotUniform(gain='relu'))

    net['maxPool1'] = lasagne.layers.MaxPool2DLayer(net['conv1'], pool_size=(2, 2))

    net['conv2'] = lasagne.layers.Conv2DLayer(
            net['maxPool1'], num_filters=32, filter_size=(5, 5),
            nonlinearity=lasagne.nonlinearities.rectify,
            W=lasagne.init.GlorotUniform(gain='relu'))

    net['maxPool2'] = lasagne.layers.MaxPool2DLayer(net['conv2'], pool_size=(2, 2))

    #net['maxPool2DropOut'] = lasagne.layers.DropoutLayer(net['maxPool2'], p=drop1)

    net['fc'] = lasagne.layers.DenseLayer(
        net['maxPool2'],
        num_units=n_hidden,
        nonlinearity=lasagne.nonlinearities.tanh)

    net['fcDropOut'] = lasagne.layers.DropoutLayer(net['fc'], p=drop2)

    net['output'] = lasagne.layers.DenseLayer(
        net['fcDropOut'],
        num_units=n_out,
        nonlinearity=lasagne.nonlinearities.softmax)
    return net

def get_Droppcnn2_model(n_in, n_hidden, n_out,drop1,drop2,drop3):
    net = dict()
    face = False
    if (face):
        net['input'] = lasagne.layers.InputLayer(shape=(None, 1, 62, 47))
    else:
        net['input'] = lasagne.layers.InputLayer(shape=(None, 1, 28, 28))

    net['inputDropOut'] = lasagne.layers.DropoutLayer(net['input'], p=drop1)

    net['conv1'] = lasagne.layers.Conv2DLayer(net['inputDropOut'], num_filters=32, filter_size=(5, 5),
            pad='same',
            nonlinearity=lasagne.nonlinearities.rectify,
            W=lasagne.init.GlorotUniform(gain='relu'))

    net['maxPool1'] = lasagne.layers.MaxPool2DLayer(net['conv1'], pool_size=(2, 2))

    net['conv2'] = lasagne.layers.Conv2DLayer(
            net['maxPool1'], num_filters=32, filter_size=(5, 5),
            nonlinearity=lasagne.nonlinearities.rectify,
            W=lasagne.init.GlorotUniform(gain='relu'))

    net['maxPool2'] = lasagne.layers.MaxPool2DLayer(net['conv2'], pool_size=(2, 2))

    #net['maxPool2DropOut'] = lasagne.layers.DropoutLayer(net['maxPool2'], p=drop1)

    net['fc'] = lasagne.layers.DenseLayer(
        net['maxPool2'],
        num_units=n_hidden,
        nonlinearity=lasagne.nonlinearities.tanh)

    net['fcDropOut'] = lasagne.layers.DropoutLayer(net['fc'], p=drop2)

    net['output'] = lasagne.layers.DenseLayer(
        net['fcDropOut'],
        num_units=n_out,
        nonlinearity=lasagne.nonlinearities.softmax)
    return net

def get_softmax_model(n_in, n_out):
    net = dict()
    net['input'] = lasagne.layers.InputLayer((None, n_in))
    net['output'] = lasagne.layers.DenseLayer(
        net['input'],
        num_units=n_out,
        nonlinearity=lasagne.nonlinearities.softmax)
    return net

def train_model(dataset, n_hidden=50, batch_size=100, epochs=100, learning_rate=0.01, model='cnn', l2_ratio=1e-7,
          rtn_layer=True):
    print("classfier.py -- train_model")

    myTest = (model == 'softmax')

    train_x, train_y, test_x, test_y = dataset
    #print(train_x)
    n_in = train_x.shape[1]
    silent = False
    #print(train_x.shape)
    n_out = len(np.unique(train_y))

    if batch_size > len(train_y):
        batch_size = len(train_y)
    if (not myTest):
        print('Building model with {} training data, {} classes...'.format(len(train_x), n_out))
    if model=='cnn' or model=='cnn2' or model=='Droppcnn' or  model=='Droppcnn2':
        input_var = T.tensor4('x')
    else:
        input_var = T.matrix('x')
    target_var = T.ivector('y')
    if model == 'cnn':

        print('Using conv neural network...')
        net = get_cnn_model(n_in, n_hidden, n_out)

    elif model == 'cnn2':
        print('Using conv neural network...')
        net = get_cnn2_model(n_in, n_hidden, n_out)
    elif model == 'nn':
        print('Using neural network...')
        net = get_nn_model(n_in, n_hidden, n_out)
    elif model == 'Dropnn':
        print('Using neural network with Dropout')
        net = get_Dropnn_model(n_in, n_hidden, n_out,0.75,0.5)
    elif model == 'Droppcnn':
        print('Using neural network with Dropout')
        net = get_Droppcnn_model(n_in, n_hidden, n_out,0.2,0.5,0.5)

    elif model == 'Droppcnn2':
        print('Using Conv neural network with Dropout')
        net = get_Droppcnn2_model(n_in, n_hidden, n_out,0.5,0.5,0.5)
    else:
        if (not myTest and not silent):
            print('Using softmax regression...')
        net = get_softmax_model(n_in, n_out)

    net['input'].input_var = input_var

    output_layer = net['output']
    # create loss function
    prediction = lasagne.layers.get_output(output_layer)
    loss = lasagne.objectives.categorical_crossentropy(prediction, target_var)
    loss = loss.mean() + l2_ratio * lasagne.regularization.regularize_network_params(output_layer,
                                                                                lasagne.regularization.l2)
    # create parameter update expressions
    params = lasagne.layers.get_all_params(output_layer, trainable=True)
    updates = lasagne.updates.adam(loss, params, learning_rate=learning_rate)
    train_fn = theano.function([input_var, target_var], loss, updates=updates)
    # use trained network for predictions
    test_prediction = lasagne.layers.get_output(output_layer, deterministic=True)
    test_fn = theano.function([input_var], test_prediction)
    if (not myTest and not silent):
        print('Training...')
    counter = 1
    for epoch in range(epochs):
        print(epoch)
        loss = 0
        for input_batch, target_batch in iterate_minibatches(train_x, train_y, batch_size):
            #input_batch = (np.reshape(input_batch,(len(input_batch),3,32,32)))
            #print("classfier.py -- func -- train_model -- iterate_minibatches")

            loss += train_fn(input_batch, target_batch)

        loss = round(loss, 4)
        if (not myTest and not silent):
            print('Epoch {}, train loss {}'.format(epoch, loss))

        counter = counter +1
    pred_y = []
    for input_batch, _ in iterate_minibatches(train_x, train_y, batch_size, shuffle=False):
        #input_batch = (np.reshape(input_batch,(len(input_batch),3,32,32)))
        pred = test_fn(input_batch)
        pred_y.append(np.argmax(pred, axis=1))
    pred_y = np.concatenate(pred_y)
    if (not myTest and not silent):
        print(classification_report(train_y, pred_y))

    if test_x is not None:
        if (not myTest and not silent):
            print('Testing...')
        pred_y = []

        if batch_size > len(test_y):
            batch_size = len(test_y)

        for input_batch, _ in iterate_minibatches(test_x, test_y, batch_size, shuffle=False):
            #input_batch = (np.reshape(input_batch,(len(input_batch),3,32,32)))
            pred = test_fn(input_batch)
            pred_y.append(np.argmax(pred, axis=1))
        pred_y = np.concatenate(pred_y)
        if (not myTest and not silent):
            print('Testing Accuracy: {}'.format(accuracy_score(test_y, pred_y)))

    print(classification_report(test_y, pred_y))

    if ( myTest):
        pres= round(precision_score(test_y, pred_y,average='weighted'),4)
        recall= round(recall_score(test_y, pred_y,average='weighted'),4)
        accuracy= round(accuracy_score(test_y, pred_y),5)
        print("precision",pres)
        print("recall",recall)
        print("accuracy",accuracy)
        return pres,recall, accuracy
        # return classification_report(test_y, pred_y) #precision_recall_fscore_support(test_y, pred_y,average=None)

    if rtn_layer:

        return output_layer
    else:
        return pred_y

def load_dataset(train_feat, train_label, test_feat=None, test_label=None):

    train_x = np.genfromtxt(train_feat, delimiter=',', dtype='float32')
    train_y = np.genfromtxt(train_label, dtype='int32')

    min_y = np.min(train_y)
    train_y -= min_y
    if test_feat is not None and test_label is not None:
        test_x = np.genfromtxt(train_feat, delimiter=',', dtype='float32')
        test_y = np.genfromtxt(train_label, dtype='int32')
        test_y -= min_y
    else:
        test_x = None
        test_y = None
    return train_x, train_y, test_x, test_y

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('train_feat', type=str)
    parser.add_argument('train_label', type=str)
    parser.add_argument('--test_feat', type=str, default=None)
    parser.add_argument('--test_label', type=str, default=None)
    parser.add_argument('--model', type=str, default='nn')
    parser.add_argument('--learning_rate', type=float, default=0.01)
    parser.add_argument('--batch_size', type=int, default=100)
    parser.add_argument('--n_hidden', type=int, default=50)
    parser.add_argument('--epochs', type=int, default=100)
    args = parser.parse_args()
    print(vars(args))
    dataset = load_dataset(args.train_feat, args.train_label, args.test_feat, args.train_label)
    train_model(dataset,
          model=args.model,
          learning_rate=args.learning_rate,
          batch_size=args.batch_size,
          n_hidden=args.n_hidden,
          epochs=args.epochs)

if __name__ == '__main__':
    main()


# Tai Sakuma <tai.sakuma@gmail.com>
import time
import itertools
import threading
import multiprocessing

import pytest

from atpbar import atpbar
from atpbar import register_reporter, find_reporter, flush, disable

##__________________________________________________________________||
@pytest.mark.parametrize('niterations', [10, 1, 0])
def test_one_loop(mock_create_presentation, niterations):
    disable()
    for i in atpbar(range(niterations)):
        pass
    presentations = mock_create_presentation.presentations
    assert 0 == len(presentations)

##__________________________________________________________________||
def test_nested_loops(mock_create_presentation):
    disable()
    for i in atpbar(range(4)):
        for j in atpbar(range(3)):
            pass

##__________________________________________________________________||
def run_with_threading(nthreads=3, niterations=[5, 5, 5]):
    def task(n, name):
        for i in atpbar(range(n), name=name):
            time.sleep(0.0001)
    threads = [ ]
    for i in range(nthreads):
        name = 'thread {}'.format(i)
        n = niterations[i]
        t = threading.Thread(target=task, args=(n, name))
        t.start()
        threads.append(t)
    for t in threads:
        t.join()
    flush()

@pytest.mark.parametrize('niterations', [[5, 4, 3], [5, 0, 1], [0], [1]])
@pytest.mark.parametrize('nthreads', [3, 1, 0])
def test_threading(mock_create_presentation, nthreads, niterations):
    disable()

    # make niterations as long as nthreads. repeat if necessary
    niterations = list(itertools.chain(*itertools.repeat(niterations, nthreads//len(niterations)+1)))[:nthreads]

    run_with_threading(nthreads, niterations)

    presentations = mock_create_presentation.presentations
    assert 0 == len(presentations)

##__________________________________________________________________||
def run_with_multiprocessing(nprocesses, ntasks, niterations):
    def task(n, name):
        for i in atpbar(range(n), name=name):
            time.sleep(0.0001)
    def worker(reporter, task, queue):
        register_reporter(reporter)
        while True:
            args = queue.get()
            if args is None:
                queue.task_done()
                break
            task(*args)
            queue.task_done()
    reporter = find_reporter()
    queue = multiprocessing.JoinableQueue()
    for i in range(nprocesses):
        p = multiprocessing.Process(target=worker, args=(reporter, task, queue))
        p.start()
    for i in range(ntasks):
        name = 'task {}'.format(i)
        n = niterations[i]
        queue.put((n, name))
    for i in range(nprocesses):
        queue.put(None)
        queue.join()
    flush()

@pytest.mark.xfail()
@pytest.mark.parametrize('niterations', [[5, 4, 3], [5, 0, 1], [0], [1]])
@pytest.mark.parametrize('ntasks', [3, 1, 0])
@pytest.mark.parametrize('nprocesses', [4, 1])
def test_multiprocessing(mock_create_presentation, nprocesses, ntasks, niterations):
    disable()

    # make niterations as long as ntasks. repeat if necessary
    niterations = list(itertools.chain(*itertools.repeat(niterations, ntasks//len(niterations)+1)))[:ntasks]

    run_with_multiprocessing(nprocesses, ntasks, niterations)

    presentations = mock_create_presentation.presentations
    assert 0 == len(presentations)

##__________________________________________________________________||
def test_call_twice():
    disable()
    disable()

##__________________________________________________________________||