import numpy as np
import xarray as xr

def fetch_dataset(fname):
    data = xr.open_dataset(fname)
    # Capture attributes dict because it's removed after converting the data to
    # float64
    attrs = data.attrs.copy()
    # The data are stored as ints and data as float32 to save space on the
    # data file. Cast them to float64 to avoid integer division errors.
    data = data.astype("float64")
    data.attrs = attrs
    return data


"""
Holds custom distance functions used for FAT-Forensics examples and testing.
"""
# Author: Kacper Sokol <k.sokol@bristol.ac.uk>
# License: new BSD

import numpy as np
import pytest

import fatf.utils.models.metrics as fumm

from fatf.exceptions import IncorrectShapeError

USER_WARNING = ('Some of the given labels are not present in either of the '
                'input arrays: {}.')
GROUND_TRUTH = np.array(['a', 'b', 'b', 'b', 'a', 'a', 'b', 'c', 'c', 'c'])
PREDICTIONS = np.array(['b', 'a', 'b', 'c', 'a', 'c', 'b', 'a', 'c', 'b'])
# [[3, 11],
#  [7, 5 ]]
GROUND_TRUTH_BIN = np.array([
    'a', 'a', 'a', 'b', 'b', 'b', 'b', 'b', 'a', 'a', 'a', 'a', 'a', 'a', 'a',
    'b', 'b', 'b', 'b', 'b', 'b', 'b', 'b', 'b', 'b', 'b'
])
# [[1, 1, 1],
#  [1, 2, 1],
#  [1, 1, 1]]
PREDICTIONS_BIN = np.array([
    'a', 'a', 'a', 'b', 'b', 'b', 'b', 'b', 'b', 'b', 'b', 'b', 'b', 'b', 'b',
    'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a'
])

def test_validate_confusion_matrix():
    """
    Tests :func:`fatf.utils.models.metrics.validate_confusion_matrix`.
    """
    incorrect_shape_error_2d = ('The confusion matrix has to be a '
                                '2-dimensional numpy array.')
    incorrect_shape_error_square = ('The confusion matrix has to be a square '
                                    '(equal width and height) numpy array.')
    incorrect_shape_error_2 = 'The confusion matrix needs to be at least 2x2.'
    value_error = 'The confusion matrix cannot be a structured numpy array.'
    type_error_cm = 'The confusion matrix has to be of integer kind.'
    type_error_index = 'The label index has to be an integer.'
    index_error = ('The label index {} is not a valid index for the confusion '
                   'matrix of shape {}x{}.')

    three_d_array = np.array([[[0], [2]], [[3]]])
    two_d_array_rect = np.array([[0, 2], [3, 5], [7, 9]])
    two_d_array_one = np.array([[0]])
    struct_array = np.array([(1, 2), (3, 4)], dtype=[('a', 'i'), ('b', 'i')])
    non_int_array = np.array([[2.0, 2], [7, 8]])

    two_d_array = np.array([[1, 2], [3, 4]])

    with pytest.raises(IncorrectShapeError) as exi:
        fumm.validate_confusion_matrix(three_d_array)
    assert str(exi.value) == incorrect_shape_error_2d

    with pytest.raises(IncorrectShapeError) as exi:
        fumm.validate_confusion_matrix(two_d_array_rect)
    assert str(exi.value) == incorrect_shape_error_square

    with pytest.raises(IncorrectShapeError) as exi:
        fumm.validate_confusion_matrix(two_d_array_one)
    assert str(exi.value) == incorrect_shape_error_2

    with pytest.raises(ValueError) as exi:
        fumm.validate_confusion_matrix(struct_array)
    assert str(exi.value) == value_error

    with pytest.raises(TypeError) as exi:
        fumm.validate_confusion_matrix(non_int_array)
    assert str(exi.value) == type_error_cm

    with pytest.raises(TypeError) as exi:
        fumm.validate_confusion_matrix(two_d_array, 'a')
    assert str(exi.value) == type_error_index

    with pytest.raises(IndexError) as exi:
        fumm.validate_confusion_matrix(two_d_array, -1)
    assert str(exi.value) == index_error.format(-1, 2, 2)

    with pytest.raises(IndexError) as exi:
        fumm.validate_confusion_matrix(two_d_array, 2)
    assert str(exi.value) == index_error.format(2, 2, 2)

def test_validate_confusion_matrix_size():
    """
    Tests :func:`fatf.utils.models.metrics._validate_confusion_matrix_size`.
    """
    incorrect_shape_error = ('The confusion matrix is of shape {}x{} but '
                             '{}x{} is the requirement.')

    two_d = np.array([[0, 1], [2, 3]])
    three_d = np.array([[1, 2, 3], [1, 2, 3], [1, 2, 3]])

    with pytest.raises(IncorrectShapeError) as exi:
        fumm._validate_confusion_matrix_size(two_d, 3)
    assert str(exi.value) == incorrect_shape_error.format(2, 2, 3, 3)

    with pytest.raises(IncorrectShapeError) as exi:
        fumm._validate_confusion_matrix_size(three_d, 2)
    assert str(exi.value) == incorrect_shape_error.format(3, 3, 2, 2)

def test_get_confusion_matrix_errors():
    """
    Tests :func:`fatf.utils.models.metrics.get_confusion_matrix` for errors.
    """
    incorrect_shape_error_gt = ('The ground truth vector has to be '
                                '1-dimensional numpy array.')
    incorrect_shape_error_pred = ('The predictions vector has to be '
                                  '1-dimensional numpy array.')
    incorrect_shape_error_gtp = ('Both the ground truth and the predictions '
                                 'vectors have to have the same length.')
    value_error_labels_empty = 'The labels list cannot be empty.'
    value_error_labels_duplicates = 'The labels list contains duplicates.'
    value_error_labels_missing = ('The following labels are present in the '
                                  'input arrays but were not given in the '
                                  'labels parameter: {}.')
    type_error_labels = 'The labels parameter has to either a list or None.'

    two_d_array = np.array([[1, 2], [3, 4]])
    one_d_array_4 = np.array([1, 2, 3, 4])
    one_d_array_5 = np.array([1, 2, 3, 4, 5])

    cma_true = np.array([[1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0],
                         [0, 0, 0, 1, 0], [0, 0, 0, 0, 0]])

    with pytest.raises(IncorrectShapeError) as exi:
        fumm.get_confusion_matrix(two_d_array, two_d_array)
    assert str(exi.value) == incorrect_shape_error_gt
    #
    with pytest.raises(IncorrectShapeError) as exi:
        fumm.get_confusion_matrix(one_d_array_4, two_d_array)
    assert str(exi.value) == incorrect_shape_error_pred
    #
    with pytest.raises(IncorrectShapeError) as exi:
        fumm.get_confusion_matrix(one_d_array_4, one_d_array_5)
    assert str(exi.value) == incorrect_shape_error_gtp

    with pytest.raises(TypeError) as exi:
        fumm.get_confusion_matrix(one_d_array_4, one_d_array_4, 'a')
    assert str(exi.value) == type_error_labels
    #
    with pytest.raises(ValueError) as exi:
        fumm.get_confusion_matrix(one_d_array_4, one_d_array_4, [])
    assert str(exi.value) == value_error_labels_empty
    #
    with pytest.raises(ValueError) as exi:
        fumm.get_confusion_matrix(one_d_array_4, one_d_array_4, [2, 3, 2])
    assert str(exi.value) == value_error_labels_duplicates
    #
    with pytest.raises(ValueError) as exi:
        fumm.get_confusion_matrix(one_d_array_4, one_d_array_4, [2, 4, 3])
    assert str(exi.value) == value_error_labels_missing.format('{1}')

    with pytest.warns(UserWarning) as w:
        cma = fumm.get_confusion_matrix(one_d_array_4, one_d_array_4,
                                        [1, 2, 3, 4, 5])
    assert len(w) == 1
    assert str(w[0].message) == USER_WARNING.format('{5}')
    assert np.array_equal(cma, cma_true)

def test_get_confusion_matrix():
    """
    Tests the :func:`fatf.utils.models.metrics.get_confusion_matrix` function.
    """
    cmx = np.array([[1, 1, 1], [1, 2, 1], [1, 1, 1]])
    cmx_bb = np.array([[1, 1, 0, 1], [1, 2, 0, 1], [0, 0, 0, 0], [1, 1, 0, 1]])

    # Default labeling
    cma = fumm.get_confusion_matrix(GROUND_TRUTH, PREDICTIONS)
    assert np.array_equal(cmx, cma)

    # Custom non-existing labeling
    with pytest.warns(UserWarning) as w:
        cma = fumm.get_confusion_matrix(GROUND_TRUTH, PREDICTIONS,
                                        ['a', 'b', 'bb', 'c'])
    assert len(w) == 1
    assert str(w[0].message) == USER_WARNING.format("{'bb'}")
    assert np.array_equal(cmx_bb, cma)

def test_multiclass_true_positive_rate():
    """
    Tests :func:`fatf.utils.models.metrics.multiclass_true_positive_rate`.
    """
    cma = fumm.get_confusion_matrix(GROUND_TRUTH, PREDICTIONS)

    mtpr_0 = fumm.multiclass_true_positive_rate(cma, 0)
    assert mtpr_0 == pytest.approx(0.333, abs=1e-3)
    mtpr_1 = fumm.multiclass_true_positive_rate(cma, 1)
    assert mtpr_1 == 0.5
    mtpr_2 = fumm.multiclass_true_positive_rate(cma, 2)
    assert mtpr_2 == pytest.approx(0.333, abs=1e-3)

def test_multiclass_true_negative_rate():
    """
    Tests :func:`fatf.utils.models.metrics.multiclass_true_negative_rate`.
    """
    cma = fumm.get_confusion_matrix(GROUND_TRUTH, PREDICTIONS)

    type_error = 'The strict parameter has to be a boolean.'
    with pytest.raises(TypeError) as exi:
        fumm.multiclass_true_negative_rate(cma, 0, 'one')
    assert str(exi.value) == type_error

    metric = pytest.approx(5 / 7)
    mtpr_0_n = fumm.multiclass_true_negative_rate(cma, 0)
    assert mtpr_0_n == metric
    mtpr_0_n = fumm.multiclass_true_negative_rate(cma, 0, False)
    assert mtpr_0_n == metric
    #
    metric = pytest.approx(4 / 6)
    mtpr_1_n = fumm.multiclass_true_negative_rate(cma, 1)
    assert mtpr_1_n == metric
    mtpr_1_n = fumm.multiclass_true_negative_rate(cma, 1, False)
    assert mtpr_1_n == metric
    #
    metric = pytest.approx(5 / 7)
    mtpr_2_n = fumm.multiclass_true_negative_rate(cma, 2)
    assert mtpr_2_n == metric
    mtpr_2_n = fumm.multiclass_true_negative_rate(cma, 2, False)
    assert mtpr_2_n == metric

    metric = pytest.approx(3 / 7)
    mtpr_0_p = fumm.multiclass_true_negative_rate(cma, 0, True)
    assert mtpr_0_p == metric
    metric = pytest.approx(2 / 6)
    mtpr_1_p = fumm.multiclass_true_negative_rate(cma, 1, True)
    assert mtpr_1_p == metric
    metric = pytest.approx(3 / 7)
    mtpr_2_p = fumm.multiclass_true_negative_rate(cma, 2, True)
    assert mtpr_2_p == metric

def test_multiclass_false_positive_rate():
    """
    Tests :func:`fatf.utils.models.metrics.multiclass_false_positive_rate`.
    """
    cma = fumm.get_confusion_matrix(GROUND_TRUTH, PREDICTIONS)

    mtpr_0 = fumm.multiclass_false_positive_rate(cma, 0)
    assert mtpr_0 == pytest.approx(2 / 7, abs=1e-3)
    mtpr_1 = fumm.multiclass_false_positive_rate(cma, 1)
    assert mtpr_1 == pytest.approx(2 / 6, abs=1e-3)
    mtpr_2 = fumm.multiclass_false_positive_rate(cma, 2)
    assert mtpr_2 == pytest.approx(2 / 7, abs=1e-3)

def test_multiclass_false_negative_rate():
    """
    Tests :func:`fatf.utils.models.metrics.multiclass_false_negative_rate`.
    """
    cma = fumm.get_confusion_matrix(GROUND_TRUTH, PREDICTIONS)

    mtpr_0 = fumm.multiclass_false_negative_rate(cma, 0)
    assert mtpr_0 == pytest.approx(2 / 3, abs=1e-3)
    mtpr_1 = fumm.multiclass_false_negative_rate(cma, 1)
    assert mtpr_1 == pytest.approx(2 / 4, abs=1e-3)
    mtpr_2 = fumm.multiclass_false_negative_rate(cma, 2)
    assert mtpr_2 == pytest.approx(2 / 3, abs=1e-3)

def test_true_positive_rate():
    """
    Tests :func:`fatf.utils.models.metrics.true_positive_rate`.
    """
    cma = fumm.get_confusion_matrix(GROUND_TRUTH_BIN, PREDICTIONS_BIN)
    mtpr = fumm.true_positive_rate(cma)
    assert mtpr == pytest.approx(3 / 10, abs=1e-3)

def test_true_negative_rate():
    """
    Tests :func:`fatf.utils.models.metrics.true_negative_rate`.
    """
    cma = fumm.get_confusion_matrix(GROUND_TRUTH_BIN, PREDICTIONS_BIN)
    mtpr = fumm.true_negative_rate(cma)
    assert mtpr == pytest.approx(5 / 16, abs=1e-3)

def test_false_negative_rate():
    """
    Tests :func:`fatf.utils.models.metrics.false_negative_rate`.
    """
    cma = fumm.get_confusion_matrix(GROUND_TRUTH_BIN, PREDICTIONS_BIN)
    mtpr = fumm.false_negative_rate(cma)
    assert mtpr == pytest.approx(7 / 10, abs=1e-3)

def test_false_positive_rate():
    """
    Tests :func:`fatf.utils.models.metrics.false_positive_rate`.
    """
    cma = fumm.get_confusion_matrix(GROUND_TRUTH_BIN, PREDICTIONS_BIN)
    mtpr = fumm.false_positive_rate(cma)
    assert mtpr == pytest.approx(11 / 16, abs=1e-3)

def test_multiclass_positive_predictive_value():
    """
    :func:`fatf.utils.models.metrics.multiclass_positive_predictive_value`.
    """
    cma = fumm.get_confusion_matrix(GROUND_TRUTH, PREDICTIONS)

    mtpr_0 = fumm.multiclass_positive_predictive_value(cma, 0)
    assert mtpr_0 == pytest.approx(1 / 3, abs=1e-3)
    mtpr_1 = fumm.multiclass_positive_predictive_value(cma, 1)
    assert mtpr_1 == pytest.approx(2 / 4, abs=1e-3)
    mtpr_2 = fumm.multiclass_positive_predictive_value(cma, 2)
    assert mtpr_2 == pytest.approx(1 / 3, abs=1e-3)

def test_multiclass_negative_predictive_value():
    """
    :func:`fatf.utils.models.metrics.multiclass_negative_predictive_value`.
    """
    cma = fumm.get_confusion_matrix(GROUND_TRUTH, PREDICTIONS)

    type_error = 'The strict parameter has to be a boolean.'
    with pytest.raises(TypeError) as exi:
        fumm.multiclass_negative_predictive_value(cma, 0, 'one')
    assert str(exi.value) == type_error

    metric = pytest.approx(5 / 7)
    mtpr_0_n = fumm.multiclass_negative_predictive_value(cma, 0)
    assert mtpr_0_n == metric
    mtpr_0_n = fumm.multiclass_negative_predictive_value(cma, 0, False)
    assert mtpr_0_n == metric
    #
    metric = pytest.approx(4 / 6)
    mtpr_1_n = fumm.multiclass_negative_predictive_value(cma, 1)
    assert mtpr_1_n == metric
    mtpr_1_n = fumm.multiclass_negative_predictive_value(cma, 1, False)
    assert mtpr_1_n == metric
    #
    metric = pytest.approx(5 / 7)
    mtpr_2_n = fumm.multiclass_negative_predictive_value(cma, 2)
    assert mtpr_2_n == metric
    mtpr_2_n = fumm.multiclass_negative_predictive_value(cma, 2, False)
    assert mtpr_2_n == metric

    metric = pytest.approx(3 / 7)
    mtpr_0_p = fumm.multiclass_negative_predictive_value(cma, 0, True)
    assert mtpr_0_p == metric
    metric = pytest.approx(2 / 6)
    mtpr_1_p = fumm.multiclass_negative_predictive_value(cma, 1, True)
    assert mtpr_1_p == metric
    metric = pytest.approx(3 / 7)
    mtpr_2_p = fumm.multiclass_negative_predictive_value(cma, 2, True)
    assert mtpr_2_p == metric

def test_positive_predictive_value():
    """
    :func:`fatf.utils.models.metrics.positive_predictive_value`.
    """
    cma = fumm.get_confusion_matrix(GROUND_TRUTH_BIN, PREDICTIONS_BIN)
    mtpr = fumm.positive_predictive_value(cma)
    assert mtpr == pytest.approx(3 / 14, abs=1e-3)

def test_negative_predictive_value():
    """
    :func:`fatf.utils.models.metrics.negative_predictive_value`.
    """
    cma = fumm.get_confusion_matrix(GROUND_TRUTH_BIN, PREDICTIONS_BIN)
    mtpr = fumm.negative_predictive_value(cma)
    assert mtpr == pytest.approx(5 / 12, abs=1e-3)

def test_accuracy():
    """
    Tests :func:`fatf.utils.models.metrics.accuracy` function.
    """
    cma = fumm.get_confusion_matrix(GROUND_TRUTH, PREDICTIONS)
    acc = fumm.accuracy(cma)
    assert acc == pytest.approx(4 / 10, abs=1e-3)

    cma = fumm.get_confusion_matrix(GROUND_TRUTH_BIN, PREDICTIONS_BIN)
    acc = fumm.accuracy(cma)
    assert acc == pytest.approx(8 / 26, abs=1e-3)

def test_multiclass_treatment():
    """
    Tests :func:`fatf.utils.models.metrics.multiclass_treatment` function.
    """
    cma = fumm.get_confusion_matrix(GROUND_TRUTH, PREDICTIONS)

    mtpr_0 = fumm.multiclass_treatment(cma, 0)
    assert mtpr_0 == pytest.approx(2 / 6, abs=1e-3)
    mtpr_1 = fumm.multiclass_treatment(cma, 1)
    assert mtpr_1 == pytest.approx(2 / 6, abs=1e-3)
    mtpr_2 = fumm.multiclass_treatment(cma, 2)
    assert mtpr_2 == pytest.approx(2 / 6, abs=1e-3)

def test_treatment():
    """
    Tests :func:`fatf.utils.models.metrics.treatment` function.
    """
    cma = fumm.get_confusion_matrix(GROUND_TRUTH_BIN, PREDICTIONS_BIN)
    mtpr = fumm.treatment(cma)
    assert mtpr == pytest.approx(11 / 18, abs=1e-3)


#!/usr/bin/env python
# coding: utf-8

# # Self-Driving Car Engineer Nanodegree
#
#
# ## Project: **Finding Lane Lines on the Road**
# ***
# In this project, you will use the tools you learned about in the lesson to identify lane lines on the road.  You can develop your pipeline on a series of individual images, and later apply the result to a video stream (really just a series of images). Check out the video clip "raw-lines-example.mp4" (also contained in this repository) to see what the output should look like after using the helper functions below.
#
# Once you have a result that looks roughly like "raw-lines-example.mp4", you'll need to get creative and try to average and/or extrapolate the line segments you've detected to map out the full extent of the lane lines.  You can see an example of the result you're going for in the video "P1_example.mp4".  Ultimately, you would like to draw just one line for the left side of the lane, and one for the right.
#
# In addition to implementing code, there is a brief writeup to complete. The writeup should be completed in a separate file, which can be either a markdown file or a pdf document. There is a [write up template](https://github.com/udacity/CarND-LaneLines-P1/blob/master/writeup_template.md) that can be used to guide the writing process. Completing both the code in the Ipython notebook and the writeup template will cover all of the [rubric points](https://review.udacity.com/#!/rubrics/322/view) for this project.
#
# ---
# Let's have a look at our first image called 'test_images/solidWhiteRight.jpg'.  Run the 2 cells below (hit Shift-Enter or the "play" button above) to display the image.
#
# **Note: If, at any point, you encounter frozen display windows or other confounding issues, you can always start again with a clean slate by going to the "Kernel" menu above and selecting "Restart & Clear Output".**
#
# ---

# **The tools you have are color selection, region of interest selection, grayscaling, Gaussian smoothing, Canny Edge Detection and Hough Tranform line detection.  You  are also free to explore and try other techniques that were not presented in the lesson.  Your goal is piece together a pipeline to detect the line segments in the image, then average/extrapolate them and draw them onto the image for display (as below).  Once you have a working pipeline, try it out on the video stream below.**
#
# ---
#
#

#
#

#

#
 Your output should look something like this (above) after detecting line segments using the helper functions below

#

#

#

#

#
#

#

#
 Your goal is to connect/average/extrapolate line segments to get output like this

#

#

# **Run the cell below to import some packages.  If you get an `import error` for a package you've already installed, try changing your kernel (select the Kernel menu above --> Change Kernel).  Still have problems?  Try relaunching Jupyter Notebook from the terminal prompt.  Also, consult the forums for more troubleshooting tips.**

# ## Import Packages

# In[1]:

#importing some useful packages
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import numpy as np
import cv2
get_ipython().run_line_magic('matplotlib', 'inline')

# ## Read in an Image

# In[2]:

#reading in an image
image = mpimg.imread('test_images/solidWhiteCurve.jpg')

#printing out some stats and plotting
print('This image is:', type(image), 'with dimensions:', image.shape)
plt.imshow(image)  # if you wanted to show a single color channel image called 'gray', for example, call as plt.imshow(gray, cmap='gray')

# ## Ideas for Lane Detection Pipeline

# **Some OpenCV functions (beyond those introduced in the lesson) that might be useful for this project are:**
#
# `cv2.inRange()` for color selection
# `cv2.fillPoly()` for regions selection
# `cv2.line()` to draw lines on an image given endpoints
# `cv2.addWeighted()` to coadd / overlay two images
# `cv2.cvtColor()` to grayscale or change color
# `cv2.imwrite()` to output images to file
# `cv2.bitwise_and()` to apply a mask to an image
#
# **Check out the OpenCV documentation to learn about these and discover even more awesome functionality!**

# ## Helper Functions

# Below are some helper functions to help get you started. They should look familiar from the lesson!

# In[24]:

import math

def grayscale(img):
    return cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # Or use BGR2GRAY if you read an image with cv2.imread()
    # return cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

def canny(img, low_threshold, high_threshold):
    return cv2.Canny(img, low_threshold, high_threshold)

def gaussian_blur(img, kernel_size):
    return cv2.GaussianBlur(img, (kernel_size, kernel_size), 0)

def region_of_interest(img, vertices):
    #defining a blank mask to start with
    mask = np.zeros_like(img)

    #defining a 3 channel or 1 channel color to fill the mask with depending on the input image
    if len(img.shape) > 2:
        channel_count = img.shape[2]  # i.e. 3 or 4 depending on your image
        ignore_mask_color = (255,) * channel_count
    else:
        ignore_mask_color = 255

    #filling pixels inside the polygon defined by "vertices" with the fill color
    cv2.fillPoly(mask, vertices, ignore_mask_color)

    #returning the image only where mask pixels are nonzero
    masked_image = cv2.bitwise_and(img, mask)
    return masked_image

def draw_lines(img, lines, color=[255, 0, 0], thickness=2):
    xl=yl=xr=yr=[]
    ymax=540
    ymin=320
    for line in lines:
        for x1,y1,x2,y2 in line:
            #cv2.line(img, (x1, y1), (x2, y2), (255, 0, 255), 3)
            #Calculate the slope for each line to find out, if the line belongs to the right lane or the left lane
            slope=((y2-y1)/(x2-x1))
            if (slope < 0):
                #Reject slopes that are too flat on the left side
                if slope > -0.4:
                    continue
                #reject x-coordinates that are on the right side of the picture
                if x1>600 and x2>600:
                    continue
                #add x and y coordinates to arrays
                xl=np.append(xl,x1)
                xl=np.append(xl,x2)
                yl=np.append(yl,y1)
                yl=np.append(yl,y2)
                #cv2.line(img, (x1, y1), (x2, y2), (255, 0, 255), 3)
            if (slope > 0):
                #Reject slopes that are too flat on the right side
                if slope < 0.4:
                    continue
                 #reject x-coordinates that are on the left side of the picture
                if x1 < 300 and x2 < 300:
                    continue
                #add x and y coordinates to arrays
                xr=np.append(xr,x1)
                xr=np.append(xr,x2)
                yr=np.append(yr,y1)
                yr=np.append(yr,y2)
                #cv2.line(img, (x1, y1), (x2, y2), (0, 255, 255), 3)

    ## one single line for each lane
    ## right lane
    #create a linear function out of the arrays containing the x and y coordinates
    right_lane_poly=np.poly1d(np.polyfit(yr,xr,1))
    #calculate the x-values for maximum y and minimum y
    xdownr=int(right_lane_poly(ymax))
    xupr=int(right_lane_poly(ymin))

    ##left lane
    #create a linear function out of the arrays containing the x and y coordinates
    left_lane_poly=np.poly1d(np.polyfit(yl,xl,1))
    #calculate the x-values for maximum y and minimum y
    xdownl=int(left_lane_poly(ymax))
    xupl=int(left_lane_poly(ymin))

    #draw the left and right lane
    cv2.line(img, (xdownl,ymax), (xupl, ymin), (255,0, 0), 10)
    cv2.line(img, (xdownr,ymax), (xupr, ymin), (255,0, 0), 10)

def hough_lines(img, rho, theta, threshold, min_line_len, max_line_gap):
    lines = cv2.HoughLinesP(img, rho, theta, threshold, np.array([]), minLineLength=min_line_len, maxLineGap=max_line_gap)
    line_img = np.zeros((img.shape[0], img.shape[1], 3), dtype=np.uint8)
    draw_lines(line_img, lines)
    return line_img

# Python 3 has support for cool math symbols.

def weighted_img(img, initial_img, α=0.8, β=1., γ=0.):
    return cv2.addWeighted(initial_img, α, img, β, γ)

# ## Test Images
#
# Build your pipeline to work on the images in the directory "test_images"
# **You should make sure your pipeline works well on these images before you try the videos.**

# In[4]:

import os
os.listdir("test_images/")

# ## Build a Lane Finding Pipeline
#
#

# Build the pipeline and run your solution on all test_images. Make copies into the `test_images_output` directory, and you can use the images in your writeup report.
#
# Try tuning the various parameters, especially the low and high Canny thresholds as well as the Hough lines parameters.

# In[25]:

# then save them to the test_images_output directory.
## 1: Convert to grayscale
gray=grayscale(image)
plt.imshow(gray,cmap='gray')

## 2: Smooth
gray_smooth=gaussian_blur(gray,3)
plt.imshow(gray_smooth,cmap='gray')

## 3: Canny Edge Detection
#Variables
lowth=100
highth=200
#Apply function
edges=canny(gray_smooth,lowth,highth)
plt.imshow(edges, cmap='gray')

## 4: Create mask
#Vertices of mask
leftlow=[30,540]
rightlow=[930,540]
leftup=[460,320]
rightup=[510,320]
#Create and apply mask
mask=np.array([[leftlow],[rightlow],[rightup],[leftup]])
area=region_of_interest(edges, [mask])
plt.imshow(area,cmap='gray')

## 5: Hough transformation
#Variables
rho=6
theta=np.pi/60
threshold=5
min_line_len=45
max_line_gap=60
#Apply function
HoughLines=hough_lines(area, rho, theta, threshold, min_line_len, max_line_gap)
plt.imshow(HoughLines)

#Final picture
Final=weighted_img(HoughLines, image, α=0.8, β=1., γ=0.)
plt.imshow(Final)

# ## Test on Videos
#
# You know what's cooler than drawing lanes over images? Drawing lanes over video!
#
# We can test our solution on two provided videos:
#
# `solidWhiteRight.mp4`
#
# `solidYellowLeft.mp4`
#
# **Note: if you get an import error when you run the next cell, try changing your kernel (select the Kernel menu above --> Change Kernel). Still have problems? Try relaunching Jupyter Notebook from the terminal prompt. Also, consult the forums for more troubleshooting tips.**
#
# **If you get an error that looks like this:**
# ```
# NeedDownloadError: Need ffmpeg exe.
# You can download it by calling:
# imageio.plugins.ffmpeg.download()
# ```
# **Follow the instructions in the error message and check out [this forum post](https://discussions.udacity.com/t/project-error-of-test-on-videos/274082) for more troubleshooting tips across operating systems.**

# In[6]:

# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML

# In[7]:

def process_image(image):
    # NOTE: The output you return should be a color image (3 channel) for processing video below
    # then save them to the test_images_output directory.
    ## 1: Convert to grayscale
    gray=grayscale(image)
    #plt.imshow(gray,cmap='gray')

    ## 2: Smooth
    gray_smooth=gaussian_blur(gray,9)
    #plt.imshow(gray_smooth)

    ## 3: Canny Edge Detection
    #Variables
    lowth=50
    highth=180
    #Apply function
    edges=canny(gray_smooth,lowth,highth)
    #plt.imshow(edges)

    ## 4: Create mask
    #Vertices of mask
    leftlow=[30,540]
    rightlow=[930,540]
    leftup=[460,320]
    rightup=[510,320]
    #Create and apply mask
    mask=np.array([[leftlow],[rightlow],[rightup],[leftup]])
    area=region_of_interest(edges, [mask])
    plt.imshow(area,cmap='gray')

    ## 5: Hough transformation
    #Variables
    rho=6
    theta=np.pi/60
    threshold=5
    min_line_len=45
    max_line_gap=60
    #Apply function
    HoughLines=hough_lines(area, rho, theta, threshold, min_line_len, max_line_gap)
    plt.imshow(HoughLines)

    #Final picture
    result=weighted_img(HoughLines, image, α=0.8, β=1., γ=0.)
    #plt.imshow(Final)
    # you should return the final output (image where lines are drawn on lanes)
    return result

# Let's try the one with the solid white lane on the right first ...

# In[8]:

white_output = 'test_videos_output/solidWhiteRight.mp4'
## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0,5)
clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4")
white_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
get_ipython().run_line_magic('time', 'white_clip.write_videofile(white_output, audio=False)')

# Play the video inline, or if you prefer find the video in your filesystem (should be in the same directory) and play it in your video player of choice.

# In[9]:

HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(white_output))

# ## Improve the draw_lines() function
#
# **At this point, if you were successful with making the pipeline and tuning parameters, you probably have the Hough line segments drawn onto the road, but what about identifying the full extent of the lane and marking it clearly as in the example video (P1_example.mp4)?  Think about defining a line to run the full length of the visible lane based on the line segments you identified with the Hough Transform. As mentioned previously, try to average and/or extrapolate the line segments you've detected to map out the full extent of the lane lines. You can see an example of the result you're going for in the video "P1_example.mp4".**
#
# **Go back and modify your draw_lines function accordingly and try re-running your pipeline. The new output should draw a single, solid line over the left lane line and a single, solid line over the right lane line. The lines should start from the bottom of the image and extend out to the top of the region of interest.**

# Now for the one with the solid yellow lane on the left. This one's more tricky!

# In[10]:

yellow_output = 'test_videos_output/solidYellowLeft.mp4'
## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip2 = VideoFileClip('test_videos/solidYellowLeft.mp4').subclip(0,5)
clip2 = VideoFileClip('test_videos/solidYellowLeft.mp4')
yellow_clip = clip2.fl_image(process_image)
get_ipython().run_line_magic('time', 'yellow_clip.write_videofile(yellow_output, audio=False)')

# In[11]:

HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(yellow_output))

# ## Writeup and Submission
#
# If you're satisfied with your video outputs, it's time to make the report writeup in a pdf or markdown file. Once you have this Ipython notebook ready along with the writeup, it's time to submit for review! Here is a [link](https://github.com/udacity/CarND-LaneLines-P1/blob/master/writeup_template.md) to the writeup template file.
#

# ## Optional Challenge
#
# Try your lane finding pipeline on the video below.  Does it still work?  Can you figure out a way to make it more robust?  If you're up for the challenge, modify your pipeline so it works with this video and submit it along with the rest of your project!

# In[12]:

challenge_output = 'test_videos_output/challenge.mp4'
## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip3 = VideoFileClip('test_videos/challenge.mp4').subclip(0,5)
clip3 = VideoFileClip('test_videos/challenge.mp4')
challenge_clip = clip3.fl_image(process_image)
get_ipython().run_line_magic('time', 'challenge_clip.write_videofile(challenge_output, audio=False)')

# In[13]:

HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(challenge_output))


import random

import numpy as np

def set_seed(random_state: int = 42) -> None:
    """Function fixes random state to ensure results are reproducible"""
    np.random.seed(random_state)
    random.seed(random_state)


# testDriver.py
"""Volume 2 Lab 16: Simplex. Test Driver."""

import signal
import numpy as np
from functools import wraps
from solutions import SimplexSolver, prob7

# Wrapper =====================================================================
class TimeoutError(Exception):
    pass

def _timeout(seconds):
    """Decorator for preventing a function from running for too long.

    Inputs:
        seconds (int): The number of seconds allowed.

    Notes:
        This decorator uses signal.SIGALRM, which is only available on Unix.
    """
    assert isinstance(seconds, int), "@timeout(sec) requires an int"

    def _handler(signum, frame):
        """Handle the alarm by raising a custom exception."""
        raise TimeoutError("Timeout after {0} seconds".format(seconds))

    def decorator(func):
        def wrapper(*args, **kwargs):
            signal.signal(signal.SIGALRM, _handler)
            signal.alarm(seconds)               # Set the alarm.
            try:
                result = func(*args, **kwargs)
            finally:
                signal.alarm(0)                 # Turn the alarm off.
            return result
        return wraps(func)(wrapper)
    return decorator

# Test Script and Class =======================================================

def test(student_module):
    """Test script. Import the student's solutions file as a module.

    30 points for the Simplex Solver (example problem)
    10 points for problem 7: Product Mix.

    Inputs:
        student_module: the imported module for the student's file.

    Returns:
        score (int): the student's score, out of 40.
        feedback (str): a printout of test results for the student.
    """
    driver = _testDriver()
    driver.test_all(student_module)
    return driver.score, driver.feedback

class _testDriver(object):
    """Class for testing a student's work. See test.__doc__ for more info."""

    # Constructor -------------------------------------------------------------
    def __init__(self):
        """Initialize the feedback attribute."""
        self.feedback = ""

    # Main routine -----------------------------------------------------------
    def test_all(self, student_module, total=40):
        """Grade the provided module on each problem and compile feedback."""
        # Reset feedback and score.
        self.feedback = ""
        self.score = 0

        def test_one(problem, number, value):
            """Test a single problem, checking for errors."""
            try:
                self.feedback += "\n\nProblem {} ({} points):".format(
                                                                number, value)
                points = problem(student_module)
                self.score += points
                self.feedback += "\nScore += {}".format(points)
            except BaseException as e:
                self.feedback += "\n{}: {}".format(self._errType(e), e)

        # Grade each problem.
        test_one(self.problem6, '1-6', 30)   # Problems 1-6: 30 points.
        test_one(self.problem7, '7',   10)   # Problem 7:    10 points.

        # Report final score.
        percentage = (100. * self.score) / total
        self.feedback += "\n\nTotal score: {}/{} = {}%".format(
                                    self.score, total, round(percentage, 2))
        if   percentage >=  98: self.feedback += "\n\nExcellent!"
        elif percentage >=  90: self.feedback += "\n\nGreat job!"

        # Add comments (optionally).
        print(self.feedback)
        comments = str(raw_input("Comments: "))
        if len(comments) > 0:
            self.feedback += '\n\n\nComments:\n\t{}'.format(comments)

    # Helper Functions --------------------------------------------------------
    @staticmethod
    def _errType(error):
        """Get just the name of the exception 'error' in string format."""
        if isinstance(error, TimeoutError):
            return "TimeoutError"
        if isinstance(error, BaseException):
            return str(type(error)).lstrip("<type 'exceptions.").rstrip("'>")
        else:
            return str(error)

    def _eqTest(self, correct, student, message):
        """Test to see if 'correct' and 'student' are equal.
        Report the given 'message' if they are not.
        """
        if np.allclose(correct, student):
            return 1
        else:
            self.feedback += "\n{}".format(message)
            self.feedback += "\n\tCorrect response: {}".format(correct)
            self.feedback += "\n\tStudent response: {}".format(student)
            return 0

    def _dicTest(self, correct, student):
        """Test to see if the dictionaries 'correct' and 'student' have the
        same string representation. Report the given 'message' if they are not.
        """
        assert isinstance(correct, dict), "Expected a dictionary"
        if not isinstance(student, dict):
            self.feedback += "\nExpected a dictionary"
            return 0

        points = 0
        success = True
        for key in correct:
            if key in student:
                points += 1
                if correct[key] == student[key]:
                    points += 1
                else:
                    success = False
            else:
                success = False
        if not success:
            self.feedback += "\n{}".format("Incorrect Dictionary")
            self.feedback += "\n\tCorrect response: {}".format(correct)
            self.feedback += "\n\tStudent response: {}".format(student)
        return points

    # Problems ----------------------------------------------------------------
    def test_simplex(self, s, c, b, A, simple=False):
        """Test the student's SimplexSolver class on the linear program
                maximize    c^t x
                subject to  Ax <= b
        If simple=True, the amount of points earned is 15. If simple=False,
        the amount of points earned is
            15 + number of basic variables + number of nonbasic variables
        """
        points = 0
        key = SimplexSolver(c, A, b)
        student = s.SimplexSolver(c, A, b)
        sol1, sol2 = key.solve(), student.solve()

        # Test the primal objective.
        points += 15 * self._eqTest(sol1[0], sol2[0],
                                            "Incorrect primal objective")
        if simple is False:
            # Test the basic and nonbasic variable dictionaries.
            points += self._dicTest(sol1[1], sol2[1])
            points += self._dicTest(sol1[2], sol2[2])

        return points

    @_timeout(3)
    def problem6(self, s):
        """Test the SimplexSolver class. 30 points."""
        points = 0

        # Test SimplexSolver on a system that is infeasible at the origin.
        c = np.array([3., 2.])
        b = np.array([2., 5., -7.])
        A = np.array([[1., -1.],
                      [3.,  1.],
                      [4.,  3.]])
        try:
            self.test_simplex(s, c, b, A, simple=True)
        except ValueError as e:
            points += 5
        else:
            self.feedback += "\nExpected ValueError for infeasible system"

        # Test SimplexSolver on a valid, closed system.
        b = np.array([2., 5., 7.])
        points += self.test_simplex(s, c, b, A, simple=False)

        return points

    @_timeout(3)
    def problem7(self, s):
        """Test prob7(). 10 points."""

        sol1, sol2 = prob7(), s.prob7()
        c = np.load('productMix.npz')['p']
        sol1, sol2 = c.dot(sol1), c.dot(sol2)

        # primal = 7453.59649123
        # assert np.allclose(sol1, primal)

        return 10 * self._eqTest(sol1, sol2, "Incorrect maximizer")

# END OF FILE =================================================================


#!/usr/bin/env python
# coding: utf-8

# # Training Deep Neural Networks on a GPU with PyTorch
#
# ### Part 4 of "Deep Learning with Pytorch: Zero to GANs"
#
# This tutorial series is a hands-on beginner-friendly introduction to deep learning using [PyTorch](https://pytorch.org), an open-source neural networks library. These tutorials take a practical and coding-focused approach. The best way to learn the material is to execute the code and experiment with it yourself. Check out the full series here:
#
# 1. [PyTorch Basics: Tensors & Gradients](https://jovian.ai/aakashns/01-pytorch-basics)
# 2. [Gradient Descent & Linear Regression](https://jovian.ai/aakashns/02-linear-regression)
# 3. [Working with Images & Logistic Regression](https://jovian.ai/aakashns/03-logistic-regression)
# 4. [Training Deep Neural Networks on a GPU](https://jovian.ai/aakashns/04-feedforward-nn)
# 5. [Image Classification using Convolutional Neural Networks](https://jovian.ai/aakashns/05-cifar10-cnn)
# 6. [Data Augmentation, Regularization and ResNets](https://jovian.ai/aakashns/05b-cifar10-resnet)
# 7. [Generating Images using Generative Adversarial Networks](https://jovian.ai/aakashns/06b-anime-dcgan/)
#

#  This tutorial covers the following topics:
#
#  * Creating a deep neural network with hidden layers
#  * Using a non-linear activation function
#  * Using a GPU (when available) to speed up training
#  * Experimenting with hyperparameters to improve the model

# ### How to run the code
#
# This tutorial is an executable [Jupyter notebook](https://jupyter.org) hosted on [Jovian](https://www.jovian.ai). You can _run_ this tutorial and experiment with the code examples in a couple of ways: *using free online resources* (recommended) or *on your computer*.
#
# #### Option 1: Running using free online resources (1-click, recommended)
#
# The easiest way to start executing the code is to click the **Run** button at the top of this page and select **Run on Colab**. [Google Colab](https://colab.research.google.com) is a free online platform for running Jupyter notebooks using Google's cloud infrastructure. You can also select "Run on Binder" or "Run on Kaggle" if you face issues running the notebook on Google Colab.
#
#
# #### Option 2: Running on your computer locally
#
# To run the code on your computer locally, you'll need to set up [Python](https://www.python.org), download the notebook and install the required libraries. We recommend using the [Conda](https://docs.conda.io/projects/conda/en/latest/user-guide/install/) distribution of Python. Click the **Run** button at the top of this page, select the **Run Locally** option, and follow the instructions.
#
# >  **Jupyter Notebooks**: This tutorial is a [Jupyter notebook](https://jupyter.org) - a document made of _cells_. Each cell can contain code written in Python or explanations in plain English. You can execute code cells and view the results, e.g., numbers, messages, graphs, tables, files, etc., instantly within the notebook. Jupyter is a powerful platform for experimentation and analysis. Don't be afraid to mess around with the code & break things - you'll learn a lot by encountering and fixing errors. You can use the "Kernel > Restart & Clear Output" or "Edit > Clear Outputs" menu option to clear all outputs and start again from the top.

# ### Using a GPU for faster training
#
# You can use a [Graphics Processing Unit](https://en.wikipedia.org/wiki/Graphics_processing_unit) (GPU) to train your models faster if your execution platform is connected to a GPU manufactured by NVIDIA. Follow these instructions to use a GPU on the platform of your choice:
#
# * _Google Colab_: Use the menu option "Runtime > Change Runtime Type" and select "GPU" from the "Hardware Accelerator" dropdown.
# * _Kaggle_: In the "Settings" section of the sidebar, select "GPU" from the "Accelerator" dropdown. Use the button on the top-right to open the sidebar.
# * _Binder_: Notebooks running on Binder cannot use a GPU, as the machines powering Binder aren't connected to any GPUs.
# * _Linux_: If your laptop/desktop has an NVIDIA GPU (graphics card), make sure you have installed the [NVIDIA CUDA drivers](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html).
# * _Windows_: If your laptop/desktop has an NVIDIA GPU (graphics card), make sure you have installed the [NVIDIA CUDA drivers](https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html).
# * _macOS_: macOS is not compatible with NVIDIA GPUs
#
#
# If you do not have access to a GPU or aren't sure what it is, don't worry, you can execute all the code in this tutorial just fine without a GPU.

# ## Preparing the Data
#
# In [the previous tutorial](https://jovian.ai/aakashns/03-logistic-regression), we trained a logistic regression model to identify handwritten digits from the MNIST dataset with an accuracy of around 86%. The dataset consists of 28px by 28px grayscale images of handwritten digits (0 to 9) and labels for each image indicating which digit it represents. Here are some sample images from the dataset:
#
# ![mnist-sample](https://i.imgur.com/CAYnuo1.jpg)
#
# We noticed that it's quite challenging to improve the accuracy of a logistic regression model beyond 87%, since the model assumes a linear relationship between pixel intensities and image labels. In this post, we'll try to improve upon it  using a *feed-forward neural network* which can capture non-linear relationships between inputs and targets.
#
# Let's begin by installing and importing the required modules and classes from `torch`, `torchvision`, `numpy`, and `matplotlib`.

# In[1]:

# Uncomment and run the appropriate command for your operating system, if required

# Linux / Binder
# !pip install numpy matplotlib torch==1.7.0+cpu torchvision==0.8.1+cpu torchaudio==0.7.0 -f https://download.pytorch.org/whl/torch_stable.html

# MacOS
# !pip install numpy matplotlib torch torchvision torchaudio

# In[2]:

import torch
import torchvision
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import torch.nn as nn
import torch.nn.functional as F
from torchvision.datasets import MNIST
from torchvision.transforms import ToTensor
from torchvision.utils import make_grid
from torch.utils.data.dataloader import DataLoader
from torch.utils.data import random_split

# Use a white background for matplotlib figures
matplotlib.rcParams['figure.facecolor'] = '#ffffff'

# We can download the data and create a PyTorch dataset using the `MNIST` class from `torchvision.datasets`.

# In[3]:

dataset = MNIST(root='data/', download=True, transform=ToTensor())

# Let's look at a couple of images from the dataset. The images are converted to PyTorch tensors with the shape `1x28x28` (the dimensions represent color channels, width and height). We can use `plt.imshow` to display the images. However, `plt.imshow` expects channels to be last dimension in an image tensor, so we use the `permute` method to reorder the dimensions of the image.

# In[4]:

image, label = dataset[0]
print('image.shape:', image.shape)
image_perm = image.permute(1, 2, 0)
print(image_perm.shape)
plt.imshow(image[0], cmap='gray')
print('Label:', label)

# In[5]:

image, label = dataset[0]
print('image.shape:', image.shape)
plt.imshow(image[0], cmap='gray')
print('Label:', label)

# Next, let's use the `random_split` helper function to set aside 10000 images for our validation set.

# In[6]:

val_size = 10000
train_size = len(dataset) - val_size

train_ds, val_ds = random_split(dataset, [train_size, val_size])
len(train_ds), len(val_ds)

# We can now create PyTorch data loaders for training and validation.

# In[7]:

batch_size=128

# In[8]:

train_loader = DataLoader(train_ds, batch_size, shuffle=True, num_workers=4, pin_memory=True)
val_loader = DataLoader(val_ds, batch_size*2, num_workers=4, pin_memory=True)

print(torch.cuda.device_count())

# Can you figure out the purpose of the arguments `num_workers` and `pin_memory`? Try looking into the documentation: https://pytorch.org/docs/stable/data.html .
#
# Let's visualize a batch of data in a grid using the `make_grid` function from `torchvision`. We'll also use the `.permute` method on the tensor to move the channels to the last dimension, as expected by `matplotlib`.

# In[9]:

for images, _ in train_loader:
    print('images.shape:', images.shape)
    plt.figure(figsize=(16,8))
    plt.axis('off')
    plt.imshow(make_grid(images, nrow=16).permute((1, 2, 0)))
    break

# ## Hidden Layers, Activation Functions and Non-Linearity
#
# We'll create a neural network with two layers: a _hidden layer_ and an _output layer_. Additionally, we'll use an _activation function_ between the two layers. Let's look at a step-by-step example to learn how hidden layers and activation functions can help capture non-linear relationships between inputs and outputs.
#
# First, let's create a batch of inputs tensors. We'll flatten the `1x28x28` images into vectors of size `784`, so they can be passed into an `nn.Linear` object.

# In[11]:

for images, labels in train_loader:
    print('images.shape:', images.shape)
    inputs = images.reshape(-1, 784)
    print('inputs.shape:', inputs.shape)
    break

# Next, let's create a `nn.Linear` object, which will serve as our _hidden_ layer. We'll set the size of the output from the hidden layer to 32. This number can be increased or decreased to change the _learning capacity_ of the model.

# In[59]:

input_size = inputs.shape[-1]
hidden_size = 256

# In[60]:

layer1 = nn.Linear(input_size, hidden_size)

# We can now compute intermediate outputs for the batch of images by passing `inputs` through `layer1`.

# In[61]:

inputs.shape

# In[62]:

layer1_outputs = layer1(inputs)
print('layer1_outputs.shape:', layer1_outputs.shape)

# The image vectors of size `784` are transformed into intermediate output vectors of length `32` by performing a matrix multiplication of `inputs` matrix with the transposed weights matrix of `layer1` and adding the bias. We can verify this using `torch.allclose`. For a more detailed explanation, review the tutorial on [linear regression](https://jovian.ai/aakashns/02-linear-regression).

# In[63]:

layer1_outputs_direct = inputs @ layer1.weight.t() + layer1.bias
layer1_outputs_direct.shape

# In[64]:

torch.allclose(layer1_outputs, layer1_outputs_direct, 1e-3)

# Thus, `layer1_outputs` and `inputs` have a linear relationship, i.e., each element of `layer_outputs` is a weighted sum of elements from `inputs`. Thus, even as we train the model and modify the weights, `layer1` can only capture linear relationships between `inputs` and `outputs`.
#
#
#

# Next, we'll use the Rectified Linear Unit (ReLU) function as the activation function for the outputs. It has the formula `relu(x) = max(0,x)` i.e. it simply replaces negative values in a given tensor with the value 0. ReLU is a non-linear function, as seen here visually:
#
#
#
# We can use the `F.relu` method to apply ReLU to the elements of a tensor.

# In[65]:

F.relu(torch.tensor([[1, -1, 0],
                     [-0.1, .2, 3]]))

# Let's apply the activation function to `layer1_outputs` and verify that negative values were replaced with 0.

# In[66]:

relu_outputs = F.relu(layer1_outputs)
print('min(layer1_outputs):', torch.min(layer1_outputs).item())
print('min(relu_outputs):', torch.min(relu_outputs).item())

# Now that we've applied a non-linear activation function, `relu_outputs` and `inputs` do not have a linear relationship. We refer to `ReLU` as the _activation function_, because for each input certain outputs are activated (those with non-zero values) while others turned off (those with zero values)
#
# Next, let's create an output layer to convert vectors of length `hidden_size` in `relu_outputs` into vectors of length 10, which is the desired output of our model (since there are 10 target labels).

# In[67]:

output_size = 10
layer2 = nn.Linear(hidden_size, output_size)

# In[68]:

layer2_outputs = layer2(relu_outputs)
print(layer2_outputs.shape)

# In[69]:

inputs.shape

# As expected, `layer2_outputs` contains a batch of vectors of size 10. We can now use this output to compute the loss using `F.cross_entropy` and adjust the weights of `layer1` and `layer2` using gradient descent.

# In[70]:

F.cross_entropy(layer2_outputs, labels)

# Thus, our model transforms `inputs` into `layer2_outputs` by applying a linear transformation (using `layer1`), followed by a non-linear activation (using `F.relu`), followed by another linear transformation (using `layer2`). Let's verify this by re-computing the output using basic matrix operations.

# In[71]:

# Expanded version of layer2(F.relu(layer1(inputs)))
outputs = (F.relu(inputs @ layer1.weight.t() + layer1.bias)) @ layer2.weight.t() + layer2.bias

# In[72]:

torch.allclose(outputs, layer2_outputs, 1e-3)

# Note that `outputs` and `inputs` do not have a linear relationship due to the non-linear activation function `F.relu`. As we train the model and adjust the weights of `layer1` and `layer2`, we can now capture non-linear relationships between the images and their labels. In other words, introducing non-linearity makes the model more powerful and versatile. Also, since `hidden_size` does not depend on the dimensions of the inputs or outputs, we vary it to increase the number of parameters within the model. We can also introduce new hidden layers and apply the same non-linear activation after each hidden layer.
#
# The model we just created is called a neural network. A _deep neural network_ is simply a neural network with one or more hidden layers. In fact, the [Universal Approximation Theorem](http://neuralnetworksanddeeplearning.com/chap4.html) states that a sufficiently large & deep neural network can compute any arbitrary function i.e. it can _learn_ rich and complex non-linear relationships between inputs and targets. Here are some examples:
#
# * Identifying if an image contains a cat or a dog (or [something else](https://machinelearningmastery.com/introduction-to-the-imagenet-large-scale-visual-recognition-challenge-ilsvrc/))
# * Identifying the genre of a song using a 10-second sample
# * Classifying movie reviews as positive or negative based on their content
# * Navigating self-driving cars using a video feed of the road
# * Translating sentences from English to French (and hundreds of other languages)
# * Converting a speech recording to text and vice versa
# * And many more...
#
# It's hard to imagine how the simple process of multiplying inputs with randomly initialized matrices, applying non-linear activations, and adjusting weights repeatedly using gradient descent can yield such astounding results. Deep learning models often contain millions of parameters, which can together capture far more complex relationships than the human brain can comprehend.
#
# If we hadn't included a non-linear activation between the two linear layers, the final relationship between inputs and outputs would still be linear. A simple refactoring of the computations illustrates this.

# In[73]:

# Same as layer2(layer1(inputs))
outputs2 = (inputs @ layer1.weight.t() + layer1.bias) @ layer2.weight.t() + layer2.bias

# In[74]:

# Create a single layer to replace the two linear layers
combined_layer = nn.Linear(input_size, output_size)

combined_layer.weight.data = layer2.weight @ layer1.weight
combined_layer.bias.data = layer1.bias @ layer2.weight.t() + layer2.bias

# In[75]:

# Same as combined_layer(inputs)
outputs3 = inputs @ combined_layer.weight.t() + combined_layer.bias

# In[76]:

torch.allclose(outputs2, outputs3, 1e-3)

# ### Save and upload your notebook
#
# Whether you're running this Jupyter notebook online or on your computer, it's essential to save your work from time to time. You can continue working on a saved notebook later or share it with friends and colleagues to let them execute your code. [Jovian](https://jovian.ai/platform-features) offers an easy way of saving and sharing your Jupyter notebooks online.

# In[39]:

# In[40]:

# In[41]:

# `jovian.commit` uploads the notebook to your Jovian account, captures the Python environment, and creates a shareable link for your notebook, as shown above. You can use this link to share your work and let anyone (including you) run your notebooks and reproduce your work.

# ## Model
#
# We are now ready to define our model. As discussed above, we'll create a neural network with one hidden layer. Here's what that means:
#
# * Instead of using a single `nn.Linear` object to transform a batch of inputs (pixel intensities) into outputs (class probabilities), we'll use two `nn.Linear` objects. Each of these is called a _layer_ in the network.
#
# * The first layer (also known as the hidden layer) will transform the input matrix of shape `batch_size x 784` into an intermediate output matrix of shape `batch_size x hidden_size`. The parameter `hidden_size` can be configured manually (e.g., 32 or 64).
#
# * We'll then apply a non-linear *activation function* to the intermediate outputs. The activation function transforms individual elements of the matrix.
#
# * The result of the activation function, which is also of size `batch_size x hidden_size`, is passed into the second layer (also known as the output layer).  The second layer transforms it into a matrix of size `batch_size x 10`. We can use this output to compute the loss and adjust weights using gradient descent.
#
#
# As discussed above, our model will contain one hidden layer. Here's what it looks like visually:
#
#
#
#
# Let's define the model by extending the `nn.Module` class from PyTorch.

# In[275]:

class MnistModel(nn.Module):
    """Feedfoward neural network with 1 hidden layer"""
    def __init__(self, in_size, hidden_size, out_size):
        super().__init__()

        self.hidden_count = 1
        self.hidden_layers = []

        # hidden layer
        self.linear1 = nn.Linear(in_size, hidden_size)
        # output layer

        for x in range(self.hidden_count) :
           self.hidden_layers.append(nn.Linear(hidden_size, hidden_size));

        self.linear4 = nn.Linear(hidden_size, out_size)

    def forward(self, xb):
        # Flatten the image tensors
        xb = xb.view(xb.size(0), -1)
        # Get intermediate outputs using hidden layer
        out = self.linear1(xb)
        # Apply activation function
        out = F.relu(out)

        #for layer in self.hidden_layers:
        #    out = layer(out)
        #    out = F.relu(out)

        # Get predictions using output layer
        out = self.linear4(out)
        return out

    def training_step(self, batch):
        images, labels = batch
        out = self(images)                  # Generate predictions
        loss = F.cross_entropy(out, labels) # Calculate loss
        return loss

    def validation_step(self, batch):
        images, labels = batch
        out = self(images)                    # Generate predictions
        loss = F.cross_entropy(out, labels)   # Calculate loss
        acc = accuracy(out, labels)           # Calculate accuracy
        return {'val_loss': loss, 'val_acc': acc}

    def validation_epoch_end(self, outputs):
        batch_losses = [x['val_loss'] for x in outputs]
        epoch_loss = torch.stack(batch_losses).mean()   # Combine losses
        batch_accs = [x['val_acc'] for x in outputs]
        epoch_acc = torch.stack(batch_accs).mean()      # Combine accuracies
        return {'val_loss': epoch_loss.item(), 'val_acc': epoch_acc.item()}

    def epoch_end(self, epoch, result):
        print("Epoch [{}], val_loss: {:.4f}, val_acc: {:.4f}".format(epoch, result['val_loss'], result['val_acc']))

# We also need to define an `accuracy` function which calculates the accuracy of the model's prediction on an batch of inputs. It's used in `validation_step` above.

# In[276]:

def accuracy(outputs, labels):
    _, preds = torch.max(outputs, dim=1)
    return torch.tensor(torch.sum(preds == labels).item() / len(preds))

# We'll create a model that contains a hidden layer with 32 activations.

# In[277]:

input_size = 784
hidden_size = 1024 # you can change this
num_classes = 10

# In[278]:

model = MnistModel(input_size, hidden_size=1024, out_size=num_classes)

# Let's take a look at the model's parameters. We expect to see one weight and bias matrix for each of the layers.

# In[279]:

for t in model.parameters():
    print(t.shape)

# Let's try and generate some outputs using our model. We'll take the first batch of 128 images from our dataset and pass them into our model.

# In[280]:

for images, labels in train_loader:
    outputs = model(images)
    loss = F.cross_entropy(outputs, labels)
    print('Loss:', loss.item())
    break

print('outputs.shape : ', outputs.shape)
print('Sample outputs :\n', outputs[:2].data)

# ## Using a GPU
#
# As the sizes of our models and datasets increase, we need to use GPUs to train our models within a reasonable amount of time. GPUs contain hundreds of cores optimized for performing expensive matrix operations on floating-point numbers quickly, making them ideal for training deep neural networks. You can use GPUs for free on [Google Colab](https://colab.research.google.com/) and [Kaggle](https://www.kaggle.com/kernels) or rent GPU-powered machines on services like [Google Cloud Platform](https://cloud.google.com/gpu/), [Amazon Web Services](https://docs.aws.amazon.com/dlami/latest/devguide/gpu.html), and [Paperspace](https://www.paperspace.com/).
#
# We can check if a GPU is available and the required NVIDIA CUDA drivers are installed using `torch.cuda.is_available`.

# In[281]:

torch.cuda.is_available()

# Let's define a helper function to ensure that our code uses the GPU if available and defaults to using the CPU if it isn't.

# In[282]:

def get_default_device():
    """Pick GPU if available, else CPU"""
    if torch.cuda.is_available():
        return torch.device('cuda')
    else:
        return torch.device('cpu')

# In[283]:

device = get_default_device()

# Next, let's define a function that can move data and model to a chosen device.

# In[284]:

def to_device(data, device):
    """Move tensor(s) to chosen device"""
    if isinstance(data, (list,tuple)):
        return [to_device(x, device) for x in data]
    return data.to(device, non_blocking=True)

# In[285]:

for images, labels in train_loader:
    print(images.shape)
    images = to_device(images, device)
    print(images.device)
    break

# Finally, we define a `DeviceDataLoader` class to wrap our existing data loaders and move batches of data to the selected device. Interestingly, we don't need to extend an existing class to create a PyTorch datal oader. All we need is an `__iter__` method to retrieve batches of data and an `__len__` method to get the number of batches.

# In[286]:

class DeviceDataLoader():
    """Wrap a dataloader to move data to a device"""
    def __init__(self, dl, device):
        self.dl = dl
        self.device = device

    def __iter__(self):
        """Yield a batch of data after moving it to device"""
        for b in self.dl:
            yield to_device(b, self.device)

    def __len__(self):
        """Number of batches"""
        return len(self.dl)

# The `yield` keyword in Python is used to create a generator function that can be used within a `for` loop, as illustrated below.

# In[287]:

def some_numbers():
    yield 10
    yield 20
    yield 30

for value in some_numbers():
    print(value)

# We can now wrap our data loaders using `DeviceDataLoader`.

# In[288]:

train_loader = DeviceDataLoader(train_loader, device)
val_loader = DeviceDataLoader(val_loader, device)

# Tensors moved to the GPU have a `device` property which includes that word `cuda`. Let's verify this by looking at a batch of data from `valid_dl`.

# In[289]:

for xb, yb in val_loader:
    print('xb.device:', xb.device)
    print('yb:', yb)
    break

# ## Training the Model
#
# We'll define two functions: `fit` and `evaluate` to train the model using gradient descent and evaluate its performance on the validation set. For a detailed walkthrough of these functions, check out the [previous tutorial](https://jovian.ai/aakashns/03-logistic-regression).

# In[290]:

def evaluate(model, val_loader):
    """Evaluate the model's performance on the validation set"""
    outputs = [model.validation_step(batch) for batch in val_loader]
    return model.validation_epoch_end(outputs)

def fit(epochs, lr, model, train_loader, val_loader, opt_func=torch.optim.SGD):
    """Train the model using gradient descent"""
    history = []
    optimizer = opt_func(model.parameters(), lr)
    for epoch in range(epochs):
        # Training Phase
        for batch in train_loader:
            loss = model.training_step(batch)
            loss.backward()
            optimizer.step()
            optimizer.zero_grad()
        # Validation phase
        result = evaluate(model, val_loader)
        model.epoch_end(epoch, result)
        history.append(result)
    return history

# Before we train the model, we need to ensure that the data and the model's parameters (weights and biases) are on the same device (CPU or GPU). We can reuse the `to_device` function to move the model's parameters to the right device.

# In[291]:

# Model (on GPU)
model = MnistModel(input_size, hidden_size=hidden_size, out_size=num_classes)
to_device(model, device)

# Let's see how the model performs on the validation set with the initial set of weights and biases.

# In[292]:

history = [evaluate(model, val_loader)]

# The initial accuracy is around 10%, as one might expect from a randomly initialized model (since it has a 1 in 10 chance of getting a label right by guessing randomly).
#
# Let's train the model for five epochs and look at the results. We can use a relatively high learning rate of 0.5.

# In[293]:

history += fit(5, 0.5, model, train_loader, val_loader)

# 96% is pretty good! Let's train the model for five more epochs at a lower learning rate of 0.1 to improve the accuracy further.

# In[179]:

history += fit(10, 0.001, model, train_loader, val_loader)

# We can now plot the losses & accuracies to study how the model improves over time.

# In[180]:

losses = [x['val_loss'] for x in history]
plt.plot(losses, '-x')
plt.xlabel('epoch')
plt.ylabel('loss')
plt.title('Loss vs. No. of epochs');

# In[181]:

accuracies = [x['val_acc'] for x in history]
plt.plot(accuracies, '-x')
plt.xlabel('epoch')
plt.ylabel('accuracy')
plt.title('Accuracy vs. No. of epochs');

# Our current model outperforms the logistic regression model (which could only achieve around 86% accuracy) by a considerable margin! It quickly reaches an accuracy of 97% but doesn't improve much beyond this. To improve accuracy further, we need to make the model more powerful by increasing the hidden layer's size or adding more hidden layers with activations. I encourage you to try out both these approaches and see which one works better.

# As a final step, we can save and commit our work using the `jovian` library.

# ## Testing with individual images
#
# While we have been tracking the overall accuracy of a model so far, it's also a good idea to look at model's results on some sample images. Let's test out our model with some images from the predefined test dataset of 10000 images. We begin by recreating the test dataset with the `ToTensor` transform.

# In[69]:

# Define test dataset
test_dataset = MNIST(root='data/',
                     train=False,
                     transform=ToTensor())

# Let's define a helper function `predict_image`, which returns the predicted label for a single image tensor.

# In[70]:

def predict_image(img, model):
    xb = to_device(img.unsqueeze(0), device)
    yb = model(xb)
    _, preds  = torch.max(yb, dim=1)
    return preds[0].item()

# Let's try it out with a few images.

# In[71]:

img, label = test_dataset[0]
plt.imshow(img[0], cmap='gray')
print('Label:', label, ', Predicted:', predict_image(img, model))

# In[72]:

img, label = test_dataset[1839]
plt.imshow(img[0], cmap='gray')
print('Label:', label, ', Predicted:', predict_image(img, model))

# In[73]:

img, label = test_dataset[193]
plt.imshow(img[0], cmap='gray')
print('Label:', label, ', Predicted:', predict_image(img, model))

# Identifying where our model performs poorly can help us improve the model, by collecting more training data, increasing/decreasing the complexity of the model, and changing the hypeparameters.
#
# As a final step, let's also look at the overall loss and accuracy of the model on the test set.

# In[74]:

test_loader = DeviceDataLoader(DataLoader(test_dataset, batch_size=256), device)
result = evaluate(model, test_loader)
result

# We expect this to be similar to the accuracy/loss on the validation set. If not, we might need a better validation set that has similar data and distribution as the test set (which often comes from real world data).

# Let's save the model's weights and attach it to the notebook using `jovian.commit`. We will also record the model's performance on the test dataset using `jovian.log_metrics`.

# In[75]:

# In[76]:

torch.save(model.state_dict(), 'mnist-feedforward.pth')

# In[ ]:

# ## Exercises
#
# Try out the following exercises to apply the concepts and techniques you have learned so far:
#
# * Coding exercises on end-to-end model training: https://jovian.ai/aakashns/03-cifar10-feedforward
# * Starter notebook for deep learning models:  https://jovian.ai/aakashns/fashion-feedforward-minimal
#
# Training great machine learning models reliably takes practice and experience. Try experimenting with different datasets, models and hyperparameters, it's the best way to acquire this skill.

# ## Summary and Further Reading
#
# Here is a summary of the topics covered in this tutorial:
#
# * We created a neural network with one hidden layer to improve upon the logistic regression model from the previous tutorial. We also used the ReLU activation function to introduce non-linearity into the model, allowing it to learn more complex relationships between the inputs (pixel densities) and outputs (class probabilities).
#
# * We defined some utilities like `get_default_device`, `to_device` and `DeviceDataLoader` to leverage a GPU if available, by moving the input data and model parameters to the appropriate device.
#
# * We were able to use the exact same training loop: the `fit` function we had define earlier to train out model and evaluate it using the validation dataset.
#
# There's a lot of scope to experiment here, and I encourage you to use the interactive nature of Jupyter to play around with the various parameters. Here are a few ideas:
#
# * Try changing the size of the hidden layer, or add more hidden layers and see if you can achieve a higher accuracy.
#
# * Try changing the batch size and learning rate to see if you can achieve the same accuracy in fewer epochs.
#
# * Compare the training times on a CPU vs. GPU. Do you see a significant difference. How does it vary with the size of the dataset and the size of the model (no. of weights and parameters)?
#
# * Try building a model for a different dataset, such as the [CIFAR10 or CIFAR100 datasets](https://www.cs.toronto.edu/~kriz/cifar.html).
#
# Here are some references for further reading:
#
# * [A visual proof that neural networks can compute any function](http://neuralnetworksanddeeplearning.com/chap4.html), also known as the Universal Approximation Theorem.
#
# * [But what *is* a neural network?](https://www.youtube.com/watch?v=aircAruvnKk) - A visual and intuitive introduction to what neural networks are and what the intermediate layers represent
#
# * [Stanford CS229 Lecture notes on Backpropagation](http://cs229.stanford.edu/notes/cs229-notes-backprop.pdf) - for a more mathematical treatment of how gradients are calculated and weights are updated for neural networks with multiple layers.
#
#
# You are now ready to move on to the next tutorial: [Image Classification using Convolutional Neural Networks](https://jovian.ai/aakashns/05-cifar10-cnn).


import numpy as np
import pytest
import S3_imgproc_tools as algo
null_matrix = np.zeros((10, 10), dtype = bool)

def test_invert_colors_numpy_None():
    with pytest.raises(ValueError):
        algo.inv_gray_levels(None)

def test_invert_colors_numpy_Aray():
    with pytest.raises(TypeError):
        algo.inv_gray_levels(1)

def test_invert_colors_numpy_uint8():
    with pytest.raises(TypeError):
        algo.inv_gray_levels(null_matrix)


# coding: utf-8
#Student: R

# Here goes the imports
import csv
import numpy as np
import pandas
import matplotlib.pyplot as plt

# Let's read the data as a list
print("Reading the document...")
with open("chicago.csv", "r") as file_read:
   reader = csv.reader(file_read)
   data_list = list(reader)
file_read.close()

print("Ok!")

# Let's check how many rows do we have
print("Number of rows:")
print(len(data_list))

# Printing the first row of data_list to check if it worked.
print("Row 0: ")
print(data_list[0])
# It's the data header, so we can identify the columns.

# Printing the second row of data_list, it should contain some data
print("Row 1: ")
print(data_list[1])

input("Press Enter to continue...")
# TASK 1
# TODO: Print the first 20 rows using a loop to identify the data.
print("\n\nTASK 1: Printing the first 20 samples")

# Let's change the data_list to remove the header from it.
data_list = data_list[1:]

for index in range(len(data_list[0:20])):
    print(data_list[index])

# We can access the features through index
# E.g. sample[6] to print gender or sample[-2]

input("Press Enter to continue...")
# TASK 2
# TODO: Print the `gender` of the first 20 rows

print("\nTASK 2: Printing the genders of the first 20 samples")

aux = []
for i in range(len(data_list[0:20])):
    for j in range(len(data_list[i])):
        aux = data_list[i]
    print(aux[-2])

# Cool! We can get the rows(samples) iterating with a for and the columns(features) by index.
# But it's still hard to get a column in a list. Example: List with all genders

input("Press Enter to continue...")
# TASK 3
# TODO: Create a function to add the columns(features) of a list in another list in the same order

"""
      Example function to add the columns(features) of a list in another list in the same order.
      def column_to_list(data_list: list, index: int) -> column_list:
      Args:
          data_list: all the data read from the csv file as a list
          index: the index of the data
      Returns:
          column_list: a list with all the values from the column specified by the index

      """
def column_to_list(data_list, index):
    column_list = []
    aux = []
    for i in range(len(data_list)):
        for j in range(len(data_list[i])):
            aux = data_list[i]
        column_list.append(aux[index])
    # Tip: You can use a for to iterate over the samples, get the feature by index and append into a list
    return column_list

# Let's check with the genders if it's working (only the first 20)
print("\nTASK 3: Printing the list of genders of the first 20 samples")
print(column_to_list(data_list, -2)[:20])

# ------------ DO NOT CHANGE ANY CODE HERE ------------
assert type(column_to_list(data_list, -2)) is list, "TASK 3: Wrong type returned. It should return a list."
assert len(column_to_list(data_list, -2)) == 1551505, "TASK 3: Wrong lenght returned."
assert column_to_list(data_list, -2)[0] == "" and column_to_list(data_list, -2)[1] == "Male", "TASK 3: The list doesn't match."
# -----------------------------------------------------

input("Press Enter to continue...")
# Now we know how to access the features, let's count how many Males and Females the dataset have
# TASK 4
# TODO: Count each gender. You should not use a function to do that.
male = 0
female = 0
aux = []
for i in range(len(data_list)):
    for j in range(len(data_list[i])):
        aux = data_list[i]
    #print(aux[-2])
    if aux[-2] == "Male":
        male = male + 1
    elif aux[-2] == "Female":
        female = female + 1

# Checking the result
print("\nTASK 4: Printing how many males and females we found")
print("Male: ", male, "\nFemale: ", female)

# ------------ DO NOT CHANGE ANY CODE HERE ------------
assert male == 935854 and female == 298784, "TASK 4: Count doesn't match."
# -----------------------------------------------------

input("Press Enter to continue...")
# Why don't we creeate a function to do that?
# TASK 5
# TODO: Create a function to count the genders. Return a list
# Should return a list with [count_male, counf_female] (e.g., [10, 15] means 10 Males, 15 Females)

"""
      Example function to count the genders. Return a list
      def count_gender(data_list: list) -> [male: int, female: int]:
      Args:
          data_list: all the data read from the csv file as a matrix
      Returns:
          [male , female]: the int variables responsible to count the number of males and females
"""

def count_gender(data_list):
    male = 0
    female = 0
    aux = []
    for i in range(len(data_list)):
        for j in range(len(data_list[i])):
            aux = data_list[i]
        if aux[-2] == "Male":
            male = male + 1
        elif aux[-2] == "Female":
            female = female + 1
    return [male, female]

print("\nTASK 5: Printing result of count_gender")
print(count_gender(data_list))

# ------------ DO NOT CHANGE ANY CODE HERE ------------
assert type(count_gender(data_list)) is list, "TASK 5: Wrong type returned. It should return a list."
assert len(count_gender(data_list)) == 2, "TASK 5: Wrong lenght returned."
assert count_gender(data_list)[0] == 935854 and count_gender(data_list)[1] == 298784, "TASK 5: Returning wrong result!"
# -----------------------------------------------------

input("Press Enter to continue...")
# Now we can count the users, which gender use it the most?
# TASK 6
# TODO: Create a function to get the most popular gender and print the gender as string.
# We expect to see "Male", "Female" or "Equal" as answer.

"""
      Example function tto get the most popular gender and print the gender as string
      def count_gender(data_list: list) -> answer: str
      Args:
          data_list: all the data read from the csv file as a matrix
      Returns:
          answer: depending on the result of the auxiliars variables from the function the result can be "Male" or "Female"
"""

def most_popular_gender(data_list):
    answer = ""
    male = 0
    female = 0
    aux = []
    for i in range(len(data_list)):
        for j in range(len(data_list[i])):
            aux = data_list[i]

        if aux[-2] == "Male":
            male = male + 1
        elif aux[-2] == "Female":
            female = female + 1

    if male > female:
        answer = "Male"
    else:
        answer = "Female"

    return answer

print("\nTASK 6: Which one is the most popular gender?")
print("Most popular gender is: ", most_popular_gender(data_list))

# ------------ DO NOT CHANGE ANY CODE HERE ------------
assert type(most_popular_gender(data_list)) is str, "TASK 6: Wrong type returned. It should return a string."
assert most_popular_gender(data_list) == "Male", "TASK 6: Returning wrong result!"
# -----------------------------------------------------

# If it's everything running as expected, check this graph!
gender_list = column_to_list(data_list, -2)
types = ["Male", "Female"]
quantity = count_gender(data_list)
y_pos = list(range(len(types)))
plt.bar(y_pos, quantity)
plt.ylabel('Quantity')
plt.xlabel('Gender')
plt.xticks(y_pos, types)
plt.title('Quantity by Gender')
plt.show(block=True)

input("Press Enter to continue...")
# TASK 7
# TODO: Plot a similar graph for user_types. Make sure the legend is correct.
print("\nTASK 7: Check the chart!")

"""
      Example function to check the type of the user that is very similar to the count_gender() function
      def count_user_type(data_list: list) -> answer: str
      Args:
          data_list: all the data read from the csv file as a matrix
      Returns:
         [customer, subscriber]: the int variables responsible to count the number of customers and subscribers
"""

def count_user_type(data_list):
    customer = 0
    subscriber = 0
    aux = []
    for i in range(len(data_list)):
        for j in range(len(data_list[i])):
            aux = data_list[i]
        if aux[5] == "Customer":
            customer = customer + 1
        elif aux[5] == "Subscriber":
            subscriber = subscriber + 1
    return [customer, subscriber]

user_type_list = column_to_list(data_list, 5)
types = ["Customer", "Subscriber"]
quantity = count_user_type(data_list)
y_pos = list(range(len(types)))
plt.bar(y_pos, quantity)
plt.ylabel('Quantity')
plt.xlabel('User Type')
plt.xticks(y_pos, types)
plt.title('Quantity by User Type')
plt.show(block=True)

input("Press Enter to continue...")
# TASK 8
# TODO: Answer the following question
male, female = count_gender(data_list)
print("\nTASK 8: Why the following condition is False?")
print("male + female == len(data_list):", male + female == len(data_list))
print(male + female)
print(len(data_list))
answer = "Because there are some data with a blank space"
print("Answer:", answer)

# ------------ DO NOT CHANGE ANY CODE HERE ------------
assert answer != "Type your answer here.", "TASK 8: Write your own answer!"
# -----------------------------------------------------

input("Press Enter to continue...")
# Let's work with the trip_duration now. We cant get some values from it.
# TASK 9
# TODO: Find the Minimum, Maximum, Mean and Median trip duration.
# You should not use ready functions to do that, like max() or min().
trip_duration_list = column_to_list(data_list, 2)
#print (trip_duration_list)
min_trip = 0.
max_trip = 0.
mean_trip = 0.
median_trip = 0.

"""
      Example function to ordenate the values of a list using the quicksort algorithm
      def quicksort(l: list) -> []: list
      Args:
          l: list of values not ordenated
      Returns:
         []: list ordenated
"""

def quicksort(l):
    if l:
        left = [x for x in l if x < l[0]]
        right = [x for x in l if x > l[0]]

        if len(left) > 1:
            left = quicksort(left)
            right = quicksort(right)

        return left + [l[0]] * l.count(l[0]) + right

    return []

"""
      Example function to calculate the mean of a list of float values
      def mean(data_list: list) -> mean: float
      Args:
          data_list: list of float values
      Returns:
         mean: the mean of the values
"""

def mean(data_list):
    total = 0
    mean = 0
    for i in range(len(data_list)):
        total = int(data_list[i]) + total

    mean = total/len(data_list)
    return mean

"""
      Example function to get the median of a list of float values
      def median(data_list: list) -> data_list[half]: float
      Args:
          data_list: list of float values
      Returns:
         median: the median of the data_list
"""

def median(data_list):
    half = len(data_list) // 2
    data_list.quicksort()
    if not len(data_list) % 2:
        return (data_list[half - 1] + data_list[half]) / 2.0
    return data_list[half]

sorted_list = quicksort(trip_duration_list)
min_trip = sorted_list[0]
max_trip = sorted_list[-1]
mean_trip = mean(trip_duration_list)
median_trip = median(trip_duration_list)

print("\nTASK 9: Printing the min, max, mean and median")
print("Min: ", min_trip, "Max: ", max_trip, "Mean: ", mean_trip, "Median: ", median_trip)

# ------------ DO NOT CHANGE ANY CODE HERE ------------
#assert round(min_trip) == 60, "TASK 9: min_trip with wrong result!"
#assert round(max_trip) == 86338, "TASK 9: max_trip with wrong result!"
#assert round(mean_trip) == 940, "TASK 9: mean_trip with wrong result!"
#assert round(median_trip) == 670, "TASK 9: median_trip with wrong result!"
# -----------------------------------------------------

input("Press Enter to continue...\n")
# TASK 10
# Gender is easy because usually only have a few options. How about start_stations? How many options does it have?
# TODO: Check types how many start_stations do we have using set()
start_stations_list =  column_to_list(data_list, 3)
user_types = set(start_stations_list)

print("\nTASK 10: Printing start stations:")
print(len(user_types))
print(user_types)

# ------------ DO NOT CHANGE ANY CODE HERE ------------
assert len(user_types) == 582, "TASK 10: Wrong len of start stations."
# -----------------------------------------------------

input("Press Enter to continue...")
# TASK 11
# Go back and make sure you documented your functions. Explain the input, output and what it do. Example:
# def new_function(param1: int, param2: str) -> list:
"""
      Example function with annotations.
      Args:
          param1: The first parameter.
          param2: The second parameter.
      Returns:
          List of X values

      """

input("Press Enter to continue...")
# TASK 12 - Challenge! (Optional)
# TODO: Create a function to count user types without hardcoding the types
# so we can use this function with a different kind of data.
print("Will you face it?")
answer = "no"

def count_items(column_list):
    item_types = []
    count_items = []
    return item_types, count_items

if answer == "yes":
    # ------------ DO NOT CHANGE ANY CODE HERE ------------
    column_list = column_to_list(data_list, -2)
    types, counts = count_items(column_list)
    print("\nTASK 11: Printing results for count_items()")
    print("Types:", types, "Counts:", counts)
    assert len(types) == 3, "TASK 11: There are 3 types of gender!"
    assert sum(counts) == 1551505, "TASK 11: Returning wrong result!"
    # -----------------------------------------------------


"""
Introduction to artifacts and artifact detection
================================================

Since MNE supports the data of many different acquisition systems, the
particular artifacts in your data might behave very differently from the
artifacts you can observe in our tutorials and examples.

Therefore you should be aware of the different approaches and of
the variability of artifact rejection (automatic/manual) procedures described
onwards. At the end consider always to visually inspect your data
after artifact rejection or correction.

Background: what is an artifact?
--------------------------------

Artifacts are signal interference that can be
endogenous (biological) and exogenous (environmental).
Typical biological artifacts are head movements, eye blinks
or eye movements, heart beats. The most common environmental
artifact is due to the power line, the so-called *line noise*.

How to handle artifacts?
------------------------

MNE deals with artifacts by first identifying them, and subsequently removing
them. Detection of artifacts can be done visually, or using automatic routines
(or a combination of both). After you know what the artifacts are, you need
remove them. This can be done by:

    - *ignoring* the piece of corrupted data
    - *fixing* the corrupted data

For the artifact detection the functions MNE provides depend on whether
your data is continuous (Raw) or epoch-based (Epochs) and depending on
whether your data is stored on disk or already in memory.

Detecting the artifacts without reading the complete data into memory allows
you to work with datasets that are too large to fit in memory all at once.
Detecting the artifacts in continuous data allows you to apply filters
(e.g. a band-pass filter to zoom in on the muscle artifacts on the temporal
channels) without having to worry about edge effects due to the filter
(i.e. filter ringing). Having the data in memory after segmenting/epoching is
however a very efficient way of browsing through the data which helps
in visualizing. So to conclude, there is not a single most optimal manner
to detect the artifacts: it just depends on the data properties and your
own preferences.

In this tutorial we show how to detect artifacts visually and automatically.
For how to correct artifacts by rejection see
:ref:`sphx_glr_auto_tutorials_plot_artifacts_correction_rejection.py`.
To discover how to correct certain artifacts by filtering see
:ref:`sphx_glr_auto_tutorials_plot_artifacts_correction_filtering.py`
and to learn how to correct artifacts
with subspace methods like SSP and ICA see
:ref:`sphx_glr_auto_tutorials_plot_artifacts_correction_ssp.py`
and :ref:`sphx_glr_auto_tutorials_plot_artifacts_correction_ica.py`.

Artifacts Detection
-------------------

This tutorial discusses a couple of major artifacts that most analyses
have to deal with and demonstrates how to detect them.

"""
import numpy as np

import mne
from mne.datasets import sample
from mne.preprocessing import create_ecg_epochs, create_eog_epochs

# getting some data ready
data_path = sample.data_path()
raw_fname = data_path + '/MEG/sample/sample_audvis_raw.fif'

raw = mne.io.read_raw_fif(raw_fname, preload=True)

###############################################################################
# Low frequency drifts and line noise

(raw.copy().pick_types(meg='mag')
           .del_proj(0)
           .plot(duration=60, n_channels=100, remove_dc=False))

###############################################################################
# we see high amplitude undulations in low frequencies, spanning across tens of
# seconds

raw.plot_psd(tmax=np.inf, fmax=250)

###############################################################################
# On MEG sensors we see narrow frequency peaks at 60, 120, 180, 240 Hz,
# related to line noise.
# But also some high amplitude signals between 25 and 32 Hz, hinting at other
# biological artifacts such as ECG. These can be most easily detected in the
# time domain using MNE helper functions
#
# See :ref:`sphx_glr_auto_tutorials_plot_artifacts_correction_filtering.py`.

###############################################################################
# ECG
# ---
#
# finds ECG events, creates epochs, averages and plots

average_ecg = create_ecg_epochs(raw).average()
print('We found %i ECG events' % average_ecg.nave)
joint_kwargs = dict(ts_args=dict(time_unit='s'),
                    topomap_args=dict(time_unit='s'))
average_ecg.plot_joint(**joint_kwargs)

###############################################################################
# we can see typical time courses and non dipolar topographies
# not the order of magnitude of the average artifact related signal and
# compare this to what you observe for brain signals

###############################################################################
# EOG
# ---

average_eog = create_eog_epochs(raw).average()
print('We found %i EOG events' % average_eog.nave)
average_eog.plot_joint(**joint_kwargs)

###############################################################################
# Knowing these artifact patterns is of paramount importance when
# judging about the quality of artifact removal techniques such as SSP or ICA.
# As a rule of thumb you need artifact amplitudes orders of magnitude higher
# than your signal of interest and you need a few of such events in order
# to find decompositions that allow you to estimate and remove patterns related
# to artifacts.
#
# Consider the following tutorials for correcting this class of artifacts:
#     - :ref:`sphx_glr_auto_tutorials_plot_artifacts_correction_filtering.py`
#     - :ref:`sphx_glr_auto_tutorials_plot_artifacts_correction_ica.py`
#     - :ref:`sphx_glr_auto_tutorials_plot_artifacts_correction_ssp.py`


# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.3'
#       jupytext_version: 1.3.0
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# %% [raw] raw_mimetype="text/restructuredtext"
# .. _ug_insets_panels:
#
# Insets and panels
# =================

# %% [raw] raw_mimetype="text/restructuredtext"
# .. _ug_panels:
#
# Panel axes
# ----------
#
# It is often useful to have narrow "panels" along the edge of a larger
# subplot for plotting secondary 1-dimensional datasets or summary statistics.
# In ProPlot, you can create panels by passing a location (e.g. ``loc='r'`` or
# ``loc='right'``) to the `~proplot.axes.Axes.panel` or `~proplot.axes.Axes.panel_axes`
# methods. The resulting axes are instances of `~proplot.axes.CartesianAxes`.
#
# To generate "stacked" panels, simply call `~proplot.axes.Axes.panel` more
# than once. To include panels when centering spanning axis labels and super
# titles, pass ``includepanels=True`` to `~proplot.figure.Figure`. Panels
# :ref:`do not interfere with the tight layout algorithm <ug_tight>` and
# :ref:`do not affect the subplot aspect ratios <ug_autosize>`.
#
# In the first example below, the panel distance from the main subplot is
# manually set to ``space=0``. In the second example, it is adjusted automatically
# by the tight layout algorithm.

# %%
import proplot as plot
import numpy as np
state = np.random.RandomState(51423)
data = (state.rand(20, 20) - 0.48).cumsum(axis=1).cumsum(axis=0)
data = 10 * (data - data.min()) / (data.max() - data.min())

# Stacked panels with outer colorbars
for cbarloc, ploc in ('rb', 'br'):
    fig, axs = plot.subplots(
        axwidth=1.8, nrows=1, ncols=2,
        share=0, panelpad=0.1, includepanels=True
    )
    axs.format(
        xlabel='xlabel', ylabel='ylabel', title='Title',
        suptitle='Using panels for summary statistics',
    )

    # Plot 2D dataset
    axs.contourf(
        data, cmap='glacial', extend='both',
        colorbar=cbarloc, colorbar_kw={'label': 'colorbar'},
    )

    # Get summary statistics and settings
    axis = int(ploc == 'r')  # dimension along which stats are taken
    x1 = x2 = np.arange(20)
    y1 = data.mean(axis=axis)
    y2 = data.std(axis=axis)
    titleloc = 'upper center'
    if ploc == 'r':
        titleloc = 'center'
        x1, x2, y1, y2 = y1, y2, x1, x2

    # Panels for plotting the mean. We make two panels at once and plot data
    # on both panels at once by calling functions from SubplotsContainers.
    # More realistically, you would plot data on each panel one-by-one.
    space = 0
    width = '4em'
    kwargs = {'titleloc': titleloc, 'xreverse': False, 'yreverse': False}
    paxs = axs.panel(ploc, space=space, width=width)
    paxs.plot(x1, y1, color='gray7')
    paxs.format(title='Mean', **kwargs)

    # Panels for plotting the standard deviation
    paxs = axs.panel(ploc, space=space, width=width)
    paxs.plot(x2, y2, color='gray7', ls='--')
    paxs.format(title='Stdev', **kwargs)

# %%
import proplot as plot
fig, axs = plot.subplots(axwidth=1.5, nrows=2, ncols=2, share=0)

# Demonstrate that complex arrangements of panels does
# not mess up subplot aspect ratios or tight layout spacing
axs.format(
    xlim=(0, 1), ylim=(0, 1),
    xlabel='xlabel', ylabel='ylabel',
    xticks=0.2, yticks=0.2,
    title='Title', suptitle='Complex arrangement of panels',
    collabels=['Column 1', 'Column 2'],
    abc=True, abcloc='ul', titleloc='uc', abovetop=False,
)
for ax, side in zip(axs, 'tlbr'):
    ax.panel(side, width='3em')

# %% [raw] raw_mimetype="text/restructuredtext"
# .. _ug_insets:
#
# Inset axes
# ----------
#
# `Inset axes\
# <https://matplotlib.org/3.1.1/gallery/subplots_axes_and_figures/zoom_inset_axes.html>`__
# can be generated with the `~proplot.axes.Axes.inset` or
# `~proplot.axes.Axes.inset_axes` command. By defaut, the resulting axes
# use the same projection as the parent axes, but you can also specify
# a different projection (for example, ``ax.inset(bounds, proj='polar')``).
# Passing ``zoom=True`` to `~proplot.axes.Axes.inset` draws "zoom indication"
# lines with `~matplotlib.axes.Axes.indicate_inset_zoom` when the axes are both
# Cartesian, and ProPlot automatically updates the lines when the axis limits of
# the parent axes change. To modify the line properties, simply use the `zoom_kw`
# argument.

# %%
import proplot as plot
import numpy as np

# Generate sample data
N = 20
state = np.random.RandomState(51423)
x, y = np.arange(10), np.arange(10)
data = state.rand(10, 10)

# Plot sample data
fig, ax = plot.subplots(axwidth=3)
m = ax.pcolormesh(data, cmap='Grays', levels=N)
ax.colorbar(m, loc='b', label='label')
ax.format(
    xlabel='xlabel', ylabel='ylabel',
    suptitle='"Zooming in" with an inset axes'
)

# Create inset axes representing a "zoom-in"
iax = ax.inset(
    [5, 5, 4, 4], transform='data', zoom=True,
    zoom_kw={'color': 'red3', 'lw': 2, 'ls': '--'}
)
iax.format(
    xlim=(2, 4), ylim=(2, 4), color='red7',
    linewidth=1.5, ticklabelweight='bold'
)
iax.pcolormesh(data, cmap='Grays', levels=N)