"""
==============================================================================
07. Save and load T1-weighted MRI scan along with anatomical landmarks in BIDS
==============================================================================

When working with MEEG data in the domain of source localization, we usually
have to deal with aligning several coordinate systems, such as the coordinate
systems of ...

- the head of a study participant
- the recording device (in the case of MEG)
- the anatomical MRI scan of a study participant

The process of aligning these frames is also called coregistration, and is
performed with the help of a transformation matrix, called ``trans`` in MNE.

In this tutorial, we show how ``MNE-BIDS`` can be used to save a T1 weighted
MRI scan in BIDS format, and to encode all information of the ``trans`` object
in a BIDS compatible way.

Finally, we will automatically reproduce our ``trans`` object from a BIDS
directory.

See the documentation pages in the MNE docs for more information on
`source alignment and coordinate frames <mne_source_coords_>`_

.. note:: For this example you will need to install ``matplotlib`` and
          ``nilearn`` on top of your usual ``mne-bids`` installation.

"""
# Authors: Stefan Appelhoff <stefan.appelhoff@mailbox.org>
#          Alex Rockhill <aprockhill206@gmail.com>
#          Alex Gramfort <alexandre.gramfort@inria.fr>
# License: BSD (3-clause)

# %%
# Let's import everything we need for this example:

import os.path as op
import shutil

import numpy as np
import matplotlib.pyplot as plt

from nilearn.plotting import plot_anat

import mne
from mne.datasets import sample
from mne.source_space import head_to_mri

from mne_bids import (write_raw_bids, BIDSPath, write_anat, get_anat_landmarks,
                      get_head_mri_trans, print_dir_tree)

# %%
# We will be using the `MNE sample data <mne_sample_data_>`_ and write a basic
# BIDS dataset. For more information, you can checkout the respective
# :ref:`example <ex-convert-mne-sample>`.

data_path = sample.data_path()
event_id = {'Auditory/Left': 1, 'Auditory/Right': 2, 'Visual/Left': 3,
            'Visual/Right': 4, 'Smiley': 5, 'Button': 32}
raw_fname = op.join(data_path, 'MEG', 'sample', 'sample_audvis_raw.fif')
events_data = op.join(data_path, 'MEG', 'sample', 'sample_audvis_raw-eve.fif')
output_path = op.abspath(op.join(data_path, '..', 'MNE-sample-data-bids'))
fs_subjects_dir = op.join(data_path, 'subjects')  # FreeSurfer subjects dir

# %%
# To ensure the output path doesn't contain any leftover files from previous
# tests and example runs, we simply delete it.
#
# .. warning:: Do not delete directories that may contain important data!
#

if op.exists(output_path):
    shutil.rmtree(output_path)

# %%
# Read the input data and store it as BIDS data.

raw = mne.io.read_raw_fif(raw_fname)
raw.info['line_freq'] = 60  # specify power line frequency as required by BIDS

sub = '01'
ses = '01'
task = 'audiovisual'
run = '01'
bids_path = BIDSPath(subject=sub, session=ses, task=task,
                     run=run, root=output_path)
write_raw_bids(raw, bids_path, events_data=events_data,
               event_id=event_id, overwrite=True)

# %%
# Print the directory tree
print_dir_tree(output_path)

# %%
# Writing T1 image
# ----------------
#
# Now let's assume that we have also collected some T1 weighted MRI data for
# our subject. And furthermore, that we have already aligned our coordinate
# frames (using e.g., the `coregistration GUI`_) and obtained a transformation
# matrix :code:`trans`.

# Get the path to our MRI scan
t1_fname = op.join(fs_subjects_dir, 'sample', 'mri', 'T1.mgz')

# Load the transformation matrix and show what it looks like
trans_fname = op.join(data_path, 'MEG', 'sample',
                      'sample_audvis_raw-trans.fif')
trans = mne.read_trans(trans_fname)
print(trans)

# %%
# We can save the MRI to our existing BIDS directory and at the same time
# create a JSON sidecar file that contains metadata, we will later use to
# retrieve our transformation matrix :code:`trans`. The metadata will here
# consist of the coordinates of three anatomical landmarks (LPA, Nasion and
# RPA (=left and right preauricular points) expressed in voxel coordinates
# w.r.t. the T1 image.

# First create the BIDSPath object.
t1w_bids_path = BIDSPath(subject=sub, session=ses, root=output_path,
                         suffix='T1w')

# use ``trans`` to transform landmarks from the ``raw`` file to
# the voxel space of the image
landmarks = get_anat_landmarks(
    t1_fname,  # path to the MRI scan
    info=raw.info,  # the MEG data file info from the same subject as the MRI
    trans=trans,  # our transformation matrix
    fs_subject='sample',  # FreeSurfer subject
    fs_subjects_dir=fs_subjects_dir,  # FreeSurfer subjects directory
)

# We use the write_anat function
t1w_bids_path = write_anat(
    image=t1_fname,  # path to the MRI scan
    bids_path=t1w_bids_path,
    landmarks=landmarks,  # the landmarks in MRI voxel space
    verbose=True  # this will print out the sidecar file
)
anat_dir = t1w_bids_path.directory

# %%
# Let's have another look at our BIDS directory
print_dir_tree(output_path)

# %%
# Our BIDS dataset is now ready to be shared. We can easily estimate the
# transformation matrix using ``MNE-BIDS`` and the BIDS dataset.
# This function converts the anatomical landmarks stored in the T1 sidecar
# file into FreeSurfer surface RAS space, and aligns the landmarks in the
# electrophysiology data with them. This way your electrophysiology channel
# locations can be transformed to surface RAS space using the ``trans`` which
# is crucial for source localization and other uses of the FreeSurfer surfaces.
#
# .. note:: If this dataset were shared with you, you would first have to use
#           the T1 image as input for the FreeSurfer recon-all, see
#           :ref:`tut-freesurfer-mne`.
estim_trans = get_head_mri_trans(bids_path=bids_path, fs_subject='sample',
                                 fs_subjects_dir=fs_subjects_dir)

# %%
# Finally, let's use the T1 weighted MRI image and plot the anatomical
# landmarks Nasion, LPA, and RPA onto the brain image. For that, we can
# extract the location of Nasion, LPA, and RPA from the MEG file, apply our
# transformation matrix :code:`trans`, and plot the results.

# Get Landmarks from MEG file, 0, 1, and 2 correspond to LPA, NAS, RPA
# and the 'r' key will provide us with the xyz coordinates. The coordinates
# are expressed here in MEG Head coordinate system.
pos = np.asarray((raw.info['dig'][0]['r'],
                  raw.info['dig'][1]['r'],
                  raw.info['dig'][2]['r']))

# We now use the ``head_to_mri`` function from MNE-Python to convert MEG
# coordinates to MRI scanner RAS space. For the conversion we use our
# estimated transformation matrix and the MEG coordinates extracted from the
# raw file. `subjects` and `subjects_dir` are used internally, to point to
# the T1-weighted MRI file: `t1_mgh_fname`. Coordinates are is mm.
mri_pos = head_to_mri(pos=pos,
                      subject='sample',
                      mri_head_t=estim_trans,
                      subjects_dir=fs_subjects_dir)

# Our MRI written to BIDS, we got `anat_dir` from our `write_anat` function
t1_nii_fname = op.join(anat_dir, 'sub-01_ses-01_T1w.nii.gz')

# Plot it
fig, axs = plt.subplots(3, 1, figsize=(7, 7), facecolor='k')
for point_idx, label in enumerate(('LPA', 'NAS', 'RPA')):
    plot_anat(t1_nii_fname, axes=axs[point_idx],
              cut_coords=mri_pos[point_idx, :],
              title=label, vmax=160)
plt.show()

# %%
# Writing FLASH MRI image
# -----------------------
#
# We can write another types of MRI data such as FLASH images for BEM models

flash_fname = op.join(fs_subjects_dir, 'sample', 'mri', 'flash', 'mef05.mgz')

flash_bids_path = \
    BIDSPath(subject=sub, session=ses, root=output_path, suffix='FLASH')

write_anat(
    image=flash_fname,
    bids_path=flash_bids_path,
    verbose=True
)

# %%
# Writing defaced and anonymized T1 image
# ---------------------------------------
#
# We can deface the MRI for anonymization by passing ``deface=True``.
t1w_bids_path = write_anat(
    image=t1_fname,  # path to the MRI scan
    bids_path=bids_path,
    landmarks=landmarks,
    deface=True,
    overwrite=True,
    verbose=True  # this will print out the sidecar file
)
anat_dir = t1w_bids_path.directory

# Our MRI written to BIDS, we got `anat_dir` from our `write_anat` function
t1_nii_fname = op.join(anat_dir, 'sub-01_ses-01_T1w.nii.gz')

# Plot it
fig, ax = plt.subplots()
plot_anat(t1_nii_fname, axes=ax, title='Defaced', vmax=160)
plt.show()

# %%
# Writing defaced and anonymized FLASH MRI image
# ----------------------------------------------
#
# Defacing the FLASH works just like the T1 as long as they are aligned.

# use ``trans`` to transform landmarks from the ``raw`` file to
# the voxel space of the image
landmarks = get_anat_landmarks(
    flash_fname,  # path to the FLASH scan
    info=raw.info,  # the MEG data file info from the same subject as the MRI
    trans=trans,  # our transformation matrix
    fs_subject='sample',  # freesurfer subject
    fs_subjects_dir=fs_subjects_dir,  # freesurfer subjects directory
)

flash_bids_path = write_anat(
    image=flash_fname,  # path to the MRI scan
    bids_path=flash_bids_path,
    landmarks=landmarks,
    deface=True,
    overwrite=True,
    verbose=True  # this will print out the sidecar file
)

# Our MRI written to BIDS, we got `anat_dir` from our `write_anat` function
flash_nii_fname = op.join(anat_dir, 'sub-01_ses-01_FLASH.nii.gz')

# Plot it
fig, ax = plt.subplots()
plot_anat(flash_nii_fname, axes=ax, title='Defaced', vmax=700)
plt.show()

# %%
# Using manual landmark coordinates in scanner RAS
# ------------------------------------------------
#
# You can also find landmarks with a 3D image viewer (e.g. FreeView) if you
# have not aligned the channel locations (including fiducials) using the
# coregistration GUI or if this is just more convenient.
#
# .. note:: In FreeView, you need to use "RAS" and not "TkReg RAS" for this.
#           You can also use voxel coordinates but, in FreeView, they
#           are integers and so not as precise as the "RAS" decimal numbers.
flash_ras_landmarks = \
    np.array([[-74.53102838, 19.62854953, -52.2888194],
              [-1.89454315, 103.69850925, 4.97120376],
              [72.01200673, 21.09274883, -57.53678375]]) / 1e3  # mm -> m

landmarks = mne.channels.make_dig_montage(
    lpa=flash_ras_landmarks[0],
    nasion=flash_ras_landmarks[1],
    rpa=flash_ras_landmarks[2],
    coord_frame='ras'
)

flash_bids_path = write_anat(
    image=flash_fname,  # path to the MRI scan
    bids_path=flash_bids_path,
    landmarks=landmarks,
    deface=True,
    overwrite=True,
    verbose=True  # this will print out the sidecar file
)

# Plot it
fig, ax = plt.subplots()
plot_anat(flash_nii_fname, axes=ax, title='Defaced', vmax=700)
plt.show()

# %%
# .. LINKS
#
# .. _coregistration GUI:
#    https://mne.tools/stable/auto_tutorials/forward/20_source_alignment.html#defining-the-headmri-trans-using-the-gui
# .. _mne_source_coords:
#    https://mne.tools/stable/auto_tutorials/source-modeling/plot_source_alignment.html
# .. _mne_sample_data:
#    https://mne.tools/stable/overview/datasets_index.html#sample
#


# coding: utf-8

# # Image Classification
# In this project, you'll classify images from the [CIFAR-10 dataset](https://www.cs.toronto.edu/~kriz/cifar.html).  The dataset consists of airplanes, dogs, cats, and other objects. You'll preprocess the images, then train a convolutional neural network on all the samples. The images need to be normalized and the labels need to be one-hot encoded.  You'll get to apply what you learned and build a convolutional, max pooling, dropout, and fully connected layers.  At the end, you'll get to see your neural network's predictions on the sample images.
# ## Get the Data
# Run the following cell to download the [CIFAR-10 dataset for python](https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz).

# In[1]:

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
from urllib.request import urlretrieve
from os.path import isfile, isdir
from tqdm import tqdm
import problem_unittests as tests
import tarfile

cifar10_dataset_folder_path = 'cifar-10-batches-py'

# Use Floyd's cifar-10 dataset if present
floyd_cifar10_location = '/input/cifar-10/python.tar.gz'
if isfile(floyd_cifar10_location):
    tar_gz_path = floyd_cifar10_location
else:
    tar_gz_path = 'cifar-10-python.tar.gz'

class DLProgress(tqdm):
    last_block = 0

    def hook(self, block_num=1, block_size=1, total_size=None):
        self.total = total_size
        self.update((block_num - self.last_block) * block_size)
        self.last_block = block_num

if not isfile(tar_gz_path):
    with DLProgress(unit='B', unit_scale=True, miniters=1, desc='CIFAR-10 Dataset') as pbar:
        urlretrieve(
            'https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz',
            tar_gz_path,
            pbar.hook)

if not isdir(cifar10_dataset_folder_path):
    with tarfile.open(tar_gz_path) as tar:
        tar.extractall()
        tar.close()

tests.test_folder_path(cifar10_dataset_folder_path)

# ## Explore the Data
# The dataset is broken into batches to prevent your machine from running out of memory.  The CIFAR-10 dataset consists of 5 batches, named `data_batch_1`, `data_batch_2`, etc.. Each batch contains the labels and images that are one of the following:
# * airplane
# * automobile
# * bird
# * cat
# * deer
# * dog
# * frog
# * horse
# * ship
# * truck
#
# Understanding a dataset is part of making predictions on the data.  Play around with the code cell below by changing the `batch_id` and `sample_id`. The `batch_id` is the id for a batch (1-5). The `sample_id` is the id for a image and label pair in the batch.
#
# Ask yourself "What are all possible labels?", "What is the range of values for the image data?", "Are the labels in order or random?".  Answers to questions like these will help you preprocess the data and end up with better predictions.

# In[2]:

get_ipython().magic('matplotlib inline')
get_ipython().magic("config InlineBackend.figure_format = 'retina'")

import helper
import numpy as np

# Explore the dataset
batch_id = 1
sample_id = 5
helper.display_stats(cifar10_dataset_folder_path, batch_id, sample_id)

# ## Implement Preprocess Functions
# ### Normalize
# In the cell below, implement the `normalize` function to take in image data, `x`, and return it as a normalized Numpy array. The values should be in the range of 0 to 1, inclusive.  The return object should be the same shape as `x`.

# In[3]:

def normalize(x):
    """
    Normalize a list of sample image data in the range of 0 to 1
    : x: List of image data.  The image shape is (32, 32, 3)
    : return: Numpy array of normalize data
    """
    return x / 255

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_normalize(normalize)

# ### One-hot encode
# Just like the previous code cell, you'll be implementing a function for preprocessing.  This time, you'll implement the `one_hot_encode` function. The input, `x`, are a list of labels.  Implement the function to return the list of labels as One-Hot encoded Numpy array.  The possible values for labels are 0 to 9. The one-hot encoding function should return the same encoding for each value between each call to `one_hot_encode`.  Make sure to save the map of encodings outside the function.
#
# Hint: Don't reinvent the wheel.

# In[5]:

def one_hot_encode(x):
    """
    One hot encode a list of sample labels. Return a one-hot encoded vector for each label.
    : x: List of sample Labels
    : return: Numpy array of one-hot encoded labels
    """
    return np.eye(10)[x]

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_one_hot_encode(one_hot_encode)

# ### Randomize Data
# As you saw from exploring the data above, the order of the samples are randomized.  It doesn't hurt to randomize it again, but you don't need to for this dataset.

# ## Preprocess all the data and save it
# Running the code cell below will preprocess all the CIFAR-10 data and save it to file. The code below also uses 10% of the training data for validation.

# In[5]:

"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
# Preprocess Training, Validation, and Testing Data
helper.preprocess_and_save_data(cifar10_dataset_folder_path, normalize, one_hot_encode)

# # Check Point
# This is your first checkpoint.  If you ever decide to come back to this notebook or have to restart the notebook, you can start from here.  The preprocessed data has been saved to disk.

# In[6]:

"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
import pickle
import problem_unittests as tests
import helper

# Load the Preprocessed Validation data
valid_features, valid_labels = pickle.load(open('preprocess_validation.p', mode='rb'))

# ## Build the network
# For the neural network, you'll build each layer into a function.  Most of the code you've seen has been outside of functions. To test your code more thoroughly, we require that you put each layer in a function.  This allows us to give you better feedback and test for simple mistakes using our unittests before you submit your project.
#
# >**Note:** If you're finding it hard to dedicate enough time for this course each week, we've provided a small shortcut to this part of the project. In the next couple of problems, you'll have the option to use classes from the [TensorFlow Layers](https://www.tensorflow.org/api_docs/python/tf/layers) or [TensorFlow Layers (contrib)](https://www.tensorflow.org/api_guides/python/contrib.layers) packages to build each layer, except the layers you build in the "Convolutional and Max Pooling Layer" section.  TF Layers is similar to Keras's and TFLearn's abstraction to layers, so it's easy to pickup.
#
# >However, if you would like to get the most out of this course, try to solve all the problems _without_ using anything from the TF Layers packages. You **can** still use classes from other packages that happen to have the same name as ones you find in TF Layers! For example, instead of using the TF Layers version of the `conv2d` class, [tf.layers.conv2d](https://www.tensorflow.org/api_docs/python/tf/layers/conv2d), you would want to use the TF Neural Network version of `conv2d`, [tf.nn.conv2d](https://www.tensorflow.org/api_docs/python/tf/nn/conv2d).
#
# Let's begin!
#
# ### Input
# The neural network needs to read the image data, one-hot encoded labels, and dropout keep probability. Implement the following functions
# * Implement `neural_net_image_input`
#  * Return a [TF Placeholder](https://www.tensorflow.org/api_docs/python/tf/placeholder)
#  * Set the shape using `image_shape` with batch size set to `None`.
#  * Name the TensorFlow placeholder "x" using the TensorFlow `name` parameter in the [TF Placeholder](https://www.tensorflow.org/api_docs/python/tf/placeholder).
# * Implement `neural_net_label_input`
#  * Return a [TF Placeholder](https://www.tensorflow.org/api_docs/python/tf/placeholder)
#  * Set the shape using `n_classes` with batch size set to `None`.
#  * Name the TensorFlow placeholder "y" using the TensorFlow `name` parameter in the [TF Placeholder](https://www.tensorflow.org/api_docs/python/tf/placeholder).
# * Implement `neural_net_keep_prob_input`
#  * Return a [TF Placeholder](https://www.tensorflow.org/api_docs/python/tf/placeholder) for dropout keep probability.
#  * Name the TensorFlow placeholder "keep_prob" using the TensorFlow `name` parameter in the [TF Placeholder](https://www.tensorflow.org/api_docs/python/tf/placeholder).
#
# These names will be used at the end of the project to load your saved model.
#
# Note: `None` for shapes in TensorFlow allow for a dynamic size.

# In[7]:

import tensorflow as tf

def neural_net_image_input(image_shape):
    """
    Return a Tensor for a bach of image input
    : image_shape: Shape of the images
    : return: Tensor for image input.
    """
    image = tf.placeholder(tf.float32 , shape=[None , *image_shape], name='x')
    return image

def neural_net_label_input(n_classes):
    """
    Return a Tensor for a batch of label input
    : n_classes: Number of classes
    : return: Tensor for label input.
    """
    labels = tf.placeholder(tf.float32, shape=(None, n_classes), name='y')
    return labels

def neural_net_keep_prob_input():
    """
    Return a Tensor for keep probability
    : return: Tensor for keep probability.
    """
    ans = tf.placeholder(tf.float32, name='keep_prob')
    return ans

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tf.reset_default_graph()
tests.test_nn_image_inputs(neural_net_image_input)
tests.test_nn_label_inputs(neural_net_label_input)
tests.test_nn_keep_prob_inputs(neural_net_keep_prob_input)

# ### Convolution and Max Pooling Layer
# Convolution layers have a lot of success with images. For this code cell, you should implement the function `conv2d_maxpool` to apply convolution then max pooling:
# * Create the weight and bias using `conv_ksize`, `conv_num_outputs` and the shape of `x_tensor`.
# * Apply a convolution to `x_tensor` using weight and `conv_strides`.
#  * We recommend you use same padding, but you're welcome to use any padding.
# * Add bias
# * Add a nonlinear activation to the convolution.
# * Apply Max Pooling using `pool_ksize` and `pool_strides`.
#  * We recommend you use same padding, but you're welcome to use any padding.
#
# **Note:** You **can't** use [TensorFlow Layers](https://www.tensorflow.org/api_docs/python/tf/layers) or [TensorFlow Layers (contrib)](https://www.tensorflow.org/api_guides/python/contrib.layers) for **this** layer, but you can still use TensorFlow's [Neural Network](https://www.tensorflow.org/api_docs/python/tf/nn) package. You may still use the shortcut option for all the **other** layers.

# In[8]:

def conv2d_maxpool(x_tensor, conv_num_outputs, conv_ksize, conv_strides, pool_ksize, pool_strides):
    """
    Apply convolution then max pooling to x_tensor
    :param x_tensor: TensorFlow Tensor
    :param conv_num_outputs: Number of outputs for the convolutional layer
    :param conv_ksize: kernal size 2-D Tuple for the convolutional layer
    :param conv_strides: Stride 2-D Tuple for convolution
    :param pool_ksize: kernal size 2-D Tuple for pool
    :param pool_strides: Stride 2-D Tuple for pool
    : return: A tensor that represents convolution and max pooling of x_tensor
    """

    _, input_width, input_height, input_depth = x_tensor.get_shape().as_list()

    weights = tf.Variable(tf.truncated_normal([*conv_ksize, input_depth, conv_num_outputs], mean=0.0, stddev=0.05, dtype=tf.float32))
    biases = tf.Variable(tf.zeros(conv_num_outputs), dtype=tf.float32)

    conv = tf.nn.conv2d(input=x_tensor, filter=weights, strides=[1, *conv_strides, 1], padding='SAME')
    conv = tf.nn.bias_add(conv, biases)
    conv = tf.nn.max_pool(conv, ksize=[1, *pool_ksize, 1], strides=[1, *pool_strides, 1], padding='SAME')
    conv = tf.nn.elu(conv)

    return conv

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_con_pool(conv2d_maxpool)

# ### Flatten Layer
# Implement the `flatten` function to change the dimension of `x_tensor` from a 4-D tensor to a 2-D tensor.  The output should be the shape (*Batch Size*, *Flattened Image Size*). Shortcut option: you can use classes from the [TensorFlow Layers](https://www.tensorflow.org/api_docs/python/tf/layers) or [TensorFlow Layers (contrib)](https://www.tensorflow.org/api_guides/python/contrib.layers) packages for this layer. For more of a challenge, only use other TensorFlow packages.

# In[9]:

def flatten(x_tensor):
    """
    Flatten x_tensor to (Batch Size, Flattened Image Size)
    : x_tensor: A tensor of size (Batch Size, ...), where ... are the image dimensions.
    : return: A tensor of size (Batch Size, Flattened Image Size).
    """
#     return tf.contrib.layers.flatten(x_tensor)
    shapes = x_tensor.get_shape().as_list()
    batch_size = shapes[0]
    if batch_size is None:
        batch_size = -1
    size = 1
    for i in shapes[1:]:
        size *= i
    return tf.reshape(x_tensor, [batch_size, size])

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_flatten(flatten)

# ### Fully-Connected Layer
# Implement the `fully_conn` function to apply a fully connected layer to `x_tensor` with the shape (*Batch Size*, *num_outputs*). Shortcut option: you can use classes from the [TensorFlow Layers](https://www.tensorflow.org/api_docs/python/tf/layers) or [TensorFlow Layers (contrib)](https://www.tensorflow.org/api_guides/python/contrib.layers) packages for this layer. For more of a challenge, only use other TensorFlow packages.

# In[10]:

def fully_conn(x_tensor, num_outputs):
    """
    Apply a fully connected layer to x_tensor using weight and bias
    : x_tensor: A 2-D tensor where the first dimension is batch size.
    : num_outputs: The number of output that the new tensor should be.
    : return: A 2-D tensor where the second dimension is num_outputs.
    """
#     return tf.contrib.layers.fully_connected(inputs = x_tensor, num_outputs=num_outputs)

    size = x_tensor.get_shape().as_list()[1]
    weights = tf.Variable(tf.truncated_normal([size, num_outputs], mean=0, stddev=0.1))
    bias = tf.Variable(tf.zeros(num_outputs))

    layer = tf.matmul(x_tensor, weights)
    layer = tf.nn.bias_add(layer, bias)

    layer = tf.nn.elu(layer)

    return layer

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_fully_conn(fully_conn)

# ### Output Layer
# Implement the `output` function to apply a fully connected layer to `x_tensor` with the shape (*Batch Size*, *num_outputs*). Shortcut option: you can use classes from the [TensorFlow Layers](https://www.tensorflow.org/api_docs/python/tf/layers) or [TensorFlow Layers (contrib)](https://www.tensorflow.org/api_guides/python/contrib.layers) packages for this layer. For more of a challenge, only use other TensorFlow packages.
#
# **Note:** Activation, softmax, or cross entropy should **not** be applied to this.

# In[11]:

def output(x_tensor, num_outputs):
    """
    Apply a output layer to x_tensor using weight and bias
    : x_tensor: A 2-D tensor where the first dimension is batch size.
    : num_outputs: The number of output that the new tensor should be.
    : return: A 2-D tensor where the second dimension is num_outputs.
    """
#     return tf.contrib.layers.fully_connected(inputs=x_tensor, num_outputs=num_outputs, activation_fn=None)
    size = x_tensor.get_shape().as_list()[1]
    weights = tf.Variable(tf.truncated_normal([size, num_outputs], mean=0, stddev=0.1))
    bias = tf.Variable(tf.zeros(num_outputs))

    layer = tf.matmul(x_tensor, weights)
    layer = tf.nn.bias_add(layer, bias)

    return layer

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_output(output)

# ### Create Convolutional Model
# Implement the function `conv_net` to create a convolutional neural network model. The function takes in a batch of images, `x`, and outputs logits.  Use the layers you created above to create this model:
#
# * Apply 1, 2, or 3 Convolution and Max Pool layers
# * Apply a Flatten Layer
# * Apply 1, 2, or 3 Fully Connected Layers
# * Apply an Output Layer
# * Return the output
# * Apply [TensorFlow's Dropout](https://www.tensorflow.org/api_docs/python/tf/nn/dropout) to one or more layers in the model using `keep_prob`.

# In[12]:

def conv_net(x, keep_prob):
    """
    Create a convolutional neural network model
    : x: Placeholder tensor that holds image data.
    : keep_prob: Placeholder tensor that hold dropout keep probability.
    : return: Tensor that represents logits
    """
    x = conv2d_maxpool(x, 32, (3, 3), (1, 1), (2, 2), (2, 2))
    x = conv2d_maxpool(x, 64, (3, 3), (1, 1), (2, 2), (2, 2))
    x = tf.nn.dropout(x, keep_prob)
    x = conv2d_maxpool(x, 128, (3, 3), (1, 1), (2, 2), (2, 2))
    x = conv2d_maxpool(x, 256, (3, 3), (1, 1), (2, 2), (2, 2))

    x = flatten(x)

    x = fully_conn(x, 1024)
    x = tf.nn.dropout(x, keep_prob)
    x = fully_conn(x, 512)
    x = tf.nn.dropout(x, keep_prob)
    out = output(x, 10)

    return out

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""

##############################
## Build the Neural Network ##
##############################

# Remove previous weights, bias, inputs, etc..
tf.reset_default_graph()

# Inputs
x = neural_net_image_input((32, 32, 3))
y = neural_net_label_input(10)
keep_prob = neural_net_keep_prob_input()

# Model
logits = conv_net(x, keep_prob)

# Name logits Tensor, so that is can be loaded from disk after training
logits = tf.identity(logits, name='logits')

# Loss and Optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y))
optimizer = tf.train.AdamOptimizer().minimize(cost)

# Accuracy
correct_pred = tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32), name='accuracy')

tests.test_conv_net(conv_net)

# ## Train the Neural Network
# ### Single Optimization
# Implement the function `train_neural_network` to do a single optimization.  The optimization should use `optimizer` to optimize in `session` with a `feed_dict` of the following:
# * `x` for image input
# * `y` for labels
# * `keep_prob` for keep probability for dropout
#
# This function will be called for each batch, so `tf.global_variables_initializer()` has already been called.
#
# Note: Nothing needs to be returned. This function is only optimizing the neural network.

# In[13]:

def train_neural_network(session, optimizer, keep_probability, feature_batch, label_batch):
    """
    Optimize the session on a batch of images and labels
    : session: Current TensorFlow session
    : optimizer: TensorFlow optimizer function
    : keep_probability: keep probability
    : feature_batch: Batch of Numpy image data
    : label_batch: Batch of Numpy label data
    """
    session.run(optimizer, feed_dict={x: feature_batch, y: label_batch, keep_prob: keep_probability})

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_train_nn(train_neural_network)

# ### Show Stats
# Implement the function `print_stats` to print loss and validation accuracy.  Use the global variables `valid_features` and `valid_labels` to calculate validation accuracy.  Use a keep probability of `1.0` to calculate the loss and validation accuracy.

# In[14]:

def print_stats(session, feature_batch, label_batch, cost, accuracy):
    """
    Print information about loss and validation accuracy
    : session: Current TensorFlow session
    : feature_batch: Batch of Numpy image data
    : label_batch: Batch of Numpy label data
    : cost: TensorFlow cost function
    : accuracy: TensorFlow accuracy function
    """
    loss = session.run(cost, feed_dict={x: feature_batch, y: label_batch, keep_prob: 1.})
    valid_acc = session.run(accuracy, feed_dict={x: valid_features, y: valid_labels, keep_prob:1.})
    print('Loss: {:>10.4f} Validation Accuracy: {:.6f}'.format(loss, valid_acc))

# ### Hyperparameters
# Tune the following parameters:
# * Set `epochs` to the number of iterations until the network stops learning or start overfitting
# * Set `batch_size` to the highest number that your machine has memory for.  Most people set them to common sizes of memory:
#  * 64
#  * 128
#  * 256
#  * ...
# * Set `keep_probability` to the probability of keeping a node using dropout

# In[15]:

# TODO: Tune Parameters
epochs = 20
batch_size = 128
keep_probability = 0.5

# ### Train on a Single CIFAR-10 Batch
# Instead of training the neural network on all the CIFAR-10 batches of data, let's use a single batch. This should save time while you iterate on the model to get a better accuracy.  Once the final validation accuracy is 50% or greater, run the model on all the data in the next section.

# In[16]:

"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
print('Checking the Training on a Single Batch...')
with tf.Session() as sess:
    # Initializing the variables
    sess.run(tf.global_variables_initializer())

    # Training cycle
    for epoch in range(epochs):
        batch_i = 1
        for batch_features, batch_labels in helper.load_preprocess_training_batch(batch_i, batch_size):
            train_neural_network(sess, optimizer, keep_probability, batch_features, batch_labels)
        print('Epoch {:>2}, CIFAR-10 Batch {}:  '.format(epoch + 1, batch_i), end='')
        print_stats(sess, batch_features, batch_labels, cost, accuracy)

# ### Fully Train the Model
# Now that you got a good accuracy with a single CIFAR-10 batch, try it with all five batches.

# In[17]:

"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
save_model_path = './image_classification'

print('Training...')
with tf.Session() as sess:
    # Initializing the variables
    sess.run(tf.global_variables_initializer())

    # Training cycle
    for epoch in range(epochs):
        # Loop over all batches
        n_batches = 5
        for batch_i in range(1, n_batches + 1):
            for batch_features, batch_labels in helper.load_preprocess_training_batch(batch_i, batch_size):
                train_neural_network(sess, optimizer, keep_probability, batch_features, batch_labels)
            print('Epoch {:>2}, CIFAR-10 Batch {}:  '.format(epoch + 1, batch_i), end='')
            print_stats(sess, batch_features, batch_labels, cost, accuracy)

    # Save Model
    saver = tf.train.Saver()
    save_path = saver.save(sess, save_model_path)

# # Checkpoint
# The model has been saved to disk.
# ## Test Model
# Test your model against the test dataset.  This will be your final accuracy. You should have an accuracy greater than 50%. If you don't, keep tweaking the model architecture and parameters.

# In[18]:

"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
get_ipython().magic('matplotlib inline')
get_ipython().magic("config InlineBackend.figure_format = 'retina'")

import tensorflow as tf
import pickle
import helper
import random

# Set batch size if not already set
try:
    if batch_size:
        pass
except NameError:
    batch_size = 64

save_model_path = './image_classification'
n_samples = 4
top_n_predictions = 3

def test_model():
    """
    Test the saved model against the test dataset
    """

    test_features, test_labels = pickle.load(open('preprocess_test.p', mode='rb'))
    loaded_graph = tf.Graph()

    with tf.Session(graph=loaded_graph) as sess:
        # Load model
        loader = tf.train.import_meta_graph(save_model_path + '.meta')
        loader.restore(sess, save_model_path)

        # Get Tensors from loaded model
        loaded_x = loaded_graph.get_tensor_by_name('x:0')
        loaded_y = loaded_graph.get_tensor_by_name('y:0')
        loaded_keep_prob = loaded_graph.get_tensor_by_name('keep_prob:0')
        loaded_logits = loaded_graph.get_tensor_by_name('logits:0')
        loaded_acc = loaded_graph.get_tensor_by_name('accuracy:0')

        # Get accuracy in batches for memory limitations
        test_batch_acc_total = 0
        test_batch_count = 0

        for train_feature_batch, train_label_batch in helper.batch_features_labels(test_features, test_labels, batch_size):
            test_batch_acc_total += sess.run(
                loaded_acc,
                feed_dict={loaded_x: train_feature_batch, loaded_y: train_label_batch, loaded_keep_prob: 1.0})
            test_batch_count += 1

        print('Testing Accuracy: {}\n'.format(test_batch_acc_total/test_batch_count))

        # Print Random Samples
        random_test_features, random_test_labels = tuple(zip(*random.sample(list(zip(test_features, test_labels)), n_samples)))
        random_test_predictions = sess.run(
            tf.nn.top_k(tf.nn.softmax(loaded_logits), top_n_predictions),
            feed_dict={loaded_x: random_test_features, loaded_y: random_test_labels, loaded_keep_prob: 1.0})
        helper.display_image_predictions(random_test_features, random_test_labels, random_test_predictions)

test_model()

# ## Why 50-70% Accuracy?
# You might be wondering why you can't get an accuracy any higher. First things first, 50% isn't bad for a simple CNN.  Pure guessing would get you 10% accuracy. However, you might notice people are getting scores [well above 70%](http://rodrigob.github.io/are_we_there_yet/build/classification_datasets_results.html#43494641522d3130).  That's because we haven't taught you all there is to know about neural networks. We still need to cover a few more techniques.
# ## Submitting This Project
# When submitting this project, make sure to run all the cells before saving the notebook.  Save the notebook file as "dlnd_image_classification.ipynb" and save it as a HTML file under "File" -> "Download as".  Include the "helper.py" and "problem_unittests.py" files in your submission.


#!/usr/bin/env python
# coding: utf-8

# In[1]:

# Here is the function to fetch the data from the github repository. It is useful in particular if
# data changes regularly, as it allows you to write a small script that you can run whenever you need
# to fetch the latest data
import os
import tarfile
from six.moves import urllib

# In[2]:

# DOWNLOAD_ROOT is the which contains the .tgz file of data (write it as it is)
DOWNLOAD_ROOT = "https://raw.githubusercontent.com/ageron/handson-ml/master/datasets/housing/"
# HOUSING_PATH is the path where .tgz file will be downloaded in your system (it can be changed)
HOUSING_PATH = "/home/cipher/aston/datum/O'reilly_handson_ml/chapter_2"
# HOUSING_URL is the download path of the .tgz file (write it as it is)
HOUSING_URL = DOWNLOAD_ROOT + "housing.tgz"

# In[3]:

def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):
    if not os.path.isdir(housing_path):
        os.makedirs(housing_path)
    tgz_path = os.path.join(housing_path, "housing.tgz")
    urllib.request.urlretrieve(housing_url, tgz_path)
    housing_tgz = tarfile.open(tgz_path)
    housing_tgz.extractall(path=housing_path)
    housing_tgz.close()

# In[4]:

fetch_housing_data(HOUSING_URL, HOUSING_PATH)

# In[5]:

# Load the data using pandas
import pandas as pd

# In[6]:

def load_housing_data(housing_path=HOUSING_PATH):
    csv_path = os.path.join(housing_path, "housing.csv")
    return pd.read_csv(csv_path)

# In[7]:

housing = load_housing_data()
housing.head()

# In[8]:

# info() method is useful to get a quick description of data
housing.info()

# Here we can clearly see there are 20640 instances in the dataset, of which total_bedrooms attribute has
# only 20,433 instances meaning 207 instances are missing this feature. Also ocean_proximity attribute type is
# object & when you look at the top five rows, you probably noticed that the values in ocean_proximity column
# were repetitive which means that it is a categorical attribute.

# In[9]:

# Now to find out what categories exist you can use
housing["ocean_proximity"].value_counts()

# In[10]:

# Another method describe() shows a summary of the numerical attributes
housing.describe()

# In[11]:

# Another way to get a feel of the type of data you are dealing with is to plot a histogram for each
# numerical attribute
get_ipython().run_line_magic('matplotlib', 'inline')
import matplotlib.pyplot as plt
housing.hist(bins=50, figsize=(20,15))
plt.show()

# In[12]:

# The given dataset should be split into train_set and test_set which are used to train the model and test
# the model respectively. Scikit-Learn provides train_test_split function to do so.
from sklearn.model_selection import train_test_split
train_set, test_set = train_test_split(housing, test_size=0.2, random_state=42)

# Here test_size ensures 20% of the dataset is picked up randomly as test_set
# The test_set should be representative of the whole dataset, which is ensured by Scikit-Learn. This is called
# stratified sampling.

# In[13]:

# median_income is a very important attribute to predict median housing prices. We need to ensure that the
# test_set is representative of the various categories of income in whole dataset. Since median_income is a
# continuos numerical attribute, you first need to create and income attribute.
# Looking at the median_income histogram we see most median_income values are clustered around $20,000-$50,000,
# but some median_income go far beyond $60,000.
import numpy as np
housing["income_cat"] = np.ceil(housing["median_income"]/1.5)
housing["income_cat"].where(housing["income_cat"] < 5, 5.0, inplace=True)

# This creates an income category attribute by dividing the median_income by 1.5 (to limit the number of
# income categories), and rounding up using ceil (to have discrete categories), and then merging all the
# categories greater than 5 into category 5.

housing.head()

# In[14]:

# Now to do stratified sampling based on income category is to use Scikit_Learn's StratifiedShuffleSplit class.
from sklearn.model_selection import StratifiedShuffleSplit

split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
for train_index, test_index in split.split(housing, housing["income_cat"]):
    strat_train_set = housing.loc[train_index]
    strat_test_set = housing.loc[test_index]

# For more information, see Readme.md
# To see if it worked as expected, you can start by lookig at the income category proportions in full dataset
housing["income_cat"].value_counts()/len(housing)

# In[15]:

# Now to remove the income_cat attribute so the data is back to its original state:
for set_ in (strat_train_set, strat_test_set):
    set_.drop("income_cat", axis=1, inplace=True)

# In[16]:

# Our train data looks like
strat_train_set.head()

# In[17]:

# Let's get into more detail of the dataset but first make a copy of the train dataset.
housing = strat_train_set.copy()
housing.plot(kind="scatter", x="longitude", y="latitude")

# This looks like California but it is hard to see any pattern.

# In[18]:

# Setting the alpha option makes it easier to visualize the places where there is high density.
housing.plot(kind="scatter", x="longitude", y="latitude", alpha=0.1)

# The alpha blending value, between 0 (transparent) and 1 (opaque).
# You can clearly see the high-density areas, namely the Bay Area and around Los Angeles and San Diego,
# plus a long line of fairly high density in the Central Valley.

# In[19]:

# The radius of each circle represents the district's population (option s), and the color represents the price
# (option c). Here we will use a predefined color map (option cmap) called jet, which ranges from
# blue (low values) to red (high values).
housing.plot(kind="scatter", x="longitude", y="latitude", alpha=0.4, s=housing["population"]/100,
              label="population", figsize=(10,7), c="median_house_value", cmap=plt.get_cmap("jet"),
              colorbar=True
             )
plt.legend()

# This image tell you that the housing prices are very much related to the location ad the population density.

# In[20]:

# To find out the standard correation coefficient (also called Pearson's r) use
corr_matrix = housing.corr()
corr_matrix["median_house_value"].sort_values(ascending=False)

# Here we see a strong positive correlation between median_house_value and median_income i.e. median house
# value tends to go up when the median income goes up.
# There is a small negative correlation between the latitude and the median house value i.e. prices have a slight
# tendency to go down as you go north.

# In[21]:

# Experimenting with Attribute Combinations
# Total number of bedrooms in a district is not very useful if you don't know how many households there are. What
# you really want is the number of rooms per household.
# Similarly, the total number of bedrooms and the population per household also seems like an interesting
# attribute combination to look at
housing["rooms_per_household"] = housing["total_rooms"]/housing["households"]
housing["bedrooms_per_room"] = housing["total_bedrooms"]/housing["total_rooms"]
housing["population_per_household"] = housing["population"]/housing["households"]

# And now let's look at correlation matrix again:
corr_matrix = housing.corr()
corr_matrix["median_house_value"].sort_values(ascending=False)

# The new bedrooms_per_room attribute is much more correlated with the median house value than the total number
# of rooms or bedrooms. Apparently houses with a lower bedroom/room ratio tend to be more expensive.

# In[22]:

# Now let's separate the predictors and the labels
housing = strat_train_set.drop("median_house_value", axis=1)
housing_labels = strat_train_set["median_house_value"].copy()

# In[23]:

# Scikit-Learn provides a handy class to take care of missing values: SimpleImputer.
# First, you need to create an Imputer instance, specifying that you want to replace each attribute's missing
# values with the median of that attribute.
from sklearn.impute import SimpleImputer
imputer = SimpleImputer(strategy="median")

# Since meadian can only be computed on numerical atttributes, we need to create a copy of the data without the
# text attribute ocean_proximity
housing_num = housing.drop("ocean_proximity", axis=1)

# Now you can fit the imputer instance to the training data using the fit() method:
imputer.fit(housing_num)

# The imputer has simply computed the median of each attribute and stored the result in its statistics_
# instance variable.
imputer.statistics_

# In[24]:

# Now you can use this "trained" imputer to transform the training set by replacing missing values by the
# learned medians. The result is a plain Numpy array.
X = imputer.transform(housing_num)

# In[25]:

# Earlier we left out the categorical attribute ocean_proximity because it is a text attribute so we cannot
# compute its median. Let's convert these text labels to numbers using Scikit-Learn's transformer for this
# task called LabelEncoder
from sklearn.preprocessing import LabelEncoder
encoder = LabelEncoder()
housing_cat = housing["ocean_proximity"]
housing_cat_encoded = encoder.fit_transform(housing_cat)
housing_cat_encoded

# In[26]:

# Although Scikit-Learn provied many useful transformers, you will need to write yourown. All you need is to
# create a class and implement three methods: fit() (returning self), transform(), and fit_transform().
# You can get the last one for free by simply adding TransformerMixin as base class. Also, if you add
# BaseEstimator as a base class (and avoid *args and **kargs in your constructor) you will get two extra methods
# (get_params() and set_params()) that will be useful for automatic hyperparameter tuning.
from sklearn.base import BaseEstimator, TransformerMixin
rooms_ix, bedrooms_ix, population_ix, household_ix = 3, 4, 5, 6

class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
    def __init__(self, add_bedrooms_per_room = True):  # no *args pr **kargs
        self.add_bedrooms_per_room = add_bedrooms_per_room
    def fit(self, X, y=None):
        return self  # nothing else to do
    def transform(self, X, y=None):
        rooms_per_household = X[:, rooms_ix]/X[:, household_ix]
        population_per_household = X[:, population_ix]/X[:, household_ix]
        if self.add_bedrooms_per_room:
            bedrooms_per_room = X[:, bedrooms_ix]/X[:, rooms_ix]
            return np.c_[X, rooms_per_household, population_per_household, bedrooms_per_room]
        else:
            return np.c_[X, rooms_per_household, population_per_household]

attr_adder = CombinedAttributesAdder(add_bedrooms_per_room=False)
housing_extra_attribs = attr_adder.transform(housing.values)

# In[27]:

# There are many data transformations steps that need to be executed in the right order. Fortunately, Scikit-Learn
# provides the Pipeline class to help with such sequences of transformations. Here is a small pipeline for the
# numerical attributes:
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler

num_pipeline = Pipeline([
    ('imputer', SimpleImputer(strategy="median")),
    ('attribs_adder', CombinedAttributesAdder()),
    ('std_scaler', StandardScaler()),
 ])

housing_num_tr = num_pipeline.fit_transform(housing_num)

# Here StandardScaler is used for standarization for the purpose of feature scaling i.e. to get all attributes
# to have the same scale. First it subtracts the mean value (so standarized values always have a zero mean),
# and then it divides by the variance so that the resulting distribution have zero variance.

# In[28]:

# Now it would be nice if we could feed a Pandas DataFrame directly into our pipeline, instead of having to first
# manually extract the numerical columns into a NumPy array.
from sklearn.base import BaseEstimator, TransformerMixin

class DataFrameSelector(BaseEstimator, TransformerMixin):
    def __init__(self, attribute_names):
        self.attribute_names = attribute_names
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        return X[self.attribute_names].values

# Our DataFrameSelector will transform the data by selecting the desired attributes, dropping the rest, and
# converting the resulting DataFrame to a NumPy array.

# In[29]:

# By default LabelBinarizer takes two input now after update to Scikit-Learn 0.18.0 because they said
# LabelBinarizer is meant fot labels only and not for features. To use LabelBinarizer here, you can create you own
class MyLabelBinarizer(TransformerMixin):
    def __init__(self, *args, **kwargs):
        self.encoder = LabelBinarizer(*args, **kwargs)
    def fit(self, x, y=0):
        self.encoder.fit(x)
        return self
    def transform(self, x, y=0):
        return self.encoder.transform(x)

# In[30]:

# You can write another pipeline for the categorical attributes as well by simply selecting the categorical
# attributes using a DataFrameSelector and then applying a LabelBinarizer.
from sklearn.preprocessing import LabelBinarizer
num_attribs = list(housing_num)
cat_attribs = ["ocean_proximity"]

num_pipeline = Pipeline([
    ('selector', DataFrameSelector(num_attribs)),
    ('imputer', SimpleImputer(strategy="median")),
    ('attribs_adder', CombinedAttributesAdder()),
    ('std_scaler', StandardScaler()),
 ])

cat_pipeline = Pipeline([
    ('selector', DataFrameSelector(cat_attribs)),
    ('label_binarizer', MyLabelBinarizer()),
])

# In[31]:

# To join above two pipelines use Scikit-Learn's FeatureUnion class
from sklearn.pipeline import FeatureUnion

full_pipeline = FeatureUnion(transformer_list=[
    ("num_pipeline", num_pipeline),
    ("cat_pipeline", cat_pipeline),
])

# And you can run the whole pipeline simply by:
housing_prepared = full_pipeline.fit_transform(housing)
housing_prepared.shape

# In[32]:

# Let's train a Linear Regression model
from sklearn.linear_model import LinearRegression

lin_reg = LinearRegression()
lin_reg.fit(housing_prepared, housing_labels)

#

# In[33]:

# Done! We now have a working Linear Regression model. Let's try it out on a few instances from training set.
some_data = housing.iloc[:5]
some_labels = housing_labels.iloc[:5]
some_data_prepared = full_pipeline.transform(some_data)
print("Predictions:", lin_reg.predict(some_data_prepared))

# It works, although the predictions are not exactly accurate.

# In[34]:

print("Labels:", list(some_labels))

# In[35]:

# Let's measure this regression model's RMSE on the whole training set using Scikit-Learn's mean_squared_error
# function
from sklearn.metrics import mean_squared_error
housing_predictions = lin_reg.predict(housing_prepared)
lin_mse = mean_squared_error(housing_labels, housing_predictions)
lin_rmse = np.sqrt(lin_mse)
lin_rmse

# This is not clearly a good score as median_housing_values range between $120,000 and $265,000, so a
# prediction error of $68,628 is not very satisfying. This is an example of a model underfitting the training
# data. To overcome this we can use a more powerful model, feed the training algorithm with better features, or
# to reduce the constraints on the model.

# In[36]:

# Let's try a more complex model. Let's train a DecisionTreeRegressor.
from sklearn.tree import DecisionTreeRegressor
tree_reg = DecisionTreeRegressor()
tree_reg.fit(housing_prepared, housing_labels)

# In[37]:

# Let's evaluate it on the training set
housing_predictions = tree_reg.predict(housing_prepared)
tree_mse = mean_squared_error(housing_labels, housing_predictions)
tree_rmse = np.sqrt(tree_mse)
tree_rmse

# Wait, what? No error at all? Model has badly overfit the data.
# We need to use part of the training set for training, and part for model validation.

# In[38]:

# One way to evaluate Decision Tree model would be to use the train_test_split function to split the training set
# into a smaller training set and a validation set, then train your mdoels against the smaller training set and
# evaluate them against the validation set.
# A great alternative is to use Scikit-Learn's cross-validation feature.
# Following code performs K-fold cross-validation: it randomly splits the training set into 10 distinct subsets
# called folds, then it trains and evaluates the Decision Tree model 10 times picking a different fold for
# evaluation every time and training on the other 9 folds.
from sklearn.model_selection import cross_val_score
scores = cross_val_score(tree_reg, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)
tree_rmse_scores = np.sqrt(-scores)

# For moer info see Readme.md
# The result is an array containing the 10 evaluation scores. Let's look at the results:
def display_scores(scores):
    print("Scores:", scores)
    print("Mean:", scores.mean())
    print("Standard Deviation:", scores.std())

display_scores(tree_rmse_scores)

# For more information see Readme.md
# Now the Decision Tree doiesn't look as good as it did earlier. In fact, it seems to perform worse than the
# Linear Regression model. The Decision Tree has a score of approximately 71,379, generally ± 2,458.

# In[39]:

# Let's compute the same scores for the Linnear Regression model
lin_scores = cross_val_score(lin_reg, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)
lin_rmse_scores = np.sqrt(-lin_scores)
display_scores(lin_rmse_scores)

# In[40]:

# Let's try one last mdoel now: the RandomForestRegressor. Random Forest works by training many Decision Trees
# on random subsets of the features, then averaging out their predictions. Building a model on top of many other
# models is called Ensemble Learning.
from sklearn.ensemble import RandomForestRegressor
forest_reg = RandomForestRegressor()
forest_reg.fit(housing_prepared, housing_labels)

# Let's compute the scores for RandomForestRegressor model
scores = cross_val_score(forest_reg, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)
forest_rmse_scores = np.sqrt(-scores)
display_scores(forest_rmse_scores)

# This is much better. However, note that the score on the training set is still much lower than on the validation
# sets, meaning that the model is still overfitting the training set.

# In[46]:

# Let's try a Support Machine Vector regressor with linear kernel
from sklearn.svm import SVR
svr_regl = SVR(kernel="linear")
svr_regl.fit(housing_prepared, housing_labels)

scores = cross_val_score(svr_regl, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)
svrl_rmse_scores = np.sqrt(-scores)
display_scores(svrl_rmse_scores)

# This is the best till now. See

# In[42]:

# Let's try another Support Machine Vector regressor with rbf kernel
from sklearn.svm import SVR
svr_reg = SVR(kernel="rbf")
svr_reg.fit(housing_prepared, housing_labels)

scores = cross_val_score(svr_reg, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)
svr_rmse_scores = np.sqrt(-scores)
display_scores(svr_rmse_scores)

# In[76]:

# You should save every model you experiment with, so you can come back easily to any model you want.
from sklearn.externals import joblib

joblib.dump(lin_reg, "lin_reg.pkl")
joblib.dump(tree_reg, "tree_reg.pkl")
joblib.dump(forest_reg, "forest_reg.pkl")
joblib.dump(svr_reg, "svr_reg.pkl")      # svr_reg for SVM with default kernel i.e., kernel="rbf"
joblib.dump(svr_regl, "svr_regl.pkl")    # svr_regl for SVM with kernel="linear"

# In[74]:

# Now to load any model type
my_model_loaded = joblib.load("forest_reg.pkl")
my_model_loaded

# In[80]:

# To fine tune your model i.e., to get a great combination of hyperparameter values, you can use Scikit-Learn's
# GridSearchCV. All you need to do is tell it which hyperparameters you want it to experiment with, and what
# values to try out, and it will evaluate all the possible combinations of hyperparameter values, using
# cross-validation.
# The following code searches for the best combination of hyperparameter values for the RandomForestRegressor:
from sklearn.model_selection import GridSearchCV

param_grid = [
    {'n_estimators': [3, 10, 30], 'max_features': [2, 4, 6, 8]},
    {'bootstrap': [False], 'n_estimators': [3, 10], 'max_features': [2, 3, 4]},
]

forest_reg = RandomForestRegressor()

grid_search = GridSearchCV(forest_reg, param_grid, cv=5, scoring='neg_mean_squared_error')
grid_search.fit(housing_prepared, housing_labels)

# This param_grid tells Scikit-Learn to first evaluate all 3 X 4 = 12 combinations of n_estimators and
# max_features hyperparameter values specified in the first dict, then try all 2 X 3 = 6 combinations of
# hyperparameter values in the second dict, but this time with the bootstrap hyperparameter set to False instead
# of True (which is the default value of this parameter).
# All in all, the grid will explore 12 + 6 = 18 combinations of RandomForestRegressor hyperparamter values, and it
# will train each model five times i.e., there will be 18 X 5 = 90 rounds of training!

grid_search.best_params_
# This gives out the best combination of parameters like this:

# In[79]:

# The evaluation scores are also available
cvres = grid_search.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
    print(np.sqrt(-mean_score), params)

# In this example, we obtain the best solution by setting the max_features hyperparameter to 6, and then
# n_estimators hyperparameter to 30. The RMSE score for this combination is 49889, which is slightly better than
# the score you go tearlier using the default hyperparameter values (which was 52927).
# For Randomized search see Readme.mdf

# In[86]:

# GridSearchCV can also reveal the relative importance of each attribute for making accurate predictions:
feature_importances = grid_search.best_estimator_.feature_importances_
feature_importances
extra_attribs = ["rooms_per_hhold", "pop_per_hhold", "bedrooms_per_room"]
cat_one_hot_attribs = list(encoder.classes_)
attributes = num_attribs + extra_attribs + cat_one_hot_attribs
sorted(zip(feature_importances, attributes), reverse=True)

# cat_one_attribs contains the subcategories of ocean_proximity separately.
# The purpose of zip() is to map the similar index of multiple containers so that they can be used just
# using as single entity.

# In[91]:

# Evaluate the final model on the test set; just set the predictors and the labels from your test run, run your
# full_pipeline to transform the data, and evaluate the final model on the test set.
final_model = grid_search.best_estimator_

X_test = strat_test_set.drop("median_house_value", axis=1)
y_test = strat_test_set["median_house_value"].copy()

X_test_prepared = full_pipeline.transform(X_test)

final_predictions = final_model.predict(X_test_prepared)

final_mse = mean_squared_error(y_test, final_predictions)
final_rmse = np.sqrt(final_mse)
final_rmse

# In[92]:

# Congratulations for your first working model.

# In[ ]:


"""
A Sage extension which adds sage-specific features:

* magics
  - %loadfile
  - %attach
  - %mode (like %maxima, etc.)
* preparsing of input
  - also make runfile and attach magics so that the '%' is optional, but encouraged
* loading Sage library
* running init.sage
* changing prompt to Sage prompt
* Display hook

TESTS:

We test that preparsing is off for ``%runfile``, on for ``%time``::

    sage: import os, re
    sage: from sage.misc.interpreter import get_test_shell
    sage: from sage.misc.misc import tmp_dir
    sage: shell = get_test_shell()
    sage: TMP = tmp_dir()

The temporary directory should have a name of the form
``.../12345/...``, to demonstrate that file names are not
preparsed when calling ``%runfile``. ::

    sage: bool(re.search('/[0-9]+/', TMP))
    True
    sage: tmp = os.path.join(TMP, 'run_cell.py')
    sage: f = open(tmp, 'w'); f.write('a = 2\n'); f.close()
    sage: shell.run_cell('%runfile '+tmp)
    sage: shell.run_cell('a')
    2

In contrast, input to the ``%time`` magic command is preparsed::

    sage: shell.run_cell('%time 594.factor()')
    CPU times: user ...
    Wall time: ...
    2 * 3^3 * 11
"""

from IPython.core.hooks import TryNext
from IPython.core.magic import Magics, magics_class, line_magic
import os
import sys
import sage
import sage.all
from sage.misc.interpreter import preparser
from sage.misc.preparser import preparse

@magics_class
class SageMagics(Magics):

    @line_magic
    def runfile(self, s):
        r"""
        Loads the code contained in the file ``s``. This is designed
        to be used from the command line as ``%runfile /path/to/file``.

        :param s: file to be loaded
        :type s: string

        EXAMPLES::

            sage: import os
            sage: from sage.misc.interpreter import get_test_shell
            sage: from sage.misc.misc import tmp_dir
            sage: shell = get_test_shell()
            sage: tmp = os.path.join(tmp_dir(), 'run_cell.py')
            sage: f = open(tmp, 'w'); f.write('a = 2\n'); f.close()
            sage: shell.run_cell('%runfile '+tmp)
            sage: shell.run_cell('a')
            2
        """
        from sage.misc.preparser import load_wrap
        return self.shell.ex(load_wrap(s, attach=False))

    @line_magic
    def attach(self, s):
        r"""
        Attaches the code contained in the file ``s``. This is
        designed to be used from the command line as
        ``%attach /path/to/file``.

        :param s: file to be attached
        :type s: string

        EXAMPLES::

            sage: import os
            sage: from sage.misc.interpreter import get_test_shell
            sage: shell = get_test_shell()
            sage: tmp = os.path.normpath(os.path.join(SAGE_TMP, 'run_cell.py'))
            sage: f = open(tmp, 'w'); f.write('a = 2\n'); f.close()
            sage: shell.run_cell('%attach ' + tmp)
            sage: shell.run_cell('a')
            2
            sage: sleep(1)  # filesystem timestamp granularity
            sage: f = open(tmp, 'w'); f.write('a = 3\n'); f.close()

        Note that the doctests are never really at the command prompt, so
        we call the input hook manually::

            sage: shell.run_cell('from sage.misc.inputhook import sage_inputhook')
            sage: shell.run_cell('sage_inputhook()')
            ### reloading attached file run_cell.py modified at ... ###
            0

            sage: shell.run_cell('a')
            3
            sage: shell.run_cell('detach(%r)'%tmp)
            sage: shell.run_cell('attached_files()')
            []
            sage: os.remove(tmp)
        """
        from sage.misc.preparser import load_wrap
        return self.shell.ex(load_wrap(s, attach=True))

    @line_magic
    def iload(self, s):
        """
        A magic command to interactively load a file as in MAGMA.

        :param s: the file to be interactively loaded
        :type s: string

        .. note::

            Currently, this cannot be completely doctested as it
            relies on :func:`raw_input`.

        EXAMPLES::

            sage: ip = get_ipython()           # not tested: works only in interactive shell
            sage: ip.magic_iload('/dev/null')  # not tested: works only in interactive shell
            Interactively loading "/dev/null"  # not tested: works only in interactive shell
        """
        try:
            name = str(eval(s))
        except Exception:
            name = s.strip()

        try:
            F = open(name)
        except IOError:
            raise ImportError, 'could not open file "%s"'%name

        shell = self.shell

        #We need to update the execution count so that the history for the
        #iload command and the history for the first line of the loaded
        #file are not written to the history database with the same line
        #number (execution count).  This happens since the execution count
        #is updated only after the magic command is run.
        shell.execution_count += 1

        print 'Interactively loading "%s"'%name

        # The following code is base on IPython's
        # InteractiveShell.interact,
        more = False
        for line in F.readlines():
            prompt = shell.prompt_manager.render('in' if not more else 'in2', color=True)
            raw_input(prompt.encode('utf-8') + line.rstrip())

            shell.input_splitter.push(line)
            more = shell.input_splitter.push_accepts_more()
            if not more:
                source, source_raw = shell.input_splitter.source_raw_reset()
                shell.run_cell(source_raw, store_history=True)

    _magic_display_status = "simple"
    @line_magic
    def display(self, mode):
        """
        A magic command to switch between simple display and ASCII art display.

        :param mode: the mode (``ascii_art`` (and optionally a ``width``) or ``simple``)
        :type s: string

        How to use: if you want activate the ASCII art mod::

            sage: from sage.misc.interpreter import get_test_shell
            sage: shell = get_test_shell()
            sage: shell.run_cell('%display ascii_art')

        That means you don't have to use :func:`ascii_art` to get an ASCII art
        output::

            sage: shell.run_cell("i = var('i')")
            sage: shell.run_cell('sum(i^2*x^i, i, 0, 10)')
                 10       9       8       7       6       5       4      3      2
            100*x   + 81*x  + 64*x  + 49*x  + 36*x  + 25*x  + 16*x  + 9*x  + 4*x  + x

        Then when you want return in 'textual mode'::

            sage: shell.run_cell('%display simple')
            sage: shell.run_cell('sum(i^2*x^i, i, 0, 10)')
            100*x^10 + 81*x^9 + 64*x^8 + 49*x^7 + 36*x^6 + 25*x^5 + 16*x^4 + 9*x^3 + 4*x^2 + x

        Sometime you could have to use a special output width and you
        could specify it::

            sage: shell.run_cell('%display ascii_art')
            sage: shell.run_cell('StandardTableaux(4).list()')
            [
            [                                                                  1  4    1  3
            [                 1  3  4    1  2  4    1  2  3    1  3    1  2    2       2
            [   1  2  3  4,   2      ,   3      ,   4      ,   2  4,   3  4,   3   ,   4
            <BLANKLINE>
                        1 ]
                1  2    2 ]
                3       3 ]
            ,   4   ,   4 ]
            sage: shell.run_cell('%display ascii_art 50')
            sage: shell.run_cell('StandardTableaux(4).list()')
            [
            [
            [                 1  3  4    1  2  4    1  2  3
            [   1  2  3  4,   2      ,   3      ,   4      ,
            <BLANKLINE>
                                                      1 ]
                              1  4    1  3    1  2    2 ]
              1  3    1  2    2       2       3       3 ]
              2  4,   3  4,   3   ,   4   ,   4   ,   4 ]
            sage: shell.run_cell('%display simple')
        """
        import displayhook, ascii_art
        args_split = mode.split(" ")
        if len(args_split) < 2:
            if mode == "":
                self._magic_display_status = "ascii_art" \
                    if self._magic_display_status == "simple" else "simple"
            else:
                self._magic_display_status = mode
            ascii_art.MAX_WIDTH = None
        else:
            self._magic_display_status =  args_split[0]
            assert(args_split[0] == "ascii_art"), "if a width is given then the mode must be `ascii_art`"
            try:
                ascii_art.MAX_WIDTH = int(args_split[1])
            except Exception:
                raise AttributeError("Second argument must be a non-negative integer")
        try:
            displayhook.SPTextFormatter.set_display(self._magic_display_status)
        except Exception:
            print mode, args_split
            raise AttributeError("First argument must be `simple` or `ascii_art` or the method must be call without argument")

import displayhook
class SageCustomizations(object):
    startup_code = """from sage.all_cmdline import *
from sage.misc.interpreter import sage_prompt
"""

    def __init__(self, shell=None):
        """
        Initialize the Sage plugin.
        """
        self.shell = shell
        self.auto_magics = SageMagics(shell)
        shell.register_magics(self.auto_magics)
        displayhook.SPTextFormatter = displayhook.SagePlainTextFormatter(config=shell.config)
        shell.display_formatter.formatters['text/plain'] = displayhook.SPTextFormatter
        from sage.misc.edit_module import edit_devel
        self.shell.set_hook('editor', edit_devel)
        self.init_inspector()
        self.init_line_transforms()
        self.register_interface_magics()

        import sage.misc.inputhook
        sage.misc.inputhook.install()

        # right now, the shutdown hook calling quit_sage() doesn't
        # work when we run doctests that involve creating test shells.
        # The test run segfaults right when it exits, complaining
        # about a bad memory access in the pari_close() function.
        #self.set_quit_hook()

        if os.environ.get('SAGE_IMPORTALL', 'yes') != 'yes':
            return

        self.init_environment()

    def register_interface_magics(self):
        """Register magics for each of the Sage interfaces"""
        from sage.misc.superseded import deprecation
        interfaces = [(name, obj)
                      for name, obj in sage.interfaces.all.__dict__.items()
                      if isinstance(obj, sage.interfaces.interface.Interface)]

        for real_name, obj in interfaces:
            def tmp(line, name=real_name):
                self.shell.run_cell('%s.interact()' % name)
            tmp.__doc__ = "Interact with %s" % real_name
            self.shell.register_magic_function(tmp, magic_name=real_name)

            obj_name = obj.name()
            if real_name != obj_name:
                def tmp_deprecated(line, name=real_name, badname=obj_name):
                    deprecation(6288, 'Use %%%s instead of %%%s.' % (name,
                                                                     badname))
                    self.shell.run_cell('%s.interact()' % name)
                tmp_deprecated.__doc__ = "Interact with %s" % real_name
                self.shell.register_magic_function(tmp_deprecated, magic_name=obj_name)

    def set_quit_hook(self):
        """
        Set the exit hook to cleanly exit Sage.  This does not work in all cases right now.
        """
        def quit(shell):
            import sage
            sage.all.quit_sage()
        self.shell.set_hook('shutdown_hook', quit)

    def init_environment(self):
        """
        Set up Sage command-line environment
        """
        try:
            self.shell.run_cell('from sage.all import Integer, RealNumber')
        except Exception:
            import traceback
            print "Error importing the Sage library"
            traceback.print_exc()
            print
            print "To debug this, you can run:"
            print 'sage -ipython -i -c "import sage.all"'
            print 'and then type "%debug" to enter the interactive debugger'
            sys.exit(1)
        self.shell.run_cell(self.startup_code)
        self.run_init()

    def run_init(self):
        """
        Run Sage's initial startup file.
        """
        startup_file = os.environ.get('SAGE_STARTUP_FILE', '')
        if os.path.exists(startup_file):
            with open(startup_file, 'r') as f:
                self.shell.run_cell(f.read(), store_history=False)

    def init_inspector(self):
        # Ideally, these would just be methods of the Inspector class
        # that we could override; however, IPython looks them up in
        # the global :class:`IPython.core.oinspect` module namespace.
        # Thus, we have to monkey-patch.
        from sage.misc import sagedoc, sageinspect
        import IPython.core.oinspect
        IPython.core.oinspect.getdoc = sageinspect.sage_getdoc #sagedoc.my_getdoc
        IPython.core.oinspect.getsource = sagedoc.my_getsource
        IPython.core.oinspect.getargspec = sageinspect.sage_getargspec

    def init_line_transforms(self):
        """
        Set up transforms (like the preparser).
        """
        import sage
        import sage.all
        from interpreter import (SagePreparseTransformer,
                                 sage_prompt_transformer,
                                 magic_transformer)
        for s in (self.shell.input_splitter, self.shell.input_transformer_manager):
            s.physical_line_transforms.extend([sage_prompt_transformer()])
            s.logical_line_transforms.insert(0, magic_transformer())
            s.python_line_transforms.extend([SagePreparseTransformer()])
        preparser(True)

# from http://stackoverflow.com/questions/4103773/efficient-way-of-having-a-function-only-execute-once-in-a-loop
from functools import wraps
def run_once(f):
    """Runs a function (successfully) only once.

    The running can be reset by setting the `has_run` attribute to False
    """
    @wraps(f)
    def wrapper(*args, **kwargs):
        if not wrapper.has_run:
            result = f(*args, **kwargs)
            wrapper.has_run = True
            return result
    wrapper.has_run = False
    return wrapper

@run_once
def load_ipython_extension(ip):
    """Load the extension in IPython."""
    # this modifies ip
    SageCustomizations(shell=ip)