#!/usr/bin/env python3

#courses/udacity_intro_to_tensorflow_for_deep_learning/l08c01_common_patterns.ipynb

import os

import matplotlib
matplotlib.use('TkAgg')
import numpy as np
import matplotlib.pyplot as plt

import tensorflow as tf
keras = tf.keras

from savefit import *

def plot_series(time, series, format="-", start=0, end=None, label=None):
    plt.plot(time[start:end], series[start:end], format, label=label)
    plt.xlabel("Time")
    plt.ylabel("Value")
    if label:
        plt.legend(fontsize=14)
    plt.grid(True)

## Trend and Seasonality
def trend(time, slope=0):
    return slope * time

# Let's create a time series that just trends upward:

time = np.arange(4 * 365 + 1)
baseline = 10
series = baseline + trend(time, 0.1)

plt.figure(figsize=(10, 6))
plot_series(time, series)
plt.show()

#time
#series

#Now let's generate a time series with a seasonal pattern:

def seasonal_pattern(season_time):
    """Just an arbitrary pattern, you can change it if you wish"""
    return np.where(season_time < 0.4,
                    np.cos(season_time * 2 * np.pi),
                    1 / np.exp(3 * season_time))

def seasonality(time, period, amplitude=1, phase=0):
    """Repeats the same pattern at each period"""
    season_time = ((time + phase) % period) / period
    return amplitude * seasonal_pattern(season_time)

amplitude = 40
series = seasonality(time, period=365, amplitude=amplitude)

plt.figure(figsize=(10, 6))
plot_series(time, series)
plt.show()

# Now let's create a time series with both trend and seasonality:

slope = 0.05
series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)

plt.figure(figsize=(10, 6))
plot_series(time, series)
plt.show()

## Noise
#
# In practice few real-life time series have such a smooth signal. They usually
# have some noise, and the signal-to-noise ratio can sometimes be very low. Let's
# generate some white noise:

def white_noise(time, noise_level=1, seed=None):
    rnd = np.random.RandomState(seed)
    return rnd.randn(len(time)) * noise_level

noise_level = 5
noise = white_noise(time, noise_level, seed=42)

plt.figure(figsize=(10, 6))
plot_series(time, noise)
plt.show()

# Now let's add this white noise to the time series:

series += noise

plt.figure(figsize=(10, 6))
plot_series(time, series)
plt.show()

split_time = 1000
time_train = time[:split_time]
x_train = series[:split_time]
time_valid = time[split_time:]
x_valid = series[split_time:]

#courses/udacity_intro_to_tensorflow_for_deep_learning/l08c02_naive_forecasting.ipynb

# All right, this looks realistic enough for now. Let's try to forecast it. We
# will split it into two periods: the training period and the validation period
# (in many cases, you would also want to have a test period). The split will be
# at time step 1000.

## Naive Forecast
naive_forecast = series[split_time - 1:-1]

plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid, label="Series")
plot_series(time_valid, naive_forecast, label="Forecast")
plt.show()

# Let's zoom in on the start of the validation period:

plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid, start=0, end=150, label="Series")
plot_series(time_valid, naive_forecast, start=1, end=151, label="Forecast")
plt.show()

#You can see that the naive forecast lags 1 step behind the time series.

# Now let's compute the mean absolute error between the forecasts and the
# predictions in the validation period:

errors = naive_forecast - x_valid
abs_errors = np.abs(errors)
mae = abs_errors.mean()
mae

# That's our baseline, now let's try a moving average.

#courses/udacity_intro_to_tensorflow_for_deep_learning/l08c03_moving_average.ipynb

# Now let's compute the mean absolute error between the forecasts and the
# predictions in the validation period:

keras.metrics.mean_absolute_error(x_valid, naive_forecast).numpy()

# That's our baseline, now let's try a moving average.

## Moving Average

def moving_average_forecast(series, window_size):
  """Forecasts the mean of the last few values.
     If window_size=1, then this is equivalent to naive forecast"""
  forecast = []
  for time in range(len(series) - window_size):
    forecast.append(series[time:time + window_size].mean())
  return np.array(forecast)

def moving_average_forecast(series, window_size):
  """Forecasts the mean of the last few values.
     If window_size=1, then this is equivalent to naive forecast
     This implementation is *much* faster than the previous one"""
  mov = np.cumsum(series)
  mov[window_size:] = mov[window_size:] - mov[:-window_size]
  return mov[window_size - 1:-1] / window_size

moving_avg = moving_average_forecast(series, 30)[split_time - 30:]

plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid, label="Series")
plot_series(time_valid, moving_avg, label="Moving average (30 days)")
plt.show()

keras.metrics.mean_absolute_error(x_valid, moving_avg).numpy()

# That's worse than naive forecast! The moving average does not anticipate trend
# or seasonality, so let's try to remove them by using differencing. Since the
# seasonality period is 365 days, we will subtract the value at time *t* – 365
# from the value at time *t*.

diff_series = (series[365:] - series[:-365])
diff_time = time[365:]

plt.figure(figsize=(10, 6))
plot_series(diff_time, diff_series, label="Series(t) – Series(t–365)")
plt.show()

#Focusing on the validation period:

plt.figure(figsize=(10, 6))
plot_series(time_valid, diff_series[split_time - 365:], label="Series(t) – Series(t–365)")
plt.show()

# Great, the trend and seasonality seem to be gone, so now we can use the moving average:

diff_moving_avg = moving_average_forecast(diff_series, 50)[split_time - 365 - 50:]

plt.figure(figsize=(10, 6))
plot_series(time_valid, diff_series[split_time - 365:], label="Series(t) – Series(t–365)")
plot_series(time_valid, diff_moving_avg, label="Moving Average of Diff")
plt.show()

# Now let's bring back the trend and seasonality by adding the past values from t – 365:

diff_moving_avg_plus_past = series[split_time - 365:-365] + diff_moving_avg

plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid, label="Series")
plot_series(time_valid, diff_moving_avg_plus_past, label="Forecasts")
plt.show()

keras.metrics.mean_absolute_error(x_valid, diff_moving_avg_plus_past).numpy()

# Better than naive forecast, good. However the forecasts look a bit too random,
# because we're just adding past values, which were noisy. Let's use a moving
# averaging on past values to remove some of the noise:

diff_moving_avg_plus_smooth_past = moving_average_forecast(series[split_time - 370:-359], 11) + diff_moving_avg

plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid, label="Series")
plot_series(time_valid, diff_moving_avg_plus_smooth_past, label="Forecasts")
plt.show()

keras.metrics.mean_absolute_error(x_valid, diff_moving_avg_plus_smooth_past).numpy()

# That's starting to look pretty good! Let's see if we can do better with a Machine Learning model.

# courses/udacity_intro_to_tensorflow_for_deep_learning/l08c04_time_windows.ipynb

## Setup

## Time Windows

# First, we will train a model to forecast the next step given the previous 20
# steps, therefore, we need to create a dataset of 20-step windows for training.

dataset = tf.data.Dataset.range(10)
for val in dataset:
    print(val.numpy())

dataset = tf.data.Dataset.range(10)
dataset = dataset.window(5, shift=1)
for window_dataset in dataset:
    for val in window_dataset:
        print(val.numpy(), end=" ")
    print()

dataset = tf.data.Dataset.range(10)
dataset = dataset.window(5, shift=1, drop_remainder=True)
for window_dataset in dataset:
    for val in window_dataset:
        print(val.numpy(), end=" ")
    print()

dataset = tf.data.Dataset.range(10)
dataset = dataset.window(5, shift=1, drop_remainder=True)
dataset = dataset.flat_map(asis(lambda window: window.batch(5)))
for window in dataset:
    print(window.numpy())

dataset = tf.data.Dataset.range(10)
dataset = dataset.window(5, shift=1, drop_remainder=True)
dataset = dataset.flat_map(asis(lambda window: window.batch(5)))
dataset = dataset.map(asis(lambda window: (window[:-1], window[-1:])))
for x, y in dataset:
    print(x.numpy(), y.numpy())

dataset = tf.data.Dataset.range(10)
dataset = dataset.window(5, shift=1, drop_remainder=True)
dataset = dataset.flat_map(asis(lambda window: window.batch(5)))
dataset = dataset.map(asis(lambda window: (window[:-1], window[-1:])))
dataset = dataset.shuffle(buffer_size=10)
for x, y in dataset:
    print(x.numpy(), y.numpy())

dataset = tf.data.Dataset.range(10)
dataset = dataset.window(5, shift=1, drop_remainder=True)
dataset = dataset.flat_map(asis(lambda window: window.batch(5)))
dataset = dataset.map(asis(lambda window: (window[:-1], window[-1:])))
dataset = dataset.shuffle(buffer_size=10)
dataset = dataset.batch(2).prefetch(1)
for x, y in dataset:
    print("x =", x.numpy())
    print("y =", y.numpy())

def window_dataset(series, window_size, batch_size=32,
                   shuffle_buffer=1000):
    dataset = tf.data.Dataset.from_tensor_slices(series)
    dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)
    dataset = dataset.flat_map(asis(lambda window: window.batch(window_size + 1)))
    dataset = dataset.shuffle(shuffle_buffer)
    dataset = dataset.map(asis(lambda window: (window[:-1], window[-1])))
    dataset = dataset.batch(batch_size).prefetch(1)
    return dataset

# Forecasting with machine learning
# courses/udacity_intro_to_tensorflow_for_deep_learning/l08c05_forecasting_with_machine_learning.ipynb

## Forecasting with Machine Learning
# First, we will train a model to forecast the next step given the previous 30 steps, therefore, we need to create a dataset of 30-step windows for training.

### Linear Model

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = window_dataset(x_train, window_size)
valid_set = window_dataset(x_valid, window_size)

model = keras.models.Sequential([
  keras.layers.Dense(1, input_shape=[window_size])
])
optimizer = keras.optimizers.SGD(lr=1e-5, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
model,history = savefit(model, train_set, epochs=100, validation_data=valid_set, verbose=0)

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = window_dataset(x_train, window_size)

model = keras.models.Sequential([
  keras.layers.Dense(1, input_shape=[window_size])
])

lr_schedule = keras.callbacks.LearningRateScheduler(
    lambda epoch: 1e-6 * 10**(epoch / 30))
optimizer = keras.optimizers.SGD(lr=1e-6, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
model,history = savefit(model, train_set, epochs=100, callbacks=[lr_schedule],verbose=0)

if history:
  plt.semilogx(history.history["lr"], history.history["loss"])
  plt.axis([1e-6, 1e-3, 0, 20])

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = window_dataset(x_train, window_size)
valid_set = window_dataset(x_valid, window_size)

model = keras.models.Sequential([
  keras.layers.Dense(1, input_shape=[window_size])
])
optimizer = keras.optimizers.SGD(lr=1e-5, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
early_stopping = keras.callbacks.EarlyStopping(patience=10)
model,history = savefit(model, train_set, epochs=500,
          validation_data=valid_set,
          callbacks=[early_stopping],verbose=0)

def model_forecast(model, series, window_size):
    ds = tf.data.Dataset.from_tensor_slices(series)
    ds = ds.window(window_size, shift=1, drop_remainder=True)
    ds = ds.flat_map(asis(lambda w: w.batch(window_size)))
    ds = ds.batch(32).prefetch(1)
    forecast = model.predict(ds)
    return forecast

lin_forecast = model_forecast(model
                              , series[split_time - window_size:-1]
                              , window_size)[:, 0]

lin_forecast.shape

plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid)
plot_series(time_valid, lin_forecast)
plt.show()

keras.metrics.mean_absolute_error(x_valid, lin_forecast).numpy()

### Dense Model Forecasting

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = window_dataset(x_train, window_size)

model = keras.models.Sequential([
  keras.layers.Dense(10, activation="relu", input_shape=[window_size]),
  keras.layers.Dense(10, activation="relu"),
  keras.layers.Dense(1)
])

lr_schedule = keras.callbacks.LearningRateScheduler(
    lambda epoch: 1e-7 * 10**(epoch / 20))
optimizer = keras.optimizers.SGD(lr=1e-7, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
model,history = savefit(model, train_set, epochs=100, callbacks=[lr_schedule],verbose=0)

if history:
  plt.semilogx(history.history["lr"], history.history["loss"])
  plt.axis([1e-7, 5e-3, 0, 30])
  plt.show()

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = window_dataset(x_train, window_size)
valid_set = window_dataset(x_valid, window_size)

model = keras.models.Sequential([
  keras.layers.Dense(10, activation="relu", input_shape=[window_size]),
  keras.layers.Dense(10, activation="relu"),
  keras.layers.Dense(1)
])

optimizer = keras.optimizers.SGD(lr=1e-5, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
early_stopping = keras.callbacks.EarlyStopping(patience=10)
model,history = savefit(model, train_set, epochs=500,
          validation_data=valid_set,
          callbacks=[early_stopping],verbose=0)

dense_forecast = model_forecast(
    model,
    series[split_time - window_size:-1],
    window_size)[:, 0]

plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid)
plot_series(time_valid, dense_forecast)
plt.show()

keras.metrics.mean_absolute_error(x_valid, dense_forecast).numpy()

# Forecasting with an RNN
# courses/udacity_intro_to_tensorflow_for_deep_learning/l08c06_forecasting_with_rnn.ipynb

## Simple RNN Forecasting

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = window_dataset(x_train, window_size, batch_size=128)

model = keras.models.Sequential([
  keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=-1),
                      input_shape=[None]),
  keras.layers.SimpleRNN(100, return_sequences=True),
  keras.layers.SimpleRNN(100),
  keras.layers.Dense(1),
  keras.layers.Lambda(lambda x: x * 200.0)
])
lr_schedule = keras.callbacks.LearningRateScheduler(
    lambda epoch: 1e-7 * 10**(epoch / 20))
optimizer = keras.optimizers.SGD(lr=1e-7, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
model,history = savefit(model, train_set, epochs=100, callbacks=[lr_schedule],verbose=0)

if history:
  plt.semilogx(history.history["lr"], history.history["loss"])
  plt.axis([1e-7, 1e-4, 0, 30])

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = window_dataset(x_train, window_size, batch_size=128)
valid_set = window_dataset(x_valid, window_size, batch_size=128)

model = keras.models.Sequential([
  keras.layers.Lambda(lambda x: tf.expand_dims(x, axis=-1),
                      input_shape=[None]),
  keras.layers.SimpleRNN(100, return_sequences=True),
  keras.layers.SimpleRNN(100),
  keras.layers.Dense(1),
  keras.layers.Lambda(lambda x: x * 200.0)
])
optimizer = keras.optimizers.SGD(lr=1.5e-6, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
early_stopping = keras.callbacks.EarlyStopping(patience=50)
model_checkpoint = keras.callbacks.ModelCheckpoint(
    "my_checkpoint", save_best_only=True)
model,history = savefit(model,train_set, epochs=500,
          validation_data=valid_set,
          callbacks=[early_stopping, model_checkpoint],verbose=0)

model = keras.models.load_model("my_checkpoint")

rnn_forecast = model_forecast(
    model,
    series[split_time - window_size:-1],
    window_size)[:, 0]

plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid)
plot_series(time_valid, rnn_forecast)
plt.show()

keras.metrics.mean_absolute_error(x_valid, rnn_forecast).numpy()

## Sequence-to-Sequence Forecasting

def seq2seq_window_dataset(series, window_size, batch_size=32,
                           shuffle_buffer=1000):
    series = tf.expand_dims(series, axis=-1)
    ds = tf.data.Dataset.from_tensor_slices(series)
    ds = ds.window(window_size + 1, shift=1, drop_remainder=True)
    ds = ds.flat_map(asis(lambda w: w.batch(window_size + 1)))
    ds = ds.shuffle(shuffle_buffer)
    ds = ds.map(asis(lambda w: (w[:-1], w[1:])))
    return ds.batch(batch_size).prefetch(1)

for X_batch, Y_batch in seq2seq_window_dataset(tf.range(10), 3,
                                               batch_size=1):
    print("X:", X_batch.numpy())
    print("Y:", Y_batch.numpy())

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = seq2seq_window_dataset(x_train, window_size,
                                   batch_size=128)

model = keras.models.Sequential([
  keras.layers.SimpleRNN(100, return_sequences=True,
                         input_shape=[None, 1]),
  keras.layers.SimpleRNN(100, return_sequences=True),
  keras.layers.Dense(1),
  keras.layers.Lambda(lambda x: x * 200)
])
lr_schedule = keras.callbacks.LearningRateScheduler(
    lambda epoch: 1e-7 * 10**(epoch / 30))
optimizer = keras.optimizers.SGD(lr=1e-7, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
model,history = savefit(model,train_set, epochs=100, callbacks=[lr_schedule],verbose=0)

if history:
  plt.semilogx(history.history["lr"], history.history["loss"])
  plt.axis([1e-7, 1e-4, 0, 30])

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = seq2seq_window_dataset(x_train, window_size,
                                   batch_size=128)
valid_set = seq2seq_window_dataset(x_valid, window_size,
                                   batch_size=128)

model = keras.models.Sequential([
  keras.layers.SimpleRNN(100, return_sequences=True,
                         input_shape=[None, 1]),
  keras.layers.SimpleRNN(100, return_sequences=True),
  keras.layers.Dense(1),
  keras.layers.Lambda(lambda x: x * 200.0)
])
optimizer = keras.optimizers.SGD(lr=1e-6, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
early_stopping = keras.callbacks.EarlyStopping(patience=10)
model,history = savefit(model, train_set, epochs=500,
          validation_data=valid_set,
          callbacks=[early_stopping],verbose=0)

rnn_forecast = model_forecast(model, series[..., np.newaxis], window_size)
rnn_forecast = rnn_forecast[split_time - window_size:-1, -1, 0]

plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid)
plot_series(time_valid, rnn_forecast)
plt.show()

keras.metrics.mean_absolute_error(x_valid, rnn_forecast).numpy()

#courses/udacity_intro_to_tensorflow_for_deep_learning/l08c07_forecasting_with_stateful_rnn.ipynb

def sequential_window_dataset(series, window_size):
    series = tf.expand_dims(series, axis=-1)
    ds = tf.data.Dataset.from_tensor_slices(series)
    ds = ds.window(window_size + 1, shift=window_size, drop_remainder=True)
    ds = ds.flat_map(asis(lambda window: window.batch(window_size + 1)))
    ds = ds.map(asis(lambda window: (window[:-1], window[1:])))
    return ds.batch(1).prefetch(1)

for X_batch, y_batch in sequential_window_dataset(tf.range(10), 3):
    print(X_batch.numpy(), y_batch.numpy())

class ResetStatesCallback(keras.callbacks.Callback):
    def on_epoch_begin(self, epoch, logs):
        self.model.reset_states()

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = sequential_window_dataset(x_train, window_size)

model = keras.models.Sequential([
  keras.layers.SimpleRNN(100, return_sequences=True, stateful=True,
                         batch_input_shape=[1, None, 1]),
  keras.layers.SimpleRNN(100, return_sequences=True, stateful=True),
  keras.layers.Dense(1),
  keras.layers.Lambda(lambda x: x * 200.0)
])
lr_schedule = keras.callbacks.LearningRateScheduler(
    lambda epoch: 1e-8 * 10**(epoch / 30))
reset_states = ResetStatesCallback()
optimizer = keras.optimizers.SGD(lr=1e-8, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
model,history = savefit(model, train_set, epochs=100,
                    callbacks=[lr_schedule, reset_states], verbose=0)

if history:
  plt.semilogx(history.history["lr"], history.history["loss"])
  plt.axis([1e-8, 1e-4, 0, 30])

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = sequential_window_dataset(x_train, window_size)
valid_set = sequential_window_dataset(x_valid, window_size)

model = keras.models.Sequential([
  keras.layers.SimpleRNN(100, return_sequences=True, stateful=True,
                         batch_input_shape=[1, None, 1]),
  keras.layers.SimpleRNN(100, return_sequences=True, stateful=True),
  keras.layers.Dense(1),
  keras.layers.Lambda(lambda x: x * 200.0)
])
optimizer = keras.optimizers.SGD(lr=1e-7, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
reset_states = ResetStatesCallback()
model_checkpoint = keras.callbacks.ModelCheckpoint(
    "my_checkpoint.h5", save_best_only=True)
early_stopping = keras.callbacks.EarlyStopping(patience=50)
model,history = savefit(model, train_set, epochs=500,
          validation_data=valid_set,
          callbacks=[early_stopping, model_checkpoint, reset_states], verbose=0)

model = keras.models.load_model("my_checkpoint.h5")

model.reset_states()
rnn_forecast = model.predict(series[np.newaxis, :, np.newaxis])
rnn_forecast = rnn_forecast[0, split_time - 1:-1, 0]

rnn_forecast.shape

plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid)
plot_series(time_valid, rnn_forecast)
plt.show()

keras.metrics.mean_absolute_error(x_valid, rnn_forecast).numpy()

# courses/udacity_intro_to_tensorflow_for_deep_learning/l08c08_forecasting_with_lstm.ipynb
## LSTM RNN Forecasting

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = sequential_window_dataset(x_train, window_size)

model = keras.models.Sequential([
  keras.layers.LSTM(100, return_sequences=True, stateful=True,
                    batch_input_shape=[1, None, 1]),
  keras.layers.LSTM(100, return_sequences=True, stateful=True),
  keras.layers.Dense(1),
  keras.layers.Lambda(lambda x: x * 200.0)
])
lr_schedule = keras.callbacks.LearningRateScheduler(
    lambda epoch: 1e-8 * 10**(epoch / 20))
reset_states = ResetStatesCallback()
optimizer = keras.optimizers.SGD(lr=1e-8, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
model,history = savefit(model, train_set, epochs=100,
                    callbacks=[lr_schedule, reset_states],verbose=0)

if history:
  plt.semilogx(history.history["lr"], history.history["loss"])
  plt.axis([1e-8, 1e-4, 0, 30])

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = sequential_window_dataset(x_train, window_size)
valid_set = sequential_window_dataset(x_valid, window_size)

model = keras.models.Sequential([
  keras.layers.LSTM(100, return_sequences=True, stateful=True,
                         batch_input_shape=[1, None, 1]),
  keras.layers.LSTM(100, return_sequences=True, stateful=True),
  keras.layers.Dense(1),
  keras.layers.Lambda(lambda x: x * 200.0)
])
optimizer = keras.optimizers.SGD(lr=5e-7, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
reset_states = ResetStatesCallback()
model_checkpoint = keras.callbacks.ModelCheckpoint(
    "my_checkpoint.h5", save_best_only=True)
early_stopping = keras.callbacks.EarlyStopping(patience=50)
model,history = savefit(model, train_set, epochs=500,
          validation_data=valid_set,
          callbacks=[early_stopping, model_checkpoint, reset_states],verbose=0)

model = keras.models.load_model("my_checkpoint.h5")

rnn_forecast = model.predict(series[np.newaxis, :, np.newaxis])
rnn_forecast = rnn_forecast[0, split_time - 1:-1, 0]

plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid)
plot_series(time_valid, rnn_forecast)
plt.show()

keras.metrics.mean_absolute_error(x_valid, rnn_forecast).numpy()

# courses/udacity_intro_to_tensorflow_for_deep_learning/l08c09_forecasting_with_cnn.ipynb

## Preprocessing With 1D-Convolutional Layers

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = seq2seq_window_dataset(x_train, window_size,
                                   batch_size=128)

model = keras.models.Sequential([
  keras.layers.Conv1D(filters=32, kernel_size=5,
                      strides=1, padding="causal",
                      activation="relu",
                      input_shape=[None, 1]),
  keras.layers.LSTM(32, return_sequences=True),
  keras.layers.LSTM(32, return_sequences=True),
  keras.layers.Dense(1),
  keras.layers.Lambda(lambda x: x * 200)
])
lr_schedule = keras.callbacks.LearningRateScheduler(
    lambda epoch: 1e-8 * 10**(epoch / 20))
optimizer = keras.optimizers.SGD(lr=1e-8, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
model,history = savefit(model, train_set, epochs=100, callbacks=[lr_schedule],verbose=0)

if history:
  plt.semilogx(history.history["lr"], history.history["loss"])
  plt.axis([1e-8, 1e-4, 0, 30])

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = seq2seq_window_dataset(x_train, window_size,
                                   batch_size=128)
valid_set = seq2seq_window_dataset(x_valid, window_size,
                                   batch_size=128)

model = keras.models.Sequential([
  keras.layers.Conv1D(filters=32, kernel_size=5,
                      strides=1, padding="causal",
                      activation="relu",
                      input_shape=[None, 1]),
  keras.layers.LSTM(32, return_sequences=True),
  keras.layers.LSTM(32, return_sequences=True),
  keras.layers.Dense(1),
  keras.layers.Lambda(lambda x: x * 200)
])
optimizer = keras.optimizers.SGD(lr=1e-5, momentum=0.9)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])

model_checkpoint = keras.callbacks.ModelCheckpoint(
    "my_checkpoint.h5", save_best_only=True)
early_stopping = keras.callbacks.EarlyStopping(patience=50)
model,history = savefit(model, train_set, epochs=500,
          validation_data=valid_set,
          callbacks=[early_stopping, model_checkpoint],verbose=0)

model = keras.models.load_model("my_checkpoint.h5")

rnn_forecast = model_forecast(model, series[:,  np.newaxis], window_size)
rnn_forecast = rnn_forecast[split_time - window_size:-1, -1, 0]

plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid)
plot_series(time_valid, rnn_forecast)
plt.show()

keras.metrics.mean_absolute_error(x_valid, rnn_forecast).numpy()

## Fully Convolutional Forecasting

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 64
train_set = seq2seq_window_dataset(x_train, window_size,
                                   batch_size=128)

model = keras.models.Sequential()
model.add(keras.layers.InputLayer(input_shape=[None, 1]))
for dilation_rate in (1, 2, 4, 8, 16, 32):
    model.add(
      keras.layers.Conv1D(filters=32,
                          kernel_size=2,
                          strides=1,
                          dilation_rate=dilation_rate,
                          padding="causal",
                          activation="relu")
    )
model.add(keras.layers.Conv1D(filters=1, kernel_size=1))
lr_schedule = keras.callbacks.LearningRateScheduler(
    lambda epoch: 1e-4 * 10**(epoch / 30))
optimizer = keras.optimizers.Adam(lr=1e-4)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])
model,history = savefit(model, train_set, epochs=100, callbacks=[lr_schedule],verbose=0)

if history:
  plt.semilogx(history.history["lr"], history.history["loss"])
  plt.axis([1e-4, 1e-1, 0, 30])

keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 64
train_set = seq2seq_window_dataset(x_train, window_size,
                                   batch_size=128)
valid_set = seq2seq_window_dataset(x_valid, window_size,
                                   batch_size=128)

model = keras.models.Sequential()
model.add(keras.layers.InputLayer(input_shape=[None, 1]))
for dilation_rate in (1, 2, 4, 8, 16, 32):
    model.add(
      keras.layers.Conv1D(filters=32,
                          kernel_size=2,
                          strides=1,
                          dilation_rate=dilation_rate,
                          padding="causal",
                          activation="relu")
    )
model.add(keras.layers.Conv1D(filters=1, kernel_size=1))
optimizer = keras.optimizers.Adam(lr=3e-4)
model.compile(loss=keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])

model_checkpoint = keras.callbacks.ModelCheckpoint(
    "my_checkpoint.h5", save_best_only=True)
early_stopping = keras.callbacks.EarlyStopping(patience=50)
model,history = savefit(model, train_set, epochs=500,
                    validation_data=valid_set,
                    callbacks=[early_stopping, model_checkpoint],verbose=0)

model = keras.models.load_model("my_checkpoint.h5")

cnn_forecast = model_forecast(model, series[..., np.newaxis], window_size)
cnn_forecast = cnn_forecast[split_time - window_size:-1, -1, 0]

plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid)
plot_series(time_valid, cnn_forecast)
plt.show()

keras.metrics.mean_absolute_error(x_valid, cnn_forecast).numpy()


# Databricks notebook source
# MAGIC %md
# MAGIC ScaDaMaLe Course [site](https://lamastex.github.io/scalable-data-science/sds/3/x/) and [book](https://lamastex.github.io/ScaDaMaLe/index.html)

# COMMAND ----------

# MAGIC %md
# MAGIC # Analyse the eigenvalue spectrum

# COMMAND ----------

# MAGIC %md
# MAGIC - Load the singular values computed in 03_compute_rsvd, sort them and convert to eigenvalues taking the square
# MAGIC - Plot the spectrum for each graph in a semi-log plot for comparison

# COMMAND ----------

import pyspark.sql.functions as F
import numpy as np
import pandas as pd
import seaborn as sns
%matplotlib inline
import matplotlib.pyplot as plt

# COMMAND ----------

# MAGIC %md
# MAGIC ### Function for getting sorted eigenvalues of graph Laplacian L from singular values of incidence matrix B

# COMMAND ----------

def to_eigen(singular_values):
  singular_values = singular_values.sort_values(by='value', ascending=False)
  eigen_values = np.power(singular_values, 2)
  return eigen_values

# COMMAND ----------

# MAGIC %md
# MAGIC ### Get eigenvalues of Ethereum graph

# COMMAND ----------

data_path = "/projects/group21/test_ethereum_SingularValues"
singular_values_eth = spark.read.format('parquet').load(data_path).toPandas()
eigen_values_eth = to_eigen(singular_values_eth)

# COMMAND ----------

# MAGIC %md
# MAGIC ### Get eigenvalues of Erdös-Renyi graphs

# COMMAND ----------

eigen_values_uniform = []
for i in range(10):
  data_path = "/projects/group21/uniform_random_graph_SingularValues" + str(i)
  singular_values = spark.read.format('parquet').load(data_path).toPandas()
  eigen_values_uniform.append(to_eigen(singular_values))

# COMMAND ----------

# MAGIC %md
# MAGIC ### Get eigenvalues of R-MAT graphs

# COMMAND ----------

eigen_values_rmat = []
for i in range(10):
  data_path = "/projects/group21/rmat_random_graph_SingularValues" + str(i)
  singular_values = spark.read.format('parquet').load(data_path).toPandas()
  eigen_values_rmat.append(to_eigen(singular_values))

# COMMAND ----------

# MAGIC %md
# MAGIC ### Plot sorted eigenvalues for all graphs

# COMMAND ----------

colors = sns.color_palette()
fig, ax = plt.subplots(figsize=(16, 9))
x = np.arange(len(eigen_values_eth))
ax = sns.lineplot(x=x, y=eigen_values_eth.to_numpy().ravel(), color=colors[0], label='ethereum')
for i in range(9):
  ax = sns.lineplot(x=x, y=eigen_values_uniform[i].to_numpy().ravel(), color=colors[1], alpha=0.4)
  ax = sns.lineplot(x=x, y=eigen_values_rmat[i].to_numpy().ravel(), color=colors[2], alpha=0.4)

ax = sns.lineplot(x=x, y=eigen_values_uniform[9].to_numpy().ravel(), color=colors[1], alpha=0.4, label='erdös-renyi')
ax = sns.lineplot(x=x, y=eigen_values_rmat[9].to_numpy().ravel(), color=colors[2], alpha=0.4, label='rmat')
ax.set_yscale('log')
ax.legend()

# COMMAND ----------

# MAGIC %md
# MAGIC ## Conclusion
# MAGIC
# MAGIC We observe a large descrepency in the spectrums between the Erdös-Renyi, R-MAT and Ethereum transaction graphs.
# MAGIC As can be expected, the spectrum of the Erdös-Renyi graphs is almost constant due to the isotropy of the graph topology.
# MAGIC The Ethereum transaction graph has very large eigenvalues compared to the random graphs.
# MAGIC A likely explanation is the presence of nodes of very high degree in the graph.
# MAGIC
# MAGIC We can see that the R-MAT graph lies in between uniform Erdös-Renyi and Ethereum graph.
# MAGIC This is also as expected since the R-MAT model is designed to better mimic the behaviour of real graphs.
# MAGIC In this project we used the default parameters for the R-MAT graph and it is likely that with further experimentation one could find a setting which better fit the spectum of the transaction graph.


# mutlinomial_distribution
# a generalization of binomial distribution
# outcomes of muti-nomial scenarios unlike binomial
# where scenarios must be only one of two.
# e.g. Blood type of a population, dice roll outcome

"""
It has three parameters:
n - number of possible outcomes(e.g. 6 for dice roll).
pvals - list of probabilities of outcomes(e.g. [1/6, 1/6, 1/6, 1/6, 1/6, 1/6] for dice roll)
size - The shape of the returned array
"""

# draw out a sample for dice roll

from numpy import random

x = random.multinomial(n = 6, pvals =[1/6, 1/6, 1/6, 1/6, 1/6, 1/6])

print(x)

# visualization of multinomial distributions

# from numpy import random
import matplotlib.pyplot as plt
import seaborn as sns

sns.distplot(random.multinomial(n = 6, pvals =[1/6, 1/6, 1/6, 1/6, 1/6, 1/6], size = 1000), hist = False)
plt.show()


from utils import util
import numpy as np

# 一维梯度下降
def gd(eta):
    x = 10
    results = [x]
    for i in range(x):
        x -= eta * 2 * x  # 一维目标函数f(x) = x * x
        results.append(x)
    print('epoch 10, x:', x)
    return results

# 显示一维轨迹
def show_trace(res):
    n = max(abs(min(res)), abs(max(res)), 10)
    f_line = np.arange(-n, n, 0.1)
    util.set_figsize()
    util.plt.plot(f_line, [x * x for x in f_line])  # 显示f(x)=x*x的图像
    util.plt.plot(res, [x * x for x in res], '-o')  # 显示梯度下降图像
    util.plt.xlabel('x')
    util.plt.ylabel('f(x)')
    util.plt.show()

# 使用二维梯度下降来训练
def train_2d(trainer):
    x1, x2, s1, s2 = -5, -2, 0, 0
    results = [(x1, x2)]
    for i in range(20):
        x1, x2, s1, s2 = trainer(x1, x2, s1, s2)
        results.append((x1, x2))
    print('epoch %d, x1 %f, x2 %f' % (20, x1, x2))
    return results

# 显示二维轨迹
def show_trace_2d(f, results):
    util.plt.plot(*zip(*results), '-o', color='#ff7f0e')
    x1, x2 = np.meshgrid(np.arange(-5.5, 1.0, 0.1), np.arange(-3.0, 1.0, 0.1))
    util.plt.contour(x1, x2, f(x1, x2), colors='#1f77b4')
    util.plt.xlabel('x1')
    util.plt.ylabel('x2')
    util.plt.show()

# 二维目标函数
def f_2d(x1, x2):
    return x1 ** 2 + 2 * x2 ** 2

# 二维梯度下降
def gd_2d(x1, x2, s1, s2):
    return (x1 - eta * 2 * x1, x2 - eta * 4 * x2, 0, 0)

# 二维随机梯度下降（增加随机噪声来模拟随机梯度下降）
def sgd_2d(x1, x2, s1, s2):
    return (x1 - eta * (2 * x1 + np.random.normal(0.1)),
            x2 - eta * (4 * x2 + np.random.normal(0.1)), 0, 0)

if __name__ == '__main__':
    # 一维梯度下降
    # 学习率eta=0.2, 学习率合适，在x=0.06处已经效果明显
    res = gd(0.2)
    show_trace(res)

    # 学习率eta=0.05, 学习率过小，梯度下降较慢
    res = gd(0.05)
    show_trace(res)

    # 学习率eta=1.1, 学习率过大，左右摇摆
    res = gd(1.1)
    show_trace(res)

    # 二维梯度下降
    eta = 0.1
    f = f_2d
    trainer = gd_2d
    res2 = train_2d(trainer)
    show_trace_2d(f, res2)

    # 二维随机梯度下降
    show_trace_2d(f_2d, train_2d(sgd_2d))


import numpy as np

from optimizer import Optimizer

class Sr1(Optimizer):
    """
    Quasi-Newton algorithm with Symmetric Rank 1 (SR1) update. See
        https://arxiv.org/pdf/2002.00657.pdf
    for a formal description and convergence proof of a similar method.

    The stability condition is from
        p. 145 in (J. Nocedal and S. J. Wright, "Numerical Optimization", 2nd edtion)

    Arguments:
        L (float, optional): an upper bound on the smoothness constant
            to initialize the Hessian estimate
        hess_estim (float array of shape (dim, dim)): initial Hessian estimate
        lr (float, optional): stepsize (default: 1)
        stability_const (float, optional): a constant from [0, 1) that ensures a curvature
            condition before updating the Hessian-inverse estimate (default: 1e-8)
    """

    def __init__(self, L=None, hess_estim=None, lr=1, stability_const=1e-8, *args, **kwargs):
        super(Sr1, self).__init__(*args, **kwargs)
        if L is None and hess_estim is None:
            L = self.loss.smoothness
            if L is None:
                raise ValueError("Either smoothness constant L or Hessian estimate must be provided")
        if not 0 <= stability_const < 1:
            raise ValueError("Invalid stability parameter: {}".format(stability_const))
        self.lr = lr
        self.L = L
        self.B = hess_estim
        self.stability_const = stability_const

    def step(self):
        self.grad = self.loss.gradient(self.x)
        x_new = self.x - self.lr * self.B_inv @ self.grad
        if self.line_search is not None:
            x_new = self.line_search(self.x, x_new)

        s = x_new - self.x
        grad_new = self.loss.gradient(x_new)
        y = grad_new - self.grad
        self.grad = grad_new
        Bs = self.B @ s
        sBs = s @ Bs
        B_inv_y = self.B_inv @ y
        y_B_inv_y = y @ B_inv_y
        y_s = y @ s
        if abs(y_s-sBs) > self.stability_const * self.loss.norm(s) * self.loss.norm(y-Bs) and y_s!=y_B_inv_y:
            self.B += np.outer(y-Bs, y-Bs) / (y_s-sBs)
            self.B_inv += np.outer(s-B_inv_y, s-B_inv_y) / (y_s-y_B_inv_y)
        self.x = x_new

    def init_run(self, *args, **kwargs):
        super(Sr1, self).init_run(*args, **kwargs)
        if self.B is None:
            self.B = self.L * np.eye(self.loss.dim)
            self.B_inv = 1 / self.L * np.eye(self.loss.dim)
        else:
            self.B_inv = np.linalg.pinv(self.B)
        self.grad = self.loss.gradient(self.x)


'''
### Question 1

Level 1

Question:
Write a program which will find all such numbers which are divisible by 7 but are not a multiple of 5, between 2000 and 3200 (both included).
The numbers obtained should be printed in a comma-separated sequence on a single line.

Hints:
Consider use range(#begin, #end) method
'''

numbersList = []
for num in range(2000,3201):
    if num % 7 == 0 and num % 5 != 0:
            numbersList.append(num)
print(numbersList)

'''
### Question 2
Level 1

Question:
Write a program which can compute the factorial of a given numbers.
The results should be printed in a comma-separated sequence on a single line.
Suppose the following input is supplied to the program:
8
Then, the output should be:
40320

Hints:
In case of input data being supplied to the question, it should be assumed to be a console input.

'''

def factorical_calculator(inputNumber):

    if inputNumber < 2:
        return inputNumber

    return inputNumber * factorical_calculator(inputNumber-1)

print(factorical_calculator(8))

'''
### Question 3
Level 1

Question:
With a given integral number n, write a program to generate a dictionary that contains (i, i*i) such that is an integral number between 1 and n (both included). and then the program should print the dictionary.
Suppose the following input is supplied to the program:
8
Then, the output should be:
{1: 1, 2: 4, 3: 9, 4: 16, 5: 25, 6: 36, 7: 49, 8: 64}

Hints:
In case of input data being supplied to the question, it should be assumed to be a console input.
Consider use dict()
'''

def create_dictionary(inputNumber):

    newDictionary = {}

    for num in range(1,inputNumber+1):
        newDictionary[str(num)] = str(num*num)

    return newDictionary

print(create_dictionary(8))

'''
### Question 4
Level 1

Question:
Write a program which accepts a sequence of comma-separated numbers from console and generate a list and a tuple which contains every number.
Suppose the following input is supplied to the program:
34,67,55,33,12,98
Then, the output should be:
['34', '67', '55', '33', '12', '98']
('34', '67', '55', '33', '12', '98')

Hints:
In case of input data being supplied to the question, it should be assumed to be a console input.
tuple() method can convert list to tuple
'''

userInput = input('Please input a sequence of comma-separated numbers')
#userInput = '34,67,55,33,12,98'
splitedNumbersList = userInput.split(',')
splitedNumbersTuple = tuple(splitedNumbersList)
print(splitedNumbersList,splitedNumbersTuple)

'''
### Question 6
Level 2

Question:
Write a program that calculates and prints the value according to the given formula:
Q = Square root of [(2 * C * D)/H]
Following are the fixed values of C and H:
C is 50. H is 30.
D is the variable whose values should be input to your program in a comma-separated sequence.
Example
Let us assume the following comma separated input sequence is given to the program:
100,150,180
The output of the program should be:
18,22,24

Hints:
If the output received is in decimal form, it should be rounded off to its nearest value (for example, if the output received is 26.0, it should be printed as 26)
In case of input data being supplied to the question, it should be assumed to be a console input.
'''
import math
userInput = input('Please input a sequence of comma-separated numbers')
#userInput = '100,150,180'
splitedNumbers = userInput.split(',')
calculatedValues = [int(math.sqrt((2 * 50 * int(num))/30)) for num in splitedNumbers]

print(calculatedValues)

'''
### Question 7
Level 2

Question:
Write a program which takes 2 digits, X,Y as input and generates a 2-dimensional array. The element value in the i-th row and j-th column of the array should be i*j.
Note: i=0,1.., X-1; j=0,1,¡Y-1.
Example
Suppose the following inputs are given to the program:
3,5
Then, the output of the program should be:
[[0, 0, 0, 0, 0], [0, 1, 2, 3, 4], [0, 2, 4, 6, 8]]

Hints:
Note: In case of input data being supplied to the question, it should be assumed to be a console input in a comma-separated form.
'''

import numpy
def create_array(x,y):

    newArray = numpy.zeros([x,y])

    for i in range(0,x):
        for j in range(0,y):
            newArray[i,j] = int(i*j)

    return newArray

print(create_array(3,5))

'''
### Question 8
Level 2

Question:
Write a program that accepts a comma separated sequence of words as input and prints the words in a comma-separated sequence after sorting them alphabetically.
Suppose the following input is supplied to the program:
without,hello,bag,world
Then, the output should be:
bag,hello,without,world

Hints:
In case of input data being supplied to the question, it should be assumed to be a console input.
'''

def arrange_words(wordsList):

    seperatedWords = wordsList.split(',')
    return sorted(seperatedWords))

print(arrange_words(input('Please input a sequence of comma-separated numbers')))

'''
### Question 9
Level 2

Question£º
Write a program that accepts sequence of lines as input and prints the lines after making all characters in the sentence capitalized.
Suppose the following input is supplied to the program:
Hello world
Practice makes perfect
Then, the output should be:
HELLO WORLD
PRACTICE MAKES PERFECT

Hints:
In case of input data being supplied to the question, it should be assumed to be a console input.
'''

while True:
    singleLine = input()
    print(singleLine.upper() + '\n')

'''
### Question 10
Level 2

Question:
Write a program that accepts a sequence of whitespace separated words as input and prints the words after removing all duplicate words and sorting them alphanumerically.
Suppose the following input is supplied to the program:
hello world and practice makes perfect and hello world again
Then, the output should be:
again and hello makes perfect practice world

Hints:
In case of input data being supplied to the question, it should be assumed to be a console input.
We use set container to remove duplicated data automatically and then use sorted() to sort the data.
'''

def arrange_words_again(wordsList):

    seperatedWords = wordsList.split(' ')
    return sorted(set(seperatedWords))

print(arrange_words_again('hello world and practice makes perfect and hello world again'))

'''
### Question 11
Level 2

Question:
Write a program which accepts a sequence of comma separated 4 digit binary numbers as its input and then check whether they are divisible by 5 or not. The numbers that are divisible by 5 are to be printed in a comma separated sequence.
Example:
0100,0011,1010,1001
Then the output should be:
1010
Notes: Assume the data is input by console.

Hints:
In case of input data being supplied to the question, it should be assumed to be a console input.
'''

def is_divisible_by_5(numbersList):

    divisibleNumbers = []

    for number in numbersList:
        if int(number)%5==0:
            divisibleNumbers.append(number)

    return divisibleNumbers

print(is_divisible_by_5(input('Please input a sequence of comma separated 4 digit binary numbers')))
#print(is_divisible_by_5(['0100','0011','1010','1001']))

'''
### Question 12
Level 2

Question:
Write a program, which will find all such numbers between 1000 and 3000 (both included) such that each digit of the number is an even number.
The numbers obtained should be printed in a comma-separated sequence on a single line.

Hints:
In case of input data being supplied to the question, it should be assumed to be a console input.
'''
divisibleNumbers = []

for num in range(1000,3001):
    if num%2 == 0:
        divisibleNumbers.append(num)

print(divisibleNumbers)

'''
### Question 13
Level 2

Question:
Write a program that accepts a sentence and calculate the number of letters and digits.
Suppose the following input is supplied to the program:
hello world! 123
Then, the output should be:
LETTERS 10
DIGITS 3

Hints:
In case of input data being supplied to the question, it should be assumed to be a console input.
'''
import string
def calculate_sum_of_letters_and_digits(userSentence):

    counterLetters=0
    counterDigits=0

    for char in userSentence:
        if char in string.ascii_letters:
            counterLetters+=1
        if char in string.digits:
            counterDigits+=1

    print('LETTERS ' + str(counterLetters) + '\n DIGITS ' + str(counterDigits))

calculate_sum_of_letters_and_digits(input('Please input a sentence containning letters and digits'))
#calculate_sum_of_letters_and_digits('hello world! 123')

'''
### Question 14
Level 2

Question:
Write a program that accepts a sentence and calculate the number of upper case letters and lower case letters.
Suppose the following input is supplied to the program:
Hello world!
Then, the output should be:
UPPER CASE 1
LOWER CASE 9

Hints:
In case of input data being supplied to the question, it should be assumed to be a console input.
'''

import string
def calculate_sum_of_upper_and_lower_case_letters(userSentence):

    counterUpper=0
    counterLower=0

    for char in userSentence:
        if char in string.ascii_uppercase:
            counterUpper+=1
        if char in string.ascii_lowercase:
            counterLower+=1

    print('UPPER CASE ' + str(counterUpper) + '\nLOWER CASE ' + str(counterLower))

calculate_sum_of_upper_and_lower_case_letters(input('Please input a sentence containning letters only'))
#calculate_sum_of_upper_and_lower_case_letters('Hello world!')

'''
### Question 17
Level 2

Question:
Write a program that computes the net amount of a bank account based a transaction log from console input. The transaction log format is shown as following:
D 100
W 200

D means deposit while W means withdrawal.
Suppose the following input is supplied to the program:
D 300
D 300
W 200
D 100
Then, the output should be:
500

Hints:
In case of input data being supplied to the question, it should be assumed to be a console input.
'''
import string
def calculate_bank_status(userSentence):

    finalSum = 0

    x = 'D 300\nD 300\nW 200\nD 100'
    y = x.splitlines()

    for singleLine in y:

        z = singleLine.split(' ')

        if z[0] == 'D':
            finalSum += int(z[1])

        if z[0] == 'W':
            finalSum -= int(z[1])

    return finalSum

calculate_bank_status(input('Please input bank actions'))
#print(calculate_bank_status('D 300\nD 300\nW 200\nD 100'))

'''
### Question 18
Level 3

Question:
A website requires the users to input username and password to register. Write a program to check the validity of password input by users.
Following are the criteria for checking the password:
1. At least 1 letter between [a-z]
2. At least 1 number between [0-9]
1. At least 1 letter between [A-Z]
3. At least 1 character from [$#@]
4. Minimum length of transaction password: 6
5. Maximum length of transaction password: 12
Your program should accept a sequence of comma separated passwords and will check them according to the above criteria. Passwords that match the criteria are to be printed, each separated by a comma.
Example
If the following passwords are given as input to the program:
ABd1234@1,a F1#,2w3E*,2We3345
Then, the output of the program should be:
ABd1234@1

Hints:
In case of input data being supplied to the question, it should be assumed to be a console input.
'''

import re
def check_password(inputPassword)

    charConditions = re.compile(r'[^a-zA-Z0-9\$\#\@]')
    if len(inputPassword) > 12 or len(inputPassword) < 6 or not bool(charConditions.search(inputPassword))
        return

    return inputPassword

userNameInput = input('Please input user name\n'))
passwordInput = input('Please input password\n'))
check_password(passwordInput)
#check_password('ABd1234@1')

'''
### Question 20
Level 3

Question:
Define a class with a generator which can iterate the numbers, which are divisible by 7, between a given range 0 and n.

Hints:
Consider use yield
'''

class Iterator:
  def __init__(self, rangeValue):

      self.divisibleSeven = []

      for num in range (0,rangeValue+1):
          if num % 7 == 0 :
              self.divisibleSeven.append(num)

newIterator = Iterator(int(input('Please input number\n')))
print(newIterator.divisibleSeven)


'''
Arithmetic with numpy arrays
-----------------------------------

Arrays are important because they enable you to express batch operations on data
without writing any for loops. This is also called as vectorization. Any arithmetic
operations between equal-size arrays applies the operation element-wise
'''
import numpy as np

arr = np.array([[1.2,3.4, 5.6],[4,5,6]])
print(arr)

print(arr * arr)
print(1/arr)
print(arr ** 0.5)
print(arr - arr)

arr2 = np.array([[1.22,3.24, 15.6],[14,5,6]])

print(arr2 > arr) # comparing elementwise of similar sized arrays yeilds boolean array


#!/usr/bin/python3
# -*- coding: utf-8 -*-

import numpy as np

a = np.array([
    [1, 1, 1],
    [2, 2, 2],
    [3, 3, 3]
    ])

a1 = [
    [1, 1, 1],
    [2, 2, 2],
    [3, 3, 3]
    ]

b = np.array([
    [1, 0, 1],
    [2, 0, 2],
    [3, 0, 3]
    ])

b1 = [
    [1, 0, 1],
    [2, 0, 2],
    [3, 0, 3]
    ]

# 转变成矩阵
c = np.mat(a)
d = np.mat(b)
f = np.mat(a1)
g = np.mat(b1)

print("a = ", a, "\na1 = ", a1)
print("b = ", b, "\nb1 = ", b1)
print("a*2 = ", a*2, "\na1*2 = ", a1*2) # 这个结果很特别， a1*2元素多了一倍，元素值没有变化
print("a+b = ", a + b)
print("a1+b1 = ", a1 + b1) # 两个数组集合的合并
print("a*b = ", a * b) # 数组相乘， not 矩阵相乘
#  print(a1 * b1) # 错误

print("c = ", c)
print("d = ", d)
print("c*d = ", c * d) # 矩阵相乘
print("f = ", f)
print("g = ", g)
print("f*g = ", f * g)


# -*- coding: utf-8 -*-
"""
Coursework 1: Linear regression
"""

import numpy as np
from pandas.io.parsers import read_csv
from matplotlib import cm
import matplotlib.pyplot as plt

# Download data
def carga_csv(file_name):
    valores=read_csv(file_name,header=None).values
    return valores.astype(float)

# Normal equation for exact computation of theta
def ecuacion_normal(X, Y):
    X_t = np.transpose(X)
    Aux = np.dot(X_t,X)
    X_plus = np.linalg.pinv(Aux).dot(X_t)
    return X_plus.dot(Y)

# Normalize dataset
def normalizar(X):
    mu = np.mean(X,  axis=0)
    sigma = np.std(X, axis=0)
    X_norm = (X - mu) / sigma
    return X_norm, mu, sigma

# Compute the gradient of the cost function
def gradiente(X, Y, theta):
    res = np.zeros((np.shape(X)[1],1))
    m = np.shape(X)[0]
    H = np.dot(X,theta)
    res = 1/m * np.dot(np.transpose(X), H-Y)
    return res

# Compute the cost function
def coste(X, Y, theta):
    H = np.dot(X, theta)
    Aux = (H - Y)**2
    return Aux.sum() / (2*len(X))

# Compute gradient descent for a maximum of maxIter iterations or until
# the norm of the gradient is lower than eps
def descenso_gradiente(X, Y, alpha, maxIter, eps):
    theta = np.zeros((np.shape(X)[1],1))
    grad = gradiente(X, Y, theta)
    it = 0
    costes = []
    while np.linalg.norm(grad) > eps and it < maxIter:
        theta = theta - alpha*grad
        grad = gradiente(X, Y, theta)
        costes.append(coste(X, Y, theta))
        it += 1
    return theta, costes

#%%
"""
One variable case
"""
datos = carga_csv('ex1data1.csv')
X = datos[:,:-1]
m = np.shape(X)[0]
Y = datos[:,-1]

# Add the column of 1s
X=np.hstack([np.ones([m,1]),X])
Y = Y[:,np.newaxis]

alpha = 0.01
maxIter = 1500
eps = 1e-3
Thetas, costes = descenso_gradiente(X, Y, alpha, maxIter, eps)

# Plot data an line
x = np.linspace(0,23,23)
plt.scatter(X[:,1],Y,s=15)
plt.plot(x, Thetas[0] + Thetas[1]*x, c='r')
plt.xlabel('Población de la ciudad en 10 000s')
plt.ylabel('Ingresos en $10 000s')

#%%
"""
Plot cost function
"""
# Plot function in 3D
fig = plt.figure()
ax = fig.gca(projection = '3d')

Th0 = np.arange(-10,10,0.05)
Th1 = np.arange(-1,4,0.05)
Th0m,Th1m = np.meshgrid(Th0,Th1)
cost = np.zeros((len(Th1),len(Th0)))
for i in range(len(Th1)) :
    for j in range(len(Th0)):
        cost[i][j] = coste(X,Y, np.array([[Th0[j]],[Th1[i]]]))
curve = ax.plot_surface(Th0m,Th1m,cost, cmap = cm.rainbow)
ax.view_init(elev = 15, azim=230)
plt.xlabel(r'$\theta_{0}$')
plt.ylabel(r'$\theta_{1}$')
ax.set_zlabel(r'$J(\theta)$',rotation=0)

# Plot function as contours
plt.figure()
plt.contour(Th0m,Th1m,cost, np.logspace(-2,3,20))
plt.scatter(Thetas[0],Thetas[1], marker='X',c='r')
plt.xlabel(r'$\theta_{0}$')
plt.ylabel(r'$\theta_{1}$')

#%%
"""
Check for multivariate case different learning rates alpha
"""
datos = carga_csv('ex1data2.csv')
X_old = datos[:,:-1]
m = np.shape(X_old)[0]
Y_old=datos[:,-1]

# Normalize data
X, mu_x, sigma_x = normalizar(X_old)

# Add the column of 1s
X_old = np.hstack([np.ones([m,1]),X_old])
X = np.hstack([np.ones([m,1]),X])
Y = Y_old[:,np.newaxis]

alpha_arr = [0.3,0.1,0.03,0.01,0.003,0.001]
maxIter = 1500
eps = 1e-3
plt.figure()
for alpha in alpha_arr:
    Thetas, costes = descenso_gradiente(X, Y, alpha, maxIter, eps)
    plt.plot(costes, label = ("With alpha = " + str(alpha)))
plt.legend()
plt.xlabel('Iterations')
plt.ylabel(r'$J(\theta)$')
#%%
"""
Multivariate case
"""
datos = carga_csv('ex1data2.csv')
X_old = datos[:,:-1]
m = np.shape(X_old)[0]
Y_old=datos[:,-1]

# Normalize data
X, mu_x, sigma_x = normalizar(X_old)

# Add the column of 1s
X_old = np.hstack([np.ones([m,1]),X_old])
X = np.hstack([np.ones([m,1]),X])
Y = Y_old[:,np.newaxis]

alpha = 0.3
maxIter = 1500
eps = 1e-3
Thetas, costes = descenso_gradiente(X, Y, alpha, maxIter, eps)

# Compute with the normal equation and compare
th = ecuacion_normal(X_old,Y)
test = np.array([1650,3])
test_n = (test - mu_x)/sigma_x
test = np.hstack([1,test])
test_n = np.hstack([1,test_n])
price = np.dot(test,th)[0]
price_n = np.dot(test_n,Thetas)[0]
print("The value computed with the normal equation for x = [1650,3] is y = ", price)
print("The value computed with gradient descent for x = [1650,3] is y = ", price_n)
print("The relative distance between the two values is ", np.abs((price - price_n)/ price))


#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# author: bigfoolliu

"""
神经网络常用的激励函数(激活函数)：

如果不用激励函数那么每一层节点的输入都是上层输出的线性函数，无论层数，与没有隐藏层效果相当，
引入非线性函数作为激励函数，可以使输入逼近任意函数，神经网络的表达能力就更加强大

如何选择激活函数：
1. 深度学习往往需要大量时间来处理大量数据，因此总体上尽量选择具有zero-cntered特点的激活函数，可以加快模型的收敛速度；
2. 使用relu需要小心设置learning rate,否则易出现'dead'神经元；
3. 尽量不用sigmoid，可以试试tanh,预期效果不如relu和maxout
"""
import math
import matplotlib.pyplot as plt
import numpy as np

def sigmoid(z):
    """
    常用非线性激活函数之一
    数学形式：f(z)=1/(1+e ** (-z))
    特点：能够将输入的连续实值变换为0和1之间的输出，如果是非常大的负数，则输出为0，如果是非常大的正数则输出为1
    缺点：深度神经网络中梯度反向传播时导致梯度爆炸(概率较小)和梯度消失(概率较大);输出(output)不是0均值(0-centered);解析式中含有幂运算，
    规模大的深度网络会较大增加训练时间
    :param z:
    :return:
    """
    ret = 1 / (1 + math.e ** (-z))
    return ret

def tanh(x):
    """
    数学形式：tanh(x)=(e ** x - e ** (-x)) / (e ** x + e ** (-x))
    """
    ret = (math.e ** x - math.e ** (-x)) / (math.e ** x + math.e ** (-x))
    return ret

def relu(x):
    """
    数学形式：relu=max(0, x)
    :param x: array
    """
    ret = []
    for i in x:
        ret.append(max(0, i))
    return np.array(ret)

def leaky_relu(x, a=0.01):
    """
    数学形式：f(x)=max(ax, x), a通常为0.01
    :param x: array
    :param a: float
    """
    ret = []
    for i in x:
        ret.append(max(a * i, i))
    return np.array(ret)

def elu(x, a=0.01):
    """
    Exponential Liner Units
    数学形式：f(x)=x (x>0); a(e ** x - 1) (x<=0)
    """
    ret = []
    for i in x:
        if i > 0:
            ret.append(i)
        else:
            ret.append(a * (math.e ** i - 1))
    return np.array(ret)

def max_out():
    """
    是深度学习网络中的一层网络，就像池化层，卷积层一样
    """
    pass

# 测试用
# z = float(input("输入z:"))
# print("[INFO]sigmoid ret:{}".format(sigmoid(z)))

fig = plt.figure()
ax = plt.Axes(fig, [-10, 10, 1, 1])
z = np.linspace(-10, 10, num=500)

plt.subplot(231)
# sigmoid
sigmoid_ret = sigmoid(z)
plt.plot(z, sigmoid_ret, color="r", label="sigmoid")
# tanh
tanh_ret = tanh(z)
plt.plot(z, tanh_ret, color="g", label="tanh")
plt.xlabel("x")
plt.ylabel("y")
plt.legend()

# relu
plt.subplot(232)
relu_ret = relu(z)
plt.plot(z, relu_ret, color="b", label="relu")
plt.xlabel("x")
plt.ylabel("y")
plt.legend()

# leaky_relu
plt.subplot(233)
leaky_relu_ret = leaky_relu(z, 0.02)
plt.plot(z, leaky_relu_ret, color="r", label="leaky_relu")
plt.xlabel("x")
plt.ylabel("y")
plt.legend()

# elu
plt.subplot(234, xlabel="x", ylabel="y")
elu_ret = elu(z, 0.02)
plt.plot(z, elu_ret, color="g", label="elu")
plt.legend()
plt.show()


import os
import numpy as np
import matplotlib.pyplot as plt

f0 = 440.0

σ = 0.03
Σ = 3.0
K = 5
decr = 1

def repr_classe(f0,fmin,fmax):
    if fmin <= f0 < fmax: return f0
    elif f0 < fmin: return repr_classe(2*f0,fmin,fmax)
    elif f0 >= fmax: return repr_classe(f0/2,fmin,fmax)
f0 = repr_classe(f0,261.0,522.0)

#######################
# Spectre en gaussiennes

n = np.arange(0,16,0.001)
p0 = np.log2(261)
# f0 = 440
freq = np.exp(np.log(2)*n)

E = np.exp(-(n - p0)**2 / (2 * Σ**2))
norm = 0
S = np.zeros(np.shape(n))

for k in range(1,K+1):
    f = repr_classe(k*f0,261.0,522.0)
    p = np.log2(f)
    for i in range(-9,9):
        if 0 < p +i < 16:
            S += (1/k**decr) * np.exp(-(n - (p+i))**2 / (2 * σ**2))
    if np.log2(k).is_integer():
        norm += 1/k**decr

plot = plt.figure()
plt.xscale('log',basex = 2)
plt.plot(freq,norm*E, ls = '--')
# plt.plot(freq,S*E, label = 'Pitch-class {} Hz'.format(int(f0)))
plt.plot(freq,S*E, label = 'Pitch-class La')
plt.xlabel('Frequence (Hz)')
plt.ylabel('Amplitude')
plt.title('Spectre de Shepard' + '\nK : {}, σ : {}, decr : {}'.format(K,σ,decr))
plt.legend(frameon=True, framealpha=0.75)
plt.show()

#######################
# Spectre en pics

# f0 = 440.0
#
# Σ = 3.0
# K, decr = 5, 1
# n = np.arange(0,16,0.001)
# p0 = np.log2(261)
# E = np.exp(-(n - p0)**2 / (2 * Σ**2))
# # f0 = 440
# freq = np.exp(np.log(2)*n)
#
#
#
# def spectre_pic(f0):
#
#     dic_spectre = {}
#     def repr_classe(f0,fmin,fmax):
#         if fmin <= f0 < fmax: return f0
#         elif f0 < fmin: return repr_classe(2*f0,fmin,fmax)
#         elif f0 >= fmax: return repr_classe(f0/2,fmin,fmax)
#     f = repr_classe(f0,261.0,522.0)
#     p0 = np.log2(261)
#     Σ = 3.0
#
#     # Construction de dic_spectre
#     for k in range(1,K+1):
#         f = repr_classe(k*f0,261.0,522.0)
#         p = np.log2(f)
#         for i in range(-8,8):
#             if 0 < p + i < 16:
#                 fp = f*2**i
#                 if fp in dic_spectre:
#                     dic_spectre[fp] += (1/k**decr)* np.exp(-(np.log2(fp) - p0)**2 / (2 * Σ**2))
#                 else : dic_spectre[fp] = (1/k**decr)* np.exp(-(np.log2(fp) - p0)**2 / (2 * Σ**2))
#
#     return dic_spectre
#
# norm = 0
# for k in range(1,K+1):
#     if np.log2(k).is_integer():
#         norm += 1/k**decr
#
# dic_spectre = spectre_pic(f0)
# # print(dic_spectre)
#
# plot = plt.figure()
# plt.xscale('log',basex = 2)
# plt.plot(freq,norm*E, ls = '--')
# leg = False
# for f in dic_spectre:
#     if not leg:
#         # plt.vlines(f, 0, dic_spectre[f], alpha=0.9, label = 'Pitch-class {} Hz'.format(int(f0)))
#         plt.vlines(f, 0, dic_spectre[f], alpha=0.9, label = 'Pitch-class La')
#         leg = True
#     else: plt.vlines(f, 0, dic_spectre[f], alpha=0.9)
# # plt.plot(freq,S*E)
# plt.xlabel('Frequence (Hz)')
# plt.ylabel('Amplitude')
# plt.title('Spectre de Shepard en pics' + '\nK : {}, decr : {}'.format(K,decr))
# plt.legend(frameon=True, framealpha=0.75)
# plt.show()


'''
Distance measures (not necessarily metrics) between probability
distributions.
'''

import numpy as np

def kullback_leibler(p, q):
    '''
    Calculates the Kullback--Leibler divergence between two discrete
    probability distributions.

    :param p: First discrete probability distribution
    :param q: Second discrete probability distribution

    :return: Value of Kullback--Leibler divergence
    '''

    # TODO: this should not be necessary
    p = np.abs(p)
    q = np.abs(q)

    p += 1e-8
    q += 1e-8

    return np.sum(p * np.log(q / p))

    # FIXME: why does this not work? Even if the two conditions are
    # chained together...
    #return np.sum(np.where(p != 0, p * np.log(q / p), 0))

def jensen_shannon(p, q):
    '''
    Calculates the Jensen--Shannon divergence between two discrete
    probability distributions.

    :param p: First discrete probability distribution
    :param q: Second discrete probability distribution

    :return: Value of Jensen--Shannon divergence
    '''

    return 0.5 * (kullback_leibler(p, q) + kullback_leibler(q, p))


# 3.10 矩阵与线性代数运算

import numpy as np
m = np.matrix([[1,-2,3],[0,4,5],[7,8,-9]])
print(m)

# Return transpose
print(m.T)

# Return inverse
print(m.I)

# Create a vector and multiply
v = np.matrix([[2],[3],[4]])
print(v)
print(m * v)

import numpy.linalg

# Determinant
numpy.linalg.det(m)

# Eigenvalues
numpy.linalg.eigvals(m)

# Solve for x in mx = v
x = numpy.linalg.solve(m, v)
print(x)
print(m * x)
print(v)


# -*- coding: utf-8 -*-

import numpy as np

def compute_PCA(X, mean=True, k=0, A=None, normalize=True):
    """
    PCA w.r.t A-norm, where A is a positive definite matrix.
    X : column-wise data set with size n-by-m, i.e. m samples of n dimension
    k : number of modes required (not include mean)
    normalize : whether to normalize eigenvectors w.r.t A
    Output :
        phi : n-by-(k+1)
        w   : eigenvalues
    """

    n, m = X.shape
    phi = np.zeros((n, k + 1))
    if mean:
        phi[:, 0] = X.mean(1)
        X = X - np.tile(phi[:, 0].reshape((-1, 1)), (1, m))
        s3 = 'Nonzero mean'
    else:
        s3 = 'Zero mean'

    if A is None:
        w, v = np.linalg.eigh(X.transpose() @ X)
        s1 = 'without norm'
    else:
        w, v = np.linalg.eigh(X.transpose() @ A @ X)
        s1 = 'with norm-A'
    w = w[::-1]
    v = v[:, ::-1]
    w = np.sqrt(np.absolute(w))
    if normalize:
        phi[:, 1:] = (X @ v[:, 0:k]) / np.tile(w[0:k].reshape((1, -1)), (n, 1))
        s2 = 'normalized eigenvectors'
    else:
        phi[:, 1:] = X @ v[:, 0:k]
        s2 = 'unnormalized eigenvectors'
    # print('Done PCA : (X %d-by-%d)' % (n, m) + s2 + ' ' + s1 + '. ' + s3 + ' + %d dominant eigenvectors' % (k))
    return phi, w


import numpy as np

########################################Ex-1#####################################

A = np.array([3,4,6,10,24,89,45,43,46,99,100])

div3 = A[A%3!=0]
print("Elements of A not divisible by 3:")
print(div3)

div5 = A[A%5==0]
print("Elements of A divisible by 5:")
print(div5)

print("Elements of A, which are divisible by 3 and 5:")
print(A[(A%3==0) & (A%5==0)])
print("------------------")

#

A[A%3==0] = 42
print("""New values of A after setting the elements of A,
which are divisible by 3, to 42:""")
print(A)

#######################################################################################

#####Calculate prime number between 0 and 100 using boolean array #####################
import numpy as np
is_prime = np.ones((100,), dtype=bool)

# Cross out 0 and 1 which are not primes:
is_prime[:2] = 0

# cross out its higher multiples (sieve of Eratosthenes):
nmax = int(np.sqrt(len(is_prime)))
for i in range(2, nmax):
    is_prime[2*i::i] = False

print(np.nonzero(is_prime))

###################################################################################


import numpy as np
import sys
import matplotlib.pyplot as pl

## returns the predicted y (based on feature dataset (X) and weights (Theta))
def f(X,th):
    return X.dot(th.T)

## returns the cost (residual error) for predicted y
def J(X,y,th):
  m=X.shape[0]
  y_hat=X.dot(th.T)
  cost=np.sum((y_hat-y)**2)/(2*m)
  return cost

## finds the "good" values for the weigths (Theta)
## Gradient descent (optmization function)
def GD (X,y,th,alpha,niters):
  m=X.shape[0]
  cost=np.zeros((niters,1))
  for i in range(0,niters):
    y_hat=X.dot(th.T)
    error=((y_hat-y)*X).sum(0)/m
    th = th-(alpha*(error))
    cost[i]=J(X,y,th)
  return th,cost

def featureScaling(X):
  X=X-np.min(X,0)
  den=np.max(X,0)-np.min(X,0)
  return X/den

def RMSE(X,y,th):
	m=X.shape[0]
	y_hat=f(X,th)
	error=((y_hat-y)**2).sum()/m
	return np.sqrt(error)

if __name__ == "__main__":
    if len(sys.argv) < 1:
        print('Usage: python %s <dataset file name (full path)>' % sys.argv[0])
        exit(0)

    try:
        fin=open(sys.argv[1]) ## open the dataset file
    except:
        print('Could not open',sys.argv[1])
        exit(0)
    ## rows will be a list of list, e.g., [[example 1],[example 2],...[example m]]
    rows=[l.split(',') for l in fin.readlines()] ## the values are separated by comma
    m=len(rows) ## how many lists rows has, i.e., how many examples in the dataset
    ## how many features are in a given list
    d = len(rows[0])-1 ## we subtract the result from 1 to discard the label (y)
    y = np.array([l[d:d+1] for l in rows],dtype=float) ## vector of dataset labels
    X = np.array([l[0:d] for l in rows],dtype=float) ## matrix of dataset features
    ### Feature scaling
    Xori=X.copy() ## save the original X
    X=featureScaling(X)
    Theta=np.zeros((1,d+1)) ## Initialize Theta with zeros
    X=np.insert(X,0,1,axis=1) ## inserts a column of 1's
    tsize=int(m*0.7) ## size of the training set
    Xtr=X[:tsize,:] ## from the first row to tsize-1, all colmuns
    Xte=X[tsize:,:] ## from the tsize row to the end, all columns
    ytr=y[:tsize]
    yte=y[tsize:]
    ### Call gradient descent to find the "good" values to Theta
    Theta,cost=GD(Xtr,ytr,Theta,0.000001,2000)
    pl.plot(cost)
    pl.show()
    print('RMSE:',RMSE(Xte,yte,Theta))
    if d>1:
        print('Sorry, we cannot plot more than 1 feature')
        exit(0)
    ## predict new labels from training set
    y_hat=Xtr.dot(Theta.T)
    ## Plot the predict y's in relation to the ground-truth (y)
    ## first the X regarding the ground-truth
    pl.plot(Xtr[:,1:],ytr,'.',c='b')
    ## now X regarding the predict values
    pl.plot(Xtr[:,1:],y_hat,'-',c='g')
    pl.show()
    ## Now, let's check the performance on the test set
    ## y_hat is built from Xte, i.e., the test set
    y_hat=Xte.dot(Theta.T)
    pl.plot(Xte[:,1:],yte,'.',c='b')
    ## now X regarding the predict values
    pl.plot(Xte[:,1:],y_hat,'-',c='g')
    pl.show()
    print("Test cost", J(Xte,yte,Theta))


import numpy as np
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter
from mpl_toolkits.mplot3d import Axes3D

def matyas(x):
   return 0.26*(x[0]**2+x[1]**2)-0.48*x[0]*x[1]

#def booth(x):
#   return (x[0]+2*x[1]-7)**2+(2*x[0]+x[1]-5)**2

x = np.linspace(-10,10,100)
y = np.linspace(-10,10,100)
x, y = np.meshgrid(x, y)
z = matyas([x,y])

fig = plt.figure()
ax = fig.gca(projection='3d')
surf = ax.plot_surface(x, y, z, rstride=1, cstride=1,
                      cmap=cm.RdBu,linewidth=0, antialiased=False)

ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))

fig.colorbar(surf, shrink=0.5, aspect=10)

plt.show()

x0 = np.array([-10, 10])
NelderMeadOptimizeResults = minimize(matyas, x0, method='nelder-mead',
               options={'xatol': 1e-8, 'disp': True})

print(NelderMeadOptimizeResults.x)

x0 = np.array([-10, 10])
PowellOptimizeResults = minimize(matyas, x0, method='Powell',
               options={'xtol': 1e-8, 'disp': True})

print(PowellOptimizeResults.x)


#!/usr/bin/env python
# integeral.py
import numpy as num

def integral(x,y):
  """
ROUTINE:        INTEGRAL

USEAGE:         RESULT = INTEGRAL( X, Y )

PURPOSE:        Integrate tabulated data using Simpson's rule
                with 3-point Lagragian interpolation. Data may be
                regularly sampled in X, or irregularly sampled.

INPUT:
  X             Vector of x axis points.
                (Elements must be unique and monotonically increasing)

  Y             Vector of corresponding Y axis points.

KEYWORD_INPUT:  None.

OUTPUT:         Result of integration.

EXAMPLE:
       Example 1:
       Define 11 x-values on the closed interval [0.0 , 0.8].
       X = [ 0.0, .12, .22, .32, .36, .40, .44, .54, .64, .70, .80 ]

       Define 11 f-values corresponding to x(i).
       F = [ 0.200000, 1.30973, 1.30524, 1.74339, 2.07490, 2.45600, $
              2.84299,  3.50730, 3.18194, 2.36302, 0.231964 ]

       Compute the integral.
       RESULT = INTEGRAL( X, F )

       In this example, the f-values are generated from a known function,
       (f = .2 + 25*x - 200*x^2 + 675*x^3 - 900*x^4 + 400*x^5)

       The Multiple Application Trapezoid Method yields;  result = 1.5648
       The Multiple Application Simpson's Method yields;  result = 1.6036
              IDL User Library INT_TABULATED.PRO yields;  result = 1.6232
                                    INTEGRAL.PRO yields;  result = 1.6274
         The Exact Solution (4 decimal accuracy) yields;  result = 1.6405

AUTHOR:         Liam Gumley, CIMSS/SSEC (liam.gumley@ssec.wisc.edu)
                Based on a FORTRAN-77 version by Paul van Delst, CIMSS/SSEC
                22-DEC-95

REVISIONS:      None.
  """
  n = x.size
  x0 = x[0:n-2]
  x1 = x[1:n-1]
  x2 = x[2:n-0]
  y0 = y[0:n-2]
  y1 = y[1:n-1]
  y2 = y[2:n-0]
  #
  # compute interpolation delta and midpoint arrays
  #
  dx = x1-x0
  xmid = 0.5*(x1+x0)
  #
  # compute 3 point lagrange interpolation
  #
  l0 = ((xmid-x1)/(x0-x1))*((xmid-x2)/(x0-x2))
  l1 = ((xmid-x0)/(x1-x0))*((xmid-x2)/(x1-x2))
  l2 = ((xmid-x0)/(x2-x0))*((xmid-x1)/(x2-x1))
  ymid = y0*l0 + y1*l1 + y2*l2;
  #
  # compute integral sum
  #
  integ = sum(1.0/6.0*dx*(y0+4.0*ymid+y1))
  #
  # handle last 3 points similarly
  #
  x0 = x[n-3]
  x1 = x[n-2]
  x2 = x[n-1]
  y0 = y[n-3]
  y1 = y[n-2]
  y2 = y[n-1]
  dx = x2 - x1
  xmid = 0.5*(x2+x1)
  l0 = ((xmid-x1)/(x0-x1))*((xmid-x2)/(x0-x2))
  l1 = ((xmid-x0)/(x1-x0))*((xmid-x2)/(x1-x2))
  l2 = ((xmid-x0)/(x2-x0))*((xmid-x1)/(x2-x1))
  ymid = y0*l0 + y1*l1 + y2*l2;
  integ = integ + 1.0/6.0*dx*(y1+4.0*ymid+y2)
  return integ

if __name__ == '__main__':
  print(integral.__doc__)
  X = num.array((0.0, .12, .22, .32, .36, .40, .44, .54, .64, .70, .80))
  Y = num.array((0.200000, 1.30973, 1.30524, 1.74339, 2.07490, 2.45600,
                 2.84299,  3.50730, 3.18194, 2.36302, 0.231964))
  i = integral(X,Y)
  print(i)


import numpy as np
from scipy.signal import detrend

def GetPrincipalComponents(x,y,z):

	mux = np.mean(x)
	muy = np.mean(y)
	muz = np.mean(z)

	sxx = np.sum((x-mux)**2)
	sxy = np.sum((x-mux)*(y-muy))
	sxz = np.sum((x-mux)*(z-muz))
	syy = np.sum((y-muy)**2)
	syz = np.sum((y-muy)*(z-muz))
	szz = np.sum((z-muz)**2)

	data = np.array([x-mux,y-muy,z-muz]).transpose()

	#ScatterMatrix=np.array([[sxx,sxy,sxz],[sxy,syy,syz],[sxz,syz,szz]])
	#CoVarMatrix=np.cov([x-mux,y-muy,z-muz])
	#Russell calls the scatter matrix a variance matrix
	#the covariance matrix = variance/(N-1)
	#according to a website these should both provide the same eigenvalues
	#though in this case they don't seem to
	ScatterMatrix = np.array([[sxx,sxy,sxz],[sxy,syy,syz],[sxz,syz,szz]])
	CoVarMatrix = np.cov(data.T)

	# eigenvectors and eigenvalues for the from the covariance matrix
	EigValCov, EigVecCov = np.linalg.eig(CoVarMatrix)

	srt = (EigValCov.argsort())[::-1]
	eigs = EigValCov[srt]
	vecs = EigVecCov[:,srt]

	return (eigs,vecs.T)

def RotateToPrincipalComponents(x,y,z):

	eigs,vecs = GetPrincipalComponents(x,y,z)

	nx = x*vecs[0,0] + y*vecs[0,1] + z*vecs[0,2]
	ny = x*vecs[1,0] + y*vecs[1,1] + z*vecs[1,2]
	nz = x*vecs[2,0] + y*vecs[2,1] + z*vecs[2,2]

	return (nx,ny,nz)

def SlidingPCA(t,x,y,z,Wind,Slip,Detrend=True):

	Tlen = np.size(t)

	Trange = t[Tlen-1]-t[0]
	Nwind = np.int32((Trange-Wind)/Slip)+1

	MinVec = np.zeros((Nwind,3),dtype='float32')+np.float32(np.nan)
	MidVec = np.zeros((Nwind,3),dtype='float32') +np.float32(np.nan)
	MaxVec = np.zeros((Nwind,3),dtype='float32') +np.float32(np.nan)
	Eigs = np.zeros((Nwind,3),dtype='float32')+np.float32(np.nan)

	Twind = np.arange(Nwind,dtype='float32')*Slip + Wind/2.0 + t[0]

	LenW = np.int32(Wind/0.05)/2
	for i in range(0,Nwind):
		use0 = np.int32(i*Slip/0.05)
		use = use0+np.arange(LenW*2)
		if np.max(use) >= np.size(t):
			bad = 1
		else:
			bad = np.where((np.isfinite(x) == False) | (np.isfinite(y) == False) | (np.isfinite(z) == False))[0]
		if np.size(bad) == 0:
			if Detrend:
				eigs,vecs = GetPrincipalComponents(detrend(x[use]),detrend(y[use]),detrend(z[use]))
			else:
				eigs,vecs = GetPrincipalComponents(x[use],y[use],z[use])
			Eigs[i] = eigs
			MaxVec[i] = vecs[0]
			MidVec[i] = vecs[1]
			MinVec[i] = vecs[2]

	return (Twind,Eigs,MinVec,MidVec,MaxVec)

def CovarianceMatrix(x):
	'''
	Input x must have shape (m,n), where m is the number of samples and n is the number of features

	Output is a (n,n) matrix
	'''
	mu = np.mean(x,axis=0)
	cov = np.dot((x-mu).T,(x-mu))/(x.shape[0]-1)

	return cov

def EigenValues(x):

	cov = CovarianceMatrix(x)
	EigVal,EigVec = np.linalg.eig(cov)

	srt = np.argsort(EigVal)[::-1]

	return EigVal[srt],EigVec[:,srt]

def PCA(x,nComp=None,EigVal=None,EigVec=None,ReturnEigVal=False,ReturnEigVec=False):
	if EigVal is None or EigVec is None:
		EigVal,EigVec = EigenValues(x)
	ev = np.abs(EigVal)

	if not nComp is None:
		#either a real or integer
		if np.int32(nComp) == nComp:
			#easy - just select number of components
			EigVal1 = EigVal[:np.int32(nComp)]
			EigVec1 = EigVec[:,:np.int32(nComp)]
			ev1 = ev[:np.int32(nComp)]
		else:
			#hopefully a floating point, in which case assume it is a number between 0.0 and 1.0
			cs = np.cumsum(ev)/np.sum(ev)
			use = np.where(cs >= nComp)[0]
			if use.size > 1:
				ind = use[1]
			elif use.size == 1:
				ind = use[0]
			else:
				ind = ev.size-1
			EigVal1 = EigVal[:ind+1]
			EigVec1 = EigVec[:,:ind+1]
			ev1 = ev[:ind+1]
	else:
		EigVec1 = EigVec
	x_transformed = np.dot(x,EigVec1)

	if not ReturnEigVec and not ReturnEigVal:
		out = x_transformed
	else:
		out = [x_transformed]
	if ReturnEigVal:
		out.append(EigVal)

	if ReturnEigVec:
		out.append(EigVec)

	return out


import numpy as np
from astropy.io import fits
import matplotlib.pyplot as plt
import time

def median_bins(data, B):
  mean = np.mean(data)
  std = np.std(data)
  minval = mean - std

  bins = np.zeros(B)
  bin_width = (2*std)/B
  smaller = 0
  for i in data:
    binmax = minval
    if i < minval:
      smaller+=1
    else:
      for j in range(B):
        binmax+=bin_width
        if i < binmax:
          bins[j] += 1
          break
  return (mean, std, smaller, bins)

def median_approx(data, B):
  mean, std, total, bins = median_bins(data,B)
  minval = mean - std
  bin_width = (2*std)/B
  target = (len(data)+1)/2
  for i in range(len(bins)):
    total+=bins[i]
    if total>=target:
      return (i*bin_width) + minval + (bin_width/2)
  return (i*bin_width) + minval + (bin_width/2)

# You can use this to test your functions.
# Any code inside this `if` statement will be ignored by the automarker.
if __name__ == '__main__':
  # Run your functions with the first example in the question.
  print(median_bins([1, 1, 3, 2, 2, 6], 3))
  print(median_approx([1, 1, 3, 2, 2, 6], 3))

  # Run your functions with the second example in the question.
  print(median_bins([1, 5, 7, 7, 3, 6, 1, 1], 4))
  print(median_approx([1, 5, 7, 7, 3, 6, 1, 1], 4))


'''
4.4.1 경사법(경사 하강법) gradient method

손실함수의 값이 최솟값이 되는 가중치와 편향을 찾는것이 목표. 기울기를 이용해 손실함수의 최솟값을 찾는 방법이 경사법.
다만 기울기가 가리키는 방향이 항상 손실함수의 최솟값이 존재하는 방향은 아니다.

최솟값에 도달할수도, 극솟값(특정 범위내에서의 최솟값)에 도달할수도, 안장점에 도달할수도,
최악으로는 고원 : 플래토 plateau 에 도달하여 학습이 정체기에 빠질수도 있다.

기울기의 방향이 항상 의도한대로 손실함수의 최솟값을 찾는것은 아니지만 일단은 기울기의 방향으로 가는것이 손실함수의 값을
줄이는 방법은 맞다.

경사법 : 경사법은 현 위치에서 기울어진 방향으로 일정 거리만큼 이동한다.
기울기가 음수이면 기울기 방향으로 나아가면 손실함수가 작아지는 방향으로 나아가는 것이고
기울기가 양수이면 기울기 반대방향으로 나아가면 손실함수가 작아지는 방향으로 나아간다.
나아간 위치에서 다시 기울기를 구하여 손실함수가 작아지는 방향으로 나아가는 것이 경사법이다.

x0 = x0 - eta * (df/dx0)
x1 = x1 - eta * (df/dx1)
...
xn = xn - eta * (df/dxn)

그래디언트가 음수이면 - eta 가 곱해져 양수가 되고, 그래디언트가 양수이면 - eta 가 곱해져 음수가 되어
x 값은 손실함수가 작아지는 방향으로 나아가게된다.

에타 eta 를 학습률 learning rate 이라고 한다. 한번의 학습으로 가중치, 편향의 값을 얼마만큼 갱신할지를 결정하게된다.
'''

import numpy as np
from ch4.numerical_differentiation import numerical_gradient

def gradient_descent(f, init_x, lr = 0.01, step_num = 100):
    x = init_x.copy()
    for i in range(step_num):
        grad = numerical_gradient(f, x)
        x -= grad * lr
    return x

def func_2(x) :
    return np.sum(x**2)

init_x = np.array([-3., 4.])
print(gradient_descent(func_2, init_x = init_x, lr = 0.1, step_num = 100))  # [ -6.11110793e-10   8.14814391e-10] == [ 0. 0.]
# 너무 큰 학습률
print(gradient_descent(func_2, init_x = init_x, lr = 10.0, step_num = 100)) # [ -2.58983747e+13  -1.29524862e+12]
# 너무 작은 학습률
print(gradient_descent(func_2, init_x = init_x, lr = 1e-10, step_num = 100))    # [-2.99999994  3.99999992]

## 학습률이 너무 크면 발산해버리고, 학습률이 너무 작으면 갱신이 제대로 되지 않는다.

'''
학습률같은 매개변수를 하이퍼파라미터 : 초매개변수 hyper parameter 라고 한다.
가중치, 편향 등은 신경망의 학습에 의해 자동으로 최적값을 갖게 되지만
학습률등의 하이퍼파라미터는 사람이 경험에 의거해 직접 설정해야하는 매개변수이다.
일반적으로 값을 바꿔보며 여러번의 시험을 통해 최적의 값을 찾는 과정을 거쳐야한다.
'''

'''
신경망에서 학습을 위한 경사감소법 수행시 기울기는 손실함수를 각 가중치로 편미분한 값을 사용한다.

W = [[w11 w21 w31]
     [w12 w22 w32]]
일 때
gradient =  [[dL/dw11 dL/dw21 dL/dw31]
             [dL/dw12 dL/dw22 dL/dw32]]
각 가중치에 대해 손실함수를 편미분한다. 이는 각 가중치 w 를 조금씩 변화시켰을때 손실함수 L 이 얼마나 변화하는가이다.

'''


import numpy as np

def solver(dt, T, f, f_0, f_1):
    """
    Solve u'=f by the Forward Euler method and by ordinary and
    Strang splitting: f(u) = f_0(u) + f_1(u).
    """
    Nt = int(round(T/float(dt)))
    t = np.linspace(0, Nt*dt, Nt+1)
    u_FE = np.zeros(len(t))
    u_split1 = np.zeros(len(t))  # 1st-order splitting
    u_split2 = np.zeros(len(t))  # 2nd-order splitting
    u_split3 = np.zeros(len(t))  # 2nd-order splitting w/exact f_0

    # Set initial values
    u_FE[0] = 0.1
    u_split1[0] = 0.1
    u_split2[0] = 0.1
    u_split3[0] = 0.1

    for n in range(len(t)-1):
        # Forward Euler method
        u_FE[n+1] = u_FE[n] + dt*f(u_FE[n])

        # --- Ordinary splitting ---
        # First step
        u_s_n = u_split1[n]
        u_s = u_s_n + dt*f_0(u_s_n)
        # Second step
        u_ss_n = u_s
        u_ss = u_ss_n + dt*f_1(u_ss_n)
        u_split1[n+1] = u_ss

        # --- Strang splitting ---
        # First step
        u_s_n = u_split2[n]
        u_s = u_s_n + dt/2.*f_0(u_s_n)
        # Second step
        u_sss_n = u_s
        u_sss = u_sss_n + dt*f_1(u_sss_n)
        # Third step
        u_ss_n = u_sss
        u_ss = u_ss_n + dt/2.*f_0(u_ss_n)
        u_split2[n+1] = u_ss

        # --- Strang splitting using exact integrator for u'=f_0 ---
        # First step
        u_s_n = u_split3[n]
        u_s = u_s_n*np.exp(dt/2.)  # exact
        # Second step
        u_sss_n = u_s
        u_sss = u_sss_n + dt*f_1(u_sss_n)
        # Third step
        u_ss_n = u_sss
        u_ss = u_ss_n*np.exp(dt/2.)  # exact
        u_split3[n+1] = u_ss

    return u_FE, u_split1, u_split2, u_split3, t

def solver_compact(dt, T, f, f_0, f_1):
    """
    As solver, but shorter code in the splitting steps.
    """
    Nt = int(round(T/float(dt)))
    t = np.linspace(0, Nt*dt, Nt+1)
    u_FE = np.zeros(len(t))
    u_split1 = np.zeros(len(t))  # 1st-order splitting
    u_split2 = np.zeros(len(t))  # 2nd-order splitting
    u_split3 = np.zeros(len(t))  # 2nd-order splitting w/exact f_0

    # Set initial values
    u_FE[0] = 0.1
    u_split1[0] = 0.1
    u_split2[0] = 0.1
    u_split3[0] = 0.1

    for n in range(len(t)-1):
        # Forward Euler method
        u_FE[n+1] = u_FE[n] + dt*f(u_FE[n])

        # Ordinary splitting
        u_s = u_split1[n] + dt*f_0(u_split1[n])
        u_split1[n+1] = u_s + dt*f_1(u_s)

        # Strang splitting
        u_s = u_split2[n] + dt/2.*f_0(u_split2[n])
        u_sss = u_s + dt*f_1(u_s)
        u_split2[n+1] = u_sss + dt/2.*f_0(u_sss)

        # Strang splitting using exact integrator for u'=f_0
        u_s = u_split3[n]*np.exp(dt/2.)  # exact
        u_ss = u_s + dt*f_1(u_s)
        u_split3[n+1] = u_ss*np.exp(dt/2.)

    return u_FE, u_split1, u_split2, u_split3, t

def demo(dt=0.2):
    u_exact = lambda t: 1./(9*np.exp(-t) + 1)
    u_FE, u_split1, u_split2, u_split3, t = solver(
        dt, 8, f = lambda u: u*(1-u),
        f_0=lambda u: u, f_1=lambda u: -u**2)

    import matplotlib.pyplot as plt
    plt.plot(t, u_FE, 'r-', t, u_split1, 'b-',
             t, u_split2, 'g-', t, u_split3, 'y-',
             t, u_exact(t), 'k--')
    plt.legend(['no split', 'split', 'strang', r'strang w/exact $f_0$', 'exact'],
               loc='lower right')
    plt.xlabel('t'); plt.ylabel('u')
    plt.title('Time step: %g' % dt)
    plt.savefig('tmp1.png'); plt.savefig('tmp1.pdf')
    plt.show()

def test_solver():
    np.set_printoptions(precision=15)
    u_FE_expected = np.array([float(x) for x in list("""
[ 0.1                0.118              0.1388152          0.162724308049792
  0.189973329573694  0.220750022298569  0.255153912289319
  0.293163990955874  0.334607764028414  0.379136845684478
  0.426215265270258  0.47512642785443   0.525002688936174
  0.574877662045366  0.62375632919069   0.67069320338774   0.714865969451186
  0.755632492485546  0.792562898242672  0.825444288357041
  0.854261491392197]"""[2:-1].split())])
    u_split1_expected = np.array([float(x) for x in list("""
[ 0.1                0.11712            0.1365934768128    0.158538732137911
  0.183007734044179  0.209963633605659  0.239259958824966
  0.270625296155645  0.303657796722007  0.33783343550351   0.37253027072271
  0.407068029717088  0.440758773984993  0.472961259290703
  0.503130113545367  0.530851841841463  0.555862750949651
  0.578048082546307  0.597425498363755  0.614118436921094
  0.628325385390187]"""[2:-1].split())])
    u_split2_expected = np.array([float(x) for x in list("""
[ 0.1                0.118338           0.139461146546647  0.1635705540078
  0.190798102531391  0.221174981642529  0.254599657026743
  0.290810238700023  0.329367696455926  0.369656716957107  0.41090943878828
  0.452253464828953  0.492779955548098  0.531621845295344
  0.568028513268957  0.601423369535241  0.631435056657206
  0.657899755122731  0.68083880192866   0.700420209898537
  0.716913803147616]"""[2:-1].split())])
    u_split3_expected = np.array([float(x) for x in list("""
[ 0.1                0.119440558200865  0.142033597399576
  0.168033940732164  0.197614356585594  0.230823935778256
  0.267544980228041  0.307455512661905  0.350006879617614
  0.394426527229859  0.439753524322399  0.484908174583947  0.5287881185737
  0.570374606392027  0.608827962442147  0.643553318053156
  0.674226057210925  0.700777592993228  0.723351459147494
  0.742244162719702  0.757844497686122]"""[2:-1].split())])
    for func in solver, solver_compact:
        u_FE, u_split1, u_split2, u_split3, t = solver(
            dt=0.2, T=4, f=lambda u: u*(1-u),
            f_0=lambda u: u, f_1=lambda u: -u**2)
        for quantity in 'u_FE', 'u_split1', 'u_split2', 'u_split3':
            diff = np.abs(eval(quantity + '_expected') -
                          eval(quantity)).max()
            assert diff < 1E-14

if __name__ == '__main__':
    test_solver()
    demo(0.05)


import numpy as np
from contextlib import suppress

def convolve_linear(signal, filter, output_size):
    out = np.zeros(output_size)
    sum = 0

    for i in range(output_size[0]):
        for j in range(output_size[1]):
            for k in range(max(0, i-filter.shape[0]), i+1):
                for l in range(max(0, j-filter.shape[1]), j+1):
                    with suppress(IndexError):
                        sum += signal[k, l] * filter[i-k, j-l]
            out[i, j] = sum
            sum = 0

    return out

def create_gaussian_kernel(kernel_size):
    kernel = np.zeros((kernel_size, kernel_size))

    # The center must be offset by 0.5 to find the correct index
    center = kernel_size*0.5 + 0.5

    sigma = np.sqrt(0.1*kernel_size)

    def kernel_function(x, y):
        return np.exp(-((x-center+1)**2 + (y-center+1)**2)/(2*sigma**2))

    kernel = np.fromfunction(kernel_function, (kernel_size, kernel_size))
    return kernel / np.linalg.norm(kernel)

def create_sobel_operators():
    Sx = np.dot([[1.0], [2.0], [1.0]], [[-1.0, 0.0, 1.0]]) / 9
    Sy = np.dot([[-1.0], [0.0], [1.0]], [[1.0, 2.0, 1.0]]) / 9

    return Sx, Sy

def sum_matrix_dimensions(mat1, mat2):
    return (mat1.shape[0] + mat2.shape[0],
            mat1.shape[1] + mat2.shape[1])

def compute_sobel(signal):
    Sx, Sy = create_sobel_operators()

    Gx = convolve_linear(signal, Sx, sum_matrix_dimensions(signal, Sx))
    Gy = convolve_linear(signal, Sy, sum_matrix_dimensions(signal, Sy))

    return np.sqrt(np.power(Gx, 2) + np.power(Gy, 2))

def create_circle(image_resolution, grid_extents, radius):
    out = np.zeros((image_resolution, image_resolution))

    for i in range(image_resolution):
        x_position = ((i * grid_extents / image_resolution)
                      - 0.5 * grid_extents)
        for j in range(image_resolution):
            y_position = ((j * grid_extents / image_resolution)
                          - 0.5 * grid_extents)
            if x_position ** 2 + y_position ** 2 <= radius ** 2:
                out[i, j] = 1.0

    return out

def main():

    # Random distribution in x
    x = np.random.rand(100, 100)

    # Gaussian signals
    def create_gaussian_signals(i, j):
        return np.exp(-(((i-50)/100) ** 2 +
                        ((j-50)/100) ** 2) / .01)
    y = np.fromfunction(create_gaussian_signals, (100, 100))

    # Normalization is not strictly necessary, but good practice
    x /= np.linalg.norm(x)
    y /= np.linalg.norm(y)

    # full convolution, output will be the size of x + y
    full_linear_output = convolve_linear(x, y, sum_matrix_dimensions(x, y))

    # simple boundaries
    simple_linear_output = convolve_linear(x, y, x.shape)

    np.savetxt("full_linear.dat", full_linear_output)
    np.savetxt("simple_linear.dat", simple_linear_output)

    # creating simple circle and 2 different Gaussian kernels
    circle = create_circle(50, 2, 0.5)

    circle = circle / np.linalg.norm(circle)

    small_kernel = create_gaussian_kernel(3)
    large_kernel = create_gaussian_kernel(25)

    small_kernel_output = convolve_linear(circle, small_kernel,
                                          sum_matrix_dimensions(circle,
                                                                small_kernel))

    large_kernel_output = convolve_linear(circle, large_kernel,
                                          sum_matrix_dimensions(circle,
                                                                large_kernel))

    np.savetxt("small_kernel.dat", small_kernel_output)
    np.savetxt("large_kernel.dat", large_kernel_output)

    circle = create_circle(50, 2, 0.5)

    # Normalization
    circle = circle / np.linalg.norm(circle)

    # using the circle for sobel operations as well
    sobel_output = compute_sobel(circle)

    np.savetxt("sobel_output.dat", sobel_output)


from numpy import *
import numpy as np

def sigmoid(x):
    return 1.0/(1.0 + exp(-x))

def sigmoidGrad(x):
    return sigmoid(x)*(1-sigmoid(x))

def softmax(x):
    #x = x-x.max(axis=1).reshape(x.shape[0],1)
    #b = np.sum(np.exp(x),axis=1)
    #x = np.exp(x)/b.reshape(x.shape[0],1)
    x = x-x.max()
    b = np.sum(np.exp(x))
    x = np.exp(x)/b
    return x

def make_onehot(i, n):
    y = np.zeros(n)
    y[i] = 1
    return y

class MultinomialSampler(object):
    """
    Fast (O(log n)) sampling from a discrete probability
    distribution, with O(n) set-up time.
    """

    def __init__(self, p, verbose=False):
        n = len(p)
        p = p.astype(float) / sum(p)
        self._cdf = cumsum(p)

    def sample(self, k=1):
        rs = np.random.random(k)
        # binary search to get indices
        return np.searchsorted(self._cdf, rs)

    def __call__(self, **kwargs):
        return self.sample(**kwargs)

    def reconstruct_p(self):
        """
        Return the original probability vector.
        Helpful for debugging.
        """
        n = len(self._cdf)
        p = np.zeros(n)
        p[0] = self._cdf[0]
        p[1:] = (self._cdf[1:] - self._cdf[:-1])
        return p

def multinomial_sample(p):
    """
    Wrapper to generate a single sample,
    using the above class.
    """
    return MultinomialSampler(p).sample(1)[0]


"""
File: adaptive_trapezint.py
Exercise 3.8
Copyright (c) 2016 Andrew Malfavon
License: MIT
Computes an approximation of an integral.
"""

import numpy as np

#Approximate integral using n number of trapazoids.
def trapezint(f, a, b, n):
    sum = 0
    h = (b - a) / float(n)
    for i in range(n):
        sum += (h / 2) * (f(a + i * h) + f(a + (i + 1) * h))
    return sum

#Approximate the second derivative of a function.
def double_prime(f, x, h=1.0E-5):
    return (f(x - h) - 2 * f(x) + f(x + h)) / float(h**2)

#Find the maximum second derivative between two values by using n iterations.
def max_double_prime(f, a, b, n=10000):
    maximum = abs(double_prime(f, a))
    for i in range(1, n):
        new_maximum = abs(double_prime(f, a + i  * ((b-a)/float((n)))))
        if new_maximum > maximum:
            maximum = new_maximum
    return maximum

def adaptive_trapezint(f, a, b, eps=1.0E-5):
    h = np.sqrt((12 * eps) / ((b - a) * max_double_prime(f, a, b)))
    n = (b - a) / h
    intn = int(n)#make calculated n an integer
    return trapezint(f, a, b, intn), intn

#Functions to test the maximum second derivative.
def func_x_cubed(x):
    return x**3

def func_sin(x):
    return np.sin(x)

def func_cos(x):
    return np.cos(x)

#Test chain rule.
def func_e_sin(x):
    return np.exp(np.sin(x))

def test_max():
    assert abs(max_double_prime(func_x_cubed, 1, 5) - 30) < 0.01
    assert abs(max_double_prime(func_sin, 0, 2 * np.pi) - 1) < 0.01
    assert abs(max_double_prime(func_cos, 0, (3 * np.pi) / float(2)) - 1) < 0.01

def table():
    #display the function and interval with the error and estimate n
    print '    Function' + '                    ' + 'Error' +  '            ' + 'Estimated n'
    #labels for the table
    cos = adaptive_trapezint(func_cos, 0, np.pi)#functions from the book
    sin1 = adaptive_trapezint(func_sin, 0, np.pi)
    sin2 = adaptive_trapezint(func_sin, 0, np.pi / 2.0)
    cos_phrase = 'cos(x) from 0 to pi'#labels for the functions
    sin1_phrase = 'sin(x) from 0 to pi'
    sin2_phrase = 'sin(x) from 0 to pi/2'
    space = '        '#blank spaces for formatting the table
    space2 = '      '
    print '{}{}{}{}{}'.format(cos_phrase, space, cos[0], space, cos[1])#puts the table together
    print '{}{}{}{}{}'.format(sin1_phrase, space, abs(sin1[0] - 2), space, sin1[1])
    print '{}{}{}{}{}'.format(sin2_phrase, space2, abs(sin2[0] - 1), space, sin2[1])


# -*- coding: utf-8 -*-
"""Utility module."""
import numpy as np

def make_prob_dist(freq, precision=10**(-10)):
    """
    Return a fine probability distribution based on given frequencies of states.

    :param freq: inputted array showing rate of occurrence
    :type freq: list or np.array
    :param precision: shows the min threshold of probabilies
    :type precision: float
    :return: a np.array giving the probability disterbution
    """
    freq = np.array(freq)
    final_dist = freq / np.sum(freq)
    zero_probe_idx = final_dist < precision
    final_dist[zero_probe_idx] = 0
    final_dist = final_dist / np.sum(final_dist)
    return np.real(final_dist)

def is_prob_dist(dist, precision=10**(-10)):
    """
    Check if given array is a probability distribution.

    :param dist: given array
    :type dist: list or np.array
    :return: True if dist is a probability distribution
    """
    dist = np.array(dist)
    if np.sum(dist < 0) != 0:
        return False
    if np.abs(np.sum(dist) - 1) > precision:
        return False
    return True

def is_valid_vector_type(vector):
    """
    Check if given vector is a valid one.

    :param vector: given vector
    :type vector: anything
    :return: True if vector is list or numpy array
    """
    return isinstance(vector, (list, np.ndarray))


# coding: utf-8
import numpy as np
import os
import matplotlib.pyplot as plt

def sigmoid(s):
    return 1/(1+np.exp(-s))

def gradient_w(x,y,w):
    g_e_in=0
    for i in range(len(x)):
        g_e_in += sigmoid(np.dot(-y[i]*x[i],w))*(-y[i]*x[i])
    return g_e_in/len(x)

def SGD(x,y,w,i):
    g_e_in = sigmoid(np.dot(-y[i]*x[i],w))*(-y[i]*x[i])
    return g_e_in

def sign(s):
    data = []
    for i in range(len(s)):
        if s[i] >= 0.5:
            data.append(1)
        else:
            data.append(-1)
    return np.array(data)

def error(h,true):
    error_rate = 0
    for i in range(len(h)):
        if h[i]!=true[i]:
            error_rate += 1
    return error_rate/len(h)

data = np.loadtxt(os.path.join('hw3_train.dat.txt'))
x_train = data[:,0:-1]
y_train = data[:,-1]
data = np.loadtxt(os.path.join('hw3_test.dat.txt'))
x_test = data[:,0:-1]
y_test = data[:,-1]
'''add x0=1'''
x_train = np.concatenate((np.ones((len(x_train),1)),x_train),axis=1)
x_test = np.concatenate((np.ones((len(x_test),1)),x_test),axis=1)
"initialization"
l_rate19 = 0.01
l_rate20 = 0.001
iteration = 2000
w = np.zeros(21)
SGD_w = np.zeros(21)
error_rate = []
E_out_error = []
index = []
"star training"
for i_ter in range(iteration):

    "Gradient descent start"
    E_in = sign(sigmoid(np.dot(x_train,w)))
    E_out = sign(sigmoid(np.dot(x_test,w)))
    grad_w = gradient_w(x_train,y_train,w)
    w = w - l_rate19*grad_w
    #if i_ter%1==0 or i_ter==(iteration-1):
    error_rate.append([error(E_in,y_train)])
    E_out_error.append([error(E_out,y_test)])

    "Stochastic Gradient Descent start"
    E_in = sign(sigmoid(np.dot(x_train,SGD_w)))
    E_out = sign(sigmoid(np.dot(x_test,SGD_w)))
    grad_w = SGD(x_train,y_train,SGD_w,i_ter%len(x_train))
    SGD_w = SGD_w - l_rate20*grad_w
    #if i_ter%1==0 or i_ter==(iteration-1):
    index.append(i_ter)
    error_rate[len(error_rate)-1].append(error(E_in,y_train))
    E_out_error[len(error_rate)-1].append(error(E_out,y_test))
    print("iteration = %d | (gd,SGD) error rate = (%3f,%3f)" %(i_ter,error_rate[len(error_rate)-1][0],error_rate[len(error_rate)-1][1]))

error_rate=np.array(error_rate)
E_out_error=np.array(E_out_error)
index=np.array(index)
"Plot E_in"
plt.plot(index,error_rate[:,0],"r",label="GD l_rate=0.01")
plt.plot(index,error_rate[:,1],"b",label="SGD l_rate=0.001")
plt.xlabel('E_in with GD & SGD')
plt.ylabel('Error rate')
plt.title('Problem 4')
plt.legend()
plt.savefig('Problem4.png')
plt.show()
"Plot E_out"
plt.plot(index,E_out_error[:,0],"r",label="GD l_rate=0.01")
plt.plot(index,E_out_error[:,1],"b",label="SGD l_rate=0.001")
plt.xlabel('E_out with GD & SGD')
plt.ylabel('Error rate')
plt.title('Problem 5')
plt.legend()
plt.savefig('Problem5.png')
plt.show()


import unittest

import numpy as np

from altFACS.hyperlog import *

#Define fixings (input data)
test_array = np.array([-100, -10, -1, 0, 1, 10, 100, 1000, 10000])

hlog_result = np.array([-3.66959885e+02, -3.67371939e+01, -3.67407265e+00,  0.00000000e+00,
                        3.67407265e+00,  3.67371939e+01,  3.66959885e+02,  3.42866112e+03,
                        7.20802323e+03])

#How accurate should equalities be
places = None
delta  = 0.0001 #use delta rather than places as we are dealing wiht log scales.

class TestHyperlog(unittest.TestCase):

    def test_hlog(self):
        #Test the default behaviour of hlog is as expected
        for n in range(len(test_array)):
            self.assertAlmostEqual(hlog_result[n], hlog(test_array[n]), places, "Default parameters should give this expected result.", delta)

if __name__ == "__main__":
    unittest.main(argv=['first-arg-is-ignored'], exit=False)