# Copyright 2019 ChangyuLiu Authors. All Rights Reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""To read data efficiently it can be helpful to serialize your data and store
 it in a set of files (100-200MB each) that can each be read linearly.
"""
import tensorflow as tf

import numpy as np
import IPython.display as display

"""## `tf.Example`

### Data types for `tf.Example`

Fundamentally a `tf.Example` is a `{"string": tf.train.Feature}` mapping.

The `tf.train.Feature` message type can accept one of the following three types (See the [`.proto` file]((https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto) for reference). Most other generic types can be coerced into one of these.

1. `tf.train.BytesList` (the following types can be coerced)

- `string`
- `byte`

1. `tf.train.FloatList` (the following types can be coerced)

- `float` (`float32`)
- `double` (`float64`)

1. `tf.train.Int64List` (the following types can be coerced)

- `bool`
- `enum`
- `int32`
- `uint32`
- `int64`
- `uint64`

In order to convert a standard TensorFlow type to a `tf.Example`-compatible `tf.train.Feature`, you can use the following shortcut functions:

Each function takes a scalar input value and returns a `tf.train.Feature` containing one of the three `list` types above.
"""

# The following functions can be used to convert a value to a type compatible
# with tf.Example.

def _bytes_feature(value):
  """Returns a bytes_list from a string / byte."""
  if isinstance(value, type(tf.constant(0))):
      value = value.numpy()  # BytesList won't unpack a string from an EagerTensor.
  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def _float_feature(value):
  """Returns a float_list from a float / double."""
  return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))

def _int64_feature(value):
  """Returns an int64_list from a bool / enum / int / uint."""
  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

"""Note: To stay simple, this example only uses scalar inputs. The simplest way to handle non-scalar features is to use `tf.serialize_tensor` to convert tensors to binary-strings. Strings are scalars in tensorflow. Use `tf.parse_tensor` to convert the binary-string back to a tensor.

Below are some examples of how these functions work. Note the varying input types and the standardizes output types. If the input type for a function does not match one of the coercible types stated above, the function will raise an exception (e.g. `_int64_feature(1.0)` will error out, since `1.0` is a float, so should be used with the `_float_feature` function instead).
"""

print(_bytes_feature(b'test_string'))
print(_bytes_feature(u'test_bytes'.encode('utf-8')))

print(_float_feature(np.exp(1)))

print(_int64_feature(True))
print(_int64_feature(1))

"""All proto messages can be serialized to a binary-string using the `.SerializeToString` method."""

feature = _float_feature(np.exp(1))

feature.SerializeToString()

"""### Creating a `tf.Example` message

Suppose you want to create a `tf.Example` message from existing data. In practice, the dataset may come from anywhere, but the procedure of creating the `tf.Example` message from a single observation will be the same.

1. Within each observation, each value needs to be converted to a `tf.train.Feature` containing one of the 3 compatible types, using one of the functions above.

1. We create a map (dictionary) from the feature name string to the encoded feature value produced in #1.

1. The map produced in #2 is converted to a [`Features` message](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/feature.proto#L85).

In this notebook, we will create a dataset using NumPy.

This dataset will have 4 features.
- a boolean feature, `False` or `True` with equal probability
- an integer feature uniformly randomly chosen from `[0, 5)`
- a string feature generated from a string table by using the integer feature as an index
- a float feature from a standard normal distribution

Consider a sample consisting of 10,000 independently and identically distributed observations from each of the above distributions.
"""

# the number of observations in the dataset
n_observations = int(1e4)

# boolean feature, encoded as False or True
feature0 = np.random.choice([False, True], n_observations)

# integer feature, random from 0 .. 4
feature1 = np.random.randint(0, 5, n_observations)

# string feature
strings = np.array([b'cat', b'dog', b'chicken', b'horse', b'goat'])
feature2 = strings[feature1]

# float feature, from a standard normal distribution
feature3 = np.random.randn(n_observations)

"""Each of these features can be coerced into a `tf.Example`-compatible type using one of `_bytes_feature`, `_float_feature`, `_int64_feature`. We can then create a `tf.Example` message from these encoded features."""

def serialize_example(feature0, feature1, feature2, feature3):
  """
  Creates a tf.Example message ready to be written to a file.
  """
  # Create a dictionary mapping the feature name to the tf.Example-compatible
  # data type.
  feature = {
      'feature0': _int64_feature(feature0),
      'feature1': _int64_feature(feature1),
      'feature2': _bytes_feature(feature2),
      'feature3': _float_feature(feature3),
  }

  # Create a Features message using tf.train.Example.

  example_proto = tf.train.Example(features=tf.train.Features(feature=feature))
  return example_proto.SerializeToString()

"""For example, suppose we have a single observation from the dataset, `[False, 4, bytes('goat'), 0.9876]`. We can create and print the `tf.Example` message for this observation using `create_message()`. Each single observation will be written as a `Features` message as per the above. Note that the `tf.Example` [message](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/example.proto#L88) is just a wrapper around the `Features` message."""

# This is an example observation from the dataset.

example_observation = []

serialized_example = serialize_example(False, 4, b'goat', 0.9876)

"""To decode the message use the `tf.train.Example.FromString` method."""

example_proto = tf.train.Example.FromString(serialized_example)

"""## TFRecords Format Details

A TFRecord file contains a sequence of records. The file can only be read sequentially.

Each record contains a byte-string, for the data-payload, plus the data-length, and  CRC32C (32-bit CRC using the Castagnoli polynomial) hashes for integrity checking.

Each record has the format

  uint64 length
  uint32 masked_crc32_of_length
  byte   data[length]
  uint32 masked_crc32_of_data

The records are concatenated together to produce the file. CRCs are
[described here](https://en.wikipedia.org/wiki/Cyclic_redundancy_check), and
the mask of a CRC is

  masked_crc = ((crc >> 15) | (crc << 17)) + 0xa282ead8ul

Note: There is no requirement to use `tf.Example` in TFRecord files. `tf.Example` is just a method of serializing dictionaries to byte-strings. Lines of text, encoded  image data, or serialized tensors (using `tf.io.serialize_tensor`, and
`tf.io.parse_tensor` when loading). See the `tf.io` module for more options.

## TFRecord files using `tf.data`

The `tf.data` module also provides tools for reading and writing data in tensorflow.

### Writing a TFRecord file

The easiest way to get the data into a dataset is to use the `from_tensor_slices` method.

Applied to an array, it returns a dataset of scalars.
"""

tf.data.Dataset.from_tensor_slices(feature1)

"""Applies to a tuple of arrays, it returns a dataset of tuples:"""

features_dataset = tf.data.Dataset.from_tensor_slices((feature0, feature1, feature2, feature3))

# Use `take(1)` to only pull one example from the dataset.
for f0, f1, f2, f3 in features_dataset.take(1):
  print(f0)
  print(f1)
  print(f2)
  print(f3)

"""Use the `tf.data.Dataset.map` method to apply a function to each element of a `Dataset`.

The mapped function must operate in TensorFlow graph mode: It must operate on and return `tf.Tensors`. A non-tensor function, like `create_example`, can be wrapped with `tf.py_function` to make it compatible.

Using `tf.py_function` requires that you specify the shape and type information that is otherwise unavailable:
"""

def tf_serialize_example(f0, f1, f2, f3):
  tf_string = tf.py_function(
      serialize_example,
      (f0, f1, f2, f3),  # pass these args to the above function.
      tf.string)      # the return type is `tf.string`.
  return tf.reshape(tf_string, ())  # The result is a scalar

tf_serialize_example(f0, f1, f2, f3)

"""Apply this function to each element in the dataset:"""

serialized_features_dataset = features_dataset.map(tf_serialize_example)

def generator():
  for features in features_dataset:
      yield serialize_example(*features)

serialized_features_dataset = tf.data.Dataset.from_generator(
  generator, output_types=tf.string, output_shapes=())

"""And write them to a TFRecord file:"""

filename = 'test.tfrecord'
writer = tf.data.experimental.TFRecordWriter(filename)
writer.write(serialized_features_dataset)

"""### Reading a TFRecord file

We can also read the TFRecord file using the `tf.data.TFRecordDataset` class.

More information on consuming TFRecord files using `tf.data` can be found [here](https://www.tensorflow.org/guide/datasets#consuming_tfrecord_data).

Using `TFRecordDataset`s can be useful for standardizing input data and optimizing performance.
"""

filenames = [filename]
raw_dataset = tf.data.TFRecordDataset(filenames)

"""At this point the dataset contains serialized `tf.train.Example` messages. When iterated over it returns these as scalar string tensors.

Use the `.take` method to only show the first 10 records.

Note: iterating over a `tf.data.Dataset` only works with eager execution enabled.
"""

for raw_record in raw_dataset.take(10):
  print(repr(raw_record))

"""These tensors can be parsed using the function below.

Note: The `feature_description` is necessary here because datasets use graph-execution, and need this description to build their shape and type signature.
"""

# Create a description of the features.
feature_description = {
  'feature0': tf.io.FixedLenFeature([], tf.int64, default_value=0),
  'feature1': tf.io.FixedLenFeature([], tf.int64, default_value=0),
  'feature2': tf.io.FixedLenFeature([], tf.string, default_value=''),
  'feature3': tf.io.FixedLenFeature([], tf.float32, default_value=0.0),
}

def _parse_function(example_proto):
  # Parse the input tf.Example proto using the dictionary above.
  return tf.io.parse_single_example(example_proto, feature_description)

"""Or use `tf.parse example` to parse a whole batch at once.

Apply this finction to each item in the dataset using the `tf.data.Dataset.map` method:
"""

parsed_dataset = raw_dataset.map(_parse_function)

"""Use eager execution to display the observations in the dataset. There are 10,000 observations in this dataset, but we only display the first 10. The data is displayed as a dictionary of features. Each item is a `tf.Tensor`, and the `numpy` element of this tensor displays the value of the feature."""

for parsed_record in parsed_dataset.take(10):
  print(repr(parsed_record))

"""Here, the `tf.parse_example` function unpacks the `tf.Example` fields into standard tensors.

## TFRecord files in python

The `tf.io` module also contains pure-Python functions for reading and writing TFRecord files.

### Writing a TFRecord file

Now write the 10,000 observations to the file `test.tfrecords`. Each observation is converted to a `tf.Example` message, then written to file. We can then verify that the file `test.tfrecords` has been created.
"""

# Write the `tf.Example` observations to the file.
with tf.io.TFRecordWriter(filename) as writer:
  for i in range(n_observations):
      example = serialize_example(feature0[i], feature1[i], feature2[i], feature3[i])
      writer.write(example)

"""### Reading a TFRecord file

These serialized tensores can be easily parsed using `tf.train.Example.ParseFromString`
"""

filenames = [filename]
raw_dataset = tf.data.TFRecordDataset(filenames)

for raw_record in raw_dataset.take(1):
  example = tf.train.Example()
  example.ParseFromString(raw_record.numpy())
  print(example)

"""## Walkthrough: Reading/Writing Image Data

This is an example of how to read and write image data using TFRecords. The purpose of this is to show how, end to end, input data (in this case an image) and write the data as a TFRecord file, then read the file back and display the image.

This can be useful if, for example, you want to use several models on the same input dataset. Instead of storing the image data raw, it can be preprocessed into the TFRecords format, and that can be used in all further processing and modelling.

First, let's download [this image](https://commons.wikimedia.org/wiki/File:Felis_catus-cat_on_snow.jpg) of a cat in the snow and [this photo](https://upload.wikimedia.org/wikipedia/commons/f/fe/New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg) of the Williamsburg Bridge, NYC under construction.

### Fetch the images
"""

cat_in_snow = tf.keras.utils.get_file('320px-Felis_catus-cat_on_snow.jpg', 'https://storage.googleapis.com/download.tensorflow.org/example_images/320px-Felis_catus-cat_on_snow.jpg')
williamsburg_bridge = tf.keras.utils.get_file('194px-New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg', 'https://storage.googleapis.com/download.tensorflow.org/example_images/194px-New_East_River_Bridge_from_Brooklyn_det.4a09796u.jpg')

display.display(display.Image(filename=cat_in_snow))
display.display(display.HTML('Image cc-by: Von.grzanka'))

display.display(display.Image(filename=williamsburg_bridge))
display.display(display.HTML('From Wikimedia'))

"""### Write the TFRecord file

As we did earlier, we can now encode the features as types compatible with `tf.Example`. In this case, we will not only store the raw image string as a feature, but we will store the height, width, depth, and an arbitrary `label` feature, which is used when we write the file to distinguish between the cat image and the bridge image. We will use `0` for the cat image, and `1` for the bridge image.
"""

image_labels = {
  cat_in_snow: 0,
  williamsburg_bridge: 1,
}

# This is an example, just using the cat image.
image_string = open(cat_in_snow, 'rb').read()

label = image_labels[cat_in_snow]

# Create a dictionary with features that may be relevant.

def image_example(image_string, label):
  image_shape = tf.image.decode_jpeg(image_string).shape

  feature = {
      'height': _int64_feature(image_shape[0]),
      'width': _int64_feature(image_shape[1]),
      'depth': _int64_feature(image_shape[2]),
      'label': _int64_feature(label),
      'image_raw': _bytes_feature(image_string),
  }

  return tf.train.Example(features=tf.train.Features(feature=feature))

for line in str(image_example(image_string, label)).split('\n')[:15]:
  print(line)
print('...')

"""We see that all of the features are now stores in the `tf.Example` message. Now, we functionalize the code above and write the example messages to a file, `images.tfrecords`."""

# Write the raw image files to images.tfrecords.
# First, process the two images into tf.Example messages.
# Then, write to a .tfrecords file.
record_file = 'images.tfrecords'
with tf.io.TFRecordWriter(record_file) as writer:
  for filename, label in image_labels.items():
      image_string = open(filename, 'rb').read()
      tf_example = image_example(image_string, label)
      writer.write(tf_example.SerializeToString())

"""### Read the TFRecord file

We now have the file `images.tfrecords`. We can now iterate over the records in the file to read back what we wrote. Since, for our use case we will just reproduce the image, the only feature we need is the raw image string. We can extract that using the getters described above, namely `example.features.feature['image_raw'].bytes_list.value[0]`. We also use the labels to determine which record is the cat as opposed to the bridge.
"""

raw_image_dataset = tf.data.TFRecordDataset('images.tfrecords')

# Create a dictionary describing the features.
image_feature_description = {
  'height': tf.io.FixedLenFeature([], tf.int64),
  'width': tf.io.FixedLenFeature([], tf.int64),
  'depth': tf.io.FixedLenFeature([], tf.int64),
  'label': tf.io.FixedLenFeature([], tf.int64),
  'image_raw': tf.io.FixedLenFeature([], tf.string),
}

def _parse_image_function(example_proto):
  # Parse the input tf.Example proto using the dictionary above.
  return tf.io.parse_single_example(example_proto, image_feature_description)

parsed_image_dataset = raw_image_dataset.map(_parse_image_function)

"""Recover the images from the TFRecord file:"""

for image_features in parsed_image_dataset:
  image_raw = image_features['image_raw'].numpy()
  display.display(display.Image(data=image_raw))


# -*- coding: utf-8 -*-
"""LS_DS_431_RNN_and_LSTM_Assignment.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1jfDqfw7tYRxJyXHrqDOZ8JrsN-Vt9-ev

## *Data Science Unit 4 Sprint 3 Assignment 1*

# Recurrent Neural Networks and Long Short Term Memory (LSTM)

![Monkey at a typewriter](https://upload.wikimedia.org/wikipedia/commons/thumb/3/3c/Chimpanzee_seated_at_typewriter.jpg/603px-Chimpanzee_seated_at_typewriter.jpg)

It is said that [infinite monkeys typing for an infinite amount of time](https://en.wikipedia.org/wiki/Infinite_monkey_theorem) will eventually type, among other things, the complete works of Wiliam Shakespeare. Let's see if we can get there a bit faster, with the power of Recurrent Neural Networks and LSTM.

This text file contains the complete works of Shakespeare: https://www.gutenberg.org/files/100/100-0.txt

Use it as training data for an RNN - you can keep it simple and train character level, and that is suggested as an initial approach.

Then, use that trained RNN to generate Shakespearean-ish text. Your goal - a function that can take, as an argument, the size of text (e.g. number of characters or lines) to generate, and returns generated text of that size.

Note - Shakespeare wrote an awful lot. It's OK, especially initially, to sample/use smaller data and parameters, so you can have a tighter feedback loop when you're trying to get things running. Then, once you've got a proof of concept - start pushing it more!
"""

from tensorflow.keras.callbacks import LambdaCallback, EarlyStopping, TensorBoard
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM

import numpy as np
import random
import sys
import requests
import os
import datetime

url = "https://www.gutenberg.org/files/100/100-0.txt"

r = requests.get(url)

r.encoding = r.apparent_encoding

data = r.text

data[:100]

data = data.split('\r\n')

data[:100]

# this creates a length of 43 sequences with 130 embedding length maybe ??? unique characters
toc = [l.strip() for l in data[44:130:2]]

type(toc)

# Skip the Table of Contents
data = data[135:]

toc

len(toc)

# Fixing Titles
toc[9] = 'THE LIFE OF KING HENRY V'
toc[18] = 'MACBETH'
toc[24] = 'OTHELLO, THE MOOR OF VENICE'
toc[34] = 'TWELFTH NIGHT: OR, WHAT YOU WILL'

locations = {id_:{'title':title, 'start':-99} for id_,title in enumerate(toc)}

print(locations)

# Start
for e,i in enumerate(data):
    for t,title in enumerate(toc):
        if title in i:
            locations[t].update({'start':e})

locations

import pandas as pd
# orient id takes out the first 10 row assuming it is the index
df_toc = pd.DataFrame.from_dict(locations, orient='index')

df_toc

df_toc['end'] = df_toc['start'].shift(-1).apply(lambda x: x-1)

df_toc['end']

df_toc['start']

df_toc.loc[42, 'end'] = len(data)

len(data)

df_toc['end'].tail()

df_toc.tail()

df_toc['text'] = df_toc.apply(lambda x: '\r\n'.join(data[ x['start'] : int(x['end']) ]), axis=1)

pd.set_option('display.max.rows', 999)
pd.set_option('display.max.columns', 999)
pd.set_option('display.max_colwidth', -1)
df_toc['text']

df_toc['text_len'] = df_toc['text'].apply(len)
print(df_toc['text_len'])
print(df_toc['text'][0])

# divide b/w plays and sonets
sonets = data[:2776]
plays = data[2777:]

len(plays)

data

def long_lines(lst_ln):
    clean = []

    for ln in lst_ln:

        if len(ln) == 0:
            pass
        else:
            pct = len(ln.strip(' ')) / len(ln)

            if pct >= .5:
                clean.append(ln.lstrip())

    return clean

# May Not be Needed
sonets = long_lines(sonets)
plays = long_lines(plays)

"""## Word Encoding

This is just a start, and is not complete yet.
"""

vocab = list(set("\r\n".join(plays).split()))
words = [line.split() for line in plays]

vocab

len(words)

words = words[:20000]

from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Embedding, LSTM, Dense, Dropout, Bidirectional
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam
from tensorflow.keras import regularizers
import tensorflow.keras.utils as ku
import numpy as np
import tensorflow as tf
import pickle

tokenizer = Tokenizer()
tokenizer.fit_on_texts(words)
total_words = len(tokenizer.word_index) + 1
print(total_words)

tokenizer.fit_on_texts(vocab)
total_words2 = len(tokenizer.word_index) + 1
print(total_words2)

input_sequences_train = []
input_sequences_test = []
for lines in words:
  token_list = tokenizer.texts_to_sequences([lines])[0]
  for i in range(1,len(token_list)):
    n_gram_sequence = token_list[:i+1]
    input_sequences_train.append(n_gram_sequence)
for lines2 in vocab:
  token_list2 = tokenizer.texts_to_sequences([lines2])[0]
  for i in range(1,len(token_list2)):
    n_gram_sequence2 = token_list[:i+1]
    input_sequences_test.append(n_gram_sequence2)

input_sequences_train

# train
# pad sequences
max_sequence_len = max([len(x) for x in input_sequences_train])
print(max_sequence_len)
input_sequences = np.array(pad_sequences(input_sequences_train, maxlen=max_sequence_len, padding='pre'))

# test
# pad sequences
max_sequence_len2 = max([len(x) for x in input_sequences_test])
print(max_sequence_len2)
input_sequences2 = np.array(pad_sequences(input_sequences_test, maxlen=max_sequence_len2, padding='pre'))

# create predictors and label
predictors, label = input_sequences[:,:-1],input_sequences[:,-1]

label = ku.to_categorical(label, num_classes=total_words)

model = Sequential()
model.add(Embedding(total_words, 20, input_length=max_sequence_len-1))
model.add(Bidirectional(LSTM(200, return_sequences = True)))
model.add(Dropout(0.2))
model.add(LSTM(100))
model.add(Dense(total_words/2, activation='relu', kernel_regularizer=regularizers.l2(0.001)))
model.add(Dense(total_words, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
print(model.summary())

breakpoint()

"""## Character Encoding

Using the technique shown in lecture.
"""

text = '\r\n'.join(sonets)

chars = list(set(text))

char_int = {c:i for i,c in enumerate(chars)}
int_char = {i:c for i,c in enumerate(chars)}

print(f"Our corpus contains {len(chars)} unique characters.")

# Create the Sequence Data

maxlen = 150
step = 1

encoded = [char_int[c] for c in text]

sequences = [] # Each element is 40 characters long
next_chars = [] # One element for each sequence

for i in range(0, len(encoded) - maxlen, step):
    sequences.append(encoded[i : i + maxlen])
    next_chars.append(encoded[i + maxlen])

print('sequences:', len(sequences))

import numpy as np

# Specify x & y

x = np.zeros((len(sequences), maxlen, len(chars)), dtype=np.bool)
y = np.zeros((len(sequences), len(chars)), dtype=np.bool)

for i, sequence in enumerate(sequences):
    for t, char in enumerate(sequence):
        x[i,t,char] = 1

    y[i, next_chars[i]] = 1

x.shape

# build the model: a single LSTM

model = Sequential()
model.add(LSTM(256, input_shape=(maxlen, len(chars)), dropout=0.2))
model.add(Dense(len(chars), activation='softmax'))

model.compile(loss='categorical_crossentropy', optimizer='nadam')

model.summary()

def sample(preds):
    # helper function to sample an index from a probability array
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / 1
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)

def on_epoch_end(epoch, _):
    # Function invoked at end of each epoch. Prints generated text.

    print()
    print('----- Generating text after Epoch: %d' % epoch)

    start_index = random.randint(0, len(text) - maxlen - 1)

    generated = ''

    sentence = text[start_index: start_index + maxlen]
    generated += sentence

    print('----- Generating with seed: "' + sentence + '"')
    sys.stdout.write(generated)

    for i in range(400):
        x_pred = np.zeros((1, maxlen, len(chars)))
        for t, char in enumerate(sentence):
            x_pred[0, t, char_int[char]] = 1

        preds = model.predict(x_pred, verbose=0)[0]
        next_index = sample(preds)
        next_char = int_char[next_index]

        sentence = sentence[1:] + next_char

        sys.stdout.write(next_char)
        sys.stdout.flush()
    print()

print_callback = LambdaCallback(on_epoch_end=on_epoch_end)

logdir = os.path.join("logs", datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
tensorboard_callback = TensorBoard(logdir, histogram_freq=1)

model.fit(x, y,
          batch_size=1024,
          validation_split=.2,
          epochs=100,
          callbacks=[print_callback,
                     #EarlyStopping(min_delta=.02, monitor='val_loss', patience=10),
                     tensorboard_callback])

# Commented out IPython magic to ensure Python compatibility.
# %load_ext tensorboard

# Commented out IPython magic to ensure Python compatibility.
# %tensorboard --logdir logs

"""# Resources and Stretch Goals

## Stretch goals:
- Refine the training and generation of text to be able to ask for different genres/styles of Shakespearean text (e.g. plays versus sonnets)
- Train a classification model that takes text and returns which work of Shakespeare it is most likely to be from
- Make it more performant! Many possible routes here - lean on Keras, optimize the code, and/or use more resources (AWS, etc.)
- Revisit the news example from class, and improve it - use categories or tags to refine the model/generation, or train a news classifier
- Run on bigger, better data

## Resources:
- [The Unreasonable Effectiveness of Recurrent Neural Networks](https://karpathy.github.io/2015/05/21/rnn-effectiveness/) - a seminal writeup demonstrating a simple but effective character-level NLP RNN
- [Simple NumPy implementation of RNN](https://github.com/JY-Yoon/RNN-Implementation-using-NumPy/blob/master/RNN%20Implementation%20using%20NumPy.ipynb) - Python 3 version of the code from "Unreasonable Effectiveness"
- [TensorFlow RNN Tutorial](https://github.com/tensorflow/models/tree/master/tutorials/rnn) - code for training a RNN on the Penn Tree Bank language dataset
- [4 part tutorial on RNN](http://www.wildml.com/2015/09/recurrent-neural-networks-tutorial-part-1-introduction-to-rnns/) - relates RNN to the vanishing gradient problem, and provides example implementation
- [RNN training tips and tricks](https://github.com/karpathy/char-rnn#tips-and-tricks) - some rules of thumb for parameterizing and training your RNN
"""


# file: dataset_reader.py
#
# Contains helper class dataset
# which is able to return mini batches of a desired
# size of <label, image> pairs
#
# For this the class constructor needs to be passed
# a imagePath in which it expects a subfolder for each
# image category.
#
# It automatically will traverse each subfolder recursively
# in order to generate a image list of the form
# e.g. [['cow', 'cow8439.jpeg'], ['dog', 'dog02.jpeg'], ...]
#
# Images will be read only using OpenCV's imread() function
# when preparing a mini-batch.
# They are not loaded all at once which is good,
# if we want to train on several 10.000 of images.
# (e.g. 1024 images of 1MB each --> 1 GB already!)
#
# The code here is inspired by:
# Wang Xinbo's AlexNet implementation:
# see https://github.com/SidHard/tfAlexNet
#
# ---
# Prof. Dr. Juergen Brauer, www.juergenbrauer.org

import numpy as np
import os
import cv2

RESIZE_WIDTH, RESIZE_HEIGHT = 227,227

class dataset:

    nr_img_channels = -1

    '''
    Checks which image categories are stored
    in the subfolder of <imagePath>:
    e.g. 'cows', 'dogs'
    Then prepares lists of filenames and label information
    for the image files found in the imagePath
    '''
    def __init__(self, imagePath, extensions, nr_img_channels):

        self.nr_img_channels = nr_img_channels

        # 1. prepare image list with category information
        #    self.data = [['cow', 'cowimg01.jpeg'],
        #                 ['dog', 'dogimg3289.jpeg], ...]
        print("\n")
        print("Searching in folder", imagePath, "for images")
        self.data = createImageList(imagePath, extensions)
        NrImgs = len(self.data)
        print("Found", NrImgs, "images")
        print("Here are the first 5 and the last 5 images and "
              "their corresponding categories I found:")
        for i in range(0,5):
                print(self.data[i])
        for i in range(NrImgs-5,NrImgs):
                print(self.data[i])

        # 2. shuffle the data
        np.random.shuffle(self.data)
        self.num_images = len(self.data)
        self.next_image_nr = 0

        # 3. use zip function to unzip the data into two lists
        #    see https://docs.python.org/3.3/library/functions.html#zip
        self.labels, self.filenames = zip(*self.data)

        # 4. show some random images
        for i in range(0,5):
                rnd_idx = np.random.randint(NrImgs)
                rnd_filename = self.filenames[ rnd_idx ]
                print("random filename = ", rnd_filename)
                img = cv2.imread( rnd_filename )
                img = self.preprocess(img)
                #img = cv2.resize(img, (RESIZE_WIDTH, RESIZE_HEIGHT))
                img_name = "example image " + str(i)
                cv2.imshow(img_name, img)
                cv2.moveWindow(img_name, 300+i*250,100);
        cv2.waitKey(1000)
        cv2.destroyAllWindows()

        # 5. get a list of all categories,
        #    e.g. ['cows', 'dogs']
        category_list = np.unique(self.labels)

        # 6. how many categories are there?
        self.num_labels = len(category_list)

        # 7. prepare a dictionary to map
        #   category names to category numbers
        self.category2label = \
            dict(zip(category_list, range(len(category_list))))

        # 8. and the other way round:
        #    prepare a dictionary {} to map category numbers
        #    to category names
        self.label2category =\
            {l: c for c, l in self.category2label.items()}

        # 9. prepare list of ground truth labels
        #    where we can find the ground truth label for
        #    image i at the i-th position in the list
        self.labels = [self.category2label[l] for l in self.labels]

    '''
    Returns the number of images
    available by this dataset object
    '''
    def __len__(self):
        return self.num_images

    '''
    Returns a onehot NumPy array,
    where all entries are set to 0
    but to 1 for the right category
    '''
    def onehot(self, label):
        v = np.zeros(self.num_labels)
        v[label] = 1
        return v

    '''
    Are there further images available?
    '''
    def hasNextRecord(self):
        return self.next_image_nr < self.num_images

    '''
    Resizes the specified OpenCV image to a fixed size
    Converts it to a NumPy array
    Converts the values from [0,255] to [0,1]
    '''
    def preprocess(self, img):

        # convert image to gray-scale
        if self.nr_img_channels == 1:
            img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

        # preprocess image by resizing it to 227x227
        pp = cv2.resize(img, (RESIZE_WIDTH, RESIZE_HEIGHT))

        # and convert OpenCV representation to Numpy array
        # note: asarray does not copy data!
        #       see
        pp = np.asarray(pp, dtype=np.float32)

        # map values from [0,255] to [0,1]
        pp /= 255

        # prepare array of shape width x height x nr_img_channels array
        pp = pp.reshape((pp.shape[0], pp.shape[1], self.nr_img_channels))
        return pp

    '''
    Returns a (label, image) tuple
     where label is a one-hot teacher vector (list)
     e.g. [0,1] if there are two categories
     and
     image is a NumPy array
     of shape (width, height, 3)
    '''
    def get_next_record(self):

        # will return the next training pair
        # consisting of the input image and a one-hot/teacher label vector
        if not self.hasNextRecord():

            # Ups! We are at the end of the image list!

            # So generate new random order of images

            # randomly shuffle the data again
            np.random.shuffle(self.data)
            self.next_image_nr = 0
            self.labels, self.filenames = zip(*self.data)
            category = np.unique(self.labels)
            self.num_labels = len(category)
            self.category2label = dict(zip(category, range(len(category))))
            self.label2category = {l: c for c, l in self.category2label.items()}

            # prepare ground-truth label information for all images
            # according to the newly shuffled order of the images
            self.labels = [self.category2label[l] for l in self.labels]

        # prepare one-hot teacher vector for the output neurons
        label = self.onehot(self.labels[self.next_image_nr])

        # read in the image using OpenCVs imread()
        # function and then preprocess it
        # (i.e., resize it, convert it to a NumPy array,
        #  convert values from [0,255] to [0,1])
        img_filename = self.filenames[self.next_image_nr]
        img_as_np_array = self.preprocess(cv2.imread(img_filename))

        # prepare next image nr to return
        self.next_image_nr += 1

        # prepare a (label, image) tuple
        return label, img_as_np_array

    '''
    Given a batch size, this function
    first creates a list of (label, image)
    tuples called <records>
    [(0, img-of-cow), (1, img-of-dog), (0, img-of-cow), ...]
    and then returns the labels and images as separate
    tuples
    '''
    def nextBatch(self, batch_size):

        # creates a mini-batch of the desired size
        records = []
        for i in range(batch_size):
            record = self.get_next_record()
            if record is None:
                break
            records.append(record)
        labels, imgs = zip(*records)
        return labels, imgs

'''
Helper function to provide a list of
all label info and image files in all
subfolders in the the given <imagePath>
'''
def createImageList(imagePath, extensions):

    # 1. start with an empty list of labels/filenames
    labels_and_filenames = []

    # 2. each subfolder name in imagePath is considered to be
    #    a class label in stored in categoryList
    categoryList = [None]
    categoryList = [c for c in sorted(os.listdir(imagePath))
                    if c[0] != '.' and
                    os.path.isdir(os.path.join(imagePath, c))]

    # 3. for each of the categories
    for category in categoryList:
        print("subfolder/category found =", category)
        if category:
            walkPath = os.path.join(imagePath, category)
        else:
            walkPath = imagePath
            category = os.path.split(imagePath)[1]

        # create a generator
        w = _walk(walkPath)

        # step through all directories and subdirectories
        while True:

            # get names of dirs and filenames of current dir
            try:
                dirpath, dirnames, filenames = next(w)
            except StopIteration:
                break

            # don't enter directories that begin with '.'
            for d in dirnames[:]:
                if d.startswith('.'):
                    dirnames.remove(d)

            dirnames.sort()

            # ignore files that begin with '.'
            filenames = [f for f in filenames if not f.startswith('.')]
            # only load images with the right extension
            filenames = [f for f in filenames if os.path.splitext(f)[1].lower() in extensions]
            filenames.sort()

            for f in filenames:
                labels_and_filenames.append([category, os.path.join(dirpath, f)])

    # labels_and_filenames will be a list of
    # two-tuples [category, filename]
    return labels_and_filenames

def _walk(top):
    """
    This is a (recursive) directory tree generator.
    What is a generator?
    See:
    http://stackoverflow.com/questions/231767/what-does-the-yield-keyword-do
    In short:
    - generators are iterables that can be iterated only once
    - their values are not stored in contrast e.g. to a list
    - 'yield' is 'like' return
    """

    # 1. collect directory names in dirs and
    #    non-directory names (filenames) in nondirs
    names = os.listdir(top)
    dirs, nondirs = [], []
    for name in names:
        if os.path.isdir(os.path.join(top, name)):
            dirs.append(name)
        else:
            nondirs.append(name)

    # 2. "return" information about directory names and filenames
    yield top, dirs, nondirs

    # 3. recursively process each directory found in current top
    #    directory
    for name in dirs:
        path = os.path.join(top, name)
        for x in _walk(path):
            yield x


import numpy as np

my_list = [10,20,30,40]

arr = np.array(my_list)

print('Elements :')
for ele in arr:
    print(ele)

# Printing Elements using for loop

new_list = [[10,20,30],[40,50,60],[70,80,90]]
matrix = np.array(new_list)

print('Elements without list')

for row in matrix:
    print(row)

print('\nprinting each element of row')
for row in matrix:
    for ele in row:
        print(ele)


#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import unittest
import numpy as np
import paddle.fluid.core as core
from op_test import OpTest, skip_check_grad_ci
import paddle
import paddle.fluid as fluid
from paddle.fluid import Program, program_guard

class TestDropoutOp(OpTest):
    def setUp(self):
        self.op_type = "dropout"
        self.inputs = {'X': np.random.random((32, 64)).astype("float32")}
        self.attrs = {'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False}
        self.outputs = {
            'Out': self.inputs['X'],
            'Mask': np.ones((32, 64)).astype('uint8')
        }

    def test_check_output(self):
        self.check_output()

    def test_check_grad_normal(self):
        self.check_grad(['X'], 'Out')

class TestDropoutOpInput1d(OpTest):
    def setUp(self):
        self.op_type = "dropout"
        self.inputs = {'X': np.random.random((2000, )).astype("float32")}
        self.attrs = {'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False}
        self.outputs = {
            'Out': self.inputs['X'],
            'Mask': np.ones((2000)).astype('uint8')
        }

    def test_check_output(self):
        self.check_output()

    def test_check_grad_normal(self):
        self.check_grad(['X'], 'Out')

class TestDropoutOp2(TestDropoutOp):
    def setUp(self):
        self.op_type = "dropout"
        self.inputs = {'X': np.random.random((32, 64)).astype("float32")}
        self.attrs = {'dropout_prob': 1.0, 'fix_seed': True, 'is_test': False}
        self.outputs = {
            'Out': np.zeros((32, 64)).astype('float32'),
            'Mask': np.zeros((32, 64)).astype('uint8')
        }

class TestDropoutOp3(TestDropoutOp):
    def setUp(self):
        self.op_type = "dropout"
        self.inputs = {'X': np.random.random((32, 64, 2)).astype("float32")}
        self.attrs = {'dropout_prob': 0.0, 'fix_seed': True, 'is_test': False}
        self.outputs = {
            'Out': self.inputs['X'],
            'Mask': np.ones((32, 64, 2)).astype('uint8')
        }

@skip_check_grad_ci(reason="For inference, check_grad is not required.")
class TestDropoutOp4(OpTest):
    def setUp(self):
        self.op_type = "dropout"
        self.inputs = {'X': np.random.random((32, 64)).astype("float32")}
        self.attrs = {'dropout_prob': 0.35, 'fix_seed': True, 'is_test': True}
        self.outputs = {
            'Out': self.inputs['X'] * (1.0 - self.attrs['dropout_prob'])
        }

    def test_check_output(self):
        self.check_output()

@skip_check_grad_ci(reason="For inference, check_grad is not required.")
class TestDropoutOp5(OpTest):
    def setUp(self):
        self.op_type = "dropout"
        self.inputs = {'X': np.random.random((32, 64, 3)).astype("float32")}
        self.attrs = {'dropout_prob': 0.75, 'is_test': True}
        self.outputs = {
            'Out': self.inputs['X'] * (1.0 - self.attrs['dropout_prob'])
        }

    def test_check_output(self):
        self.check_output()

class TestDropoutOp6(TestDropoutOp):
    def setUp(self):
        self.op_type = "dropout"
        self.inputs = {'X': np.random.random((32, 64)).astype("float32")}
        self.attrs = {
            'dropout_prob': 1.0,
            'fix_seed': True,
            'is_test': False,
            'dropout_implementation': 'upscale_in_train'
        }
        self.outputs = {
            'Out': np.zeros((32, 64)).astype('float32'),
            'Mask': np.zeros((32, 64)).astype('uint8')
        }

class TestDropoutOp7(TestDropoutOp):
    def setUp(self):
        self.op_type = "dropout"
        self.inputs = {'X': np.random.random((32, 64, 2)).astype("float32")}
        self.attrs = {
            'dropout_prob': 0.0,
            'fix_seed': True,
            'is_test': False,
            'dropout_implementation': 'upscale_in_train'
        }
        self.outputs = {
            'Out': self.inputs['X'],
            'Mask': np.ones((32, 64, 2)).astype('uint8')
        }

@skip_check_grad_ci(reason="For inference, check_grad is not required.")
class TestDropoutOp8(OpTest):
    def setUp(self):
        self.op_type = "dropout"
        self.inputs = {'X': np.random.random((32, 64)).astype("float32")}
        self.attrs = {
            'dropout_prob': 0.35,
            'fix_seed': True,
            'is_test': True,
            'dropout_implementation': 'upscale_in_train'
        }
        self.outputs = {'Out': self.inputs['X']}

    def test_check_output(self):
        self.check_output()

@skip_check_grad_ci(reason="For inference, check_grad is not required.")
class TestDropoutOp9(OpTest):
    def setUp(self):
        self.op_type = "dropout"
        self.inputs = {'X': np.random.random((32, 64, 3)).astype("float32")}
        self.attrs = {
            'dropout_prob': 0.75,
            'is_test': True,
            'dropout_implementation': 'upscale_in_train'
        }
        self.outputs = {'Out': self.inputs['X']}

    def test_check_output(self):
        self.check_output()

class TestDropoutOpWithSeed(OpTest):
    def setUp(self):
        self.op_type = "dropout"
        self.inputs = {
            "X": np.random.random((32, 64)).astype("float32"),
            "Seed": np.asarray(
                [125], dtype="int32")
        }
        self.attrs = {'dropout_prob': 0.0, }
        self.outputs = {
            'Out': self.inputs['X'],
            'Mask': np.ones((32, 64)).astype('uint8')
        }

    def test_check_output(self):
        self.check_output()

    def test_check_grad_normal(self):
        self.check_grad(['X'], 'Out', max_relative_error=0.05)

@unittest.skipIf(
    not core.is_compiled_with_cuda() or not core.op_support_gpu("dropout"),
    "core is not compiled with CUDA or core is not support dropout")
@skip_check_grad_ci(reason="For inference, check_grad is not required.")
class TestFP16DropoutOp(OpTest):
    def setUp(self):
        self.op_type = "dropout"
        self.init_test_case()

        x = np.random.random(self.input_size).astype("float16")
        out = x * (1.0 - self.prob)
        self.inputs = {'X': OpTest.np_dtype_to_fluid_dtype(x)}
        self.attrs = {
            'dropout_prob': self.prob,
            'fix_seed': self.fix_seed,
            'is_test': True
        }
        self.outputs = {'Out': out}

    def init_test_case(self):
        self.input_size = [32, 64]
        self.prob = 0.35
        self.fix_seed = True

    def test_check_output(self):
        self.check_output_with_place(core.CUDAPlace(0), atol=1e-3)

@unittest.skipIf(
    not core.is_compiled_with_cuda() or not core.op_support_gpu("dropout"),
    "core is not compiled with CUDA or core is not support dropout")
@skip_check_grad_ci(reason="For inference, check_grad is not required.")
class TestFP16DropoutOp2(TestFP16DropoutOp):
    def init_test_case(self):
        self.input_size = [32, 64, 3]
        self.prob = 0.75
        self.fix_seed = False

class TestDropoutOpError(unittest.TestCase):
    def test_errors(self):
        with program_guard(Program(), Program()):

            def test_Variable():
                # the input of dropout must be Variable.
                x1 = fluid.create_lod_tensor(
                    np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace())
                fluid.layers.dropout(x1, dropout_prob=0.5)

            self.assertRaises(TypeError, test_Variable)

            def test_dtype():
                # the input dtype of dropout must be float16 or float32 or float64
                # float16 only can be set on GPU place
                x2 = fluid.layers.data(
                    name='x2', shape=[3, 4, 5, 6], dtype="int32")
                fluid.layers.dropout(x2, dropout_prob=0.5)

            self.assertRaises(TypeError, test_dtype)

class TestDropoutFAPI(unittest.TestCase):
    def setUp(self):
        np.random.seed(123)
        self.places = [fluid.CPUPlace()]
        if core.is_compiled_with_cuda():
            self.places.append(fluid.CUDAPlace(0))

    def check_static_result(self, place):
        with fluid.program_guard(fluid.Program(), fluid.Program()):
            input = fluid.data(name="input", shape=[40, 40], dtype="float32")
            res1 = paddle.nn.functional.dropout(x=input, p=0., training=False)
            res2 = paddle.nn.functional.dropout(
                x=input, p=0., axis=0, training=True, mode='upscale_in_train')
            res3 = paddle.nn.functional.dropout(
                x=input, p=0., axis=0, training=True, mode='downscale_in_infer')
            res4 = paddle.nn.functional.dropout(
                x=input, p=0., axis=0, training=False, mode='upscale_in_train')
            res5 = paddle.nn.functional.dropout(
                x=input,
                p=0.,
                axis=0,
                training=False,
                mode='downscale_in_infer')
            res6 = paddle.nn.functional.dropout(
                x=input,
                p=0.,
                axis=[0, 1],
                training=True,
                mode='upscale_in_train')
            res7 = paddle.nn.functional.dropout(
                x=input,
                p=0.,
                axis=[0, 1],
                training=True,
                mode='downscale_in_infer')
            res8 = paddle.nn.functional.dropout(
                x=input,
                p=0.,
                axis=[0, 1],
                training=False,
                mode='upscale_in_train')
            res9 = paddle.nn.functional.dropout(
                x=input,
                p=0.,
                axis=[0, 1],
                training=False,
                mode='downscale_in_infer')
            res10 = paddle.nn.functional.dropout(x=input, p=1., training=True)
            res11 = paddle.fluid.layers.dropout(x=input, dropout_prob=0.)

            in_np = np.random.random([40, 40]).astype("float32")
            res_np = in_np
            res_np2 = np.zeros_like(in_np)

            exe = fluid.Executor(place)
            res_list = [
                res1, res2, res3, res4, res5, res6, res7, res8, res9, res11
            ]
            for res in res_list:
                fetches = exe.run(fluid.default_main_program(),
                                  feed={"input": in_np},
                                  fetch_list=[res])
                self.assertTrue(np.allclose(fetches[0], res_np))
            fetches2 = exe.run(fluid.default_main_program(),
                               feed={"input": in_np},
                               fetch_list=[res10])
            self.assertTrue(np.allclose(fetches2[0], res_np2))

    def test_static(self):
        for place in self.places:
            self.check_static_result(place=place)

    def test_dygraph(self):
        for place in self.places:
            with fluid.dygraph.guard(place):
                in_np = np.random.random([40, 40]).astype("float32")
                res_np = in_np
                res_np2 = np.zeros_like(in_np)
                input = fluid.dygraph.to_variable(in_np)

                res1 = paddle.nn.functional.dropout(
                    x=input, p=0., training=False)
                res2 = paddle.nn.functional.dropout(
                    x=input,
                    p=0.,
                    axis=0,
                    training=True,
                    mode='upscale_in_train')
                res3 = paddle.nn.functional.dropout(
                    x=input,
                    p=0.,
                    axis=0,
                    training=True,
                    mode='downscale_in_infer')
                res4 = paddle.nn.functional.dropout(
                    x=input,
                    p=0.,
                    axis=0,
                    training=False,
                    mode='upscale_in_train')
                res5 = paddle.nn.functional.dropout(
                    x=input,
                    p=0.,
                    axis=0,
                    training=False,
                    mode='downscale_in_infer')
                res6 = paddle.nn.functional.dropout(
                    x=input,
                    p=0.,
                    axis=[0, 1],
                    training=True,
                    mode='upscale_in_train')
                res7 = paddle.nn.functional.dropout(
                    x=input,
                    p=0.,
                    axis=[0, 1],
                    training=True,
                    mode='downscale_in_infer')
                res8 = paddle.nn.functional.dropout(
                    x=input,
                    p=0.,
                    axis=[0, 1],
                    training=False,
                    mode='upscale_in_train')
                res9 = paddle.nn.functional.dropout(
                    x=input,
                    p=0.,
                    axis=[0, 1],
                    training=False,
                    mode='downscale_in_infer')
                res10 = paddle.nn.functional.dropout(
                    x=input, p=1., training=True)
                dropout = paddle.fluid.dygraph.Dropout(p=0, )
                res11 = dropout(input)

            res_list = [
                res1, res2, res3, res4, res5, res6, res7, res8, res9, res11
            ]
            for res in res_list:
                self.assertTrue(np.allclose(res.numpy(), res_np))
            self.assertTrue(np.allclose(res10.numpy(), res_np2))

class TestDropoutFAPIError(unittest.TestCase):
    def test_errors(self):
        with program_guard(Program(), Program()):

            def test_Variable():
                # the input of dropout must be Variable.
                x1 = fluid.create_lod_tensor(
                    np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace())
                paddle.nn.functional.dropout(x1, p=0.5)

            self.assertRaises(TypeError, test_Variable)

            def test_Variable2():
                # the input of dropout must be Variable.
                x1 = fluid.create_lod_tensor(
                    np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace())
                paddle.nn.functional.dropout(x1, p=0.5, axis=0)

            self.assertRaises(TypeError, test_Variable2)

            def test_dtype():
                # the input dtype of dropout must be float32 or float64
                # float16 only can be set on GPU place
                xr = fluid.data(name='xr', shape=[3, 4, 5, 6], dtype="int32")
                paddle.nn.functional.dropout(xr, p=0.5)

            self.assertRaises(TypeError, test_dtype)

            def test_pdtype():
                # p should be int or float
                x2 = fluid.data(name='x2', shape=[3, 4, 5, 6], dtype="float32")
                paddle.nn.functional.dropout(x2, p='0.5')

            self.assertRaises(TypeError, test_pdtype)

            def test_pvalue():
                # p should be 0.<=p<=1.
                x2 = fluid.data(name='x2', shape=[3, 4, 5, 6], dtype="float32")
                paddle.nn.functional.dropout(x2, p=1.2)

            self.assertRaises(ValueError, test_pvalue)

            def test_mode():
                # mode should be 'downscale_in_infer' or 'upscale_in_train'
                x2 = fluid.data(name='x2', shape=[3, 4, 5, 6], dtype="float32")
                paddle.nn.functional.dropout(x2, mode='abc')

            self.assertRaises(ValueError, test_mode)

            def test_axis():
                # axis should be int or list
                x2 = fluid.data(name='x2', shape=[3, 4, 5, 6], dtype="float32")
                paddle.nn.functional.dropout(x2, axis=1.2)

            self.assertRaises(TypeError, test_axis)

            def test_axis_max():
                # maximum of axis should less than dimensions of x
                x2 = fluid.data(name='x2', shape=[3, 4, 5, 6], dtype="float32")
                paddle.nn.functional.dropout(x2, axis=[0, 5])

            self.assertRaises(ValueError, test_axis_max)

            def test_axis_min():
                # minimum of axis should greater equal than 0
                x2 = fluid.data(name='x2', shape=[3, 4, 5, 6], dtype="float32")
                paddle.nn.functional.dropout(x2, axis=[0, -1])

            self.assertRaises(ValueError, test_axis_min)

            def test_axis_len():
                # length of axis should not greater than dimensions of x
                x2 = fluid.data(name='x2', shape=[3, 4, 5, 6], dtype="float32")
                paddle.nn.functional.dropout(x2, axis=[0, 1, 2, 3, 4])

            self.assertRaises(ValueError, test_axis_len)

class TestDropoutCAPI(unittest.TestCase):
    def setUp(self):
        np.random.seed(123)
        self.places = [fluid.CPUPlace()]
        if core.is_compiled_with_cuda():
            self.places.append(fluid.CUDAPlace(0))

    def test_dygraph(self):
        for place in self.places:
            with fluid.dygraph.guard(place):
                input_np = np.random.random([40, 40]).astype("float32")
                result_np = input_np
                input = fluid.dygraph.to_variable(input_np)
                m = paddle.nn.Dropout(p=0.)
                m.eval()
                result = m(input)
                self.assertTrue(np.allclose(result.numpy(), result_np))

class TestDropout2DFAPI(unittest.TestCase):
    def setUp(self):
        np.random.seed(123)
        self.places = [fluid.CPUPlace()]
        if core.is_compiled_with_cuda():
            self.places.append(fluid.CUDAPlace(0))

    def check_static_result(self, place):
        with fluid.program_guard(fluid.Program(), fluid.Program()):
            input = fluid.data(
                name="input", shape=[2, 3, 4, 5], dtype="float32")
            res1 = paddle.nn.functional.dropout2d(
                x=input, p=0., training=False, data_format='NCHW')
            res2 = paddle.nn.functional.dropout2d(
                x=input, p=0., training=False, data_format='NHWC')

            in_np = np.random.random([2, 3, 4, 5]).astype("float32")
            res_np = in_np

            exe = fluid.Executor(place)
            res_list = [res1, res2]
            for res in res_list:
                fetches = exe.run(fluid.default_main_program(),
                                  feed={"input": in_np},
                                  fetch_list=[res])
                self.assertTrue(np.allclose(fetches[0], res_np))

    def test_static(self):
        for place in self.places:
            self.check_static_result(place=place)

    def test_dygraph(self):
        for place in self.places:
            with fluid.dygraph.guard(place):
                in_np = np.random.random([2, 3, 4, 5]).astype("float32")
                res_np = in_np
                input = fluid.dygraph.to_variable(in_np)

                res1 = paddle.nn.functional.dropout2d(
                    x=input, p=0., training=False, data_format='NCHW')
                res2 = paddle.nn.functional.dropout2d(
                    x=input, p=0., training=False, data_format='NHWC')

            res_list = [res1, res2]
            for res in res_list:
                self.assertTrue(np.allclose(res.numpy(), res_np))

class TestDropout2DFAPIError(unittest.TestCase):
    def test_errors(self):
        with program_guard(Program(), Program()):

            def test_xdim():
                # dimentions of x should be 4
                x = fluid.data(name='x1', shape=[2, 3, 4, 5, 6], dtype="int32")
                paddle.nn.functional.dropout2d(x)

            self.assertRaises(ValueError, test_xdim)

            def test_dataformat():
                # data_format should be 'NCHW' or 'NHWC'
                x = fluid.data(name='x2', shape=[2, 3, 4, 5], dtype="int32")
                paddle.nn.functional.dropout2d(x, data_format='CNHW')

            self.assertRaises(ValueError, test_dataformat)

class TestDropout2DCAPI(unittest.TestCase):
    def setUp(self):
        np.random.seed(123)
        self.places = [fluid.CPUPlace()]
        if core.is_compiled_with_cuda():
            self.places.append(fluid.CUDAPlace(0))

    def test_dygraph(self):
        for place in self.places:
            with fluid.dygraph.guard(place):
                input_np = np.random.random([2, 3, 4, 5]).astype("float32")
                result_np = input_np
                input = fluid.dygraph.to_variable(input_np)
                m = paddle.nn.Dropout2D(p=0.)
                m.eval()
                result = m(input)
                self.assertTrue(np.allclose(result.numpy(), result_np))

class TestDropout3DFAPI(unittest.TestCase):
    def setUp(self):
        np.random.seed(123)
        self.places = [fluid.CPUPlace()]
        if core.is_compiled_with_cuda():
            self.places.append(fluid.CUDAPlace(0))

    def check_static_result(self, place):
        with fluid.program_guard(fluid.Program(), fluid.Program()):
            input = fluid.data(
                name="input", shape=[2, 3, 4, 5, 6], dtype="float32")
            res1 = paddle.nn.functional.dropout3d(
                x=input, p=0., training=False, data_format='NCDHW')
            res2 = paddle.nn.functional.dropout3d(
                x=input, p=0., training=False, data_format='NDHWC')

            in_np = np.random.random([2, 3, 4, 5, 6]).astype("float32")
            res_np = in_np

            exe = fluid.Executor(place)
            res_list = [res1, res2]
            for res in res_list:
                fetches = exe.run(fluid.default_main_program(),
                                  feed={"input": in_np},
                                  fetch_list=[res])
                self.assertTrue(np.allclose(fetches[0], res_np))

    def test_static(self):
        for place in self.places:
            self.check_static_result(place=place)

    def test_dygraph(self):
        for place in self.places:
            with fluid.dygraph.guard(place):
                in_np = np.random.random([2, 3, 4, 5, 6]).astype("float32")
                res_np = in_np
                input = fluid.dygraph.to_variable(in_np)

                res1 = paddle.nn.functional.dropout3d(
                    x=input, p=0., training=False, data_format='NCDHW')
                res2 = paddle.nn.functional.dropout3d(
                    x=input, p=0., training=False, data_format='NDHWC')

            res_list = [res1, res2]
            for res in res_list:
                self.assertTrue(np.allclose(res.numpy(), res_np))

class TestDropout3DFAPIError(unittest.TestCase):
    def test_errors(self):
        with program_guard(Program(), Program()):

            def test_xdim():
                # dimentions of x should be 5
                x = fluid.data(name='x1', shape=[2, 3, 4, 5], dtype="int32")
                paddle.nn.functional.dropout3d(x)

            self.assertRaises(ValueError, test_xdim)

            def test_dataformat():
                # data_format should be 'NCDHW' or 'NDHWC'
                x = fluid.data(name='x2', shape=[2, 3, 4, 5, 6], dtype="int32")
                paddle.nn.functional.dropout3d(x, data_format='CNDHW')

            self.assertRaises(ValueError, test_dataformat)

class TestDropout3DCAPI(unittest.TestCase):
    def setUp(self):
        np.random.seed(123)
        self.places = [fluid.CPUPlace()]
        if core.is_compiled_with_cuda():
            self.places.append(fluid.CUDAPlace(0))

    def test_dygraph(self):
        for place in self.places:
            with fluid.dygraph.guard(place):
                input_np = np.random.random([2, 3, 4, 5, 6]).astype("float32")
                result_np = input_np
                input = fluid.dygraph.to_variable(input_np)
                m = paddle.nn.Dropout3D(p=0.)
                m.eval()
                result = m(input)
                self.assertTrue(np.allclose(result.numpy(), result_np))

class TestAlphaDropoutFAPI(unittest.TestCase):
    def setUp(self):
        np.random.seed(123)
        self.places = [fluid.CPUPlace()]
        if core.is_compiled_with_cuda():
            self.places.append(fluid.CUDAPlace(0))

    def check_static_result(self, place):
        with fluid.program_guard(fluid.Program(), fluid.Program()):
            input = fluid.data(name="input", shape=[40, 40], dtype="float32")
            res1 = paddle.nn.functional.alpha_dropout(x=input, p=0.)
            res2 = paddle.nn.functional.alpha_dropout(
                x=input, p=0., training=False)
            res3 = paddle.nn.functional.alpha_dropout(x=input, p=1.)

            in_np = np.random.random([40, 40]).astype("float32")
            res_np = in_np
            res_np3 = np.zeros_like(in_np)

            exe = fluid.Executor(place)
            res_list = [res1, res2]
            for res in res_list:
                fetches = exe.run(fluid.default_main_program(),
                                  feed={"input": in_np},
                                  fetch_list=[res])
                self.assertTrue(np.allclose(fetches[0], res_np))
            fetches = exe.run(fluid.default_main_program(),
                              feed={"input": in_np},
                              fetch_list=[res3])
            self.assertTrue(np.allclose(fetches[0], res_np3))

    def test_static(self):
        for place in self.places:
            self.check_static_result(place=place)

    def test_dygraph(self):
        for place in self.places:
            with fluid.dygraph.guard(place):
                in_np = np.random.random([40, 40]).astype("float32")
                res_np = in_np
                res_np3 = np.zeros_like(in_np)
                input = fluid.dygraph.to_variable(in_np)

                res1 = paddle.nn.functional.alpha_dropout(x=input, p=0.)
                res2 = paddle.nn.functional.alpha_dropout(
                    x=input, p=0., training=False)
                res3 = paddle.nn.functional.alpha_dropout(x=input, p=1.)

            res_list = [res1, res2]
            for res in res_list:
                self.assertTrue(np.allclose(res.numpy(), res_np))
            self.assertTrue(np.allclose(res3.numpy(), res_np3))

class TestAlphaDropoutFAPIError(unittest.TestCase):
    def test_errors(self):
        with program_guard(Program(), Program()):

            def test_Variable():
                # the input of dropout must be Variable.
                x1 = fluid.create_lod_tensor(
                    np.array([-1, 3, 5, 5]), [[1, 1, 1, 1]], fluid.CPUPlace())
                paddle.nn.functional.alpha_dropout(x1, p=0.5)

            self.assertRaises(TypeError, test_Variable)

            def test_dtype():
                # the input dtype of dropout must be float32 or float64
                xr = fluid.data(name='xr', shape=[3, 4, 5, 6], dtype="int32")
                paddle.nn.functional.alpha_dropout(xr)

            self.assertRaises(TypeError, test_dtype)

            def test_pdtype():
                # p should be int or float
                x2 = fluid.data(name='x2', shape=[3, 4, 5, 6], dtype="float32")
                paddle.nn.functional.alpha_dropout(x2, p='0.5')

            self.assertRaises(TypeError, test_pdtype)

            def test_pvalue():
                # p should be 0.<=p<=1.
                x2 = fluid.data(name='x2', shape=[3, 4, 5, 6], dtype="float32")
                paddle.nn.functional.alpha_dropout(x2, p=1.2)

            self.assertRaises(ValueError, test_pvalue)

class TestAlphaDropoutCAPI(unittest.TestCase):
    def setUp(self):
        np.random.seed(123)
        self.places = [fluid.CPUPlace()]
        if core.is_compiled_with_cuda():
            self.places.append(fluid.CUDAPlace(0))

    def test_dygraph(self):
        for place in self.places:
            with fluid.dygraph.guard(place):
                input_np = np.random.random([40, 40]).astype("float32")
                result_np = input_np
                input = fluid.dygraph.to_variable(input_np)
                m = paddle.nn.AlphaDropout(p=0.)
                m.eval()
                result = m(input)
                self.assertTrue(np.allclose(result.numpy(), result_np))

if __name__ == '__main__':
    unittest.main()


# python src/chapter3/chapter3_2.py
# python3 src/chapter3/chapter3_2.py
from __future__ import division, absolute_import, print_function
import sys
import math

from copy import copy
from copy import deepcopy

import numpy as np
from numpy import arange

from matplotlib.pyplot import plot
from matplotlib.pyplot import figure
from matplotlib.pyplot import show

class Chapter3_2:
    '''
    CLRS 第三章 3.2 算法函数和笔记
    '''
    def note(self):
        '''
        Summary
        =
        Print chapter3.2 note

        Example
        =
        >>> Chapter3_2().note()
        '''
        print('3.2 标准记号和常用函数')
        print('单调性：一个函数f(n)是单调递增的，若m<=n,则有f(m)<=f(n)，反之单调递减，将小于等于号换成小于号，即变为严格不等式，则函数是严格单调递增的')
        print('下取整(floor)和上取整(ceiling)')
        print('取模运算(modular arithmetic)')
        print('多项式定义及其性质')
        print('指数式定义及其性质')
        print('任何底大于1的指数函数比任何多项式函数增长得更快')
        print('对数定义及其性质')
        print('阶乘定义及其性质')
        print('计算机工作者常常认为对数的底取2最自然，因为很多算法和数据结构都涉及到对问题进行二分')
        print('任意正的多项式函数都比多项对数函数增长得快')
        print('斯特林近似公式：n!=sqrt(2*pi*n)*(n/e)^n*(1+Θ(1/n))')
        print('阶乘函数的一个更紧确的上界和下界：')
        print('n!=o(n^n) n!=ω(2^n) lg(n!)=Θ(nlgn)')
        print('函数迭代的定义和性质')
        print('多重对数函数：用记号lg * n(读作n的log星)来表示多重对数，定义为lg * n=min(i>=0;lg^(i)n<=1)')
        print('多重函数是一种增长很慢的函数')
        print('lg * 2 = 1; lg * 4 = 2; lg * 16 = 3; lg * 65536 = 4; lg * 2^65536 = 5')
        print('宇宙中可以观察到的原子数目估计约有10^80，远远小于2^65536,因此很少会遇到一个使lg * n > 5的一个n输入规模')
        print('斐波那契数列：F0 = 0 F1 = 1 F(i) = F(i-1) + F(i-2),产生的序列为0,1,1,2,3,5,8,13,21,34,55,……')
        print('斐波那契数列和黄金分割率φ以及共轭有关系')
        print('φ=((1+sqrt(5))/2=1.61803 和它的共轭(1-sqrt(5))/2=-0.61803)')
        print('练习题和思考题略')
        # python src/chapter3/chapter3_2.py
        # python3 src/chapter3/chapter3_2.py
        return self

if __name__ == '__main__':
    print('Run main : single chapter three!')
    Chapter3_2().note()
else:
    pass


"""
Tests for opds.py
"""
import os

from astropy.io import fits
import astropy.units as u
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pysiaf
import pytest
import webbpsf

def test_enable_adjustable_ote():
    """ Some basic tests of the OTE LOM"""
    nc = webbpsf.NIRCam()
    nc, ote = webbpsf.enable_adjustable_ote(nc)

    # did this produce an OTE object?
    assert isinstance(ote, webbpsf.opds.OTE_Linear_Model_WSS), "Didn't get an OTE object back"

    # can we compute the rms?
    rms = ote.rms()

    # and can we move a mirror?

    ote.move_seg_local('B1', piston=10, clocking=200)

    assert ote.segment_state[6, 2] == 10, "Couldn't piston"
    assert ote.segment_state[6, 3] == 200, "Couldn't clock"

    # did that misalignment make things much worse?

    assert ote.rms() > rms*10, "Huge piston offset didn't make the WFE much worse"

# The following "truth" values" are based off the global focus and A1 Hexike coeffs
#   that will be returned for some time after a maximum slew using the time
#   constants and amplitudes established by the model in Fall 2018 (last updates
#   to otelm/thermal_OPD_fitting_parameters_9H_um.fits)
# Random scaling factor used below
SCALING_FACTOR = 0.5
# Coefficients for SM based on 1 day after maximum slew, first with no scaling
#   factor, and second with a scaling factor as specified above, all predicted with
#   above file. The below truth values are in units of METERS
GLOBAL_FOCUS = [-1.8251043541410904e-08]
GLOBAL_FOCUS2 = [GLOBAL_FOCUS[0] * SCALING_FACTOR]
# Coeffcients for A1 based on 1 day after maximum slew with no scaling predicted
#   using above file
COEFFS_A1 = np.array([-3.52633363e-09, -2.90050902e-09, 1.25432196e-09, -7.43319098e-12,
                      -5.82462948e-11, -1.27115922e-10, -1.91541104e-12, 3.64760396e-11,
                      4.97176630e-13])
# Coeffcients for A4 based on 5 hours after maximum slew with no scaling,
#   start_angle=5. and end_angle=15., predicted using above file
# Updated on 9/18/2020
COEFFS_A4 = np.array([ 3.89238932e-10,  1.80333109e-10,  1.18632814e-10,  4.42108030e-13,
                      -3.22871622e-11,  6.93619028e-12, -1.08202005e-13,  1.15018494e-12,
                      5.25664635e-14])
# Default slew angles
START_ANGLE = -5.
END_ANGLE = 45.

# Parameters to test the thermal model
tm_parameters = ([1 * u.day, 'SM', None, START_ANGLE, END_ANGLE, GLOBAL_FOCUS],
                 [1 * u.day, 'SM', SCALING_FACTOR, START_ANGLE, END_ANGLE, GLOBAL_FOCUS2],
                 [24, 'SM', None, START_ANGLE, END_ANGLE, GLOBAL_FOCUS],
                 [0.0 * u.day, 'SM', None, START_ANGLE, END_ANGLE, [0.0]],
                 [1 * u.day, 'A1', None, START_ANGLE, END_ANGLE, COEFFS_A1],
                 [0.0 * u.day, 'A1', None, START_ANGLE, END_ANGLE, np.zeros(9)],
                 [1.0 * u.day, 'D1', None, START_ANGLE, END_ANGLE, [0.0]],
                 [5 * u.hour, 'A4', None, 5., 15., COEFFS_A4])
@pytest.mark.parametrize('time, seg, scaling, start_angle, end_angle, coeff_truth', tm_parameters)
def test_get_thermal_slew_coeffs(time, seg, scaling, start_angle, end_angle,
                                 coeff_truth):
    """ Test that the OTE Thermal model is outputting the correct values
    These tests will go through the following (in order as listed in
    thermal_model_parameters):

     1. Test for SM with defaults
     2. Test for SM with a scaling factor
     3. Test for SM if no units specified for delta_time
     4. Test for SM with no delta_time
     5. Test for PM segment with defaults
     6. Test for PM segment with no delta_time
     7. Test for PM segment that is not in list of segnames
     8. Test for PM segment with start and end angles
    """
    delta_time = time
    # Create the thermal model
    otelm = webbpsf.opds.OTE_Linear_Model_WSS()
    otelm.thermal_slew(delta_time, start_angle, end_angle, scaling, case='EOL')
    coeffs = otelm._get_thermal_slew_coeffs(segid=seg)
    # Pull out coefficients
    if isinstance(coeffs, float):
        coeffs = [coeffs]
    # Assert the coefficents
    for coeff, truth in zip(coeffs, coeff_truth):
        #assert np.round(coeff, decimals=4) == np.round(truth, decimals=4)
        coeff /= 1e-9 # Convert to nm so we are not dealing with such small numbers
        truth /= 1e-9 # Convert to nm so we are not dealing with such small numbers
        assert np.isclose(coeff, truth), "Coeffs do not match expected value after day slew."

def test_thermal_slew_partial_angle():
    """ total slew shoudl give same total amplitude if broken into smaller jumps"""

    otelm = webbpsf.opds.OTE_Linear_Model_WSS()

    start_angle = -5
    mid_angle = 20
    end_angle = 45

    delta_time = 1 * u.hr

    # One large slew
    otelm.thermal_slew(delta_time, start_angle, end_angle, case='EOL')
    cf_full = np.array([otelm._get_thermal_slew_coeffs(segid=seg) for seg in otelm.segnames[0:18]])

    # Small slew 1
    otelm.thermal_slew(delta_time, start_angle, mid_angle, case='EOL')
    cf_all1 = np.array([otelm._get_thermal_slew_coeffs(segid=seg) for seg in otelm.segnames[0:18]])

    # Small slew 2
    otelm.thermal_slew(delta_time, mid_angle, end_angle, case='EOL')
    cf_all2 = np.array([otelm._get_thermal_slew_coeffs(segid=seg) for seg in otelm.segnames[0:18]])
    cf_tot = cf_all1 + cf_all2

    # Multiply by 1E9 so we're not dealing with small numbers
    assert np.allclose(1e9*cf_full, 1e9*cf_tot), "should get same total coefficients for one big slew or if broken into two parts"

def test_thermal_slew_update_opd():
    ''' Test that running webbpsf.opds.OTE_Linear_Model_WSS.thermal_slew() will
        give the expected output

        '''
    otelm = webbpsf.opds.OTE_Linear_Model_WSS()
    otelm.thermal_slew(delta_time=1.0*u.day, case='EOL')

    # the exact value expected is affected by which version of the linear model is used.
    if otelm._segment_masks_version < 3:
        # rev V pupil segment mask file, labeled as VERSION=2 in jwpupil_segments.fits
        expected_max = 41.3338  # nanometers, expected value for peak.
                                # value derived by kjbrooks based on thermal model coefficients
        expected_rms = 11.13    # nm
                                # value derived by mperrin based on evaluation of opd map in this case
    else:
        # rev W pupil segment mask file, labeled as VERSION=3 in jwpupil_segments.fits
        # Values here are by mperrin based on evaluation of the same exact linear model code as above
        # changing only the data file $WEBBPSF_DATA/jwpupil_segments.fits to the newer version
        expected_max = 40.7763  # nanometers, expected value for peak
        expected_rms = 11.24    # nm
    assert np.isclose(np.max(otelm.opd)/1e-9, expected_max, rtol=1e-3), "OPD max does not match expected value after 1 day slew."
    assert np.isclose(otelm.rms(), expected_rms, rtol=1e-3), "OPD rms does not match expected value after 1 day slew."

def test_thermal_slew_reproducibility():
    """ If you call the thermal slew model multiple times, the OPD values should depend
    only on the LAST set of function call parameters. Not on the full time history.

    See issue #338
    """
    ote = webbpsf.opds.OTE_Linear_Model_WSS()

    ote.thermal_slew(12*u.hour, start_angle=-5, end_angle=45, case='EOL')
    opd1 = ote.opd.copy()

    ote.thermal_slew(24*u.hour, start_angle=-5, end_angle=45, case='EOL')
    opd2 = ote.opd.copy()

    ote.thermal_slew(12*u.hour, start_angle=-5, end_angle=45, case='EOL')
    opd3 = ote.opd.copy()

    assert np.allclose(opd1, opd2)==False, "OPDs expected to differ didn't"
    assert np.allclose(opd1, opd3), "OPDs expected to match didn't"

def test_update_opd():
    ''' The start of what should be many tests of this function'''

    # Test the very basics
    ote = webbpsf.opds.OTE_Linear_Model_WSS()
    ote.update_opd()
    assert np.max(ote.opd) == 0.0

    # can we add a deterministic frill drift?
    requested_wfe = 5
    ote.apply_frill_drift(requested_wfe)
    assert np.allclose(ote.rms(), requested_wfe, rtol=0.1), "Frill WFE amplitude not as expected"
    ote.apply_frill_drift(0.0)

    # can we add a deterministic IEC drift?
    requested_wfe = 15
    ote.apply_iec_drift(requested_wfe)
    assert np.allclose(ote.rms(), requested_wfe, rtol=0.1), "IEC WFE amplitude not as expected"

    # Todo test random drifts

def test_move_sur(plot=False):
    """ Test we can move mirrors using Segment Update Requests
    """
    import webbpsf
    import os
    import glob
    surdir = os.path.join(webbpsf.__path__[0], 'tests', 'surs')
    surs = glob.glob(surdir+'/*sur.xml')

    nrc = webbpsf.NIRCam()
    nrc.filter='F212N'
    nrc, ote = webbpsf.enable_adjustable_ote(nrc)
    ote.zero(zero_original=True)

    for s in surs:
        print("Testing "+s)
        ote.reset()
        ote.move_sur(s)
        # the coarse phasing SUR is a no-op after 3 groups; all others have some effect
        if 'coarse_phasing' not in s:
            assert not np.allclose(ote.segment_state, 0), "Expected some segments to be moved"

        ote.move_sur(s, reverse=True)
        assert np.allclose(ote.segment_state, 0), "Reversing moves didn't bring us back to zero"

    # Test every DOF on A1-1 and SM and check the OTE state updated accordingly
    s = glob.glob(surdir+'/example_alldof_A1-SM_sur.xml')[0]
    print("Testing "+s)
    ote.reset()
    ote.move_sur(s)
    assert np.allclose(ote.segment_state[0],  [1, 2, 3, 4, 5, 6])
    assert np.allclose(ote.segment_state[-1], [1, 2, 3, 4, 5, 0])

    # Test moving one at a time. This test relies on specifics of what's in the image stacking SUR.
    s = glob.glob(surdir+'/example_image_stacking*sur.xml')[0]
    print("Testing moving one group at a time with "+s)
    ote.reset()
    sur = webbpsf.surs.SUR(s)

    ngroups = len(sur.groups)
    oldstate = ote.segment_state.copy()

    for igrp in range(1, ngroups+1):
        print("Group {} should move segment {}".format(igrp, 2*igrp+6))
        ote.move_sur(s, group=igrp)

        movedsegs = np.abs((ote.segment_state - oldstate).sum(axis=1))
        assert (movedsegs!=0).sum()==1, "Only expected one segment to move"
        whichmoved = np.argmax(movedsegs)+1
        print ("Moved segment", whichmoved)
        assert whichmoved == 2*igrp+6, "An unexpected segment moved"
        oldstate = ote.segment_state.copy()
        if plot:
            psf = nrc.calc_psf(fov_pixels=256, add_distortion=False)
            plt.figure()
            ote.display_opd(title="After Group {}".format(igrp))
            plt.figure()
            webbpsf.display_psf(psf, ext=1, title="After Group {}".format(igrp))

def test_single_seg_psf(segmentid=1):
    """Test calculation of a single segment PSF, including options to remove piston/tip/tilt as used by MIRAGE

    """

    nrc = webbpsf.NIRCam()
    nrc.filter = 'F212N'
    nrc, ote = webbpsf.enable_adjustable_ote(nrc)
    ote.zero(zero_original=True)

    segname = webbpsf.constants.SEGNAMES_WSS_ORDER[segmentid-1][0:2]

    ote.move_seg_local(segname, xtilt=1, piston=-1)

    pupil = webbpsf.webbpsf_core.one_segment_pupil(segmentid)
    ote.amplitude = pupil[0].data

    psf = nrc.calc_psf(nlambda=1)

    ote.remove_piston = True
    ote.update_opd()
    psf_rm_piston = nrc.calc_psf(nlambda=1)
    assert np.allclose(psf[0].data, psf_rm_piston[0].data), "Piston removal should not affect the overall PSF"

    assert np.allclose( webbpsf.measure_centroid(psf), webbpsf.measure_centroid(psf_rm_piston)), "centroid should not shift"

    ote.remove_piston_tip_tilt = True
    ote.update_opd()
    psf_rm_ptt = nrc.calc_psf(nlambda=1)
    assert not np.allclose(psf[0].data, psf_rm_ptt[0].data), "Piston/Tip/Tip removal should shift the overall PSF"
    assert np.abs(webbpsf.measure_centroid(psf)[0] - webbpsf.measure_centroid(psf_rm_ptt)[0]) > 40, "centroid should shift susbtantially with/without tip/tilt removal"

def test_get_zernike_coeffs_from_smif():
    """
    Test that the OTE SM Influence function returns expected Hexike coefficients.
    """

    # Create an instance of the OTE linear model
    otelm = webbpsf.opds.OTE_Linear_Model_WSS()

    # Case 1: otelm.v2v3 is None, should return None
    otelm._apply_sm_field_dependence_model()
    assert ( otelm._apply_sm_field_dependence_model() is None)

    # Case 2: check coefficient at control point; should return zeros.
    assert( np.allclose(otelm._get_zernike_coeffs_from_smif(0., 0.), np.asarray([0.]*9) ))

    # Case 3: dx=1, dy=1, SM Poses all equal to 1 um
    telfer_zern = [-0.055279643, -0.037571947, -0.80840763, -0.035680581, -0.0036747300, 0.0033910640] # Taken from Telfer's tool
    # Convert Telfer's Zernikes to Hexikes:
    hexikes = [-telfer_zern[1],
               2.*telfer_zern[0] - (60984./69531.)*telfer_zern[5],
               telfer_zern[2],
               (33./25)*telfer_zern[3],
               (-33./25)*telfer_zern[4],
               (1386./860.)*telfer_zern[5]]

    otelm.segment_state[-1, :] = 1.0

    assert (np.allclose(otelm._get_zernike_coeffs_from_smif(1.0, 1.0)[3:], hexikes, rtol=1e-3))

    # Case 4: test at MIRIM_FP1MIMF field point
    otelm.ote_ctrl_pt = pysiaf.Siaf('NIRCAM')['NRCA3_FP1'].reference_point('tel') *u.arcsec
    otelm.v2v3 = pysiaf.Siaf('MIRI')['MIRIM_FP1MIMF'].reference_point('tel') *u.arcsec
    telfer_zern_mirim_fp1mimf = np.asarray( [-0.25066019, 0.22840080, -0.53545999, -0.024227464, -0.0025191352, 0.00050082553]) # Taken from Telfer's tool
    # Convert Telfer's Zernikes to Hexikes:
    hexikes = hexikes = [-telfer_zern_mirim_fp1mimf[1],
                         2.*telfer_zern_mirim_fp1mimf[0] - (60984./69531.)*telfer_zern_mirim_fp1mimf[5],
                         telfer_zern_mirim_fp1mimf[2],
                         (33./25)*telfer_zern_mirim_fp1mimf[3],
                         (-33./25)*telfer_zern_mirim_fp1mimf[4],
                         (1386./860.)*telfer_zern_mirim_fp1mimf[5]]

    otelm.segment_state[-1, :] = [300., 400., 100., 200., 5., 0.]
    dx =-(otelm.v2v3[0] - otelm.ote_ctrl_pt[0]).to(u.rad).value
    dy = (otelm.v2v3[1] - otelm.ote_ctrl_pt[1]).to(u.rad).value

    assert (np.allclose(otelm._get_zernike_coeffs_from_smif(dx, dy)[3:], hexikes, rtol=1e-3))

def test_segment_tilt_signs(fov_pix = 50, plot=False, npix=1024):
    """Test that segments move in the direction expected when tilted.

    The local coordinate systems are non-obvious, to say the least. This verifies
    sign conventions and coordinates are consistent in the linear optical model and
    optical propagation code.

    """

    if plot:
        fig, axs = plt.subplots(3, 5, figsize=(14,9))#, sharex = True, sharey = True)

    nrc = webbpsf.NIRCam()

    ote = webbpsf.opds.OTE_Linear_Model_WSS(npix=npix)
    nrc.include_si_wfe = False # not relevant for this test

    tilt = 1.0

	# We im for relatively minimalist PSF calcs, to reduce test runtime
    psf_kwargs = {'monochromatic': 2e-6,
                  'fov_pixels': fov_pix,
                  'oversample': 1,
                  'add_distortion': False}

    # Which way are things expected to move?
    #
    # A1:  +X rotation -> -Y pixels (DMS), +Y rotation -> -X pixels
    # B1: +X rotation -> +Y pixels, +Y rotation -> +X pixels
    # C1: +X rotation -> +X/+Y pixels, +Y rotation -> -Y/+X pixels
    # (for C1, A/B means A is the sqrt(3)/2 component, B is the 1/2 component)
    #
    # The above derived from Code V models by R. Telfer, subsequently cross checked by Perrin

    for i, iseg in enumerate(['A1', 'B1', 'C1']):
        ote.zero()

        pupil = webbpsf.webbpsf_core.one_segment_pupil(iseg, npix=npix)

        ote.amplitude = pupil[0].data
        nrc.pupil = ote

        # CENTERED PSF:
        psf = nrc.calc_psf(**psf_kwargs)
        cen_ref = webbpsf.measure_centroid(psf, boxsize=10, threshold=1)

        ote.move_seg_local(iseg, xtilt=tilt)
        # XTILT PSF:
        psfx = nrc.calc_psf(**psf_kwargs)
        cen_xtilt = webbpsf.measure_centroid(psfx, boxsize=10, threshold=1)

        if iseg.startswith("A"):
            assert cen_xtilt[0] < cen_ref[0], "Expected A1:  +X rotation -> -Y pixels (DMS coords)"
            assert np.isclose(cen_xtilt[1], cen_ref[1], atol=1), "Expected A1:  +X rotation -> no change in X"
        elif iseg.startswith("B"):
            assert cen_xtilt[0] > cen_ref[0], "Expected B1: +X rotation -> +Y pixels (DMS coords)"
            assert np.isclose(cen_xtilt[1], cen_ref[1], atol=1), "Expected B1:  +X rotation -> no change in Y"
        elif iseg.startswith("C"):
            assert cen_xtilt[0] > cen_ref[0], "Expected C1: +X rotation -> +X/+Y pixels"
            assert cen_xtilt[1] > cen_ref[1], "Expected C1: +X rotation -> +X/+Y pixels"

        if plot:
            axs[i, 0].imshow(psf[0].data, norm=matplotlib.colors.LogNorm(vmax=1e-2, vmin=1e-5), origin="lower")
            axs[i, 0].set_title(iseg+": centered")
            axs[i, 0].axhline(y=fov_pix/2)
            axs[i, 0].axvline(x=fov_pix/2)
            # PLOT RESULTING OPD:
            im = axs[i, 1].imshow(ote.opd, vmin=-4e-6, vmax=4e-6, origin="lower")
            axs[i, 1].set_title("OPD (yellow +)")
            axs[i, 2].imshow(psfx[0].data, norm=matplotlib.colors.LogNorm(vmax=1e-2, vmin=1e-5), origin="lower")
            axs[i, 2].set_title(iseg+": xtilt {} um".format(tilt))
            axs[i, 2].axhline(y=fov_pix/2)
            axs[i, 2].axvline(x=fov_pix/2)

        ote.zero()
        ote.move_seg_local(iseg, ytilt=tilt)
        # YTILT PSF:
        psfy = nrc.calc_psf(**psf_kwargs)
        cen_ytilt = webbpsf.measure_centroid(psfy, boxsize=10, threshold=1)

        if iseg.startswith("A"):
            assert cen_ytilt[1] < cen_ref[1], "Expected A1:  +Y rotation -> -X pixels (DMS coords)"
            assert np.isclose(cen_ytilt[0], cen_ref[0], atol=1), "Expected A1:  +Y rotation -> no change in Y"
        elif iseg.startswith("B"):
            assert cen_ytilt[0] > cen_ref[0], "Expected B1: +Y rotation -> +X pixels(DMS coords)"
            assert np.isclose(cen_ytilt[0], cen_ref[0], atol=1), "Expected B1:  +Y rotation -> no change in Y"
        elif iseg.startswith("C"):
            assert cen_ytilt[0] < cen_ref[0], "Expected C1: +Y rotation -> -Y/+X pixels"
            assert cen_ytilt[1] > cen_ref[1], "Expected C1: +Y rotation -> -Y/+X pixels"

        # PLOT RESULTING OPD:
        if plot:
            im = axs[i, 3].imshow(ote.opd, vmin=-4e-6, vmax=4e-6, origin="lower")
            axs[i, 3].set_title("OPD (yellow +)")
            axs[i, 4].imshow(psfy[0].data, norm=matplotlib.colors.LogNorm(vmax=1e-2, vmin=1e-5), origin="lower")
            axs[i, 4].set_title(iseg+": ytilt {} um".format(tilt))
            axs[i, 4].axhline(y=fov_pix/2)
            axs[i, 4].axvline(x=fov_pix/2)

def test_segment_tilt_signs_2048npix():
    """ Re-run same test as above, but with a different value for npix

    This verifies the LOM works as expected for a size other than 1024 pixels
    """
    test_segment_tilt_signs(npix=2048)

def test_changing_npix():
    '''
    Test that using different npix will result in same PSF
    '''
    # Create a NIRCam instance using the default npix=1024
    nircam_1024 = webbpsf.NIRCam()
    nircam_1024.pupilopd = None # Set to none so I don't have to worry about making new OPDs
    psf_1024 = nircam_1024.calc_psf(oversample=2, nlambda=1, add_distortion=False)

    # Create a NIRCam instance using npix=2048
    npix = 2048
    nircam_2048 = webbpsf.NIRCam()
    nircam_2048.pupil = os.path.join(webbpsf.utils.get_webbpsf_data_path(),
                                     f'jwst_pupil_RevW_npix{npix}.fits.gz')
    nircam_2048.pupilopd = None # Set to none so I don't have to worry about making new OPDs
    psf_2048 = nircam_2048.calc_psf(oversample=2, nlambda=1, add_distortion=False)

    # Let's check individual pixel values, at least where the PSF is not too dim.
    # Check all pixels which have > 1e-6 of the total flux (we can safely ignore pixels with very low intensity)
    mask = psf_1024[0].data>1e-6
    assert np.allclose(psf_1024[0].data[mask], psf_2048[0].data[mask], rtol=0.01), 'Pixel values differ by more than 1%'

    # Let's check that the total flux in the PSF does not change much.
    #  (A small amount is acceptable and not surprising, since higher resolution improves the fidelity at which
    #   we model light that is scattered by segment edges to very wide angles outside of the simulated PSF FOV)
    assert np.isclose(psf_1024[0].data.sum(), psf_2048[0].data.sum(), rtol=0.005), "PSF total flux should not change much"

    # Let's also check a derived property of the whole PSF: the FWHM.
    # The FWHM should be very close to identical for the two PSFs.
    assert np.isclose(webbpsf.measure_fwhm(psf_1024), webbpsf.measure_fwhm(psf_2048), rtol=0.0001), "PSF FWHM should not vary for different npix"

import numpy,import astropy,from astropy


#!/usr/bin/env python
# coding: utf-8

# # VacationPy
# ----
#
# #### Note
# * Keep an eye on your API usage. Use https://developers.google.com/maps/reporting/gmp-reporting as reference for how to monitor your usage and billing.
#
# * Instructions have been included for each segment. You do not have to follow them exactly, but they are included to help you think through the steps.

# In[2]:

get_ipython().system(' pip3 install gmaps')

# In[19]:

get_ipython().system(' jupyter nbextension enable --py gmaps')

# In[20]:

# Dependencies and Setup
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import requests
import gmaps
import os

# Import API key
from api_keys import g_key

# In[ ]:

# ### Store Part I results into DataFrame
# * Load the csv exported in Part I to a DataFrame

# In[24]:

city_data_df = pd.read_csv('output_data/cities.csv')
city_data_df.head()

# ### Humidity Heatmap
# * Configure gmaps.
# * Use the Lat and Lng as locations and Humidity as the weight.
# * Add Heatmap layer to map.

# In[25]:

#config gmaps
gmaps.configure(api_key=g_key)

# In[26]:

#heatmap of humidity
locations = city_data_df[['Lat', 'Lng']]
humidity = city_data_df['Humidity']
fig = gmaps.figure()
heat_layer = gmaps.heatmap_layer(locations, weights=humidity, dissipating=False, max_intensity=300, point_radius=5)

fig.add_layer(heat_layer)
fig

# ### Create new DataFrame fitting weather criteria
# * Narrow down the cities to fit weather conditions.
# * Drop any rows will null values.

# In[32]:

#Narrow down the cities to fit weather conditions and Drop any rows will null values
narrowed_city_df = city_data_df.loc[(city_data_df['Max Temp'] < 80) & (city_data_df['Max Temp'] > 70)                                     & (city_data_df['Wind Speed'] < 10)                                     & (city_data_df['Cloudiness'] == 0)]
narrowed_city_df.dropna()

narrowed_city_df

# ### Hotel Map
# * Store into variable named `hotel_df`.
# * Add a "Hotel Name" column to the DataFrame.
# * Set parameters to search for hotels with 5000 meters.
# * Hit the Google Places API for each city's coordinates.
# * Store the first Hotel result into the DataFrame.
# * Plot markers on top of the heatmap.

# In[33]:

# Create DataFrame named hotel_df and Add a "Hotel Name" column to the DataFrame
hotel_df = narrowed_city_df[['City', 'Country', 'Lat', 'Lng']].copy()
hotel_df['Hotel Name'] = ""
hotel_df

# In[38]:

params = {
    'radius': 5000,
    'types': 'lodging',
    'key': g_key

}

for index, row in hotel_df.iterrows():
    #get latitude and longitude
    lat = row['Lat']
    lng = row['Lng']

    params['location'] = f"{lat},{lng}"

    #use the search term 'hotel' and produce the lat/lng
    base_url = 'https://maps.googleapis.com/maps/api/place/nearbysearch/json'

    name_address = requests.get(base_url, params=params).json()

    try:
        hotel_df.loc[index, 'Hotel Name'] = name_address['results'][0]['name']
    except(KeyError, IndexError):
        print('Missing field/result... skipping...')

hotel_df

# In[39]:

# NOTE: Do not change any of the code in this cell

# Using the template add the hotel marks to the heatmap
info_box_template = """

Name

{Hotel Name}

City

{City}

Country

{Country}

"""
# Store the DataFrame Row
# NOTE: be sure to update with your DataFrame name
hotel_info = [info_box_template.format(**row) for index, row in hotel_df.iterrows()]
locations = hotel_df[["Lat", "Lng"]]

# In[40]:

# Add marker layer ontop of heat map
marker_layer = gmaps.marker_layer(locations, info_box_content=hotel_info)

fig.add_layer(marker_layer)

# Display figure
fig

# In[ ]:


"""
HeteroCL Compute APIs
=====================

**Author**: Yi-Hsiang Lai (seanlatias@github)

In this tutorial, we will show more HeteroCL compute APIs. These APIs are used
to build the algorithm. Note that in HeteroCL, the compute APIs can be used
along with the imperative DSL.
"""

import heterocl as hcl

##############################################################################
# ``hcl.compute``
# ---------------
# We have introduced this API before. This API returns a **new tensor** whose
# values are defined in an elementwise fashion. Following we show the API's
# prototype.
#
# ``compute(shape, fcompute, name, dtype)``
#
# ``shape`` defines the shape of the output tensor. ``fcompute`` is a lambda
# function that describes the elementwise definition. ``name`` and ``dtype``
# are optional. We show an example below.

hcl.init()

A = hcl.placeholder((10,), "A")
B = hcl.placeholder((10,), "B")

def compute_example(A, B):
    return hcl.compute(A.shape, lambda x: A[x]+B[x], "C")

s = hcl.create_schedule([A, B], compute_example)
print(hcl.lower(s))

##############################################################################
# ``hcl.update``
# --------------
# This API is similar to `hcl.compute` in that it defines how you **update a
# tensor** in an elementwise fashion. Note that this API does not return a
# new tensor. More specifically, the return value is `None`.
#
# ``hcl.update(tensor, fupdate, name)``
#
# ``tensor`` is the tensor we want ot update. ``fupate`` is a lambda function
# that describes the elelmentwise update behavior. ``name`` is optional. We
# show an example below that does the similar computation as `compute_example`.
# The difference is that instead of returning a new tensor `C`, we send it in
# as an input and update it in place. We can see that the generated IR is
# almost the same.

hcl.init()
A = hcl.placeholder((10,), "A")
B = hcl.placeholder((10,), "B")
C = hcl.placeholder((10,), "C")

def update_example(A, B, C):
    hcl.update(C, lambda x: A[x]+B[x], "U")

s = hcl.create_schedule([A, B, C], update_example)
print(hcl.lower(s))

##############################################################################
# ``hcl.mutate``
# -------------------
# This API allows users to describe any loops with vector code, even if the
# loop body does not have any common pattern or contains imperative DSL.
# This API is useful when we want to perform optimization.
#
# ``hcl.mutate(domain, fbody, name)``
#
# ``domain`` describes the iteration domain of our original `for` loop.
# ``fbody`` is the body statement of the `for` loop. ``name`` is optional. We
# can describe the same computation in the previous two examples using this
# API.

hcl.init()
A = hcl.placeholder((10,), "A")
B = hcl.placeholder((10,), "B")
C = hcl.placeholder((10,), "C")

def mut_example(A, B, C):
    def loop_body(x):
        C[x] = A[x] + B[x]
    hcl.mutate((10,), lambda x: loop_body(x), "M")

s = hcl.create_schedule([A, B, C], mut_example)
print(hcl.lower(s))

##############################################################################
# Note that in this example, we are not allowed to directly write the
# assignment statement inside the lambda function. This is forbidden by Python
# syntax rules.
#
# Combine Imperative DSL with Compute APIs
# ----------------------------------------
# HeteroCL allows users to write a mixed-paradigm programming application.
# This is common when performing reduction operations. Although HeteroCL
# provides APIs for simple reduction operations such as summation and finding
# the maximum number, for more complexed reduction operations such as sorting,
# we need to describe them manually. Following we show an example of finding
# the maximum two values in a tensor.

hcl.init()
A = hcl.placeholder((10,), "A")
M = hcl.placeholder((2,), "M")

def find_max_two(A, M):
    def loop_body(x):
        with hcl.if_(A[x] > M[0]):
            with hcl.if_(A[x] > M[1]):
                M[0] = M[1]
                M[1] = A[x]
            with hcl.else_():
                M[0] = A[x]
    hcl.mutate(A.shape, lambda x: loop_body(x))

s = hcl.create_schedule([A, M], find_max_two)
f = hcl.build(s)

import numpy as np

hcl_A = hcl.asarray(np.random.randint(50, size=(10,)))
hcl_M = hcl.asarray(np.array([-1, -1]))

f(hcl_A, hcl_M)

np_A = hcl_A.asnumpy()
np_M = hcl_M.asnumpy()

print(np_A)
print(np_M)

assert np.array_equal(np_M, np.sort(np_A)[-2:])


from abc import ABC, abstractmethod
import numpy as np

class Regularizer(ABC):
    @abstractmethod
    def regularize(self, weights: np.ndarray) -> None:
        pass