import time
import pandas as pd
import numpy as np

CITY_DATA = { 'chicago': 'chicago.csv',
              'new york city': 'new_york_city.csv',
              'washington': 'washington.csv' }

def get_filters():
    """
    Asks user to specify a city, month, and day to analyze.

    Returns:
        (str) city - name of the city to analyze
        (str) month - name of the month to filter by, or "all" to apply no month filter
        (str) day - name of the day of week to filter by, or "all" to apply no day filter
    """
    print('Hello! Let\'s explore some US bikeshare data!')
    # TO DO: get user input for city (chicago, new york city, washington). HINT: Use a while loop to handle invalid inputs
    city = input ('can you please choose a city from chicago, new york city, washington ')
    while city not in (CITY_DATA.keys()):
           print('please choose correct city name')
           City = input ('can you please choose a city from chicago, new york city, washington ').lower()
    # TO DO: get user input for month (all, january, february, ... , june)
    while True:
           month = input('can you please choose a  month from january to june, or type "all" to desplay all months :').lower()
           months = ['january','february','march','april','may','june']
           if month != "all" and month not in months:
                  print("please choose correct Month")
           else:
                   break
    # TO DO: get user input for day of week (all, monday, tuesday, ... sunday)
    while True:
           day = input('can you please choose from days in the week, or type "all"  to desplay all days:' ).lower()
           days = ['monday','tuesday','wednesday','thursday','friday','saturday','sunday']

           if day != 'all' and day not in days:
                  print("please choose correct day")
           else:
                   break
    print('-'*40)
    return city,month,day
city, month, day = get_filters()

def load_data(city, month, day):
    """
    Loads data for the specified city and filters by month and day if applicable.

    Args:
        (str) city - name of the city to analyze
        (str) month - name of the month to filter by, or "all" to apply no month filter
        (str) day - name of the day of week to filter by, or "all" to apply no day filter
    Returns:
        df - Pandas DataFrame containing city data filtered by month and day
    """
    #load data file into a dataframe
    df = pd.read_csv(CITY_DATA[city])
    #convert start time to datatime
    df['start time'] = pd.to_datetime(df['start time'])
    #extract month and day of week
    df['month'] = df['start time'].dt.month
    df['day_of_week'] = df['start time'].dt.day_name()
    #filter
    if month != "all":
           months = ['january','february','march','april','may','june']
           month = month.index(month) + 1
     #creat new datafram for months
    df = df[df['month'] == month]
     #creat new datafram for day of week
    if day != 'all':
            df = df[df['day_of_week'] == day.title()]
    return df
load_data(city, month, day)
def time_stats(df):

    """Displays statistics on the most frequent times of travel."""

    print('\nCalculating The Most Frequent Times of Travel...\n')
    start_time = time.time()

    # TO DO: display the most common month
    months = ['january','february','march','april','may','june']
    month = df ["month"].mode()[0]
    print('most coommon month is: {months [month]}')
    # TO DO: display the most common day of week
    day = df["day_of_week"].mode([0])
    print('most coommon day of week is: {days [day]}')
    # TO DO: display the most common start hour
    df['hour'] = df['start time'].dt.hour
    hour =df['hour'].mode(0)
    print('most coommon day of week is: {days [day]}')
    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)
time_stats(df)
def station_stats(df):
    """Displays statistics on the most popular stations and trip."""
    print('\nCalculating The Most Popular Stations and Trip...\n')
    start_time = time.time()
    # TO DO: display most commonly used start station
    most_start_station = df['start station'].mode(0)
    print('most coommon start station is: [most_start_station]')
    # TO DO: display most frequent combination of start station and end station trip
    most_trip = df['start station'] + ' , ' + df['end station'].mode(0)
    print('most frequent combination of start station and end station trip is: [most_trip]')
    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)
station_stats(df)

def trip_duration_stats(df):
    """Displays statistics on the total and average trip duration."""
    print('\nCalculating Trip Duration...\n')
    start_time = time.time()
    # TO DO: display total travel time
    total_time = df['Trip Duration'].sum()
    print('total trave time:',total_time,'seconds, or',total_time/3600,'hour')
    # TO DO: display mean travel time
    mean_time = df['Trip Duration'].mean()
    print('mean trave time:',mean_time,'seconds, or',mean_time/3600,'hour')
    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)
trip_duration_stats(df)
def user_stats(df):
       """Displays statistics on bikeshare users."""
       print('\nCalculating User Stats...\n')
       start_time = time.time()
       # TO DO: Display counts of user types
       df = ['user type'].value_counts()
       print('count of user typs:\n')
       # TO DO: Display counts of gender
       df = ['grnder'].value_counts()
       if 'Gender' in df:
              print('count of gender:\n')
       # TO DO: Display earliest, most recent, and most common year of birth
       year = df['birth year'].value_counts()
       if 'birth year' in df:
              print('earliset birth year is:{year.min()}\nmost recent is: {year.max()}\nand most common birth year is: (year.mode()[0]')
              print("\nThis took %s seconds." % (time.time() - start_time))
              print('-'*40)
user_stats(df)

def display_raw_data(df):
       """ask the user if he want to display the raw data and print 5 rows at time"""
       raw = input('\ndo you want to display raw data\n')
       if raw.lower() == "yes":
              count = 0
              while True:
                     print(df.iloc[count: count+5])
                     count += 5
                     ask = input('next 5 raws?')
                     if ask.lower() != 'yes':
                            break
display_raw_data(df)
def main():
       while True:
              display_raw_data(df)
              city, month, day = get_filters()
              df = load_data(city, month, day)
              time_stats(df)
              station_stats(df)
              trip_duration_stats(df)
              user_stats(df)

              restart = input('\nWould you like to restart? Enter yes or no.\n')
              if restart.lower() != 'yes':
                     break
       if __name__ == "__main__":
              main()


import os
import tensorflow as tf
import keras
from tensorflow import keras
from tensorflow.keras import layers
import chords_imp
import data_set
import pyaudio
import wave
import HPCP
from message import create_msg
import numpy as np
checkpoint_path = "Desktop/dino/wts/cp.ckpt"
IPC_FIFO_NAME = "Desktop/dino/ipc1"
fifo = os.open(IPC_FIFO_NAME, os.O_WRONLY)
checkpoint_dir = os.path.dirname(checkpoint_path)
inputs = keras.Input(shape=(12,))
dense = layers.Dense(1000, activation="relu")
x = dense(inputs)
x = layers.Dropout(0.3)(x)
x = layers.Dense(35, activation="relu")(x)
outputs = layers.Dense(len(chords_imp.chords))(x)
model = keras.Model(inputs=inputs, outputs=outputs)
model.load_weights(checkpoint_path)
CHUNK = 1024
FORMAT = pyaudio.paInt16
CHANNELS = 1
RATE = 44100
RECORD_SECONDS = 0.1
WAVE_OUTPUT_FILENAME = "Desktop/dino/output.wav"
while True:
    p = pyaudio.PyAudio()

    stream = p.open(format=FORMAT,
                channels=CHANNELS,
                rate=RATE,
                input=True,
                frames_per_buffer=CHUNK)

    #print("* recording")

    frames = []

    for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)):
        data = stream.read(CHUNK)
        frames.append(data)

    #print("* done recording")

    stream.stop_stream()
    stream.close()
    p.terminate()

    wf = wave.open(WAVE_OUTPUT_FILENAME, 'wb')
    wf.setnchannels(CHANNELS)
    wf.setsampwidth(p.get_sample_size(FORMAT))
    wf.setframerate(RATE)
    wf.writeframes(b''.join(frames))
    wf.close()
    chroma = HPCP.hpcp(WAVE_OUTPUT_FILENAME, norm_frames=False, win_size=4096, hop_size=1024, output='numpy')
    chroma = np.mean(chroma, axis=0)
    chroma /= sum(chroma)
    chroma = chroma.reshape((1,12))
    #print(chroma.shape)
    predictions = model.predict(chroma)
    pred_class = np.argmax(predictions, axis=1)
    #print(pred_class)
    #print("predictions array:", predictions)
    content = f"{pred_class[0]}".encode("utf8")
    msg = create_msg(content)
    os.write(fifo, msg)
os.close(fifo)


""" This module contains an abstract base class Grid"""
from abc import ABC, abstractmethod
import numpy as np
from bin.Field import Field

class Grid(ABC):
    """ Abstract base class, never directly instantiated

        AirfoilMap is a child class of this ABC
    """
    # get number of vertices
    @abstractmethod
    def get_size(self):
        pass

    @abstractmethod
    def get_geometry(self):
        pass


# -*- coding=utf-8 -*-
import pandas as pd
import numpy as np
from datetime import datetime
import tushare as ts
from dateutil.parser import parse
from aqf.utils import tushare_util

"""
金融时间序列数据处理
"""

"""
1 python 时间格式处理
1.1 Datetime 数据及转换
1.2 时间转字符串
1.3 字符串转日期
"""
"""
1.1 python 下的日期格式——Datetime 数据及转换
"""
def python_data_trans():
    now = datetime.now()  # datetime.datetime
    print('{}月{}月{}日'.format(now.year, now.month, now.day))
    print(datetime.now() - datetime(2017, 8, 19))

"""
1.2 python 下时间转日期
"""
def time2str():
    dt_time = datetime(2019, 6, 18)
    # 1- str(datetime.datetime)
    str_time1 = str(dt_time)
    # 2- datetime.datetime.strftime('format')
    str_time2 = dt_time.strftime('%d/%m/%Y')

"""
1.3 python 字符串转换为 datetime 格式
"""
def str2time():
    dt_str1 = '2019-06-08'
    # 1- datetime.strptime('str', 'format')
    dt_time1 = datetime.strptime(dt_str1, '%Y-%m-%d')
    dt_str2 = '01-06-2019'
    # 2- parse 不需要指定解析格式。
    dt_time2 = parse(dt_str2)
    print(type(dt_time2))
    print(dt_time2)

"""
2 pandas 下的 time 使用
2.1 timestamp: pandas 最基本的时间格式是 TimeStamp，使用 .to_datetime() 转换为 datetime。
2.2 DatetimeIndex: pandas 下的时间索引格式。
2.3 Period: 时期（period）指的是一段时间。
"""
def pandas_time():
    # 1- pd.to_datetime(pandas.core.series.Series)，Series 向量化将字符串转为 datetime。
    str_time = pd.Series(['2017/06/18', '2017/06/19', '2017-06-20', '2017-06-21'], name='Course_time')
    # print(type(str_time))
    dt_time = pd.to_datetime(str_time)
    # print(dt_time)

    # 2- DatetimeIndex
    # 2.1- 使用 datetime 格式创建 DatetimeIndex
    dates1 = [datetime(2019, 8, 1), datetime(2019, 8, 2)]
    dates1 = pd.DatetimeIndex(dates1)
    # 此处需注意，python 中的 datetime 转换为 pandas(Series/DataFrame) 下的时间索引 DatetimeIndex；
    df = pd.Series(np.random.randn(2), index=dates1)
    dates2 = pd.date_range('8/1/2019', periods=10)
    df = pd.Series(np.random.randn(10), index=dates2)
    # DataFrame 使用 DatetimeIndex 可以便捷选取特定年、月、日
    # print(df['2019'])
    # print(df['2019-08'])
    # print(df['2019-08-01'])
    # print(df.index)

    # 2.2- 字符串转化为 DatetimeIndex
    date_time = pd.to_datetime(['June 18, 2017', '2016-06-19',
                                '2016.6.20', None])
    # print(date_time)

    # 3- Period
    # type 为 pandas.core.indexes.period.PeriodIndex
    time_period = pd.period_range('2017-01-01', periods=12, freq='M')
    print(type(time_period))
    # type 为 pandas.core.indexes.datetimes.DatetimeIndex
    date_range = pd.date_range('2017-01-01', periods=12, freq='M')
    print(type(date_range))
    # period对象的起始时间和终止时间
    time_period + 1
    # 创建PeriodIndex
    period_index = pd.period_range('1/1/2017', '12/31/2017', freq='M')
    # 以PeriodIndex为索引创建Series
    ps = pd.Series(np.random.randn(12), index=period_index)

    # 4- 应用
    data = ts.get_k_data('000001', '2018-01-01', '2109-06-06')
    data.index = pd.to_datetime(data['date'])
    del data['date']
    data.head()
    # 4.1- 获取 DataFrame 信息
    data.info()
    # 4.2- 切片
    data['2018-06-05':'2019-01-01']
    data['2018-01':'2018-02-12']
    data.loc['2018-01-04']
    # 在 DataFrame 下可使用 DateTimeIndex 可进行年、月进行索引。
    print(data['2018'])
    print(data['2018-02'])
    # DataFrame 不支持下列索引
    print(data['2018-02-01'])

"""
3 时间序列数据处理
3.1 时间数据的聚合
3.2 时间数据的频率转换
3.3 数据频率转换应用
"""
def time_series_process():
    # 1- 时间序列数据前后移动
    data = tushare_util.get_single_stock_data('000001', '2018-01-01', '2019-01-01')
    close_price = data['close']
    # 获取前一天收盘价 Series(pandas.core.series.Series)
    previous_close_price = close_price.shift(1)
    # 计算收盘价百分比。两种方式：1，使用 当天收盘价/前一天收盘价 - 1 2，
    # print((close_price / previous_close_price - 1).head())
    # 计算累计收益率
    cum_return = (close_price / previous_close_price - 1).cumprod()
    # print(cum_return.head())

    # 2- 时间序列数据的频率调整
    # 2.1- 时间数据的聚合
    # 计算某月中每日收益率的平均值
    cum_return['2018-01'].mean()
    # DataFrame.resample 主要用于升采样和降采样。
    cum_return.resample('M').mean().head()
    cum_return.resample('M').ohlc().head()
    # deprecated 方法已过期
    # cum_return.resample('M', how='mean').head()

    # 2.2- 时间数据的频率转换
    sample = cum_return[1:3]
    print(sample)
    sample_by_hour = sample.resample('H')
    print(sample_by_hour.size())
    # print(sample_by_hour.head())

    # 2.3- 数据频率转换应用实战
    tushare_util.get_single_stock_data()

if __name__ == '__main__':
    # python_data_trans()
    # time2str()
    # str2time()
    # pandas_time()
    time_series_process()


import numpy as np
import matplotlib.pyplot as plt

x=np.linespace(0,10,1000)
y=x

fig, ax=plt.subplots()
ax.plot(x,y)
ax.set_xlabel('x')
ax.set_ylabel('y')
fig.tight_layout()


# ******************************************************************************
# Copyright 2017-2018 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ******************************************************************************
"""
Unit tests for ngraph/frontends/tensorflow/tf_importer/utils
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import ngraph as ng
import pytest
from ngraph.frontends.tensorflow.tf_importer.utils import np_layout_shuffle
from ngraph.frontends.tensorflow.tf_importer.utils_broadcast import \
    broadcast_to, is_compatible_numpy_shape, is_compatible_broadcast_shape, \
    broadcasted_shape
from ngraph.frontends.tensorflow.tf_importer.utils_pos_axes import make_pos_axes
from ngraph.testing.execution import ExecutorFactory

def test_np_layout_shuffle():
    # set up
    bsz = 8
    C, H, W, N = 3, 28, 28, bsz
    C, R, S, K = 3, 5, 5, 32

    # image dim-shuffle
    np_tf_image = np.random.randn(N, H, W, C)
    np_ng_image = np_layout_shuffle(np_tf_image, "NHWC", "CDHWN")
    np_tf_image_reverse = np_layout_shuffle(np_ng_image, "CDHWN", "NHWC")
    assert np.array_equal(np_tf_image, np_tf_image_reverse)

    # filter dim-shuffle
    np_tf_weight = np.random.randn(R, S, C, K)
    np_ng_weight = np_layout_shuffle(np_tf_weight, "RSCK", "CTRSK")
    np_tf_weight_reverse = np_layout_shuffle(np_ng_weight, "CTRSK", "RSCK")
    assert np.array_equal(np_tf_weight, np_tf_weight_reverse)

@pytest.mark.parametrize("test_case", [
    [(), (), True],
    [(), (1, 2, 3), True],
    [(1, 2), (2, 2), True],
    [(3, 2), (2, 2), False],
    [(2, 1, 2, 1), (1, 1, 3), True],
    [(2, 1, 2, 1), (2, 1, 3), True],
    [(2, 1, 2, 1), (1, 3, 1), False],
])
def test_is_compatible_numpy_shape(test_case):
    left_shape, right_shape, result = test_case
    assert is_compatible_numpy_shape(left_shape, right_shape) == result

@pytest.mark.parametrize("test_case", [
    [(), (), True],
    [(), (1, 2, 3), True],
    [(1, 2, 3), (), False],
    [(1, 2), (2, 2), True],
    [(2, 2), (1, 2), False],
    [(3, 2), (2, 2), False],
    [(2, 1, 2, 1), (1, 1, 3), False],
    [(2, 1, 3), (2, 1, 2, 1), False],
    [(2, 1, 3), (4, 2, 2, 3), True],
    [(2, 1, 3), (4, 2, 2, 5), False]
])
def test_is_compatible_broadcast_shape(test_case):
    left_shape, right_shape, result = test_case
    assert is_compatible_broadcast_shape(left_shape, right_shape) == result

@pytest.config.flex_disabled(reason='Results mismatch')
@pytest.mark.transformer_dependent
@pytest.mark.parametrize("test_case", [
    [(), (1,)],
    [(), (1, 2)],
    [(1,), (2,)],
    [(1,), (2, 1)],
    [(1,), (3, 2)],
    [(2,), (3, 2)],
    [(1, 3, 1), (2, 3, 4)],
    [(3, 1, 2), (4, 3, 5, 2)],
    [(5, 1, 2, 1), (4, 5, 1, 2, 3)],
])
def test_broadcast_to(test_case):
    src_shape, dst_shape = test_case

    # numpy results
    x_np = np.array(np.random.rand(*src_shape))
    f_np = x_np + np.zeros(dst_shape)

    # ngraph results
    x_ng = ng.constant(x_np, axes=make_pos_axes(x_np.shape))
    f_ng = broadcast_to(x_ng, dst_shape)

    with ExecutorFactory() as ex:
        f_ng_comp = ex.transformer.computation(f_ng)
        f_ng_val = f_ng_comp()
        np.testing.assert_allclose(f_ng_val, f_np)

@pytest.mark.parametrize("test_case", [
    [(), (), ()],
    [(), (1,), (1,)],
    [(1,), (), (1,)],
    [(), (1, 2), (1, 2)],
    [(1, 2), (), (1, 2)],
    [(), (1, 2, 3), (1, 2, 3)],
    [(1, 2), (2, 2), (2, 2)],
    [(2, 1, 3), (4, 2, 2, 3), (4, 2, 2, 3)],
])
def test_broadcasted_shape(test_case):
    left_shape, right_shape, out_shape = test_case
    assert (broadcasted_shape(left_shape, right_shape) == out_shape)


# -*- coding: utf-8 -*-
"""
This python file contains various of functions to generate plots.
"""
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from helpers import get_plot_path

""" Lab 3 """

def plot_fitted_curve(y, x, weights, ax):
    """plot the fitted curve. x, weights should align dimension """
    ax.scatter(x, y, color='b', s=12, facecolors='none', edgecolors='r')
    xvals = np.arange(min(x) - 0.1, max(x) + 0.1, 0.1)
    f = x.dot(weights)
    ax.plot(xvals, f)
    ax.set_xlabel("x")
    ax.set_ylabel("y")
    ax.set_title("Fitted curve for x y")

def plot_train_test(train_errors, test_errors, names=['', ''], xlabel='', ylabel='',
                    lambdas=None, filename=''):
    """
    train_errors, test_errors and lambas should be list (of the same size) the respective train error and test error for a given lambda,
    * lambda[0] = 1
    * train_errors[0] = RMSE of a ridge regression on the train set
    * test_errors[0] = RMSE of the parameter found by ridge regression applied on the test set

    degree is just used for the title of the plot.
    """
    plt.semilogx(lambdas, train_errors, color='b', marker='*', label=names[0])
    plt.semilogx(lambdas, test_errors, color='r', marker='*', label=names[1])
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    plt.title(filename)
    leg = plt.legend(loc=1, shadow=True)
    leg.draw_frame(False)
    plt.show()
    plt.savefig(get_plot_path("train_test " + filename))

""" Lab 4 """

def cross_validation_visualization(params, mse_tr, mse_te, params_name='', title='', error_name=''):
    """visualization the curves of mse_tr and mse_te."""
    plt.semilogx(params, mse_tr, marker=".", color='b', label='train error')
    plt.semilogx(params, mse_te, marker=".", color='r', label='test error')
    plt.xlabel("Parameters: " + params_name)
    plt.ylabel("Error: " + error_name)
    plt.title("cross validation" + title)
    plt.legend(loc=2)
    plt.grid(True)
    plt.savefig(get_plot_path("cross_validation_" + title))
    plt.show()

def cross_validation_visualization_due(params, mse_tr, mse_te, param2, tr2, te2, params_name='', prname2='', title='',
                                       error_name=''):
    """visualization the curves of mse_tr and mse_te."""
    plt.semilogx(params, mse_tr, marker=".", color='r', label='train error ' + params_name, linestyle='solid')
    plt.semilogx(params, mse_te, marker=".", color='r', label='test error ' + params_name, linestyle='dashed')
    plt.semilogx(param2, tr2, marker=".", color='b', label='train error ' + prname2, linestyle='solid')
    plt.semilogx(param2, te2, marker=".", color='b', label='test error ' + prname2, linestyle='dashed')
    plt.xlabel("Parameters: " + params_name + " " + prname2)
    plt.ylabel("Error: " + error_name)
    plt.title("cross validation " + title)
    plt.legend(loc=2)
    plt.grid(True)
    plt.savefig(get_plot_path("cross_validation_" + title))
    plt.show()

def bias_variance_decomposition_visualization(models, rmse_tr, rmse_te, model_names=[]):
    """visualize the bias variance decomposition."""
    rmse_tr_mean = np.expand_dims(np.mean(rmse_tr, axis=0), axis=0)
    rmse_te_mean = np.expand_dims(np.mean(rmse_te, axis=0), axis=0)
    degrees = np.array(range(len(models)))
    plt.plot(
        degrees,
        rmse_tr.T,
        'b',
        linestyle="-",
        color=([0.7, 0.7, 1]),
        label='train',
        linewidth=0.3)
    plt.plot(
        degrees,
        rmse_te.T,
        'r',
        linestyle="-",
        color=[1, 0.7, 0.7],
        label='test',
        linewidth=0.3)
    plt.plot(
        degrees,
        rmse_tr_mean.T,
        'b',
        linestyle="-",
        label='train',
        linewidth=3)
    plt.plot(
        degrees,
        rmse_te_mean.T,
        'r',
        linestyle="-",
        label='test',
        linewidth=3)
    # plt.ylim(0.2, 0.7)
    plt.xlabel("degree")
    plt.ylabel("error")
    plt.title("Bias-Variance Decomposition")
    plt.savefig(get_plot_path("bias_variance"))
    plt.show()

def pca_plot(headers, pc1, pc2, title=''):
    """ Plot pca accordingly """
    fig, ax = plt.subplots()
    ax.scatter(pc1, pc2)
    for i, name in enumerate(headers[2:]):
        ax.annotate(name, (pc1[i], pc2[i]))

    ax.set_xlabel("PC 1")
    ax.set_ylabel("PC 2")
    ax.set_title("PCA plot for data")
    fig.show()
    fig.savefig(get_plot_path("pca_plot " + title))

def pca_plot_general(headers, pcs, pcs2=None, index=(0, 1), title='', color=['b', 'r'], print_name=False):
    pc1 = pcs[index[0]]
    pc2 = pcs[index[1]]
    fig, ax = plt.subplots()
    sct = ax.scatter(pc1, pc2, color=color[0])
    for i, name in enumerate(headers[2:]):
        if not print_name:
            ax.annotate('{}'.format(i - 2), (pc1[i], pc2[i]))
        else:
            ax.annotate('{}-'.format(i - 2) + name, (pc1[i], pc2[i]))

    if pcs2 is not None:
        fig.hold()
        pc1 = pcs2[index[0]]
        pc2 = pcs2[index[1]]
        sct2 = ax.scatter(pcs2[index[0]], pcs2[index[1]], color=color[1])
        for i, name in enumerate(headers[2:]):
            if not print_name:
                ax.annotate('{}'.format(i - 2), (pc1[i], pc2[i]))
            else:
                ax.annotate('{}-'.format(i - 2) + name, (pc1[i], pc2[i]))
        fig.legend((sct, sct2), ('train', 'test'))

    ax.set_xlabel("PC {}".format(index[0]))
    ax.set_ylabel("PC {}".format(index[1]))
    ax.set_title("PCA plot for data")
    fig.show()
    fig.savefig(get_plot_path("pca_plot " + title))

def histogram(label, data, headers=None, colors=['b', 'r'], print_name=True, transform=None, filename='Default.plt',
              outlier=False):
    """
    Build up histogram regarding to labels, via each dimensions.
    Stored in the path: plots/histogram
    :param ids:         index
    :param label:       y
    :param data:        data matrix
    :param headers:     headers accordingly
    :param print_name:  print name on the histogram
    :return:
    """
    hist_path = get_plot_path() + '/histogram/'
    # Generate positive and negative index
    negative_index = np.where(label < 0)[0]
    positive_index = np.where(label > 0)[0]
    nega_data = data[negative_index, :]
    posi_data = data[positive_index, :]
    if transform is None:
        transform = [lambda x: x, lambda x: np.log(x + 0.01 - np.min(x)), lambda x: np.sqrt(np.abs(x)),
                     lambda x: np.power(x, 2)]
    trans_labels = ['linear', 'log', 'sqrt|abs|', 'power']
    # Plot according to each dimensions
    headers = headers[2:]
    # Hard coded
    gs = gridspec.GridSpec(2, 2)
    assert len(headers) == len(data[0])
    for index, header in enumerate(headers):
        # fig, axs = plt.subplots(1, len(transform))
        for f_ind, f_trans in enumerate(transform):
            ax = plt.subplot(gs[int(f_ind / 2), f_ind % 2])
            ax.set_aspect('auto')
            if outlier:
                ax.hist(f_trans(nega_data[:, index]).T, bins=100, color=colors[0], alpha=0.5)
            else:
                # nega_ind = np.where(nega_data[:, index] != -999.0)
                ax.hist(f_trans(nega_data[np.where(nega_data[:, index] != -999.0)[0], index]).T,
                        bins=100, color=colors[0], alpha=0.5)
            ax.hold(True)
            if outlier:
                ax.hist(f_trans(posi_data[:, index]).T, bins=100, color=colors[1], alpha=0.8)
            else:
                ax.hist(f_trans(posi_data[np.where(posi_data[:, index] != -999.0)[0], index]).T,
                        bins=100, color=colors[1], alpha=0.8)
            if print_name:
                ax.set_xlabel("{}({})".format(trans_labels[f_ind], header))
            else:
                ax.set_xlabel(trans_labels[f_ind])
            ax.hold(False)
        plt.savefig(hist_path + "{}-{}_{}.png".format(filename, index, header))
        plt.close()


#########################################################
#
# DO NOT EDIT THIS FILE. IT IS GENERATED AUTOMATICALLY. #
# PLEASE LOOK INTO THE README FOR MORE INFORMATION.     #
#
#########################################################

# coding: utf-8

# # Control Ops Tutorial
#
# In this tutorial we show how to use control flow operators in Caffe2 and give some details on their underlying implementations.

# ### Conditional Execution

# Let's start with conditional operator. We will demostrate how to use it in two Caffe2 APIs used for building nets: NetBuilder and Brew.

# In the first example, we first define several blobs and then use 'If' operator to set value of one of them conditionally depending on values of other blobs.

# In[1]:

from caffe2.python import workspace
from caffe2.python.core import Plan, to_execution_step, Net
from caffe2.python.net_builder import ops, NetBuilder

# In[2]:

with NetBuilder() as nb:
    ops.Const(0.0, blob_out="zero")
    ops.Const(1.0, blob_out="one")
    ops.Const(0.5, blob_out="x")
    ops.Const(0.0, blob_out="y")
    with ops.IfNet(ops.GT(["x", "zero"])):
        ops.Copy("one", "y")
    with ops.Else():
        ops.Copy("zero", "y")

# Note the usage of NetBuilder's ops.IfNet and ops.Else calls: ops.IfNet accepts a blob reference or blob name as an input, it expects an input blob to have a scalar value convertible to bool, also note that optional ops.Else is at the same level as ops.IfNet and immediately follows corresponding ops.IfNet. Let's execute resulting net (execution step) and check values of blobs.

# In[3]:

plan = Plan('if_net_test')
plan.AddStep(to_execution_step(nb))
ws = workspace.C.Workspace()
ws.run(plan)
print('x = ', ws.blobs["x"].fetch())
print('y = ', ws.blobs["y"].fetch())

# Before going further, it's important to understand the semantics of execution blocks ('then' and 'else' branches in the example above), i.e. handling of reads and writes into global (defined outside of the block) and local (defined inside the block) blobs.

# NetBuilder's uses the following set of rules:
#  - In NetBuilder's syntax, blob's declaration and definition occur at the same time - when we define an operator which writes its output into a blob with a given name;
#  - NetBuilder keeps track of all operator outputs seen before current execution point in the same block and up the stack in parent blocks.
#  - If an operator writes into a previously unseen blob, it creates a **local** blob that is visible only within the current block and the subsequent children blocks. Local blobs created in a given block are effectively deleted when we exit the block. Any write into previously defined (in the same block or in the parent blocks) blob updates an originally created blob and does not result in the redefinition of a blob.
#  - Operator's input blobs have to be defined earlier in the same block or in the parent blocks up the stack.

# As a result, in order to see values computed by a block after the block is finished the corresponding blobs have to be defined outside of the block. Note, that this is one of the ways to solve the problem with uninitialized blobs (e.g. blob created by 'then' branch, but not by 'else' branch), these rules effectively force visible blobs to be always correctly initialized.

# To illustrate concepts of block semantics and provide a more sophisticated example, let's consider the following net:

# In[4]:

with NetBuilder() as nb:
    ops.Const(0.0, blob_out="zero")
    ops.Const(1.0, blob_out="one")
    ops.Const(2.0, blob_out="two")
    ops.Const(1.5, blob_out="x")
    ops.Const(0.0, blob_out="y")
    with ops.IfNet(ops.GT(["x", "zero"])):
        ops.Copy("x", "local_blob")
        with ops.IfNet(ops.LE(["local_blob", "one"])):
            ops.Copy("one", "y")
        with ops.Else():
            ops.Copy("two", "y")
    with ops.Else():
        ops.Copy("zero", "y")
        # ops.Copy("local_blob", "z") - fails with exception during net construction, local_blob is undefined

# In[5]:

plan = Plan('if_net_test_2')
plan.AddStep(to_execution_step(nb))
ws = workspace.C.Workspace()
ws.run(plan)
print('x = ', ws.blobs["x"].fetch())
print('y = ', ws.blobs["y"].fetch())
assert "local_blob" not in ws.blobs

# Brew is another Caffe2 interface used to construct nets. Unlike NetBuilder, Brew does not track the hierarchy of blocks and, as a result, we need to specify which blobs are considered local and which global when passing 'then' and 'else' models to an API call:

# In[6]:

from caffe2.python import brew
from caffe2.python.workspace import FeedBlob, RunNetOnce, FetchBlob
from caffe2.python.model_helper import ModelHelper

# In[7]:

model = ModelHelper(name="test_if_model")

model.param_init_net.ConstantFill([], ["zero"], shape=[1], value=0.0)
model.param_init_net.ConstantFill([], ["one"], shape=[1], value=1.0)
model.param_init_net.ConstantFill([], ["x"], shape=[1], value=0.5)
model.param_init_net.ConstantFill([], ["y"], shape=[1], value=0.0)
model.param_init_net.GT(["x", "zero"], "cond")

then_model = ModelHelper(name="then_test_model")
then_model.net.Copy("one", "y")

else_model = ModelHelper(name="else_test_model")
else_model.net.Copy("zero", "y")

brew.cond(
    model=model,
    cond_blob="cond", # blob with condition value
    external_blobs=["x", "y", "zero", "one"], # writes into these blobs update existing blobs
    then_model=then_model,
    else_model=else_model)

# To run resulting init and main net:

# In[8]:

RunNetOnce(model.param_init_net)
RunNetOnce(model.net)
print("x = ", FetchBlob("x"))
print("y = ", FetchBlob("y"))

# ### Loops

# Another important control flow operator is 'While' that allows repeated execution of a fragment of net. Let's consider NetBuilder's version of While first.

# In[9]:

with NetBuilder() as nb:
    ops.Const(0, blob_out="i")
    ops.Const(0, blob_out="y")
    with ops.WhileNet():
        with ops.Condition():
            ops.Add(["i", ops.Const(1)], ["i"])
            ops.LE(["i", ops.Const(7)])
        ops.Add(["i", "y"], ["y"])

# This example illustrates the usage of 'while' loop with NetBuilder. As with an 'If' operator, standard block semantic rules apply. Note the usage of ops.Condition clause that should immediately follow ops.WhileNet and contains code that is executed before each iteration. The last operator in the condition clause is expected to have a single boolean output that determines whether the other iteration is executed.

# In the example above we increment the counter ("i") before each iteration and accumulate its values in "y" blob, the loop's body is executed 7 times, the resulting blob values:

# In[10]:

plan = Plan('while_net_test')
plan.AddStep(to_execution_step(nb))
ws = workspace.C.Workspace()
ws.run(plan)
print("i = ", ws.blobs["i"].fetch())
print("y = ", ws.blobs["y"].fetch())

# Corresponding Brew example:

# In[11]:

model = ModelHelper(name="test_while_model")

model.param_init_net.ConstantFill([], ["i"], shape=[1], value=0)
model.param_init_net.ConstantFill([], ["one"], shape=[1], value=1)
model.param_init_net.ConstantFill([], ["seven"], shape=[1], value=7)
model.param_init_net.ConstantFill([], ["y"], shape=[1], value=0)

loop_model = ModelHelper(name="loop_test_model")
loop_model.net.Add(["i", "y"], ["y"])

cond_model = ModelHelper(name="cond_test_model")
cond_model.net.Add(["i", "one"], "i")
cond_model.net.LE(["i", "seven"], "cond")

brew.loop(
    model=model,
    cond_blob="cond", # explicitly specifying condition blob
    external_blobs=["cond", "i", "one", "seven", "y"],
    loop_model=loop_model,
    cond_model=cond_model # condition model is optional
)

# Corresponding blob values:

# In[12]:

RunNetOnce(model.param_init_net)
RunNetOnce(model.net)
print("i = ", FetchBlob("i"))
print("y = ", FetchBlob("y"))

# ### Backpropagation

# Both 'If' and 'While' operators support backpropagation. To illustrate how backpropagation with control ops work, let's consider the following example:

# In[14]:

import numpy as np
# _use_control_ops=True forces NetBuilder to output single net as a result
# x is external for NetBuilder, so letting nb know about it through initial_scope param
FeedBlob("x", np.array(0.5, dtype='float32'))
with NetBuilder(_use_control_ops=True, initial_scope=["x"]) as nb:
    ops.Const(0.0, blob_out="zero")
    ops.Const(1.0, blob_out="one")
    ops.Const(4.0, blob_out="y")
    ops.Const(0.0, blob_out="z")
    with ops.IfNet(ops.GT(["x", "zero"])):
        ops.Pow("y", "z", exponent=2.0)
    with ops.Else():
        ops.Pow("y", "z", exponent=3.0)

assert len(nb.get()) == 1, "Expected a single net produced"
net = nb.get()[0]
grad_map = net.AddGradientOperators(["z"])

# Output blob "z" as a function of "y" depends on the value of blob "x", if "x" is greater than zero, than "z = y^2", otherwise it is "z = y^3"

# In[15]:

RunNetOnce(net)
print("x = ", FetchBlob("x"))
print("y = ", FetchBlob("y"))
print("z = ", FetchBlob("z"))
print("y_grad = ", FetchBlob("y_grad"))

# Now, let's change value of blob "x" and rerun net:

# In[16]:

FeedBlob("x", np.array(-0.5, dtype='float32'))
RunNetOnce(net)
print("x = ", FetchBlob("x"))
print("y = ", FetchBlob("y"))
print("z = ", FetchBlob("z"))
print("y_grad = ", FetchBlob("y_grad"))

# An example illustrating backpropagation on the loop:

# In[17]:

with NetBuilder(_use_control_ops=True) as nb:
    ops.Copy(ops.Const(0), "i")
    ops.Copy(ops.Const(1), "one")
    ops.Copy(ops.Const(2), "two")
    ops.Copy(ops.Const(2.0), "x")
    ops.Copy(ops.Const(3.0), "y")
    ops.Copy(ops.Const(2.0), "z")
    # computes x^4, y^2, z^3
    with ops.WhileNet():
        with ops.Condition():
            ops.Add(["i", "one"], "i")
            ops.LE(["i", "two"])
        ops.Pow("x", "x", exponent=2.0)
        with ops.IfNet(ops.LT(["i", "two"])):
            ops.Pow("y", "y", exponent=2.0)
        with ops.Else():
            ops.Pow("z", "z", exponent=3.0)

    ops.Add(["x", "y"], "x_plus_y")
    ops.Add(["x_plus_y", "z"], "s")

assert len(nb.get()) == 1, "Expected a single net produced"
net = nb.get()[0]

grad_map = net.AddGradientOperators(["s"])

# In[18]:

workspace.RunNetOnce(net)
print("x = ", FetchBlob("x"))
print("x_grad = ", FetchBlob("x_grad")) # 4x^3
print("y = ", FetchBlob("y"))
print("y_grad = ", FetchBlob("y_grad")) # 2y
print("z = ", FetchBlob("z"))
print("z_grad = ", FetchBlob("z_grad")) # 3z^2

# ### Implementation Notes

# On the low level, Caffe2 uses the following set of operators to implement forward and backward branching and loops:
# - If - accepts *then_net* and *else_net* nets as arguments and executes one of them, depending on input condition blob value, nets are executed **in the same** workspace;
# - While - repeats execution of *loop_net* net passed as argument, net is executed in the same workspace;
# - Do - special operator that creates a separate inner workspace, setups blob mappings between outer and inner workspaces and runs a net in an inner workspace;
# - CreateScope/HasScope - special operators that create and keep track of workspaces used by Do operator.
#
# Higher level libraries that implement branching and looping (e.g. in NetBuilder, Brew), use these operators to build control flow, e.g. for 'If':
#  - do necessary sanity checks (e.g. determine which blobs are initialized and check that subnet does not read undefined blobs)
#  - wrap 'then' and 'else' branches into Do
#  - setup correct blob mappings by specifying which local names are mapped to outer blobs
#  - prepare scope structure, used by Do operator
#
# While 'If' and 'While' Caffe2 ops can be used directly without creating local block workspaces, we encourage users to use higher level Caffe2 interfaces that provide necessary correctness guarantees.
#
# Backpropagation for 'While' in general is expensive memory-wise - we have to save local workspace for every iteration of a block, including global blobs visible to the block. It is recommended that users use RecurrentNetwork operator instead in production environments.


"""
Functions to read and save MERRA reanalysis data.

Convention for function names
  Starts with - read_ : Read from OpenDAP url(s)
              - load_ : Load from locally saved files
"""

from __future__ import division
import numpy as np
import xarray as xray
import collections
import os
import pandas as pd
import urllib2
from bs4 import BeautifulSoup

import sys
sys.path.append('/home/jwalker/dynamics/python/atmos-tools')
import atmos as atm
from atmos import print_if

# ======================================================================
# Lists of variable IDs and OpenDAP urls for data files
# ======================================================================

# ----------------------------------------------------------------------
def get_varname(var_id):
    """Return the variable name in MERRA naming convention.

    Parameters
    ----------
    var_id : {'u', 'v', 'omega', 'hgt', 'T', 'q', 'ps', 'evap', 'precip'}
    """

    var_dict = {'u' : 'U', 'v' : 'V', 'omega' : 'OMEGA', 'hgt' : 'H',
                'T' : 'T', 'q' : 'QV', 'ps' : 'PS', 'evap' : 'EVAP',
                'precip' : 'PRECTOT'}

    if var_id in var_dict:
        return var_dict[var_id]
    else:
        return var_id

# ----------------------------------------------------------------------
def url_opts(var_id, version='merra'):
    """Return the dataset options to determine URLs for a variable.

    See get_url() documentation for more info.
    """

    varnm = get_varname(var_id)

    p_res = {'merra' : 'C', 'merra2' : 'N'}[version]

    optlist = {'int_t' : ('X', 'N', 'T', 'INT'),
               'int_i' : ('X', 'N', 'I', 'INT'),
               'flx_t' : ('X', 'N', 'T', 'FLX'),
               'slv_t' : ('X', 'N', 'T', 'SLV'),
               'asm_i' : ('P', p_res, 'I', 'ASM'),
               'udt_t' : ('P', p_res, 'T', 'UDT'),
               'rad_t' : ('X', 'N', 'T', 'RAD')}

    optkeys = {}
    for nm in ['UFLXQV', 'VFLXQV', 'UFLXCPT', 'VFLXCPT', 'UFLXPHI', 'VFLXPHI',
               'DQVDT_ANA']:
        optkeys[nm] = 'int_t'
    for nm in ['TQV']:
        optkeys[nm] = 'int_i'
    for nm in ['PRECTOT', 'EVAP', 'EFLUX', 'HFLUX', 'ULML', 'VLML', 'QLML',
               'TLML', 'HLML']:
        optkeys[nm] = 'flx_t'
    for nm in ['PS', 'SLP']:
        optkeys[nm] = 'slv_t'
    for nm in ['U', 'V', 'OMEGA', 'T', 'QV', 'H']:
        optkeys[nm] = 'asm_i'
    for nm in ['DUDTANA', 'DVDTANA']:
        optkeys[nm] = 'udt_t'
    for nm in [u'Var_ALBNIRDF', u'Var_SWTNTCLN', u'Var_TAUTOT', u'LWGAB',
               u'CLDTOT', u'Var_ALBNIRDR', u'Var_LWTUPCLR', u'ALBNIRDF',
               u'Var_LWGNT', u'SWTDN', u'EMIS', u'LWTUPCLRCLN', u'SWTNTCLR',
               u'CLDHGH', u'Var_LWTUPCLRCLN', u'Var_SWTNTCLRCLN',
               u'Var_LWGNTCLRCLN', u'Var_SWGNTCLRCLN', u'Var_TAULOW',
               u'LWGABCLR', u'Var_ALBVISDF', u'LWGABCLRCLN', u'Var_ALBVISDR',
               u'Var_TAUHGH', u'Var_SWGDNCLR', u'Var_SWTDN', u'LWGNTCLRCLN',
               u'Var_CLDLOW', u'SWGNTCLRCLN', u'Var_LWGABCLR', u'Var_CLDTOT',
               u'TS', u'SWGNT', u'TAUMID', u'ALBEDO', u'SWGNTCLR', u'SWGNTCLN',
               u'LWGNTCLR', u'Var_ALBEDO', u'SWGDNCLR', u'ALBVISDF',
               u'LWTUPCLR', u'TAUTOT', u'LWGNT', u'CLDLOW', u'ALBVISDR',
               u'Var_CLDMID', u'Var_LWGNTCLR', u'SWTNTCLRCLN', u'TAUHGH',
               u'TAULOW', u'Var_LWGEM', u'Var_SWGNTCLN', u'Var_TAUMID',
               u'Var_SWTNT', u'Var_SWGNT', u'Var_LWTUP', u'Var_SWTNTCLR',
               u'Var_SWGNTCLR', u'SWGDN', u'Var_LWGAB', u'LWGEM', u'Var_CLDHGH',
               u'CLDMID', u'Var_EMIS', u'SWTNTCLN', u'ALBNIRDR', u'Var_SWGDN',
               u'SWTNT', u'LWTUP', u'Var_LWGABCLRCLN', u'Var_TS']:
        optkeys[nm] = 'rad_t'
    vertical, res, time_kind, kind = optlist[optkeys[varnm]]
    opts = {'version' : version, 'vertical' : vertical, 'res' : res,
            'time_kind' : time_kind, 'kind' : kind}

    return opts

# ----------------------------------------------------------------------
def scrape_url(url, ending='.hdf.html', cut='.html'):
    """Scrape url for links with specified ending string."""

    soup = BeautifulSoup(urllib2.urlopen(url))
    links = []
    for link in soup.find_all('a'):
        links.append(link.get('href'))

    links = list(set([s for s in links if s.endswith(ending)]))
    if cut is not None:
        links = [s.split(cut)[0] for s in links]
    return links

# ----------------------------------------------------------------------
def extract_date(filename, width, ending='.hdf'):
    """Extract yyyymmdd or yyyymm from file name."""
    s = filename.split(ending)[0]
    date = s[-width:]
    return date

# ----------------------------------------------------------------------
def get_urls(years, months=None, version='merra', varnm='U', opts=None,
             monthly=False):
    """Return dict of OpenDAP urls for MERRA and MERRA-2 daily data.

    Parameters
    ----------
    years : list or np.ndarray
        List of years to extract urls for.
    months: list or np.ndarray, optional
        List of months to extract urls for.  If None, then all months (1-12)
        are extracted.
    version : {'merra', 'merra2'}, optional
        Select MERRA or MERRA-2 data.
    varnm : str, optional
        Variable ID.  If None, then the options in the input parameter opts
        are used. If not None, then opts are determined using url_opts(varnm)
        and override any value provided to input opts.
    opts : dict, optional
        Provide the dataset options rather than calling url_opts(varnm).
        The key : value pairs of opts are:
            'vertical' : 'P', 'X', 'V', or 'E'
                Vertical location : on pressure levels (P), 2-D (X), model
                layers (V), or model layer edges (E).
            'res' : 'N' or 'C'
                Horizontal resolution: native (N) or coarse (C).
            'time_kind' : 'I' or 'T'
                Instantaneous (I) or time-averaged (T) diagnostics.
            'kind' : 'ASM', 'SLV', 'FLX', or 'RAD'
                Type of dataset: assimilated 3-d (ASM), atmospheric single-level
                (SLV), surface turbulent fluxes (FLX), or surface and TOA
                radiation fluxes (RAD).
    monthly : bool, optional
        If True, return urls for monthly data.  Otherwise return urls
        for daily data.

    Returns
    -------
    urls : dict of date:url for each date in the dataset
    """

    if varnm is not None:
        opts = url_opts(varnm, version)

    # Dataset options
    version = version.lower()
    time_kind = opts['time_kind'].upper()
    res = opts['res'].upper()
    vertical = opts['vertical'].upper()
    kind = opts['kind'].upper()

    # Make dicts of years and months
    yearvals = atm.makelist(years)
    years = {y : '%d' % y for y in yearvals}
    if months is None:
        monthvals = range(1, 13)
    else:
        monthvals = atm.makelist(months)
    months = {m : '%02d' % m for m in monthvals}

    urlstr = 'http://goldsmr%d.sci.gsfc.nasa.gov/opendap/%s/%s'
    dirname = version.upper()
    if monthly:
        dirname = dirname + '_MONTHLY'
    servers = {'merra_X' : urlstr % (2, dirname, 'MA'),
               'merra' : urlstr % (3, dirname, 'MA'),
               'merra2_X' : urlstr % (4, dirname, 'M2'),
               'merra2' : urlstr % (5, dirname, 'M2')}
    version_num = {'merra' : '.5.2.0/', 'merra2' : '.5.12.4/'}
    fmts = {'merra' : '.hdf', 'merra2' : '.nc4'}

    if vertical == 'X':
        time_res = '1'
        server_key = version + '_X'
    else:
        time_res = '3'
        server_key = version
    if monthly:
        time_res = 'M'

    try:
        basedir = servers[server_key]
        vnum = version_num[version]
        fmt = fmts[version]
    except KeyError:
        raise ValueError('Invalid version %s.  Options are: merra, merra2.' %
                         version)

    basedir = basedir + time_kind + time_res + res + vertical + kind + vnum
    print('Scraping filenames from ' + basedir)

    # Helper function to make daily urls
    def daily_urls(basedir, years, months, fmt):
        url_dict = collections.OrderedDict()
        for y in years:
            for m in months:
                print(years[y] + months[m])
                dirname = basedir + years[y] + '/' + months[m] + '/'
                files = scrape_url(dirname + 'contents.html',
                                   ending=fmt + '.html')
                files.sort()
                dates = [extract_date(nm, width=8, ending=fmt) for nm in files]
                for date, nm in zip(dates, files):
                    url_dict[date] = dirname + nm
        return url_dict

    # Helper function to make monthly urls
    def monthly_urls(basedir, years, months, fmt):
        url_dict = collections.OrderedDict()
        for y in years:
            dirname = basedir + years[y] + '/'
            files = scrape_url(dirname, ending=fmt + '.html')
            files.sort()
            dates = [extract_date(nm, width=6, ending=fmt) for nm in files]
            yr_dict = {date : nm for (date, nm) in zip(dates, files)}
            for m in months:
                date = years[y] + months[m]
                url_dict[date] = dirname + yr_dict[date]
        return url_dict

    # Extract urls
    if monthly:
        urls = monthly_urls(basedir, years, months, fmt)
    else:
        urls = daily_urls(basedir, years, months, fmt)

    return urls

# ======================================================================
# All functions below need to be revised to work with new get_urls
# ======================================================================

# ----------------------------------------------------------------------
def read_daily(var_ids, year, month, days=None, concat_dim='TIME',
               subset_dict=None, verbose=True):
    """Return MERRA daily pressure-level data for selected variable(s).

    Reads daily MERRA data from OpenDAP urls and concatenates into a
    single DataArray or Dataset for the selected days of the month.

    Parameters
    ----------
    var_ids : str or list of str
        Variable ID(s).  Can be generic ID from the list below, in which
        case get_varname() is called to get the specific ID for MERRA. Or
        var_id can be the exact name as it appears in MERRA data files.
        Generic IDs:
          {'u', 'v', 'omega', 'hgt', 'T', 'q', 'ps', 'evap', 'precip'}
    year, month : int
        Numeric year and month (1-12).
    days : list of ints, optional
       Subset of days to read. If None, all days are included.
    concat_dim : str, optional
        Name of dimension for concatenation.
    subset_dict : dict of 2-tuples, optional
        Dimensions and subsets to extract.  Each entry in subset_dict
        is in the form {dim_name : (lower_or_list, upper)}, where:
        - dim_name : string
            Name of dimension to extract from.
            The dimension name can be the actual dimension name
            (e.g. 'XDim') or a generic name (e.g. 'lon') and get_coord()
            is called to find the specific name.
        - lower_or_list : scalar or list of int or float
            If scalar, then used as the lower bound for the   subset range.
            If list, then the subset matching the list will be extracted.
        - upper : int, float, or None
            Upper bound for subset range. If lower_or_list is a list,
            then upper is ignored and should be set to None.
    verbose : bool, optional
        If True, print updates while processing files.

    Returns
    -------
    data : xray.DataArray or xray.Dataset
        Daily data (3-hourly or hourly) for the month or a selected
        subset of days.
    """

    var_ids = atm.makelist(var_ids)
    var_nms = [get_varname(var_id) for var_id in var_ids]
    dataset = get_dataset(var_ids[0], 'daily')
    urls = url_list(dataset)

    if days is None:
        # All days in the month
        dates = ['%d%02d' % (year, month)]
    elif isinstance(days, int):
        # Single day
        dates = ['%d%02d%02d' % (year, month, days)]
    else:
        # Subset of days
        dates = ['%d%02d%02d' % (year, month, d) for d in days]

    paths = []
    for date in dates:
        paths.extend([urls[key] for key in urls.keys() if date in key])

    data = atm.load_concat(paths, var_nms, concat_dim, subset_dict,
                           verbose)
    return data

# ----------------------------------------------------------------------
def read_daily_eta(var_id, level, year, month, days=None, concat_dim='TIME',
                   xsub='[330:2:450]', ysub='[60:2:301]', verbose=True):
    """Return MERRA daily eta-level data for a single variable.

    Reads a single eta level of daily MERRA data from OpenDAP urls and
    concatenates into a DataArray for the selected days of the month.

    Parameters
    ----------
    var_id : str
        Variable ID.  Can be generic ID from the list below, in which
        case get_varname() is called to get the specific ID for MERRA. Or
        var_id can be the exact name as it appears in MERRA data files.
        Generic IDs:
          {'u', 'v', 'omega', 'hgt', 'T', 'q', 'ps', 'evap', 'precip'}
    level : int
        Eta level to extract (0-71).  Level 71 is near-surface and level 0
        is the top of atmosphere.
    year, month : int
        Numeric year and month (1-12).
    days : list of ints, optional
       Subset of days to read. If None, all days are included.
    concat_dim : str, optional
        Name of dimension for concatenation.
    xsub, ysub : str, optional
        Indices of longitude and latitude subsets to extract.
    verbose : bool, optional
        If True, print updates while processing files.

    Returns
    -------
    data : xray.DataArray or xray.Dataset
        Daily data (3-hourly or hourly) for the month or a selected
        subset of days.
    """

    varnm = get_varname(var_id)
    tsub = '[0:1:3]'
    zsub = '[%d:1:%d]' % (level, level)

    def datafile(year, mon, day, varnm, xsub, ysub, zsub, tsub):
        basedir = ('http://goldsmr3.sci.gsfc.nasa.gov:80/opendap/MERRA/'
                   'MAI6NVANA.5.2.0/')
        url = ('%s%d/%02d/MERRA100.prod.assim.inst6_3d_ana_Nv.%d%02d%02d.hdf'
               '?%s%s%s%s%s,XDim%s,YDim%s,Height%s,TIME%s') % (basedir, year,
               mon, year, mon, day, varnm, tsub, zsub, ysub, xsub, xsub, ysub,
               zsub, tsub)
        return url

    if days is None:
        days = range(1, atm.days_this_month(year, month) + 1)
    urls = [datafile(year, month, day, varnm, xsub, ysub, zsub, tsub) for day
            in atm.makelist(days)]

    var = atm.load_concat(urls, varnm, concat_dim, verbose=verbose)

    return var

# ----------------------------------------------------------------------
def load_daily_season(pathstr, year, season='ann', var_ids=None,
                      lat1=-90, lat2=90, lon1=0, lon2=360,
                      verbose=True, concat_dim=None):
    """Return daily data for a selected year, season and lat-lon subset.

    Loads daily data from locally saved files and concatenates it into
    a single DataArray or Dataset for that year and season.

    Parameters
    ----------
    pathstr : str
       Beginning of path for each data file, where each file name is in
       the format *yyyymm.nc.
       e.g. pathstr = '~/datastore/merra/daily/u200_'
    year : int
       Year to load.
    season : str, optional
       Season to load. Valid values are as listed in atm.season_months()
       e.g. 'jul', 'jja', 'ann'
       Default is entire year ('ann')
    var_ids : str or list of str, optional
       Variable(s) to extract. If omitted, all variables in the data are
       included and the output is a Dataset.
    lat1, lat2, lon1, lon2 : floats, optional
        Lat-lon subset to extract.
    concat_dim : str, optional
        Name of time dimension for concatenation. If None, then
        atm.get_coord() is called to get the name from the data file.
    verbose : bool, optional
        If True, print updates while processing files.

    Returns
    -------
    data : xray.DataArray or xray.Dataset
    """

    months = atm.season_months(season)
    paths = []
    for m in months:
        datestr = '%d%02d' % (year, m)
        paths.append(pathstr + datestr + '.nc')

    # Make sure longitude range is consistent with data
    with xray.open_dataset(paths[0]) as ds:
        lonmax = atm.lon_convention(atm.get_coord(ds, 'lon'))
        if concat_dim is None:
            concat_dim = atm.get_coord(ds, 'time', 'name')
    if lon2 - lon1 == 360:
        if lonmax < lon2:
            offset = -180
        elif lonmax > lon2:
            offset = 180
        else:
            offset = 0
        lon1, lon2 = lon1 + offset, lon2 + offset
    print(lon1, lon2, lonmax)

    # Load daily data
    if var_ids is None:
        var_nms = None
    else:
        var_nms = [get_varname(var_id) for var_id in atm.makelist(var_ids)]
    subset_dict = {'lat' : (lat1, lat2), 'lon' : (lon1, lon2)}
    data = atm.load_concat(paths, var_nms, concat_dim, subset_dict, verbose)

    return data

# ----------------------------------------------------------------------
def calc_fluxes(year, month,
                var_ids=['u', 'q', 'T', 'theta', 'theta_e', 'hgt'],
                concat_dim='TIME', scratchdir=None, keepscratch=False,
                verbose=True):
    """Return the monthly mean of MERRA daily fluxes.

    Reads MERRA daily data from OpenDAP urls, computes fluxes, and
    returns the monthly mean of the daily variable and its zonal and
    meridional fluxes.

    Parameters
    ----------
    year, month : int
        Numeric year and month (1-12).
    var_ids : list of str, optional
        IDs of variables to include.
    concat_dim : str, optional
        Name of dimension for concatenation.
    scratchdir : str, optional
        Directory path to store temporary files while processing data.
        If omitted, the current working directory is used.
    keepscratch : bool, optional
        If True, scratch files are kept in scratchdir. Otherwise they
        are deleted.
    verbose : bool, optional
        If True, print updates while processing files.

    Returns
    -------
    data : xray.Dataset
        Mean of daily data and the mean of the daily zonal fluxes
        (u * var) and meridional fluxes (v * var), for each variable
        in var_ids.
    """

    nms = [get_varname(nm) for nm in atm.makelist(var_ids)]
    u_nm, v_nm = get_varname('u'), get_varname('v')
    nms.extend([u_nm, v_nm])
    if 'theta' in nms:
        nms.append(get_varname('T'))
    if 'theta_e' in nms:
        nms.extend([get_varname('T'), get_varname('q')])
    nms = set(nms)

    days = range(1, atm.days_this_month(year, month) + 1)

    def scratchfile(nm, k, year, month, day):
        filestr = '%s_level%d_%d%02d%02d.nc' % (nm, k, year, month, day)
        if scratchdir is not None:
            filestr = scratchdir + '/' + filestr
        return filestr

    # Read metadata from one file to get pressure-level array
    dataset = 'p_daily'
    url = url_list(dataset, return_dict=False)[0]
    with xray.open_dataset(url) as ds:
        pname = atm.get_coord(ds, 'plev', 'name')
        plev = atm.get_coord(ds, 'plev')
        # Pressure levels in Pa for theta/theta_e calcs
        p_units = atm.pres_units(ds[pname].units)
        pres = atm.pres_convert(plev, p_units, 'Pa')

    # Get daily data (raw and calculate extended variables)
    def get_data(nms, pres, year, month, day, concat_dim, subset_dict, verbose):
        # Lists of raw and extended variables
        ids = list(nms)
        ext = []
        for var in ['theta', 'theta_e']:
            if var in ids:
                ext.append(var)
                ids.remove(var)

        # Read raw data and calculate extended variables
        data = read_daily(ids, year, month, day, concat_dim=concat_dim,
                          subset_dict=subset_dict, verbose=verbose)
        if 'theta' in ext:
            print_if('Computing potential temperature', verbose)
            T = data[get_varname('T')]
            data['theta'] = atm.potential_temp(T, pres)
        if 'theta_e' in ext:
            print_if('Computing equivalent potential temperature', verbose)
            T = data[get_varname('T')]
            q = data[get_varname('q')]
            data['theta_e'] = atm.equiv_potential_temp(T, pres, q)

        return data

    # Iterate over vertical levels
    for k, p in enumerate(plev):
        subset_dict = {pname : (p, p)}
        print_if('Pressure-level %.1f' % p, verbose)

        files = []

        for day in days:
            # Read data for this level and day
            ds = get_data(nms, pres[k], year, month, day, concat_dim,
                           subset_dict, verbose)

            # Compute fluxes
            print_if('Computing fluxes', verbose)
            u = ds[get_varname('u')]
            v = ds[get_varname('v')]
            for nm in var_ids:
                var = ds[get_varname(nm)]
                varname, attrs, _, _ = atm.meta(var)
                u_var = u * var
                v_var = v * var

                u_var.name = get_varname(u_nm) + '*' +  var.name
                units = var.attrs['units'] + ' * ' + u.attrs['units']
                u_var.attrs['units'] = units
                v_var.name = get_varname(v_nm) + '*' +  var.name
                v_var.attrs['units'] = units
                ds[u_var.name] = u_var
                ds[v_var.name] = v_var

            # Save to temporary scratch file
            filenm = scratchfile('fluxes', k, year, month, day)
            files.append(filenm)
            print_if('Saving to scratch file ' + filenm, verbose)
            ds.to_netcdf(filenm)

        # Concatenate daily scratch files
        ds = atm.load_concat(files)

        if not keepscratch:
            for f in files:
                os.remove(f)

        # Compute monthly means
        print_if('Computing monthly means', verbose)
        if k == 0:
            data = ds.mean(dim=concat_dim)
        else:
            data = xray.concat([data, ds.mean(dim=concat_dim)], dim=pname)

    for var in data.data_vars:
        data[var].attrs = ds[var].attrs

    return data

# ======================================================================
# DEPRECATED
# ======================================================================
# ----------------------------------------------------------------------
def get_dataset(var_id, time_res='daily', default='p'):
    """Return the dataset ID corresponding to the variable.

    Parameters
    ----------
    var_id : str
        Variable name.  Can be a generic ID as input to get_varname(),
        or a specific name from MERRA data files.
    time_res : {'daily', 'monthly'}
        Time resolution of dataset.
    default : {'p', 'sfc'}
        If the variable is in both pressure-level and surface flux
        data, then default to this dataset type.

    Returns
    -------
    dataset : {'p_monthly', 'p_daily', 'sfc_monthly', 'sfc_daily'}
        Name of the dataset containing the variable (pressure-level
        or surface fluxes), at the specified time resolution.
    """

    var = get_varname(var_id)

    p_vars = [u'SLP', u'PS', u'PHIS', u'H', u'O3', u'QV', u'QL', u'QI', u'RH',
              u'T', u'U', u'V', u'EPV', u'OMEGA', u'Cov_U_V', u'Cov_U_T',
              u'Cov_V_T', u'Cov_U_H', u'Cov_V_H', u'Cov_U_QV', u'Cov_V_QV',
              u'Cov_U_QL', u'Cov_V_QL', u'Cov_U_QI', u'Cov_V_QI', u'Cov_U_EPV',
              u'Cov_V_EPV', u'Cov_U_O3', u'Cov_V_O3', u'Cov_OMEGA_U',
              u'Cov_OMEGA_V', u'Cov_OMEGA_T', u'Cov_OMEGA_QV', u'Cov_OMEGA_QL',
              u'Cov_OMEGA_QI', u'Cov_OMEGA_O3', u'vsts', u'Var_SLP', u'Var_PS',
              u'Var_PHIS', u'Var_H', u'Var_O3', u'Var_QV', u'Var_QL', u'Var_QI',
              u'Var_RH', u'Var_T', u'Var_U', u'Var_V', u'Var_EPV', u'Var_OMEGA']

    sfc_vars = [u'EFLUX', u'EVAP', u'HFLUX', u'TAUX', u'TAUY', u'TAUGWX',
                u'TAUGWY', u'PBLH', u'DISPH', u'BSTAR', u'USTAR', u'TSTAR',
                u'QSTAR', u'RI', u'Z0H', u'Z0M', u'HLML', u'TLML', u'QLML',
                u'ULML', u'VLML', u'RHOA', u'SPEED', u'CDH', u'CDQ', u'CDM',
                u'CN', u'TSH', u'QSH', u'FRSEAICE', u'PRECANV', u'PRECCON',
                u'PRECLSC', u'PRECSNO', u'PRECTOT', u'PGENTOT', u'Var_EFLUX',
                u'Var_EVAP', u'Var_HFLUX', u'Var_TAUX', u'Var_TAUY',
                u'Var_TAUGWX', u'Var_TAUGWY', u'Var_PBLH', u'Var_DISPH',
                u'Var_BSTAR', u'Var_USTAR', u'Var_TSTAR', u'Var_QSTAR',
                u'Var_RI', u'Var_Z0H', u'Var_Z0M', u'Var_HLML', u'Var_TLML',
                u'Var_QLML', u'Var_ULML', u'Var_VLML', u'Var_RHOA',
                u'Var_SPEED', u'Var_CDH', u'Var_CDQ', u'Var_CDM', u'Var_CN',
                u'Var_TSH', u'Var_QSH', u'Var_PRECANV', u'Var_PRECCON',
                u'Var_PRECLSC', u'Var_PRECSNO', u'Var_PRECTOT', u'Var_PGENTOT']

    if var in p_vars and var in sfc_vars:
        dataset = default + '_' + time_res
    elif var in p_vars:
        dataset = 'p_' + time_res
    elif var in sfc_vars:
        dataset = 'sfc_' + time_res
    else:
        raise ValueError('var_id ' + var_id + ' not found.')

    return dataset


from typing import Tuple
import numpy as np
import pandas as pd
import time

def split_train_test(X: pd.DataFrame, y: pd.Series, train_proportion: float = .75) \
        -> Tuple[pd.DataFrame, pd.Series, pd.DataFrame, pd.Series]:
    """
    Randomly split given sample to a training- and testing sample

    Parameters
    ----------
    X : DataFrame of shape (n_samples, n_features)
        Data frame of samples and feature values.

    y : Series of shape (n_samples, )
        Responses corresponding samples in data frame.

    train_proportion: Fraction of samples to be split as training set

    Returns
    -------
    train_X : DataFrame of shape (ceil(train_proportion * n_samples), n_features)
        Design matrix of train set

    train_y : Series of shape (ceil(train_proportion * n_samples), )
        Responses of training samples

    test_X : DataFrame of shape (floor((1-train_proportion) * n_samples), n_features)
        Design matrix of test set

    test_y : Series of shape (floor((1-train_proportion) * n_samples), )
        Responses of test samples

    """
    # join X_y to shuffle them simultaneously and then seperate them again
    y.name = 'labels'
    X_y = X.join(y)
    X_y = X_y.sample(frac=1).reset_index(drop=True)
    X, y = X_y.iloc[:, :-1], X_y.iloc[:, -1:]

    amount = int(np.ceil(train_proportion * y.size))
    return X.iloc[:amount], y.iloc[:amount], X.iloc[amount:], y.iloc[amount:]

def confusion_matrix(a: np.ndarray, b: np.ndarray) -> np.ndarray:
    """
    Compute a confusion matrix between two sets of integer vectors

    Parameters
    ----------
    a: ndarray of shape (n_samples,)
        First vector of integers

    b: ndarray of shape (n_samples,)
        Second vector of integers

    Returns
    -------
    confusion_matrix: ndarray of shape (a_unique_values, b_unique_values)
        A confusion matrix where the value of the i,j index shows the number of times value `i` was found in vector `a`
        while value `j` vas found in vector `b`
    """
    raise NotImplementedError()

def measure_time(f):
    """
    wrapper for a function f that prints the time it ran.
    :return:
    """
    def timed(*args, **kw):
        ts = time.time()
        result = f(*args, **kw)
        te = time.time()
        print('%r %2.2f sec' % (f.__name__, te - ts))
        return result

    return timed


import numpy as np
import pytest
from hypepy import hyp_test

def test_hyp_test_integer():
    '''
    Test if the input data is numeric, not a string or boolean.
    '''
    with pytest.raises(TypeError):
        hyp_test( data = "my data", mean_0=1, alpha=0.05)

    with pytest.raises(TypeError):
        hyp_test(data = True, mean_0=1, alpha=0.05)

def test_hyp_test_missing():
    '''
    Test if the input contains missing values.
    '''
    data = np.array([0, np.nan, 2])
    with pytest.raises(ValueError):
        hyp_test(data, mean_0=1, alpha = 0.05)

def test_hyp_test_non_zero():
    '''
    Test that the length of the data is greater than 0.
    '''
    with pytest.raises(ValueError):
        data = np.array([])
        hyp_test(data, mean_0=1, alpha=0.05)