""" Runs autogluon.tabular on synthetic classification and regression datasets.
    We test various parameters to TabularPredictor() and fit()
    We then check the leaderboard:
       Did we run the expected list of models
       Did each model have the expected score (within given range)
       Did the ensembling produce the expected score (within given range)
    This helps us spot any change in model performance.
    If any changes are spotted, the script does its best to dump out new proposed score ranges
    that you can cut and paste into the tests.  Only do this once you've identified the cause!

    Potential naming confusion:
        - this is a *regression* test, to make sure no functionality has accidentally got worse (testing terminology)
        - it runs two types of TabularPredictor tests : *regression* and classification (ML terminology)

    These tests are designed to run fast, to permit them to be run on a github hook.
    Currently the 11 tests, calling TabularPredictor.fit() 20 times, run in ~8 minutes on an 8 vcore machine with no GPU.

    Format of a test:

    {   # Default regression model on a small dataset
        'name':             # some unique name
        'type':             # either 'regression' or 'classification'.  We make a dataset using
                            # scikit-learn.make_{regression,classification}
        'n_samples':        # number of rows in the training dataset.  With TEST_SIZE default of 0.5,
                            # we make an additional n_samples for testing.
        'n_features': 2,    # number of columns.
        'n_categorical': 0, # number of categorical (discrete not continuous) columns.
        ''dataset_hash' :   # Hash of synthetic dataset to ensure the dataset itself didn't change.
        'params' : [ { 'predict' : {}, 'fit' : {} },   # If an array, we call TabularPredictor multiple times with
                                                       # different parameters.
                     { 'predict' : {}, 'fit' : {} },   # Pass the additional parameters to predict(), fit() or both
                                                       # in the dicts.
                                                       # If a scalar, we only call TabularPredictor once.
                   ],
        'expected_score_range' : {                     # A list of models we expect to run, and a valid score range we
                                                       # expect from each model.
                  'CatBoost': (-7.86, 0.01),           # The first value is the lower bound, the 2nd value is a delta
                  'ExtraTreesMSE': (-7.88, 0.01),      # to compute the upper bound, e.g. ( -8.12, 0.01 ) means we
                                                       # expect the score to be from -8.12 to -8.11 inclusive.
                  'CatBoost_BAG_L1': (np.nan, np.nan), # If np.nan, we expect this model to return np.nan as the score.
        },
    },

    Testing by @willsmithorg on master AG as of 2022-02-22 - 2022-02-23:
    Tested on AWS Linux instance m5.2xlarge, amzn2-ami-kernel-5.10-hvm-2.0.20211223.0-x86_64-gp2 with
                                               (8  vcore, no GPU, Python==3.7.10, scikit-learn==1.0.2, torch==1.10.2),
    Tested on Github jenkins Linux:
                                               (?  vcore,  0 GPU, Python==3.9.10, scikit-learn==1.0.2, torch==1.10.2),
    Tested on AWS Windows instance t3.xlarge,
                                               (4  vcore,  0 GPU, Python==3.9.7 , scikit-learn==1.0.2, torch==1.10.2),
                                               - Pytorch scores are slighty different, all else same.

"""
import sys
import math
import hashlib
import random

import numpy as np
import pandas as pd
from sklearn.datasets import make_classification, make_regression
from sklearn.model_selection import train_test_split

import pytest

from autogluon.tabular import TabularDataset, TabularPredictor

tests = [
    #
    # Regressions
    #
    {   # Default regression model on a small dataset
        'name': 'small regression',
        'type': 'regression',
        'n_samples': 100,
        'n_features': 2,
        'n_categorical': 0,
        'dataset_hash' : '5850a1c21a',
        'params' : [ { 'predict' : {}, 'fit' : {} },          # All of the followiing params should return same results because they're defaults
                     { 'predict' : {}, 'fit' : { 'presets' : 'medium_quality_faster_train' } },
                     { 'predict' : {}, 'fit' : { 'presets' : 'ignore_text' } },
                     { 'predict' : {}, 'fit' : { 'hyperparameters' : 'default' } },
                     { 'predict' : { 'eval_metric' : 'root_mean_squared_error'}, 'fit' : { } },
                   ],
        'expected_score_range' : {
                  'CatBoost': (-7.86, 0.01),
                  'ExtraTreesMSE': (-7.88, 0.01),
                  'KNeighborsDist': (-8.69, 0.01),
                  'KNeighborsUnif': (-9.06, 0.01),
                  'LightGBM': (-15.55, 0.01),
                  'LightGBMLarge': (-10.43, 0.01),
                  'LightGBMXT': (-16.32, 0.01),
                  'NeuralNetFastAI': (-6.12, 0.01),
                  'NeuralNetTorch': (-4.96, 0.01),
                  'RandomForestMSE': (-9.63, 0.01),
                  'WeightedEnsemble_L2': (-5.66, 0.01),
                  'XGBoost': (-10.8, 0.01),
        },
    },
    {   # If we explictly exclude some models the others should return unchanged and the ensemble result will be changed.
        'name': 'small regression excluded models',
        'type': 'regression',
        'n_samples': 100,
        'n_features': 2,
        'n_categorical': 0,
        'dataset_hash' : '5850a1c21a',
        'params' : { 'predict' : {}, 'fit' : { 'excluded_model_types' : [ 'KNN', 'RF', 'XT', 'GBM', 'CAT', 'XGB' ] } },
        'expected_score_range' : {
                  'NeuralNetFastAI': (-6.12, 0.01),
                  'NeuralNetTorch': (-4.96, 0.01),
                  'WeightedEnsemble_L2': (-5.09, 0.01),
        },
    },
    {   # Small regression, hyperparameters = light removes some models
        'name': 'small regression light hyperparameters',
        'type': 'regression',
        'n_samples': 100,
        'n_features': 2,
        'n_categorical': 0,
        'dataset_hash' : '5850a1c21a',
        'params' : { 'predict' : {}, 'fit' : { 'hyperparameters' : 'light' } },
        'expected_score_range' : {
                  'CatBoost': (-7.86, 0.01),
                  'ExtraTreesMSE': (-7.87, 0.01),
                  'LightGBM': (-15.55, 0.01),
                  'LightGBMLarge': (-10.43, 0.01),
                  'LightGBMXT': (-16.32, 0.01),
                  'NeuralNetFastAI': (-6.12, 0.01),
                  'NeuralNetTorch': (-4.96, 0.01),
                  'RandomForestMSE': (-9.63, 0.01),
                  'WeightedEnsemble_L2': (-5.66, 0.01),
                  'XGBoost': (-10.8, 0.01),
        },
    },
    {   # Small regression, hyperparameters = very_light removes some models
        'name': 'small regression very light hyperparameters',
        'type': 'regression',
        'n_samples': 100,
        'n_features': 2,
        'n_categorical': 0,
        'dataset_hash' : '5850a1c21a',
        'params' : { 'predict' : {}, 'fit' : { 'hyperparameters' : 'very_light' } },
        'expected_score_range' : {
                  'CatBoost': (-7.86, 0.01),
                  'LightGBM': (-15.55, 0.01),
                  'LightGBMLarge': (-10.43, 0.01),
                  'LightGBMXT': (-16.32, 0.01),
                  'NeuralNetFastAI': (-6.12, 0.01),
                  'NeuralNetTorch': (-4.96, 0.01),
                  'WeightedEnsemble_L2': (-5.58, 0.01),
                  'XGBoost': (-10.8, 0.01),
        },
    },
    {   # Small regression, hyperparameters = toy removes almost all models and runs very fast
        'name': 'small regression toy hyperparameters',
        'type': 'regression',
        'n_samples': 100,
        'n_features': 2,
        'n_categorical': 0,
        'dataset_hash' : '5850a1c21a',
        'params' : { 'predict' : {}, 'fit' : { 'hyperparameters' : 'toy' } },
        'expected_score_range' : {
                  'CatBoost': (-28.39, 0.01),
                  'LightGBM': (-27.81, 0.01),
                  'NeuralNetTorch': (-27.11, 0.01),
                  'WeightedEnsemble_L2': (-19.12, 0.01),
                  'XGBoost': (-19.12, 0.01),
        },
    },
    {   # High quality preset on small datset.
        'name': 'small regression high quality',
        'type': 'regression',
        'n_samples': 100,
        'n_features': 2,
        'n_categorical': 0,
        'dataset_hash' : '5850a1c21a',
        'params' : { 'predict' : {}, 'fit' : { 'presets' : 'high_quality_fast_inference_only_refit' } },
        'expected_score_range' : {
                  'CatBoost_BAG_L1': (np.nan, np.nan),
                  'CatBoost_BAG_L1_FULL': (-7.75, 0.01),
                  'ExtraTreesMSE_BAG_L1': (-7.52, 0.01),
                  'ExtraTreesMSE_BAG_L1_FULL': (-7.52, 0.01),
                  'KNeighborsDist_BAG_L1': (-8.21, 0.01),
                  'KNeighborsDist_BAG_L1_FULL': (-8.21, 0.01),
                  'KNeighborsUnif_BAG_L1': (-8.7, 0.01),
                  'KNeighborsUnif_BAG_L1_FULL': (-8.7, 0.01),
                  'LightGBMLarge_BAG_L1': (np.nan, np.nan),
                  'LightGBMLarge_BAG_L1_FULL': (-9.94, 0.01),
                  'LightGBMXT_BAG_L1': (np.nan, np.nan),
                  'LightGBMXT_BAG_L1_FULL': (-13.03, 0.01),
                  'LightGBM_BAG_L1': (np.nan, np.nan),
                  'LightGBM_BAG_L1_FULL': (-14.17, 0.01),
                  'NeuralNetFastAI_BAG_L1': (np.nan, np.nan),
                  'NeuralNetFastAI_BAG_L1_FULL': (-5.48, 0.01),
                  'NeuralNetTorch_BAG_L1': (np.nan, np.nan),
                  'NeuralNetTorch_BAG_L1_FULL': (-5.29, 0.01),
                  'RandomForestMSE_BAG_L1': (-9.5, 0.01),
                  'RandomForestMSE_BAG_L1_FULL': (-9.5, 0.01),
                  'WeightedEnsemble_L2': (np.nan, np.nan),
                  'WeightedEnsemble_L2_FULL': (-5.29, 0.01),
                  'XGBoost_BAG_L1': (np.nan, np.nan),
                  'XGBoost_BAG_L1_FULL': (-9.76, 0.01),
        }
    },
    {   # Best quality preset on small datset.
        'name': 'small regression best quality',
        'type': 'regression',
        'n_samples': 100,
        'n_features': 2,
        'n_categorical': 0,
        'dataset_hash' : '5850a1c21a',
        'params' : { 'predict' : {}, 'fit' : { 'presets' : 'best_quality' } },
        'expected_score_range' : {
                  'CatBoost_BAG_L1': (-7.85, 0.01),
                  'ExtraTreesMSE_BAG_L1' : (-7.52, 0.01),
                  'KNeighborsDist_BAG_L1' : (-8.21, 0.01),
                  'KNeighborsUnif_BAG_L1' : (-8.70, 0.01),
                  'LightGBMLarge_BAG_L1' : (-9.44, 0.01),
                  'LightGBMXT_BAG_L1' : (-14.78, 0.01),
                  'LightGBM_BAG_L1' : (-14.92, 0.01),
                  'NeuralNetFastAI_BAG_L1' : (-5.55, 0.01),
                  'NeuralNetTorch_BAG_L1' : (-5.07, 0.01),
                  'RandomForestMSE_BAG_L1' : (-9.5, 0.01),
                  'WeightedEnsemble_L2' : (-5.05, 0.01),   # beats default, as expected
                  'XGBoost_BAG_L1' : (-9.74, 0.01),
        }
    },
    {   # Default regression model, add some categorical features.
        'name': 'small regression with categorical',
        'type': 'regression',
        'n_samples': 100,
        'n_features': 2,
        'n_categorical': 1,
        'dataset_hash' : '3e26d128e0',
        'params' : { 'predict' : {}, 'fit' : {} },          # Default params
        'expected_score_range' : {
                 'CatBoost': (-22.58, 0.01),
                 'ExtraTreesMSE': (-25.09, 0.01),
                 'KNeighborsDist': (-39.45, 0.01),
                 'KNeighborsUnif': (-35.64, 0.01),
                 'LightGBM': (-32.96, 0.01),
                 'LightGBMLarge': (-34.86, 0.01),
                 'LightGBMXT': (-32.69, 0.01),
                 'NeuralNetFastAI': (-22.11, 0.01),
                 'NeuralNetTorch': (-19.76, 0.01),
                 'RandomForestMSE': (-27.49, 0.01),
                 'WeightedEnsemble_L2': (-19.76, 0.01),
                 'XGBoost': (-24.93, 0.01),

        }
    },
    {   # Default regression model different metric
        'name': 'small regression metric mae',
        'type': 'regression',
        'n_samples': 100,
        'n_features': 2,
        'n_categorical': 0,
        'dataset_hash' : '5850a1c21a',
        'params' : { 'predict' : { 'eval_metric' : 'mean_absolute_error'}, 'fit' : { } },
        'expected_score_range' : {
                  'CatBoost': (-5.23, 0.01),
                  'ExtraTreesMSE': (-5.48, 0.01),
                  'KNeighborsDist': (-6.16, 0.01),
                  'KNeighborsUnif': (-6.61, 0.01),
                  'LightGBM': (-11.97, 0.01),
                  'LightGBMLarge': (-7.69, 0.01),
                  'LightGBMXT': (-12.37, 0.01),
                  'NeuralNetFastAI': (-4.74, 0.01),
                  'NeuralNetTorch': (-3.77, 0.01),
                  'RandomForestMSE': (-6.96, 0.01),
                  'WeightedEnsemble_L2': (-4.03, 0.01),
                  'XGBoost': (-8.32, 0.01),

        },
    },
    #
    # Classifications
    #
    {   # Default classification model on a small dataset
        'name': 'small classification',
        'type': 'classification',
        'n_samples': 400,  # With only 8 classes it's hard to compare model quality unless we have a biggest test set (and therefore train set).
        'n_features': 10,
        'n_informative': 5,
        'n_classes': 8,
        'n_categorical': 0,
        'dataset_hash' : 'be1f16df80',
        'params' : [ { 'predict' : {}, 'fit' : {} },     # All of the followiing params should return same results
                     { 'predict' : {}, 'fit' : { 'presets' : 'medium_quality_faster_train' } },
                     { 'predict' : {}, 'fit' : { 'presets' : 'ignore_text' } },
                     { 'predict' : {}, 'fit' : { 'hyperparameters' : 'default' } },
                     { 'predict' : { 'eval_metric' : 'accuracy'}, 'fit' : { } },
                   ],
        'expected_score_range' : {
                 'CatBoost': (0.245, 0.001),            # Classification scores are low numbers so we decrease the
                 'ExtraTreesEntr': (0.327, 0.001),      # tolerance to 0.001 to make sure we pick up changes.
                 'ExtraTreesGini': (0.32, 0.001),
                 'KNeighborsDist': (0.337, 0.001),
                 'KNeighborsUnif': (0.322, 0.001),
                 'LightGBM': (0.197, 0.001),
                 'LightGBMLarge': (0.265, 0.001),
                 'LightGBMXT': (0.23, 0.001),
                 'NeuralNetFastAI': (0.34, 0.001),
                 'NeuralNetTorch': (0.232, 0.001),
                 'RandomForestEntr': (0.305, 0.001),
                 'RandomForestGini': (0.295, 0.001),
                 'WeightedEnsemble_L2': (0.34, 0.001),
                 'XGBoost': (0.227, 0.001),
        }
    },
    {   # There's different logic for boolean classification so let's test that with n_classes = 2.
        'name': 'small classification boolean',
        'type': 'classification',
        'n_samples': 400,
        'n_features': 10,
        'n_informative': 5,
        'n_classes': 2,
        'n_categorical': 0,
        'dataset_hash' : '79e634aac3',
        'params' : [ { 'predict' : {}, 'fit' : {} },      # All of the followiing params should return same results
                     { 'predict' : { 'eval_metric' : 'accuracy'}, 'fit' : { } },
                   ],
        'expected_score_range' : {
                  'CatBoost': (0.61, 0.001),
                  'ExtraTreesEntr': (0.607, 0.001),
                  'ExtraTreesGini': (0.6, 0.001),
                  'KNeighborsDist': (0.61, 0.001),
                  'KNeighborsUnif': (0.61, 0.001),
                  'LightGBM': (0.632, 0.001),
                  'LightGBMLarge': (0.552, 0.001),
                  'LightGBMXT': (0.612, 0.001),
                  'NeuralNetFastAI': (0.62, 0.001),
                  'NeuralNetTorch': (0.597, 0.001),
                  'RandomForestEntr': (0.607, 0.001),
                  'RandomForestGini': (0.582, 0.001),
                  'WeightedEnsemble_L2': (0.61, 0.001),
                  'XGBoost': (0.58, 0.001),
        }
    },
]

# Lots of test data since inference is fast and we want a score that's very reflective of model quality,
# despite very fast training times.
TEST_SIZE=0.5
def make_dataset(request, seed):
    # Ensure our datasets and model calls remain deterministic.
    random.seed(seed)
    np.random.seed(seed)
    if request['type'] == 'regression':

        x, y = make_regression(n_samples = int(request['n_samples']*(1/(1-TEST_SIZE))),
                               n_features = request['n_features'],
                               noise=4) # To make it hard enough that we get better performance on slower models
    elif request['type'] == 'classification':
        x, y = make_classification(n_samples = int(request['n_samples']*(1/(1-TEST_SIZE))),
                               n_features = request['n_features'],
                               n_informative = request['n_informative'],
                               n_redundant = request['n_classes'] - request['n_informative'],
                               n_classes = request['n_classes'],
                               class_sep=0.4) # To make it hard enough that we get better performance on slower models
    else:
        assert False, "Unrecognised request type '{request['type'}'"

    dfx = pd.DataFrame(x)
    dfy = pd.DataFrame(y, columns=['label'])

    # Make some columns categorical if required.
    if request['n_categorical'] > 0:
        cols_to_convert = random.sample(set(dfx.columns.values), k=request['n_categorical'])
        for col in cols_to_convert:
            dfx[col] = dfx[col].astype(int)
            vals = np.unique(dfx[col])
            # Shuffle the categoricals so there's no pattern in their ordering.
            vals2 = vals.copy()-min(vals)
            np.random.shuffle(vals2)
            mapper = dict(zip(vals, vals2))
            dfx[col] = dfx[col].map(mapper)
            dfx[col] = dfx[col].astype("category")

    x_train, x_test, y_train, y_test = train_test_split(dfx, dfy, test_size=TEST_SIZE)
    dftrain = pd.concat([x_train, y_train], axis=1)
    dftest  = pd.concat([x_test,  y_test],  axis=1)

    return (dftrain, dftest)

# Round to given accuracy.  The 8 is to remove floating point rounding errors.
def myfloor(x, base=.01):
  return round(base * math.floor(float(x)/base),8)

@pytest.mark.regression
def inner_test_tabular(testname):

    # Find the named test
    test = None
    for t in tests:
        if t['name'] == testname:
            test = t
    assert test is not None, f"Could not find test {testname}"

    # Build the dataset
    (dftrain, dftest) = make_dataset(request=test, seed=0)

    # Check the synthetic dataset itself hasn't changed.  We round it to 3dp otherwise tiny floating point differences
    # between platforms can give a different hash that still yields same prediction scores.
    # Ultimately it doesn't matter how we do this as long as the same dataset gives the same hash function on
    # different python versions and architectures.
    current_hash = hashlib.sha256(dftrain.round(decimals=3).values.tobytes()).hexdigest()[0:10]
    proposedconfig = "Proposed new config:\n"
    proposedconfig += f"'dataset_hash' : '{current_hash}',"
    assert current_hash == test['dataset_hash'], f"Test '{testname}' input dataset has changed.  All scores will change.\n" + proposedconfig

    # Now run the Predictor 1 or more times with various parameters, and make sure we get
    # back the expected results.

    # Params can either omitted, or a single run, or a list of runs.
    if 'params' not in test:
        test['params'] = { 'predict' : {}, 'fit' : {} }
    if not isinstance(test['params'], list):
        test['params'] = [ test['params'] ]
    for params in test['params']:

        # Run this model and set of params
        predictor = TabularPredictor(label='label', **params['predict'])
        predictor.fit(dftrain, **params['fit'])
        leaderboard = predictor.leaderboard(dftest, silent=True)
        leaderboard = leaderboard.sort_values(by='model') # So we can pre-generate sample config in alphabetical order

        # Store proposed new config based on the current run, in case the developer wants to keep thee results (just cut and paste).
        proposedconfig = "Proposed new config:\n"
        proposedconfig += "'expected_score_range' : {\n";
        for model in leaderboard['model']:
            midx_in_leaderboard = leaderboard.index.values[leaderboard['model'] == model][0]
            if np.isnan(leaderboard['score_test'][midx_in_leaderboard]):
                 values = "np.nan, np.nan"
            else:
                 if model in test['expected_score_range'] and not np.isnan(test['expected_score_range'][model][1]):
                     currentprecision = test['expected_score_range'][model][1]
                 else:
                     currentprecision = 0.01
                 values = "{}, {}".format(myfloor(leaderboard['score_test'][midx_in_leaderboard], currentprecision), currentprecision)
            proposedconfig += f"    '{model}': ({values}),\n"
        proposedconfig += "},\n"

        # First validate the model list was as expected.
        assert set(leaderboard['model']) == set(test['expected_score_range'].keys()), (f"Test '{testname}' params {params} got unexpected model list.\n" + proposedconfig)

        # Now validate the scores for each model were as expected.
        all_assertions_met = True
        currentconfig = "Existing config:\n"
        currentconfig += "'expected_score_range' : {\n";
        for model in sorted(test['expected_score_range']):
            midx_in_leaderboard = leaderboard.index.values[leaderboard['model'] == model][0]
            assert leaderboard['model'][midx_in_leaderboard] == model
            expectedrange = test['expected_score_range'][model][1]
            expectedmin = test['expected_score_range'][model][0]
            expectedmax = expectedmin + expectedrange

            if np.isnan(expectedmin):
                 values = "np.nan, np.nan"
            else:
                 values = "{}, {}".format(expectedmin, expectedrange)

            if (((leaderboard['score_test'][midx_in_leaderboard] >= expectedmin) and
               (leaderboard['score_test'][midx_in_leaderboard] <= expectedmax)) or
               (np.isnan(leaderboard['score_test'][midx_in_leaderboard]) and np.isnan(expectedmin))):
                currentconfig += f"    '{model}': ({values}),\n"
            else:
                currentconfig += f"    '{model}': ({values}), # <--- not met, got {leaderboard['score_test'][midx_in_leaderboard]} \n"
                all_assertions_met = False
        currentconfig += "},\n"

        assert all_assertions_met, f"Test '{testname}', params {params} had unexpected scores:\n" + currentconfig + proposedconfig

        # Clean up this model created with specific params.
        predictor.delete_models(models_to_keep=[], dry_run=False)

# The tests are all run individually rather than in 1 big loop that simply goes through the tests dictionary.
# This is so we easily remove some tests if necessary.
@pytest.mark.parametrize("testname", [
    'small regression',
    'small regression excluded models',
    'small regression light hyperparameters',
    'small regression very light hyperparameters',
    'small regression toy hyperparameters',
    'small regression high quality',
    'small regression best quality',
    'small regression with categorical',
    'small regression metric mae',
    'small classification',
    'small classification boolean',
])

# These results have only been confirmed for Linux.  Windows is known to give different results for Pytorch.
@pytest.mark.skipif(sys.platform != 'linux', reason='Scores only confirmed on Linux')
@pytest.mark.regression
def test_tabular_score(testname):
    inner_test_tabular(testname)


"""
Copyright 2020 The OneFlow Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import unittest
from collections import OrderedDict

import numpy as np
from oneflow.test_utils.test_util import GenArgList

import oneflow as flow
import oneflow.unittest

from oneflow.test_utils.automated_test_util import *

def _test_type_as(test_case, shape, src_dtype, tgt_dtype, placement, sbp):
    np_input = np.random.rand(*shape)
    input = flow.tensor(np_input, dtype=src_dtype).to_global(placement, sbp)
    target = flow.tensor(np_input, dtype=tgt_dtype).to_global(placement, sbp)
    input = input.type_as(target)
    test_case.assertEqual(input.dtype, target.dtype)

def _test_is_floating_point(test_case, shape, dtype, placement, sbp):
    np_input = np.random.rand(*shape)
    input = flow.tensor(np_input, dtype=dtype).to_global(placement, sbp)
    output = input.is_floating_point()
    if input.dtype in (flow.float, flow.float16, flow.float32, flow.double):
        test_case.assertEqual(output, True)
    else:
        test_case.assertEqual(output, False)

@autotest(n=1, check_graph=False)
@unittest.skipIf(os.getenv("ONEFLOW_TEST_CPU_ONLY"), "only test cpu cases")
def _test_global_cuda(test_case, placement, sbp):
    x = random_tensor(2, 8, 16).to_global(placement, sbp)
    x = x.cuda()
    y = x.sum()
    return y

class TestConsistentCuda(flow.unittest.TestCase):
    @globaltest
    def test_global_cuda(test_case):
        for placement in all_placement():
            for sbp in all_sbp(placement, max_dim=2):
                _test_global_cuda(test_case, placement, sbp)

@autotest(n=1, check_graph=False)
def _test_global_cpu(test_case, placement, sbp):
    x = random_tensor(2, 8, 16).to_global(placement, sbp)
    x = x.cpu()
    y = x.sum()
    return y

# PyTorch error if open auto_backward:
# element 0 of tensors does not require grad and does not have a grad_fn
@autotest(n=1, auto_backward=False, check_graph=False)
def _test_global_long(test_case, placement, sbp):
    x = random_tensor(2, 8, 16, requires_grad=True).to_global(placement, sbp)
    y = x.long()
    test_case.assertFalse(y.oneflow.requires_grad)
    return y

@autotest(n=1, auto_backward=False, check_graph=False)
def _test_global_int(test_case, placement, sbp):
    x = random_tensor(2, 8, 16, requires_grad=True).to_global(placement, sbp)
    y = x.int()
    test_case.assertFalse(y.oneflow.requires_grad)
    return y

@autotest(n=1, auto_backward=False, check_graph=False)
def _test_global_float(test_case, placement, sbp):
    x = random_tensor(2, 8, 16, dtype=int).to_global(placement, sbp)
    y = x.float()
    return y

@autotest(n=1, auto_backward=False, check_graph=False)
def _test_global_double(test_case, placement, sbp):
    x = random_tensor(2, 8, 16, dtype=int).to_global(placement, sbp)
    y = x.double()
    return y

@autotest(n=1, auto_backward=False, check_graph=False)
def _test_global_item(test_case, placement, sbp):
    x = random_tensor(ndim=1, dim0=1, dtype=int).to_global(placement, sbp)
    y = torch.tensor(x.item())
    return y

@autotest(n=1, auto_backward=False, check_graph=False)
def _test_global_tolist(test_case, placement, sbp):
    x = random_tensor(ndim=4, dim0=8, dim1=16, dim2=24, dim3=32, dtype=int).to_global(
        placement, sbp
    )
    y = torch.tensor(x.tolist())
    return y

class TestConsistentTensorOps(flow.unittest.TestCase):
    @globaltest
    def test_global_cpu(test_case):
        for placement in all_placement():
            for sbp in all_sbp(placement, max_dim=2):
                _test_global_cpu(test_case, placement, sbp)

    @globaltest
    def test_global_long(test_case):
        for placement in all_placement():
            for sbp in all_sbp(placement, max_dim=2):
                _test_global_long(test_case, placement, sbp)

    @globaltest
    def test_global_int(test_case):
        for placement in all_placement():
            for sbp in all_sbp(placement, max_dim=2):
                _test_global_int(test_case, placement, sbp)

    @globaltest
    def test_global_float(test_case):
        for placement in all_placement():
            for sbp in all_sbp(placement, max_dim=2):
                _test_global_float(test_case, placement, sbp)

    @globaltest
    def test_global_double(test_case):
        for placement in all_placement():
            for sbp in all_sbp(placement, max_dim=2):
                _test_global_double(test_case, placement, sbp)

    @globaltest
    def test_global_item(test_case):
        for placement in all_placement():
            for sbp in all_sbp(placement, max_dim=1, except_split=True):
                _test_global_item(test_case, placement, sbp)

    @globaltest
    def test_global_tolist(test_case):
        for placement in all_placement():
            for sbp in all_sbp(placement, max_dim=4):
                _test_global_tolist(test_case, placement, sbp)

    @globaltest
    def test_type_as(test_case):
        arg_dict = OrderedDict()
        arg_dict["shape"] = [(8, 16), (8, 16, 24), (8, 16, 24, 32)]
        arg_dict["src_dtype"] = [flow.int64, flow.int32, flow.float32, flow.float64]
        arg_dict["tgt_dtype"] = [flow.int64, flow.int32, flow.float32, flow.float64]
        for arg in GenArgList(arg_dict):
            for placement in all_placement():
                for sbp in all_sbp(placement, max_dim=len(arg[0])):
                    _test_type_as(test_case, *arg, placement, sbp)

    @globaltest
    def test_is_floating_point(test_case):
        arg_dict = OrderedDict()
        arg_dict["shape"] = [(8, 16), (8, 16, 24), (8, 16, 24, 32)]
        arg_dict["dtype"] = [
            # flow.uint8, nccl don't support uint8
            flow.int8,
            flow.int32,
            flow.int64,
            flow.float32,
            flow.float64,
            flow.double,
            flow.float,
            flow.int,
        ]
        for arg in GenArgList(arg_dict):
            for placement in all_placement():
                for sbp in all_sbp(placement, max_dim=len(arg[0])):
                    _test_is_floating_point(test_case, *arg, placement, sbp)

if __name__ == "__main__":
    unittest.main()


#!/usr/bin/env python
# coding: utf-8

# # 竞赛系列：商品评论情感预测分析

# 电商平台中,很多用户都会基于自己的购物体验对商品进行评分和评论.但有些用户只给出了评论而没有评分，没有了评分的量化标准，这给商家进行数据运营与选品决策带来了困难。如何根据商品评论估计出相对应的评分，这是情感分析的问题，而得到这些信息，也有利于对应商品的生产自身竞争力的提高，以及为用户提供高质量感兴趣的商品。
# 我们将采用 PaddleNLP 对本数据集的数据进行情感分析，用 NLP 自然语言处理中常用的 Transformer 模型进行训练

# 数据简介
#
# 本数据集包括52 万件商品，1100 多个类目，142 万用户，720 万条评论/评分数据
# 本次练习赛所使用数据集基于JD的电商数据，来自WWW的JD.com E-Commerce Data，并且针对部分字段做出了一定的调整，所有的字段信息请以本练习赛提供的字段信息为准
# 评分为[1,5] 之间的整数

# 二、数据初步处理

# !pip install -U paddlenlp

# 1.解压数据
# !tar -xvf data/data96333/商品评论情感预测.gz

# **2.查看数据**
# !head 训练集.csv
#
# !head 测试集.csv
#
# !head submission.csv

# **3.重写read方法读取自定义数据集**

# In[6]:

from paddlenlp.datasets import load_dataset
from paddle.io import Dataset, Subset
from paddlenlp.datasets import MapDataset
import re

# 数据ID,用户ID,商品ID,评论时间戳,评论标题,评论内容,评分
def read(data_path):
    with open(data_path, 'r', encoding='utf-8') as in_f:
        next(in_f)
        for line in in_f:
            line = line.strip('\n')
            split_array = [i.start() for i in re.finditer(',', line)]
            id = line[:split_array[0]]
            comment_title = line[split_array[3] + 1:split_array[4]]
            comment = line[split_array[4] + 2:split_array[-2]]
            label = line[split_array[-1] + 1:]
            yield {'text': comment_title  +' '+ comment, 'label': str(int(label.split('.')[0])-1), 'qid': id}

# 数据ID,用户ID,商品ID,评论时间戳,评论标题,评论内容,评分
def read_test(data_path):
    with open(data_path, 'r', encoding='utf-8') as in_f:
        next(in_f)
        for line in in_f:
            line = line.strip('\n')
            split_array = [i.start() for i in re.finditer(',', line)]
            id = line[:split_array[0]]
            id=id.split('_')[-1]
            comment_title = line[split_array[3] + 1:split_array[4]]
            comment = line[split_array[4] + 2:split_array[-2]]
            label= '1'
            yield {'text': comment_title  +' '+ comment, 'label': label, 'qid': id}

# **4.训练集载入**

# In[3]:

# data_path为read()方法的参数
dataset_ds = load_dataset(read, data_path='训练集.csv',lazy=False)
# 在这进行划分
train_ds = Subset(dataset=dataset_ds, indices=[i for i in range(len(dataset_ds)) if i % 10 != 1])
dev_ds = Subset(dataset=dataset_ds, indices=[i for i in range(len(dataset_ds)) if i % 10 == 1])

test_ds =  load_dataset(read_test, data_path='测试集.csv',lazy=False)

# In[4]:

for i in range(5):
    print(test_ds[i])

# In[5]:

# 在转换为MapDataset类型
train_ds = MapDataset(train_ds)
dev_ds = MapDataset(dev_ds)
test_ds = MapDataset(test_ds)
print(len(train_ds))
print(len(dev_ds))
print(len(test_ds))

# **三、SKEP模型加载**

# In[7]:

# 指定模型名称一键加载模型
from paddlenlp.transformers import SkepForSequenceClassification, SkepTokenizer

model = SkepForSequenceClassification.from_pretrained(
    'skep_ernie_1.0_large_ch', num_classes=  5)
# 指定模型名称一键加载tokenizer
tokenizer = SkepTokenizer.from_pretrained('skep_ernie_1.0_large_ch')

# **四、数据NLP特征处理**

# In[ ]:

import os
from functools import partial

import numpy as np
import paddle
import paddle.nn.functional as F
from paddlenlp.data import Stack, Tuple, Pad

from utils import create_dataloader

def convert_example(example,
                    tokenizer,
                    max_seq_length=512,
                    is_test=False):

    # 将原数据处理成model可读入的格式，enocded_inputs是一个dict，包含input_ids、token_type_ids等字段
    encoded_inputs = tokenizer(
        text=example["text"], max_seq_len=max_seq_length)

    # input_ids：对文本切分token后，在词汇表中对应的token id
    input_ids = encoded_inputs["input_ids"]
    # token_type_ids：当前token属于句子1还是句子2，即上述图中表达的segment ids
    token_type_ids = encoded_inputs["token_type_ids"]

    if not is_test:
        # label：情感极性类别
        label = np.array([example["label"]], dtype="int64")
        return input_ids, token_type_ids, label
    else:
        # qid：每条数据的编号
        qid = np.array([example["qid"]], dtype="int64")
        return input_ids, token_type_ids, qid

# In[ ]:

from utils import create_dataloader
# 处理的最大文本序列长度
max_seq_length=256
# 批量数据大小
batch_size=35

train_ds = Subset(dataset=dataset_ds, indices=[i for i in range(len(dataset_ds)) if i % 10 != 1])
dev_ds = Subset(dataset=dataset_ds, indices=[i for i in range(len(dataset_ds)) if i % 10 == 1])

# 将数据处理成模型可读入的数据格式
trans_func = partial(
    convert_example,
    tokenizer=tokenizer,
    max_seq_length=max_seq_length)

# 将数据组成批量式数据，如
# 将不同长度的文本序列padding到批量式数据中最大长度
# 将每条数据label堆叠在一起
batchify_fn = lambda samples, fn=Tuple(
    Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input_ids
    Pad(axis=0, pad_val=tokenizer.pad_token_type_id),  # token_type_ids
    Stack()  # labels
): [data for data in fn(samples)]
train_data_loader = create_dataloader(
    train_ds,
    mode='train',
    batch_size=batch_size,
    batchify_fn=batchify_fn,
    trans_fn=trans_func)
dev_data_loader = create_dataloader(
    dev_ds,
    mode='dev',
    batch_size=batch_size,
    batchify_fn=batchify_fn,
    trans_fn=trans_func)

# **五、模型训练**

# **1.训练准备**

# In[ ]:

import time

from utils import evaluate

# 训练轮次
epochs = 10
# 训练过程中保存模型参数的文件夹
ckpt_dir = "skep_ckpt"
# len(train_data_loader)一轮训练所需要的step数
num_training_steps = len(train_data_loader) * epochs

# Adam优化器
optimizer = paddle.optimizer.AdamW(
    learning_rate=2e-5,
    parameters=model.parameters())
# 交叉熵损失函数
criterion = paddle.nn.loss.CrossEntropyLoss()
# accuracy评价指标
metric = paddle.metric.Accuracy()

# **2.开始训练**

# In[ ]:

# 开启训练

# 加入日志显示
from visualdl import LogWriter

writer = LogWriter("./log")
best_val_acc=0
global_step = 0
tic_train = time.time()
for epoch in range(1, epochs + 1):
    for step, batch in enumerate(train_data_loader, start=1):
        input_ids, token_type_ids, labels = batch
        # 喂数据给model
        logits = model(input_ids, token_type_ids)
        # 计算损失函数值
        loss = criterion(logits, labels)
        # 预测分类概率值
        probs = F.softmax(logits, axis=1)
        # 计算acc
        correct = metric.compute(probs, labels)
        metric.update(correct)
        acc = metric.accumulate()

        global_step += 1
        if global_step % 10 == 0:
            print(
                "global step %d, epoch: %d, batch: %d, loss: %.5f, accu: %.5f, speed: %.2f step/s"
                % (global_step, epoch, step, loss, acc,
                    10 / (time.time() - tic_train)))
            tic_train = time.time()

        # 反向梯度回传，更新参数
        loss.backward()
        optimizer.step()
        optimizer.clear_grad()

        if global_step % 100 == 0:
            # 评估当前训练的模型
            eval_loss, eval_accu = evaluate(model, criterion, metric, dev_data_loader)
            print("eval  on dev  loss: {:.8}, accu: {:.8}".format(eval_loss, eval_accu))
            # 加入eval日志显示
            writer.add_scalar(tag="eval/loss", step=global_step, value=eval_loss)
            writer.add_scalar(tag="eval/acc", step=global_step, value=eval_accu)
            # 加入train日志显示
            writer.add_scalar(tag="train/loss", step=global_step, value=loss)
            writer.add_scalar(tag="train/acc", step=global_step, value=acc)
            save_dir = "best_checkpoint"
            # 加入保存
            if eval_accu>best_val_acc:
                if not os.path.exists(save_dir):
                    os.mkdir(save_dir)
                best_val_acc=eval_accu
                print(f"模型保存在 {global_step} 步， 最佳eval准确度为{best_val_acc:.8f}！")
                save_param_path = os.path.join(save_dir, 'best_model.pdparams')
                paddle.save(model.state_dict(), save_param_path)
                fh = open('best_checkpoint/best_model.txt', 'w', encoding='utf-8')
                fh.write(f"模型保存在 {global_step} 步， 最佳eval准确度为{best_val_acc:.8f}！")
                fh.close()

# **六、预测提交结果**

# **1.测试数据集处理**

# In[ ]:

test_ds =  load_dataset(read_test, data_path='测试集.csv',lazy=False)
# 在转换为MapDataset类型
test_ds = MapDataset(test_ds)
print(len(test_ds))

# In[ ]:

import numpy as np
import paddle

# 处理测试集数据
trans_func = partial(
    convert_example,
    tokenizer=tokenizer,
    max_seq_length=max_seq_length,
    is_test=True)
batchify_fn = lambda samples, fn=Tuple(
    Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input
    Pad(axis=0, pad_val=tokenizer.pad_token_type_id),  # segment
    Stack() # qid
): [data for data in fn(samples)]
test_data_loader = create_dataloader(
    test_ds,
    mode='test',
    batch_size=batch_size,
    batchify_fn=batchify_fn,
    trans_fn=trans_func)

# **2.加载预测模型**

# In[ ]:

# 根据实际运行情况，更换加载的参数路径
params_path = 'best_checkpoint/best_model.pdparams'
if params_path and os.path.isfile(params_path):
    # 加载模型参数
    state_dict = paddle.load(params_path)
    model.set_dict(state_dict)
    print("Loaded parameters from %s" % params_path)

# **3.开始预测**

# In[ ]:

# 处理测试集数据
label_map = {0: '1', 1:'2', 2:'3', 3:'4',4:'5'}
results = []
# 切换model模型为评估模式，关闭dropout等随机因素
model.eval()
for batch in test_data_loader:
    input_ids, token_type_ids, qids = batch
    # 喂数据给模型
    logits = model(input_ids, token_type_ids)
    # 预测分类
    probs = F.softmax(logits, axis=-1)
    idx = paddle.argmax(probs, axis=1).numpy()
    idx = idx.tolist()
    labels = [label_map[i] for i in idx]
    qids = qids.numpy().tolist()
    results.extend(zip(qids, labels))

# **4.保存结果**

# In[ ]:

# 写入预测结果
with open( "submission.csv", 'w', encoding="utf-8") as f:
    # f.write("数据ID,评分\n")
    f.write("id,score\n")

    for (idx, label) in results:
        f.write('TEST_'+str(idx[0])+","+label+"\n")

# **5.检查结果**

# In[ ]:

get_ipython().system('tail 测试集.csv')


from IPython.display import HTML

HTML('''
The raw code for this IPython notebook is by default hidden for easier reading.
To toggle on/off the raw code, click here.''')

# Radioactive Decay Interactives

The following interactive figures were borrowed from [Astro Interactives](https://juancab.github.io/AstroInteractives/) and adapted for easy access into this Jupyter Book. The first one shows the random yet predictable nature of radioactive decay and its use as a time clock.  The second one illustrates Geochron plots.

## Interactive Figure 1: Model of Radioactive Decay

This first figure takes a population of 900 atoms and models their radioactive decay.

This first interactive is designed to allow you to explore what happens during radioactive decay of some isotope. An *isotope* refers to a particular version of an element.  For example, the non-radioactive Carbon-12 isotope has 6 protons and 6 neutrons in its nucleus but the radioactive Carbon-14 isotope has 6 protons and 8 neutrons in its nucleus.  Different isotopes of the same element behave identically in terms of chemistry and bonding to other atoms, but their nuclear properties can differ.

During radioactive decay, a *parent isotope* is said to decay into a *daughter isotope*.  So, for example, the parent isotope of Carbon-14 decays into Nitrogen-14.  There is no way to predict when any one nucleus of a *parent isotope* will decay into a *daughter isotope*.   That said, if you look at a large number of nuclei of a parent isotope, they exhibit a very simple property:

> **The same *fraction* of a radioactive parent isotope will decay over the same amount of time.**

The time it takes for one-half of a population of parent isotopes to decay (on average) into daughter isotopes is called the *half-life* of that isotope.  Different parent isotopes can have very different half-lifes.   **NOTE:** This interactive shows a simulation of the decay of only 900 atoms.  The radioactive decay is still modelled as occurring randomly for any one atom, so the simulation will show slightly different results on different runs!!

Some questions to consider
1. After 1 half-life, about 50% of a parent isotope should have decayed and become daughter isotope.  Use the interactive graphic below and adjust the elapsed time to figure out how long one half-life is for each isotope.  Explain your approach.
2. How much of a parent isotope is left after 2 half-lifes? 3 half-lifes? 4 half-lifes? 5 half-lifes?  Explain how you figured this out.
3. You should have found about 25% of the original amount of parent isotope is left after 2 half-lifes.  This may seem surprising since in the first half-life 50% of the original amount of parent isotope decayed.  Explain why 'less' of it decayed during the second half-life.  **HINT** Consider the very simple property of radioactive decay we highlighted above.

from IPython.display import display
import numpy as np
import bqplot as bq
import ipywidgets as widgets
import random as random
import pandas as pd
import number_formatting as nf
from math import ceil, floor, log10

## Originally developed June 2018 by Samuel Holen
##
## Edits by Juan Cabanela October 2018 to allow changes in the GUI.
## - Made the display of the precise number/faction of parent/daught atoms instructor
##   configurable.
## - Fixed a problem with the display of data, forced data to only be displayed for first
##   10 half-lifes regardless of actual generated decay times (this allows hard-coding of)
##   num_ticks and tick values.

## Pre-construct model of radioactive decay of a population
## of parent and daughter atoms.
##

# GUI Configuration Parameters
show_counts = True
max_half_lifes = 10   # maximum half-lives to graph
time_ticks = 6  # number of ticks for the time axis
# half-lives to place ticks on horizontal axis
half_life_ticks = np.linspace(0, max_half_lifes, time_ticks)

# Constants Related to decay of the parent species to the daughter species
N_parent = 900          # initial number of parent atoms (should be a perfect square)
N_daughter = 0          # initial number of daughter atoms
tau = 1                 # placeholder for the half-life of the parent species
h = 0.025               # time step (in half lives)
mu = np.log(2.) / tau   # constant for decay time distribution
Plot_all_times = True   # Plot all times (otherwise, selects only before time slider value)

# Initialize tracking of number of atoms
Parent_counts = []          # list of number of parent atoms
Dauther_counts = []          # list of number of daughter atoms

# Generate a uniform random distribution of N_parent numbers from 0 to 1
z = np.random.rand(N_parent)

# Function to convert uniform distribution of random numbers to
# a distribution weighted to model radiactive decay. The times are in number of
# half-lives.
#
# The unsorted data representing the number of half-lifes until the individual
# decay of each atom.
decay_times = -np.log(1 - z) / mu
decay_times_sorted = np.sort( decay_times )

# Genereate array of numbers of atoms left
# Adjusted so that each count contains 0 and N_parent
Parent_counts = np.arange(N_parent,-1, -1, dtype='int') # Number of parent atoms
Daughter_counts = np.ones_like(Parent_counts)
Daughter_counts = N_parent - Parent_counts   # Number of daughter atoms

#
# Construct Pandas data frames
#

# Time column adjusted to include t=0
decay_data = pd.DataFrame()
decay_data['time'] = np.concatenate((np.zeros(1), decay_times_sorted))
decay_data['Parent'] = Parent_counts
decay_data['Daughter'] = Daughter_counts

# Data array for species
species = pd.DataFrame()
species['parent_long'] = ['Generic', 'Carbon', 'Thallium','Uranium','Rubidium']
species['daughter_long'] = ['Generic', 'Nitrogen','Lead','Thorium','Strontium']
species['parent_short'] = ['Parent', 'C-14','Tl-208','U-235','Rb-87']
species['daughter_short'] = ['Daughter', 'N-14','Pb-208','Th-231','Sr-87']
species['half-lives'] = [tau, 5730, 3.053 * 60, 703.8, 48.8]
species['step-size'] = [h, 125, 0.05 * 60, 15, 1]
species['timeunits'] = ['half-lives', 'years', 'seconds', 'million years', 'billion years']

##
## Define functions to respond to controls on population plots.
##

def UpdateSpecies(change=None):
    ##
    ## Deal with possible changes of species
    ##

    # Generate a new uniform random distribution of N_parent numbers from 0 to 1 and
    # then convert uniform distribution to one weighted to model radioactive decay.
    z = np.random.rand(N_parent)
    decay_times = -np.log(1 - z) / mu
    decay_times_sorted = np.sort( decay_times )
    decay_data['time'] = np.concatenate((np.zeros(1), decay_times_sorted))

    # Reset the time to zero
    Time_slide.value = 0

    # Adjust half-life data and limits on plots appropriately
    new_index = species.loc[species.parent_long == pick_Species.value].index[0]

    # Set the half-life based the selected species
    hf = species['half-lives'][new_index]

    # Set the limit on the time sider to be 10 half-lifes
    Time_slide.max = max_half_lifes*hf

    # Set the time step on slider
    Time_slide.step = species['step-size'][new_index]

    # Updates the time slider label/units
    unit_label.value = species['timeunits'][new_index]

    # Updates the time axes on the plot
    x_time.max = Time_slide.max
    ax_x_time.scale = x_time
    ax_x_time.label = unit_label.value

    # Update tick values
    ax_x_time.tick_values = list(half_life_ticks*hf)

    # Update the legend
    parent_label_new = species['parent_short'][new_index]
    daughter_label_new = species['daughter_short'][new_index]
    line_parent.labels = [parent_label_new]
    line_daughter.labels = [daughter_label_new]

    # Update the species in the box that shows how many are present
    parent_label.value = parent_label_new + ' produced'
    daughter_label.value = daughter_label_new + ' remaining'

    # Now call the function for updating the plot in the event of a time change
    UpdateTimes()

def UpdateFraction(change=None):
    ##
    ## Deal with whether fraction view or number view selected to set scales for plot,
    ## y values for plot, and label values
    ##
    if frac_or_num.value == False:
        # Number count mode enabled

        # Update axes and scales
        fig_counts.axes = [ax_x_time, ax_y_number]
        line_parent.scales={'x': x_time, 'y': y_number}
        line_daughter.scales={'x': x_time, 'y': y_number}
        pts_parent.scales={'x': x_time, 'y': y_number}
        pts_daughter.scales={'x': x_time, 'y': y_number}
        line_time.scales={'x': x_time, 'y': y_number}

        # Update 'time line' limits
        line_time.y = [0, N_parent]
    else:
        # Fraction mode enabled

        # Updated axes and scales
        fig_counts.axes = [ax_x_time, ax_y_fraction]
        line_parent.scales={'x': x_time, 'y': y_fraction}
        line_daughter.scales={'x': x_time, 'y': y_fraction}
        pts_parent.scales={'x': x_time, 'y': y_fraction}
        pts_daughter.scales={'x': x_time, 'y': y_fraction}
        line_time.scales={'x': x_time, 'y': y_fraction}

        # Update 'time line' limits
        line_time.y = [0, 1]

    UpdateTimes()

def UpdateTimes(change=None):
    ##
    ## Deal with changes in the time slider
    ##

    # Recall half-life of this species
    species_idx = species.loc[species.parent_long == pick_Species.value].index[0]
    hf = species['half-lives'][species_idx]

    # Update time label with 3 significant figures
    Time_label.value = str(nf.SigFig(Time_slide.value, 3))

    # Get the array of times
    time_arr = hf * decay_data['time']

    # Set the times to be x values
    line_daughter.x = time_arr
    line_parent.x = line_daughter.x
    pts_daughter.x = line_daughter.x
    pts_parent.x = line_daughter.x
    line_time.x = [Time_slide.value, Time_slide.value]

    # Changes the color of population to reflect the decays
    for i in range(N_parent):
        if Time_slide.value >= decay_times[i]*hf:
            Colors[i] = 'blue'
        else:
            Colors[i] = 'red'
    population_scat.colors = Colors

    # Identify where we are in the data set
    i = 0
    while i < N_parent + 1 and hf*decay_data['time'][i] < Time_slide.value:
        i += 1
    if i > 0:
        i -= 1

    # Update selection of the parent and daughter data to be plotted
    if Plot_all_times:
        daughter_decay = decay_data['Daughter']
        parent_decay = decay_data['Parent']
    else:
        daughter_decay = decay_data['Daughter'][0:i+1]
        parent_decay = decay_data['Parent'][0:i+1]

    line_parent.y = parent_decay
    line_daughter.y = daughter_decay

    ##
    ## Deal with whether fraction view or number view selected to set scales for plot,
    ## y values for plot, and label values
    ##
    if frac_or_num.value == False:
        # Number count mode enabled

        # Update number of parent and daughter in labels
        parent_present.value = str(decay_data['Parent'][i])
        daughter_present.value = str(decay_data['Daughter'][i])
    else:
        # Fraction mode enabled

        # Update the x and y arrays for the parent and daughter lines
        line_parent.y = (1/N_parent)*parent_decay
        line_daughter.y = (1/N_parent)*daughter_decay

        # Update number of parent and daughter in labels
        parent_present.value = '{:.3f}'.format((1/N_parent)*decay_data['Parent'][i])
        daughter_present.value = '{:.3f}'.format((1/N_parent)*decay_data['Daughter'][i])

    # Update parent and daughter lines
    pts_parent.y = line_parent.y
    pts_daughter.y = line_daughter.y

    # Update the tooltip labels and formats (Bug? Doesn't appear to be working)
    #pts_parent.tooltip.labels[1] = parent_label_new
    #pts_daughter.tooltip.labels[1] = daughter_label_new
    #if frac_or_num.value == False:
    #    pts_parent.tooltip.formats[1] = '3.0f'
    #    pts_daughter.tooltip.formats[1] = '3.0f'
    #else:
    #    pts_parent.tooltip.formats[1] = '0.3f'
    #    pts_daughter.tooltip.formats[1] = '0.3f'

##
## Set up counts versus time plot
##

# Set up Species to be Generic
init_species_ind = 0

# Set up initial time
init_time = 0
init_time_idx = 0

# Set up axes
x_time = bq.LinearScale(min = 0, max=max_half_lifes)
y_number = bq.LinearScale(min = 0, max=N_parent)
y_fraction = bq.LinearScale(min = 0, max=1)

# Labels and scales for Axes
ax_x_time = bq.Axis(label=species['timeunits'][init_species_ind], scale=x_time, num_ticks = time_ticks,
                    tick_values = half_life_ticks )
ax_y_number = bq.Axis(label='Number of atoms', scale=y_number, orientation='vertical')
ax_y_fraction = bq.Axis(label='Fraction of atoms', scale=y_fraction, orientation='vertical')

# Define tooltip (Bug: doesn't allow relabeling Tooltips, also needs to apply to Scatter, not Lines)
#def_tt_parent = bq.Tooltip(fields=['x', 'y'], formats=['.2f', '3.0f'], labels=['time', species['parent_short'][init_species_ind]])
#def_tt_daughter = bq.Tooltip(fields=['x', 'y'], formats=['.2f', '3.0f'], labels=['time', species['daughter_short'][init_species_ind]])
def_tt_parent = bq.Tooltip(fields=['x', 'y'], formats=['.2f', '.3f'], labels=['time', 'amount of parent isotope'])
def_tt_daughter = bq.Tooltip(fields=['x', 'y'], formats=['.2f', '.3f'], labels=['time', 'amount of daughter isotope'])

# Define the Lines and Scatter plots
# NOTE: Scatter only necessary to allow tooltips to function.
init_x = decay_data['time']*species['half-lives'][init_species_ind]
if Plot_all_times:
    init_parent = decay_data['Parent']
    init_daughter = decay_data['Daughter']
else:
    init_parent = decay_data['Parent'][0:init_time_idx]
    init_daughter = decay_data['Daughter'][0:init_time_idx]

pts_parent = bq.Scatter(x=init_x, y=init_parent,
                      scales={'x': x_time, 'y': y_number}, marker='circle', default_size=2,
                      display_legend=False, colors=['red'], labels=[species['parent_short'][init_species_ind]],
                      tooltip=def_tt_parent)
pts_daughter = bq.Scatter(x=init_x, y=init_daughter,
                        scales={'x': x_time, 'y': y_number}, marker='circle', default_size=2,
                        display_legend=False, colors=['blue'], labels=[species['daughter_short'][init_species_ind]],
                        tooltip=def_tt_daughter)
line_parent = bq.Lines(x=init_x, y=init_parent,
                       scales={'x': x_time, 'y': y_number}, display_legend=True, colors=['red'],
                       labels=[species['parent_short'][init_species_ind]], )
line_daughter = bq.Lines(x=init_x, y=init_daughter,
                         scales={'x': x_time, 'y': y_number}, display_legend=True, colors=['blue'],
                         labels=[species['daughter_short'][init_species_ind]] )

# Set up a vertical line on this plot to indicate the current time
times_x = [init_time, init_time]
times_y = [0, N_parent]
line_time = bq.Lines(x=times_x, y=times_y,
                     scales={'x': x_time, 'y': y_number},
                     colors=['greenyellow'])

# Creates figure for plot
fig_counts = bq.Figure(axes=[ax_x_time, ax_y_number], marks=[line_parent, line_daughter, line_time, pts_parent, pts_daughter],
                       legend_location='right', legend_style={'fill': 'white'},
                       title='Counts versus Time', background_style={'fill': 'black'},
                       layout={'width': '500px', 'min_height': '400px'},
                      animation=1000)

# Slider widget to control the amount of time that has passed
# Set for generic half-life situation initially (0 to 10 half-lifes)
Time_slide = widgets.FloatSlider(
    value=0.,
    description='Time',
    min=0.,
    max=max_half_lifes,
    step=h,
    disabled=False,
    continuous_update=False,
    orientation='horizontal',
    readout=False,
    readout_format='.1f',
    layout=widgets.Layout(overflow_x='visible',
                          overflow_y='visible',
                          width='400px',
                          max_width='500px',
                          min_width='250px')
)

# Widget to display the number of parent atoms present
parent_present = widgets.Text(
    value = str(N_parent),
    style = {'description_width': 'initial'},
    #description = species['parent_short'][0]+' remaining',
    disabled = True,
    layout=widgets.Layout(overflow_x='visible',
                          overflow_y='visible',
                          width='175px',
                          max_width='300px',
                          min_width='125px')
)

# Widget to display the number of daughter atoms present
daughter_present = widgets.Text(
    value = str(0),
    style = {'description_width': 'initial'},
    #description = species['daughter_short'][0]+' produced',
    disabled = True,
    layout=widgets.Layout(overflow_x='visible',
                          overflow_y='visible',
                          width='175px',
                          max_width='300px',
                          min_width='125px')
)

# Widgets to label the time slider with units
Time_label = widgets.Label(value=str(Time_slide.value))
unit_label = widgets.Label(value=str(species['timeunits'][0]))
Composite_Time_Label = widgets.HBox([Time_label, unit_label],
                                   layout=widgets.Layout(width='200px'))

# Labels for the parent/daughter present displays
parent_label = widgets.Label(value=species['parent_short'][0]+' remaining',
                             layout=widgets.Layout(width='200px',
                                                   overflow_x='visible',
                                                   overflow_y='visible') )
daughter_label = widgets.Label(value=species['daughter_short'][0]+' produced',
                             layout=widgets.Layout(width='200px',
                                                   overflow_x='visible',
                                                   overflow_y='visible') )

# Checkbox to choose whether to display the number of each species
# or the fraction of each
frac_or_num = widgets.Checkbox(value=False, description='Display as fraction of atoms')

# Widget to allow one to choose which species to work with
pick_Species = widgets.RadioButtons(options=species['parent_long'][:],
                                    value='Generic', description='Species:', disabled=False,
                                    layout=widgets.Layout(width='250px'))

# Scale for population figure
x_sc = bq.LinearScale(min=1, max=np.sqrt(N_parent))
y_sc = bq.LinearScale(min=1, max=np.sqrt(N_parent))

# Axes for population figure
ax_x = bq.Axis(scale=x_sc, num_ticks=0)
ax_y = bq.Axis(scale=y_sc, orientation='vertical', num_ticks=0)

# Creates an array of x values: [1,2,...,30,1,2,...30,.....,1,2,...,30]
x_ls = []
for i in range(1,int(np.sqrt(N_parent))+1):
    x_ls.append(float(i))
x_ls = x_ls * int(np.sqrt(N_parent))
x_arr = np.array(x_ls)

# Creates an array of y values: [1,1,...,1,2,2,...2,......,30,30,...,30]
y_ls = []
for i in range(1,int(np.sqrt(N_parent))+1):
    y_ls += [float(i)] * int(np.sqrt(N_parent))
y_arr = np.array(y_ls)

# Creates a color array with the same number of entries as the number of atoms in
# the sample
Colors = ['red'] * N_parent

# Plot the population model
population_scat = bq.Scatter(x=x_arr, y=y_arr, scales={'x': x_sc, 'y': y_sc}, colors =['red'])

# Picking a new species resets everything
pick_Species.observe(UpdateSpecies, names=['value'])

# Update view from fraction to/from number
frac_or_num.observe(UpdateFraction, names=['value'])

# Update times
Time_slide.observe(UpdateTimes, names=['value'])
Time_label.observe(UpdateTimes, names=['value'])

# Figure for the population
fig_population = bq.Figure(title='Population of Atoms', marks=[population_scat], axes=[ax_x, ax_y],
                background_style={'fill' : 'black'},padding_x = 0.025,
                min_aspect_ratio=1, max_aspect_ratio=1)

# Boxes to organize display
parent_box = widgets.HBox([parent_label, parent_present],
                          layout=widgets.Layout(overflow_x='visible', overflow_y='visible'))
daughter_box = widgets.HBox([daughter_label, daughter_present],
                            layout=widgets.Layout(overflow_x='visible', overflow_y='visible'))
# Set visibility of the exact counts/fractions
if (show_counts == False):
    parent_box.layout.visibility = 'hidden'
    daughter_box.layout.visibility = 'hidden'

value_box = widgets.VBox([parent_box, daughter_box])
species_box = widgets.HBox([value_box, pick_Species])

slide_box = widgets.HBox([Time_slide, Composite_Time_Label])
slide_check_box = widgets.VBox([slide_box, frac_or_num])

top_box = widgets.HBox([fig_counts, fig_population],
                      layout=widgets.Layout(width='900px'))
top_box.children[0].layout.width = '450px'
top_box.children[1].layout.width = '450px'

bottom_box = widgets.HBox([species_box, slide_check_box],
                      layout=widgets.Layout(width='900px'))
bottom_box.children[0].layout.width = '450px'
bottom_box.children[1].layout.width = '450px'

# Final display
Final = widgets.VBox([top_box, bottom_box])
Final.layout.overflow = 'hidden'
display(Final)

## Interactive Figure 2: Geochron Plot

Assuming a non-radiogenic isotope (that is, an isotope that is not the result of radioactive decay) that also will not decay, its amount should be constant.  This means that for different mineral samples we can measure the ratio of parent isotope versus the non-radiogenic isotope ($P/D_i$) and daughter isotope ($D$) versus the non-radiogenic isotope ($D/D_i$) to build an geochron plot.  For example, using the following isotopes

- $D_i$ (non-radiogenic isotope of daughter element)
- $D$ (Daughter Isoptope)
- $P$ (Parent isotope)

an geochron plot could plot $D/D_i$ versus $P/D_i$.

What sets the *geochron method* (also known as the *isochron method*) apart from the just measuring parent and daughter abundances is the use of the non-radiogenic isotope of the daughter element.  This avoids the assumption of no initial daughter isotope before the rock solidified (radioactive decay can occur while rock is molten).

Some minerals in the rock incorporate the parent better than daughter which is why the initial amount of parent
isotope versus daughter isotope can vary.  We expect daughter versus non-radiogenic isotope ratio to be constant
if we pick the non-radiogenic isotope to be the same element as the daughter isotope.

With all this said, it is actually often not this simple as many daughter isotopes are themselves radioactive and decay, leading to a chain of reactions, so comparing abundances of parent to daughter isotopes is not simple.

*Note:* The idea for the geochron dating interactive came from a Isochron Diagram Java app at *ScienceCourseware.org*.  However that app had some issues in that it didn't divide by a non-radiogenic isotope (or at least didn't mention it).  In fact, they used $D_i$ for the initial amount of daughter isotope instead of the non-radiogenic isotope of the same element as the daughter isotope.

Consider the following questions:

1. Notice that the geochron shown here is using the parent and daugther isotope ratios for 4 mineral samples in a rock.  Consider the sample that starts off with the most parent isotope initially (the point initially farthest to the right).  Adjust the geochron to show the samples after 1 half-life.  How much of that parent isotope was there initially?  How much is there after 1 half-life?  Does this make sense?
2. Consider the sample that starts off with the least parent isotope initially (the point initially farthest to the left).  Adjust the geochron to show the samples after 1 half-life.  How much of that parent isotope was there initially?  How much is there after 1 half-life?  Does this make sense?
3. What fraction of the parent isotope should have decayed into daughter isotope after one half-life?  Does it matter if the mineral sample we look at started with more parent isotope than another mineral sample in the same rock?  Why or why not?
4. Why is it as time elapses that the points representing the 4 mineral samples all move toward the upper left?  Can you explain why their motions are parallel to one another?
4. Imagine you are looking at the parent isotope Rubidium-87 which decays into Strontium-87.  You examine 4 mineral samples in an meteorite and they line up in a line with a slope of 0.0666.  Determine the age of that meteorite (or rather, the time since it solidified).  Also explain how we can determine how much of the daughter isotope was in the meteorite initially.

##
## Define the various isotopes we want to consider
##

isotope_info = pd.DataFrame(columns=['Name', 'PName', 'PAbbrev', 'DName', 'DAbbrev', 'DiName', 'DiAbbrev', 'HalfLife', 'HLUnits'])
isotope_info['index'] = ['generic', 'Rb87']
isotope_info['Name'] = ['Generic', 'Rb-87->Sr-87']
isotope_info['PName'] = ['Parent', 'Rubidium-87']
isotope_info['PAbbrev'] = ['P', 'Rb-87']
isotope_info['DName'] = ['Daughter', 'Strontium-87']
isotope_info['DAbbrev'] = ['D', 'Sr-87']
isotope_info['DiName'] = ['Non-Radiogenic Isotope of Daughter Element', 'Strontium-86']
isotope_info['DiAbbrev'] = ['D_i', 'Sr-86']
isotope_info['HalfLife'] = [ 1, 48.8 ]
isotope_info['HLUnits'] = [ 'half-lives', 'Billion years']
isotope_info = isotope_info.set_index('index')

# Set initial isotope to plot
init_isotope = 'generic'

##
## Define the initial amounts of parent and daughter in the sample.
##
## In principle, I would change this depending on the isotopes we plot.  But I am only plotting
## Rb87 --> Sr-87, since that is the most classical use of this Geochron approach.
##

# Range of P to D_i fractions and initial amounts of D to D_i to consider
P2Di_min = 0.05
P2Di_max = 0.40
D2Di0_min = 0.05
D2Di0_max = 0.75

# Generate three mineral samples in different thirds of the entire range
range_P2Di  = (P2Di_max-P2Di_min)

# Create sample amounts
n_samples = 4
nums = np.array(list(range(1, n_samples+1)))
initial_samples = pd.DataFrame(index=nums)
initial_D2Di0 = D2Di0_min + (D2Di0_max - D2Di0_min) * np.random.random()
initial_samples['P2Di'] = P2Di_min + (range_P2Di/n_samples) * (nums - np.random.random(n_samples))
initial_samples['D2Di'] = initial_D2Di0*np.ones_like(nums)

##
## Define functions to call when building interactive plot
##

def amt_left(sample_in, taus):
    # Generate a sample DataFrame after tau half-lifes given an initial DataFrame
    sample = sample_in.copy(deep = True)
    sample['P2Di'] = sample_in['P2Di']*((1/2)**(taus))
    sample['D2Di'] = sample_in['D2Di'] + sample_in['P2Di']*(1 - (1/2)**(taus))
    return sample

def line_points(sample):
    global x_min, x_max, y_min, y_max, initial_D2Di0

    # Determine the end points of a line going through the sample points.
    x_range = x_max - x_min
    y_range = y_max - y_min

    # Slope (extrapolate from first two points - could be done by a fit to the points)
    slope = (sample['D2Di'][2]-sample['D2Di'][1])/(sample['P2Di'][2]-sample['P2Di'][1])
    y_final = initial_D2Di0 + slope*x_range
    x_points = (x_min, x_max)
    y_points = (initial_D2Di0, y_final)
    return x_points, y_points, slope

def init2current(samples0, samples):
    # Compute the lines connecting initital and final points for plotting
    n_pts = len(samples0)

    xlist = []
    ylist = []
    for pt in range(1, n_pts+1):
        x = np.array([ samples0['P2Di'][pt], samples['P2Di'][pt] ])
        y = np.array([ samples0['D2Di'][pt], samples['D2Di'][pt] ])
        xlist.append(x)
        ylist.append(y)

    return(xlist, ylist)

def HL_changed(change):
    global isotope, sample, initial_samples, dots_current, line_current, connectors, slope_label

    # Determine half-life of this isotope
    idx = (isotope_info.Name == isotope.value)
    HL = float(isotope_info[idx].HalfLife.tolist()[0])

    # How many half-lives have passed?  Use this to get new sample and line info
    this_tau = HL_slider.value / HL
    sample = amt_left(initial_samples, this_tau)
    x_sample, y_sample, slope =  line_points(sample)

    # Update plot
    dots_current.x = sample['P2Di']
    dots_current.y = sample['D2Di']
    line_current.x = x_sample
    line_current.y = y_sample
    slope_label.value = 'Slope: {0:0.4f}'.format(slope)
    xlist, ylist = init2current(initial_samples, sample)
    connectors.x = xlist
    connectors.y = ylist

def isotope_changed(change):
    global ax_x_P2Di, ax_y_D2Di, HL_slider, HLlabel, UnitsText, Max_half_lives

    # Extract the necessary isotope descriptors from the Pandas DataFrame
    idx = (isotope_info.Name == change.new)
    HL = float(isotope_info[idx].HalfLife.tolist()[0])
    HLUnits = isotope_info[idx].HLUnits.tolist()[0]
    PAbbrev = isotope_info[idx].PAbbrev.tolist()[0]
    DAbbrev = isotope_info[idx].DAbbrev.tolist()[0]
    DiAbbrev = isotope_info[idx].DiAbbrev.tolist()[0]

    # Get old half-life
    idx_old = (isotope_info.Name == change.old)
    HL_old = float(isotope_info[idx_old].HalfLife.tolist()[0])

    # Determine current age reading from slider and adjust to new units
    init_age = HL_slider.value

    # Hard code generic versus others
    if (change.new != isotope_info.loc['generic'].Name):
        HL_slider.description = "Time"
    else:
        HL_slider.description = "Half-lives"

    # Adjust time scales
    if (HL_old < HL):
        # Adjust maximum limits first before adjusting values (since new HL > old HL)
        HL_slider.max = Max_half_lives*HL
        HLlabel.max = HL_slider.max
        HL_slider.value = HL*(init_age/HL_old)
        HLlabel.value = HL_slider.value
    else:
        # Adjust maximum limits after adjusting values (since new HL < old HL)
        HL_slider.value = HL*(init_age/HL_old)
        HLlabel.value = HL_slider.value
        HL_slider.max = Max_half_lives*HL
        HLlabel.max = HL_slider.max

    # Set the axes and other labels to display
    UnitsText.value = HLUnits
    ax_x_P2Di.label = '{0} / {1}'.format(PAbbrev, DiAbbrev)
    ax_y_D2Di.label = '{0} / {1}'.format(DAbbrev, DiAbbrev)

##
## Set up isochron plot
##

# Largest possible fraction of decay (only go out to 5 half-lives)
Max_half_lives = 5
Max_decay_fraction = 1 - (1/2)**(Max_half_lives)

# detemine maximum and minimum values of X and Y axes
x_step = 0.05
x_min = 0
x_max = x_step * ceil(initial_samples['P2Di'][n_samples] / x_step)
y_step = 0.04
y_min = y_step * floor(initial_D2Di0 / y_step)
y_max = y_step * ceil((initial_D2Di0 + initial_samples['P2Di'][n_samples] * Max_decay_fraction) / y_step)

# Labels and scales for Axes
x_P2Di = bq.LinearScale(min = x_min, max = x_max)
y_D2Di = bq.LinearScale(min = y_min, max = y_max)
ax_x_P2Di = bq.Axis(label='P / D_i', scale=x_P2Di)
ax_y_D2Di = bq.Axis(label='D / D_i', scale=y_D2Di, orientation='vertical')

# Set up initial conditions
taus = 0    # zero half lives past
sample = amt_left(initial_samples, taus)

##
## Define the lines
##

# Initial amount of daughter line (with dots for initial amounts of parent)
x_init, y_init, slope_init =  line_points(initial_samples)
line_initial = bq.Lines(x=x_init, y=y_init, scales={'x': x_P2Di, 'y': y_D2Di},
                   line_style='dashed', colors=['red'], labels=['Initial Sample'])
dots_initial = bq.Scatter(x=initial_samples['P2Di'], y=initial_samples['D2Di'], scales={'x': x_P2Di, 'y': y_D2Di},
                   colors=['white'], stroke='red', fill= True, labels=['Initial Isochron'])

# Current quantities on isochron line
x_sample, y_sample, slope =  line_points(sample)
line_current = bq.Lines(x=x_sample, y=y_sample, scales={'x': x_P2Di, 'y': y_D2Di},
                   line_style='solid', colors=['red'], labels=['Current Isochron'])
dots_current = bq.Scatter(x=sample['P2Di'], y=sample['D2Di'], scales={'x': x_P2Di, 'y': y_D2Di},
                   colors=['red'], stroke='red', fill= True, labels=['Current Isochron'])

# Connect Initial and Current quantities on isochron line
xlist, ylist = init2current(initial_samples, sample)
connectors = bq.Lines(x=xlist, y=ylist, scales={'x': x_P2Di, 'y': y_D2Di},
                   line_style='dotted', colors=['black'])

##
## Construct plot
##
isochron = bq.Figure(axes=[ax_x_P2Di, ax_y_D2Di],
                     marks=[connectors, line_initial, dots_initial, line_current, dots_current],
                     title='Geochron Diagram',
                     layout={'width': '700px', 'height': '500px',
                             'max_width': '700px', 'max_height': '500px',
                             'min_width': '600px', 'min_height': '400px'})

##
## Construct controls
##

# Select Generic or Specific Isotopes
isotope = widgets.RadioButtons(options=list(isotope_info.Name),
                               value=isotope_info.loc[init_isotope].Name, description='Isotope:',
                               disabled=False,
                               layout=widgets.Layout(height='75px', max_height='100px', min_height='50px',
                                                    width='200px', max_width='300px',  min_width='100px'))
isotope.observe(isotope_changed, 'value')

# Slider and text field controling age
idx = (isotope_info.Name == isotope.value)
HL = float(isotope_info[idx].HalfLife.tolist()[0])
HLUnits = isotope_info[idx].HLUnits.tolist()[0]

HL_slider = widgets.FloatSlider(value=0, min=0, max=Max_half_lives*HL, step=0.02,
                                description='Half-lives', disabled=False,
                                continuous_update=False, orientation='horizontal',
                                readout=False, readout_format='.2f',
                                layout=widgets.Layout(height='75px', max_height='100px', min_height='50px',
                                                    width='200px', max_width='300px',  min_width='100px'))
HL_slider.observe(HL_changed, 'value')

# Get units value and units for age label, then apply them
HLlabel = widgets.BoundedFloatText(value = HL_slider.value, min = HL_slider.min, max = HL_slider.max,
                                   step = HL_slider.step,
                                       layout={'width': '75px', 'height': '50px',
                                               'max_width': '75px', 'max_height': '75px',
                                               'min_width': '50px', 'min_height': '50px'})
UnitsText = widgets.Label(value=HLUnits)
age_label = widgets.HBox([HLlabel, UnitsText])
# Link HL slider with this text
widgets.jslink((HL_slider, 'value'), (HLlabel, 'value'))

# Describe slope
slope_label = widgets.Label(value = 'Slope: {0:0.4f}'.format(slope),
                                       layout={'align_items':'center','align_content':'center',
                                               'justify_content':'center',
                                               'width': '100px', 'height': '50px',
                                               'max_width': '100px', 'max_height': '75px',
                                               'min_width': '50px', 'min_height': '50px'})

controls = widgets.VBox( [isotope, HL_slider, age_label, slope_label],
                        layout=widgets.Layout(align_content='center', align_items='center',
                                              justify_content='center',
                                              width='300px', height='500px',
                                              max_width='300px', max_height='500px',
                                              min_width='100px', min_height='400px',
                                              overflow_x='hidden', overflow_y='hidden') )

display(widgets.HBox( [isochron, controls] ) )


# -*- coding: utf-8 -*-
"""
Created on Mon Jul 16 09:22:00 2018
@author: Luis Solis

Serie temporal para gráficos con el módulo matplotlib

version: 0.4
"""
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl

class Time_series():
    """
    Define los datos y sus atributos para ser representados en un
        gráfico
    """
    def __init__(self, fechas: [], values: [], legend: str, marker: str = '.',
                 scatter: int = 0, slinestyle: str = '-',
                 copy_data: bool = True):
        """
        fechas: lista de dates
        values: lista de floats o integeres
        legend: leyenda de la serie
        marker: marcador
        scatter: tipo de línea
        linestyle: estilo de línea
        """
        from copy import deepcopy
        if len(fechas) < 1 or len(values) < 1:
            raise ValueError('fechas y/0 values no tienen datos')
        if len(fechas) != len(values):
            raise ValueError('fechas y values != longitud')
        if copy_data:
            self.fechas = deepcopy(fechas)
            self.values = deepcopy(values)
        else:
            self.fechas = fechas
            self.values = values

        self.legend = legend
        self.marker = marker
        self.scatter = scatter
        self.linestyle = slinestyle

    @property
    def x(self):
        return self.fechas

    @property
    def y(self):
        return self.values

    @staticmethod
    def minmax_fechas(t_series: list) -> list():
        """
        devuelve el mínimo y el máximo de cada elemento de t_series, que es
            de tipo Time_series
        args
        t_series: lista en que cada elemento es un onjeto Time_series
        dbtype: gestor de base de datos
        output
        fecha mínima y máxima de los elementos en t_series en formato str
            yyyy-mm-dd
        """
        minmax = np.array([[l.fechas[0], l.fechas[-1]] for l in t_series],
                          dtype='datetime64')
        minmax = [np.min(minmax[:, 0]), np.max(minmax[:, 1])]
        minmax = np.datetime_as_string(minmax, unit='D').tolist()
        return minmax

class Plot_time_series():

    def __init__(self, title: str, ts1: [], ylabel1: str, dst: str,
                 write_data: int, ts2: list=[], ylabel2: str=''):
        """
        Llama a la función que dibuja uno o dos gráficos por figura
        args
        st: lista de objetos Time_series que se dibujan en el gráfico principal
        title: título de la figura
        ylabel1: nombre del eje de las Y upper graph
        dst: dirección y nombre del fichero a grabar
        ts2: lista de Time_series que se dibujan en el gráfico inferior; puede
            valer []
        ylabel2: nombre del eje de las Y del gráfico inferior, si ts2 no es []
        """
        for item in ts1 + ts2:
            if not isinstance(item, Time_series):
                raise ValueError('Los elementos de ts1 y ts2 deben ser ' +\
                                 'instancias Time_series')
        if ts2:
            Plot_time_series.xy_ts_plot_2g(title, ts1, ylabel1, ts2, ylabel2,
                                           dst)
        else:
            Plot_time_series.xy_ts_plot_1g(title, ts1, ylabel1, dst)
        if write_data:
            Plot_time_series.write_data_2xml(dst, title, ts1, ylabel1,
                                             ts2, ylabel2)

    @staticmethod
    def xy_ts_plot_1g(title: str, tsu: list, ylabelu: str, dst: str):
        """
        Dibuja una figura con 1 gráfico (axis) xy
        args
        title: título de la figura
        tsu: lista de objetos Time_series para el gráfico superior
        ylabelu: label eje y gráfico superior
        tsl: lista de objetos Time_series para el gráfico inferior
        dst: nombre fichero destino (debe incluir la extensión png)
        """
        # parámetros específicos
        mpl.rc('font', size=8)
        mpl.rc('axes', labelsize=8, titlesize= 10, grid=True)
        mpl.rc('axes.spines', right=False, top=False)
        mpl.rc('xtick', direction='out', top=False)
        mpl.rc('ytick', direction='out', right=False)
        mpl.rc('lines', linewidth=0.8, linestyle='-', marker='.', markersize=4)
        mpl.rc('legend', fontsize=8, framealpha=0.5, loc='best')

        fig, ax = plt.subplots()

        plt.suptitle(title)
        ax.set_ylabel(ylabelu)

        fig.autofmt_xdate()

        for ts1 in tsu:
            ax.plot(ts1.x, ts1.y, label=ts1.legend)
            ax.legend()

        fig.savefig(dst)
        plt.close('all')
        plt.rcdefaults()

    @staticmethod
    def xy_ts_plot_2g(title: str, tsu: list, ylabelu: str, tsl: list,
                      ylabell: str, dst: str):
        """
        Dibuja una figura con 2 gráfico (axis) xy de una o más series cada uno
            que comparten el eje x. El superior es el principal y ocupa 2/3 de
            la altura de la figura. El inferior es secundario y ocupa 1/3 de la
            altura de la figura
        title: título de la figura
        tsu: lista de objetos Time_series para el gráfico superior
        ylabelu: label eje y gráfico superior
        tsl: lista de objetos Time_series para el gráfico inferior
        dst: nombre fichero destino (debe incluir la extensión png)
        """
        import matplotlib.pyplot as plt
        import matplotlib as mpl

        # parámetros específicos
        mpl.rc('font', size=8)
        mpl.rc('axes', labelsize=8, titlesize= 10, grid=True)
        mpl.rc('axes.spines', right=False, top=False)
        mpl.rc('xtick', direction='out', top=False)
        mpl.rc('ytick', direction='out', right=False)
        mpl.rc('lines', linewidth=0.8, linestyle='-', marker='.', markersize=4)
        mpl.rc('legend', fontsize=8, framealpha=0.5, loc='best')

        fig, _ = plt.subplots()

        plt.suptitle(title)
        plt.subplots_adjust(hspace=0.1, bottom=0.16, top=0.87)

        ax1 = plt.subplot2grid((3, 1), (0, 0), rowspan=2)
        ax2 = plt.subplot2grid((3, 1), (2, 0), sharex=ax1)
        ax1.set_ylabel(ylabelu)
        ax2.set_ylabel(ylabell)

        fig.autofmt_xdate()

        for ts1 in tsu:
            ax1.plot(ts1.x, ts1.y, label=ts1.legend)
            ax1.legend()

        # subplot inferior (stem)
        for ts1 in tsl:
            markerline, _, _ = ax2.stem(ts1.x, ts1.y, markerfmt=' ',
                                        basefmt=' ', label=ts1.legend,
                                        use_line_collection=True)
            markerline.set_markerfacecolor('none')
            ax2.legend()

        fig.savefig(dst)
        plt.close('all')
        plt.rcdefaults()

    @staticmethod
    def write_data_2xml(dst: str, title: str, ts1: [], ylabel1: str,
                        ts2: list=[], ylabel2: str=''):
        """
        graba un fichero xml con los datos de un gráfico xy de una o más series
        args
        dst: nombre del fichero png del gráfico
        title: título de la figura
        ts1: lista de objetos Time_series en el gráfico superior
        ylabel1: título del eje Y en el gráfico superior
        ts2: lista de objetos Time_series en el gráfico inferior
        ylabel2: título del eje Y en el gráfico inferior
        """
        from os.path import splitext

        name, ext = splitext(dst)
        fo = open(name + '.xml', 'w')
        fo.write('<?xml version="1.0" encoding="windows-1252"?>\n')
        fo.write('<fig>\n')
        fo.write('<titulo>{title}</titulo>\n')
        Plot_time_series.write_time_series_list(fo, ts1, ylabel1)
        if ts2:
            Plot_time_series.write_time_series_list(fo, ts2, ylabel2)
        fo.write('</fig>\n')
        fo.close()

    @staticmethod
    def write_time_series_list(fo, t_series, ylabel):
        """
        graba una lista de instancias Time_series
        args
        fo: objecto File, abierto
        t_series: lista de objetos Time_series; el primer elemento se
            considera la series principal
        ylabel: título del eje Y
        """
        fo.write('<xy>\n')
        fo.write(f'<eje_y>{ylabel}</eje_y>\n')
        for ts1 in t_series:
            fo.write(f'<punto>{ts1.legend}\n')
            for x1, v1 in zip(ts1.x, ts1.y):
                fo.write(f'<d>{x1}\t{v1:0.2f}</d>\n')
            fo.write('</punto>\n')
        fo.write('</xy>\n')


# -*- coding: utf-8 -*-

import pytest
import numpy as np

import pandas as pd
import pandas.util.testing as tm
from pandas import PeriodIndex

class TestPeriodIndexArithmetic(object):
    # ---------------------------------------------------------------
    # PeriodIndex.shift is used by __add__ and __sub__

    def test_pi_shift_ndarray(self):
        idx = PeriodIndex(['2011-01', '2011-02', 'NaT', '2011-04'],
                          freq='M', name='idx')
        result = idx.shift(np.array([1, 2, 3, 4]))
        expected = PeriodIndex(['2011-02', '2011-04', 'NaT', '2011-08'],
                               freq='M', name='idx')
        tm.assert_index_equal(result, expected)

        result = idx.shift(np.array([1, -2, 3, -4]))
        expected = PeriodIndex(['2011-02', '2010-12', 'NaT', '2010-12'],
                               freq='M', name='idx')
        tm.assert_index_equal(result, expected)

    def test_shift(self):
        pi1 = PeriodIndex(freq='A', start='1/1/2001', end='12/1/2009')
        pi2 = PeriodIndex(freq='A', start='1/1/2002', end='12/1/2010')

        tm.assert_index_equal(pi1.shift(0), pi1)

        assert len(pi1) == len(pi2)
        tm.assert_index_equal(pi1.shift(1), pi2)

        pi1 = PeriodIndex(freq='A', start='1/1/2001', end='12/1/2009')
        pi2 = PeriodIndex(freq='A', start='1/1/2000', end='12/1/2008')
        assert len(pi1) == len(pi2)
        tm.assert_index_equal(pi1.shift(-1), pi2)

        pi1 = PeriodIndex(freq='M', start='1/1/2001', end='12/1/2009')
        pi2 = PeriodIndex(freq='M', start='2/1/2001', end='1/1/2010')
        assert len(pi1) == len(pi2)
        tm.assert_index_equal(pi1.shift(1), pi2)

        pi1 = PeriodIndex(freq='M', start='1/1/2001', end='12/1/2009')
        pi2 = PeriodIndex(freq='M', start='12/1/2000', end='11/1/2009')
        assert len(pi1) == len(pi2)
        tm.assert_index_equal(pi1.shift(-1), pi2)

        pi1 = PeriodIndex(freq='D', start='1/1/2001', end='12/1/2009')
        pi2 = PeriodIndex(freq='D', start='1/2/2001', end='12/2/2009')
        assert len(pi1) == len(pi2)
        tm.assert_index_equal(pi1.shift(1), pi2)

        pi1 = PeriodIndex(freq='D', start='1/1/2001', end='12/1/2009')
        pi2 = PeriodIndex(freq='D', start='12/31/2000', end='11/30/2009')
        assert len(pi1) == len(pi2)
        tm.assert_index_equal(pi1.shift(-1), pi2)

    def test_shift_corner_cases(self):
        # GH#9903
        idx = pd.PeriodIndex([], name='xxx', freq='H')

        with pytest.raises(TypeError):
            # period shift doesn't accept freq
            idx.shift(1, freq='H')

        tm.assert_index_equal(idx.shift(0), idx)
        tm.assert_index_equal(idx.shift(3), idx)

        idx = pd.PeriodIndex(['2011-01-01 10:00', '2011-01-01 11:00'
                              '2011-01-01 12:00'], name='xxx', freq='H')
        tm.assert_index_equal(idx.shift(0), idx)
        exp = pd.PeriodIndex(['2011-01-01 13:00', '2011-01-01 14:00'
                              '2011-01-01 15:00'], name='xxx', freq='H')
        tm.assert_index_equal(idx.shift(3), exp)
        exp = pd.PeriodIndex(['2011-01-01 07:00', '2011-01-01 08:00'
                              '2011-01-01 09:00'], name='xxx', freq='H')
        tm.assert_index_equal(idx.shift(-3), exp)

    def test_shift_nat(self):
        idx = PeriodIndex(['2011-01', '2011-02', 'NaT', '2011-04'],
                          freq='M', name='idx')
        result = idx.shift(1)
        expected = PeriodIndex(['2011-02', '2011-03', 'NaT', '2011-05'],
                               freq='M', name='idx')
        tm.assert_index_equal(result, expected)
        assert result.name == expected.name

    def test_shift_gh8083(self):
        # test shift for PeriodIndex
        # GH#8083
        drange = pd.period_range('20130101', periods=5, freq='D')
        result = drange.shift(1)
        expected = PeriodIndex(['2013-01-02', '2013-01-03', '2013-01-04',
                                '2013-01-05', '2013-01-06'], freq='D')
        tm.assert_index_equal(result, expected)

    def test_shift_periods(self):
        # GH #22458 : argument 'n' was deprecated in favor of 'periods'
        idx = PeriodIndex(freq='A', start='1/1/2001', end='12/1/2009')
        tm.assert_index_equal(idx.shift(periods=0), idx)
        tm.assert_index_equal(idx.shift(0), idx)
        with tm.assert_produces_warning(FutureWarning,
                                        check_stacklevel=True):
            tm.assert_index_equal(idx.shift(n=0), idx)


import numpy as np
import nexusformat

def write_nexus_file(fname, image, md={}):
    """
    write the image to a NeXus HDF5 data file

    Parameters
    ----------
    fname : str
        name of the file (relative or absolute) to be written
    image : numpy array
        the image data
    md : dictionary
        key: value where value is something that can be written by h5py
             (such as str, int, float, numpy array, ...)
    """
    nx = NXroot()
    nx['/entry'] = NXentry(NXinstrument(NXdetector()))
    nx['entry/instrument/detector/image'] = NXfield(image, units='counts',
                                                    compression='gzip')
    nx['entry/data'] = NXdata()
    nx['entry/data'].makelink(nx['entry/instrument/detector/image'])
    nx['entry/data'].nxsignal = nx['entry/data/image']

    if len(md) > 0:
        # /entry/instrument/metadata (optional, for metadata)
        metadata = nx['/entry/instrument/metadata'] = NXcollection()
        for k, v in md.items():
            metadata[k] = v

    nx.save(fname, 'w')

if __name__ == "__main__":
	"""demonstrate how to use this code"""
	import epics
	prefix = "13SIM1:"
	img = epics.caget(prefix+"image1:ArrayData")
	size_x = epics.caget(prefix+"cam1:ArraySizeX_RBV")
	size_y = epics.caget(prefix+"cam1:ArraySizeY_RBV")
	# edit the full image for just the binned data
	img = img[:size_x*size_y].reshape((size_x, size_y))

	extra_information = dict(
		unique_id = epics.caget(prefix+"image1:UniqueId_RBV"),
		size_x = size_x,
		size_y = size_y,
		detector_state = epics.caget(prefix+"cam1:DetectorState_RBV"),
		bitcoin_value="15000",
	)
	write_nexus_file("example.h5", img, md=extra_information)


# Copyright 2018-2020 Xanadu Quantum Technologies Inc.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Mutable QNode, complicated primary parameters benchmark.
"""
# pylint: disable=invalid-name
import numpy as np

import pennylane as qml
import benchmark_utils as bu

def circuit(p, *, aux=0):
    """A very simple, lightweight mutable quantum circuit."""
    qml.RX(p[aux][2], wires=[0])
    return qml.expval(qml.PauliZ(0))

class Benchmark(bu.BaseBenchmark):
    """
    This benchmark attempts to measure the efficiency of :meth:`JacobianQNode._construct` for
    mutable QNodes, using an extreme case where the QNode has lots of primary parameters with
    a complicated nested structure, but relatively few auxiliary parameters, and only a few
    of the primary parameters are actually used in the circuit.

    When the QNode is constructed, a VariableRef is built for each primary parameter,
    and the qfunc re-evaluated. In this test this is meant to be time-consuming, but it is only
    strictly necessary if the auxiliary parameters change.
    The main reasons why there are significant differences in the execution speed of this test
    between different PL commits:

      * :meth:`BaseQNode._construct` should only reconstruct the QNode if the auxiliary params
        have changed.
      * Most of the primary params are not used in the circuit, hence
        :meth:`JacobianQNode._construct` should efficiently figure out that partial derivatives
        wrt. them are always zero.
    """

    name = "mutable qnode, complicated primary params"
    min_wires = 1
    n_vals = range(6, 13, 1)

    def __init__(self, device=None, verbose=False):
        super().__init__(device, verbose)
        self.qnode = None

    def setup(self):
        self.qnode = bu.create_qnode(circuit, self.device, mutable=True, interface=None)

    def benchmark(self, n=8):
        # n is the number of levels in the primary parameter tree.
        # Hence the number of primary parameters depends exponentially on n.

        def create_params(n):
            """Recursively builds a tree structure with n levels."""
            if n <= 0:
                # the leaves are arrays
                return np.random.randn(2)
            # the other nodes have two branches and a scalar
            return [create_params(n - 1), create_params(n - 1), np.random.randn()]

        p = create_params(n)

        def evaluate(aux):
            """Evaluates the qnode using the given auxiliary params."""
            res = self.qnode(p, aux=aux)
            # check the result
            assert np.allclose(res, np.cos(p[aux][2]))

        # first evaluation and construction
        evaluate(0)
        # evaluate the node several times more with a different auxiliary argument
        # (it does not matter if p changes or not, the VariableRefs handle it)
        for _ in range(1, 10):
            # If we had evaluate(i % 2) here instead the auxiliary arguments would change
            # every time, which would negate most possible speedups.
            evaluate(1)

        return True