import unittest

import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition
from chainer.utils import type_check

@testing.parameterize(*testing.product_dict(
    [
        {'in_shapes': [(3, 1, 5), (1, 2, 5)], 'out_shape': (3, 2, 5)},
        {'in_shapes': [(3, 2, 5), (5,)], 'out_shape': (3, 2, 5)},
        {'in_shapes': [(3, 2, 5), ()], 'out_shape': (3, 2, 5)},
        {'in_shapes': [(3, 2, 5), (3, 2, 5)], 'out_shape': (3, 2, 5)},
        {'in_shapes': [(), ()], 'out_shape': ()},
        {'in_shapes': [(1, 1, 1), (1,)], 'out_shape': (1, 1, 1)},
        {'in_shapes': [(1, 1, 1), ()], 'out_shape': (1, 1, 1)},
        {'in_shapes': [(3, 2, 5)], 'out_shape': (3, 2, 5)},
        {'in_shapes': [(3, 1, 5), (1, 2, 5), (3, 2, 1)],
         'out_shape': (3, 2, 5)},
    ],
    [
        {'dtype': numpy.float16},
        {'dtype': numpy.float32},
        {'dtype': numpy.float64},
    ],
))
class TestBroadcast(unittest.TestCase):

    def setUp(self):
        uniform = numpy.random.uniform
        self.data = [uniform(0, 1, shape).astype(self.dtype)
                     for shape in self.in_shapes]
        self.grads = [uniform(0, 1, self.out_shape).astype(self.dtype)
                      for _ in range(len(self.in_shapes))]

        self.check_backward_options = {}
        if self.dtype == numpy.float16:
            self.check_backward_options = {
                'eps': 2 ** -5, 'atol': 1e-3, 'rtol': 1e-2}

    def check_forward(self, data):
        xs = [chainer.Variable(x) for x in data]
        bxs = functions.broadcast(*xs)

        # When len(xs) == 1, function returns a Variable object
        if isinstance(bxs, chainer.Variable):
            bxs = (bxs,)

        for bx in bxs:
            self.assertEqual(bx.data.shape, self.out_shape)
            self.assertEqual(bx.data.dtype, self.dtype)

    def test_forward_cpu(self):
        self.check_forward(self.data)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward([cuda.to_gpu(x) for x in self.data])

    def check_backward(self, data, grads):
        gradient_check.check_backward(
            functions.Broadcast(), data, grads,
            **self.check_backward_options)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.data, self.grads)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.check_backward([cuda.to_gpu(x) for x in self.data],
                            [cuda.to_gpu(x) for x in self.grads])

class TestBroadcastTypeError(unittest.TestCase):

    def test_invalid_shape(self):
        x_data = numpy.zeros((3, 2, 5), dtype=numpy.int32)
        y_data = numpy.zeros((1, 3, 4), dtype=numpy.float32)
        x = chainer.Variable(x_data)
        y = chainer.Variable(y_data)

        with self.assertRaises(type_check.InvalidType):
            functions.broadcast(x, y)

    def test_invalid_shape_fill(self):
        x_data = numpy.zeros((3, 2, 5), dtype=numpy.int32)
        y_data = numpy.zeros(4, dtype=numpy.float32)
        x = chainer.Variable(x_data)
        y = chainer.Variable(y_data)

        with self.assertRaises(type_check.InvalidType):
            functions.broadcast(x, y)

    def test_no_args(self):
        with self.assertRaises(type_check.InvalidType):
            functions.broadcast()

@testing.parameterize(*testing.product_dict(
    [
        {'in_shape': (3, 1, 5), 'out_shape': (3, 2, 5)},
        {'in_shape': (5,), 'out_shape': (3, 2, 5)},
        {'in_shape': (3, 2, 5), 'out_shape': (3, 2, 5)},
    ],
    [
        {'dtype': numpy.float16},
        {'dtype': numpy.float32},
        {'dtype': numpy.float64},
    ],
))
class TestBroadcastTo(unittest.TestCase):

    def setUp(self):
        uniform = numpy.random.uniform
        self.data = uniform(0, 1, self.in_shape).astype(self.dtype)
        self.grad = uniform(0, 1, self.out_shape).astype(self.dtype)
        self.check_backward_options = {}
        if self.dtype == numpy.float16:
            self.check_backward_options = {
                'eps': 2 ** -5, 'atol': 1e-3, 'rtol': 1e-2}

    def check_forward(self, data):
        x = chainer.Variable(data)
        bx = functions.broadcast_to(x, self.out_shape)

        self.assertEqual(bx.data.shape, self.out_shape)

    def test_forward_cpu(self):
        self.check_forward(self.data)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.data))

    @condition.retry(3)
    def check_backward(self, data, grads):
        gradient_check.check_backward(
            functions.BroadcastTo(self.out_shape), data, grads,
            **self.check_backward_options)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.data, self.grad)

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.data), cuda.to_gpu(self.grad))

@testing.parameterize(
    {'in_shape': (3, 2, 5), 'out_shape': (5,)},
    {'in_shape': (3, 2, 5), 'out_shape': (3, 1, 5)},
    {'in_shape': (3, 2, 5), 'out_shape': (1, 3, 2, 3)},
)
class TestBroadcastToTypeCheck(unittest.TestCase):

    def setUp(self):
        uniform = numpy.random.uniform
        self.data = uniform(0, 1, self.in_shape).astype(numpy.float32)

    def test_type_check(self):
        x = chainer.Variable(self.data)
        with self.assertRaises(type_check.InvalidType):
            functions.broadcast_to(x, self.out_shape)

testing.run_module(__name__, __file__)


"""PyWENO code generation helpers."""

from sympy.printing.fcode import FCodePrinter as SympyFCodePrinter
from sympy.printing.ccode import CCodePrinter as SympyCCodePrinter
from sympy.printing.precedence import precedence

class FCodePrinter(SympyFCodePrinter):
  pass

class CCodePrinter(SympyCCodePrinter):
  def _print_Pow(self, expr):
    if expr.exp == 2:
      PREC = precedence(expr)
      s = str(self.parenthesize(expr.base, PREC))
      return '%s*%s' % (s,s)
    else:
      return super(CCodePrinter,self)._print_Pow(expr)


#!/usr/bin/env python
import sys
from collections import OrderedDict
import json
import numpy as np

from kernel_tuner import tune_kernel
from kernel_tuner.observers import BenchmarkObserver
from kernel_tuner.nvml import NVMLObserver

def tune(device_name, strategy="bayes_opt_GPyTorch_lean", strategy_options=None, verbose=True, quiet=False, simulation_mode=True):

    #setup tuning parameters
    tune_params = OrderedDict()
    tune_params["block_size_x"] = [2**i for i in range(8, 11)][::-1]    # 5
    tune_params["block_size_y"] = [2**i for i in range(6)]
    tune_params["tile_size_x"] = [1, 2, 3, 4, 5, 6, 7, 8]
    tune_params["tile_size_y"] = [1, 2, 3, 4, 5, 6, 7, 8]
    tune_params["use_shared_mem"] = [1]
    tune_params["loop_unroll_factor_x"] = [1, 2, 3, 4, 5, 6, 7, 8]
    tune_params["loop_unroll_factor_y"] = [1, 2, 3, 4, 5, 6, 7, 8]
    tune_params["use_column"] = [1]
    tune_params["use_separate_acc"] = [0]
    tune_params["n_y_blocks"] = [2**i for i in range(6)]

    def config_valid(p):
        if p["loop_unroll_factor_x"] > p["tile_size_x"] or (p["loop_unroll_factor_x"] and p["tile_size_x"] % p["loop_unroll_factor_x"] != 0):
            return False    #no need to test this loop unroll factor, as it is the same as not unrolling the loop
        if p["loop_unroll_factor_y"] > p["tile_size_y"] or (p["loop_unroll_factor_y"] and p["tile_size_y"] % p["loop_unroll_factor_y"] != 0):
            return False    #no need to test this loop unroll factor, as it is the same as not unrolling the loop
        return True

    restrictions = config_valid

    #setup test input
    alloc_size = 32 * 1024
    size = np.int32(32 * 1024)
    max_blocks = np.int32(np.ceil(size / float(np.amin(tune_params["block_size_x"]))) * np.ceil(size / float(np.amin(tune_params["block_size_y"]))))
    ndim = np.int32(2)
    A = np.random.randn(alloc_size * ndim).astype(np.float64)
    B = A + 0.00001 * np.random.randn(alloc_size * ndim).astype(np.float64)
    scale_A = np.absolute(0.01 * np.random.randn(alloc_size).astype(np.float64))
    scale_B = np.absolute(0.01 * np.random.randn(alloc_size).astype(np.float64))
    cost = np.zeros((max_blocks)).astype(np.float64)

    #setup kernel
    kernel_name = "ExpDist"
    arguments = [A, B, size, size, scale_A, scale_B, cost]
    grid_div_x = ["block_size_x", "tile_size_x"]
    grid_div_y = ["block_size_y", "tile_size_y"]

    #get number of registers
    class RegisterObserver(BenchmarkObserver):

        def get_results(self):
            return {
                "num_regs": self.dev.current_module.get_function(kernel_name).num_regs
            }

    problem_size = lambda p: (size, size if p["use_column"] == 0 else p["n_y_blocks"] * p["block_size_y"] * p["tile_size_y"])

    metrics = OrderedDict()
    metrics["registers"] = lambda p: p["num_regs"]
    metrics["clock"] = lambda p: p["core_freq"]

    def FLOPs_in_partial_reduction(p):
        num_thread_blocks = np.ceil(size / (p["block_size_x"] * p["tile_size_x"])) * np.ceil(size / (p["block_size_y"] * p["tile_size_y"]))
        ops_per_thread_block = p["block_size_x"] * p[
            "block_size_y"] / 32 * 31 + 31    #minimal number of ops per warp times number of warps + #ops for 1 final warp
        return num_thread_blocks * ops_per_thread_block

    ops_per_iteration = 35    #from Nsight profiler
    metrics["GFLOP/s"] = lambda p: ((FLOPs_in_partial_reduction(p) + ops_per_iteration * size * size) / 1e9) / (p["time"] / 1e3)

    cp = []

    kernel1, env = tune_kernel(kernel_name, "expdist.cu", problem_size, arguments, tune_params, grid_div_x=grid_div_x, grid_div_y=grid_div_y, metrics=metrics,
                               iterations=32, compiler_options=cp, cache="cache_files/expdist_" + device_name + "_processed", restrictions=restrictions,
                               strategy=strategy, strategy_options=strategy_options, simulation_mode=simulation_mode, verbose=verbose, quiet=quiet)
    return kernel1, env

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Usage: ./expdist.py [device name]")
        exit(1)

    device_name = sys.argv[1]

    tune(device_name)


"""
Tests the LIME wrapper.
"""
# Author: Alex Hepburn <ah13558@bristol.ac.uk>
#         Kacper Sokol <k.sokol@bristol.ac.uk>
# License: new BSD

import pytest

try:
    import lime
except ImportError:  # pragma: no cover
    pytest.skip(
        'Skipping lime wrapper tests -- lime missing.',
        allow_module_level=True)
else:
    del lime

import importlib
import sys

import numpy as np

import fatf.transparency.lime as ftl
import fatf.utils.models as fum
import fatf.utils.testing.imports as futi
import fatf.utils.testing.transparency as futt

from fatf.exceptions import IncompatibleModelError, IncorrectShapeError

# yapf: disable
FUTURE_WARNING = (
    'The LIME wrapper will be deprecated in FAT Forensics version '
    '0.0.3. Please consider using the TabularBlimeyLime explainer '
    'class implemented in the fatf.transparency.predictions.'
    'surrogate_explainers module instead. Alternatively, you may '
    'consider building a custom surrogate explainer using the '
    'functionality implemented in FAT Forensics -- see the *Tabular '
    'Surrogates* how-to guide for more details.')

SAMPLE = np.array([0, 1, 0.08, 0.54])
SAMPLE_STRUCT = np.array(
    [(0, 1, 0.08, 0.54)],
    dtype=[('a', 'i'), ('b', 'i'), ('c', 'f'), ('d', 'f')])[0]
CLF = fum.KNN()
CLF.fit(futt.NUMERICAL_NP_ARRAY, futt.LABELS)
CLASS_NAMES = ['class0', 'class1', 'class2']
FEATURE_NAMES = ['feat0', 'feat1', 'feat2', 'feat3']

NUMERICAL_RESULTS = {
    'class0': [('feat0 <= 0.00', -0.415),
               ('0.50 < feat1 <= 1.00', -0.280),
               ('0.07 < feat2 <= 0.22', 0.037),
               ('0.34 < feat3 <= 0.58', -0.007)],
    'class1': [('0.50 < feat1 <= 1.00', 0.202),
               ('0.07 < feat2 <= 0.22', -0.076),
               ('feat0 <= 0.00', 0.019),
               ('0.34 < feat3 <= 0.58', -0.018)],
    'class2': [('feat0 <= 0.00', 0.395),
               ('0.50 < feat1 <= 1.00', 0.077),
               ('0.07 < feat2 <= 0.22', 0.039),
               ('0.34 < feat3 <= 0.58', 0.025)]
}
CATEGORICAL_RESULTS = {
    'class0': [('feat0=0', -0.413),
               ('feat1=1', -0.282),
               ('0.07 < feat2 <= 0.22', 0.0366),
               ('0.34 < feat3 <= 0.58', -0.00717)],
    'class1': [('feat1=1', 0.2048),
               ('0.07 < feat2 <= 0.22', -0.0767),
               ('feat0=0', 0.0179),
               ('0.34 < feat3 <= 0.58', -0.018)],
    'class2': [('feat0=0', 0.395),
               ('feat1=1', 0.077),
               ('0.07 < feat2 <= 0.22', 0.039),
               ('0.34 < feat3 <= 0.58', 0.025)]
}
REGRESSION_RESULTS = [
    ('feat0 <= 0.00', 1.332),
    ('0.50 < feat1 <= 1.00', 0.767),
    ('0.34 < feat3 <= 0.58', 0.149),
    ('0.07 < feat2 <= 0.22', -0.048)
]
# yapf: enable

USER_WARNING_MODEL_PRED = ('Since both, a model and a predictive function, '
                           'are provided only the latter will be used.')
LOG_WARNING = 'The model can only be used for LIME in a regressor mode.'

CLF_NON_PROBA = futt.NonProbabilisticModel(CLF.predict)

def test_import_when_missing():
    """
    Tests importing :mod:`fatf.transparency.lime` module when LIME is missing.
    """
    assert 'fatf.transparency.lime' in sys.modules
    warning_msg = (
        'Lime package is not installed on your system. You must install it in '
        'order to use the fatf.transparency.lime module. One possibility is '
        'to install LIME alongside this package with: pip install fatf[lime].')
    with futi.module_import_tester('lime', when_missing=True):
        with pytest.warns(ImportWarning) as w:
            importlib.reload(ftl)
        assert len(w) == 1
        assert str(w[0].message) == warning_msg
    assert 'fatf.transparency.lime' in sys.modules

def test_lime_init():
    """
    Tests :mod:`fatf.transparency.lime.Lime` object initialisation.

    This only looks into cases where the initialisation would fail.
    """
    attribute_error = 'The following named parameters are not valid: {}.'
    shape_error_data = ('The data parameter must be a 2-dimensional numpy '
                        'array.')
    value_error_cat = 'LIME does not support non-numerical data arrays.'
    value_error = ("The mode must be either 'classification' or 'regression'. "
                   "'{}' given.")
    incompatible_model_error = ('LIME requires a model object to have a fit '
                                'method and optionally a predict_proba '
                                'method.')
    type_error_predictor = ('The predict_fn parameter is not callable -- it '
                            'has to be a function.')
    type_error_struct_indices = ('The categorical_features parameter either '
                                 'has to be a list, a numpy array or None.')
    incorrect_shape_struct_indices = ('categorical_features array/list is not '
                                      '1-dimensional.')
    value_error_struct_indices = ('Since categorical_features is an array of '
                                  'indices for a structured array, all of its '
                                  'elements should be strings.')
    value_error_struct_incorrect_indices = (
        'Indices given in the categorical_features parameter are not valid '
        'for the input data array.')
    #
    attribute_error_explain = ('The following named parameters are not valid: '
                               '{}.')
    incorrect_shape_error_explain = ('The instance to be explained should be '
                                     '1-dimensional.')
    value_error_explain = ('The instance to be explained should be purely '
                           'numerical -- LIME does not support categorical '
                           'features.')

    # Wrong named parameter
    with pytest.warns(FutureWarning) as warning:
        with pytest.raises(AttributeError) as exin:
            ftl.Lime(futt.NUMERICAL_NP_ARRAY, model=CLF, lorem='ipsum')
        assert str(exin.value) == attribute_error.format("{'lorem'}")
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING

    # Not a 2-dimensional array
    with pytest.warns(FutureWarning) as warning:
        with pytest.raises(IncorrectShapeError) as exin:
            ftl.Lime(np.ones((6, 4, 4)))
        assert str(exin.value) == shape_error_data
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING

    # Not a numerical array
    with pytest.warns(FutureWarning) as warning:
        with pytest.raises(ValueError) as exin:
            lime = ftl.Lime(np.ones((6, 4), dtype='U1'))
        assert str(exin.value) == value_error_cat
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING

    # A structured data array with weird categorical indices type
    with pytest.warns(FutureWarning) as warning:
        with pytest.raises(TypeError) as exin:
            ftl.Lime(futt.NUMERICAL_STRUCT_ARRAY, categorical_features='')
        assert str(exin.value) == type_error_struct_indices
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING

    # A structured data array with weird categorical indices shape
    with pytest.warns(FutureWarning) as warning:
        with pytest.raises(IncorrectShapeError) as exin:
            ftl.Lime(futt.NUMERICAL_STRUCT_ARRAY, categorical_features=[['a']])
        assert str(exin.value) == incorrect_shape_struct_indices
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING

    # A structured data array with non-textual categorical indices
    with pytest.warns(FutureWarning) as warning:
        with pytest.raises(ValueError) as exin:
            ftl.Lime(
                futt.NUMERICAL_STRUCT_ARRAY,
                categorical_features=np.array([3, 2]))
        assert str(exin.value) == value_error_struct_indices
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING

    # A structured data array with incorrect categorical indices
    with pytest.warns(FutureWarning) as warning:
        with pytest.raises(ValueError) as exin:
            ftl.Lime(
                futt.NUMERICAL_STRUCT_ARRAY,
                categorical_features=['a', 'e', 'b'])
        assert str(exin.value) == value_error_struct_incorrect_indices
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING

    # Wrong operation mode
    with pytest.warns(FutureWarning) as warning:
        with pytest.raises(ValueError) as exin:
            ftl.Lime(futt.NUMERICAL_NP_ARRAY, mode='c')
        assert str(exin.value) == value_error.format('c')
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING

    # Invalid model
    invalid_model = futt.InvalidModel()
    with pytest.warns(FutureWarning) as warning:
        with pytest.raises(IncompatibleModelError) as exin:
            ftl.Lime(
                futt.NUMERICAL_NP_ARRAY,
                model=invalid_model,
                mode='classification')
        assert str(exin.value) == incompatible_model_error
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING
    with pytest.warns(FutureWarning) as warning:
        with pytest.raises(IncompatibleModelError) as exin:
            ftl.Lime(futt.NUMERICAL_NP_ARRAY, model='a', mode='classification')
        assert str(exin.value) == incompatible_model_error
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING

    # Invalid predictive function
    with pytest.warns(FutureWarning) as warning:
        with pytest.raises(TypeError) as exin:
            ftl.Lime(
                futt.NUMERICAL_NP_ARRAY, predict_fn='a', mode='regression')
        assert str(exin.value) == type_error_predictor
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING

    ###########################################################################
    # Test explain_instance for exceptions and errors
    with pytest.warns(FutureWarning) as warning:
        lime = ftl.Lime(futt.NUMERICAL_NP_ARRAY)
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING

    # Incorrect parameter
    with pytest.raises(AttributeError) as exin:
        lime.explain_instance(SAMPLE, weird_named_argument='yes')
    assert str(exin.value) == attribute_error_explain.format(
        "{'weird_named_argument'}")

    # Incorrect shape
    with pytest.raises(IncorrectShapeError) as exin:
        lime.explain_instance(futt.NUMERICAL_STRUCT_ARRAY)
    assert str(exin.value) == incorrect_shape_error_explain

    # Not numerical
    with pytest.raises(ValueError) as exin:
        lime.explain_instance(np.ones((5, ), dtype='U1'))
    assert str(exin.value) == value_error_explain

def test_explain_instance_classification(caplog):
    """
    Tests :mod:`fatf.transparency.lime.Lime.explain_instance` method.

    These tests are for a classification task.
    """
    runtime_error_no_predictor = 'A predictive function is not available.'
    runtime_error_non_prob = ('The predictive model is not probabilistic. '
                              'Please specify a predictive function instead.')

    # Check logging
    assert len(caplog.records) == 0

    # Non-probabilistic model -- function -- probabilistic function
    with pytest.warns(None) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_NP_ARRAY,
            model=CLF_NON_PROBA,
            predict_fn=CLF.predict_proba,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 2
    assert str(warning[0].message) == FUTURE_WARNING
    assert str(warning[1].message) == USER_WARNING_MODEL_PRED
    explained = lime.explain_instance(SAMPLE, predict_fn=CLF.predict_proba)
    assert futt.is_explanation_equal_list(explained, NUMERICAL_RESULTS)
    # Non-probabilistic model -- function -- no function
    with pytest.warns(None) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_NP_ARRAY,
            model=CLF_NON_PROBA,
            predict_fn=CLF.predict_proba,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 2
    assert str(warning[0].message) == FUTURE_WARNING
    assert str(warning[1].message) == USER_WARNING_MODEL_PRED
    explained = lime.explain_instance(SAMPLE)
    assert futt.is_explanation_equal_list(explained, NUMERICAL_RESULTS)
    # Non-probabilistic model -- no function -- probabilistic function
    with pytest.warns(FutureWarning) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_NP_ARRAY,
            model=CLF_NON_PROBA,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING
    explained = lime.explain_instance(SAMPLE, predict_fn=CLF.predict_proba)
    assert futt.is_explanation_equal_list(explained, NUMERICAL_RESULTS)
    # Non-probabilistic model -- no function -- no function
    with pytest.warns(FutureWarning) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_NP_ARRAY,
            model=CLF_NON_PROBA,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING
    with pytest.raises(RuntimeError) as exin:
        lime.explain_instance(SAMPLE_STRUCT)
    assert str(exin.value) == runtime_error_non_prob

    # Check logging
    assert len(caplog.records) == 4
    for i in range(4):
        assert caplog.records[i].levelname == 'WARNING'
        assert caplog.records[i].getMessage() == LOG_WARNING

    # No model -- function -- probabilistic function
    with pytest.warns(FutureWarning) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_STRUCT_ARRAY,
            predict_fn=CLF.predict_proba,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING
    explained = lime.explain_instance(SAMPLE, predict_fn=CLF.predict_proba)
    assert futt.is_explanation_equal_list(explained, NUMERICAL_RESULTS)
    # No model -- function -- no function
    with pytest.warns(FutureWarning) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_STRUCT_ARRAY,
            predict_fn=CLF.predict_proba,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING
    explained = lime.explain_instance(SAMPLE)
    assert futt.is_explanation_equal_list(explained, NUMERICAL_RESULTS)
    # No model -- no function -- probabilistic function
    with pytest.warns(FutureWarning) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_NP_ARRAY,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING
    explained = lime.explain_instance(SAMPLE, predict_fn=CLF.predict_proba)
    assert futt.is_explanation_equal_list(explained, NUMERICAL_RESULTS)
    # No model -- no function -- no function
    with pytest.warns(FutureWarning) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_NP_ARRAY,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING
    with pytest.raises(RuntimeError) as exin:
        lime.explain_instance(SAMPLE)
    assert str(exin.value) == runtime_error_no_predictor

    # Check logging
    assert len(caplog.records) == 4

    # Probabilistic model -- probabilistic function -- empty call
    with pytest.warns(None) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_NP_ARRAY,
            model=CLF,
            predict_fn=CLF.predict_proba,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 2
    assert str(warning[0].message) == FUTURE_WARNING
    assert str(warning[1].message) == USER_WARNING_MODEL_PRED
    explained = lime.explain_instance(SAMPLE_STRUCT)
    assert futt.is_explanation_equal_list(explained, NUMERICAL_RESULTS)
    #
    # Probabilistic model -- probabilistic function -- non-empty call
    with pytest.warns(None) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_NP_ARRAY,
            model=CLF,
            predict_fn=CLF.predict_proba,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 2
    assert str(warning[0].message) == FUTURE_WARNING
    assert str(warning[1].message) == USER_WARNING_MODEL_PRED
    explained = lime.explain_instance(SAMPLE, predict_fn=CLF.predict_proba)
    assert futt.is_explanation_equal_list(explained, NUMERICAL_RESULTS)
    #
    # Probabilistic model -- no function -- empty call
    with pytest.warns(FutureWarning) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_STRUCT_ARRAY,
            model=CLF,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING
    explained = lime.explain_instance(SAMPLE)
    assert futt.is_explanation_equal_list(explained, NUMERICAL_RESULTS)
    #
    # Probabilistic model -- no function -- non-empty call
    with pytest.warns(FutureWarning) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_STRUCT_ARRAY,
            model=CLF,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING
    explained = lime.explain_instance(
        SAMPLE_STRUCT, predict_fn=CLF.predict_proba)
    assert futt.is_explanation_equal_list(explained, NUMERICAL_RESULTS)

    # Check logging
    assert len(caplog.records) == 4

    ###########################################################################
    # Test with categorical features: feat0 and feat1

    cat_feat = [0, 1]
    with pytest.warns(FutureWarning) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_NP_ARRAY,
            model=CLF,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES,
            categorical_features=cat_feat)
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING
    explained = lime.explain_instance(SAMPLE_STRUCT)
    assert futt.is_explanation_equal_list(CATEGORICAL_RESULTS, explained)

    cat_feat = ['a', 'b']
    with pytest.warns(FutureWarning) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_STRUCT_ARRAY,
            model=CLF,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES,
            categorical_features=cat_feat)
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING
    explained = lime.explain_instance(SAMPLE)
    assert futt.is_explanation_equal_list(CATEGORICAL_RESULTS, explained)

    # Check logging
    assert len(caplog.records) == 4

def test_explain_instance_regression(caplog):
    """
    Tests :mod:`fatf.transparency.lime.Lime.explain_instance` method.

    These tests are for a regression task.
    """
    # Check logging
    assert len(caplog.records) == 0

    # Regression a non-probabilistic model
    with pytest.warns(FutureWarning) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_STRUCT_ARRAY,
            mode='regression',
            model=CLF_NON_PROBA,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING
    explained = lime.explain_instance(SAMPLE)
    assert futt.is_explanation_equal_list({'a': explained},
                                          {'a': REGRESSION_RESULTS})

    # Check logging
    assert len(caplog.records) == 1
    assert caplog.records[0].levelname == 'WARNING'
    assert caplog.records[0].getMessage() == LOG_WARNING

    # Regression a probabilistic model
    with pytest.warns(FutureWarning) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_NP_ARRAY,
            mode='regression',
            model=CLF,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING
    explained = lime.explain_instance(SAMPLE_STRUCT)
    assert futt.is_explanation_equal_list({'a': explained},
                                          {'a': REGRESSION_RESULTS})

    # Regression with a model and function
    with pytest.warns(None) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_STRUCT_ARRAY,
            mode='regression',
            model=CLF,
            predict_fn=CLF_NON_PROBA.predict,
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 2
    assert str(warning[0].message) == FUTURE_WARNING
    assert str(warning[1].message) == USER_WARNING_MODEL_PRED
    explained = lime.explain_instance(SAMPLE_STRUCT)
    assert futt.is_explanation_equal_list({'a': explained},
                                          {'a': REGRESSION_RESULTS})

    # Regression without a model
    with pytest.warns(FutureWarning) as warning:
        lime = ftl.Lime(
            futt.NUMERICAL_NP_ARRAY,
            mode='regression',
            class_names=CLASS_NAMES,
            feature_names=FEATURE_NAMES)
    assert len(warning) == 1
    assert str(warning[0].message) == FUTURE_WARNING
    explained = lime.explain_instance(SAMPLE, predict_fn=CLF_NON_PROBA.predict)
    assert futt.is_explanation_equal_list({'a': explained},
                                          {'a': REGRESSION_RESULTS})

    # Check logging
    assert len(caplog.records) == 1


# Question: Find an element in a numpy array

import numpy as np

def find_element(arr, elem):

    for i in range(len(arr)):
        if arr[i] == elem:
            return i

my_array = np.array([1,2,3,4,5,6,7,8,9])
print(find_element(my_array, 6))
print(find_element(my_array, 11))


# -*- coding: utf-8 -*-

"""
@author: krakowiakpawel9@gmail.com
@site: e-smartdata.org
"""

import pandas as pd
import numpy as np

# %% creating dataset
np.random.seed(0)
df = pd.DataFrame(np.random.randn(20, 5),
                  columns=list('abcde'),
                  index=list('abcdefghijklmnoprstu'))

# %% iloc[row_indexer, column_indexer]
# integer position based

# by col
col_1 = df.iloc[:, 0]
col_2 = df.iloc[:, 1]

col_1_df = df.iloc[:, 0:1]
col_2_df = df.iloc[:, 1:2]

col_1_2 = df.iloc[:, 0:2]
col_1_3 = df.iloc[:, [0, 2]]

col_last = df.iloc[:, -1]
col_last_df = df.iloc[:, -1:]

col_by_2 = df.iloc[:, ::2]

# by row
row_1 = df.iloc[0, :]
row_1_df = df.iloc[0:1, :]

row_1_2 = df.iloc[0:2, :]
row_1_4 = df.iloc[[0, 3], :]

row_last = df.iloc[-1, :]
row_last_df = df.iloc[-1:, :]

row_by_2 = df.iloc[::2, :]
row_by_3 = df.iloc[::3, :]


# Copyright 2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

import numpy as np
import pytest

import mindspore.context as context
import mindspore.nn as nn
from mindspore import Tensor
from mindspore.ops import operations as P

context.set_context(mode=context.GRAPH_MODE, device_target="CPU")

class MaskedFillNet(nn.Cell):
    def __init__(self):
        super(MaskedFillNet, self).__init__()
        self.maskedfill = P.MaskedFill()

    def construct(self, inputs, mask, value):
        return self.maskedfill(inputs, mask, value)

def maskedfill_fun(ntype):
    maskedfill_net = MaskedFillNet()

    inputs = Tensor(np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).astype(ntype))
    mask = Tensor(np.array([[True, True, False, True], [False, False, True, False]]).astype(np.bool))
    value = Tensor(np.array(22).astype(ntype))
    expect = np.array([[22, 22, 3, 22], [5, 6, 22, 8]]).astype(ntype)
    output = maskedfill_net(inputs, mask, value)
    assert (output.asnumpy() == expect).all()

    mask = Tensor(np.array([[True, True, True, True], [True, True, True, True]]).astype(np.bool))
    value = Tensor(np.array(1).astype(ntype))
    expect = np.array([[1, 1, 1, 1], [1, 1, 1, 1]]).astype(ntype)
    output = maskedfill_net(inputs, mask, value)
    assert (output.asnumpy() == expect).all()

    mask = Tensor(np.array([[False, False, False, False], [False, False, False, False]]).astype(np.bool))
    value = Tensor(np.array(22).astype(ntype))
    expect = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]).astype(ntype)
    output = maskedfill_net(inputs, mask, value)
    assert (output.asnumpy() == expect).all()

    # BroadCast
    mask = Tensor(np.array([True, True, False, True]).astype(np.bool))
    value = Tensor(np.array(22).astype(ntype))
    expect = np.array([[22, 22, 3, 22], [22, 22, 7, 22]]).astype(ntype)
    output = maskedfill_net(inputs, mask, value)
    assert (output.asnumpy() == expect).all()

@pytest.mark.level0
@pytest.mark.platform_x86_cpu
@pytest.mark.env_onecard
def test_maskedfill_float():
    """
    Feature: Test MaskedFill op.
    Description: Test MaskedFill with float input.
    Expectation: The result match to expect.
    """
    maskedfill_fun(np.float32)

@pytest.mark.level0
@pytest.mark.platform_x86_cpu
@pytest.mark.env_onecard
def test_maskedfill_float16():
    """
    Feature: Test MaskedFill op.
    Description: Test MaskedFill with float16 input.
    Expectation: The result match to expect.
    """
    maskedfill_fun(np.float16)

@pytest.mark.level0
@pytest.mark.platform_x86_cpu
@pytest.mark.env_onecard
def test_maskedfill_int():
    """
    Feature: Test MaskedFill op.
    Description: Test MaskedFill with int input.
    Expectation: The result match to expect.
    """
    maskedfill_fun(np.int32)

@pytest.mark.level0
@pytest.mark.platform_x86_cpu
@pytest.mark.env_onecard
def test_maskedfill_int8():
    """
    Feature: Test MaskedFill op.
    Description: Test MaskedFill with int8 input.
    Expectation: The result match to expect.
    """
    maskedfill_fun(np.int8)

def maskedfill_value(value):
    maskedfill_net = MaskedFillNet()
    inputs = Tensor(np.array([1, 2, 3, 4]).astype(np.float32))
    mask = Tensor(np.array([True, True, False, True]).astype(np.bool))
    expect = np.array([0.5, 0.5, 3, 0.5]).astype(np.float32)
    output = maskedfill_net(inputs, mask, value)
    assert (output.asnumpy() == expect).all()

@pytest.mark.level0
@pytest.mark.platform_x86_cpu
@pytest.mark.env_onecard
def test_maskedfill_float_value():
    """
    Feature: Test MaskedFill op.
    Description: Test MaskedFill with float value.
    Expectation: The result match to expect.
    """
    maskedfill_value(0.5)

@pytest.mark.level0
@pytest.mark.platform_x86_cpu
@pytest.mark.env_onecard
def test_maskedfill_tensor_value():
    """
    Feature: Test MaskedFill op.
    Description: Test MaskedFill with tensor input.
    Expectation: The result match to expect.
    """
    maskedfill_value(Tensor(0.5))


# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Functions for Auto SParsity (ASP) training and inference.
"""

import copy
import numpy as np
from paddle.fluid import framework, global_scope, program_guard, layers
from paddle.fluid.initializer import ConstantInitializer
from paddle.fluid.contrib import sparsity
from paddle.fluid import core

__all__ = ['ASPHelper']

class OpRelacementInfo(object):
    def __init__(self, source_type, target_type,
                param_shape_related_attrs={}, constant_attrs={},
                source_param_input_name='Y', source_param_idx=0,
                source_data_input_name='X', source_data_idx=0):
        self.source_type = source_type
        self.target_type = target_type
        self.param_shape_related_attrs = param_shape_related_attrs
        self.constant_attrs = constant_attrs
        self.source_param_input_name = source_param_input_name
        self.source_param_idx = source_param_idx
        self.source_data_input_name = source_data_input_name
        self.source_data_idx = source_data_idx

    def is_executable(self, block, op):
        return True

class MulSparseOpRelacementInfo(OpRelacementInfo):
    def __init__(self, source_type,
                source_param_input_name='Y', source_param_idx=0,
                source_data_input_name='X', source_data_idx=0):

        param_shape_related_attrs = {'m':1, 'k':0, 'lda':1, 'ldb':0, 'ldc':1}
        constant_attrs={'is_col_major':True, 'is_transpose_Y':True, 'switch_XY':True}

        super(MulSparseOpRelacementInfo, self).__init__(
            source_type=source_type, target_type='mul_sparse',
            param_shape_related_attrs=param_shape_related_attrs,
            constant_attrs=constant_attrs,
            source_param_input_name=source_param_input_name,
            source_param_idx=source_param_idx,
            source_data_input_name=source_data_input_name,
            source_data_idx=source_data_idx
        )

    def is_executable(self, block, op):
        param_name = op.input(self.source_param_input_name)[self.source_param_idx]
        param = block.var(param_name)

        data_name = op.input(self.source_data_input_name)[self.source_data_idx]
        data = block.var(data_name)

        if param is None or data is None:
            return False
        if param.dtype != core.VarDesc.VarType.FP16 and \
           data.dtype != core.VarDesc.VarType.FP16:
           return False
        if (param.shape[1] % 8) or \
           (param.shape[0] % 32):
           return False

        return True

class ASPHelper(object):
    r"""
    ASPHelper is a collection of Auto SParsity (ASP) functions to enable

    1. training models with weights in 2:4 sparse pattern from scratch.
    2. pruning well-trained models into 2:4 sparse pattern for fine-tuning.
    """

    MASKE_APPENDDED_NAME = '_asp_mask'
    SUPPORTED_LAYERS = {'fc':'w_0', 'linear':'w_0', 'conv2d':'w_0'}

    DENSE_SPARSE_OP_MAP = {
        'mul':MulSparseOpRelacementInfo(source_type='mul'),
        'matmul':MulSparseOpRelacementInfo(source_type='matmul')
    }

    SPARSE_OP_PARAM_INPUT_NAME_MAP = {
        'mul_sparse':'Y'
    }

    __mask_vars = {}
    __masks = {}
    __compressed_cache = {}
    __excluded_layers = []

    @staticmethod
    def get_mask_name(param_name):
        r"""
        Return mask name by given parameter name :attr:`param_name`.

        Args:
            param_name (string): The name of parameter.
        Returns:
            string: The mask name of :attr:`param_name`.
        """
        return param_name + ASPHelper.MASKE_APPENDDED_NAME

    @staticmethod
    def get_vars(main_program):
        r"""
        Get all parameters in :attr:`main_program` excluded ASP mask Variables.

        Args:
            main_program (Program): Program with model definition and its parameters.
        Returns:
            list: A list of parameter Variables in :attr:`main_program` (excluded ASP mask Variables).
        """
        var_list = []
        for param in main_program.global_block().all_parameters():
            if ASPHelper.MASKE_APPENDDED_NAME not in param.name:
                var_list.append(param)
        return var_list

    @classmethod
    def set_excluded_layers(cls, param_names):
        r"""
        Set parameter name of layers which would not be pruned as sparse weights.

        Args:
            param_names (list): A list contains names of parameters.
        """
        cls.__excluded_layers = copy.deepcopy(param_names)

    @classmethod
    def is_supported_layer(cls, param_name):
        r"""
        Verify if given :attr:`param_name` is supported by ASP.

        Args:
            param_name (string): The name of parameter.
        Returns:
            bool: True if it is supported, else False.
        Examples:
            .. code-block:: python

              import paddle.fluid as fluid
              from paddle.fluid.contrib.sparsity import ASPHelper

              main_program = fluid.Program()
              startup_program = fluid.Program()

              with fluid.program_guard(main_program, startup_program):
                  input_data = fluid.layers.data(name='data', shape=[None, 128])
                  fc = fluid.layers.fc(input=input_data, num_flatten_dims=-1, size=32, act=None)

              for param in main_program.global_block().all_parameters():
                  ASPHelper.is_supported_layer(param.name)
              # fc_0.w_0 -> True
              # fc_0.b_0 -> False
        """
        if ASPHelper.MASKE_APPENDDED_NAME in param_name:
            return False

        for layer in cls.__excluded_layers:
            if layer in param_name:
                return False

        for name in ASPHelper.SUPPORTED_LAYERS:
            if name in param_name and \
               ASPHelper.SUPPORTED_LAYERS[name] in param_name:
               return True
        return False

    @classmethod
    def minimize(cls, loss, optimizer, main_program, start_program):
        r"""
        This function is a decorator of `minimize` function in `Optimizer`.
        There are three steps:

        1. Call :attr:`optimizer`.minimize(:attr:`loss`)
        2. Create sparse mask Tensors according to supported layers in :attr:`main_program`.
        3. Insert masking ops in the end of parameters update.

        Args:
            loss (Variable): A Variable containing the value to minimize.
            optimizer (Optimizer): A Optimizer used for training.
            main_program (Program): Program with model definition and its parameters.
            start_program (Program): Program for initializing parameters.
        Returns:
            list: operators from :attr:`optimizer`.minimize(:attr:`loss`).
            list: pairs of parameters and their gradients.
        Examples:
            .. code-block:: python

              import paddle.fluid as fluid
              from paddle.fluid.contrib.sparsity import ASPHelper

              main_program = fluid.Program()
              start_program = fluid.Program()

              with fluid.program_guard(main_program, start_program):
                    input_data = fluid.layers.data(name='data', shape=[None, 128])
                    label = fluid.layers.data(name='label', shape=[None, 10])
                    hidden = fluid.layers.fc(input=input_data, num_flatten_dims=-1, size=32, act=None)
                    prob = fluid.layers.fc(input=hidden, num_flatten_dims=-1, size=10, act=None)
                    loss = fluid.layers.mean(fluid.layers.square_error_cost(prob, label))

                    optimizer = fluid.optimizer.SGD(learning_rate=0.1)
                    ASPHelper.minimize(loss, optimizer, main_program, start_program)
        """
        optimizer_ops, params_and_grads = optimizer.minimize(loss)
        cls.create_mask_variables(main_program, start_program, params_and_grads)
        cls.insert_sparse_mask_ops(main_program, start_program, params_and_grads)
        return optimizer_ops, params_and_grads

    @classmethod
    def create_mask_variables(cls, main_program, start_program, params_and_grads):
        r"""
        Create sparse mask Tensors according to supported layers in :attr:`main_program`.
        This function is called in second step of `ASPHelper.minimize`

        Args:
            main_program (Program): Program with model definition and its parameters.
            start_program (Program): Program for initializing parameters.
            params_and_grads (list): Variable pairs of parameters and their gradients.
        Examples:
            .. code-block:: python

              import paddle.fluid as fluid
              from paddle.fluid.contrib.sparsity import ASPHelper

              main_program = fluid.Program()
              start_program = fluid.Program()

              with fluid.program_guard(main_program, start_program):
                    input_data = fluid.layers.data(name='data', shape=[None, 128])
                    label = fluid.layers.data(name='label', shape=[None, 10])
                    hidden = fluid.layers.fc(input=input_data, num_flatten_dims=-1, size=32, act=None)
                    prob = fluid.layers.fc(input=hidden, num_flatten_dims=-1, size=10, act=None)
                    loss = fluid.layers.mean(fluid.layers.square_error_cost(prob, label))

                    optimizer = fluid.optimizer.SGD(learning_rate=0.1)

                    # Equal to ASPHelper.minimize(loss, optimizer, main_program, start_program)
                    optimizer_ops, params_and_grads = optimizer.minimize(loss)
                    ASPHelper.create_mask_variables(main_program, start_program, params_and_grads)
                    ASPHelper.insert_sparse_mask_ops(main_program, start_program, params_and_grads)
        """
        with program_guard(main_program, start_program):
            for param_and_grad in params_and_grads:
                if ASPHelper.is_supported_layer(param_and_grad[0].name):
                    mask_param = layers.create_parameter(
                                 name=param_and_grad[0].name + ASPHelper.MASKE_APPENDDED_NAME,
                                 shape=param_and_grad[0].shape, dtype=param_and_grad[0].dtype,
                                 default_initializer=ConstantInitializer(value=1.0))
                    mask_param.stop_gradient=True
                    mask_param.trainable=False
                    cls.__mask_vars[param_and_grad[0].name] = mask_param

    @classmethod
    def prune_model(cls, main_program, start_program, place, func_name='get_mask_1d_greedy', with_mask=True):
        r"""
        Pruning parameters of supported layers in :attr:`main_program` via
        specified mask generation function given by :attr:`func_name`. This
        function supports both training and inference controlled by :attr:`with_mask`.
        If :attr:`with_mask` is True, it would also prune parameter related ASP mask Variables,
        else only prunes parameters.

        *Note*: If calling this function with :attr:`with_mask`, it should call `ASPHelper.minimize`
        and initialization (`exe.run(startup_program`)) before.

        Args:
            main_program (Program): Program with model definition and its parameters.
            start_program (Program): Program for initializing parameters.
            place (fluid.CPUPlace()|fluid.CUDAPlace(N)): Device place for pruned parameter and mask Variables.
            func_name (string, optional): The name of function to generate spase masks. Defalut is `get_mask_1d_greedy`.
            with_mask (bool, optional): To prune mask Variables related to parameters or not. Ture is purning also, False is not. Defalut is True.
        Returns:
            dictionary: A dictionary with key: `parameter name` (string) and value: its corresponding mask Variable.
        Examples:
            .. code-block:: python

              import paddle.fluid as fluid
              from paddle.fluid.contrib.sparsity import ASPHelper

              main_program = fluid.Program()
              start_program = fluid.Program()

              place = fluid.CUDAPlace(0)

              with fluid.program_guard(main_program, start_program):
                    input_data = fluid.layers.data(name='data', shape=[None, 128])
                    label = fluid.layers.data(name='label', shape=[None, 10])
                    hidden = fluid.layers.fc(input=input_data, num_flatten_dims=-1, size=32, act=None)
                    prob = fluid.layers.fc(input=hidden, num_flatten_dims=-1, size=10, act=None)
                    loss = fluid.layers.mean(fluid.layers.square_error_cost(prob, label))

                    optimizer = fluid.optimizer.SGD(learning_rate=0.1)
                    ASPHelper.minimize(loss, optimizer, main_program, start_program)

              exe = fluid.Executor(place)
              exe.run(start_program)

              ASPHelper.prune_model(main_program, start_program, place, func_name="get_mask_2d_greedy")
        """
        checked_func_name = 'check_mask_1d' if '1d' in func_name else 'check_mask_2d'

        for param in main_program.global_block().all_parameters():
            if ASPHelper.is_supported_layer(param.name):
                weight_param = global_scope().find_var(param.name).get_tensor()
                weight_tensor = np.array(weight_param)
                weight_sparse_mask = sparsity.create_mask(weight_tensor.T, func_name=func_name).T
                weight_pruned_tensor = np.multiply(weight_tensor, weight_sparse_mask)
                weight_param.set(weight_pruned_tensor, place)
                assert sparsity.check_sparsity(weight_pruned_tensor.T, m=4, n=2, func_name=checked_func_name), \
                        'Pruning {} weight matrix failure!!!'.format(param.name)
                if with_mask:
                    weight_mask_param = global_scope().find_var(ASPHelper.get_mask_name(param.name))
                    assert weight_mask_param is not None, \
                        'Cannot find {} variable, please call ASPHelper.minimize' \
                        'initialization (exe.run(startup_program)) first!'.format(ASPHelper.get_mask_name(param.name))
                    weight_mask_param = weight_mask_param.get_tensor()
                    weight_mask_param.set(weight_sparse_mask, place)
                cls.__masks[param.name] = weight_sparse_mask
        return cls.__masks.copy()

    @classmethod
    def insert_sparse_mask_ops(cls, main_program, start_program, param_grads):
        r"""
        Insert masking ops in the end of parameters update.
        This function is called in third step of `ASPHelper.minimize`

        Args:
            main_program (Program): Program with model definition and its parameters.
            start_program (Program): Program for initializing parameters.
            params_and_grads (list): Variable pairs of parameters and their gradients.
        Examples:
            .. code-block:: python

              import paddle.fluid as fluid
              from paddle.fluid.contrib.sparsity import ASPHelper

              main_program = fluid.Program()
              start_program = fluid.Program()

              with fluid.program_guard(main_program, start_program):
                    input_data = fluid.layers.data(name='data', shape=[None, 128])
                    label = fluid.layers.data(name='label', shape=[None, 10])
                    hidden = fluid.layers.fc(input=input_data, num_flatten_dims=-1, size=32, act=None)
                    prob = fluid.layers.fc(input=hidden, num_flatten_dims=-1, size=10, act=None)
                    loss = fluid.layers.mean(fluid.layers.square_error_cost(prob, label))

                    optimizer = fluid.optimizer.SGD(learning_rate=0.1)

                    # Equal to ASPHelper.minimize(loss, optimizer, main_program, start_program)
                    optimizer_ops, params_and_grads = optimizer.minimize(loss)
                    ASPHelper.create_mask_variables(main_program, start_program, params_and_grads)
                    ASPHelper.insert_sparse_mask_ops(main_program, start_program, params_and_grads)
        """
        block = main_program.global_block()
        for param_grad in param_grads:
            if param_grad[0].name in cls.__mask_vars:
                block.append_op(
                    type='elementwise_mul',
                    inputs={"X": param_grad[0],
                            'Y': cls.__mask_vars[param_grad[0].name]},
                    outputs={'Out': param_grad[0]},
                    attrs={'axis': -1,
                            'use_mkldnn': False}
                )

    @classmethod
    def compress_model(cls, main_program, place):
        assert not main_program in cls.__compressed_cache, \
               'One program only need to compress model once. Called more than one would make errors'

        fake_batch_size = 128
        for param in main_program.global_block().all_parameters():
            if ASPHelper.is_supported_layer(param.name):
                shape = param.shape
                param_tensor = global_scope().find_var(param.name).get_tensor()
                core.compress_parameter(place, param_tensor, shape[1], fake_batch_size, shape[0],
                                        shape[1], shape[0], shape[1], True)

        cls.__compressed_cache[main_program] = True

        block = main_program.global_block()
        for op in block.ops:
            param_input_name = ASPHelper.SPARSE_OP_PARAM_INPUT_NAME_MAP.get(op.type, None)
            if param_input_name in op.input_names:
               for var_name in op.input(param_input_name):
                   if ASPHelper.is_supported_layer(var_name):
                       op._set_attr("is_sparse_compressed", True)
                       break

    @classmethod
    def replace_dense_to_sparse_op(cls, main_program):
        is_compressed = main_program in cls.__compressed_cache

        block = main_program.global_block()
        for op in block.ops:
            replacement_info = ASPHelper.DENSE_SPARSE_OP_MAP.get(op.type, None)
            if (replacement_info is not None) and \
               (replacement_info.source_param_input_name in op.input_names) and \
               (replacement_info.is_executable(block, op)):
                param_name = op.input(replacement_info.source_param_input_name)[replacement_info.source_param_idx]
                param = block.var(param_name)
                if (ASPHelper.is_supported_layer(param_name)) and \
                   (param is not None):
                    op.desc.set_type(replacement_info.target_type)
                    # TODO Need to be more general for future sparse conv ops.
                    op._set_attr("param_name", param.name)
                    op._set_attr("is_sparse_compressed", is_compressed)
                    for key, val in replacement_info.param_shape_related_attrs.items():
                       op._set_attr(key, param.shape[val])
                    for key, val in replacement_info.constant_attrs.items():
                       op._set_attr(key, val)


from datetime import date

import pandas as pd
import numpy as np

data = {'System Logs': 210, 'User Logs': 416, 'Debug Logs': 520}
print("<---- Create Series from dictionary ---->")
# When the data is a dict, and an index is not passed, the Series index will be ordered by
# the dict’s insertion order, if you’re using Python version >= 3.6 and Pandas version >= 0.23.
# If you’re using Python < 3.6 or Pandas < 0.23, and an index is not passed, the Series index
# will be the lexically ordered list of dict keys.
print(pd.Series(data=data))
# If an index is passed, the values in data corresponding to the labels in the index will
# be pulled out.
dict_data = {'a': 1.1, 'b': 2.2, 'c': 3.3, 'd': 4.4}
print("<---- Create Series using dictionary with specifying index ---->")
print("Series without specifying index")
print(pd.Series(data=dict_data))
print("Series with specifying index")
# NaN (not a number) is the standard missing data marker used in pandas.
print(pd.Series(data=dict_data,index=['c','b','e','a','d']))


#!/usr/bin/env python
# encoding: utf-8
#
# dss.py
#
# Created by José Sánchez-Gallego on 14 Sep 2017.

from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

from astropy.io import fits

import io
import pathlib
import urllib.request

import numpy as np

try:
    import PIL.Image
except ImportError:
    raise ImportError('this module requires pillow.')

__all__ = ['download_dss', 'DSS']

BASE_URL = ('http://stdatu.stsci.edu/cgi-bin/dss_search?'
            'v={survey_full}&r={ra:.6f}&d={dec:.6f}&e=J2000&h={height}&'
            'w={width}&f={format}&c=none&fov=NONE&v3=')

survey_full_dict = {'1r': 'poss1_red',
                    '1b': 'poss1_blue',
                    '2r': 'poss2ukstu_red',
                    '2b': 'poss2ukstu_blue',
                    '2ir': 'poss2ukstu_ir'}

class DSS(fits.HDUList):
    """Creates a DSS FITS object around a region of the sky.

    Parameters:
        ra,dec (float):
            Right Ascension and Declination around of the DSS region that will
            be created.
        size (float or list):
            The size, in arcmin, of the area to download. It can be specified
            as a single value, in which case a square region will be
            downloaded, or a list of two values, height and width.
        survey ({'2r', '2b', '2ir', '1r', '1b'}):
            The version of the survey to use for downloads.

    """

    def __init__(self, ra, dec, size=5, survey='2r', **kwargs):

        size = np.atleast_1d(size)
        assert len(size) > 0 and len(size) <= 2, 'incorrect shape for size.'

        assert ra >= 0 and ra < 360., 'invalid RA coordinate'
        assert dec > -90 and dec < 90, 'invalid Dec coordinate'

        if len(size) == 1:
            height = width = size[0]
        else:
            height, width = size

        assert survey in ['2r', '2ir', '2b', '1r', '1b'], 'invalid survey type.'

        survey_full = survey_full_dict[survey]

        self.url = BASE_URL.format(survey_full=survey_full, ra=ra, dec=dec,
                                   height=height, width=width, format='fits')

        try:
            url_data = urllib.request.urlopen(self.url)
        except urllib.request.URLError as ee:
            raise ValueError(f'cannot open URL for these parameters: {ee}')

        data = url_data.read()

        if 'Something went wrong' in str(data):
            if 'Calibration and image data not available for field data...' not in str(data):
                raise ValueError(f'survey {survey_full} does not cover coordiantes ({ra},{dec})')
            else:
                raise ValueError('unknown problem while retrieving the data.')

        fits_obj = fits.HDUList.fromstring(data)

        super(DSS, self).__init__(fits_obj)

        self._gif = None

    def writeto(self, *args, **kwargs):
        """Writes the FITS. See `~astropy.io.fits.HDUList.writeto`."""

        new_obj = fits.HDUList([ext for ext in self])

        return new_obj.writeto(*args, **kwargs)

    @property
    def gif(self):
        """Returns a PILImage_ object for this field.

        .. _PILImage: http://pillow.readthedocs.io/en/latest/reference/Image.html

        """

        if self._gif is None:

            url = self.url.replace('f=fits', 'f=gif')

            try:
                url_data = urllib.request.urlopen(url)
            except urllib.request.URLError as ee:
                raise ValueError(f'cannot open URL for these parameters: {ee}')

            data = url_data.read()

            image = PIL.Image.open(io.BytesIO(data))
            assert isinstance(image, PIL.GifImagePlugin.GifImageFile), 'incorrect image type.'

            self._gif = image

        return self._gif

def download_dss(ra, dec, path, save_gif=False, overwrite=False, **kwargs):
    """Downloads a DSS image.

    Parameters:
        ra,dec (float):
            Right Ascension and Declination of the DSS region to be downloaded.
        path (str):
            The full path where the FITS image will be saved.
        save_gif (bool):
            If ``True``, the GIF image will also be saved to the same location
            defined in ``path``.
        overwrite (bool):
            Whether the images, if they exist, should be overwritten.
        kwargs (dict):
            Other arguments to be passed to :class:`DSS`.

    Return:
        DSS object:
            A :class:`DSS` object representing the field.

    """

    dss = DSS(ra, dec, **kwargs)

    path = pathlib.Path(path)

    assert not path.is_dir(), 'path must contain a filename.'
    assert path.parent.exists(), 'directory does not exist.'

    dss.writeto(path, overwrite=overwrite)

    if save_gif:
        gif = dss.gif
        gif_path = path.with_suffix('.gif')
        if gif_path.exists():
            if overwrite:
                gif_path.unlink()
            else:
                raise FileExistsError(f'file {gif_path:s} exists and overwrite=False.')
        gif.save(gif_path)

    return dss


# -*- coding: utf-8 -*-
"""
Created on Tue Jun 12 12:12:16 2018

@author: user
"""

# 1) pandas hizli ve etkili for dataframes
# 2) csv ve text dosyalarına acip inceleyip sonuclarimiza bu dosya tiplerine rahat bir sekilde kaydedbilir.
# 3) pandas bizim isimizi kolaylastiriyor for missing data
# 4) reshape yapip datayi daha etkili bir sekilde kullanabiliriz
# 5) slicing indexing kolay
# 6) time series data analizinde cok yardimci olur
# 7) ayrica herseyden onemlisi hiz, pandas hiz acisindan optimize edilmis hizli bir kutuphanedir

import pandas as pd

dictionary = {"NAME":["ali","veli","kenan","hilal","ayse","evren"],
              "AGE":[15,16,17,33,45,66],
              "MAAS": [100,150,240,350,110,220]}

dataFrame1 = pd.DataFrame(dictionary)

head = dataFrame1.head()
tail = dataFrame1.tail()

# %%
# pandas basic method

print(dataFrame1.columns)

print(dataFrame1.info())

print(dataFrame1.dtypes)

print(dataFrame1.describe())  # numeric feature = columns (age,maas)

# %% indexing and slicing

print(dataFrame1["AGE"])
print(dataFrame1.AGE)

dataFrame1["yeni feature"] = [-1,-2,-3,-4,-5,-6]

print(dataFrame1.loc[:, "AGE"])

print(dataFrame1.loc[:3, "AGE"])

print(dataFrame1.loc[:3, "AGE":"NAME"])

print(dataFrame1.loc[:3, ["AGE","NAME"]])

print(dataFrame1.loc[::-1,:])

print(dataFrame1.loc[:,:"NAME"])

print(dataFrame1.loc[:,"NAME"]) #ACCEPT OBJECT INDEX

print(dataFrame1.iloc[:,2]) #ACCEPT INTEGER INDEX

# %% filtering

filtre1 = dataFrame1.MAAS > 200

filtrelenmis_data = dataFrame1[filtre1]

filtre2 = dataFrame1.AGE <20

dataFrame1[filtre1 & filtre2]

print(dataFrame1[dataFrame1.AGE > 60])

# %% list comprehension
import numpy as np

ortalama_maas = dataFrame1.MAAS.mean()

# ortalama_maas_np = np.mean(dataFrame1.MAAS)

dataFrame1["maas_seviyesi"] = ["dusuk" if ortalama_maas > each else "yuksek" for each in dataFrame1.MAAS]

#for each in dataFrame1.MAAS:
#    if(ortalama_maas > each):
#        print("dusuk")
#    else:
#        print("yukse")

dataFrame1.columns

dataFrame1.columns = [ each.lower() for each in dataFrame1.columns]

dataFrame1.columns = [each.split()[0]+"_"+each.split()[1] if(len(each.split())>1) else each for each in dataFrame1.columns]

# %% drop and concatenating

dataFrame1.drop(["yeni_feature"],axis=1,inplace = True)

# dataFrame1 = dataFrame1.drop(["yeni_feature"],axis=1)

data1 = dataFrame1.head()
data2 = dataFrame1.tail()

# vertical
data_concat = pd.concat([data1,data2],axis=0)

# horizontal

maas = dataFrame1.maas
age = dataFrame1.age

data_h_concat = pd.concat([maas,age],axis=1)

# %% transforming data

dataFrame1["list_comp"] = [ each*2 for each in dataFrame1.age]

# apply()

def multiply(age):
     return age*2

dataFrame1["apply_metodu"] = dataFrame1.age.apply(multiply)


# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function, division

import numpy as np
import unittest
import sys
sys.path.append("..")

import paddle
import paddle.fluid as fluid
from paddle.fluid import Program
from paddle.fluid.backward import append_backward

from op_test_xpu import XPUOpTest
from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper

paddle.enable_static()

class XPUTestWhereOp(XPUOpTestWrapper):
    def __init__(self):
        self.op_name = 'where'

    class TestXPUWhereOp(XPUOpTest):
        def setUp(self):
            self.init_config()
            self.init_data()
            self.inputs = {'Condition': self.cond, 'X': self.x, 'Y': self.y}
            self.outputs = {'Out': np.where(self.cond, self.x, self.y)}

        def init_data(self):
            self.x = np.random.uniform(-3, 5, (100)).astype(self.dtype)
            self.y = np.random.uniform(-3, 5, (100)).astype(self.dtype)
            self.cond = np.zeros((100)).astype("bool")

        def init_config(self):
            self.op_type = "where"
            self.dtype = self.in_type
            self.place = paddle.XPUPlace(0)
            self.__class__.no_need_check_grad = True

        def test_check_output(self):
            self.check_output_with_place(self.place)

    class TestXPUWhereOp2(TestXPUWhereOp):
        def init_data(self):
            self.x = np.random.uniform(-5, 5, (60, 2)).astype(self.dtype)
            self.y = np.random.uniform(-5, 5, (60, 2)).astype(self.dtype)
            self.cond = np.ones((60, 2)).astype("bool")

    class TestXPUWhereOp3(TestXPUWhereOp):
        def init_data(self):
            self.x = np.random.uniform(-3, 5, (20, 2, 4)).astype(self.dtype)
            self.y = np.random.uniform(-3, 5, (20, 2, 4)).astype(self.dtype)
            self.cond = np.array(
                np.random.randint(
                    2, size=(20, 2, 4)), dtype=bool)

support_types = get_xpu_op_support_types('where')
for stype in support_types:
    create_test_class(globals(), XPUTestWhereOp, stype)

class TestXPUWhereAPI(unittest.TestCase):
    def setUp(self):
        self.__class__.use_xpu = True
        self.place = paddle.XPUPlace(0)
        self.init_data()

    def init_data(self):
        self.shape = [10, 15]
        self.cond = np.array(np.random.randint(2, size=self.shape), dtype=bool)
        self.x = np.random.uniform(-2, 3, self.shape).astype(np.float32)
        self.y = np.random.uniform(-2, 3, self.shape).astype(np.float32)
        self.out = np.where(self.cond, self.x, self.y)

    def ref_x_backward(self, dout):
        return np.where(self.cond == True, dout, 0)

    def ref_y_backward(self, dout):
        return np.where(self.cond == False, dout, 0)

    def test_api(self):
        for x_stop_gradient in [False, True]:
            for y_stop_gradient in [False, True]:
                train_prog = fluid.Program()
                startup = fluid.Program()
                with fluid.program_guard(train_prog, startup):
                    cond = fluid.data(
                        name='cond', shape=self.shape, dtype='bool')
                    x = fluid.data(name='x', shape=self.shape, dtype='float32')
                    y = fluid.data(name='y', shape=self.shape, dtype='float32')

                    x.stop_gradient = x_stop_gradient
                    y.stop_gradient = y_stop_gradient

                    result = paddle.where(cond, x, y)
                    append_backward(fluid.layers.mean(result))

                    exe = fluid.Executor(self.place)
                    exe.run(startup)

                    fetch_list = [result, result.grad_name]
                    if x_stop_gradient is False:
                        fetch_list.append(x.grad_name)
                    if y_stop_gradient is False:
                        fetch_list.append(y.grad_name)
                    out = exe.run(
                        train_prog,
                        feed={'cond': self.cond,
                              'x': self.x,
                              'y': self.y},
                        fetch_list=fetch_list)
                    assert np.array_equal(out[0], self.out)

                    if x_stop_gradient is False:
                        assert np.array_equal(out[2],
                                              self.ref_x_backward(out[1]))
                        if y.stop_gradient is False:
                            assert np.array_equal(out[3],
                                                  self.ref_y_backward(out[1]))
                    elif y.stop_gradient is False:
                        assert np.array_equal(out[2],
                                              self.ref_y_backward(out[1]))

    def test_api_broadcast(self, use_cuda=False):
        train_prog = fluid.Program()
        startup = fluid.Program()
        with fluid.program_guard(train_prog, startup):
            x = fluid.layers.data(name='x', shape=[4, 1], dtype='float32')
            y = fluid.layers.data(name='y', shape=[4, 2], dtype='float32')
            x_i = np.array([[0.9383, 0.1983, 3.2, 1.2]]).astype("float32")
            y_i = np.array([[1.0, 1.0, 1.0, 1.0],
                            [1.0, 1.0, 1.0, 1.0]]).astype("float32")
            result = paddle.where(x > 1, x=x, y=y)

            exe = fluid.Executor(self.place)
            exe.run(startup)

            out = exe.run(train_prog,
                          feed={'x': x_i,
                                'y': y_i},
                          fetch_list=[result])
            assert np.array_equal(out[0], np.where(x_i > 1, x_i, y_i))

class TestWhereDygraphAPI(unittest.TestCase):
    def test_api(self):
        with fluid.dygraph.guard(paddle.XPUPlace(0)):
            x_i = np.array([0.9383, 0.1983, 3.2, 1.2]).astype("float32")
            y_i = np.array([1.0, 1.0, 1.0, 1.0]).astype("float32")
            cond_i = np.array([False, False, True, True]).astype("bool")
            x = fluid.dygraph.to_variable(x_i)
            y = fluid.dygraph.to_variable(y_i)
            cond = fluid.dygraph.to_variable(cond_i)
            out = paddle.where(cond, x, y)
            assert np.array_equal(out.numpy(), np.where(cond_i, x_i, y_i))

if __name__ == '__main__':
    unittest.main()