/*
# Copyright (c) 2014-2020 Reed A. Cartwright <reed@cartwright.ht>
#
# This file is part of the Ultimate Source Code Project.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
*/

#ifndef MUTK_MUTATION_HPP
#define MUTK_MUTATION_HPP

#include <mutk/memory.hpp>

#include <boost/container/static_vector.hpp>
#include <vector>

namespace mutk {

namespace detail {
enum struct Potential {
    Unit,                // All ones
    LikelihoodDiploid,   // P(Data|G)
    LikelihoodHaploid,   // P(Data|H)
    FounderDiploid,      // P(G)
    FounderHaploid,      // P(H)
    CloneDiploid,        // P(G1|G2)
    CloneHaploid,        // P(H1|H2)
    GameteDiploid,       // P(H1|G2)
    ChildDiploidDiploid, // P(G1|G2,G3)
    ChildHaploidDiploid, // P(G1|H2,G3)
    ChildDiploidHaploid, // P(G1|G2,H3)
    ChildHaploidHaploid, // P(G1|H1,H2)
    ChildSelfingDiploid, // P(G1|G2,G2)
    ChildSelfingHaploid  // P(G1|H2,H2)
};

struct potential_t {
    Potential type;
    int child;
    using data_t = std::pair<int,float>;
    boost::container::static_vector<data_t, 2> parents;
    std::vector<int> shuffle;

    potential_t() = default;
    potential_t(Potential type_arg, int child_arg) : type{type_arg}, child{child_arg} {}
    potential_t(Potential type_arg, int child_arg, int par1, float dist1) :
        type{type_arg}, child{child_arg}, parents{{par1,dist1}} {}
    potential_t(Potential type_arg, int child_arg, int par1, float dist1, int par2, float dist2) :
        type{type_arg}, child{child_arg}, parents{{par1,dist1},{par2,dist2}} {}
};
} // namespace detail

namespace mutation {

// Tags
struct any_t {};
struct mean_t {};

constexpr any_t ANY = {};
constexpr mean_t MEAN = {};

class Model {
public:
    using tensor_t = mutk::Tensor<1>;
    using matrix_t = mutk::Tensor<2>;
    using potential_t = mutk::detail::potential_t;

    virtual matrix_t CreateMatrix(int n, float t, any_t) const = 0;
    virtual matrix_t CreateMatrix(int n, float t, mean_t) const = 0;
    virtual matrix_t CreateMatrix(int n, float t, int x) const = 0;

    virtual tensor_t CreatePriorDiploid(int n) const = 0;
    virtual tensor_t CreatePriorHaploid(int n) const = 0;

    mutk::Tensor<1> CreatePotential(int n, const potential_t &potential, any_t);
    mutk::Tensor<1> CreatePotential(int n, const potential_t &potential, mean_t);
    mutk::Tensor<1> CreatePotential(int n, const potential_t &potential, int x);
};

// A k-alleles model. See Lewis 2001 and Tuffley and Steel 1997.
class KAllelesModel : public Model {
public:
    using tensor_t = Model::tensor_t;
    using matrix_t = Model::matrix_t;

    KAllelesModel(double k, double theta, double hom_bias, double het_bias, double hap_bias) :
        k_{k}, theta_{theta}, hom_bias_{hom_bias}, het_bias_{het_bias}, hap_bias_{hap_bias} {

        double e = theta/(k-1.0);

        if(k < 2.0) {
            throw std::invalid_argument("Invalid k parameter.");
        }
        if(theta < 0) {
            throw std::invalid_argument("Invalid theta parameter");
        }
        if(hom_bias > 1.0 || hom_bias < -(2.0+e)/((k-1.0)*e)) {
            throw std::invalid_argument("Invalid hom_bias parameter.");
        }
        if(het_bias > 1.0 || het_bias < -(2.0+2.0*e)/((k-2.0)*e)) {
            throw std::invalid_argument("Invalid het_bias parameter.");
        }
        if(hap_bias > 1.0 || hap_bias < -(1.0+e)/((k-1.0)*e)) {
            throw std::invalid_argument("Invalid hap_bias parameter.");
        }
    }

    virtual matrix_t CreateMatrix(int n, float t, any_t) const override;
    virtual matrix_t CreateMatrix(int n, float t, mean_t) const override;
    virtual matrix_t CreateMatrix(int n, float t, int x) const override;

    virtual tensor_t CreatePriorDiploid(int n) const override;
    virtual tensor_t CreatePriorHaploid(int n) const override;

protected:
    double k_;
    double theta_;
    double hom_bias_;
    double het_bias_;
    double hap_bias_;
};

} // namespace mutation
} // namespace mutk

#endif // MUTK_MUTATION_HPP


//
//  Copyright (c) 2000-2002
//  Joerg Walter, Mathias Koch
//
//  Permission to use, copy, modify, distribute and sell this software
//  and its documentation for any purpose is hereby granted without fee,
//  provided that the above copyright notice appear in all copies and
//  that both that copyright notice and this permission notice appear
//  in supporting documentation.  The authors make no representations
//  about the suitability of this software for any purpose.
//  It is provided "as is" without express or implied warranty.
//
//  The authors gratefully acknowledge the support of
//  GeNeSys mbH & Co. KG in producing this work.
//

#ifndef BOOST_UBLAS_FUNCTIONAL_H
#define BOOST_UBLAS_FUNCTIONAL_H

#include <functional>

#include <boost/numeric/ublas/config.hpp>
#include <boost/numeric/ublas/exception.hpp>
#include <boost/numeric/ublas/traits.hpp>
#ifdef BOOST_UBLAS_USE_DUFF_DEVICE
#include <boost/numeric/ublas/duff.hpp>
#endif
#ifdef BOOST_UBLAS_USE_SIMD
#include <boost/numeric/ublas/raw.hpp>
#else
namespace boost { namespace numeric { namespace ublas { namespace raw {
}}}}
#endif
#ifdef BOOST_UBLAS_HAVE_BINDINGS
#include <boost/numeric/bindings/traits/std_vector.hpp>
#include <boost/numeric/bindings/traits/ublas_vector.hpp>
#include <boost/numeric/bindings/traits/ublas_matrix.hpp>
#include <boost/numeric/bindings/atlas/cblas.hpp>
#endif

namespace boost { namespace numeric { namespace ublas {

    // Scalar functors

    // Unary
    template<class T>
    struct scalar_unary_functor {
        typedef T value_type;
        typedef typename type_traits<T>::const_reference argument_type;
        typedef typename type_traits<T>::value_type result_type;
    };

    template<class T>
    struct scalar_identity:
        public scalar_unary_functor<T> {
        typedef typename scalar_unary_functor<T>::argument_type argument_type;
        typedef typename scalar_unary_functor<T>::result_type result_type;

        static BOOST_UBLAS_INLINE
        result_type apply (argument_type t) {
            return t;
        }
    };
    template<class T>
    struct scalar_negate:
        public scalar_unary_functor<T> {
        typedef typename scalar_unary_functor<T>::argument_type argument_type;
        typedef typename scalar_unary_functor<T>::result_type result_type;

        static BOOST_UBLAS_INLINE
        result_type apply (argument_type t) {
            return - t;
        }
    };
    template<class T>
    struct scalar_conj:
        public scalar_unary_functor<T> {
        typedef typename scalar_unary_functor<T>::value_type value_type;
        typedef typename scalar_unary_functor<T>::argument_type argument_type;
        typedef typename scalar_unary_functor<T>::result_type result_type;

        static BOOST_UBLAS_INLINE
        result_type apply (argument_type t) {
            return type_traits<value_type>::conj (t);
        }
    };

    // Unary returning real
    template<class T>
    struct scalar_real_unary_functor {
        typedef T value_type;
        typedef typename type_traits<T>::const_reference argument_type;
        typedef typename type_traits<T>::real_type result_type;
    };

    template<class T>
    struct scalar_real:
        public scalar_real_unary_functor<T> {
        typedef typename scalar_real_unary_functor<T>::value_type value_type;
        typedef typename scalar_real_unary_functor<T>::argument_type argument_type;
        typedef typename scalar_real_unary_functor<T>::result_type result_type;

        static BOOST_UBLAS_INLINE
        result_type apply (argument_type t) {
            return type_traits<value_type>::real (t);
        }
    };
    template<class T>
    struct scalar_imag:
        public scalar_real_unary_functor<T> {
        typedef typename scalar_real_unary_functor<T>::value_type value_type;
        typedef typename scalar_real_unary_functor<T>::argument_type argument_type;
        typedef typename scalar_real_unary_functor<T>::result_type result_type;

        static BOOST_UBLAS_INLINE
        result_type apply (argument_type t) {
            return type_traits<value_type>::imag (t);
        }
    };

    // Binary
    template<class T1, class T2>
    struct scalar_binary_functor {
        typedef typename type_traits<T1>::const_reference argument1_type;
        typedef typename type_traits<T2>::const_reference argument2_type;
        typedef typename promote_traits<T1, T2>::promote_type result_type;
    };

    template<class T1, class T2>
    struct scalar_plus:
        public scalar_binary_functor<T1, T2> {
        typedef typename scalar_binary_functor<T1, T2>::argument1_type argument1_type;
        typedef typename scalar_binary_functor<T1, T2>::argument2_type argument2_type;
        typedef typename scalar_binary_functor<T1, T2>::result_type result_type;

        static BOOST_UBLAS_INLINE
        result_type apply (argument1_type t1, argument2_type t2) {
            return t1 + t2;
        }
    };
    template<class T1, class T2>
    struct scalar_minus:
        public scalar_binary_functor<T1, T2> {
        typedef typename scalar_binary_functor<T1, T2>::argument1_type argument1_type;
        typedef typename scalar_binary_functor<T1, T2>::argument2_type argument2_type;
        typedef typename scalar_binary_functor<T1, T2>::result_type result_type;

        static BOOST_UBLAS_INLINE
        result_type apply (argument1_type t1, argument2_type t2) {
            return t1 - t2;
        }
    };
    template<class T1, class T2>
    struct scalar_multiplies:
        public scalar_binary_functor<T1, T2> {
        typedef typename scalar_binary_functor<T1, T2>::argument1_type argument1_type;
        typedef typename scalar_binary_functor<T1, T2>::argument2_type argument2_type;
        typedef typename scalar_binary_functor<T1, T2>::result_type result_type;

        static BOOST_UBLAS_INLINE
        result_type apply (argument1_type t1, argument2_type t2) {
            return t1 * t2;
        }
    };
    template<class T1, class T2>
    struct scalar_divides:
        public scalar_binary_functor<T1, T2> {
        typedef typename scalar_binary_functor<T1, T2>::argument1_type argument1_type;
        typedef typename scalar_binary_functor<T1, T2>::argument2_type argument2_type;
        typedef typename scalar_binary_functor<T1, T2>::result_type result_type;

        static BOOST_UBLAS_INLINE
        result_type apply (argument1_type t1, argument2_type t2) {
            return t1 / t2;
        }
    };

    template<class T1, class T2>
    struct scalar_binary_assign_functor {
        // ISSUE Remove reference to avoid reference to reference problems
        typedef typename type_traits<typename boost::remove_reference<T1>::type>::reference argument1_type;
        typedef typename type_traits<T2>::const_reference argument2_type;
    };

    struct assign_tag {};
    struct computed_assign_tag {};

    template<class T1, class T2>
    struct scalar_assign:
        public scalar_binary_assign_functor<T1, T2> {
        typedef typename scalar_binary_assign_functor<T1, T2>::argument1_type argument1_type;
        typedef typename scalar_binary_assign_functor<T1, T2>::argument2_type argument2_type;
        typedef assign_tag assign_category;

        static BOOST_UBLAS_INLINE
        void apply (argument1_type t1, argument2_type t2) {
            t1 = t2;
        }

        template<class U1, class U2>
        static BOOST_UBLAS_INLINE
        scalar_assign<U1, U2> make_debug_functor () {
            return scalar_assign<U1, U2> ();
        }

    };
    template<class T1, class T2>
    struct scalar_plus_assign:
        public scalar_binary_assign_functor<T1, T2> {
        typedef typename scalar_binary_assign_functor<T1, T2>::argument1_type argument1_type;
        typedef typename scalar_binary_assign_functor<T1, T2>::argument2_type argument2_type;
        typedef computed_assign_tag assign_category;

        static BOOST_UBLAS_INLINE
        void apply (argument1_type t1, argument2_type t2) {
            t1 += t2;
        }

        template<class U1, class U2>
        static BOOST_UBLAS_INLINE
        scalar_plus_assign<U1, U2> make_debug_functor () {
            return scalar_plus_assign<U1, U2> ();
        }
    };
    template<class T1, class T2>
    struct scalar_minus_assign:
        public scalar_binary_assign_functor<T1, T2> {
        typedef typename scalar_binary_assign_functor<T1, T2>::argument1_type argument1_type;
        typedef typename scalar_binary_assign_functor<T1, T2>::argument2_type argument2_type;
        typedef computed_assign_tag assign_category;

        static BOOST_UBLAS_INLINE
        void apply (argument1_type t1, argument2_type t2) {
            t1 -= t2;
        }

        template<class U1, class U2>
        static BOOST_UBLAS_INLINE
        scalar_minus_assign<U1, U2> make_debug_functor () {
            return scalar_minus_assign<U1, U2> ();
        }
    };
    template<class T1, class T2>
    struct scalar_multiplies_assign:
        public scalar_binary_assign_functor<T1, T2> {
        typedef typename scalar_binary_assign_functor<T1, T2>::argument1_type argument1_type;
        typedef typename scalar_binary_assign_functor<T1, T2>::argument2_type argument2_type;
        typedef computed_assign_tag assign_category;

        static BOOST_UBLAS_INLINE
        void apply (argument1_type t1, argument2_type t2) {
            t1 *= t2;
        }

        template<class U1, class U2>
        static BOOST_UBLAS_INLINE
        scalar_multiplies_assign<U1, U2> make_debug_functor () {
            return scalar_multiplies_assign<U1, U2> ();
        }
    };
    template<class T1, class T2>
    struct scalar_divides_assign:
        public scalar_binary_assign_functor<T1, T2> {
        typedef typename scalar_binary_assign_functor<T1, T2>::argument1_type argument1_type;
        typedef typename scalar_binary_assign_functor<T1, T2>::argument2_type argument2_type;
        typedef computed_assign_tag assign_category;

        static BOOST_UBLAS_INLINE
        void apply (argument1_type t1, argument2_type t2) {
            t1 /= t2;
        }

        template<class U1, class U2>
        static BOOST_UBLAS_INLINE
        scalar_divides_assign<U1, U2> make_debug_functor () {
            return scalar_divides_assign<U1, U2> ();
        }
    };

    template<class T1, class T2>
    struct scalar_binary_swap_functor {
        typedef typename type_traits<typename boost::remove_reference<T1>::type>::reference argument1_type;
        typedef typename type_traits<typename boost::remove_reference<T2>::type>::reference argument2_type;
    };

    template<class T1, class T2>
    struct scalar_swap:
        public scalar_binary_swap_functor<T1, T2> {
        typedef typename scalar_binary_swap_functor<T1, T2>::argument1_type argument1_type;
        typedef typename scalar_binary_swap_functor<T1, T2>::argument2_type argument2_type;

        static BOOST_UBLAS_INLINE
        void apply (argument1_type t1, argument2_type t2) {
            std::swap (t1, t2);
        }

        template<class U1, class U2>
        static BOOST_UBLAS_INLINE
        scalar_swap<U1, U2> make_debug_functor () {
            return scalar_swap<U1, U2> ();
        }
    };

    // Vector functors

    // Unary returning scalar
    template<class T>
    struct vector_scalar_unary_functor {
        typedef std::size_t size_type;
        typedef std::ptrdiff_t difference_type;
        typedef T value_type;
        typedef T result_type;
    };

    template<class T>
    struct vector_sum:
        public vector_scalar_unary_functor<T> {
        typedef typename vector_scalar_unary_functor<T>::size_type size_type;
        typedef typename vector_scalar_unary_functor<T>::difference_type difference_type;
        typedef typename vector_scalar_unary_functor<T>::value_type value_type;
        typedef typename vector_scalar_unary_functor<T>::result_type result_type;

        template<class E>
        static BOOST_UBLAS_INLINE
        result_type apply (const vector_expression<E> &e) {
            result_type t = result_type (0);
            size_type size (e ().size ());
            for (size_type i = 0; i < size; ++ i)
                t += e () (i);
            return t;
        }
        // Dense case
        template<class I>
        static BOOST_UBLAS_INLINE
        result_type apply (difference_type size, I it) {
            result_type t = result_type (0);
            while (-- size >= 0)
                t += *it, ++ it;
            return t;
        }
        // Sparse case
        template<class I>
        static BOOST_UBLAS_INLINE
        result_type apply (I it, const I &it_end) {
            result_type t = result_type (0);
            while (it != it_end)
                t += *it, ++ it;
            return t;
        }
    };

    // Unary returning real scalar
    template<class T>
    struct vector_scalar_real_unary_functor {
        typedef std::size_t size_type;
        typedef std::ptrdiff_t difference_type;
        typedef T value_type;
        typedef typename type_traits<T>::real_type real_type;
        typedef real_type result_type;
    };

    template<class T>
    struct vector_norm_1:
        public vector_scalar_real_unary_functor<T> {
        typedef typename vector_scalar_real_unary_functor<T>::size_type size_type;
        typedef typename vector_scalar_real_unary_functor<T>::difference_type difference_type;
        typedef typename vector_scalar_real_unary_functor<T>::value_type value_type;
        typedef typename vector_scalar_real_unary_functor<T>::real_type real_type;
        typedef typename vector_scalar_real_unary_functor<T>::result_type result_type;

        template<class E>
        static BOOST_UBLAS_INLINE
        result_type apply (const vector_expression<E> &e) {
            real_type t = real_type ();
            size_type size (e ().size ());
            for (size_type i = 0; i < size; ++ i) {
                real_type u (type_traits<value_type>::norm_1 (e () (i)));
                t += u;
            }
            return t;
        }
        // Dense case
        template<class I>
        static BOOST_UBLAS_INLINE
        result_type apply (difference_type size, I it) {
            real_type t = real_type ();
            while (-- size >= 0) {
                real_type u (type_traits<value_type>::norm_1 (*it));
                t += u;
                ++ it;
            }
            return t;
        }
        // Sparse case
        template<class I>
        static BOOST_UBLAS_INLINE
        result_type apply (I it, const I &it_end) {
            real_type t = real_type ();
            while (it != it_end) {
                real_type u (type_traits<value_type>::norm_1 (*it));
                t += u;
                ++ it;
            }
            return t;
        }
    };
    template<class T>
    struct vector_norm_2:
        public vector_scalar_real_unary_functor<T> {
        typedef typename vector_scalar_real_unary_functor<T>::size_type size_type;
        typedef typename vector_scalar_real_unary_functor<T>::difference_type difference_type;
        typedef typename vector_scalar_real_unary_functor<T>::value_type value_type;
        typedef typename vector_scalar_real_unary_functor<T>::real_type real_type;
        typedef typename vector_scalar_real_unary_functor<T>::result_type result_type;

        template<class E>
        static BOOST_UBLAS_INLINE
        result_type apply (const vector_expression<E> &e) {
#ifndef BOOST_UBLAS_SCALED_NORM
            real_type t = real_type ();
            size_type size (e ().size ());
            for (size_type i = 0; i < size; ++ i) {
                real_type u (type_traits<value_type>::norm_2 (e () (i)));
                t +=  u * u;
            }
            return type_traits<real_type>::sqrt (t);
#else
            real_type scale = real_type ();
            real_type sum_squares (1);
            size_type size (e ().size ());
            for (size_type i = 0; i < size; ++ i) {
                real_type u (type_traits<value_type>::norm_2 (e () (i)));
                if (scale < u) {
                    real_type v (scale / u);
                    sum_squares = sum_squares * v * v + real_type (1);
                    scale = u;
                } else {
                    real_type v (u / scale);
                    sum_squares += v * v;
                }
            }
            return scale * type_traits<real_type>::sqrt (sum_squares);
#endif
        }
        // Dense case
        template<class I>
        static BOOST_UBLAS_INLINE
        result_type apply (difference_type size, I it) {
#ifndef BOOST_UBLAS_SCALED_NORM
            real_type t = real_type ();
            while (-- size >= 0) {
                real_type u (type_traits<value_type>::norm_2 (*it));
                t +=  u * u;
                ++ it;
            }
            return type_traits<real_type>::sqrt (t);
#else
            real_type scale = real_type ();
            real_type sum_squares (1);
            while (-- size >= 0) {
                real_type u (type_traits<value_type>::norm_2 (*it));
                if (scale < u) {
                    real_type v (scale / u);
                    sum_squares = sum_squares * v * v + real_type (1);
                    scale = u;
                } else {
                    real_type v (u / scale);
                    sum_squares += v * v;
                }
                ++ it;
            }
            return scale * type_traits<real_type>::sqrt (sum_squares);
#endif
        }
        // Sparse case
        template<class I>
        static BOOST_UBLAS_INLINE
        result_type apply (I it, const I &it_end) {
#ifndef BOOST_UBLAS_SCALED_NORM
            real_type t = real_type ();
            while (it != it_end) {
                real_type u (type_traits<value_type>::norm_2 (*it));
                t +=  u * u;
                ++ it;
            }
            return type_traits<real_type>::sqrt (t);
#else
            real_type scale = real_type ();
            real_type sum_squares (1);
            while (it != it_end) {
                real_type u (type_traits<value_type>::norm_2 (*it));
                if (scale < u) {
                    real_type v (scale / u);
                    sum_squares = sum_squares * v * v + real_type (1);
                    scale = u;
                } else {
                    real_type v (u / scale);
                    sum_squares += v * v;
                }
                ++ it;
            }
            return scale * type_traits<real_type>::sqrt (sum_squares);
#endif
        }
    };
    template<class T>
    struct vector_norm_inf:
        public vector_scalar_real_unary_functor<T> {
        typedef typename vector_scalar_real_unary_functor<T>::size_type size_type;
        typedef typename vector_scalar_real_unary_functor<T>::difference_type difference_type;
        typedef typename vector_scalar_real_unary_functor<T>::value_type value_type;
        typedef typename vector_scalar_real_unary_functor<T>::real_type real_type;
        typedef typename vector_scalar_real_unary_functor<T>::result_type result_type;

        template<class E>
        static BOOST_UBLAS_INLINE
        result_type apply (const vector_expression<E> &e) {
            real_type t = real_type ();
            size_type size (e ().size ());
            for (size_type i = 0; i < size; ++ i) {
                real_type u (type_traits<value_type>::norm_inf (e () (i)));
                if (u > t)
                    t = u;
            }
            return t;
        }
        // Dense case
        template<class I>
        static BOOST_UBLAS_INLINE
        result_type apply (difference_type size, I it) {
            real_type t = real_type ();
            while (-- size >= 0) {
                real_type u (type_traits<value_type>::norm_inf (*it));
                if (u > t)
                    t = u;
                ++ it;
            }
            return t;
        }
        // Sparse case
        template<class I>
        static BOOST_UBLAS_INLINE
        result_type apply (I it, const I &it_end) {
            real_type t = real_type ();
            while (it != it_end) {
                real_type u (type_traits<value_type>::norm_inf (*it));
                if (u > t)
                    t = u;
                ++ it;
            }
            return t;
        }
    };

    // Unary returning index
    template<class T>
    struct vector_scalar_index_unary_functor {
        typedef std::size_t size_type;
        typedef std::ptrdiff_t difference_type;
        typedef T value_type;
        typedef typename type_traits<T>::real_type real_type;
        typedef size_type result_type;
    };

    template<class T>
    struct vector_index_norm_inf:
        public vector_scalar_index_unary_functor<T> {
        typedef typename vector_scalar_index_unary_functor<T>::size_type size_type;
        typedef typename vector_scalar_index_unary_functor<T>::difference_type difference_type;
        typedef typename vector_scalar_index_unary_functor<T>::value_type value_type;
        typedef typename vector_scalar_index_unary_functor<T>::real_type real_type;
        typedef typename vector_scalar_index_unary_functor<T>::result_type result_type;

        template<class E>
        static BOOST_UBLAS_INLINE
        result_type apply (const vector_expression<E> &e) {
            // ISSUE For CBLAS compatibility return 0 index in empty case
            result_type i_norm_inf (0);
            real_type t = real_type ();
            size_type size (e ().size ());
            for (size_type i = 0; i < size; ++ i) {
                real_type u (type_traits<value_type>::norm_inf (e () (i)));
                if (u > t) {
                    i_norm_inf = i;
                    t = u;
                }
            }
            return i_norm_inf;
        }
        // Dense case
        template<class I>
        static BOOST_UBLAS_INLINE
        result_type apply (difference_type size, I it) {
            // ISSUE For CBLAS compatibility return 0 index in empty case
            result_type i_norm_inf (0);
            real_type t = real_type ();
            while (-- size >= 0) {
                real_type u (type_traits<value_type>::norm_inf (*it));
                if (u > t) {
                    i_norm_inf = it.index ();
                    t = u;
                }
                ++ it;
            }
            return i_norm_inf;
        }
        // Sparse case
        template<class I>
        static BOOST_UBLAS_INLINE
        result_type apply (I it, const I &it_end) {
            // ISSUE For CBLAS compatibility return 0 index in empty case
            result_type i_norm_inf (0);
            real_type t = real_type ();
            while (it != it_end) {
                real_type u (type_traits<value_type>::norm_inf (*it));
                if (u > t) {
                    i_norm_inf = it.index ();
                    t = u;
                }
                ++ it;
            }
            return i_norm_inf;
        }
    };

    // Binary returning scalar
    template<class T1, class T2, class TR>
    struct vector_scalar_binary_functor {
        typedef std::size_t size_type;
        typedef std::ptrdiff_t difference_type;
        typedef TR value_type;
        typedef TR result_type;
    };

    template<class T1, class T2, class TR>
    struct vector_inner_prod:
        public vector_scalar_binary_functor<T1, T2, TR> {
        typedef typename vector_scalar_binary_functor<T1, T2, TR>::size_type size_type ;
        typedef typename vector_scalar_binary_functor<T1, T2, TR>::difference_type difference_type;
        typedef typename vector_scalar_binary_functor<T1, T2, TR>::value_type value_type;
        typedef typename vector_scalar_binary_functor<T1, T2, TR>::result_type result_type;

        template<class E1, class E2>
        static BOOST_UBLAS_INLINE
        result_type apply (const vector_expression<E1> &e1,
                                 const vector_expression<E2> &e2,
                                 concrete_tag) {
#ifndef BOOST_UBLAS_HAVE_BINDINGS
            using namespace raw;
            size_type size (BOOST_UBLAS_SAME (e1 ().size (), e2 ().size ()));
            const T1 *data1 = data_const (e1 ());
            const T2 *data2 = data_const (e2 ());
            size_type s1 = stride (e1 ());
            size_type s2 = stride (e2 ());
            result_type t = result_type (0);
            if (s1 == 1 && s2 == 1) {
                for (size_type i = 0; i < size; ++ i)
                    t += data1 [i] * data2 [i];
            } else if (s2 == 1) {
                for (size_type i = 0, i1 = 0; i < size; ++ i, i1 += s1)
                    t += data1 [i1] * data2 [i];
            } else if (s1 == 1) {
                for (size_type i = 0, i2 = 0; i < size; ++ i, i2 += s2)
                    t += data1 [i] * data2 [i2];
            } else {
                for (size_type i = 0, i1 = 0, i2 = 0; i < size; ++ i, i1 += s1, i2 += s2)
                    t += data1 [i1] * data2 [i2];
            }
            return t;
#else
            return boost::numeric::bindings::atlas::dot (e1 (), e2 ());
#endif
        }
        template<class E1, class E2>
        static BOOST_UBLAS_INLINE
        result_type apply (const vector_expression<E1> &e1,
                                 const vector_expression<E2> &e2,
                                 abstract_tag) {
            size_type size (BOOST_UBLAS_SAME (e1 ().size (), e2 ().size ()));
            result_type t = result_type (0);
#ifndef BOOST_UBLAS_USE_DUFF_DEVICE
            for (size_type i = 0; i < size; ++ i)
                t += e1 () (i) * e2 () (i);
#else
            size_type i (0);
            DD (size, 4, r, (t += e1 () (i) * e2 () (i), ++ i));
#endif
            return t;
        }
        template<class E1, class E2>
        static BOOST_UBLAS_INLINE
        result_type apply (const vector_expression<E1> &e1,
                                 const vector_expression<E2> &e2) {
#ifdef BOOST_UBLAS_USE_SIMD
            typedef typename boost::mpl::if_<
                boost::mpl::and_<boost::is_same<typename E1::simd_category, concrete_tag>,
                                 boost::is_same<typename E2::simd_category, concrete_tag> >,
                    concrete_tag,
                    abstract_tag>::type simd_category;
#else
            typedef abstract_tag simd_category;
#endif
            return apply (e1, e2, simd_category ());
        }
        // Dense case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (difference_type size, I1 it1, I2 it2) {
            result_type t = result_type (0);
#ifndef BOOST_UBLAS_USE_DUFF_DEVICE
            while (-- size >= 0)
                t += *it1 * *it2, ++ it1, ++ it2;
#else
            DD (size, 4, r, (t += *it1 * *it2, ++ it1, ++ it2));
#endif
            return t;
        }
        // Packed case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (I1 it1, const I1 &it1_end, I2 it2, const I2 &it2_end) {
            result_type t = result_type (0);
            difference_type it1_size (it1_end - it1);
            difference_type it2_size (it2_end - it2);
            difference_type diff (0);
            if (it1_size > 0 && it2_size > 0)
                diff = it2.index () - it1.index ();
            if (diff != 0) {
                difference_type size = (std::min) (diff, it1_size);
                if (size > 0) {
                    it1 += size;
                    it1_size -= size;
                    diff -= size;
                }
                size = (std::min) (- diff, it2_size);
                if (size > 0) {
                    it2 += size;
                    it2_size -= size;
                    diff += size;
                }
            }
            difference_type size ((std::min) (it1_size, it2_size));
            while (-- size >= 0)
                t += *it1 * *it2, ++ it1, ++ it2;
            return t;
        }
        // Sparse case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (I1 it1, const I1 &it1_end, I2 it2, const I2 &it2_end, sparse_bidirectional_iterator_tag) {
            result_type t = result_type (0);
            if (it1 != it1_end && it2 != it2_end) {
                size_type it1_index = it1.index (), it2_index = it2.index ();
                while (true) {
                    difference_type compare = it1_index - it2_index;
                    if (compare == 0) {
                        t += *it1 * *it2, ++ it1, ++ it2;
                        if (it1 != it1_end && it2 != it2_end) {
                            it1_index = it1.index ();
                            it2_index = it2.index ();
                        } else
                            break;
                    } else if (compare < 0) {
                        increment (it1, it1_end, - compare);
                        if (it1 != it1_end)
                            it1_index = it1.index ();
                        else
                            break;
                    } else if (compare > 0) {
                        increment (it2, it2_end, compare);
                        if (it2 != it2_end)
                            it2_index = it2.index ();
                        else
                            break;
                    }
                }
            }
            return t;
        }
    };

    // Matrix functors

    // Binary returning vector
    template<class T1, class T2, class TR>
    struct matrix_vector_binary_functor {
        typedef std::size_t size_type;
        typedef std::ptrdiff_t difference_type;
        typedef TR value_type;
        typedef TR result_type;
    };

    template<class T1, class T2, class TR>
    struct matrix_vector_prod1:
        public matrix_vector_binary_functor<T1, T2, TR> {
        typedef typename matrix_vector_binary_functor<T1, T2, TR>::size_type size_type;
        typedef typename matrix_vector_binary_functor<T1, T2, TR>::difference_type difference_type;
        typedef typename matrix_vector_binary_functor<T1, T2, TR>::value_type value_type;
        typedef typename matrix_vector_binary_functor<T1, T2, TR>::result_type result_type;

        template<class E1, class E2>
        static BOOST_UBLAS_INLINE
        result_type apply (const matrix_expression<E1> &e1,
                                 const vector_expression<E2> &e2,
                                 size_type i, concrete_tag) {
#ifndef BOOST_UBLAS_HAVE_BINDINGS
            using namespace raw;
            size_type size = BOOST_UBLAS_SAME (e1 ().size2 (), e2 ().size ());
            const T1 *data1 = data_const (e1 ()) + i * stride1 (e1 ());
            const T2 *data2 = data_const (e2 ());
            size_type s1 = stride2 (e1 ());
            size_type s2 = stride (e2 ());
            result_type t = result_type (0);
            if (s1 == 1 && s2 == 1) {
                for (size_type j = 0; j < size; ++ j)
                    t += data1 [j] * data2 [j];
            } else if (s2 == 1) {
                for (size_type j = 0, j1 = 0; j < size; ++ j, j1 += s1)
                    t += data1 [j1] * data2 [j];
            } else if (s1 == 1) {
                for (size_type j = 0, j2 = 0; j < size; ++ j, j2 += s2)
                    t += data1 [j] * data2 [j2];
            } else {
                for (size_type j = 0, j1 = 0, j2 = 0; j < size; ++ j, j1 += s1, j2 += s2)
                    t += data1 [j1] * data2 [j2];
            }
            return t;
#else
            return boost::numeric::bindings::atlas::dot (e1 ().row (i), e2 ());
#endif
        }
        template<class E1, class E2>
        static BOOST_UBLAS_INLINE
        result_type apply (const matrix_expression<E1> &e1,
                                 const vector_expression<E2> &e2,
                                 size_type i, abstract_tag) {
            size_type size = BOOST_UBLAS_SAME (e1 ().size2 (), e2 ().size ());
            result_type t = result_type (0);
#ifndef BOOST_UBLAS_USE_DUFF_DEVICE
            for (size_type j = 0; j < size; ++ j)
                t += e1 () (i, j) * e2 () (j);
#else
            size_type j (0);
            DD (size, 4, r, (t += e1 () (i, j) * e2 () (j), ++ j));
#endif
            return t;
        }
        template<class E1, class E2>
        static BOOST_UBLAS_INLINE
        result_type apply (const matrix_expression<E1> &e1,
                                 const vector_expression<E2> &e2,
                                 size_type i) {
#ifdef BOOST_UBLAS_USE_SIMD
            typedef typename boost::mpl::if_<
                boost::mpl::and_<boost::is_same<typename E1::simd_category, concrete_tag>,
                                 boost::is_same<typename E2::simd_category, concrete_tag> >,
                    concrete_tag,
                    abstract_tag>::type simd_category;
#else
            typedef abstract_tag simd_category;
#endif
            return apply (e1, e2, i, simd_category ());
        }
        // Dense case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (difference_type size, I1 it1, I2 it2) {
            result_type t = result_type (0);
#ifndef BOOST_UBLAS_USE_DUFF_DEVICE
            while (-- size >= 0)
                t += *it1 * *it2, ++ it1, ++ it2;
#else
            DD (size, 4, r, (t += *it1 * *it2, ++ it1, ++ it2));
#endif
            return t;
        }
        // Packed case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (I1 it1, const I1 &it1_end, I2 it2, const I2 &it2_end) {
            result_type t = result_type (0);
            difference_type it1_size (it1_end - it1);
            difference_type it2_size (it2_end - it2);
            difference_type diff (0);
            if (it1_size > 0 && it2_size > 0)
                diff = it2.index () - it1.index2 ();
            if (diff != 0) {
                difference_type size = (std::min) (diff, it1_size);
                if (size > 0) {
                    it1 += size;
                    it1_size -= size;
                    diff -= size;
                }
                size = (std::min) (- diff, it2_size);
                if (size > 0) {
                    it2 += size;
                    it2_size -= size;
                    diff += size;
                }
            }
            difference_type size ((std::min) (it1_size, it2_size));
            while (-- size >= 0)
                t += *it1 * *it2, ++ it1, ++ it2;
            return t;
        }
        // Sparse case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (I1 it1, const I1 &it1_end, I2 it2, const I2 &it2_end,
                                 sparse_bidirectional_iterator_tag, sparse_bidirectional_iterator_tag) {
            result_type t = result_type (0);
            if (it1 != it1_end && it2 != it2_end) {
                size_type it1_index = it1.index2 (), it2_index = it2.index ();
                while (true) {
                    difference_type compare = it1_index - it2_index;
                    if (compare == 0) {
                        t += *it1 * *it2, ++ it1, ++ it2;
                        if (it1 != it1_end && it2 != it2_end) {
                            it1_index = it1.index2 ();
                            it2_index = it2.index ();
                        } else
                            break;
                    } else if (compare < 0) {
                        increment (it1, it1_end, - compare);
                        if (it1 != it1_end)
                            it1_index = it1.index2 ();
                        else
                            break;
                    } else if (compare > 0) {
                        increment (it2, it2_end, compare);
                        if (it2 != it2_end)
                            it2_index = it2.index ();
                        else
                            break;
                    }
                }
            }
            return t;
        }
        // Sparse packed case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (I1 it1, const I1 &it1_end, I2 it2, const I2 &/* it2_end */,
                                 sparse_bidirectional_iterator_tag, packed_random_access_iterator_tag) {
            result_type t = result_type (0);
            while (it1 != it1_end) {
                t += *it1 * it2 () (it1.index2 ());
                ++ it1;
            }
            return t;
        }
        // Packed sparse case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (I1 it1, const I1 &/* it1_end */, I2 it2, const I2 &it2_end,
                                 packed_random_access_iterator_tag, sparse_bidirectional_iterator_tag) {
            result_type t = result_type (0);
            while (it2 != it2_end) {
                t += it1 () (it1.index1 (), it2.index ()) * *it2;
                ++ it2;
            }
            return t;
        }
        // Another dispatcher
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (I1 it1, const I1 &it1_end, I2 it2, const I2 &it2_end,
                                 sparse_bidirectional_iterator_tag) {
            typedef typename I1::iterator_category iterator1_category;
            typedef typename I2::iterator_category iterator2_category;
            return apply (it1, it1_end, it2, it2_end, iterator1_category (), iterator2_category ());
        }
    };

    template<class T1, class T2, class TR>
    struct matrix_vector_prod2:
        public matrix_vector_binary_functor<T1, T2, TR> {
        typedef typename matrix_vector_binary_functor<T1, T2, TR>::size_type size_type;
        typedef typename matrix_vector_binary_functor<T1, T2, TR>::difference_type difference_type;
        typedef typename matrix_vector_binary_functor<T1, T2, TR>::value_type value_type;
        typedef typename matrix_vector_binary_functor<T1, T2, TR>::result_type result_type;

        template<class E1, class E2>
        static BOOST_UBLAS_INLINE
        result_type apply (const vector_expression<E1> &e1,
                                 const matrix_expression<E2> &e2,
                                 size_type i, concrete_tag) {
#ifndef BOOST_UBLAS_HAVE_BINDINGS
            using namespace raw;
            size_type size = BOOST_UBLAS_SAME (e1 ().size (), e2 ().size1 ());
            const T1 *data1 = data_const (e1 ());
            const T2 *data2 = data_const (e2 ()) + i * stride2 (e2 ());
            size_type s1 = stride (e1 ());
            size_type s2 = stride1 (e2 ());
            result_type t = result_type (0);
            if (s1 == 1 && s2 == 1) {
                for (size_type j = 0; j < size; ++ j)
                    t += data1 [j] * data2 [j];
            } else if (s2 == 1) {
                for (size_type j = 0, j1 = 0; j < size; ++ j, j1 += s1)
                    t += data1 [j1] * data2 [j];
            } else if (s1 == 1) {
                for (size_type j = 0, j2 = 0; j < size; ++ j, j2 += s2)
                    t += data1 [j] * data2 [j2];
            } else {
                for (size_type j = 0, j1 = 0, j2 = 0; j < size; ++ j, j1 += s1, j2 += s2)
                    t += data1 [j1] * data2 [j2];
            }
            return t;
#else
            return boost::numeric::bindings::atlas::dot (e1 (), e2 ().column (i));
#endif
        }
        template<class E1, class E2>
        static BOOST_UBLAS_INLINE
        result_type apply (const vector_expression<E1> &e1,
                                 const matrix_expression<E2> &e2,
                                 size_type i, abstract_tag) {
            size_type size = BOOST_UBLAS_SAME (e1 ().size (), e2 ().size1 ());
            result_type t = result_type (0);
#ifndef BOOST_UBLAS_USE_DUFF_DEVICE
            for (size_type j = 0; j < size; ++ j)
                t += e1 () (j) * e2 () (j, i);
#else
            size_type j (0);
            DD (size, 4, r, (t += e1 () (j) * e2 () (j, i), ++ j));
#endif
            return t;
        }
        template<class E1, class E2>
        static BOOST_UBLAS_INLINE
        result_type apply (const vector_expression<E1> &e1,
                                 const matrix_expression<E2> &e2,
                                 size_type i) {
#ifdef BOOST_UBLAS_USE_SIMD
            typedef typename boost::mpl::if_<
                boost::mpl::and_<boost::is_same<typename E1::simd_category, concrete_tag>,
                                 boost::is_same<typename E2::simd_category, concrete_tag> >,
                    concrete_tag,
                    abstract_tag>::type simd_category;
#else
            typedef abstract_tag simd_category;
#endif
            return apply (e1, e2, i, simd_category ());
        }
        // Dense case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (difference_type size, I1 it1, I2 it2) {
            result_type t = result_type (0);
#ifndef BOOST_UBLAS_USE_DUFF_DEVICE
            while (-- size >= 0)
                t += *it1 * *it2, ++ it1, ++ it2;
#else
            DD (size, 4, r, (t += *it1 * *it2, ++ it1, ++ it2));
#endif
            return t;
        }
        // Packed case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (I1 it1, const I1 &it1_end, I2 it2, const I2 &it2_end) {
            result_type t = result_type (0);
            difference_type it1_size (it1_end - it1);
            difference_type it2_size (it2_end - it2);
            difference_type diff (0);
            if (it1_size > 0 && it2_size > 0)
                diff = it2.index1 () - it1.index ();
            if (diff != 0) {
                difference_type size = (std::min) (diff, it1_size);
                if (size > 0) {
                    it1 += size;
                    it1_size -= size;
                    diff -= size;
                }
                size = (std::min) (- diff, it2_size);
                if (size > 0) {
                    it2 += size;
                    it2_size -= size;
                    diff += size;
                }
            }
            difference_type size ((std::min) (it1_size, it2_size));
            while (-- size >= 0)
                t += *it1 * *it2, ++ it1, ++ it2;
            return t;
        }
        // Sparse case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (I1 it1, const I1 &it1_end, I2 it2, const I2 &it2_end,
                                 sparse_bidirectional_iterator_tag, sparse_bidirectional_iterator_tag) {
            result_type t = result_type (0);
            if (it1 != it1_end && it2 != it2_end) {
                size_type it1_index = it1.index (), it2_index = it2.index1 ();
                while (true) {
                    difference_type compare = it1_index - it2_index;
                    if (compare == 0) {
                        t += *it1 * *it2, ++ it1, ++ it2;
                        if (it1 != it1_end && it2 != it2_end) {
                            it1_index = it1.index ();
                            it2_index = it2.index1 ();
                        } else
                            break;
                    } else if (compare < 0) {
                        increment (it1, it1_end, - compare);
                        if (it1 != it1_end)
                            it1_index = it1.index ();
                        else
                            break;
                    } else if (compare > 0) {
                        increment (it2, it2_end, compare);
                        if (it2 != it2_end)
                            it2_index = it2.index1 ();
                        else
                            break;
                    }
                }
            }
            return t;
        }
        // Packed sparse case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (I1 it1, const I1 &/* it1_end */, I2 it2, const I2 &it2_end,
                                 packed_random_access_iterator_tag, sparse_bidirectional_iterator_tag) {
            result_type t = result_type (0);
            while (it2 != it2_end) {
                t += it1 () (it2.index1 ()) * *it2;
                ++ it2;
            }
            return t;
        }
        // Sparse packed case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (I1 it1, const I1 &it1_end, I2 it2, const I2 &/* it2_end */,
                                 sparse_bidirectional_iterator_tag, packed_random_access_iterator_tag) {
            result_type t = result_type (0);
            while (it1 != it1_end) {
                t += *it1 * it2 () (it1.index (), it2.index2 ());
                ++ it1;
            }
            return t;
        }
        // Another dispatcher
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (I1 it1, const I1 &it1_end, I2 it2, const I2 &it2_end,
                                 sparse_bidirectional_iterator_tag) {
            typedef typename I1::iterator_category iterator1_category;
            typedef typename I2::iterator_category iterator2_category;
            return apply (it1, it1_end, it2, it2_end, iterator1_category (), iterator2_category ());
        }
    };

    // Binary returning matrix
    template<class T1, class T2, class TR>
    struct matrix_matrix_binary_functor {
        typedef std::size_t size_type;
        typedef std::ptrdiff_t difference_type;
        typedef TR value_type;
        typedef TR result_type;
    };

    template<class T1, class T2, class TR>
    struct matrix_matrix_prod:
        public matrix_matrix_binary_functor<T1, T2, TR> {
        typedef typename matrix_matrix_binary_functor<T1, T2, TR>::size_type size_type;
        typedef typename matrix_matrix_binary_functor<T1, T2, TR>::difference_type difference_type;
        typedef typename matrix_matrix_binary_functor<T1, T2, TR>::value_type value_type;
        typedef typename matrix_matrix_binary_functor<T1, T2, TR>::result_type result_type;

        template<class E1, class E2>
        static BOOST_UBLAS_INLINE
        result_type apply (const matrix_expression<E1> &e1,
                                 const matrix_expression<E2> &e2,
                                 size_type i, size_type j, concrete_tag) {
#ifndef BOOST_UBLAS_HAVE_BINDINGS
            using namespace raw;
            size_type size = BOOST_UBLAS_SAME (e1 ().size2 (), e2 ().size1 ());
            const T1 *data1 = data_const (e1 ()) + i * stride1 (e1 ());
            const T2 *data2 = data_const (e2 ()) + j * stride2 (e2 ());
            size_type s1 = stride2 (e1 ());
            size_type s2 = stride1 (e2 ());
            result_type t = result_type (0);
            if (s1 == 1 && s2 == 1) {
                for (size_type k = 0; k < size; ++ k)
                    t += data1 [k] * data2 [k];
            } else if (s2 == 1) {
                for (size_type k = 0, k1 = 0; k < size; ++ k, k1 += s1)
                    t += data1 [k1] * data2 [k];
            } else if (s1 == 1) {
                for (size_type k = 0, k2 = 0; k < size; ++ k, k2 += s2)
                    t += data1 [k] * data2 [k2];
            } else {
                for (size_type k = 0, k1 = 0, k2 = 0; k < size; ++ k, k1 += s1, k2 += s2)
                    t += data1 [k1] * data2 [k2];
            }
            return t;
#else
            return boost::numeric::bindings::atlas::dot (e1 ().row (i), e2 ().column (j));
#endif
        }
        template<class E1, class E2>
        static BOOST_UBLAS_INLINE
        result_type apply (const matrix_expression<E1> &e1,
                                 const matrix_expression<E2> &e2,
                                 size_type i, size_type j, abstract_tag) {
            size_type size = BOOST_UBLAS_SAME (e1 ().size2 (), e2 ().size1 ());
            result_type t = result_type (0);
#ifndef BOOST_UBLAS_USE_DUFF_DEVICE
            for (size_type k = 0; k < size; ++ k)
                t += e1 () (i, k) * e2 () (k, j);
#else
            size_type k (0);
            DD (size, 4, r, (t += e1 () (i, k) * e2 () (k, j), ++ k));
#endif
            return t;
        }
        template<class E1, class E2>
        static BOOST_UBLAS_INLINE
        result_type apply (const matrix_expression<E1> &e1,
                                 const matrix_expression<E2> &e2,
                                 size_type i, size_type j) {
#ifdef BOOST_UBLAS_USE_SIMD
            typedef typename boost::mpl::if_<
                boost::mpl::and_<boost::is_same<typename E1::simd_category, concrete_tag>,
                                 boost::is_same<typename E2::simd_category, concrete_tag> >,
                    concrete_tag,
                    abstract_tag>::type simd_category;
#else
            typedef abstract_tag simd_category;
#endif
            return apply (e1, e2, i, j, simd_category ());
        }
        // Dense case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (difference_type size, I1 it1, I2 it2) {
            result_type t = result_type (0);
#ifndef BOOST_UBLAS_USE_DUFF_DEVICE
            while (-- size >= 0)
                t += *it1 * *it2, ++ it1, ++ it2;
#else
            DD (size, 4, r, (t += *it1 * *it2, ++ it1, ++ it2));
#endif
            return t;
        }
        // Packed case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (I1 it1, const I1 &it1_end, I2 it2, const I2 &it2_end, packed_random_access_iterator_tag) {
            result_type t = result_type (0);
            difference_type it1_size (it1_end - it1);
            difference_type it2_size (it2_end - it2);
            difference_type diff (0);
            if (it1_size > 0 && it2_size > 0)
                diff = it2.index1 () - it1.index2 ();
            if (diff != 0) {
                difference_type size = (std::min) (diff, it1_size);
                if (size > 0) {
                    it1 += size;
                    it1_size -= size;
                    diff -= size;
                }
                size = (std::min) (- diff, it2_size);
                if (size > 0) {
                    it2 += size;
                    it2_size -= size;
                    diff += size;
                }
            }
            difference_type size ((std::min) (it1_size, it2_size));
            while (-- size >= 0)
                t += *it1 * *it2, ++ it1, ++ it2;
            return t;
        }
        // Sparse case
        template<class I1, class I2>
        static BOOST_UBLAS_INLINE
        result_type apply (I1 it1, const I1 &it1_end, I2 it2, const I2 &it2_end, sparse_bidirectional_iterator_tag) {
            result_type t = result_type (0);
            if (it1 != it1_end && it2 != it2_end) {
                size_type it1_index = it1.index2 (), it2_index = it2.index1 ();
                while (true) {
                    difference_type compare = it1_index - it2_index;
                    if (compare == 0) {
                        t += *it1 * *it2, ++ it1, ++ it2;
                        if (it1 != it1_end && it2 != it2_end) {
                            it1_index = it1.index2 ();
                            it2_index = it2.index1 ();
                        } else
                            break;
                    } else if (compare < 0) {
                        increment (it1, it1_end, - compare);
                        if (it1 != it1_end)
                            it1_index = it1.index2 ();
                        else
                            break;
                    } else if (compare > 0) {
                        increment (it2, it2_end, compare);
                        if (it2 != it2_end)
                            it2_index = it2.index1 ();
                        else
                            break;
                    }
                }
            }
            return t;
        }
    };

    // Unary returning scalar norm
    template<class T>
    struct matrix_scalar_real_unary_functor {
        typedef std::size_t size_type;
        typedef std::ptrdiff_t difference_type;
        typedef T value_type;
        typedef typename type_traits<T>::real_type real_type;
        typedef real_type result_type;
    };

    template<class T>
    struct matrix_norm_1:
        public matrix_scalar_real_unary_functor<T> {
        typedef typename matrix_scalar_real_unary_functor<T>::size_type size_type;
        typedef typename matrix_scalar_real_unary_functor<T>::difference_type difference_type;
        typedef typename matrix_scalar_real_unary_functor<T>::value_type value_type;
        typedef typename matrix_scalar_real_unary_functor<T>::real_type real_type;
        typedef typename matrix_scalar_real_unary_functor<T>::result_type result_type;

        template<class E>
        static BOOST_UBLAS_INLINE
        result_type apply (const matrix_expression<E> &e) {
            real_type t = real_type ();
            size_type size2 (e ().size2 ());
            for (size_type j = 0; j < size2; ++ j) {
                real_type u = real_type ();
                size_type size1 (e ().size1 ());
                for (size_type i = 0; i < size1; ++ i) {
                    real_type v (type_traits<value_type>::norm_1 (e () (i, j)));
                    u += v;
                }
                if (u > t)
                    t = u;
            }
            return t;
        }
    };
    template<class T>
    struct matrix_norm_frobenius:
        public matrix_scalar_real_unary_functor<T> {
        typedef typename matrix_scalar_real_unary_functor<T>::size_type size_type;
        typedef typename matrix_scalar_real_unary_functor<T>::difference_type difference_type;
        typedef typename matrix_scalar_real_unary_functor<T>::value_type value_type;
        typedef typename matrix_scalar_real_unary_functor<T>::real_type real_type;
        typedef typename matrix_scalar_real_unary_functor<T>::result_type result_type;

        template<class E>
        static BOOST_UBLAS_INLINE
        result_type apply (const matrix_expression<E> &e) {
            real_type t = real_type ();
            size_type size1 (e ().size1 ());
            for (size_type i = 0; i < size1; ++ i) {
                size_type size2 (e ().size2 ());
                for (size_type j = 0; j < size2; ++ j) {
                    real_type u (type_traits<value_type>::norm_2 (e () (i, j)));
                    t +=  u * u;
                }
            }
            return type_traits<real_type>::sqrt (t);
        }
    };
    template<class T>
    struct matrix_norm_inf:
        public matrix_scalar_real_unary_functor<T> {
        typedef typename matrix_scalar_real_unary_functor<T>::size_type size_type;
        typedef typename matrix_scalar_real_unary_functor<T>::difference_type difference_type;
        typedef typename matrix_scalar_real_unary_functor<T>::value_type value_type;
        typedef typename matrix_scalar_real_unary_functor<T>::real_type real_type;
        typedef typename matrix_scalar_real_unary_functor<T>::result_type result_type;

        template<class E>
        static BOOST_UBLAS_INLINE
        result_type apply (const matrix_expression<E> &e) {
            real_type t = real_type ();
            size_type size1 (e ().size1 ());
            for (size_type i = 0; i < size1; ++ i) {
                real_type u = real_type ();
                size_type size2 (e ().size2 ());
                for (size_type j = 0; j < size2; ++ j) {
                    real_type v (type_traits<value_type>::norm_inf (e () (i, j)));
                    u += v;
                }
                if (u > t)
                    t = u;
            }
            return t;
        }
    };

    // This functor computes the address translation
    // matrix [i] [j] -> storage [i * size2 + j]
    struct row_major {
        typedef std::size_t size_type;
        typedef std::ptrdiff_t difference_type;
        typedef row_major_tag orientation_category;

        static
        BOOST_UBLAS_INLINE
        size_type storage_size (size_type size1, size_type size2) {
            return size1 * size2;
        }

        // Indexing
        static
        BOOST_UBLAS_INLINE
        size_type element (size_type i, size_type size1, size_type j, size_type size2) {
            // Guarding against overflow.
            BOOST_UBLAS_CHECK ((size1 * size2) / size1 == size2, bad_size ());
            BOOST_UBLAS_CHECK (i < size1, bad_index ());
            BOOST_UBLAS_CHECK (j < size2, bad_index ());
            return i * size2 + j;
        }
        static
        BOOST_UBLAS_INLINE
        size_type address (size_type i, size_type size1, size_type j, size_type size2) {
            // Guarding against overflow.
            BOOST_UBLAS_CHECK (size1 == 0 || (size1 * size2) / size1 == size2, bad_size ());
            BOOST_UBLAS_CHECK (i <= size1, bad_index ());
            BOOST_UBLAS_CHECK (j <= size2, bad_index ());
            return i * size2 + j;
        }
        static
        BOOST_UBLAS_INLINE
        difference_type distance1 (difference_type k, size_type /* size1 */, size_type size2) {
            return size2 != 0 ? k / size2 : 0;
        }
        static
        BOOST_UBLAS_INLINE
        difference_type distance2 (difference_type k, size_type /* size1 */, size_type /* size2 */) {
            return k;
        }
        static
        BOOST_UBLAS_INLINE
        size_type index1 (difference_type k, size_type /* size1 */, size_type size2) {
            return size2 != 0 ? k / size2 : 0;
        }
        static
        BOOST_UBLAS_INLINE
        size_type index2 (difference_type k, size_type /* size1 */, size_type size2) {
            return size2 != 0 ? k % size2 : 0;
        }
        static
        BOOST_UBLAS_INLINE
        bool fast1 () {
            return false;
        }
        static
        BOOST_UBLAS_INLINE
        size_type one1 (size_type /* size1 */, size_type size2) {
            return size2;
        }
        static
        BOOST_UBLAS_INLINE
        bool fast2 () {
            return true;
        }
        static
        BOOST_UBLAS_INLINE
        size_type one2 (size_type /* size1 */, size_type /* size2 */) {
            return 1;
        }

        static
        BOOST_UBLAS_INLINE
        size_type lower_element (size_type i, size_type size1, size_type j, size_type size2) {
            BOOST_UBLAS_CHECK (i < size1, bad_index ());
            BOOST_UBLAS_CHECK (j < size2, bad_index ());
            BOOST_UBLAS_CHECK (i >= j, bad_index ());
            // sigma_i (i + 1) = (i + 1) * i / 2
            // i = 0 1 2 3, sigma = 0 1 3 6
            return ((i + 1) * i) / 2 + j;
        }
        static
        BOOST_UBLAS_INLINE
        size_type upper_element (size_type i, size_type size1, size_type j, size_type size2) {
            BOOST_UBLAS_CHECK (i < size1, bad_index ());
            BOOST_UBLAS_CHECK (j < size2, bad_index ());
            BOOST_UBLAS_CHECK (i <= j, bad_index ());
            // sigma_i (size - i) = size * i - i * (i - 1) / 2
            // i = 0 1 2 3, sigma = 0 4 7 9
            return (i * (2 * (std::max) (size1, size2) - i + 1)) / 2 + j - i;
        }

        static
        BOOST_UBLAS_INLINE
        size_type element1 (size_type i, size_type size1, size_type /* j */, size_type /* size2 */) {
            BOOST_UBLAS_CHECK (i < size1, bad_index ());
            return i;
        }
        static
        BOOST_UBLAS_INLINE
        size_type element2 (size_type /* i */, size_type /* size1 */, size_type j, size_type size2) {
            BOOST_UBLAS_CHECK (j < size2, bad_index ());
            return j;
        }
        static
        BOOST_UBLAS_INLINE
        size_type address1 (size_type i, size_type size1, size_type /* j */, size_type /* size2 */) {
            BOOST_UBLAS_CHECK (i <= size1, bad_index ());
            return i;
        }
        static
        BOOST_UBLAS_INLINE
        size_type address2 (size_type /* i */, size_type /* size1 */, size_type j, size_type size2) {
            BOOST_UBLAS_CHECK (j <= size2, bad_index ());
            return j;
        }
        static
        BOOST_UBLAS_INLINE
        size_type index1 (size_type index1, size_type /* index2 */) {
            return index1;
        }
        static
        BOOST_UBLAS_INLINE
        size_type index2 (size_type /* index1 */, size_type index2) {
            return index2;
        }
        static
        BOOST_UBLAS_INLINE
        size_type size1 (size_type size1, size_type /* size2 */) {
            return size1;
        }
        static
        BOOST_UBLAS_INLINE
        size_type size2 (size_type /* size1 */, size_type size2) {
            return size2;
        }

        // Iterating
        template<class I>
        static
        BOOST_UBLAS_INLINE
        void increment1 (I &it, size_type /* size1 */, size_type size2) {
            it += size2;
        }
        template<class I>
        static
        BOOST_UBLAS_INLINE
        void decrement1 (I &it, size_type /* size1 */, size_type size2) {
            it -= size2;
        }
        template<class I>
        static
        BOOST_UBLAS_INLINE
        void increment2 (I &it, size_type /* size1 */, size_type /* size2 */) {
            ++ it;
        }
        template<class I>
        static
        BOOST_UBLAS_INLINE
        void decrement2 (I &it, size_type /* size1 */, size_type /* size2 */) {
            -- it;
        }
    };

    // This functor computes the address translation
    // matrix [i] [j] -> storage [i + j * size1]
    struct column_major {
        typedef std::size_t size_type;
        typedef std::ptrdiff_t difference_type;
        typedef column_major_tag orientation_category;

        static
        BOOST_UBLAS_INLINE
        size_type storage_size (size_type size1, size_type size2) {
            return size1 * size2;
        }

        // Indexing
        static
        BOOST_UBLAS_INLINE
        size_type element (size_type i, size_type size1, size_type j, size_type size2) {
            // Guarding against overflow.
            BOOST_UBLAS_CHECK ((size1 * size2) / size1 == size2, bad_size ());
            BOOST_UBLAS_CHECK (i < size1, bad_index ());
            BOOST_UBLAS_CHECK (j < size2, bad_index ());
            return i + j * size1;
        }
        static
        BOOST_UBLAS_INLINE
        size_type address (size_type i, size_type size1, size_type j, size_type size2) {
            // Guarding against overflow.
            BOOST_UBLAS_CHECK (size1 == 0 || (size1 * size2) / size1 == size2, bad_size ());
            BOOST_UBLAS_CHECK (i <= size1, bad_index ());
            BOOST_UBLAS_CHECK (j <= size2, bad_index ());
            return i + j * size1;
        }
        static
        BOOST_UBLAS_INLINE
        difference_type distance1 (difference_type k, size_type /* size1 */, size_type /* size2 */) {
            return k;
        }
        static
        BOOST_UBLAS_INLINE
        difference_type distance2 (difference_type k, size_type size1, size_type /* size2 */) {
            return size1 != 0 ? k / size1 : 0;
        }
        static
        BOOST_UBLAS_INLINE
        size_type index1 (difference_type k, size_type size1, size_type /* size2 */) {
            return size1 != 0 ? k % size1 : 0;
        }
        static
        BOOST_UBLAS_INLINE
        size_type index2 (difference_type k, size_type size1, size_type /* size2 */) {
            return size1 != 0 ? k / size1 : 0;
        }
        static
        BOOST_UBLAS_INLINE
        bool fast1 () {
            return true;
        }
        static
        BOOST_UBLAS_INLINE
        size_type one1 (size_type /* size1 */, size_type /* size2 */) {
            return 1;
        }
        static
        BOOST_UBLAS_INLINE
        bool fast2 () {
            return false;
        }
        static
        BOOST_UBLAS_INLINE
        size_type one2 (size_type size1, size_type /* size2 */) {
            return size1;
        }

        static
        BOOST_UBLAS_INLINE
        size_type lower_element (size_type i, size_type size1, size_type j, size_type size2) {
            BOOST_UBLAS_CHECK (i < size1, bad_index ());
            BOOST_UBLAS_CHECK (j < size2, bad_index ());
            BOOST_UBLAS_CHECK (i >= j, bad_index ());
            // sigma_j (size - j) = size * j - j * (j - 1) / 2
            // j = 0 1 2 3, sigma = 0 4 7 9
            return i - j + (j * (2 * (std::max) (size1, size2) - j + 1)) / 2;
        }
        static
        BOOST_UBLAS_INLINE
        size_type upper_element (size_type i, size_type size1, size_type j, size_type size2) {
            BOOST_UBLAS_CHECK (i < size1, bad_index ());
            BOOST_UBLAS_CHECK (j < size2, bad_index ());
            BOOST_UBLAS_CHECK (i <= j, bad_index ());
            // sigma_j (j + 1) = (j + 1) * j / 2
            // j = 0 1 2 3, sigma = 0 1 3 6
            return i + ((j + 1) * j) / 2;
        }

        static
        BOOST_UBLAS_INLINE
        size_type element1 (size_type /* i */, size_type /* size1 */, size_type j, size_type size2) {
            BOOST_UBLAS_CHECK (j < size2, bad_index ());
            return j;
        }
        static
        BOOST_UBLAS_INLINE
        size_type element2 (size_type i, size_type size1, size_type /* j */, size_type /* size2 */) {
            BOOST_UBLAS_CHECK (i < size1, bad_index ());
            return i;
        }
        static
        BOOST_UBLAS_INLINE
        size_type address1 (size_type /* i */, size_type /* size1 */, size_type j, size_type size2) {
            BOOST_UBLAS_CHECK (j <= size2, bad_index ());
            return j;
        }
        static
        BOOST_UBLAS_INLINE
        size_type address2 (size_type i, size_type size1, size_type /* j */, size_type /* size2 */) {
            BOOST_UBLAS_CHECK (i <= size1, bad_index ());
            return i;
        }
        static
        BOOST_UBLAS_INLINE
        size_type index1 (size_type /* index1 */, size_type index2) {
            return index2;
        }
        static
        BOOST_UBLAS_INLINE
        size_type index2 (size_type index1, size_type /* index2 */) {
            return index1;
        }
        static
        BOOST_UBLAS_INLINE
        size_type size1 (size_type /* size1 */, size_type size2) {
            return size2;
        }
        static
        BOOST_UBLAS_INLINE
        size_type size2 (size_type size1, size_type /* size2 */) {
            return size1;
        }

        // Iterating
        template<class I>
        static
        BOOST_UBLAS_INLINE
        void increment1 (I &it, size_type /* size1 */, size_type /* size2 */) {
            ++ it;
        }
        template<class I>
        static
        BOOST_UBLAS_INLINE
        void decrement1 (I &it, size_type /* size1 */, size_type /* size2 */) {
            -- it;
        }
        template<class I>
        static
        BOOST_UBLAS_INLINE
        void increment2 (I &it, size_type size1, size_type /* size2 */) {
            it += size1;
        }
        template<class I>
        static
        BOOST_UBLAS_INLINE
        void decrement2 (I &it, size_type size1, size_type /* size2 */) {
            it -= size1;
        }
    };

    struct full {
        typedef std::size_t size_type;

        static
        BOOST_UBLAS_INLINE
        size_type packed_size (size_type size1, size_type size2) {
            return size1 * size2;
        }

        static
        BOOST_UBLAS_INLINE
        bool zero (size_type /* i */, size_type /* j */) {
            return false;
        }
        static
        BOOST_UBLAS_INLINE
        bool one (size_type /* i */, size_type /* j */) {
            return false;
        }
        static
        BOOST_UBLAS_INLINE
        bool other (size_type /* i */, size_type /* j */) {
            return true;
        }
    };

    struct lower {
        typedef std::size_t size_type;
        typedef lower_tag packed_category;

        static
        BOOST_UBLAS_INLINE
        size_type packed_size (size_type size1, size_type size2) {
            size_type size = (std::max) (size1, size2);
            return ((size + 1) * size) / 2;
        }

        static
        BOOST_UBLAS_INLINE
        bool zero (size_type i, size_type j) {
            return j > i;
        }
        static
        BOOST_UBLAS_INLINE
        bool one (size_type /* i */, size_type /* j */) {
            return false;
        }
        static
        BOOST_UBLAS_INLINE
        bool other (size_type i, size_type j) {
            return j <= i;
        }
        template<class F>
        static
        BOOST_UBLAS_INLINE
        size_type element (F, size_type i, size_type size1, size_type j, size_type size2) {
            return F::lower_element (i, size1, j, size2);
        }

        static
        BOOST_UBLAS_INLINE
        size_type restrict1 (size_type i, size_type j) {
            return (std::max) (i, j);
        }
        static
        BOOST_UBLAS_INLINE
        size_type restrict2 (size_type i, size_type j) {
            return (std::min) (i + 1, j);
        }
        static
        BOOST_UBLAS_INLINE
        size_type mutable_restrict1 (size_type i, size_type j) {
            return (std::max) (i, j);
        }
        static
        BOOST_UBLAS_INLINE
        size_type mutable_restrict2 (size_type i, size_type j) {
            return (std::min) (i + 1, j);
        }
    };
    struct upper {
        typedef std::size_t size_type;
        typedef upper_tag packed_category;

        static
        BOOST_UBLAS_INLINE
        size_type packed_size (size_type size1, size_type size2) {
            size_type size = (std::max) (size1, size2);
            return ((size + 1) * size) / 2;
        }

        static
        BOOST_UBLAS_INLINE
        bool zero (size_type i, size_type j) {
            return j < i;
        }
        static
        BOOST_UBLAS_INLINE
        bool one (size_type /* i */, size_type /* j */) {
            return false;
        }
        static
        BOOST_UBLAS_INLINE
        bool other (size_type i, size_type j) {
            return j >= i;
        }
        template<class F>
        static
        BOOST_UBLAS_INLINE
        size_type element (F, size_type i, size_type size1, size_type j, size_type size2) {
            return F::upper_element (i, size1, j, size2);
        }

        static
        BOOST_UBLAS_INLINE
        size_type restrict1 (size_type i, size_type j) {
            return (std::min) (i, j + 1);
        }
        static
        BOOST_UBLAS_INLINE
        size_type restrict2 (size_type i, size_type j) {
            return (std::max) (i, j);
        }
        static
        BOOST_UBLAS_INLINE
        size_type mutable_restrict1 (size_type i, size_type j) {
            return (std::min) (i, j + 1);
        }
        static
        BOOST_UBLAS_INLINE
        size_type mutable_restrict2 (size_type i, size_type j) {
            return (std::max) (i, j);
        }
    };
    struct unit_lower {
        typedef std::size_t size_type;
        typedef unit_lower_tag packed_category;

        static
        BOOST_UBLAS_INLINE
        size_type packed_size (size_type size1, size_type size2) {
            size_type size = (std::max) (size1, size2);
            return ((size + 1) * size) / 2;
        }

        static
        BOOST_UBLAS_INLINE
        bool zero (size_type i, size_type j) {
            return j > i;
        }
        static
        BOOST_UBLAS_INLINE
        bool one (size_type i, size_type j) {
            return j == i;
        }
        static
        BOOST_UBLAS_INLINE
        bool other (size_type i, size_type j) {
            return j < i;
        }
        template<class F>
        static
        BOOST_UBLAS_INLINE
        size_type element (F, size_type i, size_type size1, size_type j, size_type size2) {
            return F::lower_element (i, size1, j, size2);
        }

        static
        BOOST_UBLAS_INLINE
        size_type restrict1 (size_type i, size_type j) {
            return (std::max) (i, j);
        }
        static
        BOOST_UBLAS_INLINE
        size_type restrict2 (size_type i, size_type j) {
            return (std::min) (i + 1, j);
        }
        static
        BOOST_UBLAS_INLINE
        size_type mutable_restrict1 (size_type i, size_type j) {
            return (std::max) (i, j);
        }
        static
        BOOST_UBLAS_INLINE
        size_type mutable_restrict2 (size_type i, size_type j) {
            return (std::min) (i, j);
        }
    };
    struct unit_upper {
        typedef std::size_t size_type;
        typedef unit_upper_tag packed_category;

        static
        BOOST_UBLAS_INLINE
        size_type packed_size (size_type size1, size_type size2) {
            size_type size = (std::max) (size1, size2);
            return ((size + 1) * size) / 2;
        }

        static
        BOOST_UBLAS_INLINE
        bool zero (size_type i, size_type j) {
            return j < i;
        }
        static
        BOOST_UBLAS_INLINE
        bool one (size_type i, size_type j) {
            return j == i;
        }
        static
        BOOST_UBLAS_INLINE
        bool other (size_type i, size_type j) {
            return j > i;
        }
        template<class F>
        static
        BOOST_UBLAS_INLINE
        size_type element (F, size_type i, size_type size1, size_type j, size_type size2) {
            return F::upper_element (i, size1, j, size2);
        }

        static
        BOOST_UBLAS_INLINE
        size_type restrict1 (size_type i, size_type j) {
            return (std::min) (i, j + 1);
        }
        static
        BOOST_UBLAS_INLINE
        size_type restrict2 (size_type i, size_type j) {
            return (std::max) (i, j);
        }
        static
        BOOST_UBLAS_INLINE
        size_type mutable_restrict1 (size_type i, size_type j) {
            return (std::min) (i, j);
        }
        static
        BOOST_UBLAS_INLINE
        size_type mutable_restrict2 (size_type i, size_type j) {
            return (std::max) (i, j);
        }
    };
    struct strict_lower {
        typedef std::size_t size_type;
        typedef unit_lower_tag packed_category;

        static
        BOOST_UBLAS_INLINE
        size_type packed_size (size_type size1, size_type size2) {
            size_type size = (std::max) (size1, size2);
            return ((size + 1) * size) / 2;
        }

        static
        BOOST_UBLAS_INLINE
        bool zero (size_type i, size_type j) {
            return j >= i;
        }
        static
        BOOST_UBLAS_INLINE
        bool one (size_type /*i*/, size_type /*j*/) {
            return false;
        }
        static
        BOOST_UBLAS_INLINE
        bool other (size_type i, size_type j) {
            return j < i;
        }
        template<class F>
        static
        BOOST_UBLAS_INLINE
        size_type element (F, size_type i, size_type size1, size_type j, size_type size2) {
            return F::lower_element (i, size1, j, size2);
        }

        static
        BOOST_UBLAS_INLINE
        size_type restrict1 (size_type i, size_type j) {
            return (std::max) (i, j);
        }
        static
        BOOST_UBLAS_INLINE
        size_type restrict2 (size_type i, size_type j) {
            return (std::min) (i, j);
        }
        static
        BOOST_UBLAS_INLINE
        size_type mutable_restrict1 (size_type i, size_type j) {
            return (std::max) (i, j);
        }
        static
        BOOST_UBLAS_INLINE
        size_type mutable_restrict2 (size_type i, size_type j) {
            return (std::min) (i, j);
        }
    };
    struct strict_upper {
        typedef std::size_t size_type;
        typedef unit_upper_tag packed_category;

        static
        BOOST_UBLAS_INLINE
        size_type packed_size (size_type size1, size_type size2) {
            size_type size = (std::max) (size1, size2);
            return ((size + 1) * size) / 2;
        }

        static
        BOOST_UBLAS_INLINE
        bool zero (size_type i, size_type j) {
            return j <= i;
        }
        static
        BOOST_UBLAS_INLINE
        bool one (size_type /*i*/, size_type /*j*/) {
            return false;
        }
        static
        BOOST_UBLAS_INLINE
        bool other (size_type i, size_type j) {
            return j > i;
        }
        template<class F>
        static
        BOOST_UBLAS_INLINE
        size_type element (F, size_type i, size_type size1, size_type j, size_type size2) {
            return F::upper_element (i, size1, j, size2);
        }

        static
        BOOST_UBLAS_INLINE
        size_type restrict1 (size_type i, size_type j) {
            return (std::min) (i, j);
        }
        static
        BOOST_UBLAS_INLINE
        size_type restrict2 (size_type i, size_type j) {
            return (std::max) (i, j);
        }
        static
        BOOST_UBLAS_INLINE
        size_type mutable_restrict1 (size_type i, size_type j) {
            return (std::min) (i, j);
        }
        static
        BOOST_UBLAS_INLINE
        size_type mutable_restrict2 (size_type i, size_type j) {
            return (std::max) (i, j);
        }
    };

}}}

#endif


/****************************************************************************
 * Copyright (c) 2012-2020 by the ArborX authors                            *
 * All rights reserved.                                                     *
 *                                                                          *
 * This file is part of the ArborX library. ArborX is                       *
 * distributed under a BSD 3-clause license. For the licensing terms see    *
 * the LICENSE file in the top-level directory.                             *
 *                                                                          *
 * SPDX-License-Identifier: BSD-3-Clause                                    *
 ****************************************************************************/

#include <ArborX_BoostRTreeHelpers.hpp>
#include <ArborX_LinearBVH.hpp>
#include <ArborX_Version.hpp>

#include <Kokkos_Core.hpp>

#include <boost/program_options.hpp>

#include <chrono>
#include <cmath> // cbrt
#include <cstdlib>
#include <random>

#ifdef ARBORX_PERFORMANCE_TESTING
#include <mpi.h>
#endif

#include <benchmark/benchmark.h>
#include <point_clouds.hpp>

template <typename DeviceType>
Kokkos::View<ArborX::Point *, DeviceType>
constructPoints(int n_values, PointCloudType point_cloud_type)
{
  Kokkos::View<ArborX::Point *, DeviceType> random_points(
      Kokkos::ViewAllocateWithoutInitializing("random_points"), n_values);
  // Generate random points uniformly distributed within a box.  The edge
  // length of the box chosen such that object density (here objects will be
  // boxes 2x2x2 centered around a random point) will remain constant as
  // problem size is changed.
  auto const a = std::cbrt(n_values);
  generatePointCloud(point_cloud_type, a, random_points);

  return random_points;
}

template <typename DeviceType>
Kokkos::View<ArborX::Nearest<ArborX::Point> *, DeviceType>
makeNearestQueries(int n_values, int n_queries, int n_neighbors,
                   PointCloudType target_point_cloud_type)
{
  Kokkos::View<ArborX::Point *, DeviceType> random_points(
      Kokkos::ViewAllocateWithoutInitializing("random_points"), n_queries);
  auto const a = std::cbrt(n_values);
  generatePointCloud(target_point_cloud_type, a, random_points);

  Kokkos::View<ArborX::Nearest<ArborX::Point> *, DeviceType> queries(
      Kokkos::ViewAllocateWithoutInitializing("queries"), n_queries);
  using ExecutionSpace = typename DeviceType::execution_space;
  Kokkos::parallel_for(
      "bvh_driver:setup_knn_search_queries",
      Kokkos::RangePolicy<ExecutionSpace>(0, n_queries), KOKKOS_LAMBDA(int i) {
        queries(i) =
            ArborX::nearest<ArborX::Point>(random_points(i), n_neighbors);
      });
  return queries;
}

template <typename DeviceType>
Kokkos::View<decltype(ArborX::intersects(ArborX::Sphere{})) *, DeviceType>
makeSpatialQueries(int n_values, int n_queries, int n_neighbors,
                   PointCloudType target_point_cloud_type)
{
  Kokkos::View<ArborX::Point *, DeviceType> random_points(
      Kokkos::ViewAllocateWithoutInitializing("random_points"), n_queries);
  auto const a = std::cbrt(n_values);
  generatePointCloud(target_point_cloud_type, a, random_points);

  Kokkos::View<decltype(ArborX::intersects(ArborX::Sphere{})) *, DeviceType>
      queries(Kokkos::ViewAllocateWithoutInitializing("queries"), n_queries);
  // Radius is computed so that the number of results per query for a uniformly
  // distributed points in a [-a,a]^3 box is approximately n_neighbors.
  // Calculation: n_values*(4/3*M_PI*r^3)/(2a)^3 = n_neighbors
  double const r = std::cbrt(static_cast<double>(n_neighbors) * 6. / M_PI);
  using ExecutionSpace = typename DeviceType::execution_space;
  Kokkos::parallel_for(
      "bvh_driver:setup_radius_search_queries",
      Kokkos::RangePolicy<ExecutionSpace>(0, n_queries), KOKKOS_LAMBDA(int i) {
        queries(i) = ArborX::intersects(ArborX::Sphere{random_points(i), r});
      });
  return queries;
}

template <class TreeType>
void BM_construction(benchmark::State &state)
{
  using DeviceType = typename TreeType::device_type;
  int const n_values = state.range(0);
  auto const point_cloud_type = static_cast<PointCloudType>(state.range(1));
  auto const points = constructPoints<DeviceType>(n_values, point_cloud_type);

  for (auto _ : state)
  {
    auto const start = std::chrono::high_resolution_clock::now();
    TreeType index(points);
    auto const end = std::chrono::high_resolution_clock::now();
    std::chrono::duration<double> elapsed_seconds = end - start;
    state.SetIterationTime(elapsed_seconds.count());
  }
}

template <class TreeType>
void BM_knn_search(benchmark::State &state)
{
  using DeviceType = typename TreeType::device_type;
  int const n_values = state.range(0);
  int const n_queries = state.range(1);
  int const n_neighbors = state.range(2);
  bool const sort_predicates_int = state.range(3);
  auto const source_point_cloud_type =
      static_cast<PointCloudType>(state.range(4));
  auto const target_point_cloud_type =
      static_cast<PointCloudType>(state.range(5));

  TreeType index(
      constructPoints<DeviceType>(n_values, source_point_cloud_type));
  auto const queries = makeNearestQueries<DeviceType>(
      n_values, n_queries, n_neighbors, target_point_cloud_type);

  for (auto _ : state)
  {
    Kokkos::View<int *, DeviceType> offset("offset", 0);
    Kokkos::View<int *, DeviceType> indices("indices", 0);
    auto const start = std::chrono::high_resolution_clock::now();
    index.query(queries, indices, offset,
                ArborX::Experimental::TraversalPolicy().setPredicateSorting(
                    sort_predicates_int));
    auto const end = std::chrono::high_resolution_clock::now();
    std::chrono::duration<double> elapsed_seconds = end - start;
    state.SetIterationTime(elapsed_seconds.count());
  }
}

template <class TreeType>
void BM_radius_search(benchmark::State &state)
{
  using DeviceType = typename TreeType::device_type;
  int const n_values = state.range(0);
  int const n_queries = state.range(1);
  int const n_neighbors = state.range(2);
  int const sort_predicates_int = state.range(3);
  int const buffer_size = state.range(4);
  auto const source_point_cloud_type =
      static_cast<PointCloudType>(state.range(5));
  auto const target_point_cloud_type =
      static_cast<PointCloudType>(state.range(6));

  TreeType index(
      constructPoints<DeviceType>(n_values, source_point_cloud_type));
  auto const queries = makeSpatialQueries<DeviceType>(
      n_values, n_queries, n_neighbors, target_point_cloud_type);

  for (auto _ : state)
  {
    Kokkos::View<int *, DeviceType> offset("offset", 0);
    Kokkos::View<int *, DeviceType> indices("indices", 0);
    auto const start = std::chrono::high_resolution_clock::now();
    index.query(queries, indices, offset,
                ArborX::Experimental::TraversalPolicy()
                    .setPredicateSorting(sort_predicates_int)
                    .setBufferSize(buffer_size));
    auto const end = std::chrono::high_resolution_clock::now();
    std::chrono::duration<double> elapsed_seconds = end - start;
    state.SetIterationTime(elapsed_seconds.count());
  }
}

class KokkosScopeGuard
{
public:
  KokkosScopeGuard(int &argc, char *argv[]) { Kokkos::initialize(argc, argv); }
  ~KokkosScopeGuard() { Kokkos::finalize(); }
};

#ifndef ARBORX_PERFORMANCE_TESTING
#define REGISTER_BENCHMARK(TreeType, r1, r2, r3)                               \
  BENCHMARK_TEMPLATE(BM_construction, TreeType)                                \
      ->Args({n_values, source_point_cloud_type})                              \
      ->UseManualTime()                                                        \
      ->Unit(benchmark::kMicrosecond);                                         \
  BENCHMARK_TEMPLATE(BM_knn_search, TreeType)                                  \
      ->Args({n_values, n_queries, n_neighbors, sort_predicates_int,           \
              source_point_cloud_type, target_point_cloud_type})               \
      ->UseManualTime()                                                        \
      ->Unit(benchmark::kMicrosecond);                                         \
  BENCHMARK_TEMPLATE(BM_radius_search, TreeType)                               \
      ->Args({n_values, n_queries, n_neighbors, sort_predicates_int,           \
              buffer_size, source_point_cloud_type, target_point_cloud_type})  \
      ->UseManualTime()                                                        \
      ->Unit(benchmark::kMicrosecond);

#else
#define REGISTER_BENCHMARK(TreeType, r1, r2, r3)                               \
  BENCHMARK_TEMPLATE(BM_construction, TreeType)                                \
      ->Args({(int)r1, 0})                                                     \
      ->Args({(int)r2, 0})                                                     \
      ->Args({(int)r3, 0})                                                     \
      ->Args({(int)r1, 1})                                                     \
      ->Args({(int)r2, 1})                                                     \
      ->Args({(int)r3, 1})                                                     \
      ->UseManualTime()                                                        \
      ->Unit(benchmark::kMicrosecond);                                         \
  BENCHMARK_TEMPLATE(BM_knn_search, TreeType)                                  \
      ->Args({(int)r1, (int)r1, 10, 1, 0, 2})                                  \
      ->Args({(int)r2, (int)r2, 10, 1, 0, 2})                                  \
      ->Args({(int)r3, (int)r3, 10, 1, 0, 2})                                  \
      ->Args({(int)r1, (int)r1, 10, 1, 1, 3})                                  \
      ->Args({(int)r2, (int)r2, 10, 1, 1, 3})                                  \
      ->Args({(int)r3, (int)r3, 10, 1, 1, 3})                                  \
      ->UseManualTime()                                                        \
      ->Unit(benchmark::kMicrosecond);                                         \
  BENCHMARK_TEMPLATE(BM_radius_search, TreeType)                               \
      ->Args({(int)r1, (int)r1, 10, 1, 0, 0, 2})                               \
      ->Args({(int)r2, (int)r2, 10, 1, 0, 0, 2})                               \
      ->Args({(int)r3, (int)r3, 10, 1, 0, 0, 2})                               \
      ->Args({(int)r1, (int)r1, 10, 1, 0, 1, 3})                               \
      ->Args({(int)r2, (int)r2, 10, 1, 0, 1, 3})                               \
      ->Args({(int)r3, (int)r3, 10, 1, 0, 1, 3})                               \
      ->UseManualTime()                                                        \
      ->Unit(benchmark::kMicrosecond);
#endif

// NOTE Motivation for this class that stores the argument count and values is
// I could not figure out how to make the parser consume arguments with
// Boost.Program_options
// Benchmark removes its own arguments from the command line arguments. This
// means, that by virtue of returning references to internal data members in
// argc() and argv() function, it will necessarily modify the members. It will
// decrease _argc, and "reduce" _argv data. Hence, we must keep a copy of _argv
// that is not modified from the outside to release memory in the destructor
// correctly.
class CmdLineArgs
{
private:
  int _argc;
  std::vector<char *> _argv;
  std::vector<char *> _owner_ptrs;

public:
  CmdLineArgs(std::vector<std::string> const &args, char const *exe)
      : _argc(args.size() + 1)
      , _owner_ptrs{new char[std::strlen(exe) + 1]}
  {
    std::strcpy(_owner_ptrs[0], exe);
    _owner_ptrs.reserve(_argc);
    for (auto const &s : args)
    {
      _owner_ptrs.push_back(new char[s.size() + 1]);
      std::strcpy(_owner_ptrs.back(), s.c_str());
    }
    _argv = _owner_ptrs;
  }

  ~CmdLineArgs()
  {
    for (auto *p : _owner_ptrs)
    {
      delete[] p;
    }
  }

  int &argc() { return _argc; }

  char **argv() { return _argv.data(); }
};

int main(int argc, char *argv[])
{
#ifdef ARBORX_PERFORMANCE_TESTING
  MPI_Init(&argc, &argv);
#endif
  Kokkos::initialize(argc, argv);

  namespace bpo = boost::program_options;
  bpo::options_description desc("Allowed options");
  int n_values;
  int n_queries;
  int n_neighbors;
  int buffer_size;
  bool sort_predicates;
  std::string source_pt_cloud;
  std::string target_pt_cloud;
  // clang-format off
    desc.add_options()
        ( "help", "produce help message" )
        ( "values", bpo::value<int>(&n_values)->default_value(50000), "number of indexable values (source)" )
        ( "queries", bpo::value<int>(&n_queries)->default_value(20000), "number of queries (target)" )
        ( "predicate-sort", bpo::value<bool>(&sort_predicates)->default_value(true), "sort predicates" )
        ( "neighbors", bpo::value<int>(&n_neighbors)->default_value(10), "desired number of results per query" )
        ( "buffer", bpo::value<int>(&buffer_size)->default_value(0), "size for buffer optimization in radius search" )
        ( "source-point-cloud-type", bpo::value<std::string>(&source_pt_cloud)->default_value("filled_box"), "shape of the source point cloud"  )
        ( "target-point-cloud-type", bpo::value<std::string>(&target_pt_cloud)->default_value("filled_box"), "shape of the target point cloud"  )
        ( "no-header", bpo::bool_switch(), "do not print version and hash" )
    ;
  // clang-format on
  bpo::variables_map vm;
  bpo::parsed_options parsed = bpo::command_line_parser(argc, argv)
                                   .options(desc)
                                   .allow_unregistered()
                                   .run();
  bpo::store(parsed, vm);
  CmdLineArgs pass_further{
      bpo::collect_unrecognized(parsed.options, bpo::include_positional),
      argv[0]};
  bpo::notify(vm);

  int sort_predicates_int = (sort_predicates ? 1 : 0);

  if (!vm["no-header"].as<bool>())
  {
    std::cout << "ArborX version: " << ArborX::version() << std::endl;
    std::cout << "ArborX hash   : " << ArborX::gitCommitHash() << std::endl;
  }

  if (vm.count("help") > 0)
  {
    // Full list of options consists of Kokkos + Boost.Program_options +
    // Google Benchmark and we still need to call benchmark::Initialize() to
    // get those printed to the standard output.
    std::cout << desc << "\n";
    int ac = 2;
    char *av[] = {(char *)"ignored", (char *)"--help"};
    // benchmark::Initialize() calls exit(0) when `--help` so register
    // Kokkos::finalize() to be called on normal program termination.
    std::atexit(Kokkos::finalize);
    benchmark::Initialize(&ac, av);
    return 1;
  }

  benchmark::Initialize(&pass_further.argc(), pass_further.argv());
  // Throw if some of the arguments have not been recognized.
  std::ignore =
      bpo::command_line_parser(pass_further.argc(), pass_further.argv())
          .options(bpo::options_description(""))
          .run();

  // Google benchmark only supports integer arguments (see
  // https://github.com/google/benchmark/issues/387), so we map the string to
  // an enum.
  std::map<std::string, PointCloudType> to_point_cloud_enum;
  to_point_cloud_enum["filled_box"] = PointCloudType::filled_box;
  to_point_cloud_enum["hollow_box"] = PointCloudType::hollow_box;
  to_point_cloud_enum["filled_sphere"] = PointCloudType::filled_sphere;
  to_point_cloud_enum["hollow_sphere"] = PointCloudType::hollow_sphere;
  int source_point_cloud_type = to_point_cloud_enum.at(source_pt_cloud);
  int target_point_cloud_type = to_point_cloud_enum.at(target_pt_cloud);

#ifdef KOKKOS_ENABLE_SERIAL
  using Serial = Kokkos::Serial::device_type;
  REGISTER_BENCHMARK(ArborX::BVH<Serial>, 1e3, 1e4, 1e5);
#endif

#ifdef KOKKOS_ENABLE_OPENMP
  using OpenMP = Kokkos::OpenMP::device_type;
  REGISTER_BENCHMARK(ArborX::BVH<OpenMP>, 1e3, 1e4, 1e5);
#endif

#ifdef KOKKOS_ENABLE_THREADS
  using Threads = Kokkos::Threads::device_type;
  REGISTER_BENCHMARK(ArborX::BVH<Threads>, 1e3, 1e4, 1e5);
#endif

#ifdef KOKKOS_ENABLE_CUDA
  using Cuda = Kokkos::Cuda::device_type;
  REGISTER_BENCHMARK(ArborX::BVH<Cuda>, 1e4, 1e5, 1e6);
#endif

#ifndef ARBORX_PERFORMANCE_TESTING
  using BoostRTree = BoostExt::RTree<ArborX::Point>;
  REGISTER_BENCHMARK(BoostRTree, 1e3, 1e4, 1e5);
#endif

  benchmark::RunSpecifiedBenchmarks();

  Kokkos::finalize();
#ifdef ARBORX_PERFORMANCE_TESTING
  MPI_Finalize();
#endif

  return EXIT_SUCCESS;
}