module NormalSmoothingSplines

###### Inteface definition
export prepare_approximation, construct_approximation, approximate
export prepare_interpolation, construct_interpolation, interpolate
export evaluate, evaluate_at
export evaluate_gradient, evaluate_derivative
export NormalSpline, RK_H0, RK_H1, RK_H2

export get_epsilon, estimate_epsilon, get_cond, estimate_cond
export estimate_accuracy
######

using LinearAlgebra

abstract type ReproducingKernel end
abstract type ReproducingKernel_0 <: ReproducingKernel end
abstract type ReproducingKernel_1 <: ReproducingKernel_0 end
abstract type ReproducingKernel_2 <: ReproducingKernel_1 end

abstract type AbstractSpline end

"""
`struct NormalSpline{T, RK} <: AbstractSpline where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Define a structure containing full information of a normal spline
# Fields
- `_kernel`: a reproducing kernel spline was built with
- `_compression`: factor of transforming the original node locations into unit hypercube
- `_nodes`: transformed function value interpolation nodes
- `_nodes_b`: transformed function value approximation nodes
- `_values`: function values at interpolation nodes
- `_values_lb`: function lower bound values at approximation nodes
- `_values_ub`: function upper bound values at approximation nodes
- `_d_nodes`: transformed function directional derivative interpolation nodes
- `_d_nodes_b`: transformed function directional derivative approximation nodes
- `_es`: normalized derivative directions at interpolation nodes
- `_es_b`: normalized derivative directions at approximation nodes
- `_d_values`: function directional derivative values at interpolation nodes
- `_d_values_lb`: function lower bound directional derivative values at approximation nodes
- `_d_values_ub`: function upper bound directional derivative values at approximation nodes
- `_min_bound`: minimal bounds of the original node locations area
- `_gram`: Gram matrix of the problem
- `_chol`: Cholesky factorization of the Gram matrix
- `_mu`: spline coefficients
- `_active`: active inequality constraint numbers at solution
- `_cond`: estimation of the Gram matrix condition number
- `_ier`: An integer flag. If it is equal to 0, the optimal solution was found.
          If it is equal to 1, the approximate solution was found. QP algorithm iterations were stopped
          because of small spline norm change.
          If it is equal to 2, the approximate solution was found. QP algorithm iterations were stopped
          because of maximum allowed number of iterations was reached.
          If it is equal to -1, the solution was not calculated.
"""
struct NormalSpline{T, RK} <: AbstractSpline where {T <: AbstractFloat, RK <: ReproducingKernel_0}
    _kernel::RK
    _compression::T
    _nodes::Union{Matrix{T}, Nothing}
    _nodes_b::Union{Matrix{T}, Nothing}
    _values::Union{Vector{T}, Nothing}
    _values_lb::Union{Vector{T}, Nothing}
    _values_ub::Union{Vector{T}, Nothing}
    _d_nodes::Union{Matrix{T}, Nothing}
    _d_nodes_b::Union{Matrix{T}, Nothing}
    _es::Union{Matrix{T}, Nothing}
    _es_b::Union{Matrix{T}, Nothing}
    _d_values::Union{Vector{T}, Nothing}
    _d_values_lb::Union{Vector{T}, Nothing}
    _d_values_ub::Union{Vector{T}, Nothing}
    _min_bound::Union{Vector{T}, Nothing}
    _gram::Union{Matrix{T}, Nothing}
    _chol::Union{Cholesky{T, Matrix{T}}, Nothing}
    _mu::Union{Vector{T}, Nothing}
    _active::Union{Vector{Int}, Nothing}
    _cond::T
    _ier::Int
end

include("./NormalInterpolatingSplines.jl")
include("./ReproducingKernels.jl")
include("./GramMatrix.jl")
include("./Utils.jl")
include("./QP.jl")
include("./Interpolate.jl")
include("./Approximate.jl")

##
#include("./examples/Main.jl")
##

"""
`prepare_approximation(nodes_b::Matrix{T}, kernel::RK = RK_H0())
                       where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Prepare the approximating normal spline by constructing and factoring a Gram matrix of the problem.
Initialize the `NormalSpline` object.
# Arguments
- `nodes_b`: function value approximation nodes.
          This should be an `n×n_1_b` matrix, where `n` is dimension of the sampled space and
          `n_1_b` is the number of function value approximation nodes. It means that each column in the matrix defines one node.
- `kernel`: reproducing kernel of Bessel potential space the normal spline is constructed in.
            It must be a struct object of the following type:
              `RK_H0` if the spline is constructing as a continuous function,
              `RK_H1` if the spline is constructing as a differentiable function,
              `RK_H2` if the spline is constructing as a twice differentiable function.

Return: prepared `NormalSpline` object.
"""
function prepare_approximation(nodes_b::Matrix{T},
                               kernel::RK = RK_H0()
                              ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
     spline = _prepare_approximation(nodes_b, kernel)
     return spline
end

"""
`prepare_approximation(nodes::Matrix{T}, nodes_b::Matrix{T}, kernel::RK = RK_H0())
                       where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Prepare the approximating normal spline by constructing and factoring a Gram matrix of the problem.
Initialize the `NormalSpline` object.
# Arguments
- `nodes`: function value interpolation nodes.
           This should be an `n×n_1` matrix, where `n` is dimension of the sampled space and
           `n_1` is the number of function value interpolation nodes. It means that each column in the matrix defines one node.
- `nodes_b`: function value approximation nodes.
          This should be an `n×n_1_b` matrix, where `n` is dimension of the sampled space and
          `n_1_b` is the number of function value approximation nodes. It means that each column in the matrix defines one node.
- `kernel`: reproducing kernel of Bessel potential space the normal spline is constructed in.
            It must be a struct object of the following type:
              `RK_H0` if the spline is constructing as a continuous function,
              `RK_H1` if the spline is constructing as a differentiable function,
              `RK_H2` if the spline is constructing as a twice differentiable function.

Return: prepared `NormalSpline` object.
"""
function prepare_approximation(nodes::Matrix{T},
                               nodes_b::Matrix{T},
                               kernel::RK = RK_H0()
                              ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
     spline = _prepare_approximation(nodes, nodes_b, kernel)
     return spline
end

"""
`construct_approximation(spline::NormalSpline{T, RK},
                         values_lb::Vector{T}, values_ub::Vector{T},
                         maxiter::Int, ftol::T)
                         where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

construct the approximating normal spline by calculating its coefficients and
completely initializing the `NormalSpline` object.
# Arguments
- `spline`: the partly initialized `NormalSpline` object returned by `prepare` function.
- `values_lb`: function lower bound values at approximation nodes
- `values_ub`: function upper bound values at approximation nodes
- `maxiter`: Maximum allowed number of iterations.
- `ftol`: convergence tolerance. The iteration stops when relative spline norm change is smaller than ftol.

Return: constructed `NormalSpline` object and the number of QP algorithm iterations done.
"""
function construct_approximation(spline::NormalSpline{T, RK},
                                 values_lb::Vector{T},
                                 values_ub::Vector{T},
                                 maxiter::Int,
                                 ftol::T = T(1.e-3)
                                ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
    spline, nit_done = _construct_approximation(spline, values_lb, values_ub, maxiter, ftol)
    return spline, nit_done
end

"""
`construct_approximation(spline::NormalSpline{T, RK}, values::Vector{T},
                         values_lb::Vector{T}, values_ub::Vector{T}, maxiter::Int, ftol::T)
                         where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

construct the approximating normal spline by calculating its coefficients and
completely initializing the `NormalSpline` object.
# Arguments
- `spline`: the partly initialized `NormalSpline` object returned by `prepare` function.
- `values`: function values at interpolation nodes.
- `values_lb`: function lower bound values at approximation nodes
- `values_ub`: function upper bound values at approximation nodes
- `maxiter`: Maximum allowed number of iterations.
- `ftol`: convergence tolerance. The iteration stops when relative spline norm change is smaller than ftol.

Return: constructed `NormalSpline` object and the number of QP algorithm iterations done.
"""
function construct_approximation(spline::NormalSpline{T, RK},
                                 values::Vector{T},
                                 values_lb::Vector{T},
                                 values_ub::Vector{T},
                                 maxiter::Int,
                                 ftol::T = T(1.e-3)
                                ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
    spline, nit_done = _construct_approximation(spline, values, values_lb, values_ub, maxiter, ftol)
    return spline, nit_done
end

"""
`approximate(nodes::Matrix{T}, values::Vector{T}, nodes_b::Matrix{T}, values_lb::Vector{T}, values_ub::Vector{T},
             kernel::RK, maxiter::Int, ftol::T) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Prepare and construct the approximating normal spline.
# Arguments
- `nodes_b`: function value approximation nodes.
        This should be an `n×n_1_b` matrix, where `n` is dimension of the sampled space and
        `n_1_b` is the number of function value approximation nodes. It means that each column in the matrix defines one node.
- `values_lb`: function lower bound values at approximation nodes
- `values_ub`: function upper bound values at approximation nodes
- `kernel`: reproducing kernel of Bessel potential space the normal spline is constructed in.
            It must be a struct object of the following type:
              `RK_H0` if the spline is constructing as a continuous function,
              `RK_H1` if the spline is constructing as a differentiable function,
              `RK_H2` if the spline is constructing as a twice differentiable function.
- `maxiter`: Maximum allowed number of iterations.
- `ftol`: convergence tolerance. The iteration stops when relative spline norm change is smaller than ftol.

Return: constructed `NormalSpline` object.
"""
function approximate(nodes_b::Matrix{T},
                     values_lb::Vector{T},
                     values_ub::Vector{T},
                     kernel::RK,
                     maxiter::Int,
                     ftol::T = T(1.e-3)
                    ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
     spline = _prepare_approximation(nodes_b, kernel)
     spline, nit_done = _construct_approximation(spline, values_lb, values_ub, maxiter, ftol)
     return spline
end

"""
`approximate(nodes::Matrix{T}, values::Vector{T}, nodes_b::Matrix{T}, values_lb::Vector{T}, values_ub::Vector{T},
             kernel::RK, maxiter::Int, ftol::T) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Prepare and construct the approximating normal spline.
# Arguments
- `nodes`: function value interpolation nodes.
           This should be an `n×n_1` matrix, where `n` is dimension of the sampled space
           and `n_1` is the number of function value nodes.
           It means that each column in the matrix defines one node.
- `nodes_b`: function value approximation nodes.
        This should be an `n×n_1_b` matrix, where `n` is dimension of the sampled space and
        `n_1_b` is the number of function value approximation nodes. It means that each column in the matrix defines one node.
- `values`: function values at interpolation nodes.
- `values_lb`: function lower bound values at approximation nodes
- `values_ub`: function upper bound values at approximation nodes
- `kernel`: reproducing kernel of Bessel potential space the normal spline is constructed in.
            It must be a struct object of the following type:
              `RK_H0` if the spline is constructing as a continuous function,
              `RK_H1` if the spline is constructing as a differentiable function,
              `RK_H2` if the spline is constructing as a twice differentiable function.
- `maxiter`: Maximum allowed number of iterations.
- `ftol`: convergence tolerance. The iteration stops when relative spline norm change is smaller than ftol.

Return: constructed `NormalSpline` object.
"""
function approximate(nodes::Matrix{T},
                     values::Vector{T},
                     nodes_b::Matrix{T},
                     values_lb::Vector{T},
                     values_ub::Vector{T},
                     kernel::RK,
                     maxiter::Int,
                     ftol::T = T(1.e-3)
                    ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
     spline = _prepare_approximation(nodes, nodes_b, kernel)
     spline, nit_done = _construct_approximation(spline, values, values_lb, values_ub, maxiter, ftol)
     return spline
end

"""
`evaluate(spline::NormalSpline{T, RK}, points::Matrix{T}) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Evaluate the spline values at `points` locations.

# Arguments
- `spline: constructed `NormalSpline` object.
- `points`: locations at which spline values are evaluating.
            This should be an `n×m` matrix, where `n` is dimension of the sampled space
            and `m` is the number of locations where spline values are evaluating.
            It means that each column in the matrix defines one location.

Return: `Vector{T}` of the spline values at the locations defined in `points`.
"""
function evaluate(spline::NormalSpline{T, RK}, points::Matrix{T}
                 ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
    return _evaluate(spline, points)
end

"""
`evaluate_at(spline::NormalSpline{T, RK}, point::Vector{T}) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Evaluate the spline value at the `point` location.

# Arguments
- `spline`: constructed `NormalSpline` object.
- `point`: location at which spline value is evaluating.
           This should be a vector of size `n`, where `n` is dimension of the sampled space.

Return: spline value at the location defined in `point`.
"""
function evaluate_at(spline::NormalSpline{T, RK}, point::Vector{T}
                    ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
    return _evaluate(spline, reshape(point, :, 1))[1]
end

"""
`evaluate_gradient(spline::NormalSpline{T, RK}, point::Vector{T}) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Evaluate gradient of the spline at the location defined in `point`.

# Arguments
- `spline`: the `NormalSpline` object returned by `interpolate` or `construct` function.
- `point`: location at which gradient value is evaluating.
           This should be a vector of size `n`, where `n` is dimension of the sampled space.

Note: Gradient of spline built with reproducing kernel RK_H0 does not exist at the spline nodes.

Return: `Vector{T}` - gradient of the spline at the `point` location.
"""
function evaluate_gradient(spline::NormalSpline{T, RK},
                           point::Vector{T}
                          ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
    return _evaluate_gradient(spline, point)
end

########

"""
`get_epsilon(spline::NormalSpline{T, RK}) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Get the 'scaling parameter' of Bessel Potential space the spline was built in.
# Arguments
- `spline`: prepared `NormalSpline` object.

Return: `ε` - the 'scaling parameter'.
"""
function get_epsilon(spline::NormalSpline{T, RK}
                    ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
    return spline._kernel.ε
end

"""
`estimate_epsilon(nodes::Matrix{T}, kernel::RK = RK_H0()) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Get the estimation of the 'scaling parameter'2 of Bessel Potential space the spline being built in.
It coincides with the result returned by `get_epsilon` function.
# Arguments
- `nodes`: function value interpolation nodes.
           This should be an `n×n_1` matrix, where `n` is dimension of the sampled space
           and `n_1` is the number of function value nodes.
           It means that each column in the matrix defines one node.
- `kernel`: reproducing kernel of Bessel potential space the normal spline will be constructed in.
           It must be a struct object of the following type:
             `RK_H0` if the spline is constructing as a continuous function,
             `RK_H1` if the spline is constructing as a differentiable function,
             `RK_H2` if the spline is constructing as a twice differentiable function.
Return: estimation of `ε`.
"""
function estimate_epsilon(nodes::Matrix{T},
                          kernel::RK = RK_H0()
                         ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
    ε = _estimate_epsilon(nodes, kernel)
    return ε
end

"""
`estimate_epsilon(nodes::Matrix{T}, d_nodes::Matrix{T}, kernel::RK = RK_H1()) where {T <: AbstractFloat, RK <: ReproducingKernel_1}`

Get an the estimation of the 'scaling parameter' of Bessel Potential space the spline being built in.
It coincides with the result returned by `get_epsilon` function.
# Arguments
- `nodes`: function value interpolation nodes.
           This should be an `n×n_1` matrix, where `n` is dimension of the sampled space
           and `n_1` is the number of function value nodes.
           It means that each column in the matrix defines one node.
- `d_nodes`: function directional derivative nodes.
           This should be an `n×n_2` matrix, where `n` is dimension of the sampled space and
           `n_2` is the number of function directional derivative nodes.
- `kernel`: reproducing kernel of Bessel potential space the normal spline will be constructed in.
            It must be a struct object of the following type:
            `RK_H1` if the spline is constructing as a differentiable function,
            `RK_H2` if the spline is constructing as a twice differentiable function.

Return: estimation of `ε`.
"""
function estimate_epsilon(nodes::Matrix{T},
                          d_nodes::Matrix{T},
                          kernel::RK = RK_H1()
                         ) where {T <: AbstractFloat, RK <: ReproducingKernel_1}
    ε = _estimate_epsilon(nodes, d_nodes, kernel)
    return ε
end

"""
`estimate_cond(spline::NormalSpline{T, RK}) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Get an estimation of the Gram matrix condition number. It needs the `spline` object is prepared and requires O(N^2) operations.
(C. Brás, W. Hager, J. Júdice, An investigation of feasible descent algorithms for estimating the condition number of a matrix. TOP Vol.20, No.3, 2012.)
# Arguments
- `spline`: prepared `NormalSpline` object.

Return: an estimation of the Gram matrix condition number.
"""
function estimate_cond(spline::NormalSpline{T, RK}
                      ) where {T <: AbstractFloat, RK <: ReproducingKernel}
    return spline._cond
end

"""
`estimate_accuracy(spline::NormalSpline{T, RK}) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Assess accuracy of interpolation results by analyzing residuals.
# Arguments
- `spline`: constructed `NormalSpline` object.

Return: estimation of the number of significant digits in the interpolation result.
"""
function estimate_accuracy(spline::NormalSpline{T, RK}) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
    return _estimate_accuracy(spline)
end

############################## One-dimensional case

"""
`prepare_approximation(nodes_b::Vector{T}, kernel::RK = RK_H0())
                       where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Prepare the 1D approximating normal spline by constructing and factoring a Gram matrix of the problem.
Initialize the `NormalSpline` object.
# Arguments
- `nodes_b`: function value approximation nodes.
             This should be an `n_1_b` vector where `n_1_b` is the number of function value approximation nodes.
- `kernel`: reproducing kernel of Bessel potential space the normal spline is constructed in.
            It must be a struct object of the following type:
              `RK_H0` if the spline is constructing as a continuous function,
              `RK_H1` if the spline is constructing as a differentiable function,
              `RK_H2` if the spline is constructing as a twice differentiable function.

Return: prepared `NormalSpline` object.
"""
function prepare_approximation(nodes_b::Vector{T},
                               kernel::RK = RK_H0()
                              ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}

     spline = _prepare_approximation(Matrix(nodes_b'), kernel)
     return spline
end

"""
`prepare_approximation(nodes::Vector{T}, nodes_b::Vector{T}, kernel::RK = RK_H0())
                       where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Prepare the 1D approximating normal spline by constructing and factoring a Gram matrix of the problem.
Initialize the `NormalSpline` object.
# Arguments
- `nodes`: function value interpolation nodes.
           This should be an `n_1` vector where `n_1` is the number of function value interpolation nodes.
- `nodes_b`: function value approximation nodes.
           This should be an `n_1_b` vector where `n_1_b` is the number of function value approximation nodes.
- `kernel`: reproducing kernel of Bessel potential space the normal spline is constructed in.
            It must be a struct object of the following type:
              `RK_H0` if the spline is constructing as a continuous function,
              `RK_H1` if the spline is constructing as a differentiable function,
              `RK_H2` if the spline is constructing as a twice differentiable function.

Return: prepared `NormalSpline` object.
"""
function prepare_approximation(nodes::Vector{T},
                               nodes_b::Vector{T},
                               kernel::RK = RK_H0()
                              ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
     spline = _prepare_approximation(Matrix(nodes'), Matrix(nodes_b'), kernel)
     return spline
end

"""
`approximate(values::Vector{T}, nodes_b::Vector{T}, values_lb::Vector{T}, values_ub::Vector{T},
             kernel::RK, maxiter::Int, ftol::T) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Prepare and construct the 1D approximating normal spline.
# Arguments
- `nodes_b`: function value approximation nodes.
          This should be an `n_1_b` vector where `n_1_b` is the number of function value approximation nodes.
- `values_lb`: function lower bound values at approximation nodes
- `values_ub`: function upper bound values at approximation nodes
- `kernel`: reproducing kernel of Bessel potential space the normal spline is constructed in.
            It must be a struct object of the following type:
              `RK_H0` if the spline is constructing as a continuous function,
              `RK_H1` if the spline is constructing as a differentiable function,
              `RK_H2` if the spline is constructing as a twice differentiable function.
- `maxiter`: Maximum allowed number of iterations.
- `ftol`: convergence tolerance. The iteration stops when relative spline norm change is smaller than ftol.

Return: constructed `NormalSpline` object.
"""
function approximate(nodes_b::Vector{T},
                     values_lb::Vector{T},
                     values_ub::Vector{T},
                     kernel::RK,
                     maxiter::Int,
                     ftol::T = T(1.e-3)
               ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
     spline = _prepare_approximation(Matrix(nodes_b'), kernel)
     spline, nit_done = _construct_approximation(spline, values_lb, values_ub, maxiter, ftol)
     return spline
end

"""
`approximate(nodes::Vector{T}, values::Vector{T}, nodes_b::Vector{T}, values_lb::Vector{T}, values_ub::Vector{T},
             kernel::RK, maxiter::Int, ftol::T) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Prepare and construct the 1D approximating normal spline.
# Arguments
- `nodes`: function value interpolation nodes.
          This should be an `n_1` vector where `n_1` is the number of function value interpolation nodes.
- `values`: function values at interpolation nodes.
- `nodes_b`: function value approximation nodes.
          This should be an `n_1_b` vector where `n_1_b` is the number of function value approximation nodes.
- `values_lb`: function lower bound values at approximation nodes
- `values_ub`: function upper bound values at approximation nodes
- `kernel`: reproducing kernel of Bessel potential space the normal spline is constructed in.
            It must be a struct object of the following type:
              `RK_H0` if the spline is constructing as a continuous function,
              `RK_H1` if the spline is constructing as a differentiable function,
              `RK_H2` if the spline is constructing as a twice differentiable function.
- `maxiter`: Maximum allowed number of iterations.
- `ftol`: convergence tolerance. The iteration stops when relative spline norm change is smaller than ftol.

Return: constructed `NormalSpline` object.
"""
function approximate(nodes::Vector{T},
                     values::Vector{T},
                     nodes_b::Vector{T},
                     values_lb::Vector{T},
                     values_ub::Vector{T},
                     kernel::RK,
                     maxiter::Int,
                     ftol::T = T(1.e-3)
               ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
     spline = _prepare_approximation(Matrix(nodes'), Matrix(nodes_b'), kernel)
     spline, nit_done = _construct_approximation(spline, values, values_lb, values_ub, maxiter, ftol)
     return spline
end

"""
`evaluate(spline::NormalSpline{T, RK}, points::Vector{T}) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Evaluate the 1D spline values at the `points` locations.

# Arguments
- `spline`: constructed `NormalSpline` object.
- `points`: locations at which spline values are evaluating.
            This should be a vector of size `m` where `m` is the number of evaluating points.

Return: spline value at the `point` location.
"""
function evaluate(spline::NormalSpline{T, RK}, points::Vector{T}
                               ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
    return _evaluate(spline, Matrix(points'))
end

"""
`evaluate_at(spline::NormalSpline{T, RK}, point::T) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Evaluate the 1D spline value at the `point` location.

# Arguments
- `spline`: constructed `NormalSpline` object.
- `point`: location at which spline value is evaluating.

Return: spline value at the `point` location.
"""
function evaluate_at(spline::NormalSpline{T, RK},
                                       point::T
                                      ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
    v_points = Vector{T}(undef, 1)
    v_points[1] = point
    return _evaluate(spline, Matrix(v_points'))[1]
end

"""
`evaluate_derivative(spline::NormalSpline{T, RK}, point::T) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Evaluate the 1D spline derivative at the `point` location.

# Arguments
- `spline`: the `NormalSpline` object returned by `interpolate` or `construct` function.
- `point`: location at which spline derivative is evaluating.

Note: Derivative of spline built with reproducing kernel RK_H0 does not exist at the spline nodes.

Return: spline derivative value at the `point` location.
"""
function evaluate_derivative(spline::NormalSpline{T, RK},
                             point::T
                            ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
    v_points = Vector{T}(undef, 1)
    v_points[1] = point
    return _evaluate_gradient(spline, v_points)[1]
end

"""
`estimate_epsilon(nodes::Vector{T}, kernel::RK = RK_H0()) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Get an the estimation of the 'scaling parameter' of Bessel Potential space the spline being built in.
It coincides with the result returned by `get_epsilon` function.
# Arguments
- `nodes`: function value interpolation nodes.
- `kernel`: reproducing kernel of Bessel potential space the normal spline is constructed in.
            It must be a struct object of the following type:
              `RK_H0` if the spline is constructing as a continuous function,
              `RK_H1` if the spline is constructing as a differentiable function,
              `RK_H2` if the spline is constructing as a twice differentiable function.

Return: estimation of `ε`.
"""
function estimate_epsilon(nodes::Vector{T},
                          kernel::RK = RK_H0()
                         ) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
    ε = _estimate_epsilon(Matrix(nodes'), kernel)
    return ε
end

"""
`estimate_epsilon(nodes::Vector{T}, d_nodes::Vector{T}, kernel::RK = RK_H1()) where {T <: AbstractFloat, RK <: ReproducingKernel_1}`

Get an the estimation of the 'scaling parameter' of Bessel Potential space the spline being built in.
It coincides with the result returned by `get_epsilon` function.
# Arguments
- `nodes`: function value interpolation nodes.
- `d_nodes`: function derivative nodes.
- `kernel`: reproducing kernel of Bessel potential space the normal spline is constructed in.
            It must be a struct object of the following type:
              `RK_H1` if the spline is constructing as a differentiable function,
              `RK_H2` if the spline is constructing as a twice differentiable function.

Return: estimation of `ε`.
"""
function estimate_epsilon(nodes::Vector{T},
                          d_nodes::Vector{T},
                          kernel::RK = RK_H1()
                         ) where {T <: AbstractFloat, RK <: ReproducingKernel_1}
    ε = _estimate_epsilon(Matrix(nodes'), Matrix(d_nodes'), kernel)
    return ε
end

"""
`get_cond(nodes::Matrix{T}, kernel::RK = RK_H0()) where {T <: AbstractFloat, RK <: ReproducingKernel_0}`

Get a value of the Gram matrix spectral condition number. It is obtained by means of the matrix SVD decomposition.
# Arguments
- `nodes`: function value interpolation nodes.
           This should be an `n×n_1` matrix, where `n` is dimension of the sampled space and
           `n_1` is the number of function value nodes. It means that each column in the matrix defines one node.
- `kernel`: reproducing kernel of Bessel potential space the normal spline is constructed in.
            It must be a struct object of the following type:
              `RK_H0` if the spline is constructing as a continuous function,
              `RK_H1` if the spline is constructing as a differentiable function,
              `RK_H2` if the spline is constructing as a twice differentiable function.

Return: a value of the Gram matrix spectral condition number.
"""
function get_cond(nodes::Matrix{T}, kernel::RK) where {T <: AbstractFloat, RK <: ReproducingKernel_0}
  return _get_cond(nodes, kernel)
end

"""
`get_cond(nodes::Matrix{T}, d_nodes::Matrix{T}, es::Matrix{T}, kernel::RK = RK_H1()) where {T <: AbstractFloat, RK <: ReproducingKernel_1}`

Get a value of the Gram matrix spectral condition number. It is obtained by means of the matrix SVD decomposition.
# Arguments
- `nodes`: function value interpolation nodes.
           This should be an `n×n_1` matrix, where `n` is dimension of the sampled space and
           `n_1` is the number of function value nodes.
            It means that each column in the matrix defines one node.
- `d_nodes`: function directional derivatives nodes.
             This should be an `n×n_2` matrix, where `n` is dimension of the sampled space and
             `n_2` is the number of function directional derivative nodes.
- `es`: Directions of the function directional derivatives.
        This should be an `n×n_2` matrix, where `n` is dimension of the sampled space and
        `n_2` is the number of function directional derivative nodes.
        It means that each column in the matrix defines one direction of the function directional derivative.
- `kernel`: reproducing kernel of Bessel potential space the normal spline is constructed in.
            It must be a struct object of the following type:
              `RK_H1` if the spline is constructing as a differentiable function,
              `RK_H2` if the spline is constructing as a twice differentiable function.

Return: value of the Gram matrix spectral condition number.
"""
function get_cond(nodes::Matrix{T}, d_nodes::Matrix{T}, es::Matrix{T}, kernel::RK = RK_H1()
                 ) where {T <: AbstractFloat, RK <: ReproducingKernel_1}
    return _get_cond(nodes, d_nodes, es, kernel)
end

end # module


"""
    skewness(x; method=:moment)

Calculates the skewness using the specified method.

# Methods
- Moment (default)
- Fisher-Pearson
- Sample

# Arguments
- `x`:          Vector of values.
- `method`:     Estimation method: `:moment`, `:fisher_pearson` or `:sample`.
"""
function skewness(x; method::Symbol=:moment)
    n = length(x)
    mean_devs = x .- mean(x)

    if method == :moment
        # Moment
        return mean(mean_devs.^3) / sqrt(mean(mean_devs.^2))^3  # sqrt(x)^3 faster than x^1.5 !
    elseif method == :fisher_pearson
        # Fisher-Pearson
        if n > 2
            return sqrt(n*(n-1))/(n-2) * mean(mean_devs.^3) / sqrt(mean(mean_devs.^2))^3  # sqrt(x)^3 faster than x^1.5 !
        else
            return NaN
        end
    elseif method == :sample
        # Sample
        return n/((n-1)*(n-2)) * sum(mean_devs.^3 / sqrt(mean(mean_devs.^2))^3)
    end

    throw(ArgumentError("Passed method parameter '$(method)' is invalid, must be one of :moment, :fisher_pearson, :sample."))
end

"""
    kurtosis(x; method=:excess)

Calculates the kurtosis using on the specified method.

# Methods
- Excess (default)
- Moment
- Cornish-Fisher

# Arguments
- `x`:          Vector of values.
- `method`:     Estimation method: `:excess`, `:moment` or `:cornish_fisher`.
"""
function kurtosis(x; method::Symbol=:excess)
    n = length(x)
    mean_devs = x .- mean(x)

    if method == :excess
        # Excess
        return sum(mean_devs.^4 / mean(mean_devs.^2)^2 ) / n - 3
    elseif method == :moment
        # Moment
        return sum(mean_devs.^4 / mean(mean_devs.^2)^2 ) / n
    elseif method == :cornish_fisher
        # Cornish-Fisher
        return ((n+1)*(n-1)*((sum(x.^4)/n)/(sum(x.^2)/n)^2 -
            (3*(n-1))/(n+1)))/((n-2)*(n-3))
    end

    throw(ArgumentError("Passed method parameter '$(method)' is invalid, must be one of :excess, :moment, :cornish_fisher."))
end

"""
    lower_partial_moment(returns, threshold, n, method)

This function calculates the Lower Partial Moment (LPM) for a given threshold.

# Arguments
- `returns`:     Vector of asset returns.
- `threshold`:   Scalar value or vector denoting the threshold returns.
- `n`:           `n`-th moment to calculate.
- `method`:      One of `:full` or `:partial`. Indicates whether to use the number of all returns (`:full`), or only the number of returns below the threshold (`:partial`) in the denominator.
"""
function lower_partial_moment(returns, threshold, n, method::Symbol)
    if method == :full
        denominator = length(returns)
    elseif method == :partial
        denominator = count(returns .< threshold)
    else
        throw(ArgumentError("Passed method parameter '$(method)' is invalid, must be one of :full, :partial."))
    end
    excess = threshold .- returns
    sum(map(x -> max(0.0, x)^n, excess)) / denominator
end

"""
    higher_partial_moment(returns, threshold, n, method)

This function calculates the Higher Partial Moment (HPM) for a given threshold.

# Arguments
- `returns`:     Vector of asset returns.
- `threshold`:   Scalar value or vector denoting the threshold returns.
- `n`:           `n`-th moment to calculate.
- `method`:      One of `:full` or `:partial`. Indicates whether to use the number of all returns (`:full`), or only the number of returns above the threshold (`:partial`) in the denominator.
"""
function higher_partial_moment(returns, threshold, n, method::Symbol)
    if method == :full
        denominator = length(returns)
    elseif method == :partial
        denominator = count(returns .> threshold)
    else
        throw(ArgumentError("Passed method parameter '$(method)' is invalid, must be one of :full, :partial."))
    end
    excess = returns .- threshold
    sum(map(x -> max(0.0, x)^n, excess)) / denominator
end


# Author: Lukasz Bratos

module Diff
using Plots
gr()

export ilorazyRoznicowe, warNewton, naturalna, rysujNnfx

"""
Funkcja oblicza iloraz różnicowy

Dane:
x – wektor długości n + 1 zawierający węzły x0, ..., xn
    x[1] = x0, ..., x[n+1] = xn
f – wektor długości n + 1 zawierający wartości interpolowanej funkcji w węzłach
    f(x0), ..., f(xn)

Wyniki:
fx– wektor długości n + 1 zawierający obliczone ilorazy różnicowe
    fx[1] = f[x0],
    fx[2] = f[x0, x1], ..., fx[n] = f[x0, ..., xn−1], fx[n+1] = f[x0, ..., xn]
"""
function ilorazyRoznicowe(x::Vector{Float64}, f::Vector{Float64})
    len = length(f)
    fx = Vector{Float64}(undef, len)

    for i in 1:len
        fx[i] = f[i]
    end

    for i in 2:len
        for j in len:-1:i
            fx[j] = (fx[j] - fx[j - 1]) / (x[j] - x[j - i + 1])
        end
    end

    return fx
end

"""
Funkcja wyliczajaca wartosc wielomianu interpolacyjnego stopnia n w postaci Newtona Nn(x)
za pomoca uogolnionego algorytmu Hornera

Dane:
x – wektor długości n + 1 zawierający węzły x0, ..., xn
    x[1] = x0, ..., x[n+1] = xn
fx – wektor długości n+ 1 zawierający ilorazy różnicowe
    fx[1] = f[x0],
    fx[2] = f[x0, x1], ..., fx[n] = f[x0, ..., xn−1], fx[n+1] =f [x0, ..., xn]
t – punkt, w którym należy obliczyć wartość wielomianu

Wyniki:
nt – wartość wielomianu w punkcie t
"""
function warNewton(x::Vector{Float64}, fx::Vector{Float64}, t::Float64)
    len = length(x)
    nt = fx[len]

    for i in (len - 1):-1:1
        nt = fx[i] + (t - x[i]) * nt
    end

    return nt
end

"""
Funkcja wyliczajaca wspolczynniki wielomianu w postaci normanej z wielomianu
interpolacyjnego w postaci Newtona

Dane:
x – wektor długości n+ 1 zawierający węzły x0, ..., xn
    x[1] = x0, ..., x[n+1] = xn
fx – wektor długości n + 1 zawierający ilorazy różnicowe
    fx[1] = f[x0],
    fx[2] = f[x0, x1], ..., fx[n] = f[x0, ..., xn−1], fx[n+1] = f[x0, ..., xn]

Wyniki:
a – wektor długości n+ 1 zawierający obliczone współczynniki postaci naturalnej
    a[1] = a0,
    a[2] = a1, ..., a[n] = an−1, a[n+1] = an.
"""
function naturalna(x::Vector{Float64}, fx::Vector{Float64})
    len = length(x)
    a = Vector{Float64}(undef, len)
    a[len] = fx[len]

    for i in (len - 1):-1:1
        a[i] = fx[i] - a[i + 1] * x[i]

        for j in (i + 1):(len - 1)
            a[j] = a[j] - a[j + 1] * x[i]
        end
    end

    return a
end

"""
Funkcja rysująca wielomian interpolacyjny i interpolowaną funkcję na danym przedziale

Dane:
f – funkcja f(x) zadana jako anonimowa funkcja,
a, b – przedział interpolacji
n – stopień wielomianu interpolacyjnego
filename - nazwa pliku wynikowego

Wyniki:
– funkcja rysuje wielomian interpolacyjny i interpolowaną funkcję w przedziale [a, b]
"""
function rysujNnfx(f, a::Float64, b::Float64, n::Int, filename::String)
    dist = (b - a) / n

    x = Vector{Float64}(undef, n + 1)
    y = Vector{Float64}(undef, n + 1)

    for i in 1:(n + 1)
        x[i] = a + (i - 1) * dist
        y[i] = f(x[i]);
    end

    fx = ilorazyRoznicowe(x, y)

    numOfPoints = 100
    plot_dist = (b - a) / numOfPoints

    fx_val = Vector{Float64}(undef, numOfPoints + 1)
    wx_val = Vector{Float64}(undef, numOfPoints + 1)

    for i in 1:(numOfPoints + 1)
        t = a + (i - 1) * plot_dist
        fx_val[i] = f(t)
        wx_val[i] = warNewton(x, fx, t)
    end

    plot(range(a, stop=b, length=(numOfPoints + 1)), fx_val, color = "red", label = "f(x)")
    plot!(range(a, stop=b, length=(numOfPoints + 1)), wx_val, color = "blue", label = "w(x)")
    savefig("./plots/$filename.png")
end

end


# Classification Metrics and Plots
# This files contains various metrics used in Classifiaction problems

# One Hot encode
function onehot_encode(y, labels)
   onehot_arr = zeros(length(labels), length(y))
   for i in 1:length(y)
       onehot_arr[Int(y[i]) + 1, i] = 1
   end
   onehot_arr
end

"""
    bin_to_cat(y_pred, y_true)

Function to convert binary type of data to categorical with two categories. Return `y_pred` and `y_true` of shape `(2, length(y_pred))` as tuple. Utility function to support performance metrics like `Precision`, `Recall` etc, where the function first need to be converted to categorical form before applying metric.
"""
function bin_to_cat(y_pred, y_true)
    @assert length(y_pred) == length(y_true)
    y_pred_ = zeros(2, length(y_pred))
    y_pred_[2, :] = y_pred
    y_pred_[1, :] = 1 .- y_pred
    return (y_pred_, onehot_encode(y_true, 0:1))
end

# Onecold
function onecold(y)
    onecold_arr = zeros(size(y,2))
    argmax_vec = argmax(y, dims = 1)
    for i in 1:size(y,2)
        onecold_arr[i] = argmax_vec[i].I[1]
    end
    return onecold_arr
end

"""
    confusion_matrix(y_pred, y_true)

Function to create a confusion_matrix for classification problems based on provided `y_pred` and `y_true`. Expects `y_true`, to be onehot_enocded already.
"""
function confusion_matrix(y_pred, y_true)
    @assert size(y_pred) == size(y_true)
    label_count = size(y_true, 1)
    ŷ = onehot_encode(onecold(y_pred), 1:label_count)
    return ŷ * transpose(y_true)
end

"""
    TFPN(y_pred, y_true)

Returns `Confusion Matrix` and `True Positive`, `True Negative`, `False Positive` and `False Negative` for each class based on `y_pred` and `y_true`. Expects `y_true`, to be onehot_enocded already.
"""
function TFPN(y_pred, y_true)
    @assert size(y_pred) == size(y_true)
    label_count = size(y_true, 1)
    TP = zeros(label_count) ; TN = zeros(label_count)
    FP = zeros(label_count) ; FN = zeros(label_count)
    ConfusionMatrix = confusion_matrix(y_pred, y_true)
    for c in 1 : label_count
        TP[c] = ConfusionMatrix[c,c]
        FP[c] = sum(ConfusionMatrix[:,c]) - TP[c]
        FN[c] = sum(ConfusionMatrix[c,:]) - TP[c]
        TN[c] = sum(ConfusionMatrix) - TP[c] - FP[c] - FN[c]
    end
    return ConfusionMatrix,TP, TN, FP, FN
end

"""
    binary_accuracy(y_pred, y_true; threshold=0.5)

Calculates Averaged Binary Accuracy based on `y_pred` and `y_true`. Argument `threshold` is used to specify the minimum predicted probability `y_pred` required to be labelled as `1`. Default value set as `0.5`.
"""
function binary_accuracy(y_pred, y_true; threshold=0.5)
    @assert size(y_pred) == size(y_true)
    return sum((y_pred .>= threshold) .== y_true) / size(y_true, 1)
end

"""
    categorical_accuracy(y_pred, y_true)

Calculates Averaged Categorical Accuracy based on `y_pred` and `y_true`.
"""
function categorical_accuracy(y_pred, y_true)
    @assert size(y_pred) == size(y_true)
    return sum(onecold(y_pred) .== onecold(y_true)) / size(y_true, 2)
end

"""
    sparse_categorical(y_pred, y_true)

Calculated Sparse Categorical Accuracy based on `y_pred` and `y_true`. It evaluates the maximal true value is equal to the index of the maximal predicted value. Here, `y_true` is expected to provide only an integer (start from `0` index) as label for each data element (ie. not one hot encoded).
"""
function sparse_categorical(y_pred, y_true)
    @assert size(y_pred, 2) == length(y_true)
    return sum(onecold(y_pred) .== (y_true .+ 1) ) / size(y_true, 1)
end

"""
    top_k_categorical(y_pred, y_true; k=3)

Evaluates if the index of true value is equal to any of the indices of top k predicted values. Default value of `k` set to `3`.
"""
function top_k_categorical(y_pred, y_true; k=3)
    @assert size(y_pred) == size(y_true)
    count = 0
    sparse_y = onecold(y_true)
    for i in 1:size(y_true, 2)
        top_k = partialsortperm(y_pred[:,i], 1:k, lt= >)
        for j in 1:k
            if top_k[j] == sparse_y[i]
                count+=1
                break
            end
        end
    end
    return count / length(sparse_y)
end

"""
    top_k_sparse_categorical(y_pred, y_true; k=3)

Evaluates if the true value is equal to any of the indices of top k predicted values. Default value of `k` set to `3`. Similar to `sparse_categorical`, expects the `y_true` to provide only an integer (start from `0` index) as label for each data element (ie. not one hot encoded).
"""
function top_k_sparse_categorical(y_pred, y_true; k=3)
    count = 0
    for i in 1:length(y_true)
        top_k = partialsortperm(y_pred[:,i], 1:k, lt= >)
        for j in 1:k
            if top_k[j] == y_true[i] + 1
                count+=1
                break
            end
        end
    end
    return count / length(y_true)
end

# Calculate Instances for each class in y
function calc_instances(y)
   instances = zeros(size(y, 1))
   sparse_y = onecold(y)
   for i in 1:length(instances)
       instances[i] = sum(sparse_y .== i)
   end
   return instances
end

"""
    precision(y_pred, y_true; avg_type="macro", sample_weights=nothing)

Computes the precision of the predictions with respect to the labels.

# Arguments
 - `y_pred`: predicted values.
 - `y_true`: ground truth values on the basis of which predicted values are to be assessed.
 - `avg_type="macro"`: Type of average to be used while calculating precision of multiclass models. Can take values as `macro`, `micro` and `weighted`. Default set to `macro`.
 - `sample_weights`: Class weights to be provided when `avg_type` is set to `weighted`. Useful in case of imbalanced classes.

"""
function precision(y_pred, y_true; avg_type="macro", sample_weights=nothing)
    _,TP, TN, FP, FN = TFPN(y_pred, y_true)
    # Macro-averaged Precision
    if avg_type == "macro"
        return mean(TP ./ (TP .+ FP .+ eps(eltype(TP))))
    # Micro-averaged Precision
    elseif avg_type == "micro"
        return mean(TP) / (mean(TP) + mean(FP))
    # Weighted-Averaged Precision
    elseif avg_type == "weighted"
        weights = []
        if sample_weights != nothing
            weights = sample_weights
        else
            weights = calc_instances(y_true) / size(y_true, 2)
        end
        return mean((TP ./ (TP .+ FP .+ eps(eltype(TP)))) .* weights)
    end
end

"""
    recall(y_pred, y_true; avg_type="macro", sample_weights=nothing)

Computes the recall of the predictions with respect to the labels.

# Arguments
 - `y_pred`: predicted values.
 - `y_true`: ground truth values on the basis of which predicted values are to be assessed.
 - `avg_type="macro"`: Type of average to be used while calculating precision of multiclass models. Can take values as `macro`, `micro` and `weighted`. Default set to `macro`.
 - `sample_weights`: Class weights to be provided when `avg_type` is set to `weighted`. Useful in case of imbalanced classes.

Aliases: `sensitivity` and `detection_rate`
"""
function recall(y_pred, y_true; avg_type="macro", sample_weights=nothing)
    _,TP, TN, FP, FN = TFPN(y_pred, y_true)
    # Macro-averaged Precision
    if avg_type == "macro"
        return mean(TP ./ (TP .+ FN .+ eps(eltype(TP))))
    # Micro-averaged Precision
    elseif avg_type == "micro"
        return mean(TP) / (mean(TP) + mean(FN))
    # Weighted-Averaged Precision
    else
        weights = []
        if sample_weights != nothing
            weights = sample_weights
        else
            weights = calc_instances(y_true) / size(y_true, 2)
        end
        return mean((TP ./ (TP .+ FN .+ eps(eltype(TP)))) .* weights)
    end
end
const sensitivity = recall
const detection_rate = recall

"""
    f_beta_score(y_pred, y_true; β=1, avg_type="macro", sample_weights=nothing)

Compute fbeta score. The F_beta score is the weighted harmonic mean of precision and recall, reaching its optimal value at 1 and its worst value at 0.

# Arguments
 - `y_pred`: predicted values.
 - `y_true`: ground truth values on the basis of which predicted values are to be assessed.
 - `β=1`: the weight of precision in the combined score. If `β<1`, more weight given to `precision`, while `β>1` favors recall.
 - `avg_type="macro"`: Type of average to be used while calculating precision of multiclass models. Can take values as `macro`, `micro` and `weighted`. Default set to `macro`.
 - `sample_weights`: Class weights to be provided when `avg_type` is set to `weighted`. Useful in case of imbalanced classes.

"""
function f_beta_score(y_pred, y_true; β=1, avg_type="macro", sample_weights=nothing)
    recall_ = recall(y_pred, y_true, avg_type=avg_type, sample_weights=sample_weights)
    precision_ = precision(y_pred, y_true, avg_type=avg_type, sample_weights=sample_weights)
    return (1 + β^2) * precision_ * recall_ / (precision_ + (β^2) * recall_ + eps(eltype(y_pred)))
end

"""
    specificity(y_pred, y_true; avg_type="macro", sample_weights=nothing)

Computes the specificity of the predictions with respect to the labels.

# Arguments
 - `y_pred`: predicted values.
 - `y_true`: ground truth values on the basis of which predicted values are to be assessed.
 - `avg_type="macro"`: Type of average to be used while calculating precision of multiclass models. Can take values as `macro`, `micro` and `weighted`. Default set to `macro`.
 - `sample_weights`: Class weights to be provided when `avg_type` is set to `weighted`. Useful in case of imbalanced classes.
"""
function specificity(y_pred, y_true; avg_type="macro", sample_weights=nothing)
    _, TP, TN, FP, FN = TFPN(y_pred, y_true)
    # Macro-averaged Precision
    if avg_type == "macro"
        return mean(TN ./ (TN .+ FP .+ eps(eltype(TP))))
    # Micro-averaged Precision
    elseif avg_type == "micro"
        return mean(TN) / (mean(TN) + mean(FP))
    # Weighted-Averaged Precision
    else
        weights = []
        if sample_weights != nothing
            weights = sample_weights
        else
            weights = calc_instances(y_true) / size(y_true, 2)
        end
        return mean((TN ./ (TN .+ FP .+ eps(eltype(TP)))) .* weights)
    end
end

"""
    false_alarm_rate(y_pred, y_true; avg_type="macro", sample_weights=nothing)

Computes the false_alarm_raye of the predictions with respect to the labels as `1 - specificity(y_pred, y_true, avg_type, sample_weights)`

# Arguments
 - `y_pred`: predicted values.
 - `y_true`: ground truth values on the basis of which predicted values are to be assessed.
 - `avg_type="macro"`: Type of average to be used while calculating precision of multiclass models. Can take values as `macro`, `micro` and `weighted`. Default set to `macro`.
 - `sample_weights`: Class weights to be provided when `avg_type` is set to `weighted`. Useful in case of imbalanced classes.

See also: [`specificity`](@ref)
"""
function false_alarm_rate(y_pred, y_true; avg_type="macro", sample_weights=nothing)
    return 1 - specificity(y_pred, y_true, avg_type, sample_weights)
end

"""
    cohen_kappa(y_pred, y_true)

Measures the agreement between two raters (predicted and ground truth, here) who each classify N items into C mutually exclusive categories, using the observed data to calculate the probabilities of each observer randomly seeing each category.
If the raters are in complete agreement then κ = 1. If there is no agreement among the raters other than what would be expected by chance,
κ = 0.

Ref: [Cohen's Kappa](https://en.wikipedia.org/wiki/Cohen%27s_kappa)
"""
function cohen_kappa(y_pred, y_true)
    _, tp, tn, fp, fn = TFPN(y_pred, y_true)
    mrg_a = ((tp .+ fn) .* (tp .+ fp)) ./ (tp .+ fn .+ fp .+ tn)
    mrg_b = ((fp .+ tn) .* (fn .+ tn)) ./ (tp .+ fn .+ fp .+ tn)
    expec_agree = (mrg_a .+ mrg_b) ./ (tp .+ fn .+ fp .+ tn)
    obs_agree = (tp .+ tn) ./ (tp .+ fn .+ fp .+ tn)
    cohens_kappa = mean((obs_agree .- expec_agree) ./ (1 .- expec_agree))
    return cohens_kappa
end

"""
    statsfromTFPN(TP, TN, FP, FN)

Computes statistics in case of binary classification or one-vs-all statsitics in case of multiclass classification.

# Arguments:
 - `TP`: true positive values
 - `TN`: true negative values
 - `FP`: false positive values
 - `FN`: false negative values

Return the result stats as a dictionary.
"""
function statsfromTFPN(TP, TN, FP, FN)
    Confusion_Matrix = reshape([TP, FP, FN, TN], 2, 2)
    Precision = TP / (TP + FP + eps(eltype(TP)))
    Recall = TP / (TP + FN + eps(eltype(TP)))
    Specificity = TN / (TN + FP + eps(eltype(TP)))
    F1_score = 2 * (Precision * Recall) / (Precision + Recall + eps(eltype(TP)))
    Accuracy = (TP + TN) / (TP + TN + FP + FN)
    False_alarm_rate = 1 - Specificity
    return Dict(:Confusion_Matrix => Confusion_Matrix,
                :Precision => Precision, :Recall => Recall,
                :Specificity => Specificity, :F1_score => F1_score,
                :Accuracy => Accuracy)
end

"""
    classwise_stats(y_pred, y_true)

Computes statistics for each of the class for multiclass classification based on provided `y_pred` and `y_true`.

Return the result stats as a dictionary.
"""
function classwise_stats(y_pred, y_true)
    _, TP, TN, FP, FN = TFPN(y_pred, y_true)
    ClasswiseStats = Dict()
    for i in 1:size(y_true, 1)
        ClasswiseStats[i] = StatsfromTFPN(TP[i], TN[i], FP[i], FN[i])
    end
    return ClasswiseStats
end

"""
    global_stats(y_pred, y_true; avg_type="macro")

Computes the overall statistics based on provided `y_pred` and `y_true`. `avg_type` allows to specify the type of average to be used while evaluating the stats. Currently, it can take values as "macro" or "micro".

Return the result stats as a dictionary.
"""
function global_stats(y_pred, y_true; avg_type="macro")
    confusion_matrix_, TP, TN, FP, FN = TFPN(y_pred, y_true)
    if avg_type == "macro"
        precision = mean(TP ./ (TP .+ FP .+ eps(eltype(TP))))
        recall = mean(TP ./ (TP .+ FN .+ eps(eltype(TP))))
        f1_score = 2 * precision * recall / (precision + recall + eps(eltype(TP)))
        specificity = mean(TN ./ (TN .+ FP .+ eps(eltype(TP))))
        accuracy = categorical_accuracy(y_pred, y_true)
        false_alarm_rate = 1 - specificity
        return Dict("Confusion_Matrix" => confusion_matrix_,
                "Precision" => precision, "Recall" => recall,
                "Specificity" => specificity, "F1_score" => f1_score,
                "Accuracy" => accuracy, "False_alarm_rate" => false_alarm_rate)
    elseif avg_type == "micro"
        precision = mean(TP) / (mean(TP) + mean(FP))
        recall = mean(TP) / (mean(TP) + mean(FN))
        f1_score = 2 * precision * recall / (precision + recall + eps(eltype(TP)))
        specificity = mean(TN)/ (mean(TN) + mean(FP))
        accuracy = categorical_accuracy(y_pred, y_true)
        false_alarm_rate = 1 - specificity
        return Dict("Confusion_Matrix" => confusion_matrix_,
                "Precision" => precision, "Recall" => recall,
                "Specificity" => specificity, "F1_score" => f1_score,
                "Accuracy" => accuracy, "False_alarm_rate" => false_alarm_rate)
    end
    # TODO: add weighted stats option as above functions
end

# TODO
# Concordance and Discordance

# Receiver Operating Characterstic (ROC) Curve

# AUC


"""
    Bounds.Ellipsoid([T=Float64], N)
    Bounds.Ellipsoid(center::AbstractVector, A::AbstractMatrix)

An `N`-dimensional ellipsoid defined by

```math
(x - center)^T A (x - center) = 1
```

where `size(center) == (N,)` and `size(A) == (N,N)`.

This implementation follows the algorithm presented in Mukherjee et al. (2006).[^2]

[^2]: Pia Mukherjee, et al., 2006, ApJ 638 L51 ["A Nested Sampling Algorithm for Cosmological Model Selection"](https://iopscience.iop.org/article/10.1086/501068)
"""
mutable struct Ellipsoid{T} <: AbstractBoundingSpace{T}
    center::Vector{T}
    A::Matrix{T}
    axes::Matrix{T}
    axlens::Vector{T}
    volume::T
end

function Ellipsoid(center::AbstractVector, A::AbstractMatrix)
    axes, axlens = decompose(A)
    Ellipsoid(center, A, axes, axlens, _volume(A))
end
Ellipsoid(ndim::Integer) = Ellipsoid(Float64, ndim)
Ellipsoid(T::Type, ndim::Integer) = Ellipsoid(zeros(T, ndim), diagm(0 => ones(T, ndim)))
Ellipsoid{T}(center::AbstractVector, A::AbstractMatrix) where {T} = Ellipsoid(T.(center), T.(A))

Base.broadcastable(e::Ellipsoid) = (e,)

Base.ndims(ell::Ellipsoid) = length(ell.center)

# Returns the volume of an ellipsoid given its axes matrix
_volume(A::AbstractMatrix{T}) where {T} = T(volume_prefactor(size(A, 1))) / sqrt(det(A))
volume(ell::Ellipsoid) = ell.volume

# Returns the principal axes
axes(ell::Ellipsoid) = ell.axes

decompose(A::AbstractMatrix) = decompose(Symmetric(A))  # ensure that eigen() always returns real values

function decompose(A::Symmetric)
    E = eigen(A)
    axlens = @. 1 / sqrt(E.values)
    axes = E.vectors * Diagonal(axlens)
    return axes, axlens
end

# axes and axlens
decompose(ell::Ellipsoid) = ell.axes, ell.axlens

# Scale to new volume
function scale!(ell::Ellipsoid, factor)
    # linear factor
    f = factor^(1 / ndims(ell))
    ell.A ./= f^2
    ell.axes .*= f
    ell.axlens .*= f
    ell.volume *= factor
    return ell
end

function endpoints(ell::Ellipsoid)
    axes, axlens = decompose(ell)
        # find major axis
    major_axis = axes[:, argmax(axlens)]
    return ell.center .- major_axis, ell.center .+ major_axis
end

function Base.in(x::AbstractVector, ell::Ellipsoid)
    d = x .- ell.center
    return dot(d, ell.A * d) ≤ 1.0
end

randoffset(rng::AbstractRNG, ell::Ellipsoid{T}) where {T} = axes(ell) * randball(rng, T, ndims(ell))
Base.rand(rng::AbstractRNG, ell::Ellipsoid) = ell.center .+ randoffset(rng, ell)

fit(E::Type{<:Ellipsoid}, x::AbstractMatrix{S}; pointvol = 0) where {S} = fit(E{float(S)}, x; pointvol = pointvol)

function fit(E::Type{<:Ellipsoid{R}}, x::AbstractMatrix{S}; pointvol = 0) where {R,S}
    T = float(promote_type(R, S))
    x = T.(x)
    ndim, npoints = size(x)

    # single element is an n-sphere with pointvol volume
    if npoints == 1
        pointvol > 0 || error("Cannot compute bounding ellipsoid with one point without a valid pointvol (got $pointvol)")
        d = log(pointvol) - log(volume_prefactor(ndim))
        r = exp(d / ndim)
        A = diagm(0 => fill(1 / r^2, ndim))
        return Ellipsoid(vec(x), A)
    end
    # get estimators
    center, cov = mean_and_cov(x, 2)
    delta = x .- center
    # Covariance is smaller than r^2 by a factor of 1/(n+2)
    cov .*= ndim + 2
    # Ensure cov is nonsingular
    targetprod = (npoints * pointvol / volume_prefactor(ndim))^2
    make_eigvals_positive!(cov, targetprod)

    # get transformation matrix. Note: use pinv to avoid error when cov is all zeros
    A = pinv(cov)

    # calculate expansion factor necessary to bound each points
    f = diag(delta' * (A * delta))
    fmax = maximum(f)

    # try to avoid round-off errors s.t. furthest point obeys
    # x^T A x < 1 - √eps
    flex = 1 - sqrt(eps(T))
    if fmax > flex
        A .*= flex / fmax
    end

    ell = E(vec(center), A)

    if pointvol > 0
        minvol = npoints * pointvol
        vol = volume(ell)
        vol < minvol && scale!(ell, minvol / vol)
    end

    return ell
end

# ---------------------------------------------
# Helper functions

"""
    volume_prefactor(::Integer)

Volume constant for an n-dimensional sphere:

for n even:      (2pi)^(n    /2) / (2 * 4 * ... * n)
for n odd :  2 * (2pi)^((n-1)/2) / (1 * 3 * ... * n)
"""
function volume_prefactor(n::Integer)
    f, range = iseven(n) ? (1.0, 2:2:n) : (2.0, 3:2:n)
    for i in range
        f *= 2π / i
    end
    return f
end

# sample N samples from unit D-dimensional ball
randball(T::Type, D::Integer, N::Integer) = randball(GLOBAL_RNG, T, D, N)
function randball(rng::AbstractRNG, T::Type, D::Integer, N::Integer)
    z = randn(rng, T, D, N)
    z .*= rand(rng, T, 1, N).^(1 ./ D) ./ sqrt.(sum(p->p^2, z, dims = 1))
    return z
end

# sample from unit D-dimensional ball
randball(T::Type, D::Integer) = randball(GLOBAL_RNG, T, D)
function randball(rng::AbstractRNG, T::Type, D::Integer)
    z = randn(rng, T, D)
    z .*= rand(rng)^(1 / D) / sqrt(sum(p->p^2, z))
    return z
end

function make_eigvals_positive!(cov::AbstractMatrix, targetprod)
    E = eigen(cov)
    mask = E.values .< 1e-10
    if any(mask)
        nzprod = prod(E.values[.!mask])
        nzeros = count(mask)
        E.values[mask] .= (targetprod / nzprod)^(1 / nzeros)
        cov .= E.vectors * Diagonal(E.values) / E.vectors
    end
    return cov
end

make_eigvals_positive(cov::AbstractMatrix, targetprod) = make_eigvals_positive!(copy(cov), targetprod)


# Benchmark Optim.jl on logistic regression problems

using Optim, LineSearches
using LogisticOptTools
using Plots

const LOT = LogisticOptTools

# Choose dataset to load
DATASET = "covtype.libsvm.binary.bz2"
# Choose penalty with given coefficient λ
PENALTY = LOT.L2Penalty(1e-2)

if !isfile(DATASET)
    println("Download dataset from $LIBSVM_URL")
    download("$LIBSVM_URL/$DATASET", DATASET)
end

# Some plots utilities
"Display evolution of gradients along iteration."
function display_grad(results::Optim.OptimizationResults,
                      label::String,
                      p::Plots.Plot)
    exectime = [t.metadata["time"] for t in results.trace]
    exectime .-= exectime[1]
    grad = [t.g_norm for t in results.trace]
    plot!(p, exectime, log10.(grad), marker=true, label=label, markersize=1.)
end

"Display evolution of functions along iteration."
function display_func(results::Optim.OptimizationResults,
                      f♯,
                      label::String,
                      p::Plots.Plot)
    exectime = [t.metadata["time"] for t in results.trace]
    exectime .-= exectime[1]
    values = [t.value for t in results.trace]
    values .-= f♯
    plot!(p, exectime, log10.(values), marker=true, label=label, markersize=1.)
end

"Benchmark algorithms from Optim.jl"
function benchmark(algos, options, logreg)
    p = LOT.nfeatures(logreg)
    x = zeros(p)
    g = zeros(p)
    f, g!, _, _ = LOT.generate_callbacks(logreg)
    # Activate precompilation
    f(x)
    g!(g, x)

    results = []
    for algo in algos
        res = Optim.optimize(f, g!, x, algo, options)
        push!(results, res)
    end
    return results
end

bfgs1 = BFGS(alphaguess=InitialHagerZhang(α0=1.0),
             linesearch=LineSearches.HagerZhang())
bfgs2 = BFGS(alphaguess=InitialStatic(alpha=1.0),
             linesearch=LineSearches.MoreThuente())
cg = ConjugateGradient()

options = Optim.Options(iterations=100, store_trace=true, show_trace=true,
                        g_tol=1e-5, allow_f_increases=true)

# Load data
dataset = LOT.LogitData(DATASET, scale_data=true)
# Build logistic regression
logreg = LOT.LogisticRegressor(dataset.X, dataset.y, penalty=PENALTY, fit_intercept=true)
results = benchmark([bfgs1, bfgs2, cg], options, logreg)

# Plot evolution of gradient
pg = plot(markershape=:rect)
display_grad.(results, ["Optim BFGS HZ", "Optim BFGS MT", "Optim CG"], Ref(pg))
xlabel!(pg, "Time (s)")
ylabel!(pg, "Gradient (log scale)")

# Plot evolution of function
pf = plot(markershape=:rect)
# Get optimum
f♯ = minimum([res.minimum for res in results])
display_func.(results, f♯, ["Optim BFGS HZ", "Optim BFGS MT", "Optim CG"], Ref(pf))
xlabel!(pf, "Time (s)")
ylabel!(pf, "log(f - fopt) ")


## Packages
using Plots

## Files
include("FEM_aiyagari.jl")
include("NewtonRoot_mult.jl")

############ Plot basis functions #################
plot(x -> ψ(x, [0, 1, 3, 6],1), xlim=(0, 6), label = "ψ_1(x)")
plot!(x -> ψ(x, [0, 1, 3, 6], 2), xlim=(0, 6), label = "ψ_2(x)")
plot!(x -> ψ(x, [0, 1, 3, 6], 3), xlim=(0, 6), label = "ψ_3(x)")
plot!(x -> ψ(x, [0, 1, 3, 6], 4), xlim=(0, 6), label = "ψ_4(x)")
savefig("figs/linear_basis.png")

###################### Aiyagari ###########################
hh = Household()
θ_init = [hh.aGrid; hh.aGrid .+ 0.5]
θ_est = NewtonRoot_mult(θ->G(hh,θ),θ_init)

plot(a->a,xlim = (0.0,6.1),label = "45 degree", color = :black, linestyle = :dash,legend =:topleft)
plot!(a->get_apol(hh,a,hh.lGrid[1],θ_est),xlim = (0.0,6.1),label = "low")
plot!(a->get_apol(hh,a,hh.lGrid[2],θ_est),xlim = (0.0,6.1),label = "high")
scatter!(hh.aGrid,θ_est[1:8],label = "coefficient of basis (Low)")
scatter!(hh.aGrid,θ_est[9:16],label = "coefficient of basis (High)")
savefig("figs/aiyagari.png")


@syms 𝑥
@polyvar 𝑦

function poly(eq, v::Pair)
    α, β = value(first(v)), last(v)
    terms = collect_powers(eq, α)

    if !all(isone, denominator.(collect(keys(terms))))
        error("fractional power not supported")
    end

    sum([val*β^numerator(k) for (k,val) in terms]; init=0)
end

function poly(eq, β)
    xs = get_variables(eq)

    if length(xs) > 1
        error("more than one implicit variable.")
    elseif length(xs) == 0
        return nothing
    else
        return poly(eq, xs[1] => β)
    end
end

poly(eq) = poly(eq, 𝑦)

function sym(p::AbstractPolynomialLike, v::Pair)
    β, α = first(v), value(last(v))
    sum([α^maxdegree(t,β)*c for (t,c) in zip(terms(p), coefficients(p))]; init=0)
end

sym(p::AbstractPolynomialLike, α) = sym(p, var(p) => α)
sym(p::AbstractPolynomialLike) = sym(p, var(p) => 𝑥)

###############################################################################

# pox (power-of-x) is a symbolic function to keep track of the powers of x
# pox(k,n) means k*x^n
@syms pox(k, n)

is_pox(x) = istree(x) && operation(x)==pox
is_not_pox(x) = !is_pox(x)

get_coef(p) = is_pox(p) ? arguments(p)[1] : p
get_power(p) = is_pox(p) ? arguments(p)[2] : 0

replace_x(eq, x) = substitute(eq, Dict(x => pox(1,1)))

iscomplex(x) = x isa Complex

count_rule1 = @rule ^(pox(~k, ~n1), ~n2) => isequal(~k,1) ? pox(1, ~n1 * ~n2) : pox(^(~k,~n2), ~n1 * ~n2)
count_rule2 = @rule pox(~k1, ~n1) * pox(~k2, ~n2) => pox(~k1 * ~k2, ~n1 + ~n2)
count_rule3 = @acrule pox(~k, ~n) * ~u::is_not_pox => pox(~k * ~u, ~n)

"""
    collect_powers separates the powers of x in eq (a polynomial) and returns
    a dictionary of power => term
"""
function collect_powers(eq, x)
    eq = expand(expand_derivatives(eq))
    eq = replace_x(eq, x)
    #eq = Prewalk(PassThrough(count_rule1))(eq)
    eq = Fixpoint(Prewalk(PassThrough(Chain([count_rule1, count_rule2, count_rule3]))))(eq)

    if !istree(eq)
        return Dict{Any, Any}(0 => eq)
    elseif is_pox(eq)
        return Dict{Any, Any}(get_power(eq) => get_coef(eq))
    else
        eqs = Dict{Any, Any}()
        for term in arguments(eq)
            n = get_power(term)
            if haskey(eqs, n)
                eqs[n] = eqs[n] + get_coef(term)
            else
                eqs[n] = get_coef(term)
            end
        end

        return eqs
    end
end


"""
    Xtal._is_linearly_independent(vecs::AbstractMatrix{<:Number}) -> Bool
    Xtal._is_linearly_independent(vecs::AbstractVector{<:Number}...) -> Bool

Determines whether a set of vectors is linearly independent.

This function always returns `false` if the first dimension of the matrix is less than the second.
"""
function is_linearly_independent(vecs::AbstractMatrix{<:Number})
    size(vecs)[1] < size(vecs)[2] || return false
    return LinearAlgebra.rank(vecs) == minimum(size(vecs))
end

# Same thing but for sets of vectors
function is_linearly_independent(vecs::AbstractVector{<:Number}...)
    return is_linearly_independent(hcat(vecs...))
end

#=
"""
    _to_vectors(a::AbstractArray{T,N}) where {T,N} -> Vector{...Vector{T}}

Converts an array to a nest of vectors within vectors.
"""
function _to_vectors(a::AbstractArray{T,N}) where {T,N}

end
=#


#=

   The Euro coin problem.

   From Think Bayes, page 33ff
   """
   A statistical statement appeared in "The Guardian" on Friday January 4, 2002:
      When spun on edge 250 times, a Belgian one-euro coin
      came up heads 140 times and tails 110. 'It looks very
      suspicious to me,' said Barry Blight, a statistics lecturer
      at the London School of Economics. 'If the coin were
      unbiased, the chance of getting a result as extreme as
      that would be less than 7%.'

   But do these data give evidence that the coin is biased rather than fair?
   """

   Continues on page 41:
   """
   Exercise 4.1. Suppose that instead of observing coin tosses directly, you measure
   the outcome using an instrument that is not always correct. Specifically, suppose
   there is a probability y that an actual heads is reported as tails, or actual tails re-
   ported as heads.

   Write a class that estimates the bias of a coin given a series of outcomes and the
   value of y .
   How does the spread of the posterior distribution depend on y ?
   """

   mean(prob): 0.5213151098904925
   Distributions of variable probLt0_5 (num:0)
   0.00000 =>    7020  (0.702000)
   1.00000 =>    2980  (0.298000)

   Distributions of variable error (num:0)
   0.00000 =>    7051  (0.705100)
   1.00000 =>    2949  (0.294900)

   Cf euro_coin_problem.jl
   Cf ~/blog/euro_coin_unreliable_measurements.blog
      ~/webppl/euro_coin_unreliable_measurements.wppl
=#

using Turing, StatsPlots, DataFrames
include("jl_utils.jl")

@model function euro_coin_problem_unreliable_measurements()
    n = 250

    # Probability of throwing head
    prob ~ Beta(2,2)

    # We measure incorrect with probability 0.2
    error ~ flip(0.2)

    throwCoin ~ filldist(flip(prob),n)

    coin = tzeros(n)
    for i in 1:n
        coin[i] ~ Dirac(error == true ? 1-throwCoin[i] : throwCoin[i])
    end

    sum250 ~ Dirac(sum([coin[i] == true ? 1 : 0 for i in 1:n]))
    true ~ Dirac(sum250 == 140)
    probLt0_5 ~ Dirac(prob < 0.5)

end

model = euro_coin_problem_unreliable_measurements()

num_chains = 4

# chns = sample(model, Prior(), 10_000)
# chns = sample(model, MH(), 10_000)
chns = sample(model, PG(5), 10_000)
# chns = sample(model, SMC(), 10_000)
# chns = sample(model, IS(), 10_000)

display(chns)
# display(plot(chns))

println("mean(prob): ", mean(chns[:prob]))
show_var_dist_pct(chns, :probLt0_5)
show_var_dist_pct(chns, :error)


# ### m8.4t

using Pkg, DrWatson

@quickactivate "StatisticalRethinkingTuring"
using Turing
using StatisticalRethinking
Turing.setprogress!(false)

# Can't really set a Uniform[-Inf,Inf] on σ

# Turing model
@model ppl8_4(y) = begin
    α₁ ~ Uniform(-3000, 1000)
    α₂ ~ Uniform(-1000, 3000)
    σ ~ truncated(Cauchy(0,1), 0, Inf)

    y .~ Normal(α₁ + α₂, σ)
end

# Observations

y = rand(Normal(0,1), 100);

# Sample

m8_4t = ppl8_4(y)
nchains = 4; sampler = NUTS(0.65); nsamples=2000
chns8_4t = mapreduce(c -> sample(m8_4t, sampler, nsamples), chainscat, 1:nchains)

# Results rethinking

m8_4rethinking = "
         mean      sd     5.5%   94.5% n_eff Rhat
 a1    -861.15 558.17 -1841.89  -31.04     7 1.43
 a2     861.26 558.17    31.31 1842.00     7 1.43
 sigma    0.97   0.07     0.89    1.09     9 1.17
";

# Describe the posterior samples

chns |> display

# End of `08/m8.4t.jl`


# # Approximate Inference on Sparse GPs using VI

# Loading the necessary packages and setting seed.

using AbstractGPs, Plots, Random
Random.seed!(1234);

# Loading toy regression
# [dataset](https://github.com/GPflow/docs/blob/master/doc/source/notebooks/basics/data/regression_1D.csv)
# taken from GPFlow examples.

x = [0.8658165855998895, 0.6661700880180962, 0.8049218148148531, 0.7714303440386239,
    0.14790478354654835, 0.8666105548197428, 0.007044577166530286, 0.026331737288148638,
    0.17188596617099916, 0.8897812990554013, 0.24323574561119998, 0.028590102134105955];
y = [1.5255314337144372, 3.6434202968230003, 3.010885733911661, 3.774442382979625,
    3.3687639483798324, 1.5506452040608503, 3.790447985799683, 3.8689707574953,
    3.4933565751758713, 1.4284538820635841, 3.8715350915692364, 3.7045949061144983];
scatter(x, y, xlabel="x", ylabel="y")

# Split the observations into train and test set.

(x_train, y_train) = (x[begin:8], y[begin:8]);
(x_test, y_test) = (x[9:end], y[9:end]);

# Instantiate the kernel.

k = Matern52Kernel()

# Instantiate a Gaussian Process with the given kernel `k`.

f = GP(k);

# Instantiate a `FiniteGP`, a finite dimentional projection at the inputs of the dataset
# observed under Gaussian Noise with $\sigma = 0.001$ .

fx = f(x_train, 0.001);

# Calculating the exact posterior over `f` given `y`. The GP's kernel currently has some
# arbitrary fixed parameters.

p_fx = posterior(fx, y_train)

logpdf(p_fx(x_test), y_test)

# ## Variational Inference

# Sanity check for the Evidence Lower BOund (ELBO) implemented according to
# M. K. Titsias's _Variational learning of inducing variables in sparse Gaussian processes_

elbo(fx, y_train, f(rand(7)))

# We will be using `Optim.jl` package's `LBFGS` algorithm to maximize the given ELBO.

using Optim

# Create a helper function for optimization. It takes in the parameters (both variational
# and model parameters) and returns the negative of the ELBO.

function optim_function(params; x=x_train, y=y_train)
    kernel = ScaledKernel(
        transform(
            Matern52Kernel(),
            ScaleTransform(exp.(params[1]))
            ),
        exp.(params[2])
    )
    f = GP(kernel)
    fx = f(x, 0.1)
    return -elbo(fx, y, f(params[3:end]))
end

# Initialize the parameters (Varitational and Model parameters)
x0 = rand(7)

# Sanity check for the helper function. We intend to minimize the result of this function.

-optim_function(x0)

# Optimize using `Optim.jl` package's `LBFGS` algorithm.

opt = optimize(optim_function, x0, LBFGS())

# Optimal parameters:

opt.minimizer

# ELBO with optimal parameters. We see that there is significant improvement when compared
# to the initial parameters.

-optim_function(opt.minimizer; x=x_test, y=y_test)

# Visualize the posterior.

opt_kernel = ScaledKernel(
    transform(
        Matern52Kernel(),
        ScaleTransform(exp.(opt.minimizer[1]))
        ),
    exp.(opt.minimizer[2])
)
opt_f = GP(opt_kernel)
opt_fx = opt_f(x_train, 0.1)
ap = approx_posterior(VFE(), opt_fx, y_train, opt_f(opt.minimizer[3:end]));

# Average log-marginal-probability of data with posterior kernel parameter samples sampled
# using ESS. We can observe that there is significant improvement over exact posterior with
# default kernel parameters.

logpdf(ap(x_test), y_test)

plt = plot(ap, 0:0.001:1, label="Approx Posterior")
plot!(plt, p_fx, 0:0.001:1, label="Exact Posterior")
scatter!(
    plt,
    opt.minimizer[3:end],
    mean(rand(ap(opt.minimizer[3:end], 0.1), 100), dims=2),
    label="Pseudo-points"
)
scatter!(plt, x_train, y_train, label="Train Data")
scatter!(plt, x_test, y_test, label="Test Data")
plt


using LinearAlgebra
using LoopVectorization
using BenchmarkTools
using StaticArrays
using RecursiveFactorization
function cholfact!(A)
    # fills in lower triangular portion
    # total flops:
    #   2*n*(n+1)*(n-1)/6 + n + n*(n+1)/2
    # total assignments
    #   n*(n+1)*(n-1)/6 + (n+1)/2
    n = size(A,1)
    for j = 1:n

        # (n+1)*(n-1)/6
        for k = 1:j-1  # loop over columns
            for i = j:n
                @inbounds A[i,j] -= A[i,k] * A[j,k]
            end
        end
        ajj = sqrt(A[j,j])  # 1 flop

        # (n+1)/2 flops
        for i = j:n
            @inbounds A[i,j] /= ajj
        end
    end
    return A
end

function avxchol!(A)
    n = size(A,1)
    for j = 1:n

        # (n+1)*(n-1)/6
        for k = 1:j-1  # loop over columns
            @turbo for i = j:n
                A[i,j] -= A[i,k] * A[j,k]
            end
        end
        ajj = sqrt(A[j,j])  # 1 flop

        # (n+1)/2 flops
        @turbo for i = j:n
            A[i,j] /= ajj
        end
    end
end

function _chol_expr(j,n)
    quote
        for k = 1:$j-1  # loop over columns
            @turbo for i = $j:$n
                A[i,$j] -= A[i,k] * A[$j,k]
            end
        end
        ajj = sqrt(A[$j,$j])

        @turbo for i = $j:$n
            A[i,$j] /= ajj
        end
    end
end

@generated function avxchol_unrolled!(A, ::Val{n}) where n
    expr = [_chol_expr(j,n) for j = 1:n]
    quote
        $(Expr(:block, expr...))
    end
end

n = 50
A = randn(n,n); A = A'A;
C = zero(A)
println("Raw Loop")
@btime begin
    $C .= $A
    cholfact!($C)
end

C .= A
avxchol!(C)
LowerTriangular(C) ≈ cholesky(A).L
println("avxchol")
@btime begin
    $C .= $A
    cholfact!($C)
end

println("LAPACK")
@btime begin
    $C .= $A
    LAPACK.potrf!('L', $C)
end

println("Static Arrays")
SM = similar_type(SMatrix{n,n,Float64})
@btime cholesky($(SM(A)))
@btime begin
    Cs = $SM($A)
    $C .= cholesky(Cs).L
end

p = zeros(Int,n)
lu = copy(A)
RecursiveFactorization.lu!(lu, p)
println("RF lu")
@btime RecursiveFactorization.lu!($lu, $p)


abstract type AbstractGrid end

export OneDGrid, TwoDGrid

"""
    TwoDGrid( dimx, nx, dimy, ny)

Generate a cartesians mesh on rectangle `dimx`x `dimy` with `nx` x `ny` points

- `nx` : indices are in [1:nx]
- `ny` : indices are in [1:ny]
- `dimx = xmax - xmin`
- `dimy = ymax - ymin`
- `x, y` : node positions
- `dx, dy` : step size
"""
struct TwoDGrid <: AbstractGrid
    nx::Int
    ny::Int
    xmin::Float64
    xmax::Float64
    ymin::Float64
    ymax::Float64
    dimx::Float64
    dimy::Float64
    x::Vector{Float64}
    y::Vector{Float64}
    dx::Float64
    dy::Float64

    function TwoDGrid(xmin, xmax, nx::Int, ymin, ymax, ny::Int)
        dimx = xmax - xmin
        dimy = ymax - ymin

        x = collect(LinRange(0, dimx, nx + 1))
        y = collect(LinRange(0, dimy, ny + 1))

        dx = dimx / nx
        dy = dimy / ny

        return new(nx, ny, xmin, xmax, ymin, ymax, dimx, dimy, x, y, dx, dy)
    end
end

TwoDGrid(dimx, nx::Int, dimy, ny::Int) = TwoDGrid(0.0, dimx, nx, 0.0, dimy, ny)

"""
    TwoDGrid( xmin, xmax, nx, ymin, ymax, ny )

Simple structure to store mesh data from 2 dimensions
"""
struct OneDGrid <: AbstractGrid
    nx::Int
    xmin::Float64
    xmax::Float64
    dimx::Float64
    dx::Float64
    x::Vector{Float64}

    function OneDGrid(xmin, xmax, nx::Int)
        dimx = xmax - xmin
        dx = dimx / (nx - 1)
        x = collect(LinRange(0, dimx, nx + 1))

        return new(nx, xmin, xmax, dimx, dx, x)
    end
end

export get_x

"""
    get_x( mesh, i )

Get position
"""
function get_x(m::OneDGrid, i)
    return m.xmin + (i - 1) * m.dx
end

"""
    get_cell_and_offset( mesh, x )

Get cell and offset

We compute the cell indices where the particle is and its relative
normalized position inside the cell

"""
function get_cell_and_offset(m::OneDGrid, x)
    cell = floor(Int64, ((x - m.xmin) / m.Lx) * m.nx) + 1
    offset = (x - get_x(m, cell)) / dx

    return cell, offset
end


# ---
# title: 1362. Closest Divisors
# id: problem1362
# author: Tian Jun
# date: 2020-10-31
# difficulty: Medium
# categories: Math
# link: <https://leetcode.com/problems/closest-divisors/description/>
# hidden: true
# ---
#
# Given an integer `num`, find the closest two integers in absolute difference
# whose product equals `num + 1` or `num + 2`.
#
# Return the two integers in any order.
#
#
#
# **Example 1:**
#
#
#
#     Input: num = 8
#     Output: [3,3]
#     Explanation: For num + 1 = 9, the closest divisors are 3 & 3, for num + 2 = 10, the closest divisors are 2 & 5, hence 3 & 3 is chosen.
#
#
# **Example 2:**
#
#
#
#     Input: num = 123
#     Output: [5,25]
#
#
# **Example 3:**
#
#
#
#     Input: num = 999
#     Output: [40,25]
#
#
#
#
# **Constraints:**
#
#   * `1 <= num <= 10^9`
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


"""
	transferoperator_exact(t::AbstractTriangulation)

Compute the transfer operator from a triangulation.
"""
function transferoperator_exact(t::AbstractTriangulation)
    n_simplices = size(t.simplex_inds, 1)
    dim = size(t.points, 2)

    #=
    Mismatch for the markovity of the Markov matrix is at most δ if ϵ tolerance
	as below
    =#
    δ::Float64 = 1/10^5
    ϵ::Float64 = δ/n_simplices

    #=
    Tolerance for similary of convex expansion coefficients of simplex vertices in
    simplexintersection function.
    =#
    convex_params_tol::Float64 = 1/10^12

    TO = zeros(Float64, n_simplices, n_simplices) # intersecting volumes

    for i in 1:n_simplices
        imvol = t.volumes_im[i]
        for j in 1:n_simplices
            vol = t.volumes[j]
            if vol * imvol > 0 && (vol/imvol) > ϵ
                # Intersecting volume between these two simplices
                TO[i, j] = simplexintersection(
                                transpose(t.points[t.simplex_inds[j, :], :]),
                                transpose(t.impoints[t.simplex_inds[i, :], :])
                            ) / imvol
            end
        end
    end
    return ExactSimplexTransferOperator(TO)
end

"""
    transferoperator_exact_p(t::AbstractTriangulation)

Estimate the transfer operator from a triangulation of a state space.
Will run in parallel if `nprocs() > 1`.
"""
function transferoperator_exact_p(t::AbstractTriangulation)

    n_simplices = size(t.simplex_inds, 1)
    dim = size(t.points, 2)

    #=
    Mismatch for the markovity of the Markov matrix is at most δ if ϵ
    tolerance as below.
    =#
    δ::Float64 = 1/10^5
    ϵ::Float64 = δ/n_simplices

    #=
    Tolerance for similary of convex expansion coefficients of simplex vertices
    in simplexintersection function.
    =#
    convex_params_tol::Float64 = 1/10^12

    TO = SharedArray{Float64}(n_simplices, n_simplices)

    @sync @distributed for i in 1:n_simplices
        imvol = t.volumes_im[i]
        for j in 1:n_simplices
            vol = t.volumes[j]
            if vol * imvol > 0 && (vol/imvol) > ϵ
                # Intersecting volume between these two simplices
                TO[i, j] = simplexintersection(
                                transpose(t.points[t.simplex_inds[j, :], :]),
                                transpose(t.impoints[t.simplex_inds[i, :], :])
                            ) / imvol
            end
        end
    end

    return ExactSimplexTransferOperator(Array(TO))
end

transferoperator_triang_exact = transferoperator_exact
export transferoperator_triang_exact


# load required modules
using CategoricalArrays
using CSV
using DataFrames
using Distributions
using Random
using StatsBase

# set seed for reproducible sequence of numbers
Random.seed!(42);

# create a sample dataframe
df = DataFrame(
    Dict(
        :x1 => rand(Normal(10, 1), 100),
        :x2 => [sample(["Good", "Better", "Best"], pweights([0.2, 0.5, 0.3])) for i=1:100],
        :x3 => rand(Pareto(10, 1), 100)
    )
)

# get type of df
typeof(df)

# access 1 row
df[1, :]

# access 1 and 2 row
df[[1, 2],:]

# access x1 column vector
df[:, :x1]

# access x1 column dataframe
df[:,[:x1]]

# access column x1 and x2
df[:, [:x1, :x2]]

# slicing dataframe by indeces
df[1:2, 1:2]

# slicing dataframe by column names
df[1:2, [:x1, :x2]]

# select only x2 == "Better"
mask = [df[i,:x2] == "Better" for i=1:nrow(df)]
df[mask, :]

# select only x2 == "Better" using bit vector
df[df[:, :x2] .== "Better", :]

# generate random string vector with missing values
arr = [isodd(i) ?
    sample(["Good", "Better", "Best"], pweights([0.2, 0.4, 0.3])) :
    missing for i = 1:nrow(df)]

# create categorical array
carr = categorical(arr)

levels(carr)

CSV.write("../../data/interim/df.csv", df)

CSV.read("../../data/interim/df.csv", DataFrame)

# obtain dataframe dimensions
size(df)

# obtain dataframe row number
nrow(df)

# obtain dataframe column number
ncol(df)

# access first 5 rows
first(df, 5)

# access last 5 rows
last(df, 5)

# describe dataframe
describe(df)

# describe dataframe column
describe(df[:, :x1])

# describe dataframe column
describe(df[:, [:x1]])

# sort dataframe by x1 ascending
sort(df, :x1)

# sort dataframe by x1 descending
sort(df, :x1, rev=true)

# sort by x1 asc and x2 desc
sort(df, [:x1, :x2], rev=[false, true])

# obtain count group by x2
combine(groupby(df, :x2), nrow)

# obtain sum group by x2
combine(groupby(df, :x2), :x1 => sum, :x3 => sum)

# obtain aggregates of x1 column
combine(df, :x1 => mean, :x1 => median, :x1 => std)


#=
   Logistic regresiion
   Inspired by WebPPL's examples/logisticRegression.wppl

   Summary Statistics (for PG(15))
      parameters       mean       std   naive_se      mcse         ess      rhat   ess_per_sec
          Symbol    Float64   Float64    Float64   Float64     Float64   Float64       Float64

               m     1.0844    0.5870     0.0059    0.0077   6361.4279    1.0009      153.4390
               b     0.1315    0.9292     0.0093    0.0111   6431.4331    0.9999      155.1275
           sigma     0.8779    0.8513     0.0085    0.0107   6452.7250    0.9999      155.6411
            y[1]   -10.7253    6.0611     0.0606    0.0801   6408.9867    1.0008      154.5861
            y[2]    -5.3599    3.2295     0.0323    0.0439   6465.7579    1.0007      155.9555
            y[3]     2.5165    1.7809     0.0178    0.0200   6562.8112    1.0010      158.2964
            y[4]     6.6892    3.7534     0.0375    0.0454   6516.1303    1.0008      157.1705
            y[5]    11.0158    5.9711     0.0597    0.0746   6263.6413    1.0010      151.0804
  labels_post[1]     0.0163    0.1266     0.0013    0.0013   8339.5233    0.9999      201.1511
  labels_post[2]     0.0552    0.2284     0.0023    0.0026   8486.3592    1.0001      204.6928
  labels_post[3]     0.8459    0.3611     0.0036    0.0041   8013.9274    1.0001      193.2977
  labels_post[4]     0.9707    0.1687     0.0017    0.0018   8543.2209    1.0002      206.0643
  labels_post[5]     0.9873    0.1120     0.0011    0.0012   9349.4013    0.9999      225.5096

  The labels_post are the posterior predictive and are fairly close to labels values: [0, 0, 1, 1, 1]

=#

using Turing, StatsPlots, DataFrames

include("jl_utils.jl")

@model function logistic_regression(x, labels)
    m ~ Normal(0, 1)
    b ~ Normal(0, 1)
    sigma ~ Gamma(1, 1)

    n = length(x)
    y = tzeros(n)
    sigmoid = tzeros(n)
    for i in 1:n
        y[i] ~ Normal(m * x[i] + b, sigma)
        sigmoid[i] = 1 / (1 + exp(-1 * y[i]))
        labels[i] ~ Bernoulli(sigmoid[i])
    end

    # Post predictive of labels
    labels_post = tzeros(n)
    for i in 1:n
        labels_post[i] ~ Bernoulli(sigmoid[i])
    end
end

xs = [-10, -5, 2, 6, 10]
labels = [0, 0, 1, 1, 1]
model = logistic_regression(xs,labels)

# chns = sample(model, Prior(), 10_000)
# chns = sample(model, MH(), 10_000)
chns = sample(model, PG(15), 10_000)
# chns = sample(model, SMC(), 10_000)
# chns = sample(model, IS(), 10_000)

display(chns)


module _K2En
export K2En
using Statistics: std
    """
        K2, Ci = K2En(Sig)

    Returns the Kolmogorov entropy estimates `K2` and the correlation
    integrals `Ci` for `m` = [1,2] estimated from the data sequence `Sig`
    using the default parameters: embedding dimension = 2, time delay = 1,
    r = 0.2*SD(`Sig`), logarithm = natural

        K2, Ci = K2En(Sig::AbstractArray{T,1} where T<:Real; m::Int=2, tau::Int=1, r::Real=0.2*std(Sig,corrected=false), Logx::Real=exp(1))

    Returns the Kolmogorov entropy estimates `K2` for dimensions = [1,...,`m`]
    estimated from the data sequence `Sig` using the 'keyword' arguments:

    # Arguments:
    `m`     - Embedding Dimension, a positive integer\n
    `tau`   - Time Delay, a positive integer\n
    `r`     - Radius, a positive scalar \n
    `Logx`  - Logarithm base, a positive scalar\n

    # See also `DistEn`, `XK2En`, `MSEn`

    # References:
        [1] Peter Grassberger and Itamar Procaccia,
            "Estimation of the Kolmogorov entropy from a chaotic signal."
            Physical review A 28.4 (1983): 2591.

        [2] Lin Gao, Jue Wang  and Longwei Chen
            "Event-related desynchronization and synchronization
            quantification in motor-related EEG by Kolmogorov entropy"
            J Neural Eng. 2013 Jun;10(3):03602

    """
    function K2En(Sig::AbstractArray{T,1} where T<:Real; m::Int=2, tau::Int=1, r::Real=0.2*std(Sig,corrected=false), Logx::Real=exp(1))

    N = size(Sig)[1]
    (N>10) ? nothing : error("Sig:   must be a numeric vector")
    (m > 0) ? nothing : error("m:     must be an integer > 0")
    (tau > 0) ? nothing : error("tau:   must be an integer > 0")
    (r>0) ? nothing :  error("r:     must be a positive scalar > 0")
    (Logx>0) ? nothing :  error("Logx:     must be a positive number > 0")

    m = m+1
    Zm = zeros(N,m)
    Ci = zeros(m)
    for n = 1:m
        N2 = N-(n-1)*tau
        Zm[1:N2,n] = Sig[(n-1)*tau + 1:N]
        Norm = Inf*ones(N2-1,N2-1)
        for k = 1:N2-1
            Temp = repeat(transpose(Zm[k,1:n]),N2-k,1) - Zm[k+1:N2,1:n]
            Norm[k,k:N2-1] = sqrt.(sum(Temp.^2, dims=2))
        end
        Ci[n] = 2*sum(Norm .< r)/(N2*(N2-1))
    end

    K2 = log.(Logx, Ci[1:m-1]./Ci[2:m])/tau
    K2[isinf.(K2)] .= NaN

    return K2, Ci
    end
end

"""
Copyright 2021 Matthew W. Flood, EntropyHub

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

For Terms of Use see https://github.com/MattWillFlood/EntropyHub
"""


#using  NearestNeighbors: Metric
#import Distances: evaluate
using StatsBase: mode

#evaluate(dist::dwt, a, b) = dtw_distance(a, b, dist.w)

"""
   `dtw_distance(a, b, w)` is the basic dynamic time wraping function.
where `a` & `b` are the time series matrices and `w` is the percentage
of window for warping.
"""
function dtw_distance(a, b, w)
    l_a, l_b = length(a), length(b)
    FloatMax = maxintfloat(Float64)
    if w <= 0
        band = max(l_b, l_a)
    else
        band = floor(Int, w*max(l_b, l_a))
    end
    M = zeros(l_a+1,l_b+1)
    M[1, 2:end] .= FloatMax
    M[2:end, 1] .= FloatMax
    for k=2:l_a
        M[k, 2:end] =  (a .-  b[k]).^2
    end
    for i=2:l_a+1
        jstart = max(2, i-band)
        jstop = min(l_b+1, i+band+1)
        idx_inf_left = i-band-1

        if idx_inf_left >= 1
            M[i, idx_inf_left] = FloatMax
        end
        for j=jstart:jstop
            im = i-1
            jm = j-1
            M[i,j] = M[i,j] + min(min(M[im,j], M[i,jm]), M[im,jm])
        end
        if jstop < l_b + 1
            M[i,jstop] = FloatMax
        end
    end
    return M[l_a+1,l_b+1]
end

function Predict_new(m, X::Array, Y::Array, yplane::Array)
    k = m.n_neighbors
    n_train, serieslength = size(X)
    n_test = size(Y, 1)
    y_pred = zeros(n_test)
    DistanceMatrix = zeros(n_test, n_train)
    for i=1:n_test
        for j=1:n_train
            DistanceMatrix[i, j] = dtw_distance(X[j,:],Y[i,:], m.metric_params[1])
        end
    end
    index = select_sort(DistanceMatrix, k)
    y_index = yplane[Int.(index)]
    for i=1:n_test
        y_pred[i] = mode(y_index[i, :])
    end
    return y_pred, DistanceMatrix
end

function select_sort(A, k) # some times we get consicative indx
    n_test, n_train = size(A)
    index = zeros(n_test, k)
    for l=1:n_test
        for i=1:n_train
            if i > k
                break
            end
            min_idx = i
            for j=i+1:n_train
                if A[l, min_idx] > A[l, j]
                    min_idx = j
                end
            end
            A[l, i], A[l, min_idx] = A[l, min_idx], A[l, i]
            index[l, i] = min_idx
        end
    end
    return index
end

# Add tuning and fit!, predic interface


@doc raw"""
    LogEuclideanMetric <: RiemannianMetric

The LogEuclidean Metric consists of the Euclidean metric applied to all elements after mapping them
into the Lie Algebra, i.e. performing a matrix logarithm beforehand.
"""
struct LogEuclideanMetric <: RiemannianMetric end

@doc raw"""
    distance(M::MetricManifold{SymmetricPositiveDefinite{N},LogEuclideanMetric}, p, q)

Compute the distance on the [`SymmetricPositiveDefinite`](@ref) manifold between
`p` and `q` as a [`MetricManifold`](@ref) with [`LogEuclideanMetric`](@ref).
The formula reads

```math
    d_{\mathcal P(n)}(p,q) = \lVert \operatorname{Log} p - \operatorname{Log} q \rVert_{\mathrm{F}}
```

where $\operatorname{Log}$ denotes the matrix logarithm and
$\lVert\cdot\rVert_{\mathrm{F}}$ denotes the matrix Frobenius norm.
"""
function distance(
    M::MetricManifold{ℝ,SymmetricPositiveDefinite{N},LogEuclideanMetric},
    p,
    q,
) where {N}
    return norm(log(Symmetric(p)) - log(Symmetric(q)))
end


using  ForwardDiff,GenericLinearAlgebra,LinearAlgebra

mutable struct svdmat
    U::Any
    S::Any
    V::Any
end
export svd_lowrank

function svd_lowrank(A::Matrix, l::Int, q::Int)
    #= note:: The implementation is based on the Algorithm 5.1 from
    Halko et al, 2009.=#
    Q = get_approximate_basis(A, l, q)
    Q = conj(Q')
    B = Q * A
    U, S, V = svd(B)
    V = conj(V')
    U = Q * U
    return svdmat(U, S, V)
end

function get_approximate_basis(A::Matrix, l::Int, q::Int)
    #= note:: The implementation is based on the Algorithm 4.4 from
    Halko et al, 2009. =#
    m, n = size(A)
    omega = randn(n, l)
    Y = A * omega
    Q, R = qr(Y)
    Y_H = conj(Y')

    for i = 1:q
        Q, R = qr(Y_H * Q)
        Q, R = qr(Y * Q)
    end
    return Q
end


module FESpaceWithConstantFixedTests
using Gridap
using Gridap.Geometry
using Gridap.FESpaces
using Gridap.ReferenceFEs
using Test

domain = (0,1,0,1)
partition = (4,4)
model = CartesianDiscreteModel(domain,partition)

trian = get_triangulation(model)

order = 2

V = FESpace(model,ReferenceFE(lagrangian,Float64,order);conformity=:L2)

V0 = FESpaceWithConstantFixed(V,true,rand(1:num_free_dofs(V)))
test_single_field_fe_space(V0)

@test Gridap.FESpaces.ConstantApproach(V0) == Gridap.FESpaces.FixConstant()

uh0 = interpolate(V0) do x
    sin(4*pi*(x[1]+x[2]^2)) + 3
end
using Gridap.Visualization

#writevtk(trian,"trian",nsubcells=20,cellfields=["uh0"=>uh0])

end # module


"""
## rod_km!

This subroutine forms the stiffness matrix of a 1-d "rod" fin_el.

### Function
```julia
rod_km!(km, ea, length)
```

### Arguments
```julia
* km::Matrix{Float64}       : Element stiffness matrix (Updated)
* ea::Float64               : Element stiffness
* ell::Float64              : Element length
```
"""
function rod_km!(km::Matrix{Float64}, ea::Float64, ell::Float64)
  #
  # This subroutine forms the stiffness matrix of a 1-d "rod" fin_el.
  #
 km = [1.0 -1.0;-1.0 1.0]
 km *= ea/ell
end


using QuasiMonteCarlo, Distributions, Test
using Test

#1D
lb = 0.0
ub = 5.0
n = 5
d = 1
QuasiMonteCarlo.sample(n,lb,ub,GridSample(0.1))
QuasiMonteCarlo.sample(n,lb,ub,UniformSample())
QuasiMonteCarlo.sample(n,lb,ub,SobolSample())
QuasiMonteCarlo.sample(n,lb,ub,LatinHypercubeSample())
QuasiMonteCarlo.sample(n,lb,ub,LatticeRuleSample())
QuasiMonteCarlo.sample(20,lb,ub,LowDiscrepancySample(10))
QuasiMonteCarlo.sample(5,d,Cauchy())
QuasiMonteCarlo.sample(5,d,Normal(0,4))

#ND
lb = [0.1,-0.5]
ub = [1.0,20.0]
n = 5
d = 2

@testset "GridSample" begin
    #GridSample{T}
    s = QuasiMonteCarlo.sample(n,lb,ub,GridSample([0.1,0.5]))
    @test isa(s,Matrix{typeof(s[1][1])}) == true
    @test size(s) == (d, n)
end

@testset "UniformSample" begin
    #UniformSample()
    s = QuasiMonteCarlo.sample(n,lb,ub,UniformSample())
    @test isa(s,Matrix{typeof(s[1][1])}) == true
    @test size(s) == (d, n)
end

@testset "SobolSample" begin
    #SobolSample()
    s = QuasiMonteCarlo.sample(n,lb,ub,SobolSample())
    @test isa(s,Matrix{typeof(s[1][1])}) == true
    @test size(s) == (d, n)
end

@testset "LHS" begin
    #LHS
    s = QuasiMonteCarlo.sample(n,lb,ub,LatinHypercubeSample())
    @test isa(s,Matrix{typeof(s[1][1])}) == true
    @test size(s) == (d, n)
end

@testset "LatticeRuleSample" begin
    #LatticeRuleSample()
    s = QuasiMonteCarlo.sample(n,lb,ub,LatticeRuleSample())
    @test isa(s,Matrix{typeof(s[1][1])}) == true
    @test size(s) == (d, n)
end

@testset "LDS" begin
    #LDS
    s = QuasiMonteCarlo.sample(n,lb,ub,LowDiscrepancySample([10,3]))
    @test isa(s,Matrix{typeof(s[1][1])}) == true
    @test size(s) == (d, n)
end

@testset "Distribution 1" begin
    #Distribution 1
    s = QuasiMonteCarlo.sample(n,d,Cauchy())
    @test isa(s,Matrix{typeof(s[1][1])}) == true
    @test size(s) == (d, n)
end

@testset "Distribution 2" begin
    #Distribution 2
    s = QuasiMonteCarlo.sample(n,d,Normal(3,5))
    @test isa(s,Matrix{typeof(s[1][1])}) == true
    @test size(s) == (d, n)
end

@testset "generate_design_matrices" begin
    num_mat = 3
    Ms = QuasiMonteCarlo.generate_design_matrices(n,lb,ub,UniformSample(), num_mat)
    @test length(Ms) == num_mat
    A = Ms[1]
    B = Ms[2]
    @test isa(A, Matrix{typeof(A[1][1])}) == true
    @test size(A) == (d, n)
    @test isa(B, Matrix{typeof(B[1][1])}) == true
    @test size(B) == (d, n)
end


"""
```julia
using DifferentialEquations, Plots
function lorenz!(du,u,p,t)
 du[1] = 10.0*(u[2]-u[1])
 du[2] = u[1]*(28.0-u[3]) - u[2]
 du[3] = u[1]*u[2] - (8/3)*u[3]
end

u0 = [1.0;0.0;0.0]
tspan = (0.0,100.0)
prob = ODEProblem(lorenz!,u0,tspan)
sol = solve(prob)
plot(sol)
```
"""
module DifferentialEquations

  using Reexport

  @reexport using DiffEqBase
  @reexport using DiffEqNoiseProcess
  @reexport using RecursiveArrayTools

  @reexport using SteadyStateDiffEq
  @reexport using StochasticDiffEq
  @reexport using OrdinaryDiffEq
  @reexport using BoundaryValueDiffEq
  using Sundials
  @reexport using DelayDiffEq

  @reexport using DiffEqCallbacks
  @reexport using DiffEqJump

  @reexport using DiffEqFinancial
  @reexport using MultiScaleArrays
  @reexport using DiffEqPhysics

  @reexport using DimensionalPlotRecipes
  @reexport using ParameterizedFunctions

  using LinearAlgebra

  import DiffEqBase: solve

  include("default_solve.jl")
  include("default_arg_parsing.jl")
  include("ode_default_alg.jl")
  include("sde_default_alg.jl")
  include("dae_default_alg.jl")
  include("dde_default_alg.jl")
  include("discrete_default_alg.jl")
  include("rode_default_alg.jl")
  include("steady_state_default_alg.jl")
  include("bvp_default_alg.jl")

end # module


#!/usr/bin/julia

# Date: 21 August 2016
# https://github.com/trizen

# An efficient algorithm for computing factorial of a large number, modulus a larger number.
# Algorithm from: http://stackoverflow.com/questions/9727962/fast-way-to-calculate-n-mod-m-where-m-is-prime

function facmod(n::Int64, mod::Int64)

    f = 1
    for k in n+1:mod-1
       f *= k
       f %= mod
    end

    (-1 * invmod(f, mod) + mod) % mod
end

println(facmod(100_000_000, 100_000_037))