include("smear_FD.jl")
include("calc_entropy.jl")

function test_main(kT::Float64)
    Nstates = 5
    ev = Array{Float64}(Nstates)
    ev = [-2.4, -1.0, -0.5, -0.2, -0.19]
    efermi = 0.5*(ev[3] + ev[4])
    println("\nkT = ", kT)
    for spinpol in [true,false]
        println("\nspinpol = ", spinpol)
        Focc = smear_FD(ev, efermi, kT, is_spinpol=spinpol)
        for ist = 1:Nstates
            @printf("%18.10f %18.10f\n", ev[ist], Focc[ist])
        end
        @printf("sum(Focc) = %18.10f\n", sum(Focc))
        @printf("Entropy (-TS) = %18.10f\n", calc_entropy(Focc, kT, is_spinpol=spinpol))
    end
end

test_main(0.001)
test_main(0.01)
test_main(0.1)


"Algorithms for discovering sparse bases for the symmetry-allowed space of exchange couplings."

# ##### Math behind `basis_for_symmetry_allowed_couplings()` #####
#
# A crystal spacegroup consists of a set of symmetry operations. Each symop is
# defined by an orthogonal (rotation or reflection) matrix R and a translation
# vector T. The symop transforms every atom position `x` to a new one `x′ = Rx +
# T`, but leaves the crystal as a whole invariant. Consider some bond b=(i,j),
# defined as an ordered pair of atoms, not necessarily in the same crystal unit
# cell. Suppose this bond carries an exchange interaction of the form `S_iᵀ J
# S_j`. After applying a symop defined by (R, T), the bond b=(i,j) maps to a
# transformed bond b′=(i′,j′). This new bond must carry an exchange interaction
# of the form `S_{i′}ᵀ J′ S_{j′}`, where J′ is related to J, as we will now
# show. Besides mapping the atom j to a new position j′, the symop also
# transforms its spin vector from `S_j` to `S′_{j′} = R S_j`. Similarly, `S_iᵀ`
# maps to `S′_{i′}ᵀ = S_iᵀ Rᵀ`. The energy along a bond, however, is invariant
# under the symmetry transformation, so we require `S_iᵀ J S_j = S′_{i′}ᵀ J′
# S′_{j′}`, or
#
#   `S_iᵀ J S_j = S_iᵀ Rᵀ J′ R S_j`.
#
# This equation must be valid for arbitrary `S_i` and `S_j`, which implies `J =
# Rᵀ J′ R`, or equivalently,
#
#   J′ = R J Rᵀ
#
# because the matrix R is orthogonal. The conclusion is that specifying the
# exchange matrix J for one bond implicitly constrains the exchange matrices J′
# for all symmetry equivalent bonds.
#
# Often a symmetry operation will map a bond b into itself, or into the same
# bond but with atom positions reversed. The existence of such symops constrains
# the space of allowed exchange matrices J for the bond b. Specifically, we
# require
#
#   (1)  J = R J Rᵀ   or   (2)  Jᵀ = R J Rᵀ
#
# for every symop `s = (R, T)` that maps `b` into `b` (constraint 1), or into
# `reverse(b)` (constraint 2). The intersection of all such constraints defines
# the symmetry-allowed space of exchange matrices J. The allowed matrices can be
# expressed as a linear combination of basis matrices because all constraints
# are linear in J.
#
# It is convenient to express the constraints (1) or (2) in the form `F J = 0`,
# where F denotes the linear operator such that `F J = R J Rᵀ - J` or `F J = R J
# Rᵀ - Jᵀ`. Here, we are viewing the 3×3 matrix J as a flattened 9-dimensional
# vector, and F as a 9x9 matrix. In the first case, `F = R⊗R - I`. In the second
# case, we should replace `I` by the suitable transpose operation,
# `transpose_op_3x3`, defined explicitly below.
#
# Any linear combination of "vectors" (3x3 matrices) in the null space of F,
# i.e. `F J = 0`, satisfies the symmetry constraint (1) or (2) for the given
# symop. The singular value decomposition
#
#     F = U Σ Vᵀ
#
# can be used to produce an orthogonal basis for this null space. It is spanned
# by columns v of V corresponding to the zero singular values. The space spanned
# by v equivalently represented as a projection matrix,
#
#     P = v vᵀ
#
# When there are multiple constraints (F1, F2, ... Fn) we should take the
# intersection of the spanned spaces of P1, ... Pn. To calculate this
# intersection, form the product of the projectors:
#
#     P = P1 P2 ... Pn
#
# The allowable J values correspond to the eigenvectors of P with eigenvalue 1.
# An orthogonal basis for this space can again be calculated with an SVD.

# A 9x9 matrix that, when applied to a flattened 3x3 matrix (viewed as a
# 9-dimensional vector), generates the transposed 3x3 matrix in flattened form.
const transpose_op_3x3 = SMatrix{9, 9, Float64}([
    1 0 0  0 0 0  0 0 0
    0 0 0  1 0 0  0 0 0
    0 0 0  0 0 0  1 0 0

    0 1 0  0 0 0  0 0 0
    0 0 0  0 1 0  0 0 0
    0 0 0  0 0 0  0 1 0

    0 0 1  0 0 0  0 0 0
    0 0 0  0 0 1  0 0 0
    0 0 0  0 0 0  0 0 1
])

# Returns a projection operator P that maps to zero any symmetry-unallowed
# coupling matrix J. The space spanned by the eigenvectors of P with eigenvalue
# 1 represents the allowed coupling matrices J.
function projector_for_symop(cryst::Crystal, s::SymOp, parity::Bool)
    # Cartesian-space rotation operator corresponding to `s`
    R = cryst.lat_vecs * s.R * inv(cryst.lat_vecs)

    # Constraint is modeled as `F J = 0`
    F = kron(R, R) - (parity ? SMatrix{9, 9, Float64}(I) : transpose_op_3x3)

    # Orthogonal column vectors that span the null space of F
    v = nullspace(F; atol=1e-12)

    # Projector onto the null space of F
    P = SMatrix{9, 9, Float64}(v * v')
    return P
end

# Return an operator P that implicitly gives the space of symmetry allowed
# coupling matrices for bond b. Specifically, x is an allowed coupling if and
# only if it is an eigenvector of P with eigenvalue 1, i.e., `P x = x`.
function symmetry_allowed_couplings_operator(cryst::Crystal, b::BondRaw)
    P = SMatrix{9, 9, Float64}(I)
    for (s, parity) in symmetries_between_bonds(cryst, b, b)
        P = P * projector_for_symop(cryst, s, parity)
    end
    # Allowed coupling matrices J are simultaneously eigenvectors for all
    # projectors above, with eigenvalue 1.
    return P
end

# Check whether a coupling matrix J is consistent with symmetries of a bond
function is_coupling_valid(cryst::Crystal, b::BondRaw, J::Mat3)
    for (s, parity) in symmetries_between_bonds(cryst, b, b)
        R = cryst.lat_vecs * s.R * inv(cryst.lat_vecs)
        # TODO use symprec to handle case where matrix R is not precise
        if norm(R*J*R' - (parity ? J : J')) > 1e-12
            return false
        end
    end
    return true
end

function is_coupling_valid(cryst::Crystal, b::Bond, J::Mat3)
    return is_coupling_valid(cryst, BondRaw(cryst, b), J)
end

# Orthonormal basis of 3x3 symmetric matrices
const sym_basis = begin
    b = [diagm([1, 0, 0]),
         diagm([0, 1, 0]),
         diagm([0, 0, 1]),
         [0 1 0
          1 0 0
          0 0 0]/√2,
         [0 0 1
          0 0 0
          1 0 0]/√2,
         [0 0 0
          0 0 1
          0 1 0]/√2,]
    SMatrix{9, 6, Float64}(hcat(reshape.(b, 9)...))
end

# Orthonormal basis of 3x3 antisymmetric matrices
const asym_basis = begin
    b = [[ 0  1  0
          -1  0  0
           0  0  0]/√2,
         [ 0  0 -1
           0  0  0
           1  0  0]/√2,
         [ 0  0  0
           0  0  1
           0 -1  0]/√2]
    SMatrix{9, 3, Float64}(hcat(reshape.(b, 9)...))
end

@assert sym_basis * sym_basis' + asym_basis * asym_basis' ≈ I

# Given an m×n matrix A with empty nullspace, linearly combine the n columns to
# make them sparser. Solution was proposed here:
# https://math.stackexchange.com/q/4227648/660903 . Closely related to finding
# reduced row echolon form.
function sparsify_columns(A; atol)
    if size(A, 2) <= 1
        return A
    else
        # By assumption, the n columns of A are linearly independent
        @assert isempty(nullspace(A; atol))
        # Since row rank equals column rank, it should be possible to find n
        # linearly independent rows in A. Store these independent rows of A as
        # column vectors in indep_rows.
        indep_rows = zeros(Float64, size(A, 2), 0)
        for row in eachrow(A)
            # Add `row` to list if linearly independent with existing ones
            if isempty(nullspace(hcat(indep_rows, row); atol))
                indep_rows = hcat(indep_rows, row)
            end
        end
        return A * inv(indep_rows)'
    end
end

const _basis_elements_by_priority = [1, 5, 9, 8, 3, 4]

function _score_basis_matrix(J)
    return findfirst(i -> abs(J[i]) > 1e-12, _basis_elements_by_priority)
end

"""    basis_for_symmetry_allowed_couplings(cryst::Crystal, b::Bond)

Returns a list of ``3×3`` matrices that form a linear basis for the
symmetry-allowed coupling matrices associated with bond `b`.
"""
function basis_for_symmetry_allowed_couplings(cryst::Crystal, b::BondRaw)
    # Expected floating point precision for 9x9 matrix operations
    atol = 1e-12

    P = symmetry_allowed_couplings_operator(cryst, b)
    # Any solution to the original symmetry constraints `R J Rᵀ = J` or `R J Rᵀ
    # = Jᵀ` decomposes into purely symmetric/antisymmetric solutions. Therefore
    # we can pick a basis that separates into symmetric and antisymmetric parts.
    # We will do so by decomposing P. By construction, P = P_sym+P_asym.
    P_sym  = P *  sym_basis *  sym_basis'
    P_asym = P * asym_basis * asym_basis'

    acc_sym = SVector{9, Float64}[]
    acc_asym = SVector{9, Float64}[]

    # If any "reference" basis vectors are eigenvalues of P_sym with eigenvalue
    # 1, use them as outputs, and remove them from P_sym
    for x in eachcol(sym_basis)
        if isapprox(P_sym*x, x; atol)
            push!(acc_sym, x)
            P_sym = P_sym * (I - x*x')
        end
    end
    # Same for P_asym
    for x in eachcol(asym_basis)
        if isapprox(P_asym*x, x; atol)
            push!(acc_asym, x)
            P_asym = P_asym * (I - x*x')
        end
    end

    # Search for eigenvectors of P_sym with eigenvalue 1. These provide an
    # orthonormal basis for symmetric couplings.
    v = nullspace(P_sym-I; atol)
    v = sparsify_columns(v; atol)
    append!(acc_sym, eachcol(v))
    # Same for P_asym
    v = nullspace(P_asym-I; atol)
    v = sparsify_columns(v; atol)
    append!(acc_asym, eachcol(v))

    # Sort basis elements according to the indices where the nonzero elements
    # first appear
    sort!(acc_sym;  by=_score_basis_matrix)
    sort!(acc_asym; by=_score_basis_matrix)

    acc = [acc_sym; acc_asym]
    return map(acc) do x
        # Normalize each basis vector so that its maximum component is 1. The
        # shift by atol avoids unnecessary sign change in the case where the
        # maximum magnitude values of x appear symmetrically as ±c for some c.
        _, i = findmax(abs.(x.+atol))
        x = x / x[i]

        # Reinterpret as 3x3 matrix
        x = Mat3(reshape(x, 3, 3))

        # Double check that x indeed satifies the necessary symmetries
        @assert is_coupling_valid(cryst, b, x)

        return x
    end
end

function basis_for_symmetry_allowed_couplings(cryst::Crystal, b::Bond)
    return basis_for_symmetry_allowed_couplings(cryst, BondRaw(cryst, b))
end

"""
    all_symmetry_related_couplings_for_atom(cryst::Crystal, i::Int, b::Bond, J)

Given a reference bond `b` and coupling matrix `J` on that bond, return a list
of symmetry-equivalent bonds (constrained to start from atom `i`), and a
corresponding list of symmetry-transformed coupling matrices.
"""
function all_symmetry_related_couplings_for_atom(cryst::Crystal, i::Int, b_ref::Bond, J_ref)
    J_ref = Mat3(J_ref)
    @assert is_coupling_valid(cryst, b_ref, J_ref)

    bs = Bond[]
    Js = Mat3[]

    for b in all_symmetry_related_bonds_for_atom(cryst, i, b_ref)
        push!(bs, b)
        (s, parity) = first(symmetries_between_bonds(cryst, BondRaw(cryst, b), BondRaw(cryst, b_ref)))
        R = cryst.lat_vecs * s.R * inv(cryst.lat_vecs)
        push!(Js, R * (parity ? J_ref : J_ref') * R')
    end

    return (bs, Js)
end

"""
    all_symmetry_related_couplings(cryst::Crystal, b::Bond, J)

Given a reference bond `b` and coupling matrix `J` on that bond, return a list
of symmetry-equivalent bonds and a corresponding list of symmetry-transformed
coupling matrices.
"""
function all_symmetry_related_couplings(cryst::Crystal, b_ref::Bond, J_ref)
    J_ref = Mat3(J_ref)

    bs = Bond[]
    Js = Mat3[]

    for i in eachindex(cryst.positions)
        (bs_i, Js_i) = all_symmetry_related_couplings_for_atom(cryst, i, b_ref, J_ref)
        append!(bs, bs_i)
        append!(Js, Js_i)
    end

    return (bs, Js)
end


using CSDP

#=
# Example copied from `example/example.c`
   An example showing how to call the easy_sdp() interface to CSDP.  In
   this example, we solve the problem

      max tr(C*X)
          tr(A1*X)=1
          tr(A2*X)=2
          X >= 0       (X is PSD)

   where

    C=[2 1
       1 2
           3 0 1
           0 2 0
           1 0 3
                 0
                   0]

   A1=[3 1
       1 3
           0 0 0
           0 0 0
           0 0 0
                 1
                   0]

   A2=[0 0
       0 0
           3 0 1
           0 4 0
           1 0 5
                 0
                   1]

  Notice that all of the matrices have block diagonal structure.  The first
  block is of size 2x2.  The second block is of size 3x3.  The third block
  is a diagonal block of size 2.

=#

C = BlockMatrix(
   [2 1
    1 2],
        [3 0 1
         0 2 0
         1 0 3],
      Diagonal([0,
                  0]))

b = [1.0, 2.0]

A1 = ConstraintMatrix(1,
   [3 1
    1 3],
          [0 0 0
           0 0 0
           0 0 0],
       Diagonal([1,
                   0]))

A2 = ConstraintMatrix(2,
      [0 0
       0 0],
          [3 0 1
           0 4 0
           1 0 5],
       Diagonal([0,
                   1]))

constraints = [A.csdp for A in [A1, A2]]

CSDP.write_prob("prob.dat-s", C, b, constraints)

X, y, Z = initsoln(C, b, constraints)

pobj, dobj = easy_sdp(C, b, constraints, X, y, Z)

# Fails on Windows
#CSDP.write_sol("prob.sol", X, y, Z)


using EffectiveWaves

using Plots
# unicodeplots() # alternative plot backend
height=500
pyplot(linewidth=3, size=(2.0*height,height))

# for fixed total volfrac fraction
medium = Medium(ρ=3200.0,c = 2595.0)
ωs = 2.0*pi*LinRange(1.0e1,1.0e7,100)

volfrac = 0.12
r1 = 30.0e-8
r2 = 2*30.0e-6

sp1 = Specie(ρ=2329.,r=r1,c=8433.,volfrac = 0.06)
sp2 = Specie(ρ=2329.,r=r2,c=8433.,volfrac = volfrac-0.06)

vol = pi*sp1.num_density*sp1.r^2

rhoeff =  medium.ρ*(medium.ρ + sp1.ρ - vol*(medium.ρ - sp1.ρ))/
(medium.ρ + sp1.ρ + vol*(medium.ρ - sp1.ρ))

# rhoeff =  medium.ρ*(1.0-vol) + vol*sp1.ρ
# rhoeff =  medium.ρ*(1 + volfrac)/(1 - volfrac)

kTs = [ wavenumber_low_volfrac(ω, medium, [sp1,sp2]) for ω in ωs]
kTaprx =[ two_species_approx_wavenumber(ω, medium, [sp1,sp2]) for ω in ωs]

kTSs =[ wavenumber_low_volfrac(ω, medium, [sp1]) for ω in ωs]
kTLs = [
  begin
    mS = Medium(ρ=rhoeff, c= ωs[i]/kTSs[i])
    wavenumber_low_volfrac(ωs[i], mS, [sp2])
  end
for i in eachindex(ωs)];

# speed_arr = [ [ωs./real(kTs)], [0.029 .+ ωs./real(kTLs)]]
speed_arr = [ ωs./real(kTs), ωs./real(kTLs), ωs./real(kTaprx)]
atten_arr = imag([kTs,kTLs, kTaprx])

labs = reshape(["kT","kTL", "kTaprx"] ,1, 3);
p1 = plot(r1.*(ωs./real(medium.c))/(2pi), speed_arr, labels=labs, xlabel="a/λ", ylabel="wave speed");
p2 = plot(r1.*(ωs./real(medium.c))/(2pi), atten_arr, labels=labs, xlabel="a/λ", ylabel="attenuation");
plot(p1,p2)
gui()

speed_arr = [abs.(1 .- real(kTLs)./real(kTs))]
p1 = plot(r1.*(ωs./real(medium.c))/(2pi), speed_arr, labels=labs, xlabel="a/λ", ylabel="relative wave speed");


using Test, DiffEqOperators, SpecialFunctions

@testset "None BC DerivativeOperator" begin
    N = 10
    d_order = 1
    approx_order = 2
    A = DerivativeOperator{Float64}(d_order,approx_order,1.0,N,:None,:None)
    Amat = convert(Array,A)

    @test Amat[1,1:3] ≈ [-1.5, 2.0, -0.5]
    for row = 2:N-1
        @test Amat[row,row-1:row+1] ≈ [-0.5, 0.0, 0.5]
    end
    @test Amat[end,end-2:end] ≈ [0.5, -2.0, 1.5]

    d_order = 2
    A = DerivativeOperator{Float64}(d_order,approx_order,1.0,N,:None,:None)
    Amat = convert(Array,A)

    @test Amat[1,1:4] ≈ [2.0, -5.0, 4.0, -1.0]
    for row = 2:N-1
        @test Amat[row,row-1:row+1] ≈ [1.0, -2.0, 1.0]
    end
    @test Amat[end,end-3:end] ≈ [-1.0, 4.0, -5.0, 2.0]

    N = 10
    d_order = 1
    approx_order = 3
    A = DerivativeOperator{Float64}(d_order,approx_order,1.0,N,:None,:None)
    Amat = convert(Array,A)

    @test Amat[1,1:4] ≈ [-11/6, 3, -3/2, 1/3]
    for row = 3:N-2
        @test Amat[row,row-1:row+1] ≈ [-1/2, 0, 1/2]
    end
    @test Amat[end,end-3:end] ≈ [-1/3, 3/2, -3, 11/6]

end

@testset "None BC FiniteDifference Operator" begin
    N = 10
    d_order = 1
    approx_order = 2
    A = FiniteDifference{Float64}(d_order,approx_order,ones(N-1),N,:None,:None)
    Amat = convert(Array,A)

    @test Amat[1,1:3] ≈ [-1.5, 2.0, -0.5]
    for row = 2:N-1
        @test Amat[row,row-1:row+1] ≈ [-0.5, 0.0, 0.5]
    end
    @test Amat[end,end-2:end] ≈ [0.5, -2.0, 1.5]

    d_order = 2
    A = FiniteDifference{Float64}(d_order,approx_order,ones(N-1),N,:None,:None)
    Amat = convert(Array,A)

    @test Amat[1,1:4] ≈ [2.0, -5.0, 4.0, -1.0]
    for row = 2:N-1
        @test Amat[row,row-1:row+1] ≈ [1.0, -2.0, 1.0]
    end
    @test Amat[end,end-3:end] ≈ [-1.0, 4.0, -5.0, 2.0]

    N = 10
    d_order = 1
    approx_order = 3
    A = FiniteDifference{Float64}(d_order,approx_order,ones(N-1),N,:None,:None)
    Amat = convert(Array,A)

    @test Amat[1,1:4] ≈ [-11/6, 3, -3/2, 1/3]
    for row = 3:N-2
        @test Amat[row,row-1:row+1] ≈ [-1/2, 0, 1/2]
    end
    @test Amat[end,end-3:end] ≈ [-1/3, 3/2, -3, 11/6]

end

@testset "FiniteDifference Operator sin(x)==d/dx^4 sin(x)" begin
    N = 10
    n = 1000
    x = erf.(range(-2,stop=2,length=n))*2π
    x .+=abs(x[1])
    dx = diff(x)

    A = DiffEqOperators.FiniteDifference{Float64}(4,4,dx,length(x),:None,:None)
    boundary_points = A.boundary_point_count

    y = @. sin(x)+2 #avoid 0.0
    res = A*y
    @test res[boundary_points[1] .+ 1: N - boundary_points[2]] .+ 2.0 ≈ y[boundary_points[1] .+ 1: N - boundary_points[2]] atol=10.0^-1;

end


######### StanSample example  ###########

using StanSample, Test

bernoulli_model = "
data {
  int<lower=1> N;
  array[N] int<lower=0,upper=1> y;
}
parameters {
  real<lower=0,upper=1> theta;
}
model {
  theta ~ beta(1,1);
  y ~ bernoulli(theta);
}
";

bernoulli_data = [
  Dict("N" => 10, "y" => [0, 1, 0, 1, 0, 0, 0, 0, 0, 1]),
  Dict("N" => 10, "y" => [0, 1, 0, 0, 1, 0, 0, 0, 0, 1]),
  Dict("N" => 10, "y" => [0, 1, 0, 1, 0, 0, 0, 0, 1, 0]),
  Dict("N" => 10, "y" => [0, 0, 0, 1, 0, 0, 1, 0, 0, 1]),
]

sm = SampleModel("bernoulli", bernoulli_model)
rc = stan_sample(sm; data=bernoulli_data, delta=0.85, num_threads=1)
if success(rc)

  # Fetch the same output in the `sdf` ChainDataFrame
  sdf = read_summary(sm)

  @test sdf[sdf.parameters .== :theta, :mean][1] ≈ 0.33 rtol=0.05

end


# assert file to test polynomial implementation
using Base.Test
using Polynomials

pNULL = Poly(Float32[])
p0 = Poly([0])
p1 = Poly([1,0,0,0,0,0,0,0,0,0,0,0,0,0])
p2 = Poly([1,1,0,0])
p3 = Poly([1,2,1,0,0,0,0])
p4 = Poly([1,3,3,1,0,0])
p5 = Poly([1,4,6,4,1,0,0,0,0,0,0,0,0,0,0,0,0,0])
pN = Poly([276,3,87,15,24,0])
pR = Poly([3//4, -2//1, 1//1])
p1000 = Poly(randn(1000))

@test length(pNULL) == 0
@test length(p1000) == 1000
sprint(show, p1000)
sprint(show, pNULL)

@test p3 == Poly([1,2,1])
@test pN*10 == Poly([2760, 30, 870, 150, 240])
@test pN/10 == Poly([27.6, 0.3, 8.7, 1.5, 2.4])
@test 10*pNULL + pN == pN
@test 10*p0 + pN == pN
@test p5 + 2*p1 == Poly([3,4,6,4,1])
@test 10*pNULL - pN == -pN
@test p0 - pN == -pN
@test p5 - 2*p1 == Poly([-1,4,6,4,1])
@test p2*p2*p2 == p4
@test p2^4 == p5
@test pNULL^3 == pNULL
@test pNULL*pNULL == pNULL

@test polyval(pN, -.125) == 276.9609375
@test polyval(pNULL, 10) == 0
@test polyval(p0, -10) == 0
@test polyval(poly([1//2, 3//2]), 1//2) == 0//1
@test polyder(polyint(pN)) == pN
@test polyder(pR) == Poly([-2//1,2//1])
@test polyint(pNULL,1) == p1
@test polyint(Poly(Rational[1,2,3])) == Poly(Rational[0, 1, 1, 1])
@test polyder(p3) == Poly([2,2])
@test polyder(p1) == polyder(p0) == polyder(pNULL) == pNULL

@test poly([-1,-1]) == p3
@test roots(p0)==roots(p1)==roots(pNULL)==[]
@test roots(p2) == [-1]
a_roots = copy(pN.a)
@test all(abs(sort(roots(poly(a_roots))) - sort(a_roots)) .< 1e6)
@test length(roots(p5)) == 4
@test roots(pNULL) == []
@test roots(pR) == [1//2, 3//2]

@test pNULL + 2 == p0 + 2 == 2 + p0 == Poly([2])
@test p2 - 2 == -2 + p2 == Poly([-1,1])
@test 2 - p2 == Poly([1,-1])

p0 = Poly([0])
p1 = Poly([1])
p2 = Poly([5, 6, -3, 2 ,4])
p3 = Poly([7, -3, 2, 6])
p4 = p2 * p3
@test divrem(p4, p2) == (p3, zero(p3))
@test p3%p2 == p3
@test all((abs((p2/p3 - Poly([1/9,2/3])).a)) .< eps())
@test divrem(p0,p1) == (p0,p0)
@test divrem(p1,p1) == (p1,p0)
@test divrem(p2,p2) == (p1,p0)
@test divrem(pR, pR) == (one(pR), zero(pR))
@test_throws p1/p0
@test_throws divrem(p0,p0)

#Tests for multivariable support
pX = Poly([1, 2, 3, 4, 5])
pS1 = Poly([1, 2, 3, 4, 5], "s")
pS2 = Poly([1, 2, 3, 4, 5], 's')
pS3 = Poly([1, 2, 3, 4, 5], :s)
@test pX != pS1
@test pS1 == pS2
@test pS1 == pS3
@test_throws pS1 + pX
@test_throws pS1 - pX
@test_throws pS1 * pX
@test_throws pS1 / pX
@test_throws pS1 % pX


function noisy_circle_data(N, σ)
    r = 1 .+ σ*randn(1,N)
    θ = rand(Uniform(0, 2π), (1,N))

    rcosθ = r .* cos.(θ)
    rsinθ = r .* sin.(θ)

    data = [rcosθ; rsinθ]

    return data
end


using QuantizedStateSystems
using LinearAlgebra
using Test

@testset "QSS1 for scalar systems with no input" begin
    @test begin
        a = -1.0
        f = (x,u,t) -> a*x
        x₀ = 10.0
        tspan = (0.0,1.0)
        Δq = 0.01
        txarray,xarray,tqarray,qarray = qss1(f,x₀,tspan,Δq,[],[])
        xtrue(t) = x₀*exp(a*t)
        xtruearray = xtrue.(txarray)
        relerror = (xarray-xtruearray)./xtruearray
        norm(relerror)<0.01
    end
end

@testset "QSS1 for scalar systems with (control) inputs" begin
    @test begin
        f = (x,u,t) -> -1.0*x + u
        x₀ = 10.0
        tspan = (0.0,5.0)
        tuarray = [1.76]
        uarray = [10.0]
        Δq = 0.01
        heaviside(t,τ) = t>=τ ? 1.0 : 0.0
        xtrue(t) = x₀*exp(-t) + 10.0*(heaviside(t,1.76)*(1.0-exp(-t+1.76)))
        (txarray,xarray,tqarray,qarray) = qss1(f,x₀,tspan,Δq,tuarray,uarray)
        xtruearray = xtrue.(txarray)
        relerror = (xarray-xtruearray)./xtruearray
        norm(relerror)<0.05
    end
end


module llh

using LinearAlgebra  #: cholesky, logdet, diag, inv, triu
using AxUtil
using NNlib: softmax
using ..Misc

export gmm_llh, gmm_llh_invLT

function log_gauss_llh(X, mu, sigma; bypass=false)
    if !bypass
        out = _log_gauss_llh(X, mu, sigma)
    else
        out = - ones(size(X, 2))*Inf
    end
    return out
end

function _log_gauss_llh(X, mu, sigma)
    d = size(X,1)
    invLT = Matrix(inv(cholesky(sigma).L))
    Z = invLT*(X .- mu)
    exponent = -0.5*sum(Z.^2, dims=1)  |> dropdim1
    lognormconst = -d*log(2*pi)/2 -0.5*logdet(sigma)  #.-0.5*(-2*sum(log.(diag(invLT))))
    return exponent .+ lognormconst
end

function _log_gauss_llh_invLT(X, mu, invLT)
    d = size(X,1)
    Z = invLT*(X .- mu)
    exponent = -0.5*sum(Z.^2, dims=1)  |> dropdim1
    lognormconst = -d*log(2*pi)/2 .-0.5*(-2*sum(log.(diag(invLT))))
    return exponent .+ lognormconst
end

function gmm_llh(X, pis, mus, sigmas; thrsh_comp=0.005)
    p, n = size(X)
    k = length(pis)
    inactive_ixs = pis[:] .< thrsh_comp

    P = zeros(k, n)
    for j = 1:k
        P[j,:] = log_gauss_llh(X, mus[j,:], sigmas[:,:,j],
            bypass=inactive_ixs[j]) .+ log(pis[j])
    end
    return AxUtil.Math.logsumexpcols(P)
end

function gmm_llh_invLT(X, pis, mus, invLTs::Array{T,1}; disp=false, thrsh_comp=0.005) where T <: AbstractMatrix
    p, n = size(X)
    k = length(pis)
    inactive_ixs = pis[:] .< thrsh_comp

    P = zeros(k, n)
    for j = 1:k
        if !inactive_ixs[j]
            P[j,:] = _log_gauss_llh_invLT(X, mus[j,:], invLTs[j]) .+ log(pis[j])
        else
            P[j,:] .= -Inf
        end
    end
    return AxUtil.Math.logsumexpcols(P)
end

function responsibilities(X, mus::AbstractArray{T,2}, sigmas::AbstractArray{T, 3}, pis::AbstractArray{T, 1}) where T <: AbstractFloat
    softmax(reduce(vcat, [log_gauss_llh(X, mus[j,:], sigmas[:,:,j]) .+ log(pis[j]) for j in 1:size(mus, 1)]'))
end

end


export Pow

"""
`Pow([domainType=Float64::Type,] dim_in::Tuple)`

Elementwise power `p` non-linear operator with input dimensions `dim_in`.

"""
struct Pow{T,N,I<:Real} <: NonLinearOperator
	dim::NTuple{N,Int}
    p::I
end

function Pow(DomainType::Type, DomainDim::NTuple{N,Int}, p::I) where {N, I <: Real}
	Pow{DomainType, N, I}(DomainDim, p)
end

Pow(DomainDim::NTuple{N,Int}, p::I) where {N, I <: Real} = Pow{Float64,N,I}(DomainDim,p)

function mul!(y::AbstractArray{T,N}, L::Pow{T,N,I}, x::AbstractArray{T,N}) where {T,N,I}
	y .= x.^L.p
end

function mul!(y::AbstractArray,
              J::AdjointOperator{Jacobian{Pow{T, N, I},TT}},
              b::AbstractArray) where {T, N, I, TT <: AbstractArray{T,N}}
    L = J.A
    y .= conj.(L.A.p.*(L.x).^(L.A.p-1)).*b
end

fun_name(L::Pow) = "『"

size(L::Pow) = (L.dim, L.dim)

domainType(L::Pow{T,N}) where {T,N} = T
codomainType(L::Pow{T,N}) where {T,N} = T


# Column sampling sketch of a matrix
# LinearTimeSVD - Drineas, et  al. "Fast monte carlo algorithms for matrices ii: computing a low rank approximation to a matrix", SIAM Journal of Computing, 2006.
#
# Danny Perry (dperry@cs.utah.edu)
# April 2015

using Debug

@debug function LinearTimeSVD(A, num_cols)
	n = size(A,1)
	d = size(A,2)
	c = num_cols

	C = zeros(c,d)
	norm_est = 0
	for i=1:n
		normi = norm(A[i,:])^2
		norm_est += normi
		pc = normi/norm_est
		scale = 1/sqrt(c*pc)
		for j=1:c
			if rand() <= pc
				C[j,:] = scale * A[i,:]
			end
		end
	end
	U,S2,V = svd(C*C')
	S = sqrt(S2) # S are the approximate singular values of A
	H = (C' * V' * pinv(diagm(S)))  # H is an approximate right singular vector of A (left singular vector of A').
	return H,S
end

# same but using leverage scores
@debug function LeverageLinearTimeSVD(A, Uk,k, num_cols)
	n = size(A,1)
	d = size(A,2)
	c = num_cols

	row_lev = sum(Uk.^2,2) ./ k

	C = zeros(c,d)
	for i=1:n
		pc = row_lev[i]
		scale = 1/sqrt(c*pc)
		for j=1:c
			if rand() <= pc
				C[j,:] = scale * A[i,:]
			end
		end
	end
	U,S2,V = svd(C*C')
	S = sqrt(S2) # S are the approximate singular values of A
	H = (C' * V' * pinv(diagm(S)))  # H is an approximate right singular vector of A (left singular vector of A').
	return H,S
end


export area

"""
    area(polygon::Polygon, radius::Float64=1.0)

Return the area of the spherical polygon. Without given radius a unit sphere is assumed.

Source: en.wikipedia.org/wiki/Spherical_trigonometry # Area and spherical excess

The method has been altered to allow for the usage of an internal point given with the Polygon.
This method only works with polygons that fit within half a sphere as it uses great circle distances between
subsequent points.
"""
function area(polygon::Polygon, radius::Float64=1.0)
    point₁ = polygon.points[1]
    area_polygon_unit_sphere = 0.0
    for point₂ in polygon.points[2:end]
        n_intersectionsᵢ = length(intersection_points(Arc(polygon.inside_point,
        midpoint(point₁, point₂)), polygon))
        isodd(n_intersectionsᵢ) ? area_polygon_unit_sphere +=
        area(polygon.inside_point, point₁, point₂) : area_polygon_unit_sphere -=
        area(polygon.inside_point, point₁, point₂)

        point₁ = point₂
    end
    return area_polygon_unit_sphere * radius^2
end

"""
    area(angular_distance₁₂::Float64, angular_distance₂₃::Float64, angular_distance₁₃::Float64,
    radius::Float64=1.0)

Return the area of the spherical triangle 123 based on the angular distances [deg] 12-23-13. Without given
radius a unit sphere is assumed.

Source: mathworld.wolfram.com/SphericalTriangle.html
"""
function area(angular_distance₁₂::Float64, angular_distance₂₃::Float64, angular_distance₁₃::Float64,
    radius::Float64=1.0)
    return deg2rad(spherical_excess(angular_distance₁₂, angular_distance₁₃, angular_distance₂₃))*radius^2
end

"""
    area(point₁::Point, point₂::Point, point₃::Point, radius::Float64=1.0)

Return the area of the spherical triangle 123 based on the points point₁ - point₂ - point₃. Without given radius
a unit sphere is assumed.

Source: mathworld.wolfram.com/SphericalTriangle.html
"""
area(point₁::Point, point₂::Point, point₃::Point, radius::Float64=1.0) =
area(angular_distance(point₁, point₂), angular_distance(point₂, point₃), angular_distance(point₃, point₁),
radius)


using JuMP, Gurobi, Distributions, StatsFuns

const gurobi_env = Gurobi.Env()

# DIMENSIONS
const numVariables = 100
const numJCC = 100
const conRHS = numVariables^2

# CASES
# number of samples, replicates for scenario approximations
const NumSamples = ceil.(Int64,logspace(1,log10(50000.0),50))
const NumReplicates = 20

# FILENAMES
const objFile = "objectiveValue.txt"
const solnFile = "solution.txt"
const riskFile = "riskLevel.txt"
const riskTimeFile = "riskTime.txt"
const solnTimeFile = "solutionTime.txt"
const checkingTimeFile = "checkingTime.txt"
const sortingTimeFile = "sortingTime.txt"
const callbackFile = "numCallbacks.txt"
const numConstraintsFile = "numConstraintsEnforced.txt"

xi_mean = zeros(Float64,numVariables)
xi_covariance = 0.5*ones(Float64,numJCC,numJCC)

for j = 1:numVariables
	xi_mean[j] = j*1.0/numVariables
end

for i = 1:numJCC
	xi_covariance[i,i] = 1.0
end

xi_cov_chol_tmp = cholfact(xi_covariance)
xi_cov_chol = xi_cov_chol_tmp[:L]

xi_mean_mat = zeros(Float64,numJCC,numVariables)
for j = 1:numVariables
	xi_mean_mat[:,j] = xi_mean[j]*ones(numJCC)
end

# tailored implementation of multivariate normal distribution
function myMvNormal(numSamples::Int64)

	xi = zeros(Float64,numJCC,numVariables,numSamples)

	xi_tmp = rand(Normal(0.0,1.0),numJCC,numVariables,numSamples)
	for samp = 1:numSamples
		xi[:,:,samp] = xi_cov_chol*xi_tmp[:,:,samp] + xi_mean_mat
	end

	return xi
end

# generate random samples from multivariate normal distribution
function generateRandomSamples(numSamples::Int64)

	xi = myMvNormal(numSamples)

	return xi
end

srand(1234)

# generate samples to estimate actual risk level of solution
const numAnalyticalSamples = 20000
const xi_analytical = generateRandomSamples(numAnalyticalSamples)

srand()

# compute negative probability of all constraints being satisfied
# for given scenarios of demands d and a given decision vector x
function computeRiskLevel(x::Array{Float64},reliabilityLevel::Float64=1E-06)

	const numSamples = size(xi_analytical,3)
	simpleEst::Bool = false

	numViolated::Int64 = checkScenarioConstraints(x,xi_analytical)

	riskLevel::Float64 = Inf

	if(simpleEst)
		riskLevel = numViolated*1.0/numSamples
		return riskLevel
	else
		if(numViolated == 0)
			riskLevel = -log(reliabilityLevel)/numSamples
			return riskLevel
		end

		 # termination criterion for bisection
		gamma_tolerance::Float64 = 0.1/numSamples

		# use tail bounds on binomial distribution to estimate lower and upper bounds
		gamma_low::Float64 = (numViolated - sqrt(-numSamples*log(reliabilityLevel)/2.0))/numSamples
		if(gamma_low < 0.0)
			gamma_low = 0.0
		end
		gamma_up::Float64 = (numViolated + sqrt(-numSamples*log(reliabilityLevel)/2.0))/numSamples
		if(gamma_up > 1.0)
			gamma_up = 1.0
		end

		gamma::Float64 = 0.0
		func_value::Float64 = 0.0

		# use bisection to solve the nonlinear equation
		while (gamma_up - gamma_low > gamma_tolerance)

			gamma = (gamma_low + gamma_up)/2.0
			func_value = -reliabilityLevel
			for i = 1:numViolated
				tmp_func = i*log(gamma) + (numSamples-i)*log(1-gamma) + lfact(numSamples) - lfact(numSamples-i) - lfact(i)
				func_value += exp(tmp_func)
				if(func_value > 0)
					break
				end
			end
			if(func_value > 0)
				gamma_low = (gamma_low + gamma_up)/2.0
			else
				gamma_up = (gamma_low + gamma_up)/2.0
			end

		end

		riskLevel = gamma_up
		return riskLevel
	end

end

# determines if all constraints are satisfied for a given single scenario
# returns one if scenario constraints are all satisfied, zero otherwise
function checkScenarioConstraints(x::Array{Float64},xi::Array{Float64})

	numSamples::Int64 = size(xi,3)

	numViolated::Int64 = 0
	for iter = 1:numSamples
		for i = 1:numJCC
			con::Float64 = evaluateConstraint(i,x,xi[i,:,iter])
			if(con > 0)
				numViolated += 1
				break
			end
		end
	end

	return numViolated
end

# evaluate the constraint values for given decision vector x,
# realization of the random variables xi, and constraint scalings
function evaluateConstraint(index::Int64,x::Array{Float64},xi::Array{Float64})

	con::Float64 = norm(xi.*x)^2 - conRHS

	return con
end

# determines if all constraints are satisfied for a given single scenario
# returns one if scenario constraints are all satisfied, zero otherwise
function getScenarioConstraintViolations(x::Array{Float64},xi::SubArray{Float64,2,Array{Float64,3},Tuple{Int64,Base.Slice{Base.OneTo{Int64}},Base.Slice{Base.OneTo{Int64}}},true},pickFirstSetOfConstraints::Bool,maxNumConstraintsPerIteration::Int64)

	const numSamples = size(xi,2)
	numViolated::Int64 = 0
	constraintViolations = Float64[]
	constraintViolationIndices = Int64[]
	constraintViolationTolerance::Float64 = 1E-06

	for iter = 1:numSamples
		@views con::Float64 = norm(xi[:,iter].*x) - sqrt(conRHS)

		if(con >= constraintViolationTolerance)
			numViolated += 1
			push!(constraintViolations,con)
			push!(constraintViolationIndices,iter)
			if(pickFirstSetOfConstraints && numViolated == maxNumConstraintsPerIteration)
				break
			end
		end
	end

	return numViolated, constraintViolations, constraintViolationIndices
end

# write solution to file
function initializeFiles(details_file::String)

	writeMode::String = "w"

	open(details_file, writeMode) do f
		write(f,"Model: $modelName \n")
		write(f,"numVariables: $numVariables \n")
		write(f,"numJCC: $numJCC \n")
		write(f,"conRHS: $conRHS \n")
		write(f,"NumSamples: $NumSamples \n")
		write(f,"NumReplicates: $NumReplicates \n")
		write(f,"Method: Scenario approximations \n")
	end

end

# write solution to file
function writeSolutionToFile(dirName::String, status::Symbol, objvar_opt::Float64, x_opt::Array{Float64}, riskLevel::Float64, solutionTime::Float64, checkingTime::Float64, sortingTime::Float64, numCallbacks::Int64, numConstraintsAdded::Int64, riskTime::Float64)

	if(status == Symbol("Optimal"))
		writeMode::String = "a"

		obj_file::String = dirName * objFile
		open(obj_file, writeMode) do f
			write(f,"$objvar_opt \n")
		end

		solution_file::String = dirName * solnFile
		open(solution_file, writeMode) do f
			for i = 1:numVariables
				write(f,"$(x_opt[i]) ")
			end
			write(f,"\n")
		end

		risk_file::String = dirName * riskFile
		open(risk_file, writeMode) do f
			write(f,"$riskLevel \n")
		end

		risktime_file::String = dirName * riskTimeFile
		open(risktime_file, writeMode) do f
			write(f,"$riskTime \n")
		end

		solntime_file::String = dirName * solnTimeFile
		open(solntime_file, writeMode) do f
			write(f,"$solutionTime \n")
		end

		checktime_file::String = dirName * checkingTimeFile
		open(checktime_file, writeMode) do f
			write(f,"$checkingTime \n")
		end

		sorttime_file::String = dirName * sortingTimeFile
		open(sorttime_file, writeMode) do f
			write(f,"$sortingTime \n")
		end

		callback_file::String = dirName * callbackFile
		open(callback_file, writeMode) do f
			write(f,"$numCallbacks \n")
		end

		constraints_file::String = dirName * numConstraintsFile
		open(constraints_file, writeMode) do f
			write(f,"$numConstraintsAdded \n")
		end
	end

end

# solve the scenario approximation model
function solveScenarioApproximationModel(numScenarios::Int64,dirName::String)

	tic()

	# sample the random variables
	xi = generateRandomSamples(numScenarios)

	maxNumConstraintsPerIteration::Int64 = 10
	numConstraintsAdded::Int64 = 0

	pickFirstSetOfConstraints::Bool = false

	sortingTime::Float64 = 0.0
	checkingTime::Float64 = 0.0
	numCallbacks::Int64 = 0

	objvar_opt::Float64 = Inf
	y_opt = zeros(Float64,numVariables)
	status::Symbol = Symbol("Error")

	numViolations::Int64 = numScenarios

	constraintIndices = [Int64[] for i=1:numJCC]

	while(numViolations > 0)

		numCallbacks += 1
		numViolations = 0

		numConstraintsConsidered = zeros(Int64,numJCC)
		for i = 1:numJCC
			numConstraintsConsidered[i] = size(constraintIndices[i])[1]
		end

		mod = Model(solver=GurobiSolver(gurobi_env,Presolve=0,OutputFlag=0))

		@variable(mod, objvar)
		@variable(mod, 0 <= y[1:numVariables] <= 10^6)

		@objective(mod, Min, objvar)
		@constraint(mod, -sum(y[j] for j = 1:numVariables) <= objvar)
		@constraint(mod, [i=1:numJCC, scen=1:numConstraintsConsidered[i]], norm(xi[i,:,constraintIndices[i][scen]].*y) <= sqrt(conRHS))

		status = solve(mod)
		objvar_opt = getvalue(objvar)
		y_opt = getvalue(y)

		for i = 1:numJCC

			numViolatedConstraints::Int64 = 0
			constraintViolations = Float64[]
			constraintViolationIndices = Int64[]

			tic()

			@views numViolatedConstraints, constraintViolations, constraintViolationIndices = getScenarioConstraintViolations(y_opt,xi[i,:,:],pickFirstSetOfConstraints,maxNumConstraintsPerIteration)

			checkingTime += toq()

			numConstraintsEnforced::Int64 = min(size(constraintViolations,1),maxNumConstraintsPerIteration)

			tic()
			relevantIndices = 1:1:numConstraintsEnforced
			if(!pickFirstSetOfConstraints)
				relevantIndices = sortperm(constraintViolations, rev=true)
			end
			sortingTime += toq()

			for iter = 1:numConstraintsEnforced
				push!(constraintIndices[i],constraintViolationIndices[relevantIndices[iter]])
			end

			numConstraintsAdded += numConstraintsEnforced

			numViolations += numViolatedConstraints
		end

	end

	tic()

	riskLevel::Float64 = computeRiskLevel(y_opt)

	riskTime = toq()

	solutionTime::Float64 = toq()

	@printf "  Objective: %2.5f,  Risklevel: %.6f,  Time: %4.2f,  NumCallbacks: %d,  NumConstraints: %d \n" objvar_opt riskLevel solutionTime numCallbacks numConstraintsAdded

	writeSolutionToFile(dirName, status, objvar_opt, y_opt, riskLevel, solutionTime, checkingTime, sortingTime, numCallbacks, numConstraintsAdded, riskTime)

	return status, objvar_opt, y_opt, riskLevel, solutionTime, checkingTime, sortingTime, numCallbacks, numConstraintsAdded, riskTime
end


function eigmaxreal(A::Matrix)
  if size(A,1)<= 2
    return maximum(real(eigvals(A)))
  else
    return eigs(A,nev=1,which = :LR)[1][1]
  end
end
function eigminreal(A::Matrix)
  if size(A,1)<= 2
    return minimum(real(eigvals(A)))
  else
    return eigs(A,nev=1,which = :SR)[1][1]
  end
end


# This file calculates the first application in Y.-S. Chan, A. C. Fannjiang, and G. H. Paulino,
# Integral equations with hypersingular kernels -- theory and applications to fracture mechanics,
# Int. J. Eng. Sci., 41:683--720, 2003.

using ApproxFun, SingularIntegralEquations

x = Fun(identity)
w = 1/sqrt(1-x^2)
d = domain(x)
d2 = d^2
B = dirichlet(d)
H2 = Hilbert(d,2)
Σ = DefiniteIntegral(d)
ϵ = 2.0
K = LowRankFun((x,y)->-1./(x+y+2ϵ).^2+12(x+ϵ)./(x+y+2ϵ).^3-12(x+ϵ).^2./(x+y+2ϵ).^4,d2)
L = H2[w] + Σ[K*(w/π)]
f = -Fun(one)

uSIE = [B;L]\[zeros(2);f]
@time uSIE = [B;L]\[zeros(2);f]
println("The rank of K is: ",rank(K))
println("The length of uSIE is: ",length(uSIE))
println("The extrema of uSIE are: ",extrema(uSIE))
println("The normalized Stress Intensify Factors are: ",(uSIE/(1-x^2))(-1),"  ",(uSIE/(1-x^2))(1))


abstract AbstractMeasure{T}

abstract SymmetricMeasure{T} <: AbstractMeasure{T}

# Entropy
# =======

"""
Shannon entropy (H)
"""
immutable Entropy{T} <: SymmetricMeasure{T}
  base::T
end

call{T}(::Type{Entropy{T}}) = Entropy(T(Base.e))

## Estimate Entropy using ResidueProbability

"""
`estimate(Entropy(base), p)`

`p` should be a `ResidueProbability` table. The result type is determined by `base`.
"""
function estimate{B, T, N, UseGap}(measure::Entropy{B}, p::ResidueProbability{T, N, UseGap})
  H = zero(B)
  for i in 1:length(p)
    @inbounds pi = B(p[i])
    if pi != 0.0
      H += pi * log(pi)
    end
  end
  -H/log(measure.base)
end

"""
`estimate_on_marginal(Entropy{T}(base), p, marginal)`

This function estimate the entropy H(X) if marginal is 1, H(Y) for 2, etc.
The result type is determined by `base`.
"""
function estimate_on_marginal{B, T, N, UseGap}(measure::Entropy{B}, p::ResidueProbability{T, N, UseGap}, marginal::Int)
  H = zero(B)
  for i in 1:nresidues(p)
    @inbounds pi = B(p.marginals[i, marginal])
    if pi != 0.0
      H += pi * log(pi)
    end
  end
  -H/log(measure.base)
end

## Estimate Entropy using ResidueCount

"""
`estimate(Entropy{T}(base), n::ResidueCount)`

It's the fastest option (you don't spend time on probability calculations).
The result type is determined by the `base`.
"""
function estimate{T}(measure::Entropy{T}, n::ResidueCount)
  H = zero(T)
  total = T(n.total)
  for i in 1:length(n)
    @inbounds ni = T(n[i])
    if ni != 0.0
      H += ni * log(ni/total)
    end
  end
  (-H/total)/log(measure.base)
end

function estimate_on_marginal{T}(measure::Entropy{T}, n::ResidueCount, marginal::Int)
  H = zero(T)
  total = T(n.total)
  for i in 1:nresidues(n)
    @inbounds ni = T(n.marginals[i, marginal])
    if ni != 0.0
      H += ni * log(ni/total)
    end
  end
  (-H/total)/log(measure.base)
end

# Kullback-Leibler
# ================

"""
Kullback-Leibler (KL). This `SymmetricMeasure` has two fields.
The first is the base of the logarithm and the second is the backgroud frequency.
"""
immutable KullbackLeibler{T} <: SymmetricMeasure{T}
  base::T
  background::AbstractArray{T}
end

call{T}(::Type{KullbackLeibler{T}}, background::AbstractArray{T}) = KullbackLeibler(T(Base.e), background)

"""
`estimate(KullbackLeibler(base, background), p)`

`p` should be a `ResidueProbability` table, and `background` must have the size of `p`.
The result type is determined by `base` and `background`.
"""
function estimate{B, T, N, UseGap}(measure::KullbackLeibler{B}, p::ResidueProbability{T, N, UseGap})
  q = measure.background
  if size(q) != size(p) || length(q) != length(p)
    throw(ErrorException("p and background should have the same size and length."))
  end
  KL = zero(B)
  @inbounds for i in 1:length(p)
    pi = B(p[i])
    if pi != 0.0
      KL += pi * log(pi/q[i])
    end
  end
  KL/log(measure.base)
end

# Mutual Information
# ==================

"""
Mutual Information (MI)
"""
immutable MutualInformation{T} <: SymmetricMeasure{T}
  base::T
end

call{T}(::Type{MutualInformation{T}}) = MutualInformation(T(Base.e))

@inline _mi{T}(::Type{T}, pij, pi, pj) = ifelse(pij > zero(T) && pi > zero(T), T(pij * log(pij/(pi*pj))), zero(T))

"""
`estimate(MutualInformation(), pxy::ResidueProbability [, base])`

Calculate Mutual Information from `ResidueProbability`. The result type is determined by `base`.
"""
function estimate{B, T, UseGap}(measure::MutualInformation{B}, pxy::ResidueProbability{T, 2,UseGap})
  MI = zero(B)
  marginals = pxy.marginals
  @inbounds for j in 1:nresidues(pxy)
    pj = marginals[j,2]
    if pj > 0.0
      @inbounds @simd for i in 1:nresidues(pxy)
        MI +=  _mi(B, pxy[i,j], marginals[i,1], pj)
      end
    end
  end
  MI/log(measure.base)
end

@inline _mi{T}(N::T, nij, ni, nj) = ifelse(nij > zero(T) && ni > zero(T), T(nij * log((N * nij)/(ni * nj))), zero(T))

"""
`estimate(MutualInformation(), pxy::ResidueCount [, base])`

Calculate Mutual Information from `ResidueCount`. The result type is determined by the `base`.
It's the fastest option (you don't spend time on probability calculations).
"""
function estimate{B, T, UseGap}(measure::MutualInformation{B}, nxy::ResidueCount{T, 2,UseGap})
  MI = zero(B)
  N = B(nxy.total)
  marginals = nxy.marginals
  @inbounds for j in 1:nresidues(nxy)
    nj = marginals[j,2]
    if nj > 0.0
      @inbounds @simd for i in 1:nresidues(nxy)
        MI += _mi(N, nxy[i,j], marginals[i,1], nj)
      end
    end
  end
  (MI/N)/log(measure.base)
end

function estimate{B, T, UseGap}(measure::MutualInformation{B}, pxyz::ResidueContingencyTables{T, 3, UseGap})
  pxy = delete_dimensions(pxyz,3)
  return( estimate_on_marginal(Entropy(measure.base), pxyz, 1) + # H(X)
  estimate_on_marginal(Entropy(measure.base), pxyz ,2) + # H(Y)
  estimate_on_marginal(Entropy(measure.base), pxyz, 3) - # H(Z)
  estimate(Entropy(measure.base), pxy) - # H(X, Y)
  estimate(Entropy(measure.base), delete_dimensions!(pxy, pxyz, 2)) - # H(X, Z)
  estimate(Entropy(measure.base), delete_dimensions!(pxy, pxyz, 1)) + # H(Y, Z)
  estimate(Entropy(measure.base), pxyz) ) # H(X, Y, Z)
end

# Normalized Mutual Information by Entropy
# ========================================

"""
Normalized Mutual Information (nMI) by Entropy.

`nMI(X, Y) = MI(X, Y) / H(X, Y)`
"""
immutable MutualInformationOverEntropy{T} <: SymmetricMeasure{T}
  base::T
end

call{T}(::Type{MutualInformationOverEntropy{T}}) = MutualInformationOverEntropy(T(Base.e))

function estimate{B}(measure::MutualInformationOverEntropy{B}, table)
  H = estimate(Entropy(measure.base), table)
  if H != zero(B)
    MI = estimate(MutualInformation(measure.base), table)
    return(MI/H)
  else
    return(zero(B))
  end
end

# Pairwise Gap Percentage
# =======================

"""
`GapUnionPercentage`
"""
immutable GapUnionPercentage{T} <: SymmetricMeasure{T} end

"""
`GapIntersectionPercentage`
"""
immutable GapIntersectionPercentage{T} <: SymmetricMeasure{T} end

function estimate{B, T}(measure::GapIntersectionPercentage{B}, nxy::ResidueCount{T, 2, true})
  B(100.0) * B(nxy[21, 21]) / B(sum(nxy))
end

function estimate{B, T}(measure::GapUnionPercentage{B}, nxy::ResidueCount{T, 2, true})
  B(100.0) * B(nxy.marginals[21, 1] + nxy.marginals[21, 2] - nxy[21, 21]) / B(sum(nxy))
end


# SPDX-License-Identifier: MIT

@testset "logaddexp" begin
    @test (@inferred MarkovModels.Semirings.logaddexp(2.0, 3.0)) ≈ log(exp(2.0) + exp(3.0))
    @test (@inferred MarkovModels.Semirings.logaddexp(10002.0, 10003.0)) ≈
        10000 + MarkovModels.Semirings.logaddexp(2.0, 3.0)
end


"""This function returns the successor of any given positive integer."""
function succ(n :: Int64) :: Int64
    return n + 1;
end;

"""This function implements the fast exponentiation algorithm"""
function fast_expo(n :: Int64, exp :: Int64) :: Int64
    if expo == 0
        return n
    elseif expo % 2 == 0
        let res = fast_expo(n, div(exp, 2))
        return (res * res)
    else
        let res = fast_expo(n, div(exp, 2))
        return (res * res * n)
    end
end


@testset "LaguerrePolynomial" begin
    @testset "LaguerrePolynomial" begin
        # test construction of LaguerrePolynomial
        orders = [0,2,3]
        coeffs = [1.0, 2, 0.5]
        L = LaguerrePolynomial(orders, coeffs)
        @test L.orders == orders
        @test all(isapprox.(coeffs, L.coeffs))
        # test single Laguerre Polynomial
        L = LaguerrePolynomial(2, 1 // 2)
        @test L.orders == [2]
        @test L.coeffs == [1 // 2]

        # Laguerre Polynomial L3 with coefficient 1.
        L = LaguerrePolynomial(3)
        @test L.orders == [3]
        @test L.coeffs == [1]
        # test throw
        @test_throws DimensionMismatch LaguerrePolynomial([1,2], [2,3,2])
        @test_throws DomainError LaguerrePolynomial(-2, 1)
        # test string representation
        L = LaguerrePolynomial([2,3], [1,2])
        st = Laguerre.lashow(L)
        @test st == "(1)L2 + (2)L3"
    end

    @testset "LaguerrePolynomial addition" begin
        L1 = LaguerrePolynomial(2, 1 // 2)
        L2 = LaguerrePolynomial(3, 1.0)
        L = L1 + L2
        @test typeof(L) == LaguerrePolynomial{typeof(1.0)}
        @test L.orders == [2, 3]
        @test all(isapprox.(L.coeffs, [1 / 2, 1.0]))
        #
        L1 = LaguerrePolynomial(2, 1)
        L2 = LaguerrePolynomial(2, 1 // 3)
        L = L1 + L2
        @test typeof(L) == LaguerrePolynomial{typeof(1 // 3)}
        @test L.orders == [2]
        @test L.coeffs == [1 + 1 // 3]

        # test subtraction
        L = L1 - L2
        @test L.orders == [2]
        @test L.coeffs == [1 - 1 // 3]

    end

    @testset "LaguerrePolynomial * number" begin
        orders = [0,2,3]
        coeffs = [1, 2, 2]
        L1 = LaguerrePolynomial(orders, coeffs)
        L2 = L1 * 3
        # multiplication by a number
        @test L2.orders == [0,2,3]
        @test L2.coeffs == 3 * coeffs
        # division by a number
        @test_throws ErrorException L1 / 0
        L3 = L1 / 2
        @test L3.orders == L1.orders
        @test L3.coeffs == L1.coeffs / 2
    end

    @testset "LaguerrePolynomials product" begin
        k = 5
        L2 = LaguerrePolynomial(2, 1.0)
        Lk = LaguerrePolynomial(k, 1.0)
        L = L2 * Lk
        # L2Lk = k(k-1)/2 L_{k-2} -2k(k-1)L_{k-1}
        #   +(3k^2-k)L_k -2k(k+1)L_{k+1}
        #   (k+1)(k+2)/2 L_{k+2}
        res = k * (k - 1) / 2 * LaguerrePolynomial(k - 2)  -
        2 * k * (k - 1) * LaguerrePolynomial(k - 1) +
        (3 * k^2 - k) * LaguerrePolynomial(k) -
        2 * k * (k + 1) * LaguerrePolynomial(k + 1) +
        (k + 1) * (k + 2) / 2 * LaguerrePolynomial(k + 2)
        @test L == res

        # L1Lk = k L_{k-1} -2k L_k +(k+1) L_{k+1}
        L1 = LaguerrePolynomial(1, 1.0)
        Lk = LaguerrePolynomial(k, 1.0)
        L = L1 * Lk
        res = k * LaguerrePolynomial(k - 1) - 2 * k * LaguerrePolynomial(k) + (k + 1) * LaguerrePolynomial(k + 1)
        @test L ==  res
        @test Lk * LaguerrePolynomial(0) == Lk
    end
end


n = 10 # rand(1:10)
A = ones(Float64, n, n)
@test matrixdepot("circul", ones(Float64,n)) ≈ A

x = [1:n;];
A = matrixdepot("circul", x)
# test symmetry
@test A + A' == (A + A')'
B = matrixdepot("circul", n)
C = matrixdepot("circul", [1:n;], n)
@test B ≈ C

println("'circul' passed test..." )


# sort an array - aplying a function to the sort process
a1 = [1, 2, 0, -2, 7, 3, -11, 12, -43]
res = sort(a1, by=abs) # sort on the basis of the elements absolute value ...
println("$a1  / $res")

# is this array sorted
a2 = [1, 3, 6, 3]
println(issorted(a1))

a3 = [(1, 2), (2, 4), (3, 1), (5, 2)]
println(issorted(a3, by=x -> x[1])) # is this array sorted on the basis of the lambda function provided - get the first element of each tuple

# sorting custom types (structs)
struct Rect
    length::Int
    width::Int
end

a4 = [Rect(12, 22), Rect(10, 18), Rect(7, 12), Rect(9, 30)]

# create a custom sort function
function comparearea(r1::Rect, r2::Rect)
    area1 = r1.length * r1.width
    area2 = r2.length * r2.width

    return area1 < area2
end

sort!(a4, lt=comparearea) # sort 'in place', provide a 'less than' function (lt)
println(a4)


Base.:+(p::Pol) = p
Base.:-(p::Pol) = Pol([-c for c in p.coeffs])

function Base.:+(p1::Pol, p2::Pol)
    maxdeg = max(deg(p1), deg(p2))
    coeffs = [p1[i]+p2[i] for i in 1:maxdeg+1]
    return Pol(coeffs)
end

Base.:+(p1::Union{Pol, Number}, p2::Union{Pol, Number}) = +(promote(p1, p2)...)
Base.:-(p1::Union{Pol, Number}, p2::Union{Pol, Number}) = p1 + (-p2)

function Base.:*(p1::Pol{T}, p2::Pol{T}) where {T<:Number}
    coeffs = zeros(T, deg(p1)+deg(p2)+1)
    @inbounds for (i, c1) in enumerate(p1.coeffs)
        @inbounds for (j, c2) in enumerate(p2.coeffs)
            coeffs[i+j-1] += c1*c2
        end
    end
    return Pol(coeffs)
end

Base.:*(p1::Union{Pol, Number}, p2::Union{Pol, Number}) = *(promote(p1, p2)...)

Base.:/(p::Pol, n::Number)  = Pol([c/n for c in p.coeffs])

function Base.:^(p::Pol, n::Integer)
    iszero(n) && return Pol([1])
    if iseven(n)
        return (p*p)^(n÷2)
    else
        return p*(p*p)^((n-1)÷2)
    end
end

function Base.divrem(a::Pol, b::Pol)
    r = a
    q = zero(a)
    d = deg(b)
    c = lc(b)
    while !iszero(r) && deg(r) >= d
        s = lc(r)/c
        coeffs = zeros(typeof(s), deg(r)-d+1)
        coeffs[end] = s
        s = Pol(coeffs)
        q, r = q + s, r - s*b
    end
    return q, r
end

function Base.div(a::Pol, b::Pol)
    q, _ = divrem(a, b)
    return q
end

function Base.rem(a::Pol, b::Pol)
    _, r = divrem(a, b)
    return r
end

function Base.gcd(a::Pol, b::Pol)
    r0 = a
    r1 = b
    while !iszero(r1)
        r1, r0 = rem(r0, r1), r1
    end
    return r0/lc(r0)
end

# comparisons

function Base.:(==)(p1::Pol, p2::Pol)
    deg(p1) == deg(p2) || return false
    @inbounds for i in 1:length(p1.coeffs)
        p1.coeffs[i] == p2.coeffs[i] || return false
    end
    return true
end

for op in (:>, :<, :>=, :<=)
    @eval Base.$op(p1::Pol, p2::Pol) = $op(deg(p1), deg(p2))
end

Base.iszero(p::Pol) = iszero(p.coeffs)


using Unitful#, UnitfulPlots
using OrdinaryDiffEq, DiffEqBase

const UNITS_TEST_ALGS = [Euler(),Midpoint(),RK4(),SSPRK104(),SSPRK22(),SSPRK33(),
    SSPRK432(),BS3(),BS5(),DP5(),DP5Threaded(),DP8(),Feagin10(),Feagin12(),Feagin14(),
    TanYam7(),Tsit5(),TsitPap8(),Vern6(),Vern7(),Vern8(),Vern9()]

f = (y,p,t) -> 0.5*y / 3.0u"s"
u0 = 1.0u"N"
prob = ODEProblem(f,u0,(0.0u"s",1.0u"s"))

sol =solve(prob,ExplicitRK())

for alg in UNITS_TEST_ALGS
  if !(typeof(alg) <: DP5Threaded)
    @show alg
    sol = solve(prob,alg,dt=1u"s"/10)
  end
end

println("Units for Number pass")

u0 = [1.0u"N" 2.0u"N"
      3.0u"N" 1.0u"N"]
f = (dy,y,p,t) -> (dy .= 0.5.*y ./ 3.0u"s")
prob = ODEProblem(f,u0,(0.0u"s",1.0u"s"))

sol =solve(prob,ExplicitRK())

for alg in UNITS_TEST_ALGS
  @show alg
  sol = solve(prob,alg,dt=1u"s"/10)
end

println("Units for 2D pass")

using Unitful, RecursiveArrayTools, DiffEqBase, OrdinaryDiffEq
using LinearAlgebra
r0 = [1131.340, -2282.343, 6672.423]u"km"
v0 = [-5.64305, 4.30333, 2.42879]u"km/s"
Δt = 86400.0*365u"s"
μ = 398600.4418u"km^3/s^2"
rv0 = ArrayPartition(r0,v0)

function f(dy, y, μ, t)
    r = norm(y.x[1])
    dy.x[1] .= y.x[2]
    dy.x[2] .= -μ .* y.x[1] / r^3
end

prob = ODEProblem(f,rv0,(0.0u"s",1.0u"s"),μ)
sol = solve(prob,Tsit5())

# coordinate: u = [position, momentum]
# parameters: p = [mass, force constanst]
function f_harmonic!(du,u,p,t)
  du[1] = u[2]/p[1]
  du[2] = -p[2]*u[1]
end

mass = 1.0u"kg"
k = 1.0u"N/m"
p = [mass, k]

u0 = [1.0u"m", 0.0u"kg*m/s"] # initial values (position, momentum)
tspan = (0.0u"s", 10.0u"s")
prob = ODEProblem(f_harmonic!, u0, tspan, p)
sol = solve(prob)


# Tests for partial information decomposition function

# Test partial information decomposition for AND gate
arr_and = [arr1[i] >= 0.5 && arr2[i] >= 0.5 ? 1.0 : 0.0 for i in 1:length(arr1)]
pid = get_partial_information_decomposition(arr1, arr2, arr_and)
@test pid["redundancy"] ≈ 0.3 atol = 0.1
@test pid["unique_1"] ≈ 0 atol = 0.1
@test pid["unique_2"] ≈ 0 atol = 0.1
@test pid["synergy"] ≈ 0.5 atol = 0.1
println("Partial information decomposition with AND gate passed.")

pid = get_partial_information_decomposition(arr1, arr2, arr_and, all_orientations = true)
@test pid["z"]["redundancy"] ≈ 0.3 atol = 0.1
@test pid["z"]["unique_1"] ≈ 0 atol = 0.1
@test pid["z"]["unique_2"] ≈ 0 atol = 0.1
@test pid["z"]["synergy"] ≈ 0.5 atol = 0.1
println("Partial information decomposition with AND gate (all orientations) passed.")

# Test redundancy for AND gate
redundancy = get_redundancy(arr1, arr2, arr_and)
@test redundancy ≈ 0.3 atol = 0.1
println("Redundancy with AND gate passed.")

redundancy = get_redundancy(arr1, arr2, arr_and, all_orientations = true)[1]
@test redundancy ≈ 0.3 atol = 0.1
println("Redundancy with AND gate (all orientations) passed.")

# Test unique information for AND gate
mutual_information = get_mutual_information(arr1, arr_and)
@test apply_unique_information_formula(mutual_information, redundancy) ≈ 0 atol = 0.1
println("Unique information with AND gate passed.")

# Test synergy for AND gate
interaction_information = get_interaction_information(arr1, arr2, arr_and)
@test apply_synergy_formula(interaction_information, redundancy) ≈ 0.5 atol = 0.1
println("Synergy with AND gate passed.")


# # Some Radial Functions

# The **Gaussian** is defined by ``φ(ρ) = \exp \left( - (αρ)^2 \right)``, where
# ``α`` is a shape parameter to fine-tune the function.

"""
    Gaussian( α = 1 ) <: RadialFunction

A `RadialFunction` with
```math
    φ(ρ) = \\exp( - (α ρ)^2 ).
```
"""
@with_kw struct Gaussian{R<:Real} <: RadialFunction
    α :: R = 1
    @assert α > 0 "The shape parameter `α` must be positive."
end

function ( φ :: Gaussian )( ρ :: Real )
    exp( - (φ.α * ρ)^2 )
end

cpd_order( :: Gaussian ) = 0
df(φ :: Gaussian, ρ :: Real) = - 2 * φ.α^2 * ρ * φ( ρ )

# The **Multiquadric** is ``φ(ρ) = - \sqrt{ 1 + (αρ)^2 }`` and also has a positive shape
# parameter. We can actually generalize it to the following form:

"""
    Multiquadric( α = 1, β = 1//2 ) <: RadialFunction

A `RadialFunction` with
```math
    φ(ρ) = (-1)^{ \\lceil β \\rceil } ( 1 + (αρ)^2 )^β
```
"""
@with_kw struct Multiquadric{R<:Real,S<:Real} <: RadialFunction
    α :: R  = 1     # shape parameter
    β :: S  = 1//2  # exponent

    @assert α > 0 "The shape parameter `α` must be positive."
    @assert β % 1 != 0 "The exponent must not be an integer."
    @assert β > 0 "The exponent must be positive."
end

function ( φ :: Multiquadric )( ρ :: Real )
    (-1)^(ceil(Int, φ.β)) * ( 1 + (φ.α * ρ)^2 )^φ.β
end

cpd_order( φ :: Multiquadric ) = ceil( Int, φ.β )
df(φ :: Multiquadric, ρ :: Real ) = (-1)^(ceil(Int, φ.β)) * 2 * φ.α * φ.β * ρ * ( 1 + (φ.α * ρ)^2 )^(φ.β - 1)

# Related is the **Inverse Multiquadric** `` φ(ρ) = (1+(αρ)^2)^{-β}``:
"""
    InverseMultiquadric( α = 1, β = 1//2 ) <: RadialFunction

A `RadialFunction` with
```math
    φ(ρ) = ( 1 + (αρ)^2 )^{-β}
```
"""
@with_kw struct InverseMultiquadric{R<:Real,S<:Real} <: RadialFunction
    α :: R  = 1
    β :: S  = 1//2

    @assert α > 0 "The shape parameter `α` must be positive."
    @assert β > 0 "The exponent must be positive."
end

function ( φ :: InverseMultiquadric )( ρ :: Real )
   ( 1 + (φ.α * ρ)^2 )^(-φ.β)
end

cpd_order( :: InverseMultiquadric ) = 0
df(φ :: InverseMultiquadric, ρ :: Real ) = - 2 * φ.α^2 * φ.β * ρ * ( 1 + (φ.α * ρ)^2 )^(-φ.β - 1)

# The **Cubic** is ``φ(ρ) = ρ^3``.
# It can also be generalized:
"""
    Cubic( β = 3 ) <: RadialFunction

A `RadialFunction` with
```math
    φ(ρ) = (-1)^{ \\lceil β \\rceil /2 } ρ^β
```
"""
@with_kw struct Cubic <: RadialFunction
    β :: Int = 3

    @assert β > 0 "The exponent `β` must be positive."
    @assert β % 2 != 0 "The exponent `β` must not be an even number."
end

function ( φ :: Cubic )( ρ :: Real )
    (-1)^ceil(Int, φ.β/2 ) * ρ^φ.β
end

cpd_order( φ :: Cubic ) = ceil( Int, φ.β/2 )
df(φ :: Cubic, ρ :: Real ) = (-1)^(ceil(Int, φ.β/2)) * φ.β * ρ^(φ.β - 1)

# The thin plate spline is usually defined via
# ``φ(ρ) = ρ^2 \log( ρ )``.
# We provide a generalized version, which defaults to
# ``φ(ρ) = - ρ^4 \log( ρ )``.
"""
    ThinPlateSpline( k = 2 ) <: RadialFunction

A `RadialFunction` with
```math
    φ(ρ) = (-1)^{k+1} ρ^{2k} \\log(ρ)
```
"""
@with_kw struct ThinPlateSpline <: RadialFunction
    k :: Int = 2

    @assert k > 0 && k % 1 == 0 "The parameter `k` must be a positive integer."
end

function (φ :: ThinPlateSpline )( ρ :: T ) where T<:Real
    ρ == 0 ? zero(T) : (-1)^(φ.k+1) * ρ^(2*φ.k) * log( ρ )
end

cpd_order( φ :: ThinPlateSpline ) = φ.k + 1
df(φ :: ThinPlateSpline, ρ :: Real ) = ρ == 0 ? 0 : (-1)^(φ.k+1) * ρ^(2*φ.k - 1) * ( 2 * φ.k * log(ρ) + 1)

# !!! note
#     The thin plate spline with `k = 1` is not differentiable at `ρ=0` but we define the derivative
#     as 0, which results in a continuous extension.


using .PDMats: PDMat

export kernelpdmat

"""
    kernelpdmat(k::Kernel, X::AbstractVector)

Compute a positive-definite matrix in the form of a `PDMat` matrix (see [PDMats.jl](https://github.com/JuliaStats/PDMats.jl)),
with the Cholesky decomposition precomputed.
The algorithm adds a diagonal "nugget" term to the kernel matrix which is increased until positive
definiteness is achieved. The algorithm gives up with an error if the nugget becomes larger than 1% of the largest value in the kernel matrix.
"""
function kernelpdmat(κ::Kernel, X::AbstractVector)
    K = kernelmatrix(κ, X)
    Kmax = maximum(K)
    α = eps(eltype(K))
    while !isposdef(K + α * I) && α < 0.01 * Kmax
        α *= 2.0
    end
    if α >= 0.01 * Kmax
        error(
            "Adding noise on the diagonal was not sufficient to build a positive-definite" *
            " matrix:\n\t- Check that your kernel parameters are not extreme\n\t- Check" *
            " that your data is sufficiently sparse\n\t- Maybe use a different kernel",
        )
    end
    return PDMat(K + α * I)
end

"""
    kernelpdmat(k::Kernel, X::AbstractMatrix; obsdim)

If `obsdim=1`, equivalent to `kernelpdmat(k, RowVecs(X))`.
If `obsdim=2`, equivalent to `kernelpdmat(k, ColVecs(X))`.

See also: [`ColVecs`](@ref), [`RowVecs`](@ref)
"""
function kernelpdmat(κ::Kernel, X::AbstractMatrix; obsdim::Union{Int,Nothing}=defaultobs)
    return kernelpdmat(κ, vec_of_vecs(X; obsdim=obsdim))
end


using ChaosTools, Test

lo = Systems.logistic(r = 3.5)
reg = trajectory(lo, 10000; Ttr = 1000)
set_parameter!(lo, 1, 3.97)
cha = trajectory(lo, 10000)
testchaos01(cha)

println("\nTesting 0-1 chaos test...")
@testset "logistic" begin
    lo = Systems.logistic(r = 3.5)
    reg = trajectory(lo, 10000; Ttr = 1000)
    set_parameter!(lo, 1, 3.97)
    cha = trajectory(lo, 10000)

    @test testchaos01(cha)
    @test !testchaos01(reg)

    cs = range(0.1, π-0.1, length = 20)

    @test testchaos01(cha, cs)
    @test !testchaos01(reg, cs)

end

@testset "Standard Map" begin
    ds = Systems.standardmap(k = 0.9)
    cs = range(0.1, π-0.1, length = 20)
    @testset "chaotic" begin
        cha = trajectory(ds, 10000)[:, 1]
        @test testchaos01(cha)
        @test testchaos01(cha, cs)
    end
    @testset "regular" begin
        reg = trajectory(ds, 20000, [π, 0.2rand()])[:, 1]
        @test !testchaos01(reg)
        @test !testchaos01(reg, cs)
    end
end

using OrdinaryDiffEq

@testset "henonheiles" begin
    sp = [0, .295456, .407308431, 0] #stable periodic orbit: 1D torus
    qp = [0, .483000, .278980390, 0] #quasiperiodic orbit: 2D torus
    ch = [0, -0.25, 0.42081, 0] # chaotic orbit
    tt = 3000
    dt = 0.8 # <- very important!
    ds = Systems.henonheiles()
    diffeq = Dict(:Ttr => 10, :abstol=>1e-9, :reltol=>1e-9, :solver => Vern9())
    cs = range(0.1, π/3-0.1, length = 50)

    @testset "stable" begin
        tr = trajectory(ds, tt, sp; dt = dt, diffeq...)[:, 1]
        @test !testchaos01(tr)
        @test !testchaos01(tr, cs)
    end
    @testset "quasiperidic" begin
        tr = trajectory(ds, tt, qp; dt = dt, diffeq...)[:, 1]
        @test !testchaos01(tr)
        @test !testchaos01(tr, cs)
    end
    @testset "chaotic" begin
        tr = trajectory(ds, tt, ch; dt = dt, diffeq...)[:, 1]
        @test testchaos01(tr, cs)
    end
end


using MDP

const rate        = [0.3 0.7 0.9]
const arrivalRate = 0.5

const serviceCost = [6 10 20]
const holdingCost = 1
const dropPenalty = 100_000

const M = 100
const A = size(rate,2)

P = [ spzeros(Float64, M+1, M+1) for u = 1:A]
C = zeros(Float64, M+1, A)

# Initialize cost matrix
C[1,:] = 0

for x = 2:M
    for u = 1:A
        C[x,u] = (x-1) * holdingCost + serviceCost[u]
    end
end

# Add expected cost for dopping packets
for u = 1:A
    C[M+1,u] = M * holdingCost + serviceCost[u] + arrivalRate * dropPenalty
end

# Initialize Probability matrix
for x = 2:M
    for u = 1:A
        P[u][x, x-1] = (1 - arrivalRate) * rate[u]
        P[u][x, x]   = (1 - arrivalRate) * (1 - rate[u]) + arrivalRate * rate[u]
        P[u][x, x+1] = arrivalRate * ( 1 - rate[u])
    end
end

for u = 1:A
    P[u][1,1] = (1 - arrivalRate)
    P[u][1,2] = arrivalRate

    P[u][M+1, M+1] = (1 - rate[u])
    P[u][M+1, M  ] = rate[u]
end

model = ProbModel(C, P; objective=:Min)

@time (v,g) = valueIteration(model; discount=0.9)

using StatsBase

println(rle(g))


# Test Rand index

using Test
using Clustering

@testset "randindex() (Rand index)" begin

a1 = [1, 1, 1, 2, 2, 2, 3, 3, 3, 3]
a2 = [1, 1, 1, 1, 2, 2, 2, 2, 2, 2]
a3 = [3, 3, 3, 2, 2, 2, 1, 1, 1, 1]

(ARI, RI, MI, HI) = randindex(a1, a1)

@test ARI ≈ 1.0 atol=1.0e-12
@test RI  ≈ 1.0 atol=1.0e-12
@test MI  ≈ 0.0 atol=1.0e-12
@test HI  ≈ 1.0 atol=1.0e-12

(ARI, RI, MI, HI) = randindex(a1, a3)

@test ARI ≈ 1.0 atol=1.0e-12
@test RI  ≈ 1.0 atol=1.0e-12
@test MI  ≈ 0.0 atol=1.0e-12
@test HI  ≈ 1.0 atol=1.0e-12

(ARI, RI, MI, HI) = randindex(a1, a2)

@test ARI ≈ 0.403669 atol=1.0e-5
@test RI  ≈ 0.711111 atol=1.0e-5
@test MI  ≈ 0.288888 atol=1.0e-5
@test HI  ≈ 0.422222 atol=1.0e-5

@test randindex(a1, a2) == randindex(a2, a1)

end


struct GradFunc
    VecOfEq
    Variables
    dt
end

function (self::GradFunc)(dt, X)
    vars = Dict()
    for i in eachindex(self.Variables)
        vars[self.Variables[i]] = X[i]
    end
    vars[self.dt] = dt
    expr =  [ModelingToolkit.substitute(i, vars) for i in self.VecOfEq]
    val = ModelingToolkit.expand_derivatives.(expr)

    return val
end

"""
Arguement
---------
- `ODE`{Function}: ODE(du,u,p,t)
- `dim`{Integer}: Number of variables (`length of u`)
- `N` {Integer}: Sections of time intervals
"""
function gradient_gen(ODE!, p, dim, N)
    N = N + 1

    @variables X[1:dim, 1:N]
    @parameters dt

    S = 0

    D = DiffusionMatrix(dim)

    for k in 2:N
        dxdt = get_dxdt(ODE!, X[:,k-1], p)
        S = S + (1/4)*dt* (Δ(X,k,dt) - dxdt)' * (D)^(-1) * (Δ(X,k,dt) - dxdt )
    end

    S = 2*S # TO get the right answer
    S = ModelingToolkit.simplify(S)

    ret = ModelingToolkit.gradient(S, collect(Iterators.flatten(X)) )[2]

    ret = ModelingToolkit.simplify(ret)
    ret=reshape(ret,dim,:) # Matrix

    return GradFunc(ret, X, dt)
end

function get_dxdt(ODE!, x, p; t=nothing)
    dx = deepcopy(x)
    ODE!(dx, x, p, t)
    return dx
end

"""
Differentiation
"""
function Δ(X,k,dt)
    return (X[:,k] - X[:, k-1])/dt
end


module MyInterpolations

  export LinearInterpolation

  immutable LinearInterpolation
    grid::Array
    vals::Array
　end

　function (f::LinearInterpolation)(x)
    i = searchsortedlast(f.grid, x)
    if i == 0 || ( i == length(f.grid) && f.grid[end] !=  x )
        return 0
 　　end
    if x == f.grid[end]
        return f.vals[end]
    end
    interpolated_value = (f.vals[i+1]-f.vals[i])/(f.grid[i+1]-f.grid[i])*(x-f.grid[i])+f.vals[i]
    return interpolated_value

　end

end


#Hermite-Gaussian Modes
HG(n,m,x,y) = hermiteh(n,√2*x)*hermiteh(m,√2*y)*exp(-x^2-y^2);

function θ(x,y)
	#θ ∈ [-π,π]
	if x ≥ 0
		atan(y/x)
	else
		if y≥ 0
			atan(y/x) + π
		else
			atan(y/x) - π
		end
	end
end;

function LG(p,l,x,y)
	#Laguerre-Gaussian Modes
    if x==0 && y==0
        0.0
    else
        (2*(x^2+y^2))^(0.5*abs(l))*laguerrel(p,abs(l),2*(x^2+y^2))*exp(-(x^2+y^2))*exp(im*l*θ(x,y))
    end
end

function airy_beam(x,α)
	#Airy beam (in a single direction)
	airyai(x)*exp(α*x)
end

#Series of obstacles to simulate difraction
function vertical_obstacle(x,y,l)
	if abs(x)<l
		0.0
	else
		1.0
	end
end

function horizontal_obstacle(x,y,l)
	if abs(y)<l
		0.0
	else
		1.0
	end
end

function circular_obstacle(x,y,r)
	if x^2+y^2<r
		0.0
	else
		1.0
	end
end

function rectangular_obstacle(x,y,a,b,c,d)
	if (a<x<b) & (c<y<d)
		0.0
	else
		1.0
	end
end


using Distributions

function evsi_an(μ = [0, 0], σ = [1, 1], nsamples = [10000, 10000])
    m   = μ[1] - μ[2]
    s   = σ .* σ
    ps  = nsamples .* s
    θ   = sqrt(s[1] * (ps[1] / (ps[1] + 1)) + s[2] * (ps[2] / (ps[2] + 1)))
    (θ * sqrt(2 / π) * exp((-m^2) / (2 * θ^2)) + m * erf(m / (θ * sqrt(2))) - abs(m)) / 2
end

function evsi_sim(μ = [0, 0], σ = [1, 1], nsamples = [10000, 10000]; nsims = 1000000)

    ndists        = length(μ)
    true_values   = Array(Float64, nsims, ndists)
    prior_weight  = Array(Float64, ndists)
    sample_weight = Array(Float64, ndists)
    sample_sd     = Array(Float64, ndists)

    for dist in 1:ndists

        true_values[:, dist] = rand(Normal(μ[dist], σ[dist]), nsims)
        prior_weight[dist]   = (1 / σ[dist]) / (nsamples[dist] + (1 / σ[dist]))
        sample_weight[dist]  = nsamples[dist] / (nsamples[dist] + (1 / σ[dist]))
        sample_sd[dist]      = sqrt(1 / nsamples[dist])

    end

    benefit = Array(Float64, nsims)

    for sim in 1:nsims

        posterior_mean = Array(Float64, ndists)

        for dist in 1:ndists
            posterior_mean[dist] = μ[dist] * prior_weight[dist]
            if nsamples[dist] > 0
                posterior_mean[dist] +=
                    rand(Normal(true_values[sim, dist], sample_sd[dist])) *
                        sample_weight[dist]
            end
        end

        benefit[sim] = true_values[sim, indmax(posterior_mean)]

    end

    mean(benefit) - maximum(μ)

end


# This file is a part of AstroLib.jl. License is MIT "Expat".
# Copyright (C) 2016 Mosè Giordano.

function _gcirc{T<:AbstractFloat}(units::Integer, ra1::T, dec1::T, ra2::T, dec2::T)
    # Convert all quantities to radians.
    if units == 0
        # All radians
        λ_1 = ra1
        λ_2 = ra2
        φ_1 = dec1
        φ_2 = dec2
    elseif units == 1
        # Right ascensions are in hours, declinations in degrees.
        λ_1 = ra1*pi/12.0
        λ_2 = ra2*pi/12.0
        φ_1 = deg2rad(dec1)
        φ_2 = deg2rad(dec2)
    elseif units == 2
        # Right ascensions and declinations are in degrees.
        λ_1 = deg2rad(ra1)
        λ_2 = deg2rad(ra2)
        φ_1 = deg2rad(dec1)
        φ_2 = deg2rad(dec2)
    else
        # In any other case throw an error.
        error("units must be 0 (radians), 1 (hours, degrees) or 2 (degrees)")
    end
    Δφ_2 = (φ_2 - φ_1) * 0.5
    Δλ_2 = (λ_2 - λ_1) * 0.5
    Δσ = 2asin(sqrt(abs2(sin(Δφ_2)) + cos(φ_1) * cos(φ_2) * abs2(sin(Δλ_2))))
    if units == 0
        return Δσ
    else
        return rad2sec(Δσ)
    end
end

"""
    gcirc(units, ra1, dec1, ra2, dec2) -> angular_distance

### Purpose ###

Computes rigorous great circle arc distances.

### Explanation ###

Input position can be either radians, sexagesimal right ascension and
declination, or degrees.

### Arguments ###

* `units`: integer, can be either 0, or 1, or 2.  Describes units of inputs and
 output:
    * 0: everything (input right ascensions and declinations, and output
      distance) is radians
    * 1: right ascensions are in decimal hours, declinations in decimal degrees,
      output distance in arc seconds
    * 2: right ascensions and declinations are in degrees, output distance in arc
      seconds
* `ra1`:  right ascension or longitude of point 1
* `dec1`: declination or latitude of point 1
* `ra2`: right ascension or longitude of point 2
* `dec2`: declination or latitude of point 2

Both `ra1` and `dec1`, and `ra2` and `dec2` can be given as 2-tuples `(ra1,
dec1)` and `(ra2, dec2)`.

### Output ###

Angular distance on the sky between points 1 and 2, as a `AbstractFloat`.  See
`units` argument above for the units.

### Method ###

"Haversine formula" see http://en.wikipedia.org/wiki/Great-circle_distance.

### Example ###

``` julia
gcirc(0, 120, -43, 175, +22)
# => 1.590442261600714
```

### Notes ###

* If `ra1`, `dec1` are scalars, and `ra2`, `dec2` are vectors, then the output
 is a vector giving the distance of each element of `ra2`, `dec2` to `ra1`,
 `dec1`.  Similarly, if `ra1`,`de1` are vectors, and `ra2`,` dec2` are scalars,
 then the output is a vector giving the distance of each element of `ra1`,
 `dec1` to `ra2`, `dec2`.  If both `ra1`, `dec1` and `ra2`, `dec2` are vectors
 then the output is a vector giving the distance of each element of `ra1`,
 `dec1` to the corresponding element of `ra2`, `dec2`.
* The function `sphdist` provides an alternate method of computing a spherical
 distance.
* The Haversine formula can give rounding errors for antipodal points.

Code of this function is based on IDL Astronomy User's Library.
"""
gcirc(units::Integer, ra1::Real, dec1::Real, ra2::Real, dec2::Real) =
    _gcirc(units, promote(float(ra1), float(dec1), float(ra2), float(dec2))...)

function gcirc{R1<:Real, D1<:Real}(units::Integer,
                                   ra1::AbstractArray{R1},
                                   dec1::AbstractArray{D1},
                                   ra2::Real,
                                   dec2::Real)
    @assert length(ra1) == length(dec1)
    dist = similar(ra1, typeof(float(one(R1))))
    for i in eachindex(ra1)
        dist[i] = gcirc(units, ra1[i], dec1[i], ra2, dec2)
    end
    return dist
end

function gcirc{R2<:Real, D2<:Real}(units::Integer,
                                   ra1::Real,
                                   dec1::Real,
                                   ra2::AbstractArray{R2},
                                   dec2::AbstractArray{D2})
    @assert length(ra2) == length(dec2)
    dist = similar(ra2, typeof(float(one(R2))))
    for i in eachindex(ra2)
        dist[i] = gcirc(units, ra1, dec1, ra2[i], dec2[i])
    end
    return dist
end

function gcirc{R1<:Real, D1<:Real, R2<:Real, D2<:Real}(units::Integer,
                                                       ra1::AbstractArray{R1},
                                                       dec1::AbstractArray{D1},
                                                       ra2::AbstractArray{R2},
                                                       dec2::AbstractArray{D2})
    @assert length(ra1) == length(dec1) == length(ra2) == length(dec2)
    dist = similar(ra1, typeof(float(one(R1))))
    for i in eachindex(ra1)
        dist[i] = gcirc(units, ra1[i], dec1[i], ra2[i], dec2[i])
    end
    return dist
end

### Tuples input
gcirc(units::Integer, radec1::Tuple{Real, Real}, ra2::Real, dec2::Real) =
    gcirc(units, radec1..., ra2, dec2)

gcirc(units::Integer, ra1::Real, dec1::Real, radec2::Tuple{Real, Real}) =
    gcirc(units, ra1, dec1, radec2...)

gcirc(units::Integer, radec1::Tuple{Real, Real}, radec2::Tuple{Real, Real}) =
    gcirc(units, radec1..., radec2...)


using DataFrames
using StatsBase

"""
	rxy(x::Real,y::Real)
	r -> sqrt{x^2 + y^2}
"""
function rxy(x::Real, y::Real)
    return sqrt(x^2 + y^2)
end

"""
	hixy(x::Real, y::Real)
	phi -> atan(y/x)
"""
function phixy(x::Real, y::Real)
    return atan(y,x)
end

"""
	phixy(hitdf::DataFrame)
	phi -> atan(y/x) where y and x are columns of the data frame
"""
function fphi(hitdf::DataFrame)
    return atan.(hitdf.y,hitdf.x)
end

"""
	dxyz(x1::Vector{<:Real}, x2::Vector{<:Real})

Distance between two points.
"""

function dxyz(x1::Vector{<:Real}, x2::Vector{<:Real})
    return sqrt((x1[1] - x2[1])^2 + (x1[2] - x2[2])^2 + (x1[3] - x2[3])^2)
end

"""
	gline2p(x1,y1,x2, y2)
	Line that goes through two points
"""
function gline2p(x1, y1, x2, y2)
    fxy(x) = y1 + (x - x1) * (y2 - y1)/(x2 -x1)
end

function gline2p(x1::Real, y1::Real, x2::Real, y2::Real)
    fxy(x::Real) = y1 + (x - x1) * (y2 - y1)/(x2 -x1)
end

"""
function wstd(x::Vector{<:Real}, q::Vector{<:Real})

Compute the std deviation in x weighted by q:
Sqrt(1/Q Sum_i (x - x_mean) * qi )
"""
function wstd(x::Vector{<:Real}, q::Vector{<:Real})
	xmean = mean(x)
	qs = sum((x.-xmean).^2 .* q)
	Q = sum(q)
	return sqrt(qs/Q)
end

"""
	mean_std(x, xmin, xmax)
	Returns mean and std for a vector x in the interval between xmin and xmax
"""
function mean_std(x::Vector{<:Real}, xmin::Real, xmax::Real)
    xx = in_range(x, xmin, xmax)
    xm = mean(xx)
    xs = StatsBase.std(xx)
    return xm, xs
end


function init_preconditioner(d, q, elType=typeof(1.0))
    D = d * (q + 1)
    P = Diagonal(ones(elType, D))
    PI = Diagonal(ones(elType, D))
    return P, PI
end

function make_preconditioners!(cache::GaussianODEFilterCache, dt)
    @unpack P, PI, d, q = cache
    make_preconditioner!(P, dt, d, q)
    make_preconditioner_inv!(PI, dt, d, q)
    return nothing
end
function make_preconditioners!(post::GaussianODEFilterPosterior, dt)
    @unpack P, PI, d, q = post
    make_preconditioner!(P, dt, d, q)
    make_preconditioner_inv!(PI, dt, d, q)
    return nothing
end

@fastmath @inbounds function make_preconditioner!(P, h, d, q)
    val = factorial(q) / h^(q + 1 / 2)
    for j in 0:q
        @simd for i in 0:d-1
            P[j+i*(q+1)+1, j+i*(q+1)+1] = val
        end
        val /= (q - j) / h
    end
    return P
end

@fastmath @inbounds function make_preconditioner_inv!(PI, h, d, q)
    val = h^(q + 1 / 2) / factorial(q)
    for j in 0:q
        @simd for i in 0:d-1
            PI[j+i*(q+1)+1, j+i*(q+1)+1] = val
        end
        val *= (q - j) / h
    end
    return PI
end


@doc """
       QuasiNewton method for local optimization (single variable)

       Supported by:
       Ph D Kelvyn B. Sánchez
       e-mail: kelvyn.baruc@gmail.com
 """ ->
function QuasiNewton(f, var; x0 = [1 2], error = 0.0001, maxiter = 100, iter = true)
      try
            # Symbolic derivatives using Module SymPy
            dfk1 = diff(f, var)

            err = Inf
            i = 1
            xk = 0
            x_num = 0

            # Numeric iterations
            iter == true ? ti = time_ns() : nothing # To measure algorothm performance

            while err > error
                  x_num = x0

                  xk = x_num[2]-(N(dfk1 |> subs(var,x_num[2]))/(N(dfk1 |> subs(var,x_num[2])) -
                        N(dfk1 |> subs(var,x_num[1]))))*(x_num[2] - x_num[1])

                  if xk > x_num[2]
                        err = abs(x_num[1] - xk)
                        x0 = [x_num[2] xk]

                  elseif xk < x_num[1]
                        err = abs(x_num[2] - xk)
                        x0 = [xk x_num[1]]

                  elseif xk < x_num[2] && xk > x_num[1]
                         err = abs(x_num[1] - xk)
                         x0 = [xk x_num[2]]
                  end

                        if iter == true
                              if i == 1
                                    println("===========================================================")
                                    println(" QuasiNewton Solver for Unconstrained Optimization (0.0.2)")
                              end

                              if i == 1 || ceil((i/1)/10) == (i/1)/10
                                    println("===========================================================")
                                    @printf("Iter\t X\t\t f(x)\t\t |err|\n")
                                    println("===========================================================")
                              end

                              @printf("%i\t %4.6e\t %4.6e\t %4.6e\n", i, xk, N(f |> subs(var,xk)), err)
                        end

                  if i == maxiter
                        throw(ErrorException("Not Solved!. Iteration limit reached."))
                  end

                  i += 1
            end

            println("===========================================================")
            println(" ")

            iter == true ? tf = time_ns() : nothing # To measure algorithm performance
            iter == true ? println("elapse time: ", (tf - ti)/1.0e9, " seconds") : nothing

            return xk,  N(f |> subs(var,xk))

      catch Msg
            println("===========================================================")
            println(Msg)
            return NaN, NaN
      end
end


# getSHbasis.jl
using SphericalHarmonics: computeYlm
using CoordinateTransformations: SphericalFromCartesian
using MIRTjim: jim

# export numSH getSHbasis

"""
    c2sph(r, l, m)

Evaluate the `(l,m)` spherical harmonic at one spatial location `(x, y, z)`.
"""
function c2sph(x, y, z, l, m)
	a = SphericalFromCartesian()([x,y,z]) # todo: SVector?
	(rho, ϕ, θ) = (a.r, a.θ, π/2 - a.ϕ)   # (radius, azimuth, colatitude)
	Y = computeYlm(θ, ϕ; lmax=l)
	rho^l * Y[(l,m)]
end

"""
    n = numSH(L::Int)
Number of spherical harmonics for given `L`
i.e., `sum(2*(0:L) .+ 1)`, which is 9 for `L=2`.
Order of 0th-2nd order terms (for `L=2`):
* 1 cf (center frequency, Hz)
* 2 z
* 3 x
* 4 y
* 5 z2
* 6 zx
* 7 zy
* 8 x2y2
* 9 xy
"""
function numSH(L::Int)
	sum(2*(0:L) .+ 1)
end

"""
    getSHbasis!(h, x, y, z; L=2)

Store in `h` the spherical harmonic basis up to order `L` (default 2),
evaluated at spatial location `(x, y, z)`.
Vector `h` must have length `numSH(L)`.
"""
function getSHbasis!(h::AbstractVector{<:Real}, x::Real, y::Real, z::Real; L::Int = 2)

	ic = 1
	for l = 0:L, m = 0:l
		f = c2sph(x, y, z, l, m)
		h[ic] = real(f)
		ic += 1
		if m != 0
			h[ic] = imag(f)
			ic += 1
 		end
 	end

	h[1] = 1.0 # basis for center frequency offset

	return h
end

"""
    H = getSHbasis(x, y, z; L::Int=2)

Get spherical harmonic basis up to order `L` (default 2)
evaluated at spatial location vectors `x, y, z`
See `numSH` for ordering.
Output size is `(length(x), length(y), length(z), numSH(L))`
"""
function getSHbasis(
	x::AbstractVector{<:Real},
	y::AbstractVector{<:Real},
	z::AbstractVector{<:Real};
	L::Int=2
)

	T = promote_type(eltype.([x, y, z])..., Float32) # at least Float32
	H = zeros(T, length(x), length(y), length(z), numSH(L))
	for ix=1:length(x), iy=1:length(y), iz=1:length(z)
		getSHbasis!((@view H[ix,iy,iz,:]), x[ix], y[iy], z[iz]; L)
	end
	return H
end

# convenience method for scalar inputs
getSHbasis(x::Real, y::Real, z::Real; L::Int=2) = getSHbasis([x], [y], [z]; L)

"""
    H = getSHbasis("test"; L::Int = 2)

Test function
"""
function getSHbasis(str::String; L::Int = 2,
	nx::Int=22, ny::Int=20, nz::Int=18, fov::NTuple{3,Real}=(20,20,20),
)
	rx = LinRange(-1,1,nx)*fov[1]/2
	ry = LinRange(-1,1,ny)*fov[2]/2
	rz = LinRange(-1,1,nz)*fov[3]/2
	@time H = getSHbasis(rx, ry, rz; L)
	jim(H; ncol=numSH(L), color=:jet, gui=true, line3plot=false) # compare with >> evalspharm("test")
	return H
end

# getSHbasis("test"; L = 2);


"Compute a transmission coefficient for a Gaussian absorption line with given depth.  Optionally, truncate absorption at limit_line_effect."
function std_gaussian_line(x::Number, depth::Number; limit_line_effect = Inf)
   return abs(x)>limit_line_effect ? one(x) : one(x)-depth*exp(-0.5*x^2)
end

"Compute spectrum for an Gaussian absorption line give it's location, width and depth"
function absorption_line(x::Number, location::Number, width::Number, depth::Number; limit_line_effect = Inf)
    std_gaussian_line((x-location)/width, depth, limit_line_effect=limit_line_effect)
end

"Compute product of absorption lines at one wavelength given a list of locations, widths and depths"
function absorption_lines(x::T1, locations::A2, widths::A3, depths::A4; limit_line_effect = Inf) where { T1<:Number, T2<:Number, T3<:Number, T4<:Number, A2<:AbstractVector{T2}, A3<:AbstractVector{T3}, A4<:AbstractVector{T4} }
    @assert length(locations) == length(widths) == length(depths) >= 1
    @inbounds trans = absorption_line(x, locations[1], widths[1], depths[1], limit_line_effect = limit_line_effect )
    for i in 2:length(locations)
        @inbounds trans = trans * absorption_line(x, locations[i],widths[i], depths[i], limit_line_effect = limit_line_effect )
    end
    return trans
end

function absorption_lines(x::A1, locations::A2, widths::A3, depths::A4; limit_line_effect = Inf) where { T1<:Number, T2<:Number, T3<:Number, T4<:Number, A1<:AbstractArray{T1}, A2<:AbstractVector{T2}, A3<:AbstractVector{T3}, A4<:AbstractVector{T4} }
    @assert length(locations) == length(widths) == length(depths) >= 1
    trans = absorption_line.(x, locations[1], widths[1], depths[1], limit_line_effect = limit_line_effect )
    @inbounds for i in 2:length(locations)
        trans = trans .* absorption_line.(x, locations[i],widths[i], depths[i], limit_line_effect = limit_line_effect )
    end
    return trans
end


module HWconstrained

using JuMP, NLopt, DataFrames, Ipopt
using LinearAlgebra
export data, table_NLopt, table_JuMP

### Question 1
function data(a=0.5)
n=3
p=[1, 1, 1]
e=[2, 0, 0]
s1=s2=4
z1=[1, 1, 1, 1]
z2=[0.72, 0.92, 1.12, 1.32]
z3=[0.86, 0.96, 1.06, 1.16]
z=[[1, i, j] for i in z2 for j in z3]
z = vcat(z'...)
pi = repeat([1/16], 16)
a=0.5
na=3
nc=4
ns=4
nss=16

return Dict("a"=>a,"na"=>na,"nc"=>nc,"ns"=>ns,"nss"=>nss,"e"=>e,"p"=>p,"z"=>z,"pi"=>pi)
end
#end

d=data()

####  Question 2

function obj(x::Vector,grad::Vector,data::Dict)
    A = data["a"]
    Z = data["z"]
    pi = data["pi"]
    if length(grad) > 0
        grad[1] = A*exp.(-A*x[1])
        for i in 1:3
        grad[i+1] = sum(pi .* Z[:,i] .*A.*exp.(-A.*Z*x[i+1]))
        end
    end
    return -exp.(-A*x[1])+ sum(pi.*-exp.(-A*Z*x[2:4]))
end
obj(ones(4), zeros(4), d)

function constr(x::Vector,grad::Vector,data::Dict)
    if length(grad) > 0
        grad[1] = d["a"]*exp(-d["a"]*x[1])
        grad[2:end] = d["p"]
    end
    return x[1] + sum(d["p"].*(x[2:end].-d["e"]))
end

constr(ones(4), zeros(4), d) # keep track of # function evaluations

function max_NLopt(a=0.5)
d = data(a)
e= d["e"]
optimum = Opt(:LD_MMA, 4)
lower_bounds!(optimum, [0., -Inf, -Inf, -Inf])
max_objective!(optimum, (x, g)->obj(x, g, d))
inequality_constraint!(optimum, (x, g)->constr(x, g, d), 1e-8)
ftol_rel!(optimum, 1e-8)
NLopt.optimize(optimum, vcat(0,e))
end

function table_NLopt()
d = DataFrame(a=[0.5;1.0;5.0],c = zeros(3),omega1=zeros(3),omega2=zeros(3),omega3=zeros(3),fval=zeros(3))
for i in 1:nrow(d)
xx = max_NLopt(d[i,:a])
for j in 2:ncol(d)-1
d[i,j] = xx[2][j-1]
end
d[i,end] = xx[1]
end
return d
end

end # module


using StanModels, Distributions

# ### snippet 8.25

N = 100                                          # individuals
height  = rand(Normal(10,2), N) ; # sim total height of each
leg_prop = rand(Uniform(0.4,0.5), N); # leg as proportion of height

# sim left leg as proportion + error
leg_left = leg_prop .* height .+  rand(Normal( 0 , 0.02 ), N);
# sim right leg as proportion + error
leg_right = leg_prop .* height .+  rand(Normal( 0 , 0.02 ), N);

# combine into data frame

df =  DataFrame(height=height, leg_left = leg_left, leg_right = leg_right);

# Show first 5 rows

first(df, 5)

# Define the Stan language model

m8_8s = "
data{
    int N;
    real height[N];
    real leg_right[N];
    real leg_left[N];
}
parameters{
    real a;
    real bl;
    real br;
    real sigma;
}
model{
    vector[N] mu;
    sigma ~ cauchy( 0 , 1 );
    br ~ normal( 2 , 10 );
    bl ~ normal( 2 , 10 );
    a ~ normal( 10 , 100 );
    for ( i in 1:100 ) {
        mu[i] = a + bl * leg_left[i] + br * leg_right[i];
    }
    height ~ normal( mu , sigma );
}
";

# Define the Stanmodel and set the output format to :mcmcchains.

sm = SampleModel("m8.8s", m8_8s);

# Input data for cmdstan

m8_8_data = Dict("N" => size(df, 1), "height" => df[!, :height],
    "leg_left" => df[!, :leg_left], "leg_right" => df[!, :leg_right]);

# Sample using cmdstan

(sample_file, log_file) = stan_sample(sm, data=m8_8_data);

# Describe the draws

if !(sample_file == nothing)
  chn = read_samples(sm)
  describe(chn)
end


"""
$(SIGNATURES)

Average number of sigma bound exceedings

Returns the average number of unbiased states / innovation values that exceed the ⨦σ bound of the given covariance
"""
function mean_num_sigma_bound_exceedings(state_over_time::Vector{Vector{T}}, covariance_over_time::Vector{Matrix{T}}) where T
    mean(map((x, P) -> abs.(x) .> sqrt.(diag(P)), state_over_time, covariance_over_time))
end

function mean_num_sigma_bound_exceedings(state_over_time::Vector{T}, covariance_over_time::Vector{T}) where T
    mean(map((x, P) ->  abs(x) > sqrt(P), state_over_time, covariance_over_time))
end

"""
$(SIGNATURES)

Innovation magnitude bound test (σ-bound-test)

Tests if approximately 68% of state values lie within the ⨦σ bound
"""
function sigma_bound_test(state_over_time, covariance_over_time)
    isapprox.(mean_num_sigma_bound_exceedings(state_over_time, covariance_over_time), .32, atol = .015)
end

"""
$(SIGNATURES)

Innovation magnitude bound test (2σ-bound-test)

Tests if approximately 95% of state values lie within the ⨦2σ bound
"""
function two_sigma_bound_test(state_over_time, covariance_over_time)
    isapprox.(mean_num_sigma_bound_exceedings(state_over_time, 4 .* covariance_over_time), .05, atol = .008)
end

"""
$(SIGNATURES)

Normalized innovation squared (NIS) Test

Double-tailed siginicance test with false alarm probability α = 0.05

Calculates confidence interval [r1 r2] and tests Prob{ ∑ NIS values)} ∈ [r1 r2] ∣ H_0 ) = 1 - α
with Hypothesis H_0: N * ∑ NIS values ∼ χ^2_{dof}
     dof (degree of freedom): N * m (N: window length, m: dimension of state vector)
"""
function nis_test(nis_over_time, dof)
    sum_of_nis = sum(nis_over_time)

    r1 = cquantile(Chisq(dof), .975)
    r2 = cquantile(Chisq(dof), .025)

    (sum_of_nis >= r1) && (sum_of_nis <= r2)
end

"""
$(SIGNATURES)

Normalized innovation squared (NIS)

Returns NIS-value for a single innovation sequence `seq` and its variance `var`
"""
function calc_nis(seq, var)
    dot(seq, var \ seq)
end

function calc_nis(mu::AbstractMeasurementUpdate)
    calc_nis(get_innovation(mu), mu.innovation_covariance)
end


function solve_naive(input)
    earliest = earlier = parse(Int64, input[1])
    buses = parse.(Int64, filter(x -> x != "x", split(input[2], ',')))
    idx = findfirst(id -> earlier % id == 0, buses)
    while isnothing(idx)
        earlier += 1
        idx = findfirst(id -> earlier % id == 0, buses)
    end
    buses[idx] * (earlier - earliest)
end

# Taken from https://rosettacode.org/wiki/Chinese_remainder_theorem#Julia
function chineseremainder(n::Array, a::Array)
    Π = prod(n)
    mod(sum(ai * invmod(Π ÷ ni, ni) * (Π ÷ ni) for (ni, ai) in zip(n, a)), Π)
end

function solve_with_flashbacks(input)
    # (Flashbacks OMA) Esto sale con teorema chino del resto...
    buses = map(x -> x == "x" ? 0 : parse(Int64, x), split(input[2], ','))
    n = []
    a = []
    for i in 0:length(buses) - 1
        if buses[i + 1] > 0
            push!(n, -i)
            push!(a, buses[i + 1])
        end
    end
    chineseremainder(a, n)
end

@assert solve_naive(readlines("input/test/test_day13.txt")) == 295 ["test case 1 failed"]
@assert solve_with_flashbacks(readlines("input/test/test_day13.txt")) == 1068781 ["test case 2 failed"]

println("First star: " * string(solve_naive(readlines("input/day13.txt"))))
println("Second star: " * string(solve_with_flashbacks(readlines("input/day13.txt"))))


function rouwenhorst(μ,ρ,σ,n)
    n==1 && (return [μ],[1.0])

	mu_eps=0
	q = (ρ+1)/2
	nu = ((n-1)/(1-ρ^2))^(1/2) * σ
	P = [q 1-q; 1-q q]
	for i = 2:n-1
		P = q*[P zeros(i,1);zeros(1,i+1)] + (1-q)*[zeros(i,1) P; zeros(1,i+1)]+ (1-q)*[zeros(1,i+1); P zeros(i,1)] + q*[zeros(1,i+1); zeros(i,1) P]
		P[2:i,:] = P[2:i,:]/2
	end
	x = [linspace(mu_eps/(1-ρ).-nu,mu_eps/(1-ρ).+nu,n);].+μ
	return x,P
end

function cdf_normal(x) :inline
    c = 0.5 * erfc(-x/sqrt(2))
end

function tauchen(μ,ρ,σ,N,m=3)
	if N==1
		return [μ],[1.0]
	end
	Z     = zeros(N)
	Zprob = zeros(N,N)
	a     = (1-ρ)*μ

	Z[N]  = m * sqrt(σ^2 / (1 - ρ^2))
	Z[1]  = -Z[N]
	zstep = (Z[N] - Z[1]) / (N - 1)

	for i=2:(N-1)
	    Z[i] = Z[1] + zstep * (i - 1)
	end

	Z = Z .+ a / (1-ρ)

	for j = 1:N
	    for k = 1:N
	        if k == 1
	            Zprob[j,k] = cdf_normal((Z[1] - a - ρ * Z[j] + zstep / 2) / σ)
	        elseif k == N
	            Zprob[j,k] = 1 - cdf_normal((Z[N] - a - ρ * Z[j] - zstep / 2) / σ)
	        else
	            Zprob[j,k] = cdf_normal((Z[k] - a - ρ * Z[j] + zstep / 2) / σ) - cdf_normal((Z[k] - a - ρ * Z[j] - zstep / 2) / σ)
	        end
	    end
	end
	return Z,Zprob
end


n, p = 500, 5
((X, y, θ), (X1, y1, θ1)) = generate_binary(n, p; seed=52)

@testset "Logreg" begin
    # No intercept
    λ = 5.0
    lr = LogisticRegression(λ; fit_intercept=false)
    J  = objective(lr, X, y)
    o  = LogisticLoss() + λ * L2Penalty()
    @test J(θ) == o(y, X*θ, θ)
    θ_newton   = fit(lr, X, y, solver=Newton())
    θ_newtoncg = fit(lr, X, y, solver=NewtonCG())
    θ_lbfgs    = fit(lr, X, y, solver=LBFGS())
    @test isapprox(J(θ),          282.09960, rtol=1e-5)
    @test isapprox(J(θ_newton),   280.37472, rtol=1e-5) # <- ref value
    @test isapprox(J(θ_newtoncg), 280.37472, rtol=1e-5)
    @test isapprox(J(θ_lbfgs),    280.37472, rtol=1e-5)

    # With intercept
    lr1 = LogisticRegression(λ; penalize_intercept=true)
    J   = objective(lr1, X, y1)
    θ_newton   = fit(lr1, X, y1, solver=Newton())
    θ_newtoncg = fit(lr1, X, y1, solver=NewtonCG())
    θ_lbfgs    = fit(lr1, X, y1, solver=NewtonCG())
    @test isapprox(J(θ1),         213.80970, rtol=1e-5)
    @test isapprox(J(θ_newton),   209.32539, rtol=1e-5) # <- ref value
    @test isapprox(J(θ_newtoncg), 209.32539, rtol=1e-5)
    @test isapprox(J(θ_lbfgs),    209.32539, rtol=1e-5)

    # with intercept and not penalizing it
    lr1 = LogisticRegression(λ)
    J   = objective(lr1, X, y1)
    θ_newton   = fit(lr1, X, y1, solver=Newton())
    θ_newtoncg = fit(lr1, X, y1, solver=NewtonCG())
    θ_lbfgs    = fit(lr1, X, y1, solver=NewtonCG())
    @test isapprox(J(θ1),         212.29192, rtol=1e-5)
    @test isapprox(J(θ_newton),   207.71595, rtol=1e-5) # <- ref value
    @test isapprox(J(θ_newtoncg), 207.71595, rtol=1e-5)
    @test isapprox(J(θ_lbfgs),    207.71595, rtol=1e-5)

    if DO_COMPARISONS
        # This checks that the parameters recovered using Sklearn lead
        # to a similar loss than the one given by our code to verify the
        # correctness of the code.
        lr_sk_ncg = SKLEARN_LM.LogisticRegression(C=1.0/λ, solver="newton-cg")
        lr_sk_ncg.fit(X, y1)
        θ1_sk_ncg = vcat(lr_sk_ncg.coef_[:], lr_sk_ncg.intercept_)
        lr_sk_lbfgs = SKLEARN_LM.LogisticRegression(C=1.0/λ, solver="lbfgs")
        lr_sk_lbfgs.fit(X, y1)
        θ1_sk_lbfgs = vcat(lr_sk_lbfgs.coef_[:], lr_sk_lbfgs.intercept_)
        # Comparing with ours
        @test isapprox(J(θ1_sk_ncg),   207.71595, rtol=1e-5)
        @test isapprox(J(θ1_sk_lbfgs), 207.71595, rtol=1e-5)
    end
end

n, p, c = 500, 5, 4
((X, y, θ), (X1, y1, θ1)) = generate_multiclass(n, p, c; seed=525)

@testset "Multinomial" begin
    # No intercept
    λ = 5.0
    mnr = MultinomialRegression(λ; fit_intercept=false)
    J   = objective(mnr, X, y; c=c)
    θ_newtoncg = fit(mnr, X, y, solver=NewtonCG())
    θ_lbfgs    = fit(mnr, X, y, solver=R.LBFGS())
    @test isapprox(J(θ),          419.67795, rtol=1e-5)
    @test isapprox(J(θ_newtoncg), 384.33810, rtol=1e-5) # <- ref value
    @test isapprox(J(θ_lbfgs),    384.33810, rtol=1e-5)

    #  With intercept
    mnr = MultinomialRegression(λ; penalize_intercept=true)
    J   = objective(mnr, X, y1; c=c)
    θ_newtoncg = fit(mnr, X, y1, solver=NewtonCG())
    θ_lbfgs    = fit(mnr, X, y1, solver=R.LBFGS())
    @test isapprox(J(θ),          1244.46404, rtol=1e-5)
    @test isapprox(J(θ_newtoncg),  388.75018, rtol=1e-5) # <- ref value
    @test isapprox(J(θ_lbfgs),     388.75018, rtol=1e-5)

    mnr = MultinomialRegression(λ)
    J   = objective(mnr, X, y1; c=c)
    θ_newtoncg = fit(mnr, X, y1, solver=NewtonCG())
    θ_lbfgs    = fit(mnr, X, y1, solver=R.LBFGS())
    @test isapprox(J(θ),          1242.70990, rtol=1e-5)
    @test isapprox(J(θ_newtoncg),  383.93018, rtol=1e-5) # <- ref value
    @test isapprox(J(θ_lbfgs),     383.93018, rtol=1e-5)

    if DO_COMPARISONS
        lr_sk_ncg = SKLEARN_LM.LogisticRegression(C=1.0/λ, solver="newton-cg",
                                             multi_class="multinomial")
        lr_sk_ncg.fit(X, y1)
        θ1_sk_ncg = vec(vcat(lr_sk_ncg.coef_', lr_sk_ncg.intercept_'))
        lr_sk_lbfgs = SKLEARN_LM.LogisticRegression(C=1.0/λ, solver="lbfgs",
                                               multi_class="multinomial")
        lr_sk_lbfgs.fit(X, y1)
        θ1_sk_lbfgs = vec(vcat(lr_sk_ncg.coef_', lr_sk_ncg.intercept_'))
        # Comparing with ours
        @test isapprox(J(θ1_sk_ncg),   385.67895, rtol=1e-5)
        @test isapprox(J(θ1_sk_lbfgs), 385.67895, rtol=1e-5)
    end
end

n, p = 500, 100
(_, (X, y, θ)) = generate_binary(n, p;  seed=52551, sparse=0.1)

@testset "Logreg/EN" begin
    ρ = 0.8
    α = 0.03
    λ = α * (1 - ρ) * n
    γ = α * ρ * n
    enlr = LogisticRegression(λ, γ; penalize_intercept=true)
    J    = objective(enlr, X, y)
    θ_fista = fit(enlr, X, y)
    θ_ista  = fit(enlr, X, y, solver=ISTA())
    @test isapprox(J(θ),       271.43962, rtol=1e-5)
    @test isapprox(J(θ_fista), 250.46547, rtol=1e-5) # <- ref values
    @test isapprox(J(θ_ista),  250.46547, rtol=1e-5)
    # sparsity
    @test nnz(θ)       == 7  # generating θ
    @test nnz(θ_fista) == 18
    @test nnz(θ_ista)  == 18

    # pure l1 regularization (unclear how sklearn does the mixing of l1/l2 for logreg)
    enlr = LogisticRegression(γ; penalty=:l1)
    J    = objective(enlr, X, y)
    θ_fista = fit(enlr, X, y)
    @test isapprox(J(θ),       263.76834, rtol=1e-5)
    @test isapprox(J(θ_fista), 245.65841, rtol=1e-5) # <- ref value
    @test nnz(θ_fista) == 17

    if DO_COMPARISONS
        # NOTE: this algorithm is stochastic so can't have hard tests
        PY_RND.seed(1531)
        enlr_sk = SKLEARN_LM.LogisticRegression(penalty="elasticnet", C=1.0/γ, l1_ratio=1, solver="saga")
        enlr_sk.fit(X, y)
        θ_sk = vcat(enlr_sk.coef_[:], enlr_sk.intercept_)
        @test isapprox(J(θ_sk), 245.658, rtol=1e-3)
        @test 17 ≤ nnz(θ_sk) ≤ 19
    end
end

n, p, c = 1000, 100, 3
(_, (X, y, θ)) = generate_multiclass(n, p, c;  seed=33, sparse=0.1)

@testset "Multin/EN" begin
    λ = 10
    γ = 50
    enmnr = MultinomialRegression(λ, γ; penalize_intercept=true)
    J     = objective(enmnr, X, y; c=c)
    θ_fista = fit(enmnr, X, y)
    θ_ista  = fit(enmnr, X, y)
    @test isapprox(J(θ),       1574.61741, rtol=1e-5)
    @test isapprox(J(θ_fista),  922.36776, rtol=1e-5) # <- ref value
    @test isapprox(J(θ_ista),   922.36776, rtol=1e-5)
    @test nnz(θ_fista) == 15
    @test nnz(θ_ista)  == 15

    # pure l1 regularization for sklearn comp
    enmnr = MultinomialRegression(γ; penalty=:l1)
    J     = objective(enmnr, X, y; c=c)
    θ_fista = fit(enmnr, X, y)
    @test isapprox(J(θ),       1432.37281, rtol=1e-5)
    @test isapprox(J(θ_fista),  912.25297, rtol=1e-5) # <- ref value
    @test nnz(θ_fista) == 16

    if DO_COMPARISONS
        # NOTE: this algorithm is stochastic
        PY_RND.seed(1531)
        enmnr_sk = SKLEARN_LM.LogisticRegression(penalty="elasticnet", C=1.0/γ, l1_ratio=1,
                                            solver="saga", multi_class="multinomial")
        enmnr_sk.fit(X, y)
        θ_sk = enmnr_sk.coef_'[:]
        @test isapprox(J(θ_sk), 912.39215, rtol=1e-3)
        @test 14 ≤ nnz(θ_sk) ≤ 16
    end
end


# indicator of the Stiefel manifold

export IndStiefel

@doc raw"""
**Indicator of the Stiefel manifold**

    IndStiefel()

Returns the indicator of the Stiefel manifold
```math
S_{n,p} = \left\{ X \in \mathbb{F}^{n \times p} : X^*X = I \right\}.
```
where ``\mathbb{F}`` is the real or complex field, and parameters ``n`` and ``p``
are inferred from the matrix provided as input.
"""
struct IndStiefel <: ProximableFunction end

is_set(f::IndStiefel) = true

function (f::IndStiefel)(X::AbstractMatrix{T}) where {R <: Real, T <: Union{R, Complex{R}}}
    F = svd(X)
    if all(F.S .≈ R(1))
        return R(0)
    end
    return R(Inf)
end

function prox!(Y::AbstractMatrix{T}, f::IndStiefel, X::AbstractMatrix{T}, gamma::R=R(1)) where {R <: Real, T <: Union{R, Complex{R}}}
    n, p = size(X)
    F = svd(X)
    U_sliced = view(F.U, :, 1:p)
    mul!(Y, U_sliced, F.Vt)
    return R(0)
end

function prox_naive(f::IndStiefel, X::AbstractMatrix{T}, gamma::R=R(1)) where {R, T <: Union{R, Complex{R}}}
    n, p = size(X)
    F = svd(X)
    Y = F.U[:, 1:p] * F.Vt
    return Y, R(0)
end


"""
**Basic gradient descent with fixed learning rate**
```julia
VanillaGradDescent(; η::Real=0.01)
```

Algorithm
```math
\\Delta x_t = \\eta g_t
```

"""
mutable struct VanillaGradDescent <: Optimizer
    opt_type::String
    t::Int64
    η::Float64
end

function VanillaGradDescent(; η::Real=0.01)
    @assert η > 0.0 "η must be greater than 0"

    VanillaGradDescent("Vanilla Gradient Descent", 0, η)
end

params(opt::VanillaGradDescent) = "η=$(opt.η)"

function update(opt::VanillaGradDescent, g_t::AbstractArray{T}) where {T<:Real}
    # update timestep
    opt.t += 1
    return opt.η * g_t
end