# The sum of the primes below 10 is 2 + 3 + 5 + 7 = 17.
#
# Find the sum of all the primes below two million.

using ProjectEulerSolutions

# Run Sieve of Eratosthenes to get primes and sum them.
function p010solution(n::Integer=3)::Integer
    return sum(sieve_eratosthenes(n))
end

p010 = Problems.Problem(p010solution)

Problems.benchmark(p010, 1_999_999)


using jInv.Mesh
using jInv.LinearSolvers
using Test
using KrylovMethods
using Multigrid
using LinearAlgebra
using SparseArrays

println("===  Example 2D DivSigGrad ====");

domain = [0.0, 1.0, 0.0, 1.0];
n      = [50,50];
Mr     = getRegularMesh(domain,n)
G      = getNodalGradientMatrix(Mr);
Ar     = G'*G;
Ar     = Ar + 1e-2*norm(Ar,1)*sparse(1.0I,size(Ar,2),size(Ar,2));
N      = size(Ar,2);
B      = Ar*rand(N,4);

levels      = 5;
numCores 	= 8;
maxIter     = 15;
relativeTol = 1e-2;
relaxType   = "SPAI";
relaxParam  = 1.0;
relaxPre 	= 2;
relaxPost   = 2;
cycleType   ='V';
coarseSolveType = "Julia";

MG = getMGparam(Float64,Int64,levels,numCores,maxIter,relativeTol,relaxType,relaxParam,relaxPre,relaxPost,cycleType,coarseSolveType);

sMGPCG   = getMGsolver(MG,Mr, 1,"GMRES",out=1);
X,  = solveLinearSystem(Ar,B,sMGPCG);
@test norm(Ar*X-B)/norm(B) < sMGPCG.tol

sMGPCG   = getMGsolver(MG,Mr, 1,"PCG",out=1);
X,  = solveLinearSystem(Ar,B,sMGPCG);
@test norm(Ar*X-B)/norm(B) < sMGPCG.tol


using NumericalIntegration
using Test
using InteractiveUtils # for subtypes
using StaticArrays
using HCubature # for testing n-dimensional integration

@testset "compare with analytic result" begin
    x = collect(-π : 2*π/2048 : π)
    y = sin.(x)
    p = collect(0 : π/1024 : π)
    q = sin.(p)
    for M in subtypes(IntegrationMethod)
        for T in [Float32, Float64, BigFloat]
            for (xs,ys,val,atol) in [
                                     (x,y,0,1e-4),
                                     (p,q,2,1e-4),
                                  ]
                result = @inferred integrate(T.(xs), T.(ys),M())
                @test isapprox(result, val, atol=atol)
                @test typeof(result) == T
            end
        end
    end
end

@testset "cumulative integration" begin
    x = collect(-π : 2*π/2048 : π)
    y = sin.(x)
    exact = @. -cos(x) + cos(-π)
    for M in subtypes(IntegrationMethod)
        hasmethod(cumul_integrate, Tuple{AbstractVector, AbstractVector, M}) || continue
        for T in [Float32, Float64, BigFloat]
            result = @inferred cumul_integrate(T.(x), T.(y), M())
            @test typeof(result) == Vector{T}
            @test all(isapprox.(result, exact, atol=1e-4))
        end
    end
end

@testset "n-dimensional integration testing" begin
    X = range(0,stop=2π,length=10)
    Y = range(-π,stop=π,length=10)
    Z = range(0,stop=2,length=10)

    A = Array{Float64}(undef,length(X),length(Y),length(Z))
    f(x) = sin(x[1]) + cos(x[2]) + 2x[3]

    for (k,z) in enumerate(Z)
        for (j,y) in enumerate(Y)
            for (i,x) in enumerate(X)
                A[i,j,k] = f([x,y,z])
            end
        end
    end

    @test isapprox(integrate((X,Y,Z),A), hcubature(f,[0,-π,0],[2π,π,2])[1], atol=1e-4)
end

@testset "SVector" begin
    xs = range(0, stop=1, length=9)
    ys1 = randn(9)
    ys2 = randn(9)
    ys = map(SVector, ys1, ys2)
    for M in subtypes(IntegrationMethod)
        m = M()
        res1 = integrate(xs,ys1,m)
        res2 = integrate(xs,ys2,m)
        res = @inferred integrate(xs,ys,m)
        @test res ≈ @SVector [res1, res2]
    end
end

@testset "Raising Warnings" begin
    xs = collect(-1.0 : 0.5 : 1.0)
    ys = xs.^2
    m = RombergEven()
    expwarn = "RombergEven :: final step reached, but accuracy not: 1.3333333333333335 > 1.0e-12"
    @test_logs (:warn, expwarn) integrate(xs, ys, m)
end

using Unitful

@testset "Unitful compatibility" begin
    x = LinRange(1u"s", 10u"s", 9)
    y = rand(9)*u"m/s"
    for M in subtypes(IntegrationMethod)
        @test typeof(integrate(x,y,M())) == typeof(1.0u"m")
        if hasmethod(cumul_integrate, Tuple{AbstractVector, AbstractVector, M})
            @test typeof(cumul_integrate(x,y,M())) == Vector{typeof(1.0u"m")}
        end
    end
end


export Associative, Nonassociative, associativity

"""
Trait to indicate that a binary operation • is associative over set S. Only
methods of • with the signature •(x::S, y::S) are to be considered.

That is to say, ``\forall x, y, z \in S. (x \bullet y) \bullet z = x \bullet (y
\bullet z)``. Note that in practice, this is not so absolute. Minor deviations
such as `Float64` arithmetic not being perfectly associative are allowed for
practical reasons.
"""
abstract Associative

"""
Trait to indicate that a binary operation • is not necessarily associative over
set S. Despite the name, this trait does not imply the existence of a
counterexample to associativity.
"""
abstract Nonassociative

"""
Return the associativity trait of the set S under the binary operation •.
"""
associativity(::Type, ::Function) = Nonassociative

# complex numbers are associative under + and *
# note that this is not in general true for arbitrary Numbers
associativity{N<:AbstractComplex}(::Type{N}, ::typeof(+)) = Associative
associativity{N<:AbstractComplex}(::Type{N}, ::typeof(*)) = Associative

# Strings are associative under *
associativity(::Type{String}, ::typeof(*)) = Associative


"""
    NormalCanon(η, λ)

Canonical parametrisation of the Normal distribution with canonical parameters `η` and `λ`.

The two *canonical parameters* of a normal distribution ``\\mathcal{N}(\\mu, \\sigma^2)`` with mean ``\\mu`` and
standard deviation ``\\sigma`` are ``\\eta = \\sigma^{-2} \\mu`` and ``\\lambda = \\sigma^{-2}``.
"""
struct NormalCanon{T<:Real} <: ContinuousUnivariateDistribution
    η::T       # σ^(-2) * μ
    λ::T       # σ^(-2)
    μ::T       # μ

    function NormalCanon{T}(η, λ; check_args::Bool=true) where T
        check_args && @check_args(NormalCanon, λ > zero(λ))
        new{T}(η, λ, η / λ)
    end
end

NormalCanon(η::T, λ::T; check_args::Bool=true) where {T<:Real} = NormalCanon{typeof(η/λ)}(η, λ; check_args=check_args)
NormalCanon(η::Real, λ::Real; check_args::Bool=true) = NormalCanon(promote(η, λ)...; check_args=check_args)
NormalCanon(η::Integer, λ::Integer; check_args::Bool=true) = NormalCanon(float(η), float(λ); check_args=check_args)
NormalCanon() = NormalCanon{Float64}(0.0, 1.0; check_args=false)

@distr_support NormalCanon -Inf Inf

#### Type Conversions
convert(::Type{NormalCanon{T}}, η::S, λ::S) where {T <: Real, S <: Real} = NormalCanon(T(η), T(λ))
convert(::Type{NormalCanon{T}}, d::NormalCanon{S}) where {T <: Real, S <: Real} = NormalCanon(T(d.η), T(d.λ); check_args=false)

## conversion between Normal and NormalCanon

convert(::Type{Normal}, d::NormalCanon) = Normal(d.μ, 1 / sqrt(d.λ))
convert(::Type{NormalCanon}, d::Normal) = (λ = 1 / d.σ^2; NormalCanon(λ * d.μ, λ))
meanform(d::NormalCanon) = convert(Normal, d)
canonform(d::Normal) = convert(NormalCanon, d)

#### Parameters

params(d::NormalCanon) = (d.η, d.λ)
@inline partype(d::NormalCanon{T}) where {T<:Real} = T

#### Statistics

mean(d::NormalCanon) = d.μ
median(d::NormalCanon) = mean(d)
mode(d::NormalCanon) = mean(d)

skewness(d::NormalCanon{T}) where {T<:Real} = zero(T)
kurtosis(d::NormalCanon{T}) where {T<:Real} = zero(T)

var(d::NormalCanon) = 1 / d.λ
std(d::NormalCanon) = sqrt(var(d))

entropy(d::NormalCanon) = (-log(d.λ) + log2π + 1) / 2

location(d::NormalCanon) = mean(d)
scale(d::NormalCanon) = std(d)

#### Evaluation

pdf(d::NormalCanon, x::Real) = (sqrt(d.λ) / sqrt2π) * exp(-d.λ * abs2(x - d.μ)/2)
logpdf(d::NormalCanon, x::Real) = (log(d.λ) - log2π - d.λ * abs2(x - d.μ))/2

zval(d::NormalCanon, x::Real) = (x - d.μ) * sqrt(d.λ)
xval(d::NormalCanon, z::Real) = d.μ + z / sqrt(d.λ)

cdf(d::NormalCanon, x::Real) = normcdf(zval(d,x))
ccdf(d::NormalCanon, x::Real) = normccdf(zval(d,x))
logcdf(d::NormalCanon, x::Real) = normlogcdf(zval(d,x))
logccdf(d::NormalCanon, x::Real) = normlogccdf(zval(d,x))

quantile(d::NormalCanon, p::Real) = xval(d, norminvcdf(p))
cquantile(d::NormalCanon, p::Real) = xval(d, norminvccdf(p))
invlogcdf(d::NormalCanon, lp::Real) = xval(d, norminvlogcdf(lp))
invlogccdf(d::NormalCanon, lp::Real) = xval(d, norminvlogccdf(lp))

#### Sampling

rand(rng::AbstractRNG, cf::NormalCanon) = cf.μ + randn(rng) / sqrt(cf.λ)

#### Affine transformations

function Base.:+(d::NormalCanon, c::Real)
    η, λ = params(d)
    return NormalCanon(η + c * λ, λ)
end
Base.:*(c::Real, d::NormalCanon) = NormalCanon(d.η / c, d.λ / c^2)


@testset "Test the 2-by-2 symmetric positive definite matrices" begin
using Random: seed!
# explicitly compute an easy exp
M = SymmetricPositiveDefinite(2)
x = SPDPoint([1.0 0.0; 0.0 1.0])
ξ = SPDTVector([1.0 0.0; 0.0 0.0])
η = SPDTVector([1.0 0.0; 0.0 1.0])
y = exp(M,x,ξ)
@test norm( getValue(y) - [ℯ 0;0 1.0]) ≈ 0 atol=10.0^(-16)
# check that with base the base must mach.
z = SPDPoint([1.0 0.0; 0.0 0.1])
ξE = TVectorE(ξ,x);
@test_throws DomainError exp(M,z,ξE)
# check that log is the inverse of exp and stores the base point correctly
# if that's activated
xT = MPointE(x);
ξT2 = log(M,x,y)
@test norm( getValue(ξ) - getValue(ξT2) ) ≈ 0 atol=10.0^(-16)
@test distance(M, getBasePoint(ξE), x) ≈ 0 atol=10.0^(-16)
# test norm
@test norm(M,x,ξ) ≈ distance(M,x,y) atol=10.0^(-16)
# test parallel transport
@test norm(getValue( parallelTransport(M,y,x,log(M,y,x)) ) + getValue(ξ) ) ≈ 0 atol=3*10.0^(-16)
@test parallelTransport(M,x,x,ξ) == ξ
# random
seed!(42)
@test validateMPoint(M,randomMPoint(M))
@test validateTVector(M,x, randomTVector(M,x))
@test validateTVector(M,x, randomTVector(M,x, :Gaussian))
@test validateTVector(M,x, randomTVector(M,x, :Rician))
#
# Test tangent ONB
#
n = 4
M2 = SymmetricPositiveDefinite(n)
x2 = SPDPoint(one(zeros(n,n)))
ξ2 = SPDTVector(getValue(x2))
y2 = exp(M2,x2,ξ2)
Ξ,κ = tangentONB(M2,x2,ξ2)
# test orthogonality
@test all([ dot(M2,x2,Ξ[i],Ξ[j]) for i=1:length(Ξ) for j=i+1:length(Ξ) ] .== 0)
# test normality
@test all( [1-dot(M2,x2,Ξ[i],Ξ[i]) for i=1:length(Ξ)] .< 3*10^(-16) )
Ξ2,κ2 = tangentONB(M2,x2,y2)
@test sum( norm.(Ref(M2),Ref(x2), Ξ .- Ξ2) ) ≈ 0
@test sum(κ - κ2) ≈ 0
#
# test types
@test typeofTVector(x) == SPDTVector{Float64}
@test typeofTVector(SPDPoint{Float64}) == SPDTVector{Float64}
@test typeofMPoint(ξ) == SPDPoint{Float64}
@test typeofMPoint(SPDTVector{Float64}) == SPDPoint{Float64}
#
@test typicalDistance(M) ≈ sqrt(3)
#
@test_throws DomainError validateMPoint(M2,x)
@test_throws DomainError validateMPoint(M, SPDPoint([0. 1.;-1. 0.]))
@test_throws DomainError validateMPoint(M, SPDPoint([0. 0.;0. 0.]))
@test_throws DomainError validateTVector(M,x,ξ2)
@test_throws DomainError validateTVector(M,x,SPDTVector([0. 1.; -2. 0.]) )
#
@test norm(getValue(zeroTVector(M,x))) == 0
@test validateTVector(M,x,zeroTVector(M,x))
# show
@test "$M" == "The Manifold of 2-by-2 symmetric positive definite matrices"
@test "$x" == "SPD($(getValue(x)))"
@test "$ξ" == "SPDT($(getValue(ξ)))"
# Test Matrix trait
@test getValue(x+y) == getValue(x)+getValue(y)
@test getValue(x-y) == getValue(x)-getValue(y)
@test getValue(x*y) == getValue(x)*getValue(y)
@test getValue(ξ+η) == getValue(ξ)+getValue(η)
@test getValue(ξ-η) == getValue(ξ)-getValue(η)
@test getValue(ξ*η) == getValue(ξ)*getValue(η)
@test transpose(x) == SPDPoint(Matrix(transpose(getValue(x))))
@test transpose(ξ) == SPDTVector(Matrix(transpose(getValue(ξ))))
end


module Metrics
    using ..Representation
    function calc_population_fitness!(population::Representation.Population,
                                    true_values::Array{Float64, 1},
                                    error_function, extra_params=nothing)
        for memeber in population.members
            pred_values = [kernels_to_values(x, memeber.gauss_kernels) for x in true_values]
            if extra_params != nothing && error_function == weighted_mean_abs_error
                (error, hit_ratio) = error_function(true_values,
                                        pred_values,
                                        extra_params["threshold"],
                                        extra_params["lower_weight"],
                                        extra_params["upper_weight"])
            else
                (error, hit_ratio) = error_function(true_values, pred_values)
            end
            memeber.fitness = error
            memeber.hit_ratio = hit_ratio
        end
    end

    function mean_sqr_error(true_values::Array{Float64, 1}, pred_values::Array{Float64, 1})
        error = sum((true_values - pred_values).^2)/length(true_values)
        hit_ratio = sum(true_values .== pred_values)
        return error, hit_ratio
    end

    function mean_absolute_error(true_values::Array{Float64, 1}, pred_values::Array{Float64, 1})
        error = sum(abs.(true_values - pred_values))/length(true_values)
        hit_ratio = sum(true_values .== pred_values)
        return error, hit_ratio
    end

    function weighted_mean_abs_error(true_values::Array{Float64, 1},
                                    pred_values::Array{Float64, 1},
                                    threshold::Float64,
                                    lower_weight::Float64,
                                    upper_weight::Float64)

        abs_error = abs.(true_values - pred_values)
        mask = [apply_threshold(value, threshold,
                        lower_weight, upper_weight) for value in abs_error]
        hit_ratio = sum(mask .== 1.0)
        return sum(abs_error.*mask)/length(true_values), hit_ratio

    end

    function apply_threshold(value::Float64, threshold::Float64,
                                lower_weight::Float64, upper_weight::Float64)
        return value <= threshold ? lower_weight : upper_weight
    end

    function kernels_to_values(x::Float64,
                                kernels::Array{Tuple{Float64, Float64, Float64}, 1})
        sum = Float32(0.0)
        for kernel in kernels
            (weight, c, γ) = kernel
            sum += weight * exp(-γ*(c - x)^2)
        end
        return sum
    end

end


# Gaussian models

### For prior IsoNormalCanon

function logpar(d::IsoNormalCanon)
    q = length(d)
    h = d.h
    σ2 = inv(d.J.value)
    (q * log(2π * σ2) + σ2 * vecnorm(h)^2) * 0.5
end

immutable IsoNormalCanonSuffStats
    h::Vector{Float64}  # add to h
    κ::Float64          # add to κ
end

length(ss::IsoNormalCanonSuffStats) = length(ss.h)

function posterior(d::IsoNormalCanon, ss::IsoNormalCanonSuffStats)
    length(d) == length(ss) || throw(DimensionMismatch())
    MvNormalCanon(d.h + ss.h, d.J.value + ss.κ)
end

##
# Gaussian likelihood model
#
#  x ~ N(θ, σ^2)
#
immutable IsoGaussModel <: LikelihoodModel
    σ::Float64    # the standard dev of observations
end

withparams(g::IsoGaussModel, μ::Vector{Float64}) =
    MvNormal(μ, g.σ)

function suffstats(g::IsoGaussModel, X::AbstractMatrix, inds::AbstractVector{Int})
    d, n = size(X)
    h = zeros(d)
    c = 1.0 / g.σ^2
    for i in inds
        x = view(X, :, i)
        axpy!(c, x, h)
    end
    IsoNormalCanonSuffStats(h, n * c)
end

suffstats(g::IsoGaussModel, X::AbstractMatrix) = suffstats(g, X, 1:size(X,2))


using OptimTools
using Base.Test

# test exact line search for quadratic
Q = 10*speye(40)
f(x)    = 0.5*dot(x,Q*x)
df(x)   = Q*x
d2f(x)  = Q
x0 = randn(40)

exactLS  = (f,df,fk,dfk,xk,pk)-> (-dot(pk,dfk)/dot(pk,Q*pk),1)
goldenLSf = (f,df,dk,dfk,xk,pk) -> goldenLS(f,xk,pk,0.0,5.0)

println("Test SD,NLCG,BFGS with exact line search ")
# steepest descent
x1,flag1,his1 = sd(f,df,x0,maxIter=1000,atol=1e-6,out=0,lineSearch=exactLS)
@test norm(x1)<1e-6*norm(x0)
x2,flag2,his2 = sd(f,df,x0,maxIter=1000,atol=1e-6,out=0, lineSearch=exactLS)
@test norm(x2)<1e-6
@test size(his2,1)==2 #should converge in one iteration

# nlcg
x1,flag1,his1 = nlcg(f,df,x0,maxIter=1000,atol=1e-6,out=0)
@test norm(x1)<1e-6*norm(x0)
x2,flag2,his2 = nlcg(f,df,x0,maxIter=1000,atol=1e-6,out=0, lineSearch=exactLS)
@test norm(x2)<1e-6
@test size(his2,1)==2 #should converge in one iteration

# bfgs
x1,flag1,his1 = bfgs(f,df,x0,maxIter=1000,atol=1e-6,out=0)
@test norm(x1)<1e-6*norm(x0)
x2,flag2,his2 = bfgs(f,df,x0,maxIter=1000,atol=1e-6,out=0, lineSearch=exactLS)
@test norm(x2)<1e-6
@test size(his2,1)==2 #should converge in one iteration

println("Done!")

println("Test SD,NLCG,BFGS with golden line search ")
# steepest descent
x1,flag1,his1 = sd(f,df,x0,maxIter=1000,atol=1e-6,out=0,lineSearch=goldenLSf)
@test norm(x1)<1e-6*norm(x0)
x2,flag2,his2 = sd(f,df,x0,maxIter=1000,atol=1e-6,out=0, lineSearch=goldenLSf)
@test norm(x2)<1e-6

# nlcg
x1,flag1,his1 = nlcg(f,df,x0,maxIter=1000,atol=1e-6,out=0)
@test norm(x1)<1e-6*norm(x0)
x2,flag2,his2 = nlcg(f,df,x0,maxIter=1000,atol=1e-6,out=0, lineSearch=goldenLSf)
@test norm(x2)<1e-6

# bfgs
x1,flag1,his1 = bfgs(f,df,x0,maxIter=1000,atol=1e-6,out=0)
@test norm(x1)<1e-6*norm(x0)
x2,flag2,his2 = bfgs(f,df,x0,maxIter=1000,atol=1e-6,out=0, lineSearch=goldenLSf)
@test norm(x2)<1e-6

println("Done!")


using Distributions
using ReverseDiffOverload
import Distributions.logpdf

test(dist, x, params...) = begin
    @assert isapprox(logpdf(dist, x, params...), logpdf(dist(params...), x))
    if dist <: DiscreteDistribution
        testdiff((params...) -> logpdf(dist, x, params...), params...)
    else
        testdiff((x, params...) -> logpdf(dist, x, params...), x, params...)
    end
end

#Normal
valid(::Type{Normal}, x, μ, σ) = begin
    @assert all(σ .> 0)
end
#Differentiable versions.
logpdf(::Type{Normal}, x, μ, σ) = begin
    #valid(Normal, x, μ, σ)
    σ2 = σ.*σ
    diff = x-μ
    -(log(2*pi*σ2) + diff.*diff ./ σ2)/2
end
test(Normal, 1, 2, 3)

#MvNormal
valid(::Type{MvNormal}, x::Vector, μ::Vector, Σ::Matrix) = begin
    @assert isposdef(Σ)
end
#Differentiable versions.
logpdf(::Type{MvNormal}, x::Vector, μ::Vector, Σ::Matrix) = begin
    #valid(MvNormal, x, μ, Σ)
    diff = x-μ
    -(length(x)*log(2*pi) + logdet(Σ) + (diff'/Σ)*x)/2
end
#test(MvNormal, [1., 2], [3., 4], eye(2) + ones(2,2))

#Gamma
valid(::Type{Gamma}, x, shape, scale) = begin
    @assert all(x .>= 0)
    @assert all(shape .>= 0)
    @assert all(scale .>= 0)
end
logpdf(::Type{Gamma}, x, shape, scale) = begin
    #valid(Gamma, x, shape, scale)
    -logΓ(shape) - shape.*log(scale) + (shape-1).*log(x) - x./scale
end
test(Gamma, 1, 2, 3)

#InverseGamma
valid(::Type{InverseGamma}, x, shape, scale) = begin
    @assert all(x .>= 0)
    @assert all(shape .>= 0)
    @assert all(scale .>= 0)
end
logpdf(::Type{InverseGamma}, x, shape, scale) = begin
    #valid(InverseGamma, x, shape, scale)
    -logΓ(shape) + shape.*log(scale) - (shape+1).*log(x) - scale./x
end
test(InverseGamma, 1, 2, 3)

#Bernoulli
valid(::Type{Bernoulli}, x, p) = begin
    @assert all(0 .<= p .<= 1)
    @assert all((x .== 1)|(x .== 0))
end
logpdf(::Type{Bernoulli}, x, p) = begin
    #valid(Bernoulli, x, p)
    x*log(p) + (1-x)*log(1-p)
end
test(Bernoulli, 1., 0.9)
test(Bernoulli, 0., 0.9)

#Categorical
valid(::Type{Categorical}, x, ps::Vector{Float64}) = begin
    @assert isapprox(sum(ps), 1)
    @assert all(0 .<= ps .<= 1)
end
logpdf(::Type{Categorical}, x, ps) = begin
    #valid(Categorical, x, ps)
    log(ps[x])
end
test(Categorical, 2, [0.3, 0.4, 0.3])

#Beta
valid(::Type{Beta}, x, α, β) = begin
    @assert all(α .> 0)
    @assert all(β .> 0)
    @assert all(0 .<= x .<= 1)
end
logB(α, β) = logΓ(α) + logΓ(β) - logΓ(α + β)
logpdf(::Type{Beta}, x, α, β) = begin
    #valid(Beta, x, α, β)
    (α-1).*log(x) + (β-1).*log(1-x) - logB(α, β)
end
test(Beta, 0.6, 1.3, 1.5)

#Dirichlet
valid(::Type{Dirichlet}, x, α::Vector{Float64}) = begin
    @assert all(0 .<= x)
    @assert all(0 .<= α)
end
logB(α) = sum(logΓ(α)) - logΓ(sum(α))
logpdf(::Type{Dirichlet}, x, α) = begin
    #valid(Dirichlet, x, α)
    sum((α-1).*log(x)) - logB(α)
end
test(Dirichlet, [1.2, 1.6, 1.3], [1.3, 1.6, 1.3])

#Poisson
valid(::Type{Poisson}, k, λ) = begin
    @assert all(k .>= 0)
    @assert all(λ .>= 0)
end
logpdf(::Type{Poisson}, k, λ) = begin
    #valid(Poisson, k, λ)
    k*log(λ) - log(factorial(k)) - λ
end
test(Poisson, 3, 5)

#Non-differentiable fallbacks from Distributions.jl
logpdf{D <: Distribution}(d::Type{D}, x, params...) =
    logpdf(d(params...), x)


abstract Type1FuzzySet <: AbstractFuzzySet
export Type1FuzzySet

#####################
abstract Membershipfunction <: Type1FuzzySet
export Membershipfunction

import Base: getindex

getindex(mf::Membershipfunction, range) = [mf[value]::Float64 for value in range]
getindex(mf::Membershipfunction, values...) = [mf[value]::Float64 for value in values]

export getindex

#####################
# Membership Functions
"Triangle Membership Function"
immutable TriangleMF <: Membershipfunction
	a::Float64
	b::Float64
	c::Float64

	TriangleMF(a,b,c) = a <= b <= c ? new(a,b,c) : error("Indices out of order")
end
export TriangleMF

import Base: getindex
function getindex(trimf::TriangleMF, value::Real)
	(value <= trimf.a || trimf.c <= value) && return zero(Float64)
	(value == trimf.b) && return one(Float64)

	if trimf.a < value < trimf.b
		slope = (1.0 - 0.0)/(trimf.b - trimf.a)
		return slope * (value - trimf.a)
	else
		slope = (0.0 - 1.0)/(trimf.c - trimf.b)
		return (slope * (value - trimf.b) + 1.0)
	end
end
export getindex

"Trapezoid Membership Function"
immutable TrapezoidMF <: Membershipfunction
	a::Float64
	b::Float64
	c::Float64
	d::Float64
end
export TrapezoidMF

import Base: getindex
function getindex(trapmf::TrapezoidMF, value::Real)
	(value <= trapmf.a || trapmf.d <= value) && return zero(Float64)
	(trapmf.b <= value <= trapmf.c) && return one(Float64)

	if trapmf.a < value < trapmf.b
		slope = (1 - 0)/(trapmf.b - trapmf.a)
		return slope * (value - trapmf.a)
	else
		slope = (0 - 1)/(trapmf.d - trapmf.c)
		return slope * (value - trapmf.c) + 1
	end
end
export getindex

"Generic Membership Function"
immutable GenericMF <: Membershipfunction
	mf::Function
end
export GenericMF

import Base: getindex
function getindex(gmf::GenericMF, value::Real)
	gmf.mf(value)
end
export getindex

immutable GaussMF <: Membershipfunction
	mu::Float64
	sigma::Float64
end
export GaussMF
immutable GaussMF <: Membershipfunction
	mu::Float64
	sigma::Float64
end
export GaussMF

import Base: getindex
function getindex(gmf::GaussMF, value::Real)
	GenericMF(x -> exp(-(x-gmf.mu).^2/(2.*gmf.sigma.^2)))[value]
end
export getindex

#####################


using Base.Test

N = 10

C = rand(N, N)
A = C - C'

T, Z, λ = schur(A)
@test Z*T*Z' ≈ A

##########################

A = Antisymmetric(A)
T, Z, λ = schur(A)

@test Z*T*Z' ≈ A

#############################

A = AntisymTridiagonal(rand(N))
@show A

T, Z, λ = schur(A)
@test Z*T*Z' ≈ A


module problem112

description = """
Bouncy numbers

Working from left-to-right if no digit is exceeded by the digit to its left it is called an increasing number; for example, 134468.
Similarly if no digit is exceeded by the digit to its right it is called a decreasing number; for example, 66420.
We shall call a positive integer that is neither increasing nor decreasing a "bouncy" number; for example, 155349.
Clearly there cannot be any bouncy numbers below one-hundred, but just over half of the numbers below one-thousand (525) are bouncy. In fact, the least number for which the proportion of bouncy numbers first reaches 50% is 538.
Surprisingly, bouncy numbers become more and more common and by the time we reach 21780 the proportion of bouncy numbers is equal to 90%.

Find the least number for which the proportion of bouncy numbers is exactly 99%.
"""

function increasing(n :: Integer)
  lastDigit = 9
  while n > 0
    d = n % 10
    if d > lastDigit
      return false
    end
    lastDigit = d;
    n = div(n, 10)
  end
  return true
end

function decreasing(n :: Integer)
  lastDigit = 0
  while n > 0
    d = n % 10
    if d < lastDigit
      return false
    end
    lastDigit = d;
    n = div(n, 10)
  end
  return true
end

function bouncy(n :: Integer)
  return !(increasing(n) || decreasing(n))
end

function bouncySearch(percentage :: Integer)
  n = 1
  b = 0
  while b*100 != n*percentage
    n += 1
    b += bouncy(n) ? 1 : 0
  end
  return n
end

using Base.Test

@test bouncySearch(90) == 21780

end


"""
High-order singular value decomposition (HO-SVD).
"""
function hosvd{T,N}(tensor::StridedArray{T,N}, core_dims::NTuple{N, Int};
                    pad_zeros::Bool=false, compute_error::Bool=false)
    pad_zeros || _check_tensor(tensor, core_dims)

    factors = map(1:N) do i
        X = _col_unfold(tensor, i)
        f = eigs(X'X, nev=core_dims[i])[2]
        if pad_zeros && size(f, 2) < core_dims[i] # fill missing factors with zeros
            f = hcat(f, zeros(T, size(tensor, i), core_dims[i]-size(f, 2)))
        end
        mapslices(_check_sign, f, 1)
    end

    res = Tucker((factors...), tensorcontractmatrices(tensor, factors))
    if compute_error
        _set_rel_residue(res, tensor)
    end
    return res
end

hosvd{T,N}(tensor::StridedArray{T,N}, r::Int; compute_error::Bool=false) =
    hosvd(tensor, (fill(r, N)...); compute_error=compute_error)


using FinancialToolbox, DualNumbers, FinancialFFT, FinancialMonteCarlo

A = 600.0;
N = 20000;

method = LewisMethod(A, N);
S0 = 100.0;
K = 100.0;
r = dual(0.02, 1.0);
#r=0.02;
T = 1.0;
d = 0.0;
#sigma=dual(0.2,1.0);
sigma = 0.2;
mu1 = 0.03;
sigma1 = 0.02;
p = 0.3;
lam = 5.0;
lamp = 30.0;
lamm = 20.0;
zero_rate = ZeroRate(r);

#Model = MertonProcess(sigma, lam, mu1, sigma1, Underlying(S0, d));
Model = KouProcess(sigma, lam, p, lamp, lamm, Underlying(S0, d));
#Model = BlackScholesProcess(sigma, Underlying(S0, d));
# Model = VarianceGammaProcess(sigma, 0.01, 0.01, Underlying(S0, d));
#Model = NormalInverseGaussianProcess(sigma, 0.01, 0.01, Underlying(S0, d));

EUData = EuropeanOption(T, K)
Nsim = 10_000;
Nstep = 30;
mc = MonteCarloConfiguration(Nsim, Nstep);

@time @show pricer(Model, zero_rate, method, EUData);
@time @show pricer(Model, zero_rate, mc, EUData);
#typeof(Model) <: BlackScholesProcess ? @time(@show(blsprice(S0, K, r, T, sigma, d))) : nothing;


# MIT License                                                                    #src
#                                                                                #src
# Copyright (c) 2020 François Pacaud                                             #src
#                                                                                #src
# Permission is hereby granted, free of charge, to any person obtaining a copy   #src
# of this software and associated documentation files (the "Software"), to deal  #src
# in the Software without restriction, including without limitation the rights   #src
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell      #src
# copies of the Software, and to permit persons to whom the Software is          #src
# furnished to do so, subject to the following conditions:                       #src
#                                                                                #src
# The above copyright notice and this permission notice shall be included in all #src
# copies or substantial portions of the Software.                                #src
#                                                                                #src
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR     #src
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,       #src
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE    #src
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER         #src
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,  #src
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE  #src
# SOFTWARE.                                                                      #src

# # Logistic Regression

# **Originally Contributed by**: François Pacaud

# This tutorial shows how to solve a logistic regression problem
# with JuMP. Logistic regression is a well known method in machine learning,
# useful when we want to classify binary variables with the help of
# a given set of features. To this goal,
# we find the optimal combination of features maximizing
# the (log)-likelihood onto a training set. From a modern optimization glance,
# the resulting problem is convex and differentiable. On a modern optimization
# glance, it is even conic representable.
#
# ## Formulating the logistic regression problem
#
# Suppose we have a set of training data-point $i = 1, \cdots, n$, where
# for each $i$ we have a vector of features $x_i \in \mathbb{R}^p$ and a
# categorical observation $y_i \in \{-1, 1\}$.
#
# The log-likelihood is given by
#
# ```math
# l(\theta) = \sum_{i=1}^n \log(\dfrac{1}{1 + \exp(-y_i \theta^\top x_i)})
# ```
#
# and the optimal $\theta$ minimizes the logistic loss function:
#
# ```math
# \min_{\theta}\; \sum_{i=1}^n \log(1 + \exp(-y_i \theta^\top x_i)).
# ```
#
# Most of the time, instead of solving directly the previous optimization problem, we
# prefer to add a regularization term:
#
# ```math
# \min_{\theta}\; \sum_{i=1}^n \log(1 + \exp(-y_i \theta^\top x_i)) + \lambda \| \theta \|
# ```
#
# with $\lambda \in \mathbb{R}_+$ a penalty and $\|.\|$ a norm function. By adding
# such a regularization term, we avoid overfitting on the training set and usually
# achieve a greater score in cross-validation.

# ## Reformulation as a conic optimization problem
#
# By introducing auxiliary variables $t_1, \cdots, t_n$ and $r$,
# the optimization problem is equivalent to
#
# ```math
# \begin{aligned}
# \min_{t, r, \theta} \;& \sum_{i=1}^n t_i + \lambda r \\
# \text{subject to } & \quad t_i \geq \log(1 + \exp(- y_i \theta^\top x_i)) \\
#                    & \quad r \geq \|\theta\|
# \end{aligned}
# ```
#
# Now, the trick is to reformulate the constraints $t_i \geq \log(1 + \exp(- y_i \theta^\top x_i))$
# with the help of the *exponential cone*
#
# ```math
# K_{exp} = \{ (x, y, z) \in \mathbb{R}^3 : \; y \exp(x / y) \leq z \} .
# ```
#
# Indeed, by passing to the exponential, we
# see that for all $i=1, \cdots, n$, the constraint $t_i \geq \log(1 + \exp(- y_i \theta^\top x_i))$
# is equivalent to
#
# ```math
# \exp(-t_i) + \exp(u_i - t_i) \leq 1
# ```
#
# with $u_i = -y_i \theta^\top x_i$. Then, by adding two auxiliary variables
# $z_{i1}$ and $z_{i2}$ such that $z_{i1} \geq \exp(u_i-t_i)$ and $z_{i2} \geq \exp(-t_i)$, we get
# the equivalent formulation
#
# ```math
# \left\{
# \begin{aligned}
# (u_i -t_i , 1, z_{i1}) & \in  K_{exp}  \\
# (-t_i , 1, z_{i2}) & \in  K_{exp}  \\
# z_{i1} + z_{i2} & \leq  1
# \end{aligned}
# \right.
# ```
#
# In this setting, the conic version of the logistic regression problems writes out
#
# ```math
# \begin{aligned}
# \min_{t, z, r, \theta}&  \; \sum_{i=1}^n t_i + \lambda r \\
# \text{subject to } & \quad  (u_i -t_i , 1, z_{i1})  \in  K_{exp}  \\
#                    & \quad  (-t_i , 1, z_{i2})  \in  K_{exp}  \\
#                    & \quad  z_{i1} + z_{i2}  \leq  1 \\
#                    & \quad u_i = -y_i x_i^\top \theta \\
#                    & \quad r \geq \|\theta\|
# \end{aligned}
# ```
#
# and thus encompasses $3n + p + 1$ variables and $3n + 1$ constraints ($u_i = -y_i \theta^\top x_i$
# is only a virtual constraint used to clarify the notation).
# Thus, if $n \gg 1$, we get a large number of variables and constraints.

# ## Fitting logistic regression with a conic solver
#
# It is now time to pass to the implementation. We choose SCS as a conic solver.
using JuMP
import Random
import SCS

Random.seed!(2713);

# We start by implementing a function to generate a fake dataset, and where
# we could tune the correlation between the feature variables. The function
# is a direct transcription of the one used in [this blog post](http://fa.bianp.net/blog/2013/numerical-optimizers-for-logistic-regression/).
function generate_dataset(n_samples=100, n_features=10; shift=0.0)
    X = randn(n_samples, n_features)
    w = randn(n_features)
    y = sign.(X * w)
    X .+= 0.8 * randn(n_samples, n_features) # add noise
    X .+= shift # shift the points in the feature space
    X = hcat(X, ones(n_samples, 1))
    return X, y
end

# We write a `softplus` function to formulate each constraint
# $t \geq \log(1 + \exp(u))$ with two exponential cones.
function softplus(model, t, u)
    z = @variable(model, [1:2], lower_bound=0.0)
    @constraint(model, sum(z) <= 1.0)
    @constraint(model, [u - t, 1, z[1]] in MOI.ExponentialCone())
    @constraint(model, [-t, 1, z[2]] in MOI.ExponentialCone())
end

# ### $\ell_2$ regularized logistic regression
# Then, with the help of the `softplus` function, we could write our
# optimization model. In the $\ell_2$ regularization case, the constraint
# $r \geq \|\theta\|_2$ rewrites as a second order cone constraint.
function build_logit_model(X, y, λ)
    n, p = size(X)
    model = Model()
    @variable(model, θ[1:p])
    @variable(model, t[1:n])
    for i in 1:n
        u = - (X[i, :]' * θ) * y[i]
        softplus(model, t[i], u)
    end
    ## Add ℓ2 regularization
    @variable(model, 0.0 <= reg)
    @constraint(model, [reg; θ] in MOI.SecondOrderCone(p+1))
    ## Define objective
    @objective(model, Min, sum(t) + λ * reg)
    return model
end

# We generate the dataset.
#
# !!! warning
#     Be careful here, for large n and p SCS could fail to converge!
#
n, p = 200, 10
X, y = generate_dataset(n, p, shift=10.0);

## We could now solve the logistic regression problem
λ = 10.0
model = build_logit_model(X, y, λ)
set_optimizer(model, SCS.Optimizer)
JuMP.optimize!(model)

#-

θ♯ = JuMP.value.(model[:θ])

# It appears that the speed of convergence is not that impacted by the correlation
# of the dataset, nor by the penalty $\lambda$.

# ### $\ell_1$ regularized logistic regression
#
# We now formulate the logistic problem with a $\ell_1$ regularization term.
# The $\ell_1$ regularization ensures sparsity in the optimal
# solution of the resulting optimization problem. Luckily, the $\ell_1$ norm
# is implemented as a set in `MathOptInterface`. Thus, we could easily formulate
# the sparse logistic regression problem with the help of a `MOI.NormOneCone`
# set.
function build_sparse_logit_model(X, y, λ)
    n, p = size(X)
    model = Model()
    @variable(model, θ[1:p])
    @variable(model, t[1:n])
    for i in 1:n
        u = - (X[i, :]' * θ) * y[i]
        softplus(model, t[i], u)
    end
    ## Add ℓ1 regularization
    @variable(model, 0.0 <= reg)
    @constraint(model, [reg; θ] in MOI.NormOneCone(p+1))
    ## Define objective
    @objective(model, Min, sum(t) + λ * reg)
    return model
end

## Auxiliary function to count non-null components:
count_nonzero(v::Vector; tol=1e-6) = sum(abs.(v) .>= tol)

## We solve the sparse logistic regression problem on the same dataset as before.
λ = 10.0
sparse_model = build_sparse_logit_model(X, y, λ)
set_optimizer(sparse_model, SCS.Optimizer)
JuMP.optimize!(sparse_model)

#-

θ♯ = JuMP.value.(sparse_model[:θ])
println("Number of non-zero components: ", count_nonzero(θ♯),
        " (out of ", p, " features)")

# ### Extensions
# A direct extension would be to consider the sparse logistic regression with
# *hard* thresholding, which, on contrary to the *soft* version using a $\ell_1$ regularization,
# adds an explicit cardinality constraint in its formulation:
#
# ```math
# \begin{aligned}
# \min_{\theta} & \; \sum_{i=1}^n \log(1 + \exp(-y_i \theta^\top x_i)) + \lambda \| \theta \|_2^2 \\
# \text{subject to } & \quad \| \theta \|_0 <= k
# \end{aligned}
# ```
#
# where $k$ is the maximum number of non-zero components in the vector $\theta$,
# and $\|.\|_0$ is the $\ell_0$ pseudo-norm:
#
# ```math
# \| x\|_0 = \#\{i : \; x_i \neq 0\}
# ```
#
# The cardinality constraint $\|\theta\|_0 \leq k$ could be reformulated with
# binary variables. Thus the hard sparse regression problem could be solved
# by any solver supporting mixed integer conic problems.
#


"""
# Usage

    solve(α, dx, dt, xStart, xEnd, n, κ, FiniteDiffEx())

!!! tip
    Here, if we set ``0<\\alpha\\leq 1``, the equation is the fractional diffusion equation or subdiffusion equation, whereas for ``1<\\alpha\\leq 2``, the equation is the fractional diffusion-wave equation.

Use explicit Caputo discretization method to solve the diffusion equations.

### References

```tex
@article{Murillo2011AnED,
  title={An Explicit Difference Method for Solving Fractional Diffusion and Diffusion-Wave Equations in the Caputo Form},
  author={Joaqu{\'i}n Quintana Murillo and Santos B. Yuste},
  journal={Journal of Computational and Nonlinear Dynamics},
  year={2011},
  volume={6},
  pages={021014}
}
```

Matlab version: https://github.com/awstown/Fractional-Derivative
"""
struct FiniteDiffEx <: FractionalDiffEqAlgorithm end

function solve(α, dx, dt, xStart, xEnd, n, κ, u0t, uendt, u0, ::FiniteDiffEx)
    x = collect(0:dx:xEnd)
    t = collect(0:dt:n)
    S = κ*((dt^α)/(dx^2))
    S_bar = gamma(3-α)*S

    # Preallocate
    U = zeros(Int64(n/dt + 1), round(Int, (xEnd - xStart)/dx + 1))

    # Boundry conditions
    U[:, 1] .= u0t
    U[:, end] .= uendt
    U[1, :] = u0.(x)

    k = collect(1:length(t)-2)
    bOfK = bbcoeff(k, α)
    test = Float64[]

    for m = 1:(length(t)- 1)
        # It is from 2 to end - 1 because of the boundry conditions
        U[m+1, 2:end-1], diff = nextStep(U, m, S_bar, test, bOfK)
    end
    return U
end

function bbcoeff(k, α)
    c = zeros(length(k))
    for i=1:length(k)
        if maximum(k) < 2
            @. c = (k+1)^(1-α) - k^(1-α)
        else
            @. c = (k+1)^(2-α) - k^(2-α)
        end
    end
    return c
end

# Compute values for each time step
function nextStep(U, m, S_bar, diff, bOfK)
    row_left   = U[m, 1:end-2]
    row_right  = U[m, 3:end]
    row_center = U[m, 2:end-1]

    if m < 2
        Uspatial = row_center + S_bar*(row_left - 2*row_center + row_right)
    else
        row_below  = U[m-1, 2:end-1]
        Uspatial = 2 * row_center - row_below + S_bar*(row_left - 2*row_center + row_right)
    end

    if m > 2
        if length(diff) == 0
            diff = U[m, 2:end-1] - 2*U[m-1, 2:end-1] + U[m-2, 2:end-1]
        else
            diff = [U[m, 2:end-1] .- 2 .*U[m-1, 2:end-1] .+ U[m-2, 2:end-1]; diff]
        end
        Utemporal = bOfK[1:length(diff[:, 1])]' * diff
        Unext = Uspatial .- Utemporal
    elseif m > 1
        if length(diff) == 0
            diff = U[m, 2:end-1] .- U[m-1, 2:end-1]
        else
            diff = [U[m, 2:end-1] .- U[m-1,2:end-1]; diff]
        end
        Utemporal = bOfK[1:length(diff[:, 1])]' * diff
        Unext = Uspatial .- Utemporal
    else
        Unext = Uspatial
    end
    return Unext, diff
end


module _SpecEn
export SpecEn
using FFTW: fft
using DSP: conv
    """
        Spec, BandEn = SpecEn(Sig)

    Returns the spectral entropy estimate of the full spectrum (`Spec`)
    and the within-band entropy (`BandEn`) estimated from the data sequence (`Sig`)
    using the default  parameters:
    N-point FFT = 2*len(`Sig`) + 1, normalised band edge frequencies = [0 1],
    logarithm = base 2, normalisation = w.r.t # of spectrum/band frequency values.

        Spec, BandEn = SpecEn(Sig::AbstractArray{T,1} where T<:Real; N::Int=1 + (2*size(Sig,1)), Freqs::Tuple{Real,Real}=(0,1), Logx::Real=exp(1), Norm::Bool=true)

    Returns the spectral entropy (`Spec`) and the within-band entropy (`BandEn`)
    estimate for the data sequence (`Sig`) using the specified 'keyword' arguments:

    # Arguments:
    `N`     - Resolution of spectrum (N-point FFT), an integer > 1  \n
    `Freqs` - Normalised band edge frequencies, a 2 element tuple with values \n
              in range [0 1] where 1 corresponds to the Nyquist frequency (Fs/2).
              Note: When no band frequencies are entered, BandEn == SpecEn
    `Logx`  - Logarithm base, a positive scalar (enter 0 for natural log) \n
    `Norm`  - Normalisation of `Spec` value:\n
              [false]  no normalisation.
              [true]   normalises w.r.t # of spectrum/band frequency values - default.

    For more info, see the EntropyHub guide.

    # See also `XSpecEn`, `fft`, `MSEn`,  `XMSEn`

    # References:
        [1] G.E. Powell and I.C. Percival,
            "A spectral entropy method for distinguishing regular and
            irregular motion of Hamiltonian systems."
            Journal of Physics A: Mathematical and General
            12.11 (1979): 2053.

        [2] Tsuyoshi Inouye, et al.,
            "Quantification of EEG irregularity by use of the entropy of
            the power spectrum."
            Electroencephalography and clinical neurophysiology
            79.3 (1991): 204-210.

    """
    function SpecEn(Sig::AbstractArray{T,1} where T<:Real; N::Int=1 + (2*size(Sig,1)),
        Freqs::Tuple{Real,Real}=(0,1), Logx::Real=exp(1), Norm::Bool=true)

    (size(Sig)[1] > 4) ? nothing : error("Sig:   must be a numeric vector")
    (N > 1) ? nothing :  error("N:     must be an integer > 1")
    (0<=Freqs[1]<1 && 0<Freqs[2]<=1 && Freqs[1]<Freqs[2]) ? nothing :
        error("Freq:    must be a two element tuple with values in range [0 1].
                The values must be in increasing order.")
    (Logx>0) ? nothing :   error("Logx:     must be a positive number > 0")

    Freqs = collect(Freqs)
    Fx = Int(ceil(N/2))
    Freqs = Int.(round.(Freqs.*Fx))
    Freqs[Freqs.==0] .= 1

    if Freqs[1] > Freqs[2]
        error("Lower band frequency must come first.")
    elseif Freqs[2]-Freqs[1]<1
        error("Spectrum resoution too low to determine bandwidth.")
    elseif minimum(Freqs)<0 || maximum(Freqs)>Fx
        error("Freqs must be normalized w.r.t sampling frequency [0 1].")
    end

    Temp = conv(Sig,Sig)
    N <= size(Temp,1) ? Temp = Temp[1:N]  : Temp = vcat(Temp,zeros(N-size(Temp)[1]))

    Pt = abs.(fft(Temp))
    Pxx = Pt[1:Fx]/sum(Pt[1:Fx])
    Spec = -(transpose(Pxx)*log.(Logx, Pxx))
    Pband = (Pxx[Freqs[1]:Freqs[2]])/sum(Pxx[Freqs[1]:Freqs[2]])
    BandEn = -(transpose(Pband)*log.(Logx, Pband))

    if Norm
        Spec = Spec/(log(Logx, Fx));
        BandEn = BandEn/(log(Logx, Freqs[2]-Freqs[1]+1));
    end

    return Spec, BandEn
    end

end

"""
Copyright 2021 Matthew W. Flood, EntropyHub

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

For Terms of Use see https://github.com/MattWillFlood/EntropyHub

"""


using LinearAlgebra
using Plots
using CSV
using KernelFunctions
using Distributions
using DataFrames

include("../algo/kernels.jl")
include("../algo/regularized_Picard.jl")
include("../algo/utils.jl")

"""
    convergence_exact_solution()

Visualize the convergence of regularized Picard to exact solution (see Proposition 10)
"""
function convergence_exact_solution()

    # width
    sigma = .01
    # regularizer
    lambda =  .05
    # regularizer for positive definiteness
    epsilon = 1e-10;
    # relative objective tolerance
    tol = 1e-6

    ############# load 1 DPP samples #############
    # intensity of the generating DPP
    intensity = 100; # 50 or 100
    strg_intensity = string(intensity)

    # create an array of arrays
    indices_DPP_samples = Array{Int64,1}[];

    print("Loading DPP samples from: \n")
    i = 1; # id of the DPP sample
    file_name = "data/statspats/samples/GaussDPPsample_alpha0_00p5_rho0_"*strg_intensity*"_nb_"*string(i)*".csv"
    print(file_name, " \n")
    temp = CSV.File(file_name; header=true) |> Tables.matrix
    temp = temp[:,2:3];
    id_temp = collect(1:size(temp,1));
    push!(indices_DPP_samples,id_temp);
    total_samples = temp;

    # for approximating Fredholm determinant
    indices_Fredholm_sample = indices_DPP_samples[1]

    ############# exact solution #############

    # create full kernel matrix
    x = (total_samples)'/sigma;
    k = SqExponentialKernel();
    K = kernelmatrix(k, x) + epsilon *I ;

    # Chol decomposition
    R = cholesky(K).U;
    Rinv = inv(R);

    # Exact solution
    m = size(K,1);
    X_exact = 0.5*(real(sqrt(m^2*I + 4*m*K/lambda))-m*I);
    B_exact = Rinv'*X_exact*Rinv;

    # objective exact solution
    obj_det_exact,obj_reg_exact = Picard_objective(B_exact, indices_DPP_samples, indices_Fredholm_sample, R,lambda)
    obj_exact = obj_det_exact + obj_reg_exact;

    ############# approximate solution #############

    # initial positive definite iterate
    X = randn(size(K));
    B = X*X'+ UniformScaling(epsilon);

    use_inverse = false

    n_steps = 50;
    it_max = 1;
    err = ones(n_steps,1);
    obj = ones(n_steps,1);

    for j=1:n_steps
        B, R, obj_j, i_stop = regularized_Picard(B, K, indices_DPP_samples, indices_Fredholm_sample, lambda, it_max ,tol,use_inverse)
        err[j] = norm(B-B_exact)/norm(B_exact);
        obj[j] = obj_j[end];
    end

    #############  plotting #############

    plt_error = plot(err, yaxis=:log,legend=false,ylabel = "relative error",xlabel = "iteration")
    display(plt_error)

    plt_obj = plot(obj,legend=false,ylabel = "objective",xlabel = "iteration")
    plot!(obj_exact*ones(size(obj)))
    display(plt_obj)
end


import Base.kron

# (A ⊗ B)×vec(X) = vec(BXAᵀ)
"""`kron(A, B)`
Kronecker tensor product of A and B in linear operator form, if either
or both are linear operators. If both A and B are matrices, then
`Base.kron` is used.
"""
function kron(A::AbstractLinearOperator, B::AbstractLinearOperator)
  m, n = size(A)
  p, q = size(B)
  T = promote_type(eltype(A), eltype(B))
  function prod!(res, x, α, β)
    S = promote_type(T, eltype(x))
    X = reshape(convert(Vector{S}, x), q, n)
    res .= Matrix(B * X * transpose(A))[:]
  end
  function tprod!(res, x, α, β)
    S = promote_type(T, eltype(x))
    X = reshape(convert(Vector{S}, x), p, m)
    res .= Matrix(transpose(B) * X * A)[:]
  end
  function ctprod!(res, x, α, β)
    S = promote_type(T, eltype(x))
    X = reshape(convert(Vector{S}, x), p, m)
    res .= Matrix(B' * X * conj(A))[:]
  end
  symm = issymmetric(A) && issymmetric(B)
  herm = ishermitian(A) && ishermitian(B)
  nrow, ncol = m * p, n * q
  Mv = Vector{T}(undef, nrow)
  Mtu = symm ? Mv : Vector{T}(undef, ncol)
  Maw = herm ? Mv : Vector{T}(undef, ncol)
  return LinearOperator{T}(nrow, ncol, symm, herm, prod!, tprod!, ctprod!, Mv, Mtu, Maw)
end

kron(A::AbstractMatrix, B::AbstractLinearOperator) = kron(LinearOperator(A), B)

kron(A::AbstractLinearOperator, B::AbstractMatrix) = kron(A, LinearOperator(B))


##  Test of fft, power spectrum, and Hilbert transform
##  using as example of the ElNino time series
##
##  Author Bernd Blasius (bernd.blasius@gmail.com)

module W

using CSV, PyPlot, FFTW, Statistics

include("../src/fft_power.jl")

function test_power_spectrum()
 # read in the data
  df = CSV.read("../data/ElNino_data.csv"; comment="#")
  t   = df[:,1]  # time (year)
  sst = df[:,2]  # el nino signal (sea surface temperature SST)
  x   = df[:,3]  # el nino temperature anomaly

  fs = 1 ./ (mean(diff(t)))  # mean sampling frequency
  f, pow = power_real(x,fs)

  figure("El Nino Power")
  clf()
  subplot(2,1,1)
  plot(t,x)
  ylabel("El Nino")
  xlabel("time")

  subplot(2,1,2)
  plot(f, log10.(pow))
  xlabel("frequency")
  ylabel("power")

end

function test_hilbert()
  # read in the El Nino data
  df = CSV.read("../data/ElNino_data.csv"; comment="#")
  t   = df[:,1]  # time (year)
  sst = df[:,2]  # el nino signal (sea surface temperature SST)
  x   = df[:,3]  # el nino temperature anomaly

  fs = 1 ./ (mean(diff(t)))  # mean sampling frequency
  h = hilbert(x,fs)

  figure("hilbert")
  clf()
  subplot(2,1,1)  # phase plane
  plot(t,x)  # El Nino
  plot(t,h,"r") # Hilber transform
  xlabel("time")
  ylabel("El Nino")

  subplot(2,1,2)  # phase plane
  plot(x,h)

end

#test_power_spectrum()
test_hilbert()

end # module


using CombUtils

function main()
    f1 = big(1)
    f2 = big(1)
    index = 2
    num_digits = 1
    while num_digits < 1000
        index += 1
        tmp = f2
        f2 += f1
        f1 = tmp
        num_digits = ndigits(f2)
    end
    @printf("The first index where the nth fib has 1000 digits is %d\n", index)
end

main()


function simpson_integration(x_vec,y_vec)

    sum = 0.
    dx = x_vec[2]-x_vec[1]

    #First point
    sum += 3*y_vec[1]/8 + 3*y_vec[end]/8
    sum += 7*y_vec[2]/6 + 7*y_vec[end-1]/6
    sum += 23*y_vec[3]/24 + 23*y_vec[end-2]/24

    for idx in 4:(length(x_vec)-3)
        sum += y_vec[idx]
    end
    return sum*dx
end

function simpson_integration_2D(x_vec,y_vec,z_vec)

    sum_vector = zeros(length(y_vec))

    for idx in 1:length(sum_vector)
        sum_vector[idx] = simpson_integration(x_vec,z_vec[:,idx])
    end

    sum = simpson_integration(y_vec , sum_vector)

    return sum
end

function get_field_energy( fname::String, every::Int , tlimit::Float64)

    #Get Variables
    fid         = h5open( fname , "r")

    rmin = read_attribute(fid, "rmin")
    rmax = read_attribute(fid, "rmax")
    rnodes   = read_attribute(fid, "rnodes")

    thnodes   = read_attribute(fid, "thnodes")

    #Discretize space
    dr = (rmax - rmin)/(rnodes-1)
    r = [rmin + (i-1)*dr for i in 1:rnodes]

    dth = 2*pi/(thnodes)
    theta = [ (i-1)*dth for i in 1:thnodes]

    #Create Differential Operators
    Dr  =   Diff_Operator_2D(1 ,1, [dr,dth] , [rnodes,thnodes])
    Drr =   Diff_Operator_2D(2 ,1, [dr,dth] , [rnodes,thnodes])

    Dp  =  Diff_Operator_2D_Periodic(1 ,2, [dr,dth] , [rnodes,thnodes])
    Dpp =  Diff_Operator_2D_Periodic(2 ,2, [dr,dth] , [rnodes,thnodes])

    one_over_r  = spzeros(rnodes*thnodes,rnodes*thnodes)
    r_matrix = spzeros(rnodes*thnodes,rnodes*thnodes)

    for i in 1:rnodes*thnodes
        one_over_r[i,i] = (1/r[1 + (i-1)%rnodes])
        r_matrix[i,i] = (r[1 + (i-1)%rnodes])
    end

    #Get max time iteration
    _ , tmax , max_sim_iter  = get_time_variables(fname)
    max_plot_iter = max_sim_iter

    if(tlimit <=  tmax)
        max_plot_iter = floor(Int , max_sim_iter * tlimit/tmax )
    else
        println("ATTENTION: max time input is larger than simulation time, using tmax")
    end

    # Total volume of space
    Volume = pi*( rmax^2 - rmin^2)
    #dV = dx*dy
    #I_factor = dV / Volume

    #Create storage matrix
    Energy_matrix = zeros( rnodes , thnodes )

    #Storage vectors
    E_vector = Float64[]
    t_vector = Float64[]

    t = 0
    E = 0

    #Loop to get energy
    for i in 0:every:max_plot_iter

        #Time and fields
        t, _ , _ , ψ , dψ = get_fields(fname,i)

        #Gradient term
        Psi  = reshape(ψ, rnodes*thnodes)
        dPsi = reshape(dψ, rnodes*thnodes)

        vr = reshape( r_matrix * Drr * Psi ,  (rnodes,thnodes))
        vp = reshape( r_matrix * one_over_r * Dp * Psi , (rnodes,thnodes))
        vt = reshape( r_matrix *  dPsi , (rnodes,thnodes))

        Energy_matrix = (vr.^2 .+ vp.^2 .+ vt.^2)

        E = simpson_integration_2D(r , theta ,Energy_matrix)

        #Add to storage vectors  and normalize
        push!(t_vector, t)
        push!(E_vector, E/Volume  )

    end

    return t_vector , E_vector

end


export L1ball

struct L1ball{T <: Real} <: ConvexSet
    center::Vector{T}
    radius::T
end

"""

    L1ball(c, r)

A ball with center `c` and radius `r`.
"""

function project!(s::L1ball{T}, v::Vector{T}, y::Vector{T}) where {T <: Real}
    n = length(y)
    z = sum(abs, y-s.center)
   if z <= s.radius
      v = y
   else
      w = project(Projections.Simplex(s.radius),y)
      sgn = zeros(n)
      d = y-s.center
      for i in 1:n
            if d[i] > 0
                sgn[i] = 1
            elseif d[i] == 0
                sgn[i] = 0
            else
                sgn[i] = -1
            end
      end
        v = s.center .+ sgn .* w
   end
end

function project(s::L1ball{<:Real}, y::Vector{<:Real})
    project!(s, similar(y), y)
end


s = """
73167176531330624919225119674426574742355349194934
96983520312774506326239578318016984801869478851843
85861560789112949495459501737958331952853208805511
12540698747158523863050715693290963295227443043557
66896648950445244523161731856403098711121722383113
62229893423380308135336276614282806444486645238749
30358907296290491560440772390713810515859307960866
70172427121883998797908792274921901699720888093776
65727333001053367881220235421809751254540594752243
52584907711670556013604839586446706324415722155397
53697817977846174064955149290862569321978468622482
83972241375657056057490261407972968652414535100474
82166370484403199890008895243450658541227588666881
16427171479924442928230863465674813919123162824586
17866458359124566529476545682848912883142607690042
24219022671055626321111109370544217506941658960408
07198403850962455444362981230987879927244284909188
84580156166097919133875499200524063689912560717606
05886116467109405077541002256983155200055935729725
71636269561882670428252483600823257530420752963450
"""
s2 = replace(s, "\n" => "")
a = split(s2, "")
a2 = map(x -> parse(Int, x), a)
len = length(a2)

window = 13
maxsofar = 0
for i = 1:(len-window+1)
  global maxsofar
  p = prod(a2[i:i+window-1])
  maxsofar = max(maxsofar, p)
end
println(maxsofar)


# Demonstrate NTF synthesis (synthesizeNTF)
using RSDeltaSigmaPort
j=im

#==Baseband modulator
===============================================================================#
println("\n*** 5th order, 2-level, baseband modulator")
OSR = 32

@info("Synthesizing NTF without zero-optimization..."); flush(stdout); flush(stderr)
#-------------------------------------------------------------------------------
NTF_noopt = synthesizeNTF(5, OSR, opt=0)
println("\tdone.")

plot = plotNTF(NTF_noopt, OSR, color=:blue)
plot.title = "5th-Order Modulator (No Zero-Optimization)"
saveimage(:png, "dsdemo1_o5_noopt.png", plot, AR=2/1, width=900)
displaygui(plot)

@info("Synthesizing NTF with optimized zeros..."); flush(stdout); flush(stderr)
#-------------------------------------------------------------------------------
NTF_opt = synthesizeNTF(5, OSR, opt=1)
println("\tdone.")

plot = plotNTF(NTF_opt, OSR, color=:red)
plot.title = "5th-Order Modulator (Optimized Zeros)"
saveimage(:png, "dsdemo1_o5_zopt.png", plot, AR=2/1, width=900)
displaygui(plot)

@info("Plotting NTF comparison (overlay results)")
#-------------------------------------------------------------------------------
plot = plotNTF(NTF_noopt, OSR, color=:blue)
plot = plotNTF!(plot, NTF_opt, OSR, color=:red)
plot.title = "5th-Order Modulator (Optimized Zeros - Overlay)"
saveimage(:png, "dsdemo1_o5_cmp.png", plot, AR=2/1, width=900)
displaygui(plot)

#==Bandpass modulator
===============================================================================#
println("\n*** 8th order, 2-level, bandpass modulator")
OSR = 64
order = 8
f0 = 0.125 #fs/8

function calcSTF(order, OSR, NTF, f0)
	G = _zpk(zeros(array_round(order/2)),NTF.p,1,1)
	G.k = 1/abs(evalTF(G,exp(2π*j*f0)))
	return G
end

@info("Synthesizing NTF..."); flush(stdout); flush(stderr)
#-------------------------------------------------------------------------------
NTF = synthesizeNTF(order, OSR, opt=2, f0=f0)
println("\tdone.")

@info("Plotting NTF")
#-------------------------------------------------------------------------------
STF = calcSTF(order, OSR, NTF, f0)
plot = plotNTF(NTF, OSR, color=:blue, f0=f0, STF=STF)
plot.title = "8th-Order Bandpass Modulator"
saveimage(:png, "dsdemo1_o8_bp.png", plot, AR=2/1, width=900)
displaygui(plot)

:END_OF_DEMO


# Largest prime factor

NUMBER = 600_851_475_143

function solve()
    n = NUMBER
    i = 2
    while n > 1
        if n % i == 0
            n /= i
        else
            i += 1
        end
    end
    return i
end

print(Int(solve()))


#=
The power set of a set is the set of all its subsets. Write a function that, given a set, generates its power set.

For example, given the set {1, 2, 3}, it should return {{}, {1}, {2}, {3}, {1, 2}, {1, 3}, {2, 3}, {1, 2, 3}}.

You may also use a list or array to represent a set.
=#

function get_powerset(set::Array{Int,1})
    powerset = [[]]
    for number in set
        push!(powerset, [number])
    end
    set_length = 1
    while set_length < length(set)
        newpowerset = copy(powerset)
        for arr_index in eachindex(powerset)
            arr = powerset[arr_index]
            for number in set
                if length(arr) == set_length && !in(number, arr) && number > arr[end]
                    new_array = union(arr, number)
                    push!(newpowerset, new_array)
                end
            end
        end
        powerset = newpowerset
        set_length = set_length + 1
    end
    return powerset
end


function issquarefree{I<:Integer}(d::I)
  # factor(d) is a dictionary whose keys are primes
  # and whose values are corresponding prime powers.
  # An integer is square free iff none of its prime
  # factors have power 2 or greater. This is equivalent
  # to its Mobius function being nonzero.
  mobius(d) != 0
end

"""

[Link](https://proofwiki.org/wiki/Definition:M%C3%B6bius_Function)
"""
function mobius{I<:Integer}(n::I)
  if n==1 return I(1) end
  powers = collect(values(factor(n)))
  return all(powers.== 1) ? I(-1)^length(powers) : I(0)
end

#  Division by monic polynomials

function onestep!{I<:Integer}(num::Array{I,1}, den::Array{I,1}, quo::Array{I,1})
  n = findlast(num .!= 0)
  d = findlast(den .!= 0)
  den[d] == 1 || error("denominator is not monic")
  k = n-d
  k >= 0 || error("numerator not divisible by denominator")
  quo[1+k] += num[n]
  for i=1:d
    num[i+k] -= num[n]*den[i]
  end
end

function divide_by_monic{I<:Integer}(num::Array{I,1}, den::Array{I,1})
  cnum = deepcopy(num)
  quo = zeros(I,length(num))
  while any(cnum .!= 0)
    onestep!(cnum, den, quo)
  end
  quo
end

function cyclotomic{I<:Integer}(n::I)
  n >= 1 || error("There is no cyclotomic of degree < 1")
  if n==1
    return [I(-1), I(1)]
  end
  num = zeros(I,n+1)
  num[n+1]=I(1)
  num[1] = I(-1)
  for d in 1:(n-1)
    if n%d == 0
      num = divide_by_monic(num, cyclotomic(d))
    end
  end
  d = findlast(num .!= 0)
  num[1:d]
end


# # Mesh with implicit geometry

# In this tutorial, we will learn
#
#    -  How to create a mesh of hexahedra.
#    -  How to create a mesh using an implicit definition of the geometry.

# The geometry is a ramp in the form of 3/4 part of a circle.

# The tutorial will produce files for mesh visualization in the
# [Paraview](https://www.paraview.org/) format (VTK). One can display this
# information by loading the file with `paraview.exe`. When the tutorial is
# executed in `mybinder.org`, the graphics file needs to be downloaded to your
# desktop, and then visualized locally.

# We shall use static arrays for the connectivities and the coordinates.
using StaticArrays

# We start with a parametric domain: a rectangular block, with dimensions of
# different physical units. The extent of the domain: the first
# coordinate is angular. The next two coordinates are in the plane generating
# section.
L, W, H = 1.5*pi, 3.0, 1.5
R = 4.0

# This is the number of element edges along each dimension.
M, N, K = 13, 4, 2

# Now we define the indexing of the vertices. The linear index converts the
# triple of indexes along the three dimensions into a single number. the
# dimensions correspond to the number of nodes along direction.
linix = LinearIndices((M+1, N+1, K+1))
# The serial-number function converts the triple of indexes into a single linear
# index.
sn(m, n, k) = linix[m, n, k]

# Now we create the connectivities of the individual hexahedral cells. The
# numbering is given by the topologically regular character of the block domain.
C = SVector{8, Int}[]
for k in 1:K, n in 1:N, m in 1:M
    c = (
        sn(m, n, k), sn(m+1, n, k), sn(m+1, n+1, k), sn(m, n+1, k),
        sn(m, n, k+1), sn(m+1, n, k+1), sn(m+1, n+1, k+1), sn(m, n+1, k+1)
        )
    push!(C, SVector{8, Int}(c))
end

# The locations of the vertices is also perfectly regular within the parametric
# domain. The nodes are distributed uniformly along each direction. the location
# in the physical space is computed from the parametric coordinates: the first
# coordinate is used as an angle, and `R` is the interior radius of the circular
# ramp.
loc = SVector{3, Float64}[]
for k in 1:K+1, n in 1:N+1, m in 1:M+1
    a = (m-1)/M*L
    x, y, z = (R+(n-1)/N*W) * cos(a), (R+(n-1)/N*W) * sin(a), (k-1)/K*H * (1 + a)
    push!(loc, SVector{3, Float64}(x, y, z))
end

# At this point we can create the shape collections.
using MeshCore: ShapeColl, H8, P1
# This is the collection of hexahedral shapes.
elements = ShapeColl(H8, length(C), "elements")
# And this is the collection of the vertices.
vertices = ShapeColl(P1, length(loc), "vertices")

# The geometry is attached is an attribute. Here we use vector attribute which
# actually stores three coordinates per vertex.
using MeshCore: VecAttrib
vertices.attributes["geom"] = VecAttrib(loc)
using MeshCore: IncRel
connectivities = IncRel(elements, vertices, C)

# We now write out a postprocessing file with the elements. Note that we pass
# the `connectivities` incidence relation to get the elements stored in the
# file.
using MeshSteward: vtkwrite
vtkwrite("block", connectivities)

# Now we will define another shape collection. This one will refer to geometry
# defined implicitly: through a function that returns the coordinates  of a
# vertex based on its serial number. Remember that serial numbers are stored in
# the connectivities.

vertices2 = ShapeColl(P1, prod((M+1, N+1, K+1)), "vertices2")

# The Cartesian indices will allow us to map the linear index to a Cartesian
# triple of indices.
cart = CartesianIndices((M+1, N+1, K+1))
using MeshCore: FunAttrib

# Note that this function defines the same geometry in physical space as the
# loop above. However, the geometry is not stored, it is returned as a triple of
# numbers.
access_location(i) = let
    ci  = cart[i]
    m, n, k = ci[1], ci[2], ci[3]
    a = (m-1)/M*L
    x, y, z = (R+(n-1)/N*W) * cos(a), (R+(n-1)/N*W) * sin(a), (k-1)/K*H * (1 + a)
    SVector{3, Float64}(x, y, z)
end

# The geometry attribute is defined by providing access to the locations of the
# vertices through the function `access_location`.
vertices2.attributes["geom"] = FunAttrib(0.0, prod((M+1, N+1, K+1)), access_location);
connectivities = IncRel(elements, vertices2, C)
vtkwrite("block2", connectivities)

# Now both data sets have been written out. Load them into
# [Paraview](https://www.paraview.org/) and it should be possible to verify that
# those two meshes sets overlap perfectly.

true


# Example of MLE estimation. The data is really drawn from the
# Logit DGP, so the model is well specified, and the MLE has
# the properties discussed in class. E.g., if you make n very
# large you should see that the estimator is very close to the
# true value of theta used to generate data

using Econometrics
include("LogitDGP.jl")
n = 30 # sample size
theta = [0, 0.5] # true theta for generating data
(y, x) = LogitDGP(n, theta) # generate the data

# now define things for estimation
model = theta -> logit(theta, y, x)
theta = zeros(size(x,2)) # start values for estimation

# Perform the estimation - Make sure that you examine
# the MLE estimation programs so that you see how this works
thetahat, objvalue, V, converged = mleresults(model, theta, "estimate logit model");


export getLaplacianMatrix, getConvectionMatrix

"""
function ddxc(n)

centered one-dimensional finite difference operator.

Input:

    n  - number of cells

Output:

    dx - sparse matrix for 1D derivative size(dx) = (n,n)
"""
function ddxc(n::Int)
    dx = spdiagm(-1=>fill(-1.0,n-1), 1=>fill(1.0,n-1))
    return dx
end

"""
function ddxfwd(n)

forward one-dimensional finite difference operator.

Input:

    n  - number of cells

Output:

    dx - sparse matrix for 1D derivative size(dx) = (n,n)
"""
function ddxfwd(n::Int)
    dx = spdiagm(0=>fill(-1.0,n),1=>fill(1.0,n-1))
    return dx
end

"""
function ddxbwd(n)

backward one-dimensional finite difference operator.

Input:

    n  - number of cells

Output:

    dx - sparse matrix for 1D derivative size(dx) = (n,n)
"""
function ddxbwd(n::Int)
    dx = spdiagm(-1=>fill(-1.0,n-1),0=>fill(1.0,n))
    return dx
end

"""
function getLaplacian(M::RegularMesh)

builds 2D/3D Laplacian for padded cell-centered vector including
ghost points and corners.

Input:

	M  - computational mesh

Output:

	Lap - 2D/3D Laplacian

"""
function getLaplacianMatrix(M::RegularMesh)
    if M.dim==2
		dx = Mesh.ddx(M.n[1]+1)/M.h[1]
		dy = Mesh.ddx(M.n[2]+1)/M.h[2]

		Dxx = kron(sparse(1.0I,M.n[2]+2, M.n[2]+2),dx'*dx)
		Dyy = kron(dy'*dy, sparse(I, M.n[1]+2, M.n[1]+2))
		return Dxx+Dyy
	elseif M.dim==3
		dx = Mesh.ddx(M.n[1]+1)/M.h[1]
		dy = Mesh.ddx(M.n[2]+1)/M.h[2]
		dz = Mesh.ddx(M.n[3]+1)/M.h[3]

		Dxx = kron(sparse(1.0I, M.n[3]+2, M.n[3]+2),kron(sparse(1.0I, M.n[2]+2, M.n[2]+2),dx'*dx))
		Dyy = kron(sparse(1.0I, M.n[3]+2, M.n[3]+2),kron(dy'*dy, sparse(1.0I, M.n[1]+2, M.n[1]+2)))
		Dzz = kron(dz'*dz, kron(sparse(1.0I, M.n[2]+2, M.n[2]+2), sparse(1.0I, M.n[1]+2, M.n[1]+2)))

		return Dxx + Dyy + Dzz
	end
end

"""
function getConvectionMatrix(M,v)

builds stationary convection operator on extended mesh

2D: C(v_1,v_2)     = Diagonal(v_1)*Dx + Diagonal(v_2)*Dy

3D: C(v_1,v_2,v_3) = Diagonal(v_1)*Dx + Diagonal(v_2)*Dy + Diagonal(v_3)*Dz

Input:

	M :: RegularMesh - original domain (not padded)
	v                - velocities discretized on cell-centers of extended mesh
	 				   or function that returns values (X -> vFun) where
					   X has x,y(,z) coordinates column wise.

Output:

	Conv - 2D Convection operator

"""
function getConvectionMatrix(M::RegularMesh,v::Array{Float64})

	n   = M.n
	h   = M.h
	dim = M.dim
	v  = reshape(v,prod(n.+2),dim)

	if dim==2
    	dxf = ddxfwd(n[1]+2)./(h[1])
    	dxb = ddxbwd(n[1]+2)./(h[1])
    	dyf = ddxfwd(n[2]+2)./(h[2])
    	dyb = ddxbwd(n[2]+2)./(h[2])

    	Dxf = kron(sparse(1.0I, n[2]+2, n[2]+2), dxf)
    	Dxb = kron(sparse(1.0I, n[2]+2, n[2]+2), dxb)
    	Dyf = kron(dyf, sparse(1.0I, n[1]+2, n[1]+2))
    	Dyb = kron(dyb, sparse(1.0I, n[1]+2, n[1]+2))
    	return Diagonal(max.(v[:,1],0))*Dxb + Diagonal(min.(v[:,1],0))*Dxf +
		       Diagonal(max.(v[:,2],0))*Dyb + Diagonal(min.(v[:,2],0))*Dyf
	elseif dim==3
		dxf = ddxfwd(n[1]+2)./(M.h[1])
		dxb = ddxbwd(n[1]+2)./(M.h[1])
		dyf = ddxfwd(n[2]+2)./(M.h[2])
		dyb = ddxbwd(n[2]+2)./(M.h[2])
		dzf = ddxfwd(n[3]+2)./(M.h[3])
		dzb = ddxbwd(n[3]+2)./(M.h[3])

		Dxf = kron(sparse(1.0I, n[3]+2, n[3]+2),kron(sparse(1.0I, n[2]+2, n[2]+2), dxf));
		Dxb = kron(sparse(1.0I, n[3]+2, n[3]+2),kron(sparse(1.0I, n[2]+2, n[2]+2), dxb));
		Dyf = kron(sparse(1.0I, n[3]+2, n[3]+2),kron(dyf, sparse(1.0I, n[1]+2, n[1]+2)))
		Dyb = kron(sparse(1.0I, n[3]+2, n[3]+2),kron(dyb, sparse(1.0I, n[1]+2, n[1]+2)))
		Dzf = kron(dzf, sparse(1.0I, prod(n[1:2].+2), prod(n[1:2].+2)))
		Dzb = kron(dzb, sparse(1.0I, prod(n[1:2].+2), prod(n[1:2].+2)))
    	return Diagonal(max.(v[:,1],0))*Dxb + Diagonal(min.(v[:,1],0))*Dxf +
               Diagonal(max.(v[:,2],0))*Dyb + Diagonal(min.(v[:,2],0))*Dyf +
               Diagonal(max.(v[:,3],0))*Dzb + Diagonal(min.(v[:,3],0))*Dzf
	end
end

function getConvectionMatrix(M::RegularMesh,v::Function)
	xc = getNodalGrid(getPaddedMesh(M))
	return getConvectionMatrix(M,v(xc))
end


using Penopt
using LinearAlgebra: norm
using Test

f(x) = norm(x).^2
c(x) = x - 20 * ones(length(x))
@testset "Penopt.jl" begin
    xopt,fxopt = penopt(f,c,ones(2),[0 1 2 4 8 16 32 64])
    @test  xopt ≈ [0,0] atol = 0.0001
    @test fxopt ≈ 0.0   atol = 0.001
end


using ChaoticEncryption
using TestImages
using Test

@testset "ChaoticEncryption.jl" begin

    @testset "Logistic Map" begin
        @test typeof(logistic_key(0.01, 3.97, 20)) == Array{Int64, 1}
        @test length(logistic_key(0.01, 3.97, 20)) == 20
    end

    @testset "Lorenz System of Differential Equations" begin
        @test typeof(lorenz_key(0.01, 0.02, 0.03, 20)) == Tuple{Array{Int64, 1}, Array{Int64, 1}, Array{Int64, 1}}
        @test length(lorenz_key(0.01, 0.02, 0.03, 20)) == 3
        @test length(lorenz_key(0.01, 0.02, 0.03, 20)[1]) == 20
        @test length(lorenz_key(0.01, 0.02, 0.03, 20)[2]) == 20
        @test length(lorenz_key(0.01, 0.02, 0.03, 20)[3]) == 20
    end

    @testset "Substitution Encryption" begin
        img = testimage("mandrill")
        height, width = size(img)
        keys = logistic_key(0.01, 3.97, height * width)
        substitution_encryption(img, keys; path_for_result="../test_images/encrypted.png")
        @test isfile("../test_images/encrypted.png")

        keys = logistic_key(0.01, 3.97, 20)
        @test_throws ArgumentError("Number of keys must be equal to height * width of image.") substitution_encryption(img, keys; path_for_result="../test_images/encrypted.png")
    end

    @testset "Substitution Decryption" begin
        img = testimage("mandrill")
        height, width = size(img)
        keys = logistic_key(0.01, 3.97, height * width)

        substitution_decryption("../test_images/encrypted.png", keys; path_for_result="../test_images/decrypted.png")
        @test isfile("../test_images/decrypted.png")

        substitution_decryption(img, keys; path_for_result="../test_images/decrypted.png")
        @test isfile("../test_images/decrypted.png")

        @test_throws ArgumentError("image must be of the type ::String or ::Array{RGB{N0f8},2}") substitution_decryption(5, keys; path_for_result="../test_images/decrypted.png")

        keys = logistic_key(0.01, 3.97, 20)
        @test_throws ArgumentError("Number of keys must be equal to height * width of image.") substitution_decryption(img, keys; path_for_result="../test_images/decrypted.png")

        rm("../test_images", recursive=true)
    end
end


include("../functions/Functions.jl")
include("../evaluate/Evaluate.jl")
include("../opt/GradientDescent.jl")
include("../data/data.jl")

X, y = getOldData()

lambda = 0.00001
iter = 1000

x0 = zeros(size(X,2),1)

f = f_logreg(X,y,x0,lambda)
g = g_logreg(X,y,x0,lambda)

#f = f_svm(X,y,x0,lambda)
#g = sub_g_svm()

@time (theta, status,vals1,stops1 ) = qn(x0,f,g,1e-12,iter,20,20,1e-4,.9,"bt","gd")

grad = g(theta)
grad_norm = norm(grad, Inf)
@printf("Inf-Norm of gradient is %f.\n",grad_norm)
println(evaluate(X,y,theta,"logreg",false))


using SellmeierFit
using SimpleConstants  # for micro = 1e-6
using CairoMakie

## Main section
# Load a set of measured refractive index data from https://RefractiveIndex.info.
path = "https://refractiveindex.info/tmp/data/main/SiO2/Malitson.csv"
(; λ, ε) = SellmeierFit.read(path)

# Fit the loaded data to the Sellmeier equation.
(; mdl, err) = fit_sellmeier(λ, ε)

## Analysis section
# The calculated fit parameters are the same as those shown in RefractiveIndex.info's
# Dispersion formula section.
println("B = $(mdl.str)")
println("√C (µm) = $(mdl.λres ./ micro)")
println("Error between data and fit = $err")

# Visualize the measured data and fit Sellmeier equation; note that n = √ε.
fontsize = 20
fig = Figure(; fontsize)
Axis(fig[1,1], title="Refractive Index of SiO₂", xlabel="λ (meter)", ylabel="n")
scatter!(λ, .√ε, label="Measurement", color=:black)
lines!(λ, .√mdl.(λ), label="Sellmeier equation", color=:red)
axislegend(; position=:rt)
display(fig)


using Distributions, Plots; pyplot()

alpha, beta = 8, 2
prior(lam) = pdf(Gamma(alpha, 1/beta), lam)
data = [2,1,0,0,1,0,2,2,5,2,4,0,3,2,5,0]

like(lam) = *([pdf(Poisson(lam),x) for x in data]...)
posteriorUpToK(lam) = like(lam)*prior(lam)

sig = 0.5
foldedNormalPDF(x,mu) = (1/sqrt(2*pi*sig^2))*(exp(-(x-mu)^2/2sig^2)
                                                + exp(-(x+mu)^2/2sig^2))
foldedNormalRV(mu) = abs(rand(Normal(mu,sig)))

function sampler(piProb,qProp,rvProp)
    lam = 1
    warmN, N = 10^5, 10^6
    samples = zeros(N-warmN)

    for t in 1:N
        while true
            lamTry = rvProp(lam)
            L = piProb(lamTry)/piProb(lam)
            H = min(1,L*qProp(lam,lamTry)/qProp(lamTry,lam))
            if rand() < H
                lam = lamTry
                if t > warmN
                    samples[t-warmN] = lam
                end
                break
            end
        end
    end
    return samples
end

mcmcSamples = sampler(posteriorUpToK,foldedNormalPDF,foldedNormalRV)
println("MCMC Bayes Estimate: ",mean(mcmcSamples))

stephist(mcmcSamples, bins=100,
	c=:black, normed=true, label="Histogram of MCMC samples")

lamRange = 0:0.01:10
plot!(lamRange, prior.(lamRange),
	c=:blue, label="Prior distribution")

closedFormPosterior(lam)=pdf(Gamma(alpha + sum(data),1/(beta+length(data))),lam)
plot!(lamRange, closedFormPosterior.(lamRange),
	c=:red, label="Posterior distribution",
	xlims=(0, 10), ylims=(0, 1.2),
    xlabel=L"\lambda",ylabel="Density")


@testset "TSP" begin

using STMO.TSP

X = [1 1; 1 3; 2 2; 2 1]

tsp = TravelingSalesmanProblem(X)

@test tsp isa TravelingSalesmanProblem{Int}

tour = [4, 1, 2, 3]

n = length(tour)

@test n == 4

@test Set(cities(tsp)) == Set(tour)

@test isvalid(tsp, tour)
@test !isvalid(tsp, [1, 2, 3])
@test !isvalid(tsp, [1, 2, 3, 4, 5])
@test !isvalid(tsp, [3, 1, 2, 3])

@test dist(tsp, 2, 3) ≈ sqrt(2)
@test computecost(tsp, tour) ≈ 2 + sqrt(2) + 1 + 1

@testset "swapping" begin
    cost = computecost(tsp, tour)

    for i in 1:n
        for j in 1:n
            Δc = deltaswapcost(tsp, tour, i, j)
            swap!(tour, i, j)
            @test computecost(tsp, tour) ≈ cost + Δc
            swap!(tour, i, j)
        end
    end
end

@testset "flipping" begin
    cost = computecost(tsp, tour)

    for i in 1:n
        for j in 1:n
            Δc = deltaflipcost(tsp, tour, i, j)
            flip!(tour, i, j)
            @test computecost(tsp, tour) ≈ cost + Δc
            flip!(tour, i, j)
        end
    end
end

end


# Testing:
#
#  - computation of sufficient statistics
#  - distribution fitting (i.e. estimation)
#

using Distributions
using Test, Random, LinearAlgebra

n0 = 100
N = 10^5

w = rand(n0)

# DiscreteUniform

x = rand(DiscreteUniform(10, 15), n0)
d = fit(DiscreteUniform, x)
@test isa(d, DiscreteUniform)
@test minimum(d) == minimum(x)
@test maximum(d) == maximum(x)

d = fit(DiscreteUniform, rand(DiscreteUniform(10, 15), N))
@test minimum(d) == 10
@test maximum(d) == 15

# Bernoulli

x = rand(Bernoulli(0.7), n0)

ss = suffstats(Bernoulli, x)
@test isa(ss, Distributions.BernoulliStats)
@test ss.cnt0 == n0 - count(t->t != 0, x)
@test ss.cnt1 == count(t->t != 0, x)

ss = suffstats(Bernoulli, x, w)
@test isa(ss, Distributions.BernoulliStats)
@test ss.cnt0 ≈ sum(w[x .== 0])
@test ss.cnt1 ≈ sum(w[x .== 1])

d = fit(Bernoulli, x)
p = count(t->t != 0, x) / n0
@test isa(d, Bernoulli)
@test mean(d) ≈ p

d = fit(Bernoulli, x, w)
p = sum(w[x .== 1]) / sum(w)
@test isa(d, Bernoulli)
@test mean(d) ≈ p

d = fit(Bernoulli, rand(Bernoulli(0.7), N))
@test isa(d, Bernoulli)
@test isapprox(mean(d), 0.7, atol=0.01)

# Beta

d = fit(Beta, rand(Beta(1.3, 3.7), N))
@test isa(d, Beta)
@test isapprox(d.α, 1.3, atol=0.1)
@test isapprox(d.β, 3.7, atol=0.1)

# Binomial

x = rand(Binomial(100, 0.3), n0)

ss = suffstats(Binomial, (100, x))
@test isa(ss, Distributions.BinomialStats)
@test ss.ns ≈ sum(x)
@test ss.ne == n0
@test ss.n == 100

ss = suffstats(Binomial, (100, x), w)
@test isa(ss, Distributions.BinomialStats)
@test ss.ns ≈ dot(Float64[xx for xx in x], w)
@test ss.ne ≈ sum(w)
@test ss.n == 100

d = fit(Binomial, (100, x))
@test isa(d, Binomial)
@test ntrials(d) == 100
@test succprob(d) ≈ sum(x) / (n0 * 100)

d = fit(Binomial, (100, x), w)
@test isa(d, Binomial)
@test ntrials(d) == 100
@test succprob(d) ≈ dot(x, w) / (sum(w) * 100)

d = fit(Binomial, 100, rand(Binomial(100, 0.3), N))
@test isa(d, Binomial)
@test ntrials(d) == 100
@test isapprox(succprob(d), 0.3, atol=0.01)

# Categorical

p = [0.2, 0.5, 0.3]
x = rand(Categorical(p), n0)

ss = suffstats(Categorical, (3, x))
h = Float64[count(v->v == i, x) for i = 1 : 3]
@test isa(ss, Distributions.CategoricalStats)
@test ss.h ≈ h

d = fit(Categorical, (3, x))
@test isa(d, Categorical)
@test d.K == 3
@test probs(d) ≈ h / sum(h)

d2 = fit(Categorical, x)
@test isa(d2, Categorical)
@test probs(d2) == probs(d)

ss = suffstats(Categorical, (3, x), w)
h = Float64[sum(w[x .== i]) for i = 1 : 3]
@test isa(ss, Distributions.CategoricalStats)
@test ss.h ≈ h

d = fit(Categorical, (3, x), w)
@test isa(d, Categorical)
@test probs(d) ≈ h / sum(h)

d = fit(Categorical, suffstats(Categorical, 3, x, w))
@test isa(d, Categorical)
@test probs(d) ≈ (h / sum(h))

d = fit(Categorical, rand(Categorical(p), N))
@test isa(d, Categorical)
@test isapprox(probs(d), p, atol=0.01)

# Cauchy

@test fit(Cauchy, collect(-4.0:4.0)) === Cauchy(0.0, 2.0)

# Exponential

x = rand(Exponential(0.5), n0)

ss = suffstats(Exponential, x)
@test isa(ss, Distributions.ExponentialStats)
@test ss.sx ≈ sum(x)
@test ss.sw == n0

ss = suffstats(Exponential, x, w)
@test isa(ss, Distributions.ExponentialStats)
@test ss.sx ≈ dot(x, w)
@test ss.sw == sum(w)

d = fit(Exponential, x)
@test isa(d, Exponential)
@test scale(d) ≈ mean(x)

d = fit(Exponential, x, w)
@test isa(d, Exponential)
@test scale(d) ≈ dot(x, w) / sum(w)

d = fit(Exponential, rand(Exponential(0.5), N))
@test isa(d, Exponential)
@test isapprox(scale(d), 0.5, atol=0.01)

# Normal

μ = 11.3
σ = 3.2

x = rand(Normal(μ, σ), n0)

ss = suffstats(Normal, x)
@test isa(ss, Distributions.NormalStats)
@test ss.s  ≈ sum(x)
@test ss.m  ≈ mean(x)
@test ss.s2 ≈ sum((x .- ss.m).^2)
@test ss.tw ≈ n0

ss = suffstats(Normal, x, w)
@test isa(ss, Distributions.NormalStats)
@test ss.s  ≈ dot(x, w)
@test ss.m  ≈ dot(x, w) / sum(w)
@test ss.s2 ≈ dot((x .- ss.m).^2, w)
@test ss.tw ≈ sum(w)

d = fit(Normal, x)
@test isa(d, Normal)
@test d.μ ≈ mean(x)
@test d.σ ≈ sqrt(mean((x .- d.μ).^2))

d = fit(Normal, x, w)
@test isa(d, Normal)
@test d.μ ≈ dot(x, w) / sum(w)
@test d.σ ≈ sqrt(dot((x .- d.μ).^2, w) / sum(w))

d = fit(Normal, rand(Normal(μ, σ), N))
@test isa(d, Normal)
@test isapprox(d.μ, μ, atol=0.1)
@test isapprox(d.σ, σ, atol=0.1)

import Distributions.NormalKnownMu, Distributions.NormalKnownSigma

ss = suffstats(NormalKnownMu(μ), x)
@test isa(ss, Distributions.NormalKnownMuStats)
@test ss.μ == μ
@test ss.s2 ≈ sum(abs2.(x .- μ))
@test ss.tw ≈ n0

ss = suffstats(NormalKnownMu(μ), x, w)
@test isa(ss, Distributions.NormalKnownMuStats)
@test ss.μ == μ
@test ss.s2 ≈ dot((x .- μ).^2, w)
@test ss.tw ≈ sum(w)

d = fit_mle(Normal, x; mu=μ)
@test isa(d, Normal)
@test d.μ == μ
@test d.σ ≈ sqrt(mean((x .- d.μ).^2))

d = fit_mle(Normal, x, w; mu=μ)
@test isa(d, Normal)
@test d.μ == μ
@test d.σ ≈ sqrt(dot((x .- d.μ).^2, w) / sum(w))

ss = suffstats(NormalKnownSigma(σ), x)
@test isa(ss, Distributions.NormalKnownSigmaStats)
@test ss.σ == σ
@test ss.sx ≈ sum(x)
@test ss.tw ≈ n0

ss = suffstats(NormalKnownSigma(σ), x, w)
@test isa(ss, Distributions.NormalKnownSigmaStats)
@test ss.σ == σ
@test ss.sx ≈ dot(x, w)
@test ss.tw ≈ sum(w)

d = fit_mle(Normal, x; sigma=σ)
@test isa(d, Normal)
@test d.σ == σ
@test d.μ ≈ mean(x)

d = fit_mle(Normal, x, w; sigma=σ)
@test isa(d, Normal)
@test d.σ == σ
@test d.μ ≈ dot(x, w) / sum(w)

# Uniform

x = rand(Uniform(1.2, 5.8), n0)
d = fit(Uniform, x)
@test isa(d, Uniform)
@test 1.2 <= minimum(d) <= maximum(d) <= 5.8
@test minimum(d) == minimum(x)
@test maximum(d) == maximum(x)

d = fit(Uniform, rand(Uniform(1.2, 5.8), N))
@test 1.2 <= minimum(d) <= maximum(d) <= 5.8
@test isapprox(minimum(d), 1.2, atol=0.02)
@test isapprox(maximum(d), 5.8, atol=0.02)

# Gamma

x = rand(Gamma(3.9, 2.1), n0)

ss = suffstats(Gamma, x)
@test isa(ss, Distributions.GammaStats)
@test ss.sx    ≈ sum(x)
@test ss.slogx ≈ sum(log.(x))
@test ss.tw    ≈ n0

ss = suffstats(Gamma, x, w)
@test isa(ss, Distributions.GammaStats)
@test ss.sx    ≈ dot(x, w)
@test ss.slogx ≈ dot(log.(x), w)
@test ss.tw    ≈ sum(w)

d = fit(Gamma, rand(Gamma(3.9, 2.1), N))
@test isa(d, Gamma)
@test isapprox(shape(d), 3.9, atol=0.1)
@test isapprox(scale(d), 2.1, atol=0.2)

# Geometric

x = rand(Geometric(0.3), n0)

ss = suffstats(Geometric, x)
@test isa(ss, Distributions.GeometricStats)
@test ss.sx ≈ sum(x)
@test ss.tw ≈ n0

ss = suffstats(Geometric, x, w)
@test isa(ss, Distributions.GeometricStats)
@test ss.sx ≈ dot(x, w)
@test ss.tw ≈ sum(w)

d = fit(Geometric, x)
@test isa(d, Geometric)
@test succprob(d) ≈ inv(1. + mean(x))

d = fit(Geometric, x, w)
@test isa(d, Geometric)
@test succprob(d) ≈ inv(1. + dot(x, w) / sum(w))

d = fit(Geometric, rand(Geometric(0.3), N))
@test isa(d, Geometric)
@test isapprox(succprob(d), 0.3, atol=0.01)

# Laplace

d = fit(Laplace, rand(Laplace(5.0, 3.0), N))
@test isa(d, Laplace)
@test isapprox(location(d), 5.0, atol=0.1)
@test isapprox(scale(d)   , 3.0, atol=0.2)

# Pareto

x = rand(Pareto(3., 7.), N)
d = fit(Pareto, x)

@test isa(d, Pareto)
@test isapprox(shape(d), 3., atol=0.1)
@test isapprox(scale(d), 7., atol=0.1)

# Poisson

x = rand(Poisson(8.2), n0)

ss = suffstats(Poisson, x)
@test isa(ss, Distributions.PoissonStats)
@test ss.sx ≈ sum(x)
@test ss.tw ≈ n0

ss = suffstats(Poisson, x, w)
@test isa(ss, Distributions.PoissonStats)
@test ss.sx ≈ dot(x, w)
@test ss.tw ≈ sum(w)

d = fit(Poisson, x)
@test isa(d, Poisson)
@test mean(d) ≈ mean(x)

d = fit(Poisson, x, w)
@test isa(d, Poisson)
@test mean(d) ≈ dot(Float64[xx for xx in x], w) / sum(w)

d = fit(Poisson, rand(Poisson(8.2), N))
@test isa(d, Poisson)
@test isapprox(mean(d), 8.2, atol=0.2)


using Plots
using LinearAlgebra

x = 3 # True slope
a = -2:.25:2
b = a*x + randn(length(a)) # Add noise

p1 = plot(a, x*a, label = "True line") # True relationship
scatter!(a, b, label = "Noisy data") # Noisy measurements

U, S, V = svd(reshape(collect(a), length(a), 1))
xtilde = (V*inv(diagm(S))*U'*b)[1] # Least-square fit

plot!(a, xtilde*a, label = "Regression line") # Plot fit


function getergodic(transmatrix::Array{Float64,2})
    D,V = eig(transmatrix')
    eigindex = indmin(abs.(D-1.0))
    ergodic = vec(V[:,eigindex]/sum(V[:,eigindex]))
    if any(abs.(imag(ergodic)).>1e-16)
        error("Imaginary ergodic distribution")
    else
        return real(ergodic)
    end
end

function createtransmatrix_sym(numregimes::Int)
    out = Array{SymPy.Sym}(numregimes)
    for j = 1:numregimes
        out[j] = Sym("TRANSITIONPROBABILITY_RX_RP$(j)")
    end
    return out
end

function TransitionMatrix(trans)
    # Sanity check the regime transition matrix
    numregimes = size(trans,1)
    if size(trans,2)!=size(trans,1)
        error("Regime transition matrix is not square.")
    end
    if any((abs.(sum(trans,2))-1.0).>1e-12)
        error("Rows of the regime transition matrix do not sum to 1.")
    end
    # Create a sympy version of the regime trans matrix
    syms = createtransmatrix_sym(numregimes)

    # Get the ergodic distribution of the markovian regime process
    ergodic = getergodic(trans)
    return TransitionMatrix(trans,syms,ergodic,Sym("TRANSITIONPROBABILITY"))
end


@doc """
       Newton method for local optimization (multivariable)

       Supported by:

       Ph D Kelvyn B. Sánchez
       e-mail: kelvyn.baruc@gmail.com
       GitHub: Kelvyn88
 """ ->

function Newton(f, vars, x0; error = 0.0001, maxiter = 100, iter = true)
      try
            const version = Pkg.installed("OptimJulia")
            length(vars) == 0 ? len = 1 : len = length(vars)

            length(vars) == 0 ? sep_var = 1 : sep_var = length(vars)
            const separator = "="^(9 + (sep_var + 2)*16)
            err = Inf
            count = 1
            x_num = 0
            f_num = f

            # Numeric iterations
            iter == true ? ti = time_ns() : nothing # To measure algorithm performance

            while err > error
                  x_num = x0

                  # Symbolic derivatives using Module SymPy
                  if length(vars) == 0
                        # Single variable
                        dfk = diff(f)
                  else
                        # Multivariable variable
                        dfk = Array(SymPy.Sym, 1, length(vars))
                        for i=1:length(vars)
                              dfk[i] = diff(f, vars[i])
                        end
                  end

                  f_num = f

                  # Symbolic hessian using module SymPy
                  if length(vars) == 0
                        # Single variable
                        dfk2 = diff(dfk)
                  else
                        # Multivariable variable
                        hess = hessian(f, vars)
                  end

                  # Numeric evaluations
                  if length(vars) == 0
                        # Single variable
                        dfk = dfk |> subs(vars, x_num)
                        dfk2 = dfk2 |> subs(vars,x_num)

                        # Conversion from BigInt to Float32 (avoid overflow)
                        dfk = convert(Float32, dfk)
                        dfk2 = convert(Float32, dfk2)
                  else
                        # Multivariable variable
                        for i=1:length(vars), j=1:length(vars), k=1:length(vars)
                              dfk[i] = dfk[i] |> subs(vars[k], x_num[k])
                              hess[i,j] = hess[i,j] |> subs(vars[k],x_num[k])
                        end

                        # Conversion from BigInt to Float32 (avoid overflow)
                        dfk = convert(Array{Float32,length(vars)}, dfk)
                        hess = convert(Array{Float32,length(vars)}, hess)
                  end

                  # Newton equation
                  if length(vars) == 0
                        # Single variable
                        xk = x_num - dfk/dfk2
                  else
                        # Multivariable variable
                        xk = x_num - inv(hess) * transpose(dfk)
                  end

                  err = sum(abs((xk -  x_num) ./ xk))

                  # Numeric evaluation of original function
                  for k=1:len
                        f_num = f_num |> subs(vars[k], N(xk[k]))
                  end

                  f_num = convert(Float32, f_num)

                  # Print iteration for the case of iter == true
                  if iter == true
                        if count == 1
                              # Header message and version
                              println(separator)
                              msg1 = @sprintf(" Newton Solver for Unconstrained Optimization %s ", version)
                              msg2 = @sprintf(" Ph D Kelvyn B. Sánchez ")
                              header_left = ceil((length(separator) - length(msg1))/2)
                              header_right = (length(separator) - length(msg1)) - header_left
                              msg1_left = ":"^convert(Int32,header_left)
                              msg1_right = ":"^convert(Int32,header_right)
                              msg1 = string(msg1_left, msg1, msg1_right)
                              println(msg1)

                              header_left = ceil((length(separator) - length(msg2))/2)
                              header_right = (length(separator) - length(msg2)) - header_left
                              msg2_left = ":"^convert(Int32,header_left)
                              msg2_right = ":"^convert(Int32,header_right)
                              msg2 = string(msg2_left, msg2, msg2_right)
                              println(msg2)
                        end

                        if count == 1 || ceil((count/1)/10) == (count/1)/10
                              println(separator)

                              # Header
                              msg1 = "Iter\t "

                              for i=1:len
                                    msg1 = string(msg1, vars[i], "\t\t ")
                              end

                              msg1 = string(msg1, "f(x)\t\t ","||err||\t\t ")
                              println(msg1)
                              println(separator)
                        end

                        # Custom message for iterator
                        msg2 = @sprintf("%i\t ", count)
                        for i=1:len
                              msg2 = string(msg2, @sprintf("%4.6e\t ", xk[i]))
                        end

                        msg2 = string(msg2, @sprintf("%4.6e\t ", f_num), @sprintf("%4.6e", err))
                        println(msg2)
                  end

                  # Backup of new data for next iteration
                  x0 = xk
                  x_num = xk
                  count += 1

                  if count == maxiter
                        throw(ErrorException("Not Solved!. Iteration limit reached."))
                  end
            end

            iter == true ? println(separator) : nothing
            iter == true ? tf = time_ns() : nothing # To measure algorithm performance
            iter == true ? println("elapse time: ", (tf - ti)/1.0e9, " seconds") : nothing

            return x_num,  f_num, err

      catch Msg
            println(separator)
            println(Msg)
            return NaN, NaN, NaN
      end
end


using LocallyCompetitive
using Test
using Random
using Statistics

# Testing params
M = 100
N = 100

k = 10

Random.seed!(0)

# Dictionary
basis = randn(M,N)

# Make k sparse signal
vals  = rand(M)
vals[sortperm(rand(M))[k+1:end]] .= 0

# Input signal with k sparse structure
sig = basis*vals

# Learn 'vals' based upon sig and the basis
out = LocallyCompetitive.lca(sig, basis)

recon = basis*out

sparse_orig = vals .!= 0
sparse_new  = out  .!= 0

sse_orig  = mean(recon.^2)
sse_new   = mean(sig.^2)
sse_diff  = mean((sig.-recon).^2)

# Same sparsity found in output
@test sparse_orig == sparse_new

# Reconstruction models data
@test sse_orig*0.01 > sse_diff
@test sse_new *0.01 > sse_diff


@doc """
	cgnr(A::Matrix, b::Vector, x0::Vector; prec, maxiter)

Conjugate gradient for normal equations residual method for solving the least squares problem `min norm( Ax - b )`.

- `A` is the coefficient matrix (`M-by-N`)
- `b` is the observation vector (`M`).
- `x0` is the starting vector (`N`).

The iteration stops when `norm(xnew - xold) < prec` or after at most `maxiter` iterations.
"""->
function cgnr{T<:Number}(A::AbstractMatrix{T}, b::AbstractVector{T}, x0::AbstractVector{T}; prec=LARGE_EPS, maxiter=length(x0))
	size(A,1) == length(b) || throw(DimensionMismatch())
	size(A,2) == length(x0) || throw(DimensionMismatch())
	prec >= SMALL_EPS || throw(DomainError())
	maxiter >= 1 || throw(DomainError())

	# Initialize
	x = copy(x0)
	y = A*x
	r = b - y
	z = A'*r
	p = copy(z)
	oneT = one(T)

	for iter = 1:maxiter
		ztz = norm(z)^2
		A_mul_B!(y, A, p) # y = A*p
		mu = ztz / norm(y)^2
		BLAS.axpy!(mu, p, x) # x = x + mu*p
		xdiff = mu*norm(p)

		BLAS.axpy!(-mu, y, r) # r = r - mu*y
		Ac_mul_B!(z, A, r) # z = A'*r
		tau = norm(z)^2 / ztz

		# p = z + tau*p
		scale!(p, tau)
		BLAS.axpy!(oneT, z, p)

		# Check for convergence: |xnew - xold|
		if xdiff < prec
			break
		end
	end

	return x
end

@doc """
	cgnr(T::Freq2wave, b, x0; ...) -> x

Conjugate gradient for normal equations residual method for `Freq2Wave`.
The initial point `x0` must be of the same dimension as `T`.
"""->
function cgnr(T::Freq2Wave, b::AbstractVector{Complex{Float64}}, x0::AbstractVecOrMat{Complex{Float64}}=zeros(eltype(T), wsize(T)); prec=LARGE_EPS, maxiter=max(length(x0),50))
	size(T,1) == length(b) || throw(DimensionMismatch())
	wsize(T) == size(x0) || throw(DimensionMismatch())
	prec >= SMALL_EPS || throw(DomainError())
	maxiter >= 1 || throw(DomainError())

	# Initialize
	x = copy(x0)
	y = T*x
	r = b - y
	z = T'*r
	p = copy(z)
	Cone = one(Complex{Float64})

	for iter = 1:maxiter
		ztz = vecnorm(z)^2
		A_mul_B!(y, T, p) # y = T*p
		mu = ztz / vecnorm(y)^2
		BLAS.axpy!(mu, p, x) # x = x + mu*p
		xdiff = mu*norm(p)

		BLAS.axpy!(-mu, y, r) # r = r - mu*y
		Ac_mul_B!(z, T, r) # z = T'*r
		tau = vecnorm(z)^2 / ztz

		# p = z + tau*p
		scale!(p, tau)
		BLAS.axpy!(Cone, z, p)

		# Check for convergence: |xnew - xold|
		if xdiff < prec
			break
		end
	end

	return x
end


using Mimi

@defcomp grosseconomy begin
    YGROSS  = Variable(index=[time])    # Gross output
    K       = Variable(index=[time])    # Capital
    l       = Parameter(index=[time])   # Labor
    tfp     = Parameter(index=[time])   # Total factor productivity
    s       = Parameter(index=[time])   # Savings rate
    depk    = Parameter()               # Depreciation rate on capital - Note that it has no time index
    k0      = Parameter()               # Initial level of capital
    share   = Parameter()               # Capital share

    function run_timestep(p, v, d, t)
        # Define an equation for K
        if is_first(t)
            v.K[t]  = p.k0  # Note the use of v. and p. to distinguish between variables and parameters
        else
            v.K[t]  = (1 - p.depk)^5 * v.K[t-1] + v.YGROSS[t-1] * p.s[t-1] * 5
        end

        # Define an equation for YGROSS
        v.YGROSS[t] = p.tfp[t] * v.K[t]^p.share * p.l[t]^(1-p.share)
    end
end

@defcomp emissions begin
    E       = Variable(index=[time])    # Total greenhouse gas emissions
    sigma   = Parameter(index=[time])   # Emissions output ratio
    YGROSS  = Parameter(index=[time])   # Gross output - Note that YGROSS is now a parameter

    function run_timestep(p, v, d, t)
        # Define an eqation for E
        v.E[t] = p.YGROSS[t] * p.sigma[t]   # Note the p. in front of YGROSS
    end
end

function construct_model()
	m = Model()

	set_dimension!(m, :time, collect(2015:5:2110))

	# Order matters here. If the emissions component were defined first, the model would not run.
	add_comp!(m, grosseconomy)
	add_comp!(m, emissions)

	# Update parameters for the grosseconomy component
	update_param!(m, :grosseconomy, :l, [(1. + 0.015)^t *6404 for t in 1:20])
	update_param!(m, :grosseconomy, :tfp, [(1 + 0.065)^t * 3.57 for t in 1:20])
	update_param!(m, :grosseconomy, :s, ones(20).* 0.22)
	update_param!(m, :grosseconomy, :depk, 0.1)
	update_param!(m, :grosseconomy, :k0, 130.)
	update_param!(m, :grosseconomy, :share, 0.3)

	# Update and connect parameters for the emissions component
	update_param!(m, :emissions, :sigma, [(1. - 0.05)^t *0.58 for t in 1:20])
	connect_param!(m, :emissions, :YGROSS, :grosseconomy, :YGROSS)

	return m

end #end function

model = construct_model()

run(model)

# Show model results
model[:emissions, :E]

# Or, as a DataFrame
getdataframe(model, :emissions, :E)


using LinearAlgebra

abstract type AbstractForecasting end

export StateSpaceModel

"""

Space-State model is defined through the following equations

Let ``{x_0, x_1, \\dots, x_K }`` denote the discrete-time state process,
which is typically unknown to the observer. Let ``{y_1,y_2,... ,y_K}``
represent the observation process, which describes the measurements of the
system that are available to the observer.

These state and observation processes are related following a nonlinear
state-space model: The first equation describes how the system state ``x_k``
evolves through a nonlinear dynamical model ``f_{k−1,k}`` between successive
time steps ``t_{k−1}`` and ``t_k``.
``η = {η1,η2,... ,ηK}`` is the model noise process, which accounts for
the imperfections of the model; it is assumed to be independent and
identically distributed. At each time step, ``η_k`` is assumed
to be Gaussian with zero mean and covariance matrix Q. The second equation
models how an observation ``y_k`` at time ``t_k`` is obtained from the state
``x_k`` through a linear operator ``H``. ``ε = {ε1, ε2, . . . , εk}`` is an
independent and identically distributed process representing the observation
errors; ``ε_k`` is assumed
Gaussian with zero mean and covariance matrix ``R``. Finally, the
processes ``η`` and ``ε`` are assumed to be jointly independent and independent
of the initial (background) state x0, which is assumed to be Gaussian
with mean xb and covariance B.

```math
\\left\\{
\\begin{array}{l}
X_t = m(X_{t-1}) + \\eta_t, \\\\
Y_t = H(X_t) + \\varepsilon_t,
\\end{array}
\\right.
```

- X : hidden variables
- Y : observed variables

- `dt_integration`is the numerical time step used to solve the ODE.
- `dt_states` is the number of `dt_integration` between ``X_{t-1}`` and ``X_t``.
- `dt_obs` is the number of `dt_integration` between ``Y_{t-1}`` and ``Y_t``.

"""
struct StateSpaceModel <: AbstractForecasting

    model::Function
    dt_integration::Float64
    dt_states::Int64
    dt_obs::Int64
    params::Vector{Float64}
    var_obs::Vector{Int64}
    nb_loop_train::Int64
    nb_loop_test::Int64
    sigma2_catalog::Float64
    sigma2_obs::Float64

end


using DomainSets: ×

"Does `z` agree with `val` up to the given `threshold`?"
isaccurate(z, val, threshold = 1e-5) = abs(z-val) < threshold

function test_readme_examples()
    # Examples of README.md
    threshold = 1e-5
    @test isaccurate(integral(cos, 0..1.0), 0.8414709848078965)
    @test isaccurate(integral(x -> exp(x[1]+x[2]), (0..1.0)^2), 2.9524924420120535)
    @test isaccurate(quadrature(cos, UnionDomain(0..1, 2..3))[1], 0.07329356604208204)
    @test isaccurate(integral(t -> sin(log(abs(t))), -1..1, LogPointSingularity(0.0)), -1.0000000021051316)
    @test isaccurate(integral( x -> exp(log(abs(x[1]-x[2]))), (2..3) × (1..4), DiagonalSingularity()),
2.333333333333333)
    @test isaccurate(integral(cos, ChebyshevTMeasure()), 2.403939430634413)
    @test isaccurate(integral(t -> cos(t)*1/sqrt(1-t^2), -1.0..1.0), 2.403939410869398)
    @test isaccurate(quadrature(QuadAdaptive(atol=1e-3, rtol = 1e-3), t->cos(t^2), 0..10)[1], 0.6011251848111901, 1e-3)
    @test isaccurate(integral(Q_GaussLaguerre(10), cos), 0.5000005097999486)
    @test isaccurate(integral(t -> cos(t)*exp(-t), HalfLine()), 0.5)
end

function test_some_integrals()
    z1 = sin(1.0)
    @test abs(integral(cos, 0..1.0) - z1) < 1e-8
    I, E = quadrature(cos, 0..1.0)
    @test abs(I-z1) < 1e-8
    @test E < 1e-8

    f2 = x -> exp(x[1]+x[2])
    z2 = hcubature(f2, (0,0), (1,1))[1]
    @test abs(integral(f2, (0..1)^2) - z2) < 1e-6
end

function test_integrals()
    test_some_integrals()
    test_readme_examples()
end


using FinancialMonteCarlo, Distributions, Test;
@show "Test Parameters"
S0 = 100.0;
r = 0.02;
d = 0.01;
T = 1.0;
Nsim = 100;
Nstep = 30;

sigma = 0.2;

McConfig = MonteCarloConfiguration(Nsim, Nstep);
mc1 = MonteCarloConfiguration(Nsim, Nstep, FinancialMonteCarlo.AntitheticMC());
rfCurve = ZeroRate(r);
rfCurve2 = ZeroRate([0.00, 0.02], T);

@show "Test Subordinated Brownian Motion Parameters"
drift = 0.0;
@test_throws(ErrorException, SubordinatedBrownianMotion(-sigma, drift, InverseGaussian(1.0, 1.0)))
@test_throws(ErrorException, SubordinatedBrownianMotion(-sigma, rfCurve2.r, InverseGaussian(1.0, 1.0)))


function fahrenheit_to_celsius(temp_f)
    temp_c = (temp_f - 32.0) * (5.0/9.0)
    return temp_c
end

temp_c = fahrenheit_to_celsius(100.0)
println(temp_c)