@doc """
  Splits a floating-point number `a` into two parts `a = h + l` such that the
  halves `h` and `l` are non-overlapping and `|x| ≥ |y|`. That is, both halves
  have at most `s-1` nonzero bits, where `s = ceil(p/2)` and `p` is the
  precision (`p = 24` for `Float32` and `p = 53` for `Float64`).

  References:

    * T.J. Dekker, A floating-point technique for extending the available
      precision, Numerische Mathematik 18, pp. 224–242, 1971.
  """ ->
# TODO: Extend the definition to floating-point arrays
for T in (Float32, Float64)
  @eval begin
    function fsplit(a::$T)
      # Float32: factor = 2^12 + 1 = 4097
      # Float64: factor = 2^27 + 1 = 134217729
      f = $(convert(T, 2^ceil(precision(T)/2)+1));
      c = a * f
      h = c - (c-a)
      l = a - h
      h, l
    end
  end
end

@doc """
  Computes the error-free multiplication `a * b = x + y`, where `x = fl(a * b)`,
  for floating-point numbers `a` and `b`. This algorithm uses all cross products
  between the halves generated by `fsplit` from `a` and `b`. It requires 17
  flops.

  References:

    * T.J. Dekker, A floating-point technique for extending the available
      precision, Numerische Mathematik 18, pp. 224–242, 1971.
  """ ->
# TODO: Extend the definition to floating-point arrays
function err_mul{T<:IEEE754}(a::T, b::T)
  x = a * b
  ah, al = fsplit(a)
  bh, bl = fsplit(b)
  y = al*bl - (((x - ah*bh) - al*bh) - ah*bl)
  x, y
end

@doc """
  Computes the error-free transformation `a * b = x + y`, where `x = fl(a * b)`,
  for floating-point numbers `a` and `b`. By using the fused multiply-add (FMA)
  operation, it only requires 2 flops instead of the 17 flops required by
  `err_mul` but the improvement in execution time typically a less than that.

  References:

    * T. Ogita, S.M. Rump, and S. Oishi, Accurate sum and dot product, SIAM
      Journal on Scientific Computing 26, pp. 1955–1988, 2005.
  """ ->
# TODO: Extend the definition to floating-point arrays
function err_fast_mul{T<:IEEE754}(a::T, b::T)
  x = a * b
  y = fma(a, b, -x)
  x, y
end


using Turing
using DifferentialEquations

# Load StatsPlots for visualizations and diagnostics.
using StatsPlots

using LinearAlgebra

# Set a seed for reproducibility.
using Random
Random.seed!(14);

# Define Lotka-Volterra model.
function lotka_volterra(du, u, p, t)
    # Model parameters.
    α, β, γ, δ = p
    # Current state.
    x, y = u

    # Evaluate differential equations.
    du[1] = (α - β * y) * x # prey
    du[2] = (δ * x - γ) * y # predator

    return nothing
end

# Define initial-value problem.
u0 = [1.0, 1.0]
p = [1.5, 1.0, 3.0, 1.0]
tspan = (0.0, 10.0)
prob = ODEProblem(lotka_volterra, u0, tspan, p)

# Plot simulation.
plot(solve(prob, Tsit5()))

sol = solve(prob, Tsit5(); saveat=0.1)
odedata = Array(sol) + 0.8 * randn(size(Array(sol)))

# Plot simulation and noisy observations.
plot(sol; alpha=0.3)
scatter!(sol.t, odedata'; color=[1 2], label="")

@model function fitlv(data, prob)
    # Prior distributions.
    σ ~ InverseGamma(2, 3)
    α ~ truncated(Normal(1.5, 0.5), 0.5, 2.5)
    β ~ truncated(Normal(1.2, 0.5), 0, 2)
    γ ~ truncated(Normal(3.0, 0.5), 1, 4)
    δ ~ truncated(Normal(1.0, 0.5), 0, 2)

    # Simulate Lotka-Volterra model.
    p = [α, β, γ, δ]
    predicted = solve(prob, Tsit5(); p=p, saveat=0.1)

    # Observations.
    for i in 1:length(predicted)
        data[:, i] ~ MvNormal(predicted[i], σ^2 * I)
    end

    return nothing
end

model = fitlv(odedata, prob)

# Sample 3 independent chains with forward-mode automatic differentiation (the default).
chain = sample(model, NUTS(0.65), MCMCSerial(), 1000, 3; progress=false)

plot(chain)

plot(; legend=false)
posterior_samples = sample(chain[[:α, :β, :γ, :δ]], 300; replace=false)
for p in eachrow(Array(posterior_samples))
    sol_p = solve(prob, Tsit5(); p=p, saveat=0.1)
    plot!(sol_p; alpha=0.1, color="#BBBBBB")
end

# Plot simulation and noisy observations.
plot!(sol; color=[1 2], linewidth=1)
scatter!(sol.t, odedata'; color=[1 2])

@model function fitlv2(data::AbstractVector, prob)
    # Prior distributions.
    σ ~ InverseGamma(2, 3)
    α ~ truncated(Normal(1.5, 0.5), 0.5, 2.5)
    β ~ truncated(Normal(1.2, 0.5), 0, 2)
    γ ~ truncated(Normal(3.0, 0.5), 1, 4)
    δ ~ truncated(Normal(1.0, 0.5), 0, 2)

    # Simulate Lotka-Volterra model but save only the second state of the system (predators).
    p = [α, β, γ, δ]
    predicted = solve(prob, Tsit5(); p=p, saveat=0.1, save_idxs=2)

    # Observations of the predators.
    data ~ MvNormal(predicted.u, σ^2 * I)

    return nothing
end

model2 = fitlv2(odedata[2, :], prob)

# Sample 3 independent chains.
chain2 = sample(model2, NUTS(0.45), MCMCSerial(), 5000, 3; progress=false)

plot(; legend=false)
posterior_samples = sample(chain2[[:α, :β, :γ, :δ]], 300; replace=false)
for p in eachrow(Array(posterior_samples))
    sol_p = solve(prob, Tsit5(); p=p, saveat=0.1)
    plot!(sol_p; alpha=0.1, color="#BBBBBB")
end

# Plot simulation and noisy observations.
plot!(sol; color=[1 2], linewidth=1)
scatter!(sol.t, odedata'; color=[1 2])

function delay_lotka_volterra(du, u, h, p, t)
    # Model parameters.
    α, β, γ, δ = p

    # Current state.
    x, y = u
    # Evaluate differential equations
    du[1] = α * h(p, t - 1; idxs=1) - β * x * y
    du[2] = -γ * y + δ * x * y

    return nothing
end

# Define initial-value problem.
p = (1.5, 1.0, 3.0, 1.0)
u0 = [1.0; 1.0]
tspan = (0.0, 10.0)
h(p, t; idxs::Int) = 1.0
prob_dde = DDEProblem(delay_lotka_volterra, u0, h, tspan, p);

sol_dde = solve(prob_dde; saveat=0.1)
ddedata = Array(sol_dde) + 0.5 * randn(size(sol_dde))

# Plot simulation and noisy observations.
plot(sol_dde)
scatter!(sol_dde.t, ddedata'; color=[1 2], label="")

@model function fitlv_dde(data, prob)
    # Prior distributions.
    σ ~ InverseGamma(2, 3)
    α ~ Truncated(Normal(1.5, 0.5), 0.5, 2.5)
    β ~ Truncated(Normal(1.2, 0.5), 0, 2)
    γ ~ Truncated(Normal(3.0, 0.5), 1, 4)
    δ ~ Truncated(Normal(1.0, 0.5), 0, 2)

    # Simulate Lotka-Volterra model.
    p = [α, β, γ, δ]
    predicted = solve(prob, MethodOfSteps(Tsit5()); p=p, saveat=0.1)

    # Observations.
    for i in 1:length(predicted)
        data[:, i] ~ MvNormal(predicted[i], σ^2 * I)
    end
end

model_dde = fitlv_dde(ddedata, prob_dde)

# Sample 3 independent chains.
chain_dde = sample(model_dde, NUTS(0.65), MCMCSerial(), 300, 3; progress=false)

plot(chain_dde)

plot(; legend=false)
posterior_samples = sample(chain_dde[[:α, :β, :γ, :δ]], 300; replace=false)
for p in eachrow(Array(posterior_samples))
    sol_p = solve(prob_dde, MethodOfSteps(Tsit5()); p=p, saveat=0.1)
    plot!(sol_p; alpha=0.1, color="#BBBBBB")
end

# Plot simulation and noisy observations.
plot!(sol_dde; color=[1 2], linewidth=1)
scatter!(sol_dde.t, ddedata'; color=[1 2])

using Zygote, DiffEqSensitivity

# Sample a single chain with 1000 samples using Zygote.
setadbackend(:zygote)
sample(model, NUTS(0.65), 1000; progress=false)

@model function fitlv_sensealg(data, prob)
    # Prior distributions.
    σ ~ InverseGamma(2, 3)
    α ~ truncated(Normal(1.5, 0.5), 0.5, 2.5)
    β ~ truncated(Normal(1.2, 0.5), 0, 2)
    γ ~ truncated(Normal(3.0, 0.5), 1, 4)
    δ ~ truncated(Normal(1.0, 0.5), 0, 2)

    # Simulate Lotka-Volterra model and use a specific algorithm for computing sensitivities.
    p = [α, β, γ, δ]
    predicted = solve(
        prob;
        p=p,
        saveat=0.1,
        sensealg=InterpolatingAdjoint(; autojacvec=ReverseDiffVJP(true)),
    )

    # Observations.
    for i in 1:length(predicted)
        data[:, i] ~ MvNormal(predicted[i], σ^2 * I)
    end

    return nothing
end;

model_sensealg = fitlv_sensealg(odedata, prob)

# Sample a single chain with 1000 samples using Zygote.
setadbackend(:zygote)
sample(model_sensealg, NUTS(0.65), 1000; progress=false)

u0 = [1.0, 1.0]
tspan = (0.0, 10.0)
function multiplicative_noise!(du, u, p, t)
    x, y = u
    du[1] = p[5] * x
    return du[2] = p[6] * y
end
p = [1.5, 1.0, 3.0, 1.0, 0.1, 0.1]

function lotka_volterra!(du, u, p, t)
    x, y = u
    α, β, γ, δ = p
    du[1] = dx = α * x - β * x * y
    return du[2] = dy = δ * x * y - γ * y
end

prob_sde = SDEProblem(lotka_volterra!, multiplicative_noise!, u0, tspan, p)

ensembleprob = EnsembleProblem(prob_sde)
data = solve(ensembleprob, SOSRI(); saveat=0.1, trajectories=1000)
plot(EnsembleSummary(data))

@model function fitlv_sde(data, prob)
    # Prior distributions.
    σ ~ InverseGamma(2, 3)
    α ~ truncated(Normal(1.3, 0.5), 0.5, 2.5)
    β ~ truncated(Normal(1.2, 0.25), 0.5, 2)
    γ ~ truncated(Normal(3.2, 0.25), 2.2, 4.0)
    δ ~ truncated(Normal(1.2, 0.25), 0.5, 2.0)
    ϕ1 ~ truncated(Normal(0.12, 0.3), 0.05, 0.25)
    ϕ2 ~ truncated(Normal(0.12, 0.3), 0.05, 0.25)

    # Simulate stochastic Lotka-Volterra model.
    p = [α, β, γ, δ, ϕ1, ϕ2]
    predicted = solve(prob, SOSRI(); p=p, saveat=0.1)

    # Early exit if simulation could not be computed successfully.
    if predicted.retcode !== :Success
        Turing.@addlogprob! -Inf
        return nothing
    end

    # Observations.
    for i in 1:length(predicted)
        data[:, i] ~ MvNormal(predicted[i], σ^2 * I)
    end

    return nothing
end;

model_sde = fitlv_sde(odedata, prob_sde)

setadbackend(:forwarddiff)
chain_sde = sample(
    model_sde,
    NUTS(0.25),
    5000;
    init_params=[1.5, 1.3, 1.2, 2.7, 1.2, 0.12, 0.12],
    progress=false,
)
plot(chain_sde)


### A Pluto.jl notebook ###
# v0.12.7

using Markdown
using InteractiveUtils

# ╔═╡ 6fd616f6-1ea1-11eb-3814-8bfb4a096c49
using Pkg, DrWatson

# ╔═╡ 6ffe2628-1ea1-11eb-24ea-57f985146a72
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using Distributions, Random, StatsBase, DataFrames, Plots
	Random.seed!(1)
end;

# ╔═╡ ed174bc4-1ea0-11eb-1e2f-a32874cec549
md"## Listing 3.19"

# ╔═╡ 70179b62-1ea1-11eb-13ce-7fb4d54bad4b
function prn(lambda)
    k, p = 0, 1
    while p > MathConstants.e^(-lambda)
        k += 1
        p *= rand()
    end
    return k-1
end

# ╔═╡ 0cbc946e-2164-11eb-1b0c-7d4c7a607b1e
begin
	xGrid, lambda, N = 0:16, 5.5, 10^6

	pDist = Poisson(lambda)
	bPmf = pdf.(pDist,xGrid)
	data = counts([prn(lambda) for _ in 1:N],xGrid)/N
end

# ╔═╡ 0cbcbdf2-2164-11eb-193d-2baa8a6bce0b
begin
	plot( xGrid, data,
		line=:stem, marker=:circle,
		c=:blue, ms=10, msw=0, lw=4, label="MC estimate")
	plot!( xGrid, bPmf, line=:stem,
		marker=:xcross, c=:red, ms=6, msw=0, lw=2, label="PMF",
		ylims=(0,0.2), xlabel="x", ylabel="Probability of x events")
end

# ╔═╡ 70182cd8-1ea1-11eb-094d-8d0d49cf15f3
md"## End of listing 3.19"

# ╔═╡ Cell order:
# ╟─ed174bc4-1ea0-11eb-1e2f-a32874cec549
# ╠═6fd616f6-1ea1-11eb-3814-8bfb4a096c49
# ╠═6ffe2628-1ea1-11eb-24ea-57f985146a72
# ╠═70179b62-1ea1-11eb-13ce-7fb4d54bad4b
# ╠═0cbc946e-2164-11eb-1b0c-7d4c7a607b1e
# ╠═0cbcbdf2-2164-11eb-193d-2baa8a6bce0b
# ╟─70182cd8-1ea1-11eb-094d-8d0d49cf15f3


"""
    forwardsub(L,b)

Solve the lower-triangular linear system with matrix `L` and right-hand side
vector `b`.
"""
function forwardsub(L,b)

n = size(L,1)
x = zeros(n)
for i = 1:n
    s = i > 1 ? sum( L[i,j]*x[j] for j=1:i-1 ) : 0
    x[i] = ( b[i] - s ) / L[i,i]
end

return x
end

"""
    backsub(U,b)

Solve the upper-triangular linear system with matrix `U` and right-hand side
vector `b`.
"""
function backsub(U,b)

n = size(U,1)
x = zeros(n)
for i = n:-1:1
    s = i < n ? sum( U[i,j]*x[j] for j=i+1:n ) : 0
    x[i] = ( b[i] - s ) / U[i,i]
end

return x
end

"""
    lufact(A)

Compute the LU factorization of square matrix `A`, returning the factors.
"""
function lufact(A)

n = size(A,1)
L = Matrix(Diagonal(ones(n)))    # puts ones on diagonal
U = copy(A)

# Gaussian elimination
for j = 1:n-1
  for i = j+1:n
    L[i,j] = U[i,j] / U[j,j]   # row multiplier
    U[i,j:n] = U[i,j:n] - L[i,j]*U[j,j:n]
  end
end

return L,triu(U)
end


using Pkg
using Plots
using DataFrames
pyplot()

### functions for simulations
function calc_new_state(y::AbstractArray,
    a::Dict,
    b::Dict,
    l::Dict;
    from::Float64=0.01)
    if length(y)==0
        y = [from]
    end
    y[end]+(1-y[end])*(a["walk"]*(1-exp(-b["walk"]*l["walk"]*a["walk"]*y[end]))+a["TTC"]*(1-exp(-b["TTC"]*l["TTC"]*a["TTC"]*y[end])))
end

function simul(s::Dict,
               b::Dict,
               l::Dict;
               to_perc::Bool=true,
               from::Float64=0.01,
               to::Union{Float64, Int}=0.99)

               # initial calculations
                e1 = (exp(-(s["walk"])/min(s["walk"], s["TTC"])))
                e2 = (exp(-(s["TTC"])/min(s["walk"], s["TTC"])))
                a = Dict("walk" => e1/sum([e1,e2]), "TTC" => e2/sum([e1,e2]))
                # simulation loop
                y = []
                append!(y, from)
                if to_perc
                    while y[end] <= to
                        append!(y, calc_new_state(y,a,b,l))
                    end
                else
                    for i in 1:to
                        append!(y, calc_new_state(y,a,b,l))
                    end
                end
                y
end

#####################################
### scenario 1:
s = Dict("walk" => 15, "TTC" => 10) # time of travel
l = Dict("walk" => 0.01, "TTC" => 0.02) # contagion
b = Dict("walk" => 5, "TTC" => 5) # interaction frequency

x1 = simul(s,b,l)
length(x1) - 1

### scenario 2:
s = Dict("walk" => 15, "TTC" => 10) # time of travel
l = Dict("walk" => 0.01, "TTC" => 0.02) # contagion
b = Dict("walk" => 3, "TTC" => 5) # interaction frequency

x2 = simul(s,b,l)
length(x2) - 1

# PLOT1
Plots.plot(1:length(x1), x1, linecolor="purple", lw = 2, legend=false, grid=false)
ylabel!("% infected")
xlabel!("days")
savefig("plot1.eps")

# PLOT2
Plots.plot(1:length(x2), x2, linecolor="purple", lw = 2, legend=false, grid=false)
ylabel!("% infected")
xlabel!("days")
savefig("plot2.eps")

###############################

#PLOT3
Plots.plot(1:length(x1), x1, linecolor="purple", label="Scenario 1", lw=2, grid = false, legend=:topleft)
Plots.plot!(1:length(x2), x2, linecolor="black", label="Scenario 2", lw=2 #linestyle = :dot, )
)
ylabel!("% infected")
xlabel!("days")
savefig("plot3.eps")

################################

# PLOT 4
s = Dict("walk" => 15, "TTC" => 10) # time of travel
l = Dict("walk" => 0.01, "TTC" => 0.02) # contagion
b = Dict("walk" => 3, "TTC" => 5) # interaction frequency

results_full = Dict()
results_short = DataFrame()
for i in 1:50
    s["TTC"] = i
    results_full[i] = simul(s,b,l)
    append!(results_short, DataFrame(s_ttc = [i], days = [length(results_full[i])-1]))
end

results_short # ok

Plots.plot(results_short[!,:s_ttc], results_short[!,:days], linecolor="purple", lw = 2, legend=false, grid=false)
ylabel!("days")
xlabel!("time when taking TTC")
savefig("plot4.eps")

#############################

# PLOT 5
s = Dict("walk" => 15, "TTC" => 10) # time of travel
l = Dict("walk" => 0.01, "TTC" => 0.02) # contagion
b = Dict("walk" => 3, "TTC" => 5) # interaction frequency

results_full2 = Dict()
results_short2 = DataFrame()
for i in 1:50
    s["TTC"] = i
    results_full2[i] = simul(s,b,l, to_perc=false, to=101)
    append!(results_short2, DataFrame(s_ttc = [i], infected = results_full2[i][101]))
end

results_short2 # ok

Plots.plot(results_short2[!,:s_ttc], results_short2[!,:infected], linecolor="purple", lw = 2, legend=false, grid=false)
ylabel!("% infected after 100 days")
xlabel!("time when taking TTC")
savefig("plot5.eps")


@model AHMCGaussian(y, N) = begin
    mu ~ Normal(0, 1)
    sigma ~ Truncated(Cauchy(0, 5), 0, Inf)
    y ~ MvNormal(Fill(mu, N), sigma)  # use `Fill(mu, N)` is memory freiendly
end

AHMCconfig = Turing.NUTS(1000, .85)

CmdStanGaussian = "
data {
  int<lower=0> N;
  vector[N] y;
}
parameters {
  real mu;
  real<lower=0> sigma;
}
model {
  mu ~ normal(0,1);
  sigma ~ cauchy(0,5);
  y ~ normal(mu,sigma);
}
"

CmdStanConfig = Stanmodel(
  name = "CmdStanGaussian", model=CmdStanGaussian, nchains=1,
  Sample(
    num_samples=1000, num_warmup=1000, adapt=CmdStan.Adapt(delta=0.8),
    save_warmup=true
  )
)

struct GaussianProb{TY <: AbstractVector}
    "Observations."
    y::TY
end

function (problem::GaussianProb)(θ)
    @unpack y = problem   # extract the data
    @unpack mu, sigma = θ
    N = length(y)
    logpdf(MvNormal(Fill(mu, N), sigma), y) + logpdf(Normal(0, 1), mu) +
      logpdf(Truncated(Cauchy(0, 5), 0, Inf), sigma)
end

# Define problem with data and inits.
function sampleDHMC(obs, N, nsamples, autodiff)
  p = GaussianProb(obs);
  p((mu=0.0, sigma=1.0))

  # Write a function to return properly dimensioned transformation.
  trans = as((mu=as(Real, -25, 25), sigma=asℝ₊), )

  # Use Flux for the gradient.

  P = TransformedLogDensity(trans, p)
  ∇P = ADgradient(autodiff, P)

  # Sample from the posterior.
  results = mcmc_with_warmup(Random.GLOBAL_RNG, ∇P, nsamples; reporter = NoProgressReport())

  # Undo the transformation to obtain the posterior from the chain.
  posterior = transform.(trans, results.chain)

  chns = nptochain(results, posterior)
  return chns
end

simulateGaussian(; μ=0, σ=1, Nd, kwargs...) = (y=rand(Normal(μ, σ), Nd), N=Nd)


"""
    Module Basis2DQuad

Includes DG core functions.
"""

module Basis2DQuad

export vandermonde_2D, grad_vandermonde_2D
export nodes_2D, equi_nodes_2D, quad_nodes_2D

using Basis1D
using CommonUtils
using LinearAlgebra

"""
    vandermonde_2D(N, r)

Initialize the 2D Vandermonde matrix of order N "Legendre" polynomials at
nodes (r,s)

# Examples
```jldoctest
"""
function vandermonde_2D(N, r, s)

    Np = convert(Int,(N+1)*(N+1))
    sk = 1
    V = zeros(length(r), Np);
    for i=0:N
        for j=0:N
            V[:,sk] = jacobiP(r, 0, 0, i).*jacobiP(s, 0, 0, j)
            sk += 1
        end
    end
    return V
end

"""
    gradV_2D(N, Np, r, s)

Initilize 2D gradient Vandermonde matrix of derivatives 2D "Legendre" polynomials
of order N at (r,s)

# Examples
```jldoctest
"""
function grad_vandermonde_2D(N, r, s)

    Np = convert(Int,(N+1)*(N+1))
    sk = 1
    V2Dr = zeros(length(r), Np);
    V2Ds = zeros(length(r), Np);
    for i=0:N
        for j=0:N
            V2Dr[:,sk] = grad_jacobiP(r, 0, 0, i).*jacobiP(s, 0, 0, j)
            V2Ds[:,sk] = jacobiP(r, 0, 0, i).*grad_jacobiP(s, 0, 0, j)
            sk += 1
        end
    end

    return V2Dr, V2Ds
end

# ===================================================

"""
    nodes_2D(N)

Compute optimized interpolation nodes using blend & warp method on equilateral
triangles for polynomial of order N, with Np points

# Examples
```jldoctest
"""

function nodes_2D(N)
    r1D,w1D = gauss_lobatto_quad(0,0,N)
    s,r = meshgrid(r1D)
    return r[:], s[:]
end

"""
    equi_nodes_2D(N)

Compute optimized interpolation nodes using blend & warp method on equilateral
triangles for polynomial of order N, with Np points

# Examples
```jldoctest
"""

function equi_nodes_2D(N)
    r1D = LinRange(-1,1,N+1)
    s,r = meshgrid(r1D)

    return r[:], s[:]
end

"""
    quad_nodes_2D(N)

Compute optimized interpolation nodes using blend & warp method on equilateral
triangles for polynomial of order N, with Np points

# Examples
```jldoctest
"""

function quad_nodes_2D(N)
    r1D,w1D = gauss_quad(0,0,N)
    s,r = meshgrid(r1D)
    ws,wr = meshgrid(w1D)
    w = @. wr*ws
    return r[:], s[:], w[:]
end

end


import CompEcon, BasisMatrices
using PyPlot
using PyCall

f1(x) = @. exp(-2x)
g1(x) = @. -2exp(-2x)

function univariate_interp(f=f1, g=g1)
    # fit approximant
    n = 10; a = -1; b = 1
    basis = CompEcon.fundefn(:cheb, n, a, b)
    c = CompEcon.funfitf(basis, f1)

    # graph approximation error for function and derivative
    x = BasisMatrices.nodeunif(1001, a, b)[1]
    yact = f(x)
    dact = g(x)
    yfit = CompEcon.funeval(c, basis, x)[1]
    dfit = CompEcon.funeval(c, basis, x, 1)[1]

    # Nice plot of function approximation error
    fig, ax = plt.subplots()
    ax[:plot](x, yfit-yact)
    ax[:plot](x, 0*x, "k--", linewidth=2)
    ax[:set_xlabel]("x")
    ax[:set_ylabel]("Error")
    ax[:set_title]("Error approximating function using Chebyshev")

    # Nice plot of derivative approximation error
    fig, ax = plt.subplots()
    ax[:plot](x, dfit-dact);
    ax[:plot](x, 0*x, "k--", linewidth=2)
    ax[:set_xlabel]("x")
    ax[:set_ylabel]("Error")
    ax[:set_title]("Error approximating derivative using Chebyshev")
end

f2(x) = cos.(x[:, 1]) ./ exp.(x[:, 2])
function bivaraite_interp(f=f2)
    n = [7 7]
    a = [0 0]
    b = [1 1]
    basis = CompEcon.fundefn(:cheb, n, a, b)
    c = CompEcon.funfitf(basis, f2)

    fig = plt.figure()
    nplot = [101 101]
    x, xcoord = BasisMatrices.nodeunif(nplot, a, b)
    yfit = CompEcon.funeval(c, basis, x)[1]
    error = reshape(yfit - f2(x), nplot...)
    poly3d = surf(xcoord[1], xcoord[2], error)

end

univariate_interp()
bivaraite_interp()


"""
    UtilFuncs

This module collects many conventional utility functions
which are provided within the general form ``u(X|\\theta):\\mathbb{R}^{k}\\to\\mathbb{R}``.
where ``\\theta`` is `tuple` of parameters (in specific types of utility function, they are defined as keyword parameters), and ``k`` is the dimension of utilizable goods.

There is the contents of utility functions:
1. `u_log` - logarithm (risk netural, 1 good)
2. `u_crra` - CRRA
3. `u_cara` - CARA
4. `u_ces2` - CES (2 goods)
5. `u_cesk` - CES (k goods)
6. `u_cd1` - Cobb-Douglas (1 good)
7. `u_cd2` - Cobb-Douglas (2 goods)
8. `u_cdk` - Cobb-Douglas (k goods)
9. `u_logcd2` - Logarithm Cobb-Douglas (2 goods)
10. `u_logcdk` - Logarithm Cobb-Douglas (k goods)
11. `u_leontiefk` - Leontief (k goods)

Please refer to each specific function for detailed documentations.

This module depends on:
1. nothing

@author: Clpr @ GitHub
"""
module UtilFuncs
    export u_log, u_crra, u_cara
    export u_ces2, u_cesk
    export u_cd1, u_cd2, u_cdk
    export u_logcd2, u_logcdk
    export u_leontiefk
# =================================
"""
    u_log(x::Real)

Logarithm (risk-netural) utility function for one good
where `x` is naturally required to be positive.
The expression of logarithm utility function is:
``u(x)=\\log(x),x>0``
"""
function u_log(x::Real)
    return log(x)::Real
end # u_log

# ----------------
"""
    u_crra(x::Real ; η::Real)

CRRA (constant relative risk aversion) utility function of one good
whose expression is:
``u(x|\\eta>0) = \\frac{x^{1-\\eta}}{1-\\eta}``.
The relative risk aversion coefficient ``\\eta`` must be positive.
When ``\\eta`` is one, this function degenerates to logarithm utility function.
"""
function u_crra(x::Real ; η::Real = 1.1)
    if η<0; throw(ErrorException("requires positive eta (RRA)")); end
    η == 1 ? (return u_log(x)::Real)  :  (return (x^(1-η)/(1-η))::Real)
end # u_crra

# ----------------
"""
    u_cara(x::Real ; α::Real = 1.0)

CARA (constant absolute risk aversion) utility function of one good.
whose expression is:
``u(x|\\alpha>0) = -e^{-\\alpha x}``.
The absolute risk aversion coefficient ``\\alpha`` must be positive.
"""
function u_cara(x::Real ; α::Real = 1.0)
    if α<0; throw(ErrorException("requires positive alpha (ARA)")); end
    return (-exp(-α * x))::Real
end # u_cara

# ----------------
"""
    u_ces2(x1::Real, x2::Real ; ρ::Real = 1.0, β::Real = 1.0, α::Real = 0.5 )

CES (constant elasticity of substitution) utility function whose expression is:
``u(x_1,x_2)=(\\alpha x_1^{\\rho} + \\(1-alpha) x_2^{\\rho})^{\\beta/\\rho}``
The elasticity of substitution is ``\\frac{1}{1-\\rho}``.

Special cases:
1. when ``\\rho=1``, the function has linear/perfect substitution.
2. when ``\\rho=0``, the function becomes Cobb-Douglas function of two goods, i.e. ``u(x_1,x_2)=(x_1^\\alpha x_2^{1-\\alpha})^{\\beta}``
3. when ``\\rho=-\\infty``, the function becomes Leontief (perfect complimentary), i.e. ``u(x_1,x_2)=\\beta \\min\\{ x_1,x_2 \\}``
"""
function u_ces2(x1::Real, x2::Real ; ρ::Real = 1.0, β::Real = 1.0, α::Real = 0.5 )
    if ρ == 0
        return ( ( x1^α * x2^(1-α) )^β )::Real
    elseif ρ == -Inf
        return ( β * min(x1,x2) )::Real
    end # if
    return ( ( α*x1^ρ + (1-α)*x2^ρ )^(β/ρ) )::Real
end # u_ces2

# ----------------
"""
    u_cesk(X::Vector ; W::Vector = ones(length(X)) , β::Real = 1.0, ρ::Real = 1.0 )

CES utility function of k goods whose expression is:
``u(X)=(\\sum w_i \\log(x_i)^{\\rho})^{\\beta/\\rho}``.
requiring the length of `W` must be equal to the length of `X`.
This function does not require the summation of `W` must be 1.

Special cases:
1. when ``\\rho=1``, the function has linear/perfect substitution.
2. when ``\\rho=0``, the function becomes Cobb-Douglas function
3. when ``\\rho=-\\infty``, the function becomes Leontief (perfect complimentary)

"""
function u_cesk(X::Vector ; W::Vector = ones(length(X)) , β::Real = 1.0, ρ::Real = 1.0 )
    if ρ == 0
        return ( ( u_cdk(X, W=W) )^β )::Real
    elseif ρ == -Inf
        return ( β * u_leontiefk(X) )::Real
    end # if
    local U::Real = 0.0
    for (x,w) in zip(X,W)
        U += w * x ^ ρ
    end # for
    return ( U^(β/ρ) )::Real
end # u_cesk

# ----------------
"""
    u_cd1(x::Real ; θ::Real = 1.0)

Cobb-Douglas utility function of one good whose expression is:
``u(x)=x^\\theta``
"""
function u_cd1(x::Real ; θ::Real = 1.0)
    return (x^θ)::Real
end # u_cd1

# ----------------
"""
    u_cd2(x::Real ; θ::Real = 0.5)

Cobb-Douglas utility function of two good whose expression is:
``u(x_1,x_2)=x_1^\\theta\\cdot x_2^{1-\\theta}``
"""
function u_cd2(x1::Real, x2::Real ; θ::Real = 0.5)
    return (x1^θ * x2^(1-θ))::Real
end # u_cd2

# ----------------
"""
    u_cdk(X::Vector ; W::Vector = ones(length(X)))

Cobb-Douglas utility function of k goods whose expression is:
``u(X)=\\prod \\log(x_i)^{w_i}``.
requiring the length of `W` must be equal to the length of `X`.
This function does not require the summation of `W` must be 1.
"""
function u_cdk(X::Vector ; W::Vector = ones(length(X)) )
    return exp(u_logcdk(X, W = W))::Real
end # u_cdk

# ----------------
"""
    u_logcd2(x1::Real, x2::Real ; β::Real = 1.0 )

Logarithm Cobb-Douglas utility function of k goods whose expression is:
``u(x_1,x_2)=\\log(x_1)+\\beta\\log(x_2)``
"""
function u_logcd2(x1::Real, x2::Real ; β::Real = 1.0 )
    return ( log(x1) + β * log(x2) )::Real
end

# ----------------
"""
    u_logcdk(X::Vector ; W::Vector = ones(length(X)))

Logarithm Cobb-Douglas utility function whose expression is:
``u(X)=\\sum w_i\\log(x_i)``.
requiring the length of `W` must be equal to the length of `X`.
"""
function u_logcdk(X::Vector ; W::Vector = ones(length(X)) )
    local U::Real = 0.0
    for (x,w) in zip(X,W); U += w * x; end # for
    return U::Real
end # u_logcdk

# ----------------
"""
    u_leontiefk(X::Vector)

Leontief (perfect complimentary) utility function of k goods:
``u(\\mathbf{x})=\\min\\{x_1,x_2,\\dots\\}``
"""
function u_leontiefk(X::Vector)
    return findmin(X)
end # u_leontiefk

# =================================
end # UtilFuncs


using LinearAlgebra

function matcond(n::Int, c::Float64)
# Function generates a random square matrix A of size n with
# a given condition number c.
# Inputs:
#	n: size of matrix A, n>1
#	c: condition of matrix A, c>= 1.0
#
# Usage: matcond(10, 100.0)
#
# Pawel Zielinski
        if n < 2
         error("size n should be > 1")
        end
        if c< 1.0
         error("condition number  c of a matrix  should be >= 1.0")
        end
        (U,S,V)=svd(rand(n,n))
        return U*diagm(0 =>[LinRange(1.0,c,n);])*V'
end


#Minimum cover EMS method
using JuMP,Gurobi

#define the model
m =Model(Gurobi.Optimizer)

#Let n represents the number of decision variables
n=10

#declaring variables
@variable(m, x[1:n], Bin) # binary constraint

#define the objective function
@objective(m , Min, sum(x[i] for i =1:n)) # objective is to minimize the coverage

#Since decision variables x2,x3,x8 and x10>=1, I declared them in a for-loop together
for i in [2,3,8,10]
	@constraint(m,x[i]>=1)    #(x[i] for i in [2,3,8,10]) >=1)
end

#additional constraint
@constraint(m, x[1]+x[2]>=1)    #district 2
@constraint(m, x[1]+x[3]>=1)    #district 3
@constraint(m, x[2]+x[4]>=1)	#district 7
@constraint(m, x[3]+x[4]>=1)	#district 8
@constraint(m, x[4]+x[6]>=1)	#district 10
@constraint(m, x[4]+x[5]>=1)	#district 11
@constraint(m, x[4]+x[5]+x[7]>=1)	#distict 13
@constraint(m, x[4]+x[5]+x[6]>=1)	#distict 12
@constraint(m, x[8]+x[9]>=1)		#distict 14, district 18
@constraint(m, x[6]+x[9]>=1)		#distict 15
@constraint(m, x[5]+x[6]>=1)		#distict 16
@constraint(m, x[5]+x[7]+x[10]>=1)	#distict 17
@constraint(m, x[8]+x[10]>=1)		#distict 19

print(m)

#Printing the optimal solutions obtained
println("Optimal Solutions:")
status = JuMP.optimize!(m)
println(" Objective value: ",JuMP.objective_value(m))

println("Decision variable values:")
for i=1:10
  println("x[$i] = ",   JuMP.value(x[i]))
end


function dayNumber(inDate::Date)
    return Dates.date2epochdays(inDate)::Integer
end

excelcostant = 693959;

"""
From Excel Number Format to Date

		Date=fromExcelNumberToDate(ExcelNumber)

Where:\n
		ExcelNumber = Integer representing a date in the excel format.

		Date      = Date representing the input in the Julia object format.

# Example
```julia-repl
julia> fromExcelNumberToDate(45000)
2023-03-15
```
"""
function fromExcelNumberToDate(num::Integer)::Date
    return Dates.Dates.epochdays2date(excelcostant + num)
end

"""
Actual Number of days between two dates

		ndays=daysact(Date1,Date2)

Where:\n
		Date1 = Start date.
		Date2 = End date.

		ndays      = Actual Number of days between Start Date and End Date.

# Example
```julia-repl
julia> daysact(Date(1996,10,12),Date(1998,1,10))
455
```
"""
function daysact(Date1::Date, Date2::Date)::Integer
    D1 = dayNumber(Date1)
    D2 = dayNumber(Date2)
    dayCount = D2 - D1
    return dayCount
end

function isLastOfFebruary(inDate::Date)::Bool
    return (Dates.isleapyear(Dates.year(inDate)) && (Dates.day(inDate) == 29) && (Dates.month(inDate) == 2)) || ((!Dates.isleapyear(Dates.year(inDate))) && (Dates.day(inDate) == 28) && (Dates.month(inDate) == 2))
end

currMaxImplemented = 7;

"""
Fraction of year between two Dates according the following convention

		yfr=yearfrac(Date1,Date2,basis)

Where:\n
		Date1 = Start date.
		Date2 = End date.
		basis = Integer representing the following conventions:
				- 0 = (ACT/ACT)
				- 1 = (30/360 SIA)
				- 2 = (ACT/360)
				- 3 = (ACT/365)
				- 4 = (30/360 PSA)
				- 5 = (30/360 ISDA)
				- 6 = (30E/360)
				- 7 = (ACT/365 JPN)

		yfr      = fraction of year between start and end date according to basis.

# Example
```julia-repl
julia> yearfrac(Date(1996,10,12),Date(1998,1,10),1)
1.2444444444444445
```
"""
function yearfrac(startDate::Date, endDate::Date, convention::Integer)::Real
    if (convention < 0)
        error("Negative basis are not defined, check the help")
    end
    if (convention > currMaxImplemented)
        error("Convention not implemented yet")
    end
    if (startDate > endDate)
        return -yearfrac(endDate, startDate, convention)
    elseif (startDate == endDate)
        return 0.0
    end
    yearFrac = 0.0
    y1 = Dates.year(startDate)
    m1 = Dates.month(startDate)
    y2 = Dates.year(endDate)
    m2 = Dates.month(endDate)
    d1 = Dates.day(startDate)
    d2 = Dates.day(endDate)
    if (convention == 0)#(ACT/ACT)
        Nday = daysact(startDate, endDate)
        EndOFYear = Date(1)
        if (isLastOfFebruary(startDate) && (Dates.day(startDate) == 29))
            EndOFYear = Date(Dates.year(startDate) + 1, 3, 1)
        else
            EndOFYear = Date(Dates.year(startDate) + 1, Dates.month(startDate), Dates.day(startDate))
        end
        yearFrac = (Nday) / daysact(startDate, EndOFYear)
    elseif (convention == 1)  #(30/360 SIA)
        if (isLastOfFebruary(startDate) && isLastOfFebruary(endDate))
            d2 = 30
        end
        if (isLastOfFebruary(startDate) || Dates.day(startDate) == 31)
            d1 = 30
        end
        if (d1 == 30 && d2 == 31)
            d2 = 30
        end
        dy = y2 - y1
        dm = m2 - m1
        dd = d2 - d1
        yearFrac = (360.0 * dy + 30.0 * dm + dd) / 360.0
    elseif (convention == 2)#(ACT/360)
        Nday = daysact(startDate, endDate)
        yearFrac = (Nday) / 360.0
    elseif (convention == 3)# (ACT/365)
        Nday = daysact(startDate, endDate)
        yearFrac = (Nday) / 365.0
    elseif (convention == 4)# (30/360 PSA)
        if ((Dates.day(startDate) == 31) || isLastOfFebruary(startDate))
            d1 = 30
        end
        if ((Dates.day(startDate) == 30 || isLastOfFebruary(startDate)) && Dates.day(endDate) == 31)
            d2 = 30
        end
        dy = y2 - y1
        dm = m2 - m1
        dd = d2 - d1
        yearFrac = (360.0 * dy + 30.0 * dm + dd) / 360.0

    elseif (convention == 5)#(30/360 ISDA)
        y1 = Dates.year(startDate)
        m1 = Dates.month(startDate)
        y2 = Dates.year(endDate)
        m2 = Dates.month(endDate)
        if (Dates.day(startDate) < 31)
            d1 = Dates.day(startDate)
        else
            d1 = 30
        end
        if ((Dates.day(endDate) == 31) && (d1 > 29))
            d2 = 30
        else
            d2 = Dates.day(endDate)
        end
        yearFrac = (360.0 * ((y2 - y1)) + 30.0 * ((m2 - m1)) + (d2 - d1)) / 360.0

    elseif (convention == 6)#(30E/360)
        if (d1 == 31)
            d1 = 30
        end
        if (d2 == 31)
            d2 = 30
        end
        dy = y2 - y1
        dm = m2 - m1
        dd = d2 - d1
        yearFrac = (360.0 * dy + 30.0 * dm + dd) / 360.0
    elseif (convention == 7)#(ACT/365 JPN)
        daydistance = [0; 31; 59; 90; 120; 151; 181; 212; 243; 273; 304; 334]
        dy = y2 - y1
        dd = d2 - d1
        dayCount = (365.0 * dy + daydistance[m2] - daydistance[m1] + dd)
        yearFrac = dayCount / 365.0
    end

    return yearFrac
end


doc"""
    TDist(ν)

The *Students T distribution* with `ν` degrees of freedom has probability density function

$f(x; d) = \frac{1}{\sqrt{d} B(1/2, d/2)}
\left( 1 + \frac{x^2}{d} \right)^{-\frac{d + 1}{2}}$

```julia
TDist(d)      # t-distribution with d degrees of freedom

params(d)     # Get the parameters, i.e. (d,)
dof(d)        # Get the degrees of freedom, i.e. d
```

External links

[Student's T distribution on Wikipedia](https://en.wikipedia.org/wiki/Student%27s_t-distribution)

"""
immutable TDist <: ContinuousUnivariateDistribution
    ν::Float64

    TDist(ν::Real) = (@check_args(TDist, ν > zero(ν)); new(ν))
end

@distr_support TDist -Inf Inf

#### Parameters

dof(d::TDist) = d.ν
params(d::TDist) = (d.ν,)

#### Statistics

mean(d::TDist) = d.ν > 1.0 ? 0.0 : NaN
median(d::TDist) = 0.0
mode(d::TDist) = 0.0

function var(d::TDist)
    ν = d.ν
    ν > 2.0 ? ν / (ν - 2.0) :
    ν > 1.0 ? Inf : NaN
end

skewness(d::TDist) = d.ν > 3.0 ? 0.0 : NaN

function kurtosis(d::TDist)
    ν = d.ν
    ν > 4.0 ? 6.0 / (ν - 4.0) :
    ν > 2.0 ? Inf : NaN
end

function entropy(d::TDist)
    h = 0.5 * d.ν
    h1 = h + 0.5
    h1 * (digamma(h1) - digamma(h)) + 0.5 * log(d.ν) + lbeta(h, 0.5)
end

#### Evaluation & Sampling

@_delegate_statsfuns TDist tdist ν

rand(d::TDist) = StatsFuns.Rmath.tdistrand(d.ν)

function cf(d::TDist, t::Real)
    t == 0 && return complex(1.0)
    h = d.ν * 0.5
    q = d.ν * 0.25
    t2 = t*t
    complex(2*(q*t2)^q*besselk(h,sqrt(d.ν)*abs(t))/gamma(h))
end

gradlogpdf(d::TDist, x::Float64) = -((d.ν + 1.0) * x) / (x^2 + d.ν)


using DifferentialEquations, Plots, LaTeXStrings

##
function Hénon_Heiles(du,u,p,t)
    x  = u[1]
    y  = u[2]
    dx = u[3]
    dy = u[4]
    du[1] = dx
    du[2] = dy
    du[3] = -x - 2x*y
    du[4] = y^2 - y -x^2
end

##
initial1 = [1/6,0.0,-0.2,0.0]
initial2 = [1/6,0.0 + 0.01,-0.2+0.01,0.0]

##
e,x,y,ydot = initial1
xdot = sqrt(2*e - ydot^2 - y^2 + (2//3)*y^3)
ini = [x,y,xdot,ydot]
tspan = (0,150.)
prob = ODEProblem(Hénon_Heiles, ini, tspan)
sol = solve(prob, Vern9(), abs_tol=1e-16, rel_tol=1e-16,saveat = 0.1);
plot(sol, vars=(1,2), title = "The orbit of the Hénon-Heiles system", xaxis = "x", yaxis = "y", label = false,linecolor = :blue)
scatter!([x],[y],label = false,markercolor = :blue)

##
e2,x2,y2,ydot2 = initial2
xdot2 = sqrt(2*e2 - ydot2^2 - y2^2 + (2//3)*y2^3)
ini2 = [x2,y2,xdot2,ydot2]
prob2 = ODEProblem(Hénon_Heiles, ini2, tspan)
sol2 = solve(prob2, Vern9(), abs_tol=1e-16, rel_tol=1e-16,saveat = 0.1);
plot!(sol2, vars=(1,2), label = false,linecolor = :red)
scatter!([x2],[y2],label = false,markercolor = :red)


"""
    Compression(c, d, i; σ = 1.0, μ = 0.0)

Maps ``ℝ → (-1, 1)``.

# Arguments
- `c`: the "compression", i.e., the map ``ℝ → (-1, 1)``,
- `d`: the "decompression", i.e., the map ``(-1, 1) → ℝ``,
- `i`: the jacobian of the map, i.e., ``∂d(τ) /∂τ``,
- `σ`, `μ`: control the width and offset in the numerical integrations
"""
struct Compression
	comp::Function
	decomp::Function
	inv_jacobian::Function

	σ::Float64
	μ::Float64
	function Compression(c, d, i; σ = 1.0, μ = 0.0)
		σ ≤ 0.0 && error("σ needs to be positive")
		new(c, d, i, σ, μ)
	end
end

"""
    Algebraic(; σ = 1.0, μ = 0.0)

Maps ``ℝ → (-1, 1)`` by ``t ↦ \\frac{t}{1 + |t|}``.
Equivalent symbol is `:algebraic`.
"""
Algebraic(; σ = 1.0, μ = 0.0) = Compression(
	t ->   t / (1.0 + abs(t)),
	τ ->   τ / (1.0 - abs(τ)),
	τ -> 1.0 / (1.0 - abs(τ))^2, # == (1.0 + abs( τ / (1.0 - abs(τ)) ) )^2
	σ = σ, μ = μ
)

"""
    Exponential(; σ = 1.0, μ = 0.0)

Maps ``ℝ → (-1, 1)`` by ``t ↦ \\mathrm{tanh}(t)``.
Equivalent symbol is `:exponential`.
"""
Exponential(; σ = 1.0, μ = 0.0) = Compression(
	t -> tanh(t),
	τ -> atanh(τ),
	τ -> 1.0 / (1.0 - τ^2), # == cosh(atanh(τ))^2
	σ = σ, μ = μ
)

"""
    Trigonometric(; σ = 1.0, μ = 0.0)

Maps ``ℝ → (-1, 1)`` by ``t ↦ \\frac{2}{π} \\mathrm{tan}^{-1}(t)``.
Equivalent symbol is `:trigonometric`.
"""
Trigonometric(; σ = 1.0, μ = 0.0) = Compression(
	t -> 2.0 * atan(t) / π,
	τ -> tan(π*τ/2.0),
	τ -> π * (1.0 + tan(π*τ/2.0)^2) / 2.0,
	σ = σ, μ = μ
)

Compressions = Dict(
	:algebraic     => Algebraic(),
	:exponential   => Exponential(),
	:trigonometric => Trigonometric()
)


### A Pluto.jl notebook ###
# v0.14.1

using Markdown
using InteractiveUtils

# ╔═╡ 7c62a2c4-9159-11eb-1b54-452017ea90d9
using LinearAlgebra, SparseArrays

# ╔═╡ 7ad16586-9162-11eb-1d94-5188eb8d4c5d
using DifferentialEquations, Plots

# ╔═╡ 44df6004-9158-11eb-1a63-651d7dca957b
md"# Heat equation with Robin boundary conditions

Consider a one-dimensional rod $\Omega = (0,L)$ and time $t \in (0,T).$
"

# ╔═╡ aff71d6e-9158-11eb-0559-cb7334f4a624
L = 0.1 # Length of rod

# ╔═╡ 3771dab8-9159-11eb-3a69-cd9b94a10399
Tf = 60.0 # Final simulation time

# ╔═╡ 44dbf81c-9159-11eb-2920-27ee60a7db9a
md"The rod has the physical properties
- thermal conductivity $\lambda$,
- specific heat capacity $c$,
- mass density $\rho$.

The diffusivity constant is given by $\alpha = \frac{\lambda}{c \rho}$. The heat equation is noted as

$\dot{\vartheta}(t,x) = \alpha \frac{\partial^2}{\partial x^2} \vartheta(t,x)$

for $(t,x) \in (0,T) \times \Omega$ with initial condition

$\vartheta(0,x) = \vartheta_{0}(x)$

for $x \in \overline{\Omega}$."

# ╔═╡ 86a40dc8-9158-11eb-16a3-1f776d6bde4e
begin
	λ = 45.0;    # Thermal conductivity
	ρ = 7800.0;  # Mass density
	cap = 480.0; # Specific heat capacitivity
end

# ╔═╡ 9e8ee0a2-9158-11eb-0210-4513221b7823
α = λ/(cap * ρ)   # Diffusivity

# ╔═╡ a4f1e1e2-9158-11eb-0754-a70399a341ba
md"## Boundary conditions

On both sides of the rod, heat transfer

$\phi_{t}(t,x) = - h ~ (\vartheta(t,x) - \vartheta_{amb})$

and heat radiation

$\phi_{r}(t,x) = - \epsilon \varrho ~ (\vartheta(t,x)^4 - \vartheta_{amb}^4)$

is assumed to operate. Parameter $\vartheta_{amb}$ describes the ambient temperature, $h>0$ is called the heat transfer coefficient, $\epsilon \in (0,1)$ is called emissivity and $\varrho\approx 5.67 \cdot 10^{-8}$ is known as Stefan-Boltzmann constant. For simplicity, $k=\epsilon \cdot \varrho$.

The sum of $\phi_{t}$ and $\phi_{r}$

$\phi_{out}(t,x) = \phi_{t}(t,x) + \phi_{r}(t,x) = - h ~ (\vartheta(t,x) - \vartheta_{amb}) - k ~ (\vartheta(t,x)^4 - \vartheta_{amb}^4)$

is the flux from the rod to the environment. Thus, the natural (Robin) boundary conditions are noted as

$\lambda ~ \left.\frac{\partial}{\partial x} \vartheta(\cdot, x) \right\rvert_{x = 0} \cdot \vec{n} ~=~ \lambda ~  \left. \frac{\partial}{\partial x} \vartheta(\cdot, x) \right\rvert_{x = L} \cdot \vec{n} ~=~ \phi_{out}(t,x)$

with outer [normal vector](https://en.wikipedia.org/wiki/Normal_(geometry)) $\vec{n}$ on the left or right boundary. Here, the normal vector $\vec{n} = -1$ on the left boundary at $x=0$, and  $\vec{n} = 1$ on the right boundary at $x=L$."

# ╔═╡ 7c9e9a4a-9159-11eb-3dda-59d2aa15f32a
begin
	h = 10.0; # Heat transfer coefficient
	ϵ = 0.6;  # Emissivity
	sb = 5.67*10^(-8) # Stefan-Boltzmann constant
	k = ϵ * sb; 	  # Radiation coefficient
end

# ╔═╡ c5da798e-9160-11eb-1d16-75ce792ef2fe
θamb = 298.0 # Ambient temperature in Kelvin

# ╔═╡ 7c80218c-9159-11eb-301a-95a428fb76d9
md"### Spatial approximation

The one-dimensional rod is discretized as a one-dimensional grid with N points and the finite discretization is noted by

$\Delta x = \frac{L}{N-1}.$

The resulting grid points have the position $x^{0} = 0$, $x^{n} = n~\Delta x$ and $x^{N-1} = L$. The second order derivative is approximated using the [Taylor series](https://en.wikipedia.org/wiki/Taylor_series) as

$\left. \frac{\partial^2 f(x)}{\partial x^2} \right\rvert_{x = \tilde{x}} \approx \frac{1}{\Delta x^2} \left[ f(\tilde{x} - \Delta x) - 2 f(\tilde{x}) + f(\tilde{x} + \Delta x)  \right].$

This [Finite Difference](https://en.wikipedia.org/wiki/Finite_difference_method) scheme is written in matrix-vector form as

$\frac{\partial^2 \vartheta(t,x)}{\partial x^2} \approx \frac{1}{\Delta x^2}
\begin{pmatrix}
-2 & 1 & 0 & \cdots &  & 0 \\
1 & -2 & 1 & 0  & \cdots & \vdots \\
0 & 1 & -2 & 1 & \ddots &  \\
 & & \ddots & \ddots & \ddots &  & \\
& & & 1 & -2 & 1 \\
& & &  & 1 & -2 \\
\end{pmatrix}
~
\begin{pmatrix}
\vartheta(t, x^{0}) \\
\vartheta(t, x^{1}) \\
\vdots \\
\vartheta(t, x^{N-1}) \\
\end{pmatrix}.$
"

# ╔═╡ 3487893a-915c-11eb-3a04-1fdd31009413
N = 101 # Number of grid elements

# ╔═╡ 39663578-915c-11eb-108c-2718b706729e
Δx = L/(N-1) # Finite discretization

# ╔═╡ 3468092a-915c-11eb-1719-d1b9b04b68e0
# Diffusion matrix
M = spdiagm(-1 => ones(N-1), 0 => -2*ones(N), 1 => ones(N-1));

# ╔═╡ 344c1bb6-915c-11eb-2be7-35f340f4caa8
Matrix(M)[1:5,1:5]

# ╔═╡ 342fe676-915c-11eb-3f22-1d20d5b502ed
md"### Approximated boundary conditions

The spatial approximation of the heat equation

$\frac{\partial^2 \vartheta(t,x)}{\partial x^2} \approx \frac{1}{\Delta x^2} \left( \vartheta(\cdot, x^{n-1}) - 2 ~ \vartheta(\cdot, x^{n}) + \vartheta(\cdot, x^{n+1}) \right)$

can not be evaluated directly at the grid points $x^{0}$ and $x^{N-1}$ because they depend on values from the not-existing grid points $x^{-1}$ and $x^{N}$. Therefore, the Neumann boundary condition is used to find $\vartheta(\cdot, x^{-1})$ and $\vartheta(\cdot, x^{N})$.

The Robin boundary condition states that the heat flux at both sides is described by

$\lambda \left. \frac{\partial \vartheta(\cdot,x)}{\partial x}  \right\rvert_{x = 0} \cdot \vec{n} = -1 \cdot \lambda \frac{\vartheta(\cdot, x^{1}) - \vartheta(\cdot, x^{-1})}{2 \Delta x}   = \phi_{out}(t,x^{0})$

on the left boundary and

$\lambda \left. \frac{\partial \vartheta(\cdot,x)}{\partial x}  \right\rvert_{x = L} \cdot \vec{n} = \lambda \frac{\vartheta(\cdot, x^{N}) - \vartheta(\cdot, x^{N-2})}{2 \Delta x}  = \phi_{out}(t,x^{N-1})$

on the right boundary. Reformulating both equations lead to

$\vartheta(\cdot, x^{-1}) = \vartheta(\cdot, x^{1}) + 2 \Delta x ~ \phi_{out}(t,x^{0})$

and

$\vartheta(\cdot, x^{N}) = \vartheta(\cdot, x^{N-2})  + 2 \Delta x ~ \phi_{out}(t,x^{N-1}).$

On the left boundary one yields

$\left. \frac{\partial^2 \vartheta(t,x)}{\partial x^2} \right\rvert_{x = 0} \approx \frac{1}{\Delta x^2} \left( - 2 ~ \vartheta(\cdot, x^{0}) + 2 ~ \vartheta(\cdot, x^{1}) \right) + \frac{2}{\Delta x} \phi_{out}(t,x^{0})$

and analog on the right boundary

$\left. \frac{\partial^2 \vartheta(t,x)}{\partial x^2} \right\rvert_{x = 0} \approx \frac{1}{\Delta x^2} \left(2 ~ \vartheta(\cdot, x^{N-2}) - 2 ~ \vartheta(\cdot, x^{N-1}) \right) + \frac{2}{\Delta x} \phi_{out}(t,x^{N-1}).$

The diffusion matrix and the approximated flux are noted as

$M =
\begin{pmatrix}
-2 & 2 & 0 & \cdots &  & 0 \\
1 & -2 & 1 & 0  & \cdots & \vdots \\
0 & 1 & -2 & 1 & \ddots &  \\
 & & \ddots & \ddots & \ddots &  & \\
& & & 1 & -2 & 1 \\
& & &  & 2 & -2 \\
\end{pmatrix}, \quad

\Phi_{out}(t) =
\begin{pmatrix}
\phi_{out}(t, x^{0}) \\
0 \\
\vdots \\
\\
0 \\
\phi_{out}(t, x^{N-1}) \\
\end{pmatrix}.$
"

# ╔═╡ 3414b70c-915c-11eb-2606-4d1ad425d3ab
# First row
M[1,2] = 2;

# ╔═╡ 33cd51a0-915c-11eb-3c62-efc367e08126
# Last row
M[end,end-1] = 2;

# ╔═╡ 33af0164-915c-11eb-30f0-313d926c2dae
md"### Heat equation as ODE

After the spatial approximation the heat equation has a form of

$\dot{\theta}(t) =  \frac{\alpha}{\Delta x^2} ~ M ~ \theta(t) + \frac{2 \alpha}{\Delta x} ~ \Phi_{out}(t)$

with M as diffusion matrix, temperature vector $\quad \theta(t) = \left( \vartheta(t, x^{0}), \cdots, \vartheta(t, x^{N-1}) \right)^{\top}$. This is an ordinary differential equation (ODE) that can be solved with common solvers like forward Euler method or Runge-Kutta scheme."

# ╔═╡ 328a1ed0-9160-11eb-0892-7d0a9d05a54d
# Heat Equation as ODE
function heat_eq!(dθ, θ, p, t)

	N = size(θ)
	Φout = zeros(N);

	Φout[1] = -h * (θ[1] - θamb) - k*(θ[1]^4 - θamb^4)

	Φout[end] = -h * (θ[end] - θamb) - k*(θ[end]^4 - θamb^4)

    dθ .= (1/Δx^2) * α * M * θ + (2/Δx) * α * Φout
end

# ╔═╡ 66d5aee4-9161-11eb-1ef1-2193d9c2f6a2
md"### Initial Conditions

For simplicity, the initial data of the heat equation is assumed as $\vartheta_{0}(x)= 1000$ Kelvin for all $x \in \left[0,L\right]$. This function is approximated to gain the initial conditions of the ODE

$\theta(0) = \left( 10^3, \cdots , 10^3 \right)^{\top}.$
"

# ╔═╡ 91a272be-9162-11eb-1046-8dd9e33653e5
θ₀ = 10^3 * ones(N)

# ╔═╡ 1571f3f2-9162-11eb-0b63-ff4e25a07572
md"The upper limit of the sampling time has to

$\Delta t < \frac{1}{2} \frac{\Delta x^2}{\alpha}.$

to guarantee numerical stability."

# ╔═╡ f85a2d4a-9161-11eb-0e0f-851c0750832a
Δt_ul = (0.5*Δx^2)/α # upper limit of sampling time

# ╔═╡ 5aa78f74-9162-11eb-1bd4-c96f2b90e586
Δt = 10^(-2) # Sampling time

# ╔═╡ 66ba8b22-9162-11eb-13b0-b14dfcead641
md"## Simulation"

# ╔═╡ 861018de-9162-11eb-2a01-2354acd868e2
tspan = (0.0, Tf)

# ╔═╡ e0e033e8-9162-11eb-3fe6-05565991ce77
# 1-dimensional grid
xspan = 0 : Δx : L

# ╔═╡ 01b373f0-9163-11eb-3f35-3f645825720f
prob = ODEProblem( heat_eq!, θ₀, tspan ) # ODE Problem

# ╔═╡ 0103888c-9163-11eb-16ae-371cd6d04c45
sol = solve(prob,Euler(),dt=Δt,progress=true, save_everystep=false, save_start=true) # Solving the ODE

# ╔═╡ 08dcb90e-9163-11eb-2ff0-7129add62934
plot(xspan, sol.u[2], xlabel = "Position x", ylabel="Temperature", legend=false)

# ╔═╡ Cell order:
# ╟─44df6004-9158-11eb-1a63-651d7dca957b
# ╠═aff71d6e-9158-11eb-0559-cb7334f4a624
# ╠═3771dab8-9159-11eb-3a69-cd9b94a10399
# ╟─44dbf81c-9159-11eb-2920-27ee60a7db9a
# ╠═86a40dc8-9158-11eb-16a3-1f776d6bde4e
# ╠═9e8ee0a2-9158-11eb-0210-4513221b7823
# ╟─a4f1e1e2-9158-11eb-0754-a70399a341ba
# ╠═7c9e9a4a-9159-11eb-3dda-59d2aa15f32a
# ╠═c5da798e-9160-11eb-1d16-75ce792ef2fe
# ╟─7c80218c-9159-11eb-301a-95a428fb76d9
# ╠═7c62a2c4-9159-11eb-1b54-452017ea90d9
# ╠═3487893a-915c-11eb-3a04-1fdd31009413
# ╠═39663578-915c-11eb-108c-2718b706729e
# ╠═3468092a-915c-11eb-1719-d1b9b04b68e0
# ╠═344c1bb6-915c-11eb-2be7-35f340f4caa8
# ╟─342fe676-915c-11eb-3f22-1d20d5b502ed
# ╠═3414b70c-915c-11eb-2606-4d1ad425d3ab
# ╠═33cd51a0-915c-11eb-3c62-efc367e08126
# ╟─33af0164-915c-11eb-30f0-313d926c2dae
# ╠═328a1ed0-9160-11eb-0892-7d0a9d05a54d
# ╟─66d5aee4-9161-11eb-1ef1-2193d9c2f6a2
# ╠═91a272be-9162-11eb-1046-8dd9e33653e5
# ╟─1571f3f2-9162-11eb-0b63-ff4e25a07572
# ╠═f85a2d4a-9161-11eb-0e0f-851c0750832a
# ╠═5aa78f74-9162-11eb-1bd4-c96f2b90e586
# ╠═66ba8b22-9162-11eb-13b0-b14dfcead641
# ╠═7ad16586-9162-11eb-1d94-5188eb8d4c5d
# ╠═861018de-9162-11eb-2a01-2354acd868e2
# ╠═e0e033e8-9162-11eb-3fe6-05565991ce77
# ╠═01b373f0-9163-11eb-3f35-3f645825720f
# ╠═0103888c-9163-11eb-16ae-371cd6d04c45
# ╠═08dcb90e-9163-11eb-2ff0-7129add62934


# sample from a double well in 1D using HMC
using Printf
using Random
using LinearAlgebra
using ForwardDiff
using QuadGK
using BasicMD
using Statistics

include("potentials.jl")

β = 5.0;
x₀ = [-1.0];
M = 1.0;
seed = 100;
Δt = 2e-1;
n_iters = 10^4; # number of samples
nΔt = 10^1; # number of Verlet steps per HMC iteration
n_save_iters=10;

V = x->DoubleWell(x);
cfg = ForwardDiff.GradientConfig(V, x₀);
gradV! = (gradV, x)-> ForwardDiff.gradient!(gradV, V, x, cfg);

# second moment and energy obsrevables
f₁ = x-> x[1]^2;
f₂ = x-> V(x[1]);
observables = (f₁, f₂);

sampler = HMC(V, gradV!, β, M, Δt, nΔt);

Random.seed!(100);
Xvals,_ = sample_trajectory(x₀, sampler, options=MDOptions(n_iters=n_iters,n_save_iters=n_save_iters));
f₁_estimate = mean(f₁.(Xvals));
f₂_estimate = mean(f₂.(Xvals));
@printf("f₁ estimate with %d samples: %g\n",length(Xvals), f₁_estimate);
@printf("f₂ estimate with %d samples: %g\n", length(Xvals), f₂_estimate);
Random.seed!(100);
observable_samples = sample_observables(x₀, sampler, observables, options=MDOptions(n_iters=n_iters,n_save_iters=n_save_iters));
@printf("f₁ estimate with %d samples: %g\n",length(observable_samples), mean(observable_samples[1,:]));
@printf("f₂ estimate with %d samples: %g\n", length(observable_samples), mean(observable_samples[2,:]));


"""
    gammaHSIC(X::Vector{T}, Y::Vector{T})

"""

function gammaHSIC(X::Vector{T}, Y::Vector{T}; randomSubSet = 100, kernelSize = -1) where {T<:Real}

	M = length(X)

	# get kernel sizes
	if kernelSize == -1
		sigx = estimateKernelSize(X, sampleSize = randomSubSet)
		sigy = estimateKernelSize(Y, sampleSize = randomSubSet)
	else
		sigx = kernelSize
		sigy = kernelSize
	end

	bone = ones(T, M, 1)
	H = eye(M) - 1/M * ones(T, M,M)

	K = rbfDotProduct(X, X, sigx)
	L = rbfDotProduct(Y, Y, sigy)

	# NOTE: these are slightly biased estimates of centred Gram matrices
	Kc = H*K*H
	Lc = H*L*H

	# NOTE: we fit Gamma to testStat*M
	testStat = 1/M * sum(sum(Kc'.*Lc))

	varHSIC = (1/6 * Kc.*Lc).^2
	varHSIC = 1/M/(M-1)* (  sum(sum(varHSIC)) - sum(diag(varHSIC))  ) # second subtracted term is bias correction
	varHSIC = 72*(M-4)*(M-5)/M/(M-1)/(M-2)/(M-3) * varHSIC

	K = K-diagm(0 => diag(K))
	L = L-diagm(0 => diag(L))

	muX = 1/M/(M-1)*bone'*(K*bone)
	muY = 1/M/(M-1)*bone'*(L*bone)

	mHSIC  = 1/M * ( 1 .+muX*muY - muX - muY )

	al = mHSIC^2 / varHSIC
	bet = varHSIC*M ./ mHSIC

	return 1 - gammacdf(al[1], bet[1], testStat)

end


# I.Y. Kim and O.L. de Weck.  Adaptive weighted-sum method for bi-objective optimization
# Pareto front generation. Structural and Multidisciplinary Optimization.
# Vol. 29, Num. 2, pp. 149–158, 2005.
# https://link.springer.com/article/10.1007/s00158-004-0465-1

using NSGAIII, PyPlot

function plot_pop(P)
    clf()
    p = plot(map(x -> x.y[1], P), map(x -> x.y[2], P), "bo", markersize=1)
    !isinteractive() && show()
    sleep(0.2)
end

const d = RealCoding(8, [-3, -3], [3, 3])
z1(x1, x2) = -(3(1-x1)^2 * exp(-x1^2 - (x2+1)^2) - 10(x1/5 - x1^3 - x2^5) * exp(-x1^2-x2^2) -3exp(-(x1+2)^2 - x2^2) + 0.5(2x1 + x2))
z2(x1, x2) = -(3(1+x2)^2 * exp(-x2^2 - (1-x1)^2) - 10(-x2/5 + x2^3 + x1^5) * exp(-x1^2-x2^2) - 3exp(-(2-x2)^2 - x1^2))
z(x) = begin
    x1, x2 = decode(x, d)
    z1(x1, x2), z2(x1, x2)
end
nsga(300, 20, ()->bitrand(d.nbbitstotal), z, 10, fplot = plot_pop)


### A Pluto.jl notebook ###
# v0.16.0

using Markdown
using InteractiveUtils

# ╔═╡ 5bb6cf00-9b75-4e74-bc75-6a059bc1b32d
# Učita se paket SymPy
using SymPy

# ╔═╡ efc1cf20-8276-11eb-39ba-bd62a4342e68
md"
# Integrali racionalnih funkcija
"

# ╔═╡ d52c1d09-9bee-4359-8e2c-a969a2f32ea3
# Ova naredba daje popis svih djelova paketa, uglavnom su to funkcije.
# varinfo(SymPy)

# ╔═╡ 8153dabd-5be6-4308-b7c5-1f4ed54722ab
begin
	# Izračunajmo nekoliko integrala.
	x=Sym("x")
	integrate(x^2*sin(x))
end

# ╔═╡ 10b872ad-f070-404c-9ec8-19f0bc2032c5
integrate(exp(-x^2))

# ╔═╡ 056c397f-7303-464e-9ec6-8cf893bbc98c
# Primjer rekurzivne formule. Ovo traje malo duže
integrate(1/(1+x^2)^13)

# ╔═╡ b29b5f68-8978-4b4b-9b29-4840a4097978
# Kraća rekurzivna formula
integrate(1/(1+x^2)^3)

# ╔═╡ 232dcad5-1984-4421-9c1a-001406096217
methods(diff)

# ╔═╡ a2819cd4-d0b3-476a-ab8c-40838db72d54
# Derivacija
diff(x^2*exp(sin(1/x)))

# ╔═╡ 74af42dd-9768-4e91-aa3b-b26c69d6fb68
# Racionalna funkcija trigonometrijskih funkcija (univerzalna trigonometrijska supstitucija)
integrate(1/((2+cos(x))*sin(x)))

# ╔═╡ de024987-9c44-4199-9085-b259980e94a5
md"""
#### Zadatak 1.5.e)
"""

# ╔═╡ 62c5d76c-9eef-40a9-b8c9-446720733f80
integrate(1/(4*sin(x)+3*cos(x)+5))

# ╔═╡ 379bc6e9-7a38-4f3e-8e4a-2ff80917880b
integrate(1/(sin(x)*(2+cos(x)-2*sin(x))))

# ╔═╡ 28dd8649-ed7c-47e5-b7ab-69e99dc0ce86
# Ovo traje beskonačno?
# integrate(cos(x)^3/(sin(x)^2+sin(x)))

# ╔═╡ 12904f05-67c3-4d8a-aefd-e9645e6ef0eb
# Ovo je isti zadatak kao gore, ali traje kratko. Koristi se jednostavnija supstitucija.
integrate((1-sin(x)^2)*cos(x)/(sin(x)^2+sin(x)))

# ╔═╡ bf1829c9-b2fb-4e3b-b92d-38a93513678b
# Ovo traje jako dugo i ne uspije izračunati nego vrati polazni integral
# integrate( (2*tan(x)+3)/(sin(x)^2+2*cos(x)^2))

# ╔═╡ 3732f5f7-ed44-4254-8b16-d9d860d2c04f
md"""
Uz supstituciju $t=\tan(x)$ integral postaje jednostavan:
"""

# ╔═╡ 232927bc-ac81-4225-801f-9da5e5842d6d
begin
	t=Sym("t")
	I₁=integrate( (2*t+3)/(t^2+2))
end

# ╔═╡ 13e53f50-8c7d-403c-85c5-848bf2ccbe94
begin
	# Vratimo supstituciju natrag
	I₂=convert(Function,I₁)
	I₂(tan(x))
end

# ╔═╡ d9ada9db-2651-4895-876b-704e4ed05bcb
md"""
#### Zadatak 1.6
"""

# ╔═╡ f2eb749b-51d6-4875-8e14-1e0f688cdfc7
# Ovo traje beskonačno!
# I₄=integrate((1+sinh(x))/((2+cosh(x))*(3+sinh(x))),x)
# Program treba našu pomoć!!!

# ╔═╡ 244208a5-b153-424b-8305-5b6002fd8186
md"""
__Univerzalna hiperbolna supstitucija__

$$\begin{aligned}
t&=\tanh(\frac{x}{2}), \quad x=2\mathop{\mathrm{atanh}}(t),\quad  dx=\displaystyle\frac{2}{1-t^2}dt,\\
\sinh(x)&=\displaystyle\frac{2t}{1-t^2}, \quad \cosh(x)=\displaystyle\frac{1+t^2}{1-t^2}.\end{aligned}$$

"""

# ╔═╡ 78acc71b-b314-4799-b633-4ab5b7958496
I₄=(1+sinh(x))/((2+cosh(x))*(3+sinh(x)))

# ╔═╡ 463c1ac7-e503-4507-8aca-d671acf04d36
# supstitucija za sinh(x)
I₅=subs(I₄,sinh(x),(2*t)/(1-t^2))

# ╔═╡ 1b47139d-6c06-4227-91d2-1556d2f3a9d2
# Supstitucija za cosh(x)
I₆=subs(I₅,cosh(x),(1+t^2)/(1-t^2))

# ╔═╡ 646c21be-c589-43b6-b648-3d0dea8353db
# Pomnožimo s dx
I₇=I₆*2/(1-t^2)

# ╔═╡ 34320cf9-d978-46ca-ace2-ea638cc602d7
# Integriramo racionalnu funkciju
I₈=integrate(I₇,t)

# ╔═╡ 6e0e6146-fefa-4967-938e-1755a0a04607
# Vratimo supstituciju natrag
I₉=subs(I₈,t,tanh(x/2))

# ╔═╡ a817a9d1-f9d2-4fb9-ad16-2477a0a6dea6

# ╔═╡ 00000000-0000-0000-0000-000000000001
PLUTO_PROJECT_TOML_CONTENTS = """
[deps]
SymPy = "24249f21-da20-56a4-8eb1-6a02cf4ae2e6"

[compat]
SymPy = "~1.0.52"
"""

# ╔═╡ 00000000-0000-0000-0000-000000000002
PLUTO_MANIFEST_TOML_CONTENTS = """
# This file is machine-generated - editing it directly is not advised

[[ArgTools]]
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"

[[Artifacts]]
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"

[[Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[ChainRulesCore]]
deps = ["Compat", "LinearAlgebra", "SparseArrays"]
git-tree-sha1 = "a325370b9dd0e6bf5656a6f1a7ae80755f8ccc46"
uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
version = "1.7.2"

[[CommonEq]]
git-tree-sha1 = "d1beba82ceee6dc0fce8cb6b80bf600bbde66381"
uuid = "3709ef60-1bee-4518-9f2f-acd86f176c50"
version = "0.2.0"

[[CommonSolve]]
git-tree-sha1 = "68a0743f578349ada8bc911a5cbd5a2ef6ed6d1f"
uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2"
version = "0.2.0"

[[Compat]]
deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
git-tree-sha1 = "31d0151f5716b655421d9d75b7fa74cc4e744df2"
uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
version = "3.39.0"

[[CompilerSupportLibraries_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"

[[Conda]]
deps = ["JSON", "VersionParsing"]
git-tree-sha1 = "299304989a5e6473d985212c28928899c74e9421"
uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d"
version = "1.5.2"

[[Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"

[[DelimitedFiles]]
deps = ["Mmap"]
uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"

[[Distributed]]
deps = ["Random", "Serialization", "Sockets"]
uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"

[[DocStringExtensions]]
deps = ["LibGit2"]
git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f"
uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
version = "0.8.5"

[[Downloads]]
deps = ["ArgTools", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"

[[InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

[[IrrationalConstants]]
git-tree-sha1 = "f76424439413893a832026ca355fe273e93bce94"
uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
version = "0.1.0"

[[JLLWrappers]]
deps = ["Preferences"]
git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e"
uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
version = "1.3.0"

[[JSON]]
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37"
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
version = "0.21.2"

[[LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"

[[LibCURL_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"

[[LibGit2]]
deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"

[[LibSSH2_jll]]
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"

[[Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

[[LinearAlgebra]]
deps = ["Libdl"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[LogExpFunctions]]
deps = ["ChainRulesCore", "DocStringExtensions", "IrrationalConstants", "LinearAlgebra"]
git-tree-sha1 = "34dc30f868e368f8a17b728a1238f3fcda43931a"
uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
version = "0.3.3"

[[Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"

[[MacroTools]]
deps = ["Markdown", "Random"]
git-tree-sha1 = "5a5bc6bf062f0f95e62d0fe0a2d99699fed82dd9"
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
version = "0.5.8"

[[Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"

[[MbedTLS_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"

[[Mmap]]
uuid = "a63ad114-7e13-5084-954f-fe012c677804"

[[MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"

[[NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"

[[OpenLibm_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "05823500-19ac-5b8b-9628-191a04bc5112"

[[OpenSpecFun_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1"
uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
version = "0.5.5+0"

[[Parsers]]
deps = ["Dates"]
git-tree-sha1 = "a8709b968a1ea6abc2dc1967cb1db6ac9a00dfb6"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "2.0.5"

[[Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

[[Preferences]]
deps = ["TOML"]
git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a"
uuid = "21216c6a-2e73-6563-6e65-726566657250"
version = "1.2.2"

[[Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"

[[PyCall]]
deps = ["Conda", "Dates", "Libdl", "LinearAlgebra", "MacroTools", "Serialization", "VersionParsing"]
git-tree-sha1 = "169bb8ea6b1b143c5cf57df6d34d022a7b60c6db"
uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0"
version = "1.92.3"

[[REPL]]
deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[Random]]
deps = ["Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[RecipesBase]]
git-tree-sha1 = "44a75aa7a527910ee3d1751d1f0e4148698add9e"
uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
version = "1.1.2"

[[SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"

[[Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

[[SharedArrays]]
deps = ["Distributed", "Mmap", "Random", "Serialization"]
uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"

[[Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

[[SparseArrays]]
deps = ["LinearAlgebra", "Random"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

[[SpecialFunctions]]
deps = ["ChainRulesCore", "IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"]
git-tree-sha1 = "793793f1df98e3d7d554b65a107e9c9a6399a6ed"
uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
version = "1.7.0"

[[Statistics]]
deps = ["LinearAlgebra", "SparseArrays"]
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

[[SymPy]]
deps = ["CommonEq", "CommonSolve", "LinearAlgebra", "Markdown", "PyCall", "RecipesBase", "SpecialFunctions"]
git-tree-sha1 = "1ef257ecbcab8058595a68ca36a6844b41babcbd"
uuid = "24249f21-da20-56a4-8eb1-6a02cf4ae2e6"
version = "1.0.52"

[[TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"

[[Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"

[[Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[[Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

[[VersionParsing]]
git-tree-sha1 = "80229be1f670524750d905f8fc8148e5a8c4537f"
uuid = "81def892-9a0e-5fdd-b105-ffc91e053289"
version = "1.2.0"

[[Zlib_jll]]
deps = ["Libdl"]
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"

[[nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"

[[p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
"""

# ╔═╡ Cell order:
# ╟─efc1cf20-8276-11eb-39ba-bd62a4342e68
# ╠═5bb6cf00-9b75-4e74-bc75-6a059bc1b32d
# ╠═d52c1d09-9bee-4359-8e2c-a969a2f32ea3
# ╠═8153dabd-5be6-4308-b7c5-1f4ed54722ab
# ╠═10b872ad-f070-404c-9ec8-19f0bc2032c5
# ╠═056c397f-7303-464e-9ec6-8cf893bbc98c
# ╠═b29b5f68-8978-4b4b-9b29-4840a4097978
# ╠═232dcad5-1984-4421-9c1a-001406096217
# ╠═a2819cd4-d0b3-476a-ab8c-40838db72d54
# ╠═74af42dd-9768-4e91-aa3b-b26c69d6fb68
# ╟─de024987-9c44-4199-9085-b259980e94a5
# ╠═62c5d76c-9eef-40a9-b8c9-446720733f80
# ╠═379bc6e9-7a38-4f3e-8e4a-2ff80917880b
# ╠═28dd8649-ed7c-47e5-b7ab-69e99dc0ce86
# ╠═12904f05-67c3-4d8a-aefd-e9645e6ef0eb
# ╠═bf1829c9-b2fb-4e3b-b92d-38a93513678b
# ╟─3732f5f7-ed44-4254-8b16-d9d860d2c04f
# ╠═232927bc-ac81-4225-801f-9da5e5842d6d
# ╠═13e53f50-8c7d-403c-85c5-848bf2ccbe94
# ╟─d9ada9db-2651-4895-876b-704e4ed05bcb
# ╠═f2eb749b-51d6-4875-8e14-1e0f688cdfc7
# ╟─244208a5-b153-424b-8305-5b6002fd8186
# ╠═78acc71b-b314-4799-b633-4ab5b7958496
# ╠═463c1ac7-e503-4507-8aca-d671acf04d36
# ╠═1b47139d-6c06-4227-91d2-1556d2f3a9d2
# ╠═646c21be-c589-43b6-b648-3d0dea8353db
# ╠═34320cf9-d978-46ca-ace2-ea638cc602d7
# ╠═6e0e6146-fefa-4967-938e-1755a0a04607
# ╠═a817a9d1-f9d2-4fb9-ad16-2477a0a6dea6
# ╟─00000000-0000-0000-0000-000000000001
# ╟─00000000-0000-0000-0000-000000000002


"""
    eom_logistic2(dx, x, p, n) -> function

Equations of motions for a system consisting of two coupled logistic maps where
X unidirectionally influences Y.

The parameter `c` controls how strong the dynamical forcing is. Parameters `r₁`
and `r₂` are set to the chaotic regime by default

# References
1. Diego, D., Agasøster Haaga, K., & Hannisdal, B. (2018, November 1).
Transfer entropy computation using the Perron-Frobenius operator.
Eprint ArXiv:1811.01677. Retrieved from
https://ui.adsabs.harvard.edu/#abs/2018arXiv181101677D
"""
function eom_logistic2(dx, x, p, n)
    c, r₁, r₂, σ = (p...,)
    ξ = rand() # random number from flat distribution on [0, 1]
    x, y = x[1], x[2]
    f_xy = (y +  (c*(x + σ*ξ)/2) ) / (1 + (c/2)*(1+σ))

    dx[1] = r₁ * x * (1 - x)
    dx[2] = r₂ * (f_xy) * (1 - f_xy)
    return
end

function logistic2(u₀, c, r₁, r₂, σ)
    p = [c, r₁, r₂, σ]
    DiscreteDynamicalSystem(eom_logistic2, u₀, p)
end

"""
    logistic2(;u₀ = rand(2), c = 0.1, σ = 0.05,
        r₁ = 3.78, r₂ = 3.66) -> DiscreteDynamicalSystem

Initialise a system consisting of two coupled logistic maps where X
unidirectionally influences Y. By default, the parameters `r₁` and `r₂` are set
to values yielding chaotic behaviour.

The parameter `c` controls how strong the dynamical forcing is. If `σ > 0`,
dynamical noise masking the influence of  `x` on `y` equivalent to
``\\sigma \\cdot \\xi`` is added at each iteration. Here,``\\xi`` is a draw from a
flat distribution on ``[0, 1]``. Thus, setting `σ = 0.05` is equivalent to
add dynamical noise corresponding to a maximum of ``5 \\%`` of the possible
range of values of the logistic map.

The equations of motion are

```math
\\begin{aligned}
dx &= r_1 x(1 - x) \\
dy &= r_2 f(x,y)(1 - f(x,y)),
\\end{aligned}
```
with
```math
\\begin{aligned}
f(x,y) = \\dfrac{y + \\frac{c(x \\xi )}{2}}{1 + \\frac{c}{2}(1+ \\sigma )}
\\end{aligned}
```

where

# References

1. Diego, D., Agasøster Haaga, K., & Hannisdal, B. (2018, November 1).
Transfer entropy computation using the Perron-Frobenius operator.
Eprint ArXiv:1811.01677. Retrieved from
https://ui.adsabs.harvard.edu/#abs/2018arXiv181101677D

"""
logistic2(;u₀ = rand(2), c = 0.1, r₁ = 3.78, r₂ = 3.66, σ = 0.05) =
    logistic2(u₀, c, r₁, r₂, σ)


#
#Simple example of a Poisson problem:
#
#           y
#             |
#             |           u_y = 0
#          Ly ----------------------------------
#             |                                |
#             |                                |
#             |                                |
#       u = 0 |    u - u_xx - u_yy = f         | u = 0
#             |                                |
#             |                                |
#             |                                |
#           -----------------------------------|-----  x
#                         u_y = 0               Lx
#
#Here,
#
#      u = sin(omegax x) * cos(omegay y)
#      f = ( 1 + omegax^2 + omegay^2 ) * sin(omegax x) * cos(omegay y)
#      omegax = nx pi / Lx
#      omegay = ny pi / Ly
#

using FinElt
using FinElt.PlanarPoisson
using Printf

include("params.jl")

const omegax = 3 * pi / Lx
const omegay = pi / Ly

function exact_u(x)
    return sin(omegax*x[1]) * cos(omegay*x[2])
end

function f(x)
    c = 1 + omegax^2 + omegay^2
    return c * exact_u(x)
end

ell_f(v, z) = P1SourceTimesFunc!(v, z, f)
lin_functionals = [ ("Omega", ell_f) ]

maxerr = zeros(refinements+1)
@printf("%10s  %12s  %8s  %8s\n\n",
        "N", "max error", "rate", "seconds")
for k = 0:refinements
    start = time()
    mesh = read_msh_file("rect$k.msh")
    vp = VariationalProblem(mesh, ["Left", "Right"])
    add_bilin_form!(vp, "Omega", grad_dot_grad!)
    add_bilin_form!(vp, "Omega", func_times_func!)
    add_lin_functnl!(vp, "Omega", source_times_func!, f)
    # Implicit zero boundary conditions.
    A, b = assembled_linear_system(vp)
    ufree = A \ b
    uh = complete_soln(ufree, vp)
    u = get_nodal_vals(exact_u, mesh)
    finish = time()
    maxerr[k+1] = maximum(abs.(uh-u))
    N = length(ufree)
    if k == 0
        @printf("%10d  %12.4e\n", N, maxerr[k+1])
    else
        rate = log2(maxerr[k]/maxerr[k+1])
        elapsed = finish - start
        @printf("%10d  %12.4e  %8.4f  %8.4f\n",
                N, maxerr[k+1], rate, elapsed)
    end
end


using LinearAlgebraicRepresentation
using SparseArrays
Lar = LinearAlgebraicRepresentation
L = Lar
using ViewerGL
GL = ViewerGL

function twocubegrids(n,m,p)
    V,(VV,EV,FV,CV) = Lar.cuboidGrid([n,m,p],true)
    mybox = (V,CV,FV,EV)

    twocubs = Lar.Struct([mybox, L.t(.3,.4,.5), L.r(pi/5,0,0), L.r(0,0,pi/12), mybox])

    V,CV,FV,EV = Lar.struct2lar(twocubs)
    GL.VIEW([ GL.GLGrid(V,FV, GL.COLORS[1]) ]);

    cop_EV = Lar.coboundary_0(EV::Lar.Cells);
    cop_EW = convert(Lar.ChainOp, cop_EV);
    cop_FE = Lar.coboundary_1(V, FV::Lar.Cells, EV::Lar.Cells);
    W = convert(Lar.Points, V');

    V, copEV, copFE, copCF = Lar.Arrangement.spatial_arrangement( W, cop_EW, cop_FE)

    EV = Lar.cop2lar(copEV)
    FE = [findnz(copFE[k,:])[1] for k=1:size(copFE,1)]
    FV = [collect(Set(cat(EV[e] for e in FE[f]))) for f=1:length(FE)]
    FV = convert(Lar.Cells, FV)
    W = convert(Lar.Points, V')
    WW = [[k] for k=1:size(W,2)]

    #GL.VIEW(GL.numbering(.2)((W,[WW,EV])));

    V,CVs,FVs,EVs = Lar.pols2tria(W, copEV, copFE, copCF)
    GL.VIEW(GL.GLExplode(V,FVs,1.5,1.5,1.5,99,1));
    GL.VIEW(GL.GLExplode(V,EVs,1.5,1.5,1.5,99,1));
    GL.VIEW(GL.GLExplode(V,CVs,5,5,5,99,1));

    return V, copEV, copFE, copCF
end

twocubegrids(5,5,5);


# Starting example file for Julia number types

# Julia defines a set of specific sub-types for numbers
# Signed integers: Int, Int8, Int16, Int32, Int64, BigInt
a = Int16(1)
println(typeof(a))

b = Int(2000)
println(typeof(b))

# Unsigned integers: UInt, UInt8, UInt16, UInt32, UInt64
c = UInt16(1)
println(typeof(c))

d = UInt(2000)
println(typeof(d))

# typemax() and typemin() will provide max and min values
println(typemax(Int8))
println(typemax(Int32))
println(typemax(Int64))
println(typemin(Int64))

# Use the WORDSIZE property to see what type of system this is
println(Sys.WORD_SIZE)

# Trying to assign a number too large for the type will fail and give an error
# CANNOT do: a = UInt(300)

# Special values represent Infinity and not-a-number
println(1.0 / 0.0)
println(1.0 / Inf)
println(0.0 / 0.0)

# zero() and one() functions produce values for a given type
println(zero(Float64))
println(one(UInt32))


# morlet wavelet
function morlet(F::T, fs::T; dtype=Float32) where {T}
    wtime = range(-1, stop=1, length=Int(2 * fs))
    s = 6 / ( 2 * pi * F)
    wavelet = exp.(2im * pi * F .* wtime) .* exp.(-wtime.^2 ./ (2*s^2));
    return wavelet |> Vector{Complex{dtype}}
end

"""
function dwt(data, fs, frange; reflection=false, wavelet=morlet)

Arguments:
- data
- fs
- frange

Keyword Arguments:
- reflection
- wavelet

Return result:

Note:
discrete wavelet transformation by convolution between the fft results of
original values and the wavelet function.
"""
function dwt(data::Vector{D}, fs::T, frange::Vector{T};
             reflection::Bool=false, wavelet::Function=morlet) where {D, T}
    data_fft = data
    if reflection
        data_flip = reverse(data_fft, dims=1)
        data_fft = [data_flip;data_fft;data_flip]
    end

    data_fft = data_fft |> Vector{Complex{Float32}}
    result = zeros(Complex{Float32}, size(data,1), length(frange))

    for (fidx, ftarget) in enumerate(frange)
        w = wavelet(ftarget, fs)
        if reflection
            result[:, fidx] = DSP.conv(data_fft, w)[Int(fs)+size(data,1):end-Int(fs)-size(data, 1)]
        else
            result[:, fidx] = DSP.conv(data_fft, w)[Int(fs):end-Int(fs)]
        end
    end

    result
end


mutable struct LinearRegressor
    _w::AbstractArray{Float64,1}
    _var::Float64
end

function fitting(
    regressor::LinearRegressor,
    Phi::AbstractArray{Float64,2},
    t::AbstractArray{Float64,1},
)
    # Phi is the design matrix of size (n_samples, n_feature)
    # t is the target vector of size (sample_size, 1)
    # `pinv(Phi)` = (Phi^T * Phi)^(-1) * Phi^T
    w = pinv(Phi) * t
    error = Phi * w - t
    var = sum(error .^ 2) / size(t)[1]

    regressor._w = w
    regressor._var = var
end

function predict(
    regressor::LinearRegressor,
    phi::AbstractArray{Float64,1},
    return_std::Bool,
)
    # phi is the transformed vector of size n_features
    y = transpose(phi) * regressor._w
    std = regressor._var

    if return_std == true
        return y, std
    else
        return y
    end
end

function predict(
    regressor::LinearRegressor,
    Phi::AbstractArray{Float64,2},
    return_std::Bool,
)
    # Phi is the design matrix [phi(x_1), phi(x_2), ,,, phi(x_N)]^T of size (n_samples, n_features)
    y = Phi * regressor._w
    std = regressor._var

    if return_std == true
        return y, std
    else
        return y
    end
end


export fac, stirlingNumber

caching_fac = [1]
function fac(n)
    if n == 0
        return 1
    elseif size(caching_fac, 1) >= n
        return caching_fac[n]
    end
    fac = caching_fac[size(caching_fac, 1)]
    for i = size(caching_fac, 1)+1:n
        append!(caching_fac, fac * i)
    end
    return caching_fac[n]
end

function stirlingNumber(m, k)
    if k > m || k < 0 || (m > 0 && k == 0)
        return 0
    elseif m == 0 && k == 0
        return 1
    else
        return k * stirlingNumber(m - 1, k) + stirlingNumber(m - 1, k - 1)
    end
end


Pkg.add("RDatasets")
Pkg.add("Query")
Pkg.add("StatPlots")
Pkg.add("GR")

using RDatasets, Plots, Query, DataFrames

iris = dataset("datasets", "iris")

function SplitDFbySpecies(iris, species)
    df = @from i in iris begin
         @where i.Species == species
         @select {i.PetalLength, i.PetalWidth}
         @collect DataFrame
    end
    return df
end

function AttributeDescription(df, name)
    println(name, ":\n")
    colnames = names(df)
    ncols = size(colnames)[1]
    for i in 1:ncols
        println(colnames[i], ":")
        println(" mean: ", mean(df[i]))
        println(" var:  ", var(df[i]))
    end
    println()
end

function dfmean(df::DataFrame)
    return [mean(i) for i in df.columns]
end

function dfcov(df)
    cols = df.columns
    ncols = size(cols)[1]
    ndatas = size(cols[1])[1]
    array = zeros(ncols, ncols)
    mean = dfmean(df)
    for i in 1:ncols
        for j in 1:ncols
            array[i, j] = 1/ndatas * (cols[i] .- mean[i])' * (cols[j] .- mean[j])
        end
    end
    return array
end

function scale(df, iris)
    ncols = length(df.columns)
    means = dfmean(iris[[:PetalLength, :PetalWidth]])
    cov = dfcov(iris[[:PetalLength, :PetalWidth]])
    sigmas = [sqrt(cov[i, i]) for i in 1:ncols]

    scaler(x, mu, sigma) = (x .- mu) / sigma
    array = zeros(size(df))
    for i in 1:ncols
        array[:, i] = scaler(df[i], means[i], sigmas[i])
    end
    return array
end

setosa = SplitDFbySpecies(iris, "setosa")
versicolor = SplitDFbySpecies(iris, "versicolor")
virginica = SplitDFbySpecies(iris, "virginica")

gr()
scatter(setosa[1], setosa[2]; xlabel="Petal Length", ylabel="Petal Width",
        xlims=(1, 7), ylims=(-1, 4), m=(0.5, :s), label="setosa")
scatter!(versicolor[1], versicolor[2]; m=(0.5, :c), label="versicolor")
scatter!(virginica[1], virginica[2]; m=(0.5, :v), label="virginica")
savefig("figures/unscaled.png")

setosa_scaled = scale(setosa, iris)
versicolor_scaled = scale(versicolor, iris)
virginica_scaled = scale(virginica, iris)

scatter(setosa_scaled[:, 1], setosa_scaled[:, 2]; xlabel="Petal Length", ylabel="Petal Width",
        xlims=(-1.5, 1.5), ylims=(-1.5, 1.5), m=(0.5, :s), label="setosa")
scatter!(versicolor_scaled[:, 1], versicolor_scaled[:, 2]; m=(0.5, :c), label="versicolor")
scatter!(virginica_scaled[:, 1], virginica_scaled[:, 2]; m=(0.5, :v), label="virginica")
savefig("figures/scaled.png")


function step_size(x0)
    h = sqrt.( eps.( abs.(x0) ) )
    return h
end

function jacobian_fw!(ode_fcn, x0, jac::Matrix, h_tmp, f_tmp)
    # f: R^n -> R^m
    # g_ij = df_i / dx_j

    h_tmp .= sqrt.( eps.( abs.(x0) ) ) # n-by-1
    return jacobian_fw!(ode_fcn, x0, h_tmp, jac, f_tmp)
end

function jacobian_fw!(ode_fcn, x0, h, jac::Matrix, f_tmp)
    # f: R^n -> R^m
    # g_ij = df_i / dx_j

    f0    = ode_fcn(x0)                  # m-by-1
    copyto!(f_tmp, f0)
    return jacobian_fw!(ode_fcn, x0, h, f_tmp, jac)   # m-by-n
end

function jacobian_fw!(ode_fcn, x0, h, f0, jac::Matrix)
    # f: R^n -> R^m
    # g_ij = df_i / dx_j

    x1 = copy(x0)
    for kk = 1:length(x0)
        x1[kk]     += h[kk]
        f1          = ode_fcn(x1)
        jac[:, kk]  = (f1 .- f0) ./ h[kk]
        x1[kk]     -= h[kk]
    end
    return jac
end

function time_numdiff_fw!(ode_fcn, t0, dfdt_tmp)
    h = sqrt(eps(abs(t0)))
    return time_numdiff_fw!(ode_fcn, t0, h, dfdt_tmp)
end

function time_numdiff_fw!(ode_fcn, t0, h, dfdt_tmp)
    f0 = ode_fcn(t0)
    return time_numdiff_fw!(ode_fcn, t0, h, f0, dfdt_tmp)
end

function time_numdiff_fw!(ode_fcn, t0, h, f0, dfdt_tmp)
    f1 = ode_fcn(t0 + h)
    dfdt_tmp .= (f1 .- f0) ./ h
    return dfdt_tmp
end


#Linear Kalman filter with control input example
#This example closely follows that given on "Greg Czerniak's Website". Namely
#the canonball example on this page: http://greg.czerniak.info/guides/kalman1/
#Basically we shoot a canonball into the air at a given angle of elevation.
#Using observations of the canonball's velocity and position we're going to
#track the position of the canonball.

#To use the Kalman Filter we need three things:
#1) Some observations
#2) The parameters of the Kalman Filter model describing the system process and
#the observation model. In this example we also need the control input
#parameters.
#3) An initial guess of the state.
#

#Let's import the modules required to execute the Kalman Filter
using StateSpace
using Distributions
using Gadfly
using Colors

################################################################################
#Section: Generate noisy Observations
#-------------------------------------------------------------------------------
#First we're going to provide some parameter values required to model the motion
#of the canonball.

#Set the Parameters
elevation_angle = 45.0 #Angle (measured anti-clockwise) between the ground (zero degrees) and the direction the canon ball is fired
muzzle_speed = 100.0 #Speed at which the canonball leaves the muzzle
initial_velocity = [muzzle_speed*cos(deg2rad(elevation_angle)), muzzle_speed*sin(deg2rad(elevation_angle))] #initial x and y components of the velocity
gravAcc = 9.81 #gravitational acceleration
initial_location = [0.0, 0.0] # initial position of the canonball
Δt = 0.1 #time between each measurement

#Functions describing the position of canonball
x_pos(x0::Float64, Vx::Float64, t::Float64) = x0 + Vx*t
y_pos(y0::Float64, Vy::Float64, t::Float64, g::Float64) = y0 + Vy*t - (g * t^2)/2
#Function to describe the evolution of the velocity in the vertical direction
velocityY(Vy::Float64, t::Float64, g::Float64) = Vy - g * t

#Give variances of the observation noise for the position and velocity
x_pos_var = 200.0
y_pos_var = 200.0
Vx_var = 1.0
Vy_var = 1.0

#Set the number of observations and preallocate vectors to store true and noisy
#measurement values
numObs = 145
x_pos_true = Vector{Float64}(numObs)
x_pos_obs = Vector{Float64}(numObs)
y_pos_true = Vector{Float64}(numObs)
y_pos_obs = Vector{Float64}(numObs)

Vx_true = Vector{Float64}(numObs)
Vx_obs = Vector{Float64}(numObs)
Vy_true = Vector{Float64}(numObs)
Vy_obs = Vector{Float64}(numObs)

#Generate the data (true values and noisy observations)
for i in 1:numObs
    x_pos_true[i] = x_pos(initial_location[1], initial_velocity[1], (i-1)*Δt)
    y_pos_true[i] = y_pos(initial_location[2], initial_velocity[2], (i-1)*Δt, gravAcc)
    Vx_true[i] = initial_velocity[1]
    Vy_true[i] = velocityY(initial_velocity[2], (i-1)*Δt, gravAcc)

    x_pos_obs[i] = x_pos_true[i] + randn() * sqrt(x_pos_var)
    y_pos_obs[i] = y_pos_true[i] + randn() * sqrt(y_pos_var)
    Vx_obs[i] = Vx_true[i] + randn() * sqrt(Vx_var)
    Vy_obs[i] = Vy_true[i] + randn() * sqrt(Vy_var)
end
#Create the observations vector for the Kalman filter
observations = [x_pos_obs Vx_obs y_pos_obs Vy_obs]'
#End Section: Generate noisy Observations
################################################################################

################################################################################
#Section: Describe Kalman Filter parameters
#-------------------------------------------------------------------------------

#Describe the system parameters
process_matrix = [[1.0, Δt, 0.0, 0.0] [0.0, 1.0, 0.0, 0.0] [0.0, 0.0, 1.0, Δt] [0.0, 0.0, 0.0, 1.0]]'
process_covariance = 0.01*eye(4)
observation_matrix = eye(4)
observation_covariance = 0.2*eye(4)
control_matrix = [[0.0, 0.0, 0.0, 0.0] [0.0, 0.0, 0.0, 0.0] [0.0, 0.0, 1.0, 0.0] [0.0, 0.0, 0.0, 1.0]]
control_input = [0.0, 0.0, -(gravAcc * Δt^2)/2, -(gravAcc * Δt)]

#Create an instance of the LKF with the control inputs
linCISMM = LinearGaussianCISSM(process_matrix, process_covariance, observation_matrix, observation_covariance, control_matrix, control_input)
#End Section: Describe Kalman Filter parameters
################################################################################

################################################################################
#Section: Set Initial Guess
#-------------------------------------------------------------------------------
initial_guess_state = [0.0, initial_velocity[1], 500.0, initial_velocity[2]]
initial_guess_covariance = eye(4)
initial_guess = MvNormal(initial_guess_state, initial_guess_covariance)
#End Section: Set Initial Guess
################################################################################

################################################################################
#Section: Execute Kalman Filter
#-------------------------------------------------------------------------------
filtered_state = filter(linCISMM, observations, initial_guess)
#End Section: Execute Kalman Filter
################################################################################

################################################################################
#Section: Plot Filtered results
#-------------------------------------------------------------------------------
#Here we are plotting the filtered results with Gadfly. See the Gadfly
#documentation for information about how plotting works if you are unfamiliar.
#Website: http://gadflyjl.org/
#Plot results
x_filt = Vector{Float64}(numObs)
y_filt = Vector{Float64}(numObs)
for i in 1:numObs
    current_state = filtered_state.state[i]
    x_filt[i] = current_state.μ[1]
    y_filt[i] = current_state.μ[3]
end

n = 3
getColors = distinguishable_colors(n, Color[LCHab(70, 60, 240)],
                                   transform=c -> deuteranopic(c, 0.5),
                                   lchoices=Float64[65, 70, 75, 80],
                                   cchoices=Float64[0, 50, 60, 70],
                                   hchoices=linspace(0, 330, 24))

cannonball_plot = plot(
    layer(x=x_pos_true, y=y_pos_true, Geom.line, Theme(default_color=getColors[3])),
    layer(x=[initial_guess_state[1]; x_filt], y=[initial_guess_state[3]; y_filt], Geom.line, Theme(default_color=getColors[1])),
    layer(x=x_pos_obs, y=y_pos_obs, Geom.point, Theme(default_color=getColors[2])),
    Guide.xlabel("X position"), Guide.ylabel("Y position"),
    Guide.manual_color_key("Colour Key",["Filtered Estimate", "Measurements","True Value "],[getColors[1],getColors[2],getColors[3]]),
    Guide.title("Measurement of a Canonball in Flight")
    )
#End Section: Plot Filtered results
################################################################################


"""
    jensen_alpha(asset_returns, benchmark_returns; risk_free=0.0)

Jensen's alpha, or simply alpha, is a risk-adjusted excess-performance measure. It indicates the average return of an investment above or below that predicted by the capital asset pricing model (CAPM). The measure adjusts the excess returns such that the risk is identical for the investment and benchmark.

# Arguments
- `asset_returns`:      Vector of asset returns.
- `benchmark_returns`:  Vector of benchmark returns (e.g. market portfolio returns for CAPM beta).
- `risk_free`:          Optional vector or scalar value denoting the risk-free return (must have same frequency as the provided returns, e.g. daily).

# Returns
Jensen's alpha measure.

# Sources
- Bacon, Carl (2008). Practical Portfolio Performance Measurement and Attribution, 2nd Edition, John Wiley & Sons Ltd. Page 72.
"""
function jensen_alpha(asset_returns, benchmark_returns; risk_free=0.0)
    (α, β) = capm(asset_returns, benchmark_returns; risk_free)
    α
end

"""
    modified_jensen(asset_returns, benchmark_returns; risk_free=0.0)

Divides Jensen’s alpha by the systematic risk, which measures the systematic risk-adjusted return per unit of systematic risk. See also `jensen_alpha`

# Arguments
- `asset_returns`:      Vector of asset returns.
- `benchmark_returns`:  Vector of benchmark returns (e.g. market portfolio returns for CAPM beta).
- `risk_free`:          Optional vector or scalar value denoting the risk-free return (must have same frequency as the provided returns, e.g. daily).

# Returns
Modified Jensen's alpha measure.

# Sources
- Bacon, Carl (2008). Practical Portfolio Performance Measurement and Attribution, 2nd Edition, John Wiley & Sons Ltd. Page 77.
"""
function modified_jensen(asset_returns, benchmark_returns; risk_free=0.0)
    (α, β) = capm(asset_returns, benchmark_returns; risk_free)
    α / β
end


# =========================================================================== #
# Activation Functions
# =========================================================================== #

abstract type AbstractActivationFunction
end

struct TanhActivationFunction <: AbstractActivationFunction
end

function (act_fcn::TanhActivationFunction)(x)
    return tanh(x)
end

function differentiate(act_fcn::TanhActivationFunction, x)
    return one(typeof(x)) - act_fcn(x)^2
end

function eval(act_fcn::TanhActivationFunction, x)
    return act_fcn(x)
end

struct ReluActivationFunction <: AbstractActivationFunction
end

function (act_fcn::ReluActivationFunction)(x)
    T = typeof(x)
    z = zero(T)
    return (x < z) * z + (x >= z) * x
end

function differentiate(act_fcn::ReluActivationFunction, x)
    T = typeof(x)
    z = zero(T)
    return (x < z) * z + (x >= z) * one(T)
end

function eval(act_fcn::ReluActivationFunction, x)
    return act_fcn(x)
end

struct IdentityActivationFunction <: AbstractActivationFunction
end

function (act_fcn::IdentityActivationFunction)(x)
    return x
end

function differentiate(act_fcn::IdentityActivationFunction, x)
    return one(typeof(x))
end

function eval(act_fcn::IdentityActivationFunction, x)
    return act_fcn(x)
end


"""
Return the rotation angle between two
vectors of any number of dimensions.

Note that if at least one of the vector is
the zero vector, then it returns 0.
by convention.

Arguments
----------
:Type{RotationAngle}
    must be type ``RotationAngle``

vector_1:AbstractArray
    vector 1

vector_2:AbstractArray
    vector 2

tol_near_zero=1e-7:AbstractFloat
    tolerance for being close to zero
"""
function gage(::Type{RotationAngle}, vector_1::AbstractArray, vector_2::AbstractArray,
    tol_near_zero::AbstractFloat=1e-7)
    @debug @assert length(vector_1) == length(vector_2)

    if nearly_zero(norm(vector_1,2), tol_near_zero)
        return 0.
    elseif nearly_zero(norm(vector_2,2), tol_near_zero)
        return 0.
    else
        #= developer's note: (Yuhang Wang 04/17/2016)
            Sometimes, the normalized dot product is > 1 or < -1 due to
            error in floating point division.
            Without proper handling, Julia will report domain error.
        =#
        normalized_dot = dot(vector_1/norm(vector_1,2), vector_2/norm(vector_2,2))
        if normalized_dot > 1.0
            return acos(floor(normalized_dot))
        elseif normalized_dot < -1.0
            return acos(ceil(normalized_dot))
        else
            return acos(normalized_dot)
        end
    end
end


# given a simplex and a face returns:
# +v if face is the v-th face of the simplex oriented according to the simplex
# -v if face is the v-th face of the simplex oriented oppositely to the simplex
# 0 is face is not a face of the simplex
function relorientation(face, simplex)

    v = setdiff(simplex, face)
    length(v) == 1 || return 0

    # find the position of the missing vertex
    v = v[1]
    i = Base.findfirst(simplex, v)
    s = (-1)^(i-1)

    # remove that vertex from the simplex
    face2 = Array{Int}(length(simplex)-1)
    for j in 1 : i-1
        face2[j] = simplex[j]
    end
    for j in i : length(simplex)-1
        face2[j] = simplex[j+1]
    end

    # get the permutation that maps face to face2
    #p = indexin(face, face2)
    p = [ findfirst(face2,v) for v in face ]

    return s * levicivita(p) * i
end

"""
    getcommonedge(cell1, cell2) -> e1, e2, edge

Returns in edge the common vertices of cell1 and cell2. e1 contains the index
of the vertex of cell1 opposite to this common edge, and with a plus or minus
sign depending on whether the orientation of the common edge is along or
against the internal orientation of cell1. Similar for e2.
"""
function getcommonedge(cell1, cell2)
    isct = intersect(cell1, cell2)
    relorientation(isct, cell1), relorientation(isct, cell2), isct
end