_tri(t, period) = (2 / π) * abs(asin(sin(π * (t - 1) / period)))
_sin(t, period) = abs(sin(π * (t - 1) / period))
_cycle(λ0, λ1, g) = abs(λ0 - λ1) * g + min(λ0, λ1)

"""
    Triangle{T, S<:Integer}(range0, range1, period)
    Triangle(;λ0, λ1, period)

A [triangle wave](https://en.wikipedia.org/wiki/Triangle_wave) schedule
with `period`.
The output conforms to
```text
abs(λ0 - λ1) * (2 / π) * abs(asin(sin(π * (t - 1) / period))) + min(λ0, λ1)
```

# Arguments
- `range0`/`λ0`: the first range endpoint
- `range1`/`λ1`: the second range endpoint
- `period::Integer`: the period
"""
struct Triangle{T, S<:Integer}
    range0::T
    range1::T
    period::S
end
Triangle(;λ0, λ1, period) = Triangle(λ0, λ1, period)

(schedule::Triangle)(t) = _cycle(schedule.range0, schedule.range1, _tri(t, schedule.period))

Base.eltype(::Type{<:Triangle{T}}) where T = T
Base.IteratorSize(::Type{<:Triangle}) = Base.IsInfinite()

Base.iterate(schedule::Triangle, t = 1) = schedule(t), t + 1

Base.axes(::Triangle) = (OneToInf(),)

"""
    TriangleDecay2{T, S<:Integer}(range0, range1, period)
    TriangleDecay2(;λ0, λ1, period)

A [triangle wave](https://en.wikipedia.org/wiki/Triangle_wave) schedule
with `period` and half the amplitude each cycle.
The output conforms to
```text
abs(λ0 - λ1) * Triangle(t) / (2^floor((t - 1) / period)) + min(λ0, λ1)
```
where `Triangle(t)` is `(2 / π) * abs(asin(sin(π * (t - 1) / schedule.period)))` (see [`Triangle`](#)).

# Arguments
- `range0`/`λ0`: the first range endpoint
- `range1`/`λ1`: the second range endpoint
- `period::Integer`: the period
"""
struct TriangleDecay2{T, S<:Integer}
    range0::T
    range1::T
    period::S
end
TriangleDecay2(;λ0, λ1, period) = TriangleDecay2(λ0, λ1, period)

(schedule::TriangleDecay2)(t) = _cycle(schedule.range0, schedule.range1,
                                       _tri(t, schedule.period) / (2^fld(t - 1, schedule.period)))

Base.eltype(::Type{<:TriangleDecay2{T}}) where T = T
Base.IteratorSize(::Type{<:TriangleDecay2}) = Base.IsInfinite()

Base.iterate(schedule::TriangleDecay2, t = 1) = schedule(t), t + 1

Base.axes(::TriangleDecay2) = (OneToInf(),)

"""
    TriangleExp{T, S<:Integer}(range0, range1, period, decay)
    TriangleExp(λ0, λ1, period, γ)
    TriangleExp(;λ0, λ1, period, γ)

A [triangle wave](https://en.wikipedia.org/wiki/Triangle_wave) schedule
with `period` and an exponentially decaying amplitude.
The output conforms to
```text
abs(λ0 - λ1) * Triangle(t) * γ^(t - 1) + min(λ0, λ1)
```
where `Triangle(t)` is `(2 / π) * abs(asin(sin(π * (t - 1) / schedule.period)))` (see [`Triangle`](#)).

# Arguments
- `range0`/`λ0`: the first range endpoint
- `range1`/`λ1`: the second range endpoint
- `period::Integer`: the period
- `decay`/`γ`: the decay rate
"""
struct TriangleExp{T, S<:Integer}
    range0::T
    range1::T
    period::S
    decay::T
end
TriangleExp(;λ0, λ1, period, γ) = TriangleExp(λ0, λ1, period, γ)

startvalue(schedule::TriangleExp) = schedule.tri.λ0
endvalue(schedule::TriangleExp) = schedule.tri.λ1
(schedule::TriangleExp)(t) = _cycle(schedule.range0, schedule.range1,
                                    _tri(t, schedule.period) * schedule.decay^(t - 1))

Base.eltype(::Type{<:TriangleExp{T}}) where T = T
Base.IteratorSize(::Type{<:TriangleExp}) = Base.IsInfinite()

Base.iterate(schedule::TriangleExp, t = 1) = schedule(t), t + 1

Base.axes(::TriangleExp) = (OneToInf(),)

"""
    Sin{T, S<:Integer}(range0, range1, period)
    Sin(;λ0, λ1, period)

A sine wave schedule with `period`.
The output conforms to
```text
abs(λ0 - λ1) * abs(sin(π * (t - 1) / period)) + min(λ0, λ1)
```

# Arguments
- `range0`/`λ0`: the first range endpoint
- `range1`/`λ1`: the second range endpoint
- `period::Integer`: the period
"""
struct Sin{T, S<:Integer}
    range0::T
    range1::T
    period::S
end
Sin(;λ0, λ1, period) = Sin(λ0, λ1, period)

(schedule::Sin)(t) = _cycle(schedule.range0, schedule.range1, _sin(t, schedule.period))

Base.eltype(::Type{<:Sin{T}}) where T = T
Base.IteratorSize(::Type{<:Sin}) = Base.IsInfinite()

Base.iterate(schedule::Sin, t = 1) = schedule(t), t + 1

Base.axes(::Sin) = (OneToInf(),)

"""
    SinDecay2{T, S<:Integer}(range0, range1, period)
    SinDecay2(;λ0, λ1, period)

A sine wave schedule with `period` and half the amplitude each cycle.
The output conforms to
```text
abs(λ0 - λ1) * Sin(t) / (2^floor((t - 1) / period)) + min(λ0, λ1)
```
where `Sin(t)` is `abs(sin(π * (t - 1) / period))` (see [`Sin`](#)).

# Arguments
- `range0`/`λ0`: the first range endpoint
- `range1`/`λ1`: the second range endpoint
- `period::Integer`: the period
"""
struct SinDecay2{T, S<:Integer}
    range0::T
    range1::T
    period::S
end
SinDecay2(;λ0, λ1, period) = SinDecay2(λ0, λ1, period)

(schedule::SinDecay2)(t) = _cycle(schedule.range0, schedule.range1,
                                  _sin(t, schedule.period) / (2^fld(t - 1, schedule.period)))

Base.eltype(::Type{<:SinDecay2{T}}) where T = T
Base.IteratorSize(::Type{<:SinDecay2}) = Base.IsInfinite()

Base.iterate(schedule::SinDecay2, t = 1) = schedule(t), t + 1

Base.axes(::SinDecay2) = (OneToInf(),)

"""
    SinExp{T, S<:Integer}(range0, range1, period, decay)
    SinDecay2(;λ0, λ1, period, γ)

A sine wave schedule with `period` and an exponentially decaying amplitude.
The output conforms to
```text
abs(λ0 - λ1) * Sin(t) * γ^(t - 1) + min(λ0, λ1)
```
where `Sin(t)` is `abs(sin(π * (t - 1) / period))` (see [`Sin`](#)).

# Arguments
- `range0`/`λ0`: the first range endpoint
- `range1`/`λ1`: the second range endpoint
- `period::Integer`: the period
- `decay`/`γ`: the decay rate
"""
struct SinExp{T, S<:Integer}
    range0::T
    range1::T
    period::S
    decay::T
end
SinExp(;λ0, λ1, period, γ) = SinExp(λ0, λ1, period, γ)

(schedule::SinExp)(t) = _cycle(schedule.range0, schedule.range1,
                               _sin(t, schedule.period) * schedule.decay^(t - 1))

Base.eltype(::Type{<:SinExp{T}}) where T = T
Base.IteratorSize(::Type{<:SinExp}) = Base.IsInfinite()

Base.iterate(schedule::SinExp, t = 1) = schedule(t), t + 1

Base.axes(::SinExp) = (OneToInf(),)

"""
    Cos{T, S<:Integer}(range0, range1, period)
    Cos(;λ0, λ1, period)

A cosine annealing schedule
(see ["SGDR: Stochastic Gradient Descent with Warm Restarts"](https://arxiv.org/abs/1608.03983v5))
The output conforms to
```text
abs(λ0 - λ1) * (1 + cos(π * mod(t - 1, period) / period)) / 2 + min(λ0, λ1)
```
This schedule is also referred to as "cosine annealing with warm restarts"
in machine learning literature.

# Arguments
- `range0`/`λ0`: the first range endpoint
- `range1`/`λ1`: the second range endpoint
- `period::Integer`: the period
"""
struct Cos{T, S<:Integer}
    range0::T
    range1::T
    period::S
end
Cos(;λ0, λ1, period) = Cos(λ0, λ1, period)

(schedule::Cos)(t) = _cycle(schedule.range0, schedule.range1,
                            (1 + cos(π * mod(t - 1, schedule.period) / schedule.period)) / 2)

Base.eltype(::Type{<:Cos{T}}) where T = T
Base.IteratorSize(::Type{<:Cos}) = Base.IsInfinite()

Base.iterate(schedule::Cos, t = 1) = schedule(t), t + 1

Base.axes(::Cos) = (OneToInf(),)


export dnorm, dgumbel, dgamma, dlognorm

"""
    dnorm(x::Real, μ::Real, σ::Real)

Normal distribution density function
"""
dnorm(x::Real, μ::Real, σ::Real) = exp(-0.5 * ((x - μ)/σ)^2)/(σ*sqrt2π)

"""
    dgumbel(x::Real, μ::Real, σ::Real)

Gumbel density, parameterized by mean (μ) and SD (σ) of x.
"""
function dgumbel(x::Real, μ::Real, σ::Real)
    β = (√6 * σ) / π
    α = μ - (β*γ)
    z = (x - α) / β
    exp(-(z + exp(-z)))
end

"""
    dgamma(x::Real, μ::Real, σ::Real)

Gamma density, parameterized by mean (μ) and sigma(σ) of x.
"""
function dgamma(x::Real, μ::Real, σ::Real)
    α = μ^2 / σ^2
    β = μ / σ^2
    (x^(α - 1) * exp(-β * x))  * (β^α / Γ(α))
end

"""
    dlognorm(x::Real, μ::Real, σ::Real)

Lognormal density, parameterized by mean (μ) and SD (σ) of x, NOT of log(x).
"""
function dlognorm(x::Real, μ::Real, σ::Real)
    lμ = log(μ^2 / sqrt(1 + σ^2 / μ^2))
    lσ = √(log(1 + σ^2 / μ^2))
    dnorm(log(x), lμ, lσ) / x
end


###################################
## Simple VI & GS for 2D Gaussian

using PyPlot
using Distributions

function calc_KL(mu1, lambda1, mu2, lambda2)
    D = size(mu1, 1)
    px_lnqx = 0.5 * logdet(lambda2) - 0.5 * ((mu1 - mu2)' * lambda2 * (mu1 - mu2) + trace(lambda2 * inv(lambda1)))
    px_lnpx = 0.5 * logdet(lambda1) - 0.5 * D
    KL = - (px_lnqx - px_lnpx)
    return KL[1]
end

function plot_results(result, truth)
    N = size(result, 1)
    H = Int(ceil(sqrt(N)))
    W = Int(ceil(N / H))
    for i in 1 : H
        for j in 1 : W
            n = (i - 1) * W + j
            if n <= N
                subplot(H, W, n)
                title("$n of $N")
                plot_gaussian(truth[1], truth[2], "b", "\$p(z)\$")
                plot_gaussian(result[n][1], result[n][2], "r", "\$p(z)\$")
            end
        end
    end
end

function plot_lines(X)
    D, N = size(X)
    X_d = zeros(D, 2*N + 1)
    X_d[:,1] = X[:,1]
    for i in 1 : N
        X_d[1, 2*i - 1] = X[1, i]
        X_d[1, 2*i] = X[1, i]
        X_d[2, 2*i] = X[2, i]
        X_d[2, 2*i + 1] = X[2, i]
    end
    plot(X[1,:], X[2,:], "oy")
    plot(X_d[1,1:2*N], X_d[2,1:2*N], "--y")
end

function plot_gaussian(Mu, Sigma, col, label)
    res = 100
    plot(Mu[1], Mu[2], "x", color=col)

    F = eigfact(Sigma)
    vec = F.vectors
    val = F.values
    dw = 2*pi/res
    w = dw * (0 : res)

    c = 1.0
    a = sqrt(c*val[1])
    b = sqrt(c*val[2])
    P1 = a*cos.(w)
    P2 = b*sin.(w)
    P = Mu .+ vec'*vcat(P1', P2')
    plot(P[1, :], P[2, :], "-", color=col, label=label)
end

"""
Variational inference for 2D Gauss.
"""
function main_VI()
    ## creat truth distribution
    D = 2 # dimension
    theta = 2.0*pi/12 # tilt
    A = reshape([cos.(theta), -sin.(theta),
                 sin.(theta), cos.(theta)],
                2, 2)
    mu = [0.0, 0.0]
    lambda = inv(A * inv(reshape([1,0,0,10], 2, 2)) * A')

    ## initialize
    #mu_h = randn(D)
    mu_h = [-0.5, 0.3]
    lambda_h = zeros(D,D)

    ## main iteration
    max_iter = 10
    KL = NaN * Array{Float64, 1}(max_iter)
    result = Array{Any, 1}(max_iter)
    for i in 1 : max_iter
        ## update
        mu_h[1] = mu[1] - inv(lambda[1,1])*lambda[1,2] * (mu_h[2] - mu[2])

        lambda_h[1,1] = lambda[1,1]
        mu_h[2] = mu[2] - inv(lambda[2,2])*lambda[2,1] * (mu_h[1] - mu[1])
        lambda_h[2,2] = lambda[2,2]

        ## calculate KL divergeince
        KL[i] = calc_KL(mu_h, lambda_h, mu, lambda)

        ## store the results
        result[i] = [deepcopy(mu_h), deepcopy(inv(lambda_h))]
    end

    ## visualize results
    figure("result per iteration (VI)")
    clf()
    plot_results(result, (mu, inv(lambda)))

    figure("result (VI)")
    clf()
    plot_gaussian(mu, inv(lambda), "b", "\$p(\\bf{z})\$")
    plot_gaussian(result[end][1], result[end][2], "r", "\$q(\\bf{z})\$")
    xlabel("\$z_1\$", fontsize=20)
    ylabel("\$z_2\$", fontsize=20)
    legend(fontsize=16)

    figure("KL divergence (VI)")
    clf()
    plot(1:max_iter, KL)
    ylabel("KL divergence", fontsize=16)
    xlabel("iteration", fontsize=16)
    show()
end

"""
Gibbs sampling for 2D Gauss.
"""
function main_GS()
    ## creat truth distribution
    D = 2 # dimension
    theta = 2.0*pi/12 # tilt
    A = reshape([cos.(theta), -sin.(theta),
                 sin.(theta), cos.(theta)],
                2, 2)
    mu = [0.0, 0.0]
    #lambda = inv(A * inv(reshape([1,0,0,10], 2, 2)) * A')
    lambda = inv(A * inv(reshape([1,0,0,100], 2, 2)) * A')

    ## initialize
    #max_iter = 1000
    max_iter = 50
    X = randn(D, max_iter)
    mu_h = randn(D)

    ## main iteration
    KL = NaN * Array{Float64, 1}(max_iter)
    for i in 2 : max_iter
        ## update
        mu_h[1] = mu[1] - inv(lambda[1,1])*lambda[1,2] * (X[2,i-1] - mu[2])
        X[1, i] = rand(Normal(mu_h[1], sqrt(inv(lambda[1,1]))))

        mu_h[2] = mu[2] - inv(lambda[2,2])*lambda[2,1] * (X[1,i] - mu[1])
        X[2, i] = rand(Normal(mu_h[2], sqrt(inv(lambda[2,2]))))

        if i > D
            KL[i] = calc_KL(mean(X[:,1:i], 2), inv(cov(X[:,1:i], 2)), mu, lambda)
        end
    end

    ## visualize results
    expt_mu = mean(X, 2)
    expt_Sigma = cov(X, 2)

    figure("samples (GS)")
    clf()
    plot_lines(X)
    plot_gaussian(mu, inv(lambda), "b", "\$p(\\bf{z})\$")
    plot_gaussian(expt_mu, expt_Sigma, "r", "\$q(\\bf{z})\$")
    xlabel("\$z_1\$", fontsize=20)
    ylabel("\$z_2\$", fontsize=20)
    legend(fontsize=16)

    figure("KL divergence (GS)")
    clf()
    plot(1:max_iter, KL)
    ylabel("KL divergence", fontsize=16)
    xlabel("sample size", fontsize=16)
    show()
end

main_VI()
main_GS()


using FractionalTransforms
using Test

@testset "Test FRFT" begin
    @test isapprox(real(frft([1,2,3], 0.5)), [0.100464613588213, 3.07460010423315, 1.31566432887283], atol=1e-5)
    @test isapprox(imag(frft([1,2,3], 0.5)), [-0.259409480977277, 0.199349381540632, -0.93645356561191], atol=1e-5)
end

@testset "Test FRFT auxillary functions" begin
    @testset "Test Sinc Interpolation" begin
        @test isapprox(real(sinc_interp([1, 2, 3], 3)), [1.0, 1.1577906803857638, 1.4472383504822042, 2.0, 2.6877283651812367, 3.1425747039042147, 3.0]; atol=1e-5)
        @test isapprox(imag(sinc_interp([1, 2, 3], 3)), [-4.0696311822644287e-17, -2.4671622769447922e-17, -2.522336560207922e-16, -3.331855648569948e-17, 0.0, 1.9624582529744518e-17, 3.331855648569948e-17]; atol=1e-5)
    end

    @testset "Test Frequency Shear" begin
        @test isapprox(real(freq_shear([1, 2, 3], 3)), [0.0707372016677029, 2.0, 0.2122116050031087]; atol=1e-5)
        @test isapprox(imag(freq_shear([1, 2, 3], 3)), [0.9974949866040544, 0.0, 2.9924849598121632]; atol=1e-5)
    end
end


using DiffEqFlux, Flux, OrdinaryDiffEq, Test, Optim, DiffEqSensitivity

x = Float32[0.8; 0.8]
tspan = (0.0f0,10.0f0)

ann = Chain(Dense(2,10,tanh), Dense(10,1))
p = Float32[-2.0,1.1]
p2,re = Flux.destructure(ann)
_p = [p;p2]
θ = [x;_p]

function dudt2_(u,p,t)
    x, y = u
    [(re(p[3:end])(u)[1]),p[1]*y + p[2]*x]
end

prob = ODEProblem(dudt2_,x,tspan,_p)
solve(prob,Tsit5())

function predict_rd(θ)
  Array(solve(prob,Tsit5(),u0=θ[1:2],p=θ[3:end],abstol=1e-7,reltol=1e-5,sensealg=TrackerAdjoint()))
end
loss_rd(p) = sum(abs2,x-1 for x in predict_rd(p))
l = loss_rd(θ)

cb = function (θ,l)
  println(l)
  #display(plot(solve(remake(prob,u0=Flux.data(_x),p=Flux.data(p)),Tsit5(),saveat=0.1),ylim=(0,6)))
  false
end

# Display the ODE with the current parameter values.
cb(θ,l)

loss1 = loss_rd(θ)
res = DiffEqFlux.sciml_train(loss_rd, θ, BFGS(initial_stepnorm = 0.01), cb = cb)
loss2 = res.minimum
@test 10loss2 < loss1

## Partial Neural Adjoint

u0 = Float32[0.8; 0.8]
tspan = (0.0f0,25.0f0)

ann = Chain(Dense(2,10,tanh), Dense(10,1))

p1,re = Flux.destructure(ann)
p2 = Float32[-2.0,1.1]
p3 = [p1;p2]
θ = [u0;p3]

function dudt_(du,u,p,t)
    x, y = u
    du[1] = re(p[1:41])(u)[1]
    du[2] = p[end-1]*y + p[end]*x
end
prob = ODEProblem(dudt_,u0,tspan,p3)
solve(prob,Tsit5(),abstol=1e-8,reltol=1e-6)

function predict_adjoint(θ)
  Array(solve(prob,Tsit5(),u0=θ[1:2],p=θ[3:end],saveat=0.0:1:25.0))
end
loss_adjoint(θ) = sum(abs2,x-1 for x in predict_adjoint(θ))
l = loss_adjoint(θ)

cb = function (θ,l)
  println(l)
  #display(plot(solve(remake(prob,p=Flux.data(p3),u0=Flux.data(u0)),Tsit5(),saveat=0.1),ylim=(0,6)))
  false
end

# Display the ODE with the current parameter values.
cb(θ,l)

loss1 = loss_adjoint(θ)
res1 = DiffEqFlux.sciml_train(loss_adjoint, θ, ADAM(0.01), cb = cb, maxiters = 100)
res = DiffEqFlux.sciml_train(loss_adjoint, res1.minimizer, BFGS(initial_stepnorm = 0.01), cb = cb)
loss2 = res.minimum
@test 10loss2 < loss1


using DEMC
using Random
using LinearAlgebra
using Distributions
Random.seed!(31953150)

# generate data
nobs = 100
npar = 3
Σ = ones(npar, npar)
for i = 1:npar
    for iprime = 1:i
        if i != iprime
            Σ[i, iprime] = 0.25
            Σ[iprime, i] = 0.25
        end
    end
end

DataDistribution = MvNormal(zeros(npar), Σ)
X = zeros(nobs, npar+1)
X[:,1] .= 1.
for iobs = 1:nobs
    X[iobs, 2:end] .= rand(DataDistribution)
end

β = rand(npar+1)
y = X*β + randn(nobs)

# log objective function - for correct std errors need to use optimal weights
log_obj(b) = -0.5 * sum((y.-X*b).^2)

# set up of DEMCz chain
Npar = length(β)
ndim = Npar
blockindex = [1:Npar] # parameter blocks: here choose all parameters to be updated simultaenously
Nblocks = length(blockindex)
eps_scale = 1e-5*ones(Npar) # scale of random error around DE update
γ = 2.38 # scale of DE update, 2.38 is the "optimal" number for a normal distribution
N = 3
K = 10
Z = randn((10*ndim, ndim))

# Number of iterations in Chain
Ngen = 2500
mc, Z = DEMC.demcz_sample(log_obj, Z, N, K, Ngen, Nblocks, blockindex, eps_scale, γ; verbose =false)

# drop first half of chain
keep = Int(round(Ngen/2))+1:Ngen
Ngen_burned = length(keep)
chain_burned = mc.chain[:,:,keep]
chainflat = DEMC.flatten_chain(chain_burned, N, Ngen_burned, Npar)'
bhat = mean(chainflat,dims=1)[:]
println("\n estimates: ", bhat, "\n dist to true: ", bhat - β)
# covariance of estimates
b, Σb = DEMC.mean_cov_chain(chain_burned, N, Ngen_burned, Npar)
# correct std errors for incorrect weighting matrix

# OLS
bols = (X'*X)\(X'*y)
ui = y .- X*bols
s2hat = (1/(nobs-npar)) * sum(ui.^2)
Σols = inv(X'X)*s2hat


# Firms' parameters
const A = 1
const N = 1
const α = 0.33
const β = 0.96
const δ = 0.05

"""
Compute wage rate given an interest rate, r
"""
function r_to_w(r::Real)
    return A * (1 - α) * (A * α / (r + δ)) ^ (α / (1 - α))
end

"""
Inverse demand curve for capital. The interest rate
associated with a given demand for capital K.
"""
function rd(K::Real)
    return A * α * (N / K) ^ (1 - α) - δ
end

"""
Map prices to the induced level of capital stock.

##### Arguments
- `am::Household` : Household instance for problem we want to solve
- `r::Real` : interest rate

##### Returns
- The implied level of aggregate capital
"""
function prices_to_capital_stock(am::Household, r::Real)

    # Set up problem
    w = r_to_w(r)
    setup_R!(am, r, w)
    aiyagari_ddp = DiscreteDP(am.R, am.Q, am.β)

    # Compute the optimal policy
    results = solve(aiyagari_ddp, PFI)

    # Compute the stationary distribution
    stationary_probs = stationary_distributions(results.mc)[:, 1][1]

    # Return K
    return dot(am.s_vals[:, 1], stationary_probs)
end

# Create an instance of Household
am = Household(β=β, a_max=20.0)

# Create a grid of r values at which to compute demand and supply of capital
num_points = 20
r_vals = linspace(0.005, 0.04, num_points)

# Compute supply of capital
k_vals = prices_to_capital_stock.(am, r_vals)

# Plot against demand for capital by firms
demand = rd.(k_vals)
labels =  ["demand for capital" "supply of capital"]
plot(k_vals, [demand r_vals], label=labels, lw=2, alpha=0.6)
plot!(xlabel="capital", ylabel="interest rate", xlim=(2, 14), ylim=(0.0, 0.1))


import Statistics:mean

function ParetoSub(X, perm)
    Z= X.-X[:,1]
    nondominated=reshape(any(Z.<0,dims=1),:)
    ispareto = all(any(Z[:,nondominated].>0, dims=1))
    X=X[:, nondominated]
    nondominated = perm[nondominated]
    return ispareto, nondominated, X
end

"""
    index = ParetoSet(X[, sense])
Get the Pareto set from a given set of oberservations `X`.
# Argument
- `X::AbstractMatrix{Real}` size(X)=(n_objectives, n_samples)
- `sense::AbstractVector{Union{Symbol, EGOSense}}=repeat(:Min, n_objectives)` sense of each objective, size(S)=(n_objectives,)
"""
function ParetoSet(X::AbstractMatrix)
    n,m = size(X)
    Xmin = minimum(X, dims=2)
    Xt = X.-Xmin
    Xm = mean(Xt, dims=2)
    Xn = maximum(Xt ./ (Xm.+maximum(Xm)), dims=1)
    perm = sortperm(reshape(Xn,:))
    Y = X[:, perm]
    membership = falses(m)
    while length(perm)>1
        k = perm[1]
        membership[k], perm, Y = ParetoSub(Y, perm)
    end
    membership[perm] .= true
    membership
end

ParetoSet(X, sense::AbstractVector{EGOSense}) = ParetoSet(X.*Int.(sense))

ParetoSet(X, sense::AbstractVector{Symbol}) = ParetoSet(X, sym2sense.(sense))


@testset "polynomial" begin
    rng = MersenneTwister(123456)
    x = rand(rng)*2
    v1 = rand(rng, 3)
    v2 = rand(rng, 3)
    c = randn(rng)
    @testset "LinearKernel" begin
        k = LinearKernel()
        @test kappa(k,x) ≈ x
        @test k(v1,v2) ≈ dot(v1,v2)
        @test kappa(LinearKernel(),x) == kappa(k,x)
        @test metric(LinearKernel()) == KernelFunctions.DotProduct()
        @test metric(LinearKernel(c=2.0)) == KernelFunctions.DotProduct()
        @test repr(k) == "Linear Kernel (c = 0.0)"

        # Standardised tests.
        TestUtils.test_interface(k, Float64)
        test_ADs(x->LinearKernel(c=x[1]), [c])
        test_params(LinearKernel(; c=c), ([c],))
    end
    @testset "PolynomialKernel" begin
        k = PolynomialKernel()
        @test kappa(k,x) ≈ x^2
        @test k(v1,v2) ≈ dot(v1,v2)^2
        @test kappa(PolynomialKernel(),x) == kappa(k,x)
        @test repr(k) == "Polynomial Kernel (c = 0.0, d = 2.0)"

        # Coherence tests.
        @test kappa(PolynomialKernel(d=1.0,c=c),x) ≈ kappa(LinearKernel(c=c),x)
        @test metric(PolynomialKernel()) == KernelFunctions.DotProduct()
        @test metric(PolynomialKernel(d=3.0)) == KernelFunctions.DotProduct()
        @test metric(PolynomialKernel(d=3.0,c=2.0)) == KernelFunctions.DotProduct()

        # Standardised tests.
        TestUtils.test_interface(k, Float64)
        # test_ADs(x->PolynomialKernel(d=x[1], c=x[2]),[2.0,  c])
        @test_broken "All, because of the power"
        test_params(PolynomialKernel(; d=x, c=c), ([x], [c]))
    end
end


using LinearAlgebra
using Distributions
using Plots

include("./gen_data.jl")
include("./kalman_filters.jl")

"""Experimental parameters"""

# Number of sigma points
N = 10

# Time horizon
T = 100

# Experimental parameters
transition_coeffs = 1.0
emission_coeffs = 1.0

# Noises
process_noise = 0.5
measurement_noise = 0.1

# Prior state
state0 = ([0.], reshape([1e-12], 1,1))

"Generate data"

# Generate signal
observations, states = LGDS(transition_coeffs,
                            emission_coeffs,
                            process_noise,
                            measurement_noise,
                            state0;
                            time_horizon=T)

# Check signal visually
scatter(1:T, observations, color="blue", label="observations")
plot!(1:T, states[2:end], color="red", label="states")

"""Basic Kalman filter"""

# Call filter
include("./kalman_filters.jl")
mx, Px = kalman_filter(observations,
                       transition_coeffs,
                       emission_coeffs,
                       process_noise,
                       measurement_noise,
                       state0)

# Visualize estimates
scatter(1:T, observations[1,:], color="black", label="observations")
plot!(1:T, states[1,2:end], color="red", label="latent states")
plot!(1:T, mx[:], color="purple", label="inferred")
plot!(1:T, mx[:],
      ribbon=[sqrt.(Px[1,1,:]), sqrt.(Px[1,1,:])],
      color="purple", alpha=0.1, label="")
xlabel!("time (t)")
ylabel!("signal")
savefig("./viz/LGDS_kf.png")

"Generate data"

# Nonlinearities
transition_function(x) = .1*x.^3 + 0.5*x
emission_function(x) = x

# Noises
process_noise = reshape([0.1], 1,1)
measurement_noise = reshape([0.2], 1,1)

# Generate signal
observations, states = NLGDS(transition_function,
                             emission_function,
                             process_noise,
                             measurement_noise,
                             state0;
                             time_horizon=T)

# Check signal visually
scatter(1:T, observations, color="black", label="observations")
plot!(1:T, states[2:end], color="red", label="states")

"""Nonlinear Kalman filter"""

# Call basic filter
mx1, Px1 = kalman_filter(observations,
                         transition_coeffs,
                         emission_coeffs,
                         process_noise,
                         measurement_noise,
                         state0)

# Visualize estimates
p1 = scatter(1:T, observations[1,:], color="black", label="observations", size=(1200,500))
plot!(1:T, states[1,2:end], color="red", label="latent states")
plot!(1:T, mx1[:], color="purple", label="linear")
plot!(1:T, mx1[:],
      ribbon=[sqrt.(Px1[1,1,:]), sqrt.(Px1[1,1,:])],
      color="purple", alpha=0.1, label="")
xlabel!("time (t)")
ylabel!("signal")
savefig(p1, "./viz/NLGDS_KF.png")

# Call extended filter with first-order Taylor
mx2, Px2 = extended_kalman_filter(observations,
                                  transition_function,
                                  emission_function,
                                  process_noise,
                                  measurement_noise,
                                  state0)

p2 = scatter(1:T, observations[1,:], color="black", label="observations", size=(1200,500))
plot!(1:T, states[1,2:end], color="red", label="latent states")
plot!(1:T, mx2[:], color="blue", label="extended")
plot!(1:T, mx2[:],
      ribbon=[sqrt.(Px2[1,1,:]), sqrt.(Px2[1,1,:])],
      color="blue", alpha=0.1, label="")
xlabel!("time (t)")
ylabel!("signal")
savefig(p2, "./viz/NLGDS_E1KF.png")

# Call extended filter with second-order Taylor
mx3, Px3 = extended_kalman_filter(observations,
                                  transition_function,
                                  emission_function,
                                  process_noise,
                                  measurement_noise,
                                  state0,
                                  second_order=true)

p3 = scatter(1:T, observations[1,:], color="black", label="observations", size=(1200,500))
plot!(1:T, states[1,2:end], color="red", label="latent states")
plot!(1:T, mx3[:], color="cyan", label="extended")
plot!(1:T, mx3[:],
    ribbon=[sqrt.(Px3[1,1,:]), sqrt.(Px3[1,1,:])],
    color="cyan", alpha=0.1, label="")
xlabel!("time (t)")
ylabel!("signal")
savefig(p3, "./viz/NLGDS_E2KF.png")

# Call unscented filter
mx4, Px4 = unscented_kalman_filter(observations,
                                   transition_function,
                                   emission_function,
                                   process_noise,
                                   measurement_noise,
                                   state0,
                                   α=1., κ=0., β=2.)

p4 = scatter(1:T, observations[1,:], color="black", label="observations", size=(1200,500))
plot!(1:T, states[1,2:end], color="red", label="latent states")
plot!(1:T, mx4[:], color="green", label="unscented")
plot!(1:T, mx4[:],
      ribbon=[sqrt.(Px4[1,1,:]), sqrt.(Px4[1,1,:])],
      color="green", alpha=0.1, label="")
xlabel!("time (t)")
ylabel!("signal")
savefig(p4, "./viz/NLGDS_UKF.png")

plot(p1, p2, p3, p4, layout=(4,1), size=(2000,1600))
savefig("./viz/NLGDS_filters.png")


using Test; using PyPlot; using Revise; using BenchmarkTools; using AbstractFFTs; using FFTW;

#= test function without discontinuity in derivative at t=0 =#
function funk1(x)
  return (1/2)*x^2 + (1/3)*x^3 - (1/4)*x^4
end

function funk1dot(x)
  return x + x^2 - x^3
end

function funk2(x)
  return x + (1/2)*x^2 - (1/3)*x^3
end

function funk2dot(x)
  return 1 + x - x^2
end

even(x) = x%2==0

function spectralmultiplier(x, L)
  N = length(x)
  out = similar(x, Complex{Float64})
  for k in 0:(N-1)
    if k<(N/2)
      out[k+1] = x[k+1]*2*π*im*k/L

    elseif k>(N/2)
      out[k+1] = x[k+1]*2*π*im*(k-N)/L

    end
  end
  if even(N); out[N÷2]=0.0; end

  return out
end

function specdifftest(f, fdot)
	yff = fft(f)
	yffdot = spectralmultiplier(yff, t[end])
	ydot_approx = real.(ifft(yffdot))
	plot(t, fdot)
	plot(t, ydot_approx, "--", alpha=0.2)
  return
end

dt = 0.01
t = Vector{Float64}(0.0:dt:8*pi)
#y = -cos.(t)
#ydot = sin.(t)

specdifftest(funk1.(t), funk1dot.(t))


#!/usr/bin/julia

# Trizen
# 17 January 2017
# https://github.com/trizen

# Generates the Mandelbrot set.

# See also:
#   https://en.wikipedia.org/wiki/Mandelbrot_set
#   https://trizenx.blogspot.ro/2017/01/mandelbrot-set.html

using Images

@inline function hsv2rgb(h, s, v)
    c = v * s
    x = c * (1 - abs(((h/60) % 2) - 1))
    m = v - c

    if h < 60
        r,g,b = (c, x, 0)
    elseif h < 120
        r,g,b = (x, c, 0)
    elseif h < 180
        r,g,b = (0, c, x)
    elseif h < 240
        r,g,b = (0, x, c)
    elseif h < 300
        r,g,b = (x, 0, c)
    else
        r,g,b = (c, 0, x)
    end

    (r + m), (b + m), (g + m)
end

function mandelbrot()

    w, h = 1000, 1000

    zoom  = 0.5
    moveX = 0
    moveY = 0

    maxIter = 100
    img = zeros(RGB{Float64}, h, w)

    for x in 1:w, y in 1:h
        i = maxIter
        c = Complex(
            (2*x - w) / (w * zoom) + moveX,
            (2*y - h) / (h * zoom) + moveY
        )
        z = c
        while abs(z) < 2 && (i -= 1) > 0
            z = z^2 + c
        end
        r,g,b = hsv2rgb(i / maxIter * 360, 1, i / maxIter)
        img[y,x] = RGB{Float64}(r, g, b)
    end

    println("Generating image...")
    save("mandelbrot_set.png", img)
end

mandelbrot()


export mutualinfo, Kraskov1, Kraskov2

abstract type MutualInformationEstimator <: EntropyEstimator end

"""
    mutualinfo(x, y, est; base = 2, q = 1)

Estimate mutual information between `x` and `y`, ``I^{q}(x; y)``, using the provided
entropy/probability estimator `est` from Entropies.jl, and Rényi entropy of order `q`
(defaults to `q = 1`, which is the Shannon entropy), with logarithms to the given `base`.

Both `x` and `y` can be vectors or (potentially multivariate) [`Dataset`](@ref)s.

Worth highlighting here are the estimators that compute entropies _directly_, e.g.
nearest-neighbor based methhods. The choice is between naive
estimation using the [`KozachenkoLeonenko`](@ref) or [`Kraskov`](@ref) entropy estimators,
or the improved [`Kraskov1`](@ref) and [`Kraskov2`](@ref) dedicated ``I`` estimators. The
latter estimators reduce bias compared to the naive estimators.

**Note**: only Shannon entropy is possible to use for nearest neighbor estimators, so the
keyword `q` cannot be provided; it is hardcoded as `q = 1`.

## Description

Mutual information ``I`` between ``X`` and ``Y``
is defined as

```math
I(X; Y) = \\sum_{y \\in Y} \\sum_{x \\in X} p(x, y) \\log \\left( \\dfrac{p(x, y)}{p(x)p(y)} \\right)
```

Here, we rewrite this expression as the sum of the marginal entropies, and extend the
definition of ``I`` to use generalized Rényi entropies

```math
I^{q}(X; Y) = H^{q}(X) + H^{q}(Y) - H^{q}(X, Y),
```

where ``H^{q}(\\cdot)`` is the generalized Renyi entropy of order ``q``, i.e., the
`genentropy` function from Entropies.jl.
"""
function mutualinfo end

mutualinfo(x::Vector_or_Dataset, y::Vector_or_Dataset) =
    error("Estimator missing. Please provide a valid estimator as the third argument.")

function mutualinfo(x::Vector_or_Dataset, y::Vector_or_Dataset, est; base = 2, q = 1)
    X = genentropy(Dataset(x), est; base = base, q = q)
    Y = genentropy(Dataset(y), est; base = base, q = q)
    XY = genentropy(Dataset(x, y), est; base = base, q = q)
    MI = X + Y - XY
end

include("nearestneighbor.jl")


using LinearAlgebra: norm, inv, eigvals
import ForwardDiff
using StaticArrays: setindex
using DynamicalSystems

# This code is written for `SVector`s, or out-of-place dynamical systems.
# Notation z is used for the vector (x..., p), i.e. mixed space
"""
    bifurcation_rule_form(ds) → f, J
Produce dynamic rule `f` and Jacobian function `J` in a two-argument form.
"""
function bifurcation_rule_form(ds::ContinuousDynamicalSystem)
    f = (x, p) -> ds.f(x, p, 0.0)
    J = (x, p) -> ds.jacobian(x, p, 0.0)
    return f, J
end

function bifurcation_rule_form(ds::DiscreteDynamicalSystem)
    error("TODO: Convert `f` to `f - x`.")
    f = (x, p) -> ds.f(x, p, 0.0)
    J = (x, p) -> ds.jacobian(x, p, 0.0)
    return f, J
end

# Corrector: Newton's method in mixed-space with extra fixed condition
"""
    corrector(zpred, zprev, f, J; max_steps = 200, δ = 0.9, ε = 1e-3) → (z*, success)
Implement the Newton algorithm to converge to the fixed point for given predicted
mixed state `zpred`, previously found state `zprev`, function `f`, Jacobian `J`
and stepping factor `δ`.

If iteration occurs for more than `max_steps` without convergence better than `ε`,
then iteration stops and `success` will be `false`.
"""
function corrector(zpred, zprev, f, J; δ = 0.9, max_steps = 200, ε = 1e-6)
    Δz = zpred .- zprev
    # index of variable that changed the most
    # The `iszero` clause is just to keep the first iteration at given `p0`
    i = all(iszero, Δz) ? length(Δz) : argmax(Δz .^ 2)
    c = 0
    zⱼ = zpred
    zⱼ₊₁ = newton_step!(zⱼ, zpred, i, f, J, δ)
    while norm(zⱼ₊₁ - zⱼ) > ε
        zⱼ = zⱼ₊₁
        zⱼ₊₁ = newton_step!(zⱼ, zpred, i, f, J, δ)
        c += 1
        if c > max_steps
            @warn("Newton did not converge.")
            return (zⱼ₊₁, false)
        end
    end
    return zⱼ₊₁, true
end

function newton_step!(zⱼ, zpred, i, f, J, δ)
    Jfinal = mixed_jacobian(zⱼ, i, f, J)
    xⱼ = zⱼ[1:end-1]; pⱼ = zⱼ[end]
    g = f(xⱼ, pⱼ)
    gz = vcat(g, zⱼ[i] - zpred[i])
    zⱼ₊₁ = zⱼ - δ*(inv(Jfinal))*gz
    return zⱼ₊₁
end

function mixed_jacobian(z, i, f, J)
    x = z[1:end-1]; p = z[end]
    # start creating the mixed space jacobian
    j = J(x, p)
    # to the state space jacobian add one more column, derivative towards p
    pder = ForwardDiff.derivative(p -> f(x, p), p)
    Jmixed = hcat(j, pder)
    # add the last row, which is 1 for the `i` entry, 0 everywhere else
    last_row = setindex((@SVector zeros(length(z))), 1.0, i)
    Jfinal = vcat(Jmixed, last_row')
    return Jfinal
end

# Predictor: Secant (linear extrapolation of already found bifurcation curve)
function predictor(zs, dz0)
    if length(zs) == 1
        return zs[end]
    elseif length(zs) == 2 # 1 entry is z0, 2nd entry is 1st found fixed point
        return zs[end] .+ dz0
    else
        return 2zs[end] .- zs[end-1]
    end
end

# Continuation function: perform a step of predictor-corrector and save values
function continuation!(zs, f, J; dz0, pmin, pmax)
    zpred = predictor(zs, dz0)
    (pmin ≤ zpred[end] ≤ pmax) || return false
    zˣ, success = corrector(zpred, zs[end], f, J)
    push!(zs, zˣ)
    return success
end

# Continuation loop: do continuation for a given amount of steps
function continuation(f, J, x0, p0;
        pmin, pmax, dp0, dx0, N = 1000
    )

    z0 = vcat(x0, p0); zs = [z0]; dz0 = vcat(dx0, dp0)

    ps = [p0]
    xs = Dataset([x0])
    stability = Bool[]
    for i in 1:N
        success = continuation!(zs, f, J; dz0, pmin, pmax)
        # Stop iteration if we exceed given parameter margins
        success || break
        # Detect stability of found fixed point (needs `Array` coz of StaticArrays.jl)
        eigenvalues = eigvals(Array(J(zs[end][1:end-1], zs[end][end])))
        μ = maximum(real(v) for v ∈ eigenvalues)
        isstable = μ < 0
        push!(stability, isstable)
    end
    xs = Dataset([z[1:end-1] for z in zs])
    ps = [z[end] for z in zs]
    popfirst!(xs.data); popfirst!(ps) # remove initial guess
    return xs, ps, stability
end


### A Pluto.jl notebook ###
# v0.12.16

using Markdown
using InteractiveUtils

# ╔═╡ e077ab16-22e5-11eb-15ed-5f0654bf7328
using Pkg, DrWatson

# ╔═╡ e75a9a06-22e5-11eb-2e36-8d4f62b830ed
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using Random, Distributions, Plots
	Random.seed!(0)
end

# ╔═╡ ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
md"## Listing5.05"

# ╔═╡ fa68607e-22e5-11eb-0558-c9a4d9f77426
begin
	n1, n2 = 10, 15
	N = 10^6
	mu, sigma = 10, 4
	normDist = Normal(mu,sigma)
end;

# ╔═╡ 67a42f40-3a99-11eb-2199-cb72cdee58fa
begin
	fValues = Array{Float64}(undef, N)

	for i in 1:N
		data1 = rand(normDist,n1)
		data2 = rand(normDist,n2)
		fValues[i] = var(data1)/var(data2)
	end
end

# ╔═╡ 67a46e56-3a99-11eb-1254-3d0d490364f5
begin
	fRange = 0:0.1:5
	stephist(fValues, bins=400, c=:blue, label="Simulated", normed=true)
	plot!(fRange, pdf.(FDist(n1-1, n2-1), fRange),
		c=:red, label="Analytic", xlims=(0,5), ylims=(0,0.8),
			xlabel = "F", ylabel = "Density")
end

# ╔═╡ 475ff888-22e6-11eb-2354-09f8f40a8e12
md"## End of listing5.05"

# ╔═╡ Cell order:
# ╟─ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
# ╠═e077ab16-22e5-11eb-15ed-5f0654bf7328
# ╠═e75a9a06-22e5-11eb-2e36-8d4f62b830ed
# ╠═fa68607e-22e5-11eb-0558-c9a4d9f77426
# ╠═67a42f40-3a99-11eb-2199-cb72cdee58fa
# ╠═67a46e56-3a99-11eb-1254-3d0d490364f5
# ╟─475ff888-22e6-11eb-2354-09f8f40a8e12


function simpson_integrate(u, stepsize)
    numpts = length(u)
    @assert iseven(numpts - 1)
    mid = div(numpts,2)
    integral = 0.0
    for i = 1:mid
        integral += stepsize / 3.0 * (u[2i-1] + 4.0 * u[2i] + u[2i+1])
    end
    return integral
end


using SchumakerSpline
using Dates
tol = 10*eps()

StartDate = Date(2018, 7, 21)
x = Array{Date}(undef,1000)
for i in 1:1000
    x[i] = StartDate +Dates.Day(2* (i-1))
end

function f(x::Date)
    days_between = Dates.days(x - StartDate)
    return log(days_between+1) + sqrt(days_between)
end
y = f.(x)

spline = Schumaker(x,y)
for i in 1:length(x)
    abs(evaluate(spline, x[i]) - y[2]) < tol
end
# Evaluation with a Float64.
evaluate(spline, 11.5)

# Testing second derivatives
second_derivatives = evaluate.(spline, x,2)
maximum(second_derivatives) < tol

# Testing Integrals
function analytic_integral(lhs,rhs)
    lhs_in_days = Dates.days(lhs - StartDate)
    rhs_in_days = Dates.days(rhs - StartDate)
    return (rhs_in_days+1)*log(rhs_in_days+1)-rhs_in_days + (2/3)*rhs_in_days^(3/2) - ((lhs_in_days+1)*log(lhs_in_days+1) - lhs_in_days + (2/3)*lhs_in_days^(3/2))
end

lhs = StartDate
rhs = StartDate + Dates.Month(16)
numerical_integral = evaluate_integral(spline, lhs,rhs)
analytical = analytic_integral(lhs,rhs)
abs(  analytical - numerical_integral  ) < 1

## Testing with only one date provided.
x = Array{Date}(undef, 1)
x[1] = Date(2018, 7, 21)
y = Array{Float64}(undef, 1)
y[1] = 0.0
spline = Schumaker(x,y)
abs(evaluate(spline,  Date(2018, 7, 21))) < tol
abs(evaluate(spline,  Date(2019, 7, 21))) < tol
abs(evaluate(spline,  Date(2000, 7, 21))) < tol

## Testing with two dates provided.
x = Array{Date}(undef,2)
x[1] = Date(2018, 7, 21)
x[2] = Date(2018, 8, 21)
y = Array{Float64}(undef,2)
y[1] = 0.0
y[2] = 1.0
spline = Schumaker(x,y)
abs(evaluate(spline,  Date(2018, 7, 21))) < tol
abs(evaluate(spline,  Date(2018, 7, 30))) > tol
abs(evaluate(spline,  Date(2018, 8, 21)) - y[2]) < tol
abs(evaluate(spline,  Date(2019, 8, 21)) - y[2]) > tol

spline = Schumaker(x, y , extrapolation =  (Constant,Constant))
abs(evaluate(spline,  Date(2018, 8, 21)) - y[2]) < tol
abs(evaluate(spline,  Date(2019, 8, 21)) - y[2]) < tol

## Testing with three dates provided.
x = Array{Date}(undef,3)
x[1] = Date(2018, 7, 21)
x[2] = Date(2018, 8, 21)
x[3] = Date(2018, 9, 21)
y = Array{Float64}(undef,3)
y[1] = 0.0
y[2] = 1.0
y[3] = 1.3
spline = Schumaker(x,y)
abs(evaluate(spline,  x[2]) - y[2]) < tol


using Test
using Example2018

# Polynomials form a ring
# 1. addition is associative
# 2. additive identity
# 3. additive inverse
# 4. addition commutes

@testset "Polynomials form a ring" begin

	z = Polynomial([0])

	for n in 1:100

		deg = rand(0:100)
		p = Polynomial(rand(-1000000:1000000, deg+1))

		@test p + z == p
		@test z + p == p
		@test p + (-p) == z

	end

end

# Scaler Multiplication
# 1. A scaler multiplied by a polynomial gives a polynomial
# 2. A polynomial multiplied by zero gives zero

@testset "Scaler Multiplication" begin

	z = Polynomial([0])

	for n in 1:100

		deg = rand(0:100)
		sclr = rand()
		p = Polynomial(rand(-1000000:1000000, deg+1))

		@test poly(sclr*p) == true
		@test 0*p == z

	end

end

# Derivative Test
# 1. The derivative of a polynomial must be a polynomial
# 2. If the polynomial has degree n, the derivative must have degree (n-1)

@testset "Derivative Test" begin

	for n in 1:100

		deg = rand(0:100)
		p = Polynomial(rand(-1000000:1000000, deg+1))
		dp = deriv(p)

		@test poly(dp) == true
		@test length(dp.coeffs) == deg

	end

end


export Sech

"""
`Sech([domainType=Float64::Type,] dim_in::Tuple)`

Creates an hyperbolic secant non-linear operator with input dimensions `dim_in`:
```math
\\text{sech} ( \\mathbf{x} ).
```

"""
struct Sech{T,N} <: NonLinearOperator
	dim::NTuple{N,Int}
end

function Sech(DomainType::Type, DomainDim::NTuple{N,Int}) where {N}
	Sech{DomainType,N}(DomainDim)
end

Sech(DomainDim::NTuple{N,Int}) where {N} = Sech{Float64,N}(DomainDim)
Sech(DomainDim::Vararg{Int}) = Sech{Float64,length(DomainDim)}(DomainDim)

function mul!(y::AbstractArray{T,N}, L::Sech{T,N}, x::AbstractArray{T,N}) where {T,N}
	y .= sech.(x)
end

function mul!(y::AbstractArray,
              J::AdjointOperator{Jacobian{A,TT}},
              b::AbstractArray) where {T,N, A<: Sech{T,N}, TT <: AbstractArray{T,N}}
    L = J.A
    y .= -conj.( tanh.(L.x) .* sech.(L.x) ).*b
end

fun_name(L::Sech) = "sech"

size(L::Sech) = (L.dim, L.dim)

domainType(L::Sech{T,N}) where {T,N} = T
codomainType(L::Sech{T,N}) where {T,N} = T


### A Pluto.jl notebook ###
# v0.12.6

using Markdown
using InteractiveUtils

# ╔═╡ e77948a6-1ecc-11eb-3285-7f06dcdb7cf5
using Pkg, DrWatson

# ╔═╡ e7799b9e-1ecc-11eb-311f-035de72c3603
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using StatsBase, Combinatorics, Plots
end

# ╔═╡ 6c38d2ce-1ecc-11eb-281d-0d13a832adea
md"## Listing 2.3"

# ╔═╡ fe1ee6a6-1ecc-11eb-1930-0301940bc5f7
	N = 10^3

# ╔═╡ e77a2a28-1ecc-11eb-0acb-5167936362f5
begin
	matchExists1(n) = 1 - prod([k/365 for k in 365:-1:365-n+1])
	matchExists2(n) = 1- factorial(365,365-big(n))/365^big(n)

	function bdEvent(n)
		birthdays = rand(1:365,n)
		dayCounts = counts(birthdays, 1:365)
		return maximum(dayCounts) > 1
	end

	probEst(n) = sum([bdEvent(n) for _ in 1:N])/N
end

# ╔═╡ e7885e54-1ecc-11eb-03d1-2580aad82a3d
begin
	xGrid = 1:50
	analyticSolution1 = [matchExists1(n) for n in xGrid]
	analyticSolution2 = [matchExists2(n) for n in xGrid]
	("Maximum error: $(round(maximum(abs.(analyticSolution1 - analyticSolution2)), digits=25))")
end

# ╔═╡ e788edce-1ecc-11eb-3682-b1bfc98646a0
begin
	mcEstimates = [probEst(n) for n in xGrid]

	plot(xGrid, analyticSolution1, c=:blue, label="Analytic solution")
	scatter!(xGrid, mcEstimates, c=:red, ms=6, msw=0, shape=:xcross,
		label="MC estimate", xlims=(0,50), ylims=(0, 1),
		xlabel="Number of people in room",
		ylabel="Probability of birthday match",
		legend=:topleft)
end

# ╔═╡ e7965b58-1ecc-11eb-2308-093c9f5a167e
md"## End of listing 2.3"

# ╔═╡ Cell order:
# ╟─6c38d2ce-1ecc-11eb-281d-0d13a832adea
# ╠═e77948a6-1ecc-11eb-3285-7f06dcdb7cf5
# ╠═e7799b9e-1ecc-11eb-311f-035de72c3603
# ╠═fe1ee6a6-1ecc-11eb-1930-0301940bc5f7
# ╠═e77a2a28-1ecc-11eb-0acb-5167936362f5
# ╠═e7885e54-1ecc-11eb-03d1-2580aad82a3d
# ╠═e788edce-1ecc-11eb-3682-b1bfc98646a0
# ╟─e7965b58-1ecc-11eb-2308-093c9f5a167e


### A Pluto.jl notebook ###
# v0.14.6

using Markdown
using InteractiveUtils

# ╔═╡ 7ef4b0b3-9d0a-4356-bd16-14487f14e0ab
begin
	using PlutoUI
	PlutoUI.TableOfContents(aside=true)
end

# ╔═╡ 0cd92d78-1da6-435b-ad8e-60af29502e1d
begin
	# Necessary packages
	# If a package is missing, follow the instructions.
	using LightGraphs
	using GraphPlot
	using SparseArrays
	using LinearAlgebra
	using Plots
	using Distances
	using Arpack
	using Images
	using Random
end

# ╔═╡ cb61f761-b2ee-4818-9149-f85d95b76a1e
md"""
# Spectral Graph Bipartitioning

Many data clustering problems can be interpreted as clustering of vertices of graphs. __Graph bipartitioning problem__ is to partition vertices into subsets such that the connections within subsets are stronger than the connections between different subsets.

Partition of the vertices into two subsetts is done according to signs of the eigenvectors of the second smallest eigenvalue of the Laplacian matrix.

__Prerequisites__

The reader should be familiar with the basic graph theory, linear algebra and, in particular,  eigenvalues and eigenvectors.

__Competences__

The reader should be able to apply graph spectral bipartitioning and recursive bipartitioning to data clustering problems.

__Credits.__ The notebook was initially derived from M.Sc. Thesis of Ivančica Mirošević.
"""

# ╔═╡ 5926c79a-6038-4dbc-b49a-5b05d02250a0
md"""
# Graphs

For more details, see [W. H. Haemers, Matrices and Graphs, in L. Hogben, Ed., 'Handbook of Linear Algebra', pp. 39.1-39.14, CRC Press, Boca Raton, 2014.](https://www.routledge.com/Handbook-of-Linear-Algebra/Hogben/p/book/9781138199897) and [S. Butler and F. Chung, Spectral Graph Theory, ibid., pp. 47.1-47.6](https://www.routledge.com/Handbook-of-Linear-Algebra/Hogben/p/book/9781138199897) and the references therein.

## Definitions

A __weighted graph__ is an ordered triplet $G=(V,E,\omega)$, where $V=\{1,2,3,...,n\}$ is the set of __vertices__ , $E=\{(i,j)\}$ is a set of __edges__ connecting vertices, and $\omega$ is a set of __weights__ of edges. We assume $G$ is undirected.
"""

# ╔═╡ a66a6e80-1f80-11eb-15f8-71ffaa5050e7
md"
__Adjacency matrix__ of graph $G$ is the matrix $A$ defined as

$$A_{ij}=\begin{cases} 1 \quad \textrm{if}\ (i,j)\in E, \\
0\quad  \textrm{otherwise} \end{cases}.$$

__Weight matrix__ of graph $G$ is the matrix $W$ defined as

$W_{ij}=\begin{cases} \omega(e) \quad \textrm{if}\ e=(i,j)\in E, \\
0\quad  \textrm{otherwise} \end{cases}.$

__Laplacian matrix__ of graph $G$ is the matrix

$L=D-W,$

where
$D=\mathop{\mathrm{diag}}(d_1,d_2,\ldots,d_n)$ with $d_i=\sum_{k=1}^n W_{ik}$ for $i=1,\ldots,n$.

__Normalized Laplacian matrix__ is the matrix

$L_n=D^{-1/2} L D^{-1/2}\equiv D^{-1/2} (D-W) D^{-1/2}$

(__diagonally scaled $L$__).

__Incidence matrix__ of graph $G$ is the $|V|\times |E|$ matrix $I_G$. Each row of $I_G$ corresponds to a vertex of $G$ and each column corresponds to an edge of $G$.
In the column corresponding to en edge $e=(i,j)$, all elements are zero except the ones in the $i$-th and $j$-th row, which are equal to $\sqrt{\omega(e)}$ and $-\sqrt{\omega(e)}$, respectively.
"

# ╔═╡ d4cf12ce-12fb-4562-bbea-e9b85cba94c2
md"""
### Examples

Graph types and algorithms are implemented in the package [LightGraphs.jl](https://github.com/JuliaGraphs/LightGraphs.jl).

Plotting graphs is done by the packages
[GraphPlot.jl](https://github.com/JuliaGraphs/GraphPlot.jl).

As a small inconvenience, we can only plot unweighted graphs and plot weights as edge labels.
"""

# ╔═╡ 87cca1d3-b944-4d74-bf4c-1562f34cf91a
begin
	# Sources, targets and weights
	n=7
	sn=[1,1,1,2,2,3,3,3,5,5,6]
	tn=[2,3,4,4,5,4,6,7,6,7,7]
	wn=[2,3,4,7,1,3,2,1,7,3,5]
	[sn tn wn]
end

# ╔═╡ 3d7b4a01-9a7d-4111-a45d-1d31f1ef6278
begin
	# Create the graph
	G=Graph(n)
	for i=1:length(sn)
	    add_edge!(G,sn[i],tn[i])
	end
	G
end

# ╔═╡ 3b5ed1fa-75b0-43fb-a918-4c2e8f52d039
# What is the optimal bipartition?
gplot(G, nodelabel=1:n, edgelabel=wn)

# ╔═╡ 8f7d1f6c-ee4c-407f-80d9-d2abc2af949e
begin
	# We define some functions
	function WeightMatrix(src::Array,dst::Array,weights::Array)
	    n=nv(G)
	    sparse([src;dst],[dst;src],[weights;weights],n,n)
	end

	Laplacian(W::AbstractMatrix)=spdiagm(0=>vec(sum(W,dims=2)))-W

	function NormalizedLaplacian(L::AbstractMatrix)
	    D=1.0./sqrt.(diag(L))
		Diagonal(D)*L*Diagonal(D)
	end
end

# ╔═╡ b7f127d8-e8cd-4771-8091-c88f9ffe6b08
W=WeightMatrix(sn,tn,wn)

# ╔═╡ 3f5447bb-95cd-4520-b0b9-e85f33b323bd
Matrix(W)

# ╔═╡ ecb2a46e-f9dc-4e57-812f-0a2788afb202
begin
	L=Laplacian(W)
	Matrix(L)
end

# ╔═╡ 86e5f00f-13ee-4867-b0cd-187f2c782a46
Lₙ=NormalizedLaplacian(L)

# ╔═╡ cf26dbe9-3c4f-4c95-88ef-f197f360d759
Matrix(Lₙ)

# ╔═╡ 8fcd2440-4013-4b37-a747-2c0c7f6d4aeb
issymmetric(Lₙ)

# ╔═╡ 0c8d3572-3436-446f-952a-b19d0370ab38
# Let us compute the incidence matrix
function IncidenceMatrix(G::Graph, weights::Array)
    A=zeros(nv(G),ne(G))
    k=1
    for a in edges(G)
        A[a.dst,k]=sqrt.(weights[k])
        A[a.src,k]=-sqrt(weights[k])
        k+=1
    end
    A
end

# ╔═╡ 1437aadc-1c26-4eff-bb22-9cf9c15e1f5f
Iᵧ=IncidenceMatrix(G,wn)

# ╔═╡ 6529324b-49f7-4f01-b5cd-33668979b56b
md"""
## Facts

1.  $L=I_{G}I_{G}^{T}$.

2.  $L$ is symmetric PSD matrix.

3.  $L\mathbf{1}=0$ for $\mathbf{1}=[1,...,1]^{T}$, thus $0$ is an eigenvalue of $L$  and $\mathbf{1}$ is the corresponding eigenvector.

4. If $G$ has $c$ connected components, then $L$ has $c$ eigenvalues equal to $0$.

5. For every $x\in \mathbb{R}^{n}$, it holds
$x^{T}L x=\sum\limits_{i<j}W_{ij}(x_{i}-x_{j})^{2}$.

6. For every $x\in\mathbb{R}^{n}$ and $\alpha,\beta\in\mathbb{R}$, it holds
$(\alpha x+\beta \mathbf{1})^{T} L (\alpha x+\beta \mathbf{1})
=\alpha^{2} x^{T}L x$.

7. Assume that the eigenvalues of $L$ are increasingly ordered. Then,

$$
0=\lambda_1(L)\leq \lambda_2(L)\leq \cdots \leq\lambda_{n}(L)\leq
2\max\limits_{i=1,\cdots ,n}d_{i}.$$

8.  $\sigma(L_n) \subseteq [0,2]$.
"""

# ╔═╡ a9ba64a6-bd17-4b86-acb4-e7c7d8f87dc1
md"""
### Examples
"""

# ╔═╡ 6a6f5db5-7dd4-4a26-ac1a-ef3ca3a5d6be
# Fact 1
norm(L-Iᵧ*Iᵧ')

# ╔═╡ e11bc23d-03e2-4780-9488-93f0adfc2fcb
# Facts 2 and 7
issymmetric(L), eigs(L)[1], 2*maximum(diag(L))

# ╔═╡ 7192f7d0-28b9-11eb-28dd-8db0019504ca
eigen(Matrix(L))

# ╔═╡ e5890dca-368f-437e-bc45-6a395714dae2
# Fact 3
L*ones(n)

# ╔═╡ 61a978a7-b2d2-43e7-b3f7-95bf4c4f54b9
begin
	# Fact 5
	x=rand(n)
	x'*L*x, sum([W[i,j]*(x[i]-x[j])^2 for i=1:n, j=1:n])/2
end

# ╔═╡ da7f85ed-73b0-44b9-a97f-4a8ed5e9a1d1
begin
	# Fact 6
	α,β=rand(),rand()
	(α*x+β*ones(n))'*L*(α*x+β*ones(n)), α^2*x'*L*x
end

# ╔═╡ fed7f592-be03-4baf-8ace-ff9ee78fee71
# Fact 8
eigvals(Matrix(Lₙ))

# ╔═╡ 9f31928e-c933-43bb-9a5c-bf55f29737cd
md"""
# Bipartitioning

## Definitions

Let $\pi=\{V_{1},V_{2}\}$ be a partition of $V$ with $V_1,V_2\neq \emptyset$.

__Cut__ of partition $\pi$ is the sum of weights of all
edges between $V_1$ and $V_2$,

$$\mathop{\mathrm{cut}}(\pi)\equiv \mathop{\mathrm{cut}}(V_1,V_2)=\sum\limits_{{\displaystyle i\in V_{1} \atop \displaystyle j\in V_{2}}}W_{ij}.$$

__Weight__ of vertex $i\in V$ is the sum of the weights of all egdges emanating from $i$,
$\omega(i)=\sum\limits_{j=1}^{n}W_{ij}$.

__Weight__ of a subset $\bar V\subset V$ is the sum of the weights of all vertices in $\bar V$,

$\omega(\bar V)=\sum\limits_{\displaystyle i\in\bar V} \omega(i)$.

__Proportional cut__ of partition $\pi$ is

$$
\mathop{\mathrm{pcut}}(\pi)=\displaystyle\frac{\mathop{\mathrm{cut}}(\pi)}{|V_{1}|}+\frac{\mathop{\mathrm{cut}}(\pi)}{|V_{2}|}.$$

__Normalized cut__ of partition $\pi$ is

$$
\mathop{\mathrm{ncut}}(\pi)=\displaystyle\frac{\mathop{\mathrm{cut}}(\pi)}{\omega(V_{1})}+\frac{\mathop{\mathrm{cut}}(\pi)}{\omega(V_{2})}.$$
"""

# ╔═╡ 1e6d4c72-f25d-4bf5-99d6-ae0446be25dd
md"""
### Example

Consider the following partitions (all edges have unit weights):

 $(load(\"./files/cut2.png\"))

Left partition is $\pi$, right partition is $\pi'$.

|     Cut \ Partition  | $\pi$            |  $\pi'$     |
| ------- | ---------------- | ------------|
| $\mathop{\mathrm{cut}}$  |  $2$        |     $3$        |
| $\mathop{\mathrm{pcut}}$ | $\frac{2}{1}+\frac{2}{11}=2.18$|    $\frac{3}{6}+\frac{3}{6}=1$      |
| $\mathop{\mathrm{ncut}}$ | $\frac{2}{2}+\frac{2}{50}=1.04$ |  $\frac{3}{27}+\frac{3}{25}=0.23$|
"""

# ╔═╡ 02035eed-6d49-4253-a755-68a999e7e90e
md"""
## Facts

1. The informal description of the bipartitioning problem can be formulated as two problems,

$$
\mathop{\textrm{arg min}}\limits_{\pi} \mathop{\mathrm{pcut}}(\pi) \quad \textrm{or} \quad
\mathop{\textrm{arg min}}\limits_{\pi} \mathop{\mathrm{ncut}}(\pi).$$

The first problem favors partitions into subsets with similar numbers of vertices, while the second problem favors partitions into subsets with similar weights.

2. Both problems are NP-hard.

3. __Approximate solutions can be computed by suitable relaxations in $O(n^2)$ operations.__

4. The partition $\pi$ is defined by the vector $y$ such that

$$
y_{i}=
\begin{cases}
\frac{1}{2} & \text{for } i\in V_1 \\
-\frac{1}{2} & \text{for } i\in V_2
\end{cases}$$

The proportional cut problem can be formulated as the  __discrete proportional cut__ problem

$$
\underset{\displaystyle \big|\mathbf{y}^{T}\mathbf{1} \big|\leq \beta}
{\min\limits_{\displaystyle y_{i}\in \{-\frac{1}{2},\frac{1}{2}\}}}
\frac{1}{2}\sum_{i,j}(y_{i}-y_{j})^{2}W_{ij}.$$

Parameter $\beta$ controls the number of vertices in each subset.

5. The normalized cut problem can be formulated as the __discrete normalized cut__ problem

$$
\underset{\displaystyle \big|y^{T}D\mathbf{1} \big|\leq \beta}
{\min\limits_{\displaystyle y_{i}\in \{-\frac{1}{2},\frac{1}{2}\}}}
\frac{1}{2}\sum_{i,j}(y_{i}-y_{j})^{2}W_{ij}.$$

Parameter $\beta$ controls the weights of each subset.

6. Using the Fact 5 above, the discrete proportional cut problem can be formulated as the __relaxed proportional cut__ problem

$$
\underset{\displaystyle y^{T}y=1}{\underset{\displaystyle \big| y^{T}\mathbf{1} \big|
\leq 2\frac{\beta}{\sqrt{n}}}
{\min\limits_{\displaystyle y\in \mathbb{R}^{n}}}} y^{T}L y.$$

Similarly, the discrete normalized cut problem can be formulated as the __relaxed normalized cut__ problem

$$
\underset{\displaystyle y^{T}Dy=1}{\underset{\displaystyle \big| y^{T}D\mathbf{1}\big|
\leq \displaystyle \frac{\beta}{\sqrt{\theta n}}}{\min\limits_{\displaystyle y\in
\mathbb{R}^{n}}}}y^{T}L_n y.$$

7. __The Main Theorem.__ Let $A\in \mathbb{R}^{n\times n}$ be a symmetric matrix with eigenvalues $\lambda _{1}<\lambda _{2}<\lambda_{3}\leq \cdots \leq \lambda _{n}$ and let $v^{[1]},v^{[2]},\ldots,v^{[n]}$ be the corresponding eigenvectors. For the fixed $0\leq \alpha <1$, the solution of the problem

$$
\underset{\displaystyle y^{T}y=1}{\underset{\displaystyle \left|y^{T}v^{[1]}\right|\leq \alpha}
{\min\limits_{\displaystyle y\in \mathbb{R}^{n}}}} y^{T}Ay$$

is $y=\pm \alpha v^{[1]}\pm \sqrt{1-\alpha^{2}}v^{[2]}$.

For the proof see [D. J. Higham and M. Kibble, A Unified View of Spectral Clustering, Theorem 3.1, p. 7](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.115.1591&rep=rep1&type=pdf).

8. For $0\leq \beta <\frac{n}{2}$, the solution of the relaxed proportional cut problem is

$$
y=\pm \frac{2\beta}{n\sqrt{n}}\mathbf{1}\pm
\sqrt{1-4\frac{\beta ^{2}}{n^{2}}}v^{[2]},$$

where $v^{[2]}$ is an eigenvector corresponding to $\lambda_2(L)$. $v^{[2]}$ the __Fiedler vector__. Since the first summand carries no information, $V$ is partitioned according to the signs of the components of $v^{[2]}$:

$$
V_{1}=\{i:v^{[2]}_i <0\}, \quad V_{2}=\{i:v^{[2]}_i \geq 0\}.$$

_Notice that the value of $\beta$ is irrelevant for the solution._

9. For $0\leq \beta <\sqrt{\theta n}\left\Vert D^{\frac{1}{2}}\mathbf{1} \right\Vert _{2},$ the solution of the relaxed normalized cut problem is

$$
y=\pm \frac{\beta }{\sqrt{\theta n}\left\Vert
D^{\frac{1}{2}} \mathbf{1}\right\Vert _{2}^{2}}\mathbf{1}\pm
\sqrt{1-\frac{\beta ^{2}}{ \theta n\left\Vert
D^{\frac{1}{2}}\mathbf{1}\right\Vert _{2}^{2}}}D^{-\frac{1
}{2}} v_n^{[2]},$$

where $v_n^{[2]}$ is an eigenvector corresponding to $\lambda_2(L_n)$. $V_n$ is partitioned according to the signs of the components of $v_n^{[2]}$, as above.

10. Neither of the relaxed algorithms is guaranteed to solve exactly the true proportional / normalized cut problem. However, the computed solutions are in the right direction. Whether to use proportional or normalized cut formulation, depends upon the specific problem.
"""

# ╔═╡ c188b7cf-03e7-40b3-bf90-773978c7b410
# Voila!
eigs(L,nev=2,which=:SM, v0=ones(n))

# ╔═╡ f60b51b6-79c3-4a3f-8bfb-8f198cdb92a8
# For the normalized cut
eigs(Lₙ,nev=2,which=:SM, v0=ones(n))

# ╔═╡ bfac7a55-0d82-4549-b999-7db3272eaa20
md"""
### Concentric rings

A __complete graph__ has edges connecting each pair of vertices.

To a set of points $X=\{x_{1},x_{2},\cdots ,x_{m}\}$ , where $x_{i}\in\mathbb{R}^{n}$, we assign a weighted complete graph $G=(V,E)$ with $m$ vertices, where the vertex $j\in V$ corresponds to the point $x_j\in X$.

The main idea is to assign weight of an edge $e=(i,j)$ which reflects the distance between $x_i$ and $x_j$, something like $\omega(e)=\displaystyle\frac{1}{\mathop{\mathrm{dist}}(x_i,x_j)}$.

However, this has to be implemented with care. For example, using simple Euclidean distance yield the same results as the function `kmeans()`. In this example we use Gaussian kernel, that is

$$
\omega(e)=e^{\displaystyle -\|x_i-x_j\|_2^2/\sigma^2},$$

where the choice of $\sigma$ is based on experience.

The computation of various distances is implemented in the package [Distances.jl](https://github.com/JuliaStats/Distances.jl).

We will construct the Laplace matrix directly.
"""

# ╔═╡ 80ed91ad-6f9a-4f12-a1ee-bc8dc6b58168
begin
	# Two concentric circles
	k=2
	# Center
	Random.seed!(541)
	# center=[rand(-5:5),rand(-5:5)]
	center=[0,0]
	# Radii
	radii=randperm(10)[1:k]
	# Number of points in circles
	sizes=rand(1000:2000,k)
	center,radii,sizes
end

# ╔═╡ 2d8689c9-1b9e-4360-865c-23c2c8c5dcb9
begin
	# Generate points
	X=Array{Float64}(undef,2,sum(sizes))
	csizes=cumsum(sizes)
	# Random angles
	ϕ=2*π*rand(sum(sizes))
	for i=1:csizes[1]
		X[:,i]=center+radii[1]*[cos(ϕ[i]);sin(ϕ[i])] + (rand(2).-0.5)/50
	end
	for j=2:k
		for i=csizes[j-1]+1:csizes[j]
			X[:,i]=center+radii[j]*[cos(ϕ[i]);sin(ϕ[i])] + (rand(2).-0.5)/50
		end
	end
	scatter(X[1,:],X[2,:],title="Concentric rings", aspect_ratio=1,label="Points")
end

# ╔═╡ fa4ae713-a0b2-46ce-b5f7-558b26d82019
# Weight matrix
W₁=1 ./pairwise(SqEuclidean(),X)

# ╔═╡ 760e8ca2-596f-435f-befb-140b3d204f7f
begin
	# Laplacian matrix
	m=csizes[end]
	for i=1:m
	    W₁[i,i]=0
	end
	L₁=Diagonal(vec(sum(W₁,dims=2)))-W₁
	# Check Fact 3
	norm(L₁*ones(m))
end

# ╔═╡ 75956488-76ab-4262-a32f-0c53e1b3dc17
# Notice λ₁=0
E=eigs(L₁,nev=2,which=:SM, v0=ones(m))

# ╔═╡ a55260dd-f05d-4a65-9fdf-cb3fd4638857
begin
	# Define clusters
	C=ones(Int64,m)
	C[findall(E[2][:,2].>0)].=2
	C
end

# ╔═╡ ba164ff3-5ef4-4480-8af2-9dd9a99b1e8d
# Yet another plotting function
function plotKpartresult(C::Vector,X::Array)
	scatter(aspect_ratio=1)
    k=maximum(C)
    for j=1:k
        scatter!(X[1,findall(C.==j)],X[2,findall(C.==j)],label="Cluster $j")
    end
	scatter!(aspect_ratio=1)
end

# ╔═╡ 46de28bc-73ed-495c-a2e0-f565c1ce5651
plotKpartresult(C,X)

# ╔═╡ e004e4bf-ca04-4b41-947b-9f1ecd058abb
md"""
This is the same partitioning as obtained by `kmeans()`. Let us try Gaussian kernel. A rule of thumb is: if rings are close, use $\sigma<1$, if rings are apart, use $\sigma>1$.
"""

# ╔═╡ bb9d4d98-3aff-439e-b8a3-8347cf345839
begin
	σ=0.7 # 0.1
	W₂=exp.(-pairwise(SqEuclidean(),X)/σ^2)-I
	L₂=Diagonal(vec(sum(W₂,dims=2)))-W₂
	E₂=eigs(L₂,nev=2,which=:SM, v0=ones(m))
	C₂=ones(Int64,m)
	C₂[findall(E₂[2][:,2].>0)].=2
	plotKpartresult(C₂,X)
end

# ╔═╡ 012fa54a-5171-4980-a0ba-474a22c25981
md"""
# Recursive bipartitioning

## Definitions

Let $G=(V,E)$ be a weighted graph with weights $\omega$.

Let $\pi_k =\{V_{1},V_{2},...,V_{k}\}$ be a $k$-partition of $V$, with $V_i\neq \emptyset$ for $i=1,\ldots,k$.

The previous definition of $cut(\pi)\equiv cut(\pi_2)$ extends naturally to $k$-partition.

A __cut__ of a partition $\pi_k$ is

$$
\mathop{\mathrm{cut}}(\pi_k)=\sum\limits_{\displaystyle i<j} \mathop{\mathrm{cut}}(V_{i},V_{j}),$$

where $\mathop{\mathrm{cut}}(V_{i},V_{j})$ is interpreted as a cut of the bipartition of the subgraph of $G$ with vertices $V_1\cup V_2$.

__Proportional cut__ of a partition $\pi_k$ is

$$
\mathop{\mathrm{pcut}}(\pi_k)=\underset{i<j}{\sum\limits_{i,j=1}^{k}} \left(
\frac{\mathop{\mathrm{cut}}(V_{i},V_{j})}{|V_{i}|}+\frac{\mathop{\mathrm{cut}}(V_{i},V_{j})}{|V_{j}|}\right) =
\sum_{i=1}^{k}\frac{\mathop{\mathrm{cut}}(V_{i},V\backslash V_{i})}{|V_{i}|}.$$

__Normalized cut__ of a partition $\pi_k$ is

$$
\mathop{\mathrm{ncut}}(\pi_k)=\underset{i<j}{\sum\limits_{i,j=1}^{k}} \left(
\frac{\mathop{\mathrm{cut}}(V_{i},V_{j})}{\omega(V_{i})}+\frac{\mathop{\mathrm{cut}}(V_{i},V_{j})}{\omega(V_{j})}\right) =
\sum_{i=1}^{k}\frac{\mathop{\mathrm{cut}}(V_{i},V\backslash V_{i})}{ \omega(V_{i})}.$$

## Facts

If we want to cluster vertices of graph $G=(V,E)$ into $k$ clusters, we can apply the following recursive algorithm:

1. __Initialization.__ Compute the bipartition $\pi=\{V_{1},V_{2}\}$ of $V$. Set the counter $c=2$.

2. __Recursion.__ While $c<k$ repeat:

    1. Compute the bipartition of each subset of $V$.

    2. Among all $(c+1)$-partitions, choose the one with the smallest $\mathop{\mathrm{pcut}}(\pi_{c+1})$ or $\mathop{\mathrm{ncut}}(\pi_{c+1})$, respectively.

    3. Set $c=c+1$.

3. __Stop.__

There is no guarantee for optimality of this algorithm. Clearly, the optimal $k$-partiton may be a subpartition of one of the discarded partitions.
"""

# ╔═╡ f3eb8ad4-55a8-4e76-a581-07f9a7cc75f7

# ╔═╡ Cell order:
# ╟─7ef4b0b3-9d0a-4356-bd16-14487f14e0ab
# ╟─cb61f761-b2ee-4818-9149-f85d95b76a1e
# ╟─5926c79a-6038-4dbc-b49a-5b05d02250a0
# ╟─a66a6e80-1f80-11eb-15f8-71ffaa5050e7
# ╟─d4cf12ce-12fb-4562-bbea-e9b85cba94c2
# ╠═0cd92d78-1da6-435b-ad8e-60af29502e1d
# ╠═87cca1d3-b944-4d74-bf4c-1562f34cf91a
# ╠═3d7b4a01-9a7d-4111-a45d-1d31f1ef6278
# ╠═3b5ed1fa-75b0-43fb-a918-4c2e8f52d039
# ╠═8f7d1f6c-ee4c-407f-80d9-d2abc2af949e
# ╠═b7f127d8-e8cd-4771-8091-c88f9ffe6b08
# ╠═3f5447bb-95cd-4520-b0b9-e85f33b323bd
# ╠═ecb2a46e-f9dc-4e57-812f-0a2788afb202
# ╠═86e5f00f-13ee-4867-b0cd-187f2c782a46
# ╠═cf26dbe9-3c4f-4c95-88ef-f197f360d759
# ╠═8fcd2440-4013-4b37-a747-2c0c7f6d4aeb
# ╠═0c8d3572-3436-446f-952a-b19d0370ab38
# ╠═1437aadc-1c26-4eff-bb22-9cf9c15e1f5f
# ╟─6529324b-49f7-4f01-b5cd-33668979b56b
# ╟─a9ba64a6-bd17-4b86-acb4-e7c7d8f87dc1
# ╠═6a6f5db5-7dd4-4a26-ac1a-ef3ca3a5d6be
# ╠═e11bc23d-03e2-4780-9488-93f0adfc2fcb
# ╠═7192f7d0-28b9-11eb-28dd-8db0019504ca
# ╠═e5890dca-368f-437e-bc45-6a395714dae2
# ╠═61a978a7-b2d2-43e7-b3f7-95bf4c4f54b9
# ╠═da7f85ed-73b0-44b9-a97f-4a8ed5e9a1d1
# ╠═fed7f592-be03-4baf-8ace-ff9ee78fee71
# ╟─9f31928e-c933-43bb-9a5c-bf55f29737cd
# ╟─1e6d4c72-f25d-4bf5-99d6-ae0446be25dd
# ╟─02035eed-6d49-4253-a755-68a999e7e90e
# ╠═c188b7cf-03e7-40b3-bf90-773978c7b410
# ╠═f60b51b6-79c3-4a3f-8bfb-8f198cdb92a8
# ╟─bfac7a55-0d82-4549-b999-7db3272eaa20
# ╠═80ed91ad-6f9a-4f12-a1ee-bc8dc6b58168
# ╠═2d8689c9-1b9e-4360-865c-23c2c8c5dcb9
# ╠═fa4ae713-a0b2-46ce-b5f7-558b26d82019
# ╠═760e8ca2-596f-435f-befb-140b3d204f7f
# ╠═75956488-76ab-4262-a32f-0c53e1b3dc17
# ╠═a55260dd-f05d-4a65-9fdf-cb3fd4638857
# ╠═ba164ff3-5ef4-4480-8af2-9dd9a99b1e8d
# ╠═46de28bc-73ed-495c-a2e0-f565c1ce5651
# ╟─e004e4bf-ca04-4b41-947b-9f1ecd058abb
# ╠═bb9d4d98-3aff-439e-b8a3-8347cf345839
# ╟─012fa54a-5171-4980-a0ba-474a22c25981
# ╠═f3eb8ad4-55a8-4e76-a581-07f9a7cc75f7

src/Module C - Applications/L10 Spectral Graph Bipartitioning.jl


# News Boy Problem solved by L-Shaped Benders Decomposition
# Author: Edward J. Xu, edxu96@outlook.com
# Date: April 5th, 2019
# 0. Set working path, install packages and use module -----------------------------------------------------------------
push!(LOAD_PATH, "$(homedir())/Desktop/News Boy Problem, Stochastic Programming")
cd("$(homedir())/Desktop/News Boy Problem, Stochastic Programming")
using BendersLshapedStochasMilp_EDXU
using JuMP
using GLPKMathProgInterface
# 1. Date Input --------------------------------------------------------------------------------------------------------
demand = [12, 14, 16, 18, 20, 22, 24, 26, 28, 30] # Demand of newspapers in each scenario
lengthS = length(demand)
vec_prob = [0.05, 0.10, 0.10, 0.10, 0.15, 0.15, 0.10, 0.10, 0.10, 0.05 ] # probability of scenario
c = 20 # purchase price
p = 70 # selling price
h = 10 # scrap value
y = 20
# 2. Data Transfer -----------------------------------------------------------------------------------------------------
n_x = 10  # Number of all the x
n_y = 1
vec_min_y = hcat([0])
vec_max_y = hcat([30])
vec_f = hcat(c - 10)
vec_pi = hcat(vec_prob)
mat_c = zeros(10, 1, 1)
for i = 1:10
    mat_c[i, :, :] = hcat([- 70 + 10])
end
mat_h = zeros(10, 2, 1)
for i = 1: 10
    mat_h[i, 1, :] = hcat(- demand[i])
    mat_h[i, 2, :] = hcat(0)
end
mat3_t = zeros(10, 2, 1)
for i = 1:10
    mat3_t[i, 1, :] = hcat(0)
    mat3_t[i, 2, :] = hcat(1)
end
mat3_w = - ones(10, 2, 1)
epsilon = 0.001
timesIterationMax = 500
# 3. Begin Optimization ------------------------------------------------------------------------------------------------
BendersLshaped(; n_x = n_x, n_y = n_y, vec_min_y = vec_min_y, vec_max_y = vec_max_y, vec_f = vec_f,
    vec_pi = vec_pi, mat_c = mat_c, mat_h = mat_h, mat3_t = mat3_t, mat3_w = mat3_w,
    epsilon = epsilon, timesIterationMax = timesIterationMax)
# answer: obj = - 976


using NURBS
using Base.Test

@testset "RB-Spline curve generator" begin

	@testset "rbasis" begin
		x = knot(5, 4)
		h = [1, 1, 0.25, 1, 1]
		@test typeof(rbasis(5, 4, 0.5, x, h)) == Array{Float64,1}
		@test isapprox(rbasis(5, 4, 0.5, x, h), [0.153846, 0.730769, 0.0769231, 0.0384615, 0.0], atol = 1e-5)
	end

	@testset "rbspline" begin
		b = [0 1 5/2 4 5; 1 2 0 2 0; 0 0 0 0 0]
		h = [1, 1, 0.25, 1, 1]
		@test typeof(rbspline(5, 3, 10, b, h)[1]) == Array{Float64,2}
		@test typeof(rbspline(5, 3, 10, b, h)[2]) == Array{Array{Int64,1},1}
		@test isapprox(rbspline(5, 3, 10, b, h)[1], [0.0 0.557971 0.966667 1.3 1.95455 3.04545 3.7 4.03333 4.44203 5.0; 1.0 1.50725 1.73333 1.6 1.21212 1.21212 1.6 1.6 1.04348 0.0; 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0], atol = 1e-5)
	end

	@testset "rbsplinu" begin
		b = [0 1 5/2 4 5; 1 2 0 2 0; 0 0 0 0 0]
		h = [1, 1, 0.25, 1, 1]
		@test typeof(rbsplinu(5, 3, 10, b, h)[1]) == Array{Float64,2}
		@test typeof(rbsplinu(5, 3, 10, b, h)[2]) == Array{Array{Int64,1},1}
		@test isapprox(rbsplinu(5, 3, 10, b, h)[1], [0.5 0.789855 1.03333 1.3 1.95455 3.04545 3.7 3.96667 4.21014 4.75; 1.5 1.73913 1.8 1.6 1.21212 1.21212 1.6 1.73333 1.50725 0.5; 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0], atol = 1e-4)
	end

end


using Unitful, UnitfulAstro
using PhysicalConstants.CODATA2018: h, c_0, k_B

export blackbody

"""
    blackbody(wave::Vector{<:Quantity}, T::Quantity)
    blackbody(wave::Vector{<:Real}, T::Real)

Create a blackbody spectrum using Planck's law. The curve follows the mathematical form

``B_\\lambda(T) = \\frac{2hc^2}{\\lambda^5}\\frac{1}{e^{hc/\\lambda k_B T} - 1}``

If `wave` and `T` are not `Unitful.Quantity`, they are assumed to be in angstrom and Kelvin, and the returned flux will be in units `W m^-2 Å^-1`.

The physical constants are calculated using [PhysicalConstants.jl](https://github.com/juliaphysics/physicalconstants.jl), specifically the CODATA2018 measurement set.

# References
[Planck's Law](https://en.wikipedia.org/wiki/Planck%27s_law)

# Examples
```jldoctest
julia> using Spectra, Unitful, UnitfulAstro

julia> wave = range(1, 3, length=100)u"μm"
(1.0:0.020202020202020204:3.0) μm

julia> bb = blackbody(wave, 2000u"K")
UnitfulSpectrum (100,)
  λ (μm) f (W μm^-1 m^-2)
  T: 2000 K
  name: Blackbody

julia> blackbody(ustrip.(u"angstrom", wave), 6000)
Spectrum (100,)
  T: 6000
  name: Blackbody

julia> bb.wave[argmax(bb)]
1.4444444444444444 μm

julia> 2898u"μm*K" / bb.T # See if it matches up with Wien's law
1.449 μm
```
"""
function blackbody(wave::AbstractVector{<:Quantity}, T::Quantity)
    out_unit = u"W/m^2" / unit(eltype(wave))
    flux = _blackbody(wave, T) .|> out_unit
    return spectrum(wave, flux, name = "Blackbody", T = T)
end

function blackbody(wave::AbstractVector{<:Real}, T::Real)
    flux = ustrip.(u"W/m^2/angstrom", _blackbody(wave * u"angstrom", T * u"K"))
    return spectrum(wave, flux, name = "Blackbody", T = T)
end

_blackbody(wave::AbstractVector{<:Quantity}, T::Quantity) = blackbody(T).(wave)

"""
  blackbody(T::Quantity)

Returns a function for calculating blackbody curves.
"""
blackbody(T::Quantity) = w->2h * c_0^2 / w^5 / (exp(h * c_0 / (w * k_B * T)) - 1)


@testset "Kolmogorov-Smirnov test" begin
    x = [
        392,585,410,414,384,423,456,549,455,406,470,402,416,431,435,413,468,435,
        432,438,425,392,410,498,475,431,460,440,420,422,409,417,370,500,501,478,
        454,444,447,393,446,454,500,433,444,427,412,433,460,434
    ]
    res_kstest = ks_test(x, Normal(mean(x), std(x)))
    @test res_kstest.statistic ≈ 0.119089045870544
    @test res_kstest.n_obs == 50
    @test res_kstest.pval ≈ 0.477419559223632
end


## PART 2

# Define the error functions

function σf1(v̄̂,σ̂)
    ddvs = zeros(5);
    ddvs[1] = 1 / v̄̂[2]^2;
    ddvs[2] = v̄̂[1]^2 / v̄̂[2]^4;
    return sqrt( dot( ddvs, σ̂.^2) );
end

function σf2(v̄̂,σ̂)
    ddvs = zeros(5);
    ddvs[3] = exp(v̄̂[3] - v̄̂[4])^2;
    ddvs[4] = exp(v̄̂[3] - v̄̂[4])^2;
    return sqrt( dot( ddvs, σ̂.^2) );
end

function σf3(v̄̂,σ̂)
    lv5s = log(v̄̂[5])^2;
    ddvs = zeros(5);
    ddvs[1] = lv5s / v̄̂[2]^2;
    ddvs[2] = lv5s * v̄̂[1]^2 / v̄̂[2]^4;
    ddvs[3] = lv5s / v̄̂[4]^2;
    ddvs[4] = lv5s * v̄̂[3]^2 / v̄̂[4]^4;
    ddvs[5] = ( ( v̄̂[1]/v̄̂[2] + v̄̂[3]/v̄̂[4] ) / v̄̂[5] )^2;
    return sqrt( dot( ddvs, σ̂.^2) );
end

println("Calculating σ̂f1, σ̂f2, σ̂f3 for sample sizes N1, N2");
σ̂f1N1 = σf1(v̄̂,σ̂1);
σ̂f2N1 = σf2(v̄̂,σ̂1);
σ̂f3N1 = σf3(v̄̂,σ̂1);

σ̂f1N2 = σf1(v̄̂,σ̂2);
σ̂f2N2 = σf2(v̄̂,σ̂2);
σ̂f3N2 = σf3(v̄̂,σ̂2);

println("\n\tNaive standard deviations for sample size 1,000");
println("\tσ̂f1N1 = $σ̂f1N1");
println("\tσ̂f2N1 = $σ̂f2N1");
println("\tσ̂f3N1 = $σ̂f3N1");

println("\n\tNaive standard deviations for sample size 10,000");
println("\tσ̂f1N2 = $σ̂f1N2");
println("\tσ̂f2N2 = $σ̂f2N2");
println("\tσ̂f3N2 = $σ̂f3N2");

println("\n\tTrue standard deviations for sample size 1,000");
println("\tσ̂truef1N1 = $σ̂truef1N1");
println("\tσ̂truef2N1 = $σ̂truef2N1");
println("\tσ̂truef3N1 = $σ̂truef3N1");

println("\n\tTrue standard deviations for sample size 10,000");
println("\tσ̂truef1N2 = $σ̂truef1N2");
println("\tσ̂truef2N2 = $σ̂truef2N2");
println("\tσ̂truef3N2 = $σ̂truef3N2");


"""
	cracks2moduli(E₀, nu₀, α1, α3, β11, β13, β33)

Compute 6×6 stiffness matrix (in Voight notation) for a cracked material characterised by normalised crack density parameters α1, α3, β11, β13 and β33, with solid matrix moduli E₀ and nu₀. Crack density parameters are normalised by h = 3E₀(2-nu₀)/(32(1-nu₀²)). From Sayers and Kachanov 1995.
"""
function cracks2moduli(E₀, nu₀, α1, α3, β11, β13, β33)
    S011 = 1/E₀
    S012 = -nu₀/E₀
    h = cracknormfactor(E₀, nu₀)

    D = (S011 + α3/h + β33/h)*(S011 + S012 + α1/h + 4β11/h/3) -
        2*(S012 + β13/h)^2

    c11 = (1/2)*((S011+α3/h+β33/h)/D + 1/(S011-S012+α1/h+2β11/h/3))
    c33 = (S011 + S012 + α1/h + 4β11/h/3)/D
    c44 = 1/(2S011 - 2S012 + α1/h + α3/h + 4β13/h)
    c13 = -(S012 + β13/h)/D
    c66 = 1/(S011-S012+α1/h+2β11/h/3)/2

    return stiffmatrixTI(c11, c13, c33, c44, c66)
end

"""
	cracknormfactor(E₀, ν₀)

Compute normalisation factor h=3E₀*(2-nu₀)/(32(1-nu₀²)).
"""
function cracknormfactor(E₀, ν₀)
    return 3E₀*(2-ν₀)/(32(1-ν₀^2))
end

"""
	findcracks(p::Vector{VMeasure},
                    sv::Vector{VMeasure},
                    sh::Vector{VMeasure},
                    ρ::Real,
                    E₀::Real,
                    nu₀::Real,
                    α₁₀::Tuple{Real,Real},
                    α₃₀::Tuple{Real,Real},
                    β₁₁₀::Tuple{Real,Real},
                    β₁₃₀::Tuple{Real,Real},
                    β₃₃₀::Tuple{Real,Real})

Estimate normalised crack density parameters and phase angles from measurements of group P, SV ans SH wave velocities, using quasinewton method. A priori values ()₀ are tuples with value and standard deviation (a priori gaussian).

Return α₁₁, α₃₃, β₁₁₁₁, β₁₁₃₃, β₃₃₃₃, θp, θsv, θsh, CMpost.
"""
function findcracks(p::Vector{VMeasure},
                    sv::Vector{VMeasure},
                    sh::Vector{VMeasure},
                    ρ::Real,
                    E₀::Real,
                    nu₀::Real,
                    α₁₀::Tuple{Real,Real},
                    α₃₀::Tuple{Real,Real},
                    β₁₁₀::Tuple{Real,Real},
                    β₁₃₀::Tuple{Real,Real},
                    β₃₃₀::Tuple{Real,Real})

    dobs, CDi, np, nsv, nsh = inputdobs(p,sv,sh)

    #make vector of model parameters (first guess)
    m₀ = [α₁₀[1],α₃₀[1],β₁₁₀[1],β₁₃₀[1],β₃₃₀[1]]
    σm = [α₁₀[2],α₃₀[2],β₁₁₀[2],β₁₃₀[2],β₃₃₀[2]]

    C₀ = cracks2moduli(E₀, nu₀, m₀...)
    ϵ₀, δ₀, γ₀ = moduli2thomsen(C₀)
    vp,vs = velocities0(C₀,ρ) # here density does not matter

    for x in p
        push!(m₀, phaseangleP(x.angle, vp/vs, ϵ₀, δ₀))
        push!(σm, 100)
    end
    for x in sv
        push!(m₀, phaseangleSV(x.angle, vp/vs, ϵ₀, δ₀))
        push!(σm, 100)
    end
    for x in sh
        push!(m₀, phaseangleSH(x.angle, γ₀))
        push!(σm, 100)
    end

    CM = Diagonal(σm.^2)
    CMi = inv(CM)

    m, CMpost = quasinewton(dobs, _gf, m₀, CMi, CDi, (E₀, nu₀, ρ,np,nsv,nsh))

    return m[1:5], m[6:5+np], m[6+np:5+np+nsv], m[6+np+nsv:5+np+nsv+nsh], CMpost

end

function _gf(m,E₀,nu₀,ρ,np,nsv,nsh)
    C₀ = cracks2moduli(E₀, nu₀, m[1:5]...)
    ϵ, δ, γ = moduli2thomsen(C₀)
    vp0,vs0 = velocities0(C₀,ρ)
    mt = vcat([vp0,vs0,ϵ,δ,γ], m[6:end])
    return _gt(mt, np, nsv, nsh)
end


function vector_identities(c)
    A = ContravariantVector(2*randn(3),c,unit_basis=true)
    B = ContravariantVector(2*randn(3),c,unit_basis=true)
    C = CovariantVector(2*randn(3),c,unit_basis=true)
    D = CovariantVector(2*randn(3),c,unit_basis=true)

    ϵ = eps(abs(float(one(eltype(A)))))

    A_BxC = dot(A,cross(B,C))
    @test isapprox(A_BxC, dot(cross(A,B),C), atol=500ϵ)
    @test isapprox(A_BxC, dot(B,cross(C,A)), atol=500ϵ)
    @test isapprox(A_BxC, dot(cross(B,C),A), atol=500ϵ)
    @test isapprox(A_BxC, dot(C,cross(A,B)), atol=500ϵ)
    @test isapprox(A_BxC, dot(cross(C,A),B), atol=500ϵ)

    AxBxC = cross(A,cross(B,C))
    @test isapprox(AxBxC, cross(cross(C,B),A), atol=500ϵ)
    @test isapprox(AxBxC, dot(A,C)*B - dot(A,B)*C, atol=500ϵ)

    @test isapprox(cross(A,cross(B,C)) + cross(B,cross(C,A)) + cross(C,cross(A,B)), zeros(eltype(A),length(A)), atol=2000ϵ)

    AxB_CxD = dot(cross(A,B),cross(C,D))
    @test isapprox(AxB_CxD, dot(A,C)*dot(B,D) - dot(A,D)*dot(B,C), atol=2000ϵ)

    AxBxCxD = cross(cross(A,B),cross(C,D))
    @test isapprox(AxBxCxD, (dot(cross(A,B),D)*C - dot(cross(A,B),C)*D), atol=2000ϵ)
end

@testset "Algebraic Operations" begin
    println("Testing Algebraic Identities...")
    @testset "Algebraic Identities" begin
        @testset "Cartesian" begin
            c = CartesianCoordinate([2.0,-1.0,1.5])
            vector_identities(c);
        end
        @testset "Cylindrical" begin
            c = CylindricalCoordinate([2.0,pi/4,-1.0])
            vector_identities(c);
        end
        @testset "Spherical" begin
            c = SphericalCoordinate([2.0,pi/4,pi/3])
            vector_identities(c);
        end
    end
end


include("setting.jl")

# jacobian of lorenz96
function jac_lorenz(x::Vector{Float64})::Matrix{Float64}
    # jac[i, j] = -1 when j = i
    # jac[i, j] = x[i-1] when j = i+1
    # jac[i, j] = x[i+1] - x[i-2] when j = i-1
    # jac[i, j] = -x[i-1] when j = i-2
    len = length(x)
    lenm = len - 1;
    jac = zeros(len, len)

    for i = 1:len
        ip1 = (i+1+lenm)%len+1
        im1 = (i-1+lenm)%len+1
        im2 = (i-2+lenm)%len+1
        @inbounds jac[i, i] = -1
        @inbounds jac[i, ip1] = x[im1]
        @inbounds jac[i, im1] = x[ip1] - x[im2]
        @inbounds jac[i, im2] = -x[im1]
    end
    jac
end

a = repeat([1., 3, 6, 4, 5], 3)
da = repeat([.2, .2, .2, .2, .2], 3)
b = a + da
fa = lorenz96(a, Missing, Missing)
fb = lorenz96(b, Missing, Missing)
jfa = jac_lorenz(a) * da
display(jac_lorenz(a))
println()
@show fa + jfa - fb
@assert all(abs.(fa+jfa-fb) .< 1e-6)


# Converts infix expression into reverse Polish notation
# (Only 1-digit numbers are allowed.)

operators = {
    # opname => (operator, precedence)
    '+' => (+, 6),
    '-' => (-, 6),
    '*' => (*, 7),
    '/' => (/, 7),
}

is_operator(op) = haskey(operators, op)
get_operator(ch) = operators[ch][1];
get_precedence(ch) = operators[ch][2];
PREC_OFFSET = 100

function in2post(expr::ASCIIString)
    opstack = Char[]
    precedence_stack = Int[]
    output = Char[]
    prec_offset = 0

    for ch in expr
        if ch == '('
            prec_offset += PREC_OFFSET
        elseif ch == ')'
            prec_offset -= PREC_OFFSET
        elseif is_operator(ch)
            precedence = get_precedence(ch) + prec_offset
            while !isempty(opstack) &&
                precedence <= precedence_stack[end]
                push!(output, pop!(opstack))
                pop!(precedence_stack)
            end
            push!(opstack, ch)
            push!(precedence_stack, precedence)
        else
            push!(output, ch)
        end
    end

    if prec_offset != 0
        warn("Unbalanced parentheses!")
    end
    [output, reverse(opstack)]
end

function eval_post(postexp)
    numstack = Int[]
    for ch in postexp
        if is_operator(ch)
            b = pop!(numstack);
            a = pop!(numstack);
            push!(numstack, get_operator(ch)(a, b))
        else
            push!(numstack, ch - '0');
        end
    end
    assert(length(numstack) == 1)
    numstack[end]
end

expr = strip(readline())
postexp = in2post(expr)
@printf("The converted RPN is: %s,\nwhich = %d",
    join(postexp), eval_post(postexp))

# Conclusion: Julia is far from mature enough to use in production.


@doc raw"""
```
binarize(Balanced(), img)
```
Binarizes the image using the balanced histogram thresholding method.

# Output

Returns the binarized image as an `Array{Gray{Bool},2}`.

# Details
In balanced histogram thresholding, one interprets a  bin as a  physical weight
with a mass equal to its occupancy count. The balanced histogram method involves
iterating the following three steps: (1) choose the midpoint bin index as a
"pivot",  (2) compute the combined weight to the left and right of the pivot bin
and (3) remove the leftmost bin if the left side is the heaviest, and the
rightmost bin otherwise. The algorithm stops when only a single bin remains. The
last bin determines the sought-after threshold with which the image is
binarized.

Let ``f_n`` (``n = 1 \ldots N``) denote the number of observations in the ``n``th
bin of the image histogram. The balanced histogram method constructs a sequence
of nested intervals

```math
[1,N] \cap \mathbb{Z} \supset I_2 \supset I_3 \supset \ldots \supset I_{N-1},
```
where for ``k = 2 \ldots N-1``
```math
I_k = \begin{cases}
   I_{k-1} \setminus \{\min \left( I_{k-1} \right) \} &\text{if } \sum_{n = \min \left( I_{k-1} \right)}^{I_m}f_n \gt   \sum_{n =  I_m + 1}^{ \max \left( I_{k-1} \right)} f_n, \\
   I_{k-1} \setminus \{\max \left( I_{k-1} \right) \} &\text{otherwise},
\end{cases}
```
and ``I_m = \lfloor \frac{1}{2}\left(  \min \left( I_{k-1} \right) +  \max \left( I_{k-1} \right) \right) \rfloor ``.
The final interval ``I_{N-1}`` consists of a single element which is the bin index
corresponding to the desired threshold.

If one interprets a bin as a physical weight with a mass equal to its occupancy
count, then each step of the algorithm can be conceptualised as removing the
leftmost or rightmost bin to "balance" the resulting histogram on a pivot. The
pivot is defined to be the midpoint between the start and end points of the
interval under consideration.

If it turns out that the single element in ``I_{N-1}`` equals ``1`` or ``N`` then
the original histogram must have a single peak and the algorithm has failed to
find a suitable threshold. In this case the algorithm will fall back to using
the `UnimodalRosin` method to select the threshold.

# Arguments

The function argument is described in more detail below.

##  `img`

An `AbstractArray` representing an image. The image is automatically converted
to `Gray` in order to construct the requisite graylevel histogram.

# Example

Binarize the "cameraman" image in the `TestImages` package.

```julia
using TestImages, ImageBinarization

img = testimage("cameraman")
img_binary = binarize(Balanced(), img)
```

# Reference

1. “BI-LEVEL IMAGE THRESHOLDING - A Fast Method”, Proceedings of the First International Conference on Bio-inspired Systems and Signal Processing, 2008. Available: [10.5220/0001064300700076](https://doi.org/10.5220/0001064300700076)
"""
function binarize(algorithm::Balanced,  img::AbstractArray{T,2}) where T <: Colorant
  img₀₁ = zeros(Gray{Bool}, axes(img))
  edges, counts = build_histogram(img,  256)
  t = find_threshold(HistogramThresholding.Balanced(), counts[1:end], edges)
  for i in CartesianIndices(img)
    img₀₁[i] = img[i] < t ? 0 : 1
  end
  img₀₁
end


using Printf

function distcheck(f::Function, rep::Int=10000, Δ::Int=3)
    smpl = f(rep)
    counts = Dict(k => count(smpl .== k) for k in unique(smpl))
    expected = rep / length(counts)
    lbound = expected * (1 - 0.01Δ)
    ubound = expected * (1 + 0.01Δ)
    noobs = count(x -> !(lbound ≤ x ≤ ubound), values(counts))
    if noobs > 0 warn(@sprintf "%2.4f%% values out of bounds" noobs / rep) end
    return counts
end

# Dice5 check
distcheck(x -> rand(1:5, x))
# Dice7 check
distcheck(dice7)


function boxcox(z, λ)
    reduce(&, z .> 0) || throw(DomainError("boxcox requires strictly positive data"))
	if λ == 0
		return log.(z)
	else
		return (z.^λ .- 1) ./ λ
	end
end

function inv_boxcox(z, λ)
    if λ == 0
        return exp.(z)
    else
        return (λ .* z .+ 1).^(1/λ)
    end
end

function guerrero(z; lower=-1, upper=2, s=2)
    s >= 2 || throw(ArgumentError("guerrero requires seasonality to be higher or equal to 2"))
    N = length(z)
    rz = reshape(view(z, (N % s + 1):N), N ÷ s, s)
    m = mean(rz, dims=1)
    sd = std(rz, mean=m, dims=1)
    function cv(λ)
        w = sd ./ (m.^(1 - λ))
        mw = mean(w)
        return std(w, mean=mw) / mw
    end
    return optimize(cv, lower, upper).minimizer
end

function difference(z::AbstractVector{T}; d::Integer=1,s::Integer=1) where T
    N = length(z)
    differenced = Vector{T}(undef, N)
    @inbounds @simd for i in 1:N
        differenced[i] = z[i]
    end
    @inbounds for j in 1:d
        for i in 1:N - j * s
            differenced[N - i + 1] -= differenced[N - i + 1 - s]
        end
    end
    return differenced[1 + d * s:N]
end

function integrate(z::AbstractVector{T}, z0::AbstractVector; d::Integer=1,s::Integer=1) where T
    N = length(z)
    integrated = Vector{T}(undef, N + d * s)
    N0 = length(z0)
    for i in 1:N + d * s
        integrated[i] = i > d * s ? z[i - d * s] : i > N0 ? zero(T) : z0[i]
    end
    for j in 1:d
        for i in (1 + s):N + d * s
            integrated[i] += integrated[i - s]
        end
    end
    return integrated
end

integrate(z::AbstractVector{T}; d::Integer=1,s::Integer=1) where T = integrate(z, T[], d=d, s=s)


using Plots,ApproxFun;pyplot()

x = Fun(identity,[0.,10.])
f = sin(x^2)
g = cos(x)

h = f + g^2
r = roots(h)
rp = roots(differentiate(h))
plot(h;legend=false,grid=false)
scatter!(r,h(r))
scatter!(rp,h(rp))
xlabel!("\$x\$")
ylabel!("\$h(x)\$")
xlims!(0,10)
PyPlot.savefig("extrema.png",dpi=300)
println("First image done")

x = Fun(identity,[-1000.,200.])
d = domain(x)
D = Derivative(d)
B = dirichlet(d)
L = D^2 - x
u = [B;L] \ [airyai(d.a),airyai(d.b)]
plot(u;legend=false,grid=false)						    # Requires Gadfly or PyPlot
xlabel!("\$x\$");ylabel!("\${\\rm Ai}(x)\$")
PyPlot.savefig("airy.png",dpi=300)

println("Second image done")

d = Interval(-π,π)
a = Fun(t-> 1+sin(cos(2t)),d)
D = Derivative(d)
L = D + a
f = Fun(t->exp(sin(10t)),d)
B = periodic(d,0)
uChebyshev = [B;L]\[0.,f]

d = PeriodicSegment([-π,π])
a = Fun(t-> 1+sin(cos(2t)),d)
D = Derivative(d)
L = D + a
f = Fun(t->exp(sin(10t)),d)
uFourier = L\f

length(uFourier)/length(uChebyshev),2/π
plot(real(uFourier);legend=false,grid=false)						    # Requires Gadfly or PyPlot
xlims!(d.a,d.b);xlabel!("\$t\$");ylabel!("\$u(t)\$")
PyPlot.savefig("periodic.png",dpi=300)

println("Third image done")

f = abs(Fun(sin,[-5.,5.]))
d = domain(f)
x = ApproxFun.sample(f,10000)
plot(f/sum(f);legend=false,grid=false)                           # Requires Gadfly or PyPlot
histogram!(x;normed=true,nbins=100)
xlims!(leftendpoint(d),rightendpoint(d));ylims!(0.0,0.18);xlabel!("\$x\$");ylabel!("Density")
PyPlot.savefig("Sample.png",dpi=300)

println("Fourth image done")

## Nonlinear BVP

x=Fun()
u0=0x

N=u->[u(-1.)-1.,u(1.)+0.5,0.001u''+6*(1-x^2)*u'+u^2-1.]
u=newton(N,u0)
plot(u;legend=false,grid=false)
PyPlot.savefig("nbvp.png",dpi=300)

println("Fifth image done")

## Multivariate

d = ChebyshevInterval()^2                            # Defines a rectangle

u = [dirichlet(d);lap(d)+100I]\[[1.0,1.0,1.0,1.0]]      # First four entries of rhs are
                                            # boundary conditions
surface(u,grid=false)                                     # contour plot

PyPlot.savefig("helmholtz.png",dpi=300)


using Pythia
using Test
using CSV
using DataFrames

# Using Examples from Forecasting: Principles and Practice (2nd ed.) by Rob J Hyndman and George Athanasopoulos
# Ground Truth Taken from statsmodels

### Exponential Smoothing Tests ###

## Simple Exponential Smoothing ##

# Import Data
oildata = CSV.read("fpp2_datasets/oil.csv", DataFrame)
y = oildata[!, 3]
y = y[32:49]

# Initialize Model and Make Prediction
mdl = SES(y, h = 5)
fitted_mdl = fit(mdl, v = 0)
yhat = predict(fitted_mdl)
yactual = [542.6803683882322, 542.6803683882322, 542.6803683882322, 542.6803683882322, 542.6803683882322]

@test yhat ≈ yactual rtol=0.05
# Compare yhat with ground truth

## Holt's Linear Trend Method ##

# Import Data
ausair = CSV.read("fpp2_datasets/ausair.csv", DataFrame)
y = ausair[!, 3]
y = y[21:47]

# Initialize Model and Make Prediction
mdl = Holt(y, h = 5)
fitted_mdl = fit(mdl, v = 0)
yhat = predict(fitted_mdl)
yactual = [74.59331490568717, 76.69123025401483, 78.7891456023425, 80.88706095067016, 82.98497629899784]
@test yhat ≈ yactual rtol=0.05

## Holt's Linear Method - Damped ##
mdl = Holt(y, h = 5, damped = true)
fitted_mdl = fit(mdl, v = 0)
yhat = predict(fitted_mdl)
yactual = [74.4245006855701, 76.3634111773908, 78.2926270423973, 80.21219675409546, 82.12216854362171]
@test yhat ≈ yactual rtol=0.05

## HoltWinter's Method ##

# Import Data
austourists = CSV.read("fpp2_datasets/austourists.csv", DataFrame)
y = austourists[!, 3]
y = y[25:68]

# Initialize Model and Make Prediction
mdl = HoltWinters(y, h = 5, m = 4, init_season = [9.70, -9.31, -1.69, 1.31])
fitted_mdl = fit(mdl, v = 0)
yhat = predict(fitted_mdl)
# @test yhat ≈ yactual rtol=0.05


__precompile__()

module Vandermonde

export pvand

function pvand(alpha, b)
    # x = pvand(alpha, b)
    #
    # Solves system A*x = b
    # A is Vandermonde matrix, with nonstandard definition
    # A(i,j) = alpha(j)^i
    #
    # Algorithm by Bjorck & Pereyra
    # Mathematics of Computation, Vol. 24, No. 112 (1970), pp. 893-903
    # https://doi.org/10.2307/2004623
    #
    n = length(alpha);
    x = copy(b);
    for k=1:n
        for j=n:-1:k+1
            x[j] = x[j]-alpha[k]*x[j-1];
        end
    end
    for k=n-1:-1:1
        for j=k+1:n
            x[j] = x[j]/(alpha[j]-alpha[j-k]);
        end
        for j=k:n-1
            x[j] = x[j]-x[j+1];
        end
    end
    return x
end

end


using OrdinaryDiffEq, ModelingToolkit, Test, LinearAlgebra
@parameters t
@variables y[1:3](t)
@parameters k[1:3]
D = Differential(t)

eqs = [D(y[1]) ~ -k[1] * y[1] + k[3] * y[2] * y[3],
    D(y[2]) ~ k[1] * y[1] - k[3] * y[2] * y[3] - k[2] * y[2]^2,
    0 ~ y[1] + y[2] + y[3] - 1]

@named sys = ODESystem(eqs, t, y, k)
@test_throws ArgumentError ODESystem(eqs, y[1])
M = calculate_massmatrix(sys)
@test M == [1 0 0
            0 1 0
            0 0 0]

f = ODEFunction(sys)
prob_mm = ODEProblem(f, [1.0, 0.0, 0.0], (0.0, 1e5), (0.04, 3e7, 1e4))
sol = solve(prob_mm, Rodas5(), reltol = 1e-8, abstol = 1e-8)

function rober(du, u, p, t)
    y₁, y₂, y₃ = u
    k₁, k₂, k₃ = p
    du[1] = -k₁ * y₁ + k₃ * y₂ * y₃
    du[2] = k₁ * y₁ - k₃ * y₂ * y₃ - k₂ * y₂^2
    du[3] = y₁ + y₂ + y₃ - 1
    nothing
end
f = ODEFunction(rober, mass_matrix = M)
prob_mm2 = ODEProblem(f, [1.0, 0.0, 0.0], (0.0, 1e5), (0.04, 3e7, 1e4))
sol2 = solve(prob_mm2, Rodas5(), reltol = 1e-8, abstol = 1e-8, tstops = sol.t,
             adaptive = false)

# MTK expression are canonicalized, so the floating point numbers are slightly
# different
@test Array(sol) ≈ Array(sol2)

# Test mass matrix in the identity case
eqs = [D(y[1]) ~ y[1], D(y[2]) ~ y[2], D(y[3]) ~ y[3]]

@named sys = ODESystem(eqs, t, y, k)

@test calculate_massmatrix(sys) === I


using SpecialFunctions

include("spindown_besselzeros.jl")

function u_spindown(r, t, Re)
    # Assuming unit radius, speed and viscosity
    Omega, a = 1.0, 1.0
    nu = 1.0/Re
    S = 0.0
    ds = 0.0
    relerr = 0.0
    for n=1:length(lambda)
        ds = besselj(1, lambda[n]*r/a) / (lambda[n]*besselj(0, lambda[n]))*exp(-lambda[n]^2*nu*t/a)
        S += ds
        relerr = abs(ds/S) # If convergence is nice enough
        if relerr < eps()
            break
        end
    end
    if relerr > eps()
        warn("Spindown solution not converged, relerr=", relerr)
    end
    u = -2*Omega*a*S
end

function Lu_spindown(r, t, Re)
    # Laplacian of the above
    Omega, a = 1.0, 1.0
    nu = 1.0/Re
    S = 0.0
    ds = 0.0
    relerr = 0.0
    for n=1:length(lambda)
        b = lambda[n]/a
        ds = -b^2*besselj(1, b*r) / (lambda[n]*besselj(0, lambda[n]))*exp(-lambda[n]^2*nu*t/a)
        S += ds
        relerr = abs(ds/S) # If convergence is nice enough
        if relerr < eps()
            break
        end
    end
    if relerr > eps()
        warn("Spindown solution not converged, relerr=", relerr)
    end
    u = -2*Omega*a*S
end

function Du_spindown(r, t, Re)
    # r derivative
    Omega, a = 1.0, 1.0
    nu = 1.0/Re
    S = 0.0
    ds = 0.0
    relerr = 0.0
    for n=1:length(lambda)
        b = lambda[n]/a
        ds = b/2*(besselj(0, b*r)-besselj(2, b*r)) / (lambda[n]*besselj(0, lambda[n]))*exp(-lambda[n]^2*nu*t/a)
        S += ds
        relerr = abs(ds/S) # If convergence is nice enough
        if relerr < eps()
            break
        end
    end
    if relerr > eps()
        warn("Spindown solution not converged, relerr=", relerr)
    end
    u = -2*Omega*a*S
end

function plot_spindown(t, Re)
    r = linspace(0,1,200)
    u = map(x -> u_spindown(x, t, Re), r)
    plot(r, u, label="Reference")
end


@testset "BasicEMD" begin
    @testset "BasicEMD Test 1" begin
        n = 200
        s = EMDSetting(n, 50, 10, 2)

        t = collect(range(0,1, length=n))
        x = zeros(n)

        for i=1:n
            x[i] = cos(22*pi*t[i]^2) + 6*t[i]^2
        end

        output = emd(x, s)

        @test size(output, 2) == s.m

        recovered = zeros(n)

        for i=1:size(output, 2)
            recovered += output[:, i]
        end

        @test recovered == x || isapprox(recovered, x)
    end

    @testset "BasicEMD Test 2" begin
        N = 1024
        s = EMDSetting(N, 50, 10, 10)

        function input_signal(x::Float64)
            omega = 2*pi/(N-1)
            return sin(17*omega*x)+0.5*(1.0-exp(-0.002*x))*sin(51*omega*x+1)
        end

        input = zeros(N)

        for i=1:N
            input[i] = input_signal(float(i))
        end

        output = emd(input, s)

        @test size(output, 2) == s.m

        recovered = zeros(N)

        for i=1:size(output, 2)
            recovered += output[:, i]
        end

        @test recovered == input || isapprox(recovered, input)
    end
end