using LinearAlgebra
function Sparse_matrix_SINDYPI(Theta, dx, lambda, nvars, maxiter)

normLib = zeros(1, size(Theta,2))

Thetan = deepcopy(Theta)

    for norm_k=1:size(Theta,2)
            normLib[norm_k] = norm(Thetan[:,norm_k]);
            Thetan[:,norm_k] = Thetan[:,norm_k]/normLib[norm_k];
    end

   #Epsilon = Thetan\dx

#Below:Only to be used for ill conditioned matrices, like in the glycolysis example. Check on this
Epsilon = zeros(size(Thetan,2),1)
ldiv!(Epsilon, qr(Thetan), dx)

#=
   Epsilon_c = similar(Epsilon)
   Epsilon_c .= Epsilon
=#

    for i in 1:maxiter
        index_remove = abs.(Epsilon) .<= lambda
        Epsilon[index_remove] .= zero(eltype(Epsilon))

        for j in 1:nvars
            index_retain = @. !index_remove[:,j]
            Epsilon[index_retain, j] = Thetan[:, index_retain] \ dx[:,j]
        end
   #=
    if norm(Epsilon_c - Epsilon, 2) < eps()
        break
    else
        Epsilon_c .= Epsilon
    end
=#
    end

        for norm_k=1:length(Epsilon)
            Epsilon[norm_k,:] = Epsilon[norm_k,:]/normLib[norm_k];
        end

    return Epsilon
end

#=
A. Q factor

qrf.R

Ans = inv(qrf.R)*(qrf.Q)'*dx
=#


abstract type CovKer end

"""
    SqExp(σ)

Stores the bandwidth parameter `σ` of the squared exponential covariance kernel.

See also: [`(s_e::SqExp)(x, y)`](@ref)
"""
struct SqExp{F <: Real} <: CovKer
    neg_inv_double_σ_sq::F

    SqExp(σ::F = 1.0) where F <: Real = new{F}(-F(0.5)/(σ^2))
end

"""
    (s_e::SqExp)(x, y)

Evaluates the squared exponential covariance kernel at `(x, y)`.
"""
(s_e::SqExp)(x::V1, y::V2) where {V1 <: AbstractVector, V2 <: AbstractVector} =
    exp(dot((x - y),(x - y))*s_e.neg_inv_double_σ_sq)

"""
    (s_e::SqExp)(X)

Evaluates the s_e for all `(X_i, X_j)`.
"""

function (s_e::SqExp)(x_samples::AbstractMatrix{F}) where {F<:Real}
    num_samples = size(x_samples, 2)
    K = Array{F, 2}(undef, num_samples, num_samples)
    for i in 1:num_samples
        for j in i:num_samples
            K[i, j] = s_e(x_samples[:,i], x_samples[:,j])
        end
    end
    K
end


Metric(mat::T) where {N, T<:SMatrix{N, N, Float64}} = Metric{N, T}(Symmetric(mat))

function Metric(mat::AbstractMatrix{<:Real})
    N, M = size(mat)
    @assert N == M
    return Metric(SMatrix{N, N, Float64}(mat))
end

"""
    Metric(N::Int)

The Euclidean metric in `N` dimensions.
"""
Metric(N::Int) = Metric(SMatrix{N,N}(1.0I))

function Wedge(vectors::Vector{SVector{N, Float64}}) where N
    K = length(vectors)
    Wedge{N, K}(SVector{K, SVector{N, Float64}}(vectors))
end

"""
    Wedge(s::Simplex{N, K}) where {N, K}

The wedge product of all vectors emanating from the first vertex of a simplex.
"""
function Wedge(s::Simplex{N, K}) where {N, K}
    vectors = SVector{N, Float64}[p.coords - s.points[1].coords for p in s.points[2:end]]
    return Wedge(vectors)
end

export inner_product
"""
    inner_product(m::Metric{N}, v1::SVector{N, Float64}, v2::SVector{N, Float64}) where N
    inner_product(m::Metric{N}, w1::Wedge{N, K}, w2::Wedge{N, K}) where {N, K}

Compute the inner product of two vectors with respect to the given metric.
"""
inner_product(m::Metric{N}, v1::SVector{N, Float64}, v2::SVector{N, Float64}) where N =
    transpose(v1) * m.mat * v2

function inner_product(m::Metric{N}, w1::Wedge{N, K}, w2::Wedge{N, K}) where {N, K}
    if K == 0
        return 1.0
    else
        inner_products = hcat([[inner_product(m, v1, v2) for v1 in w1.vectors]
            for v2 in w2.vectors]...)
        return det(inner_products)
    end
end

export norm_square
"""
    norm_square(m::Metric{N}, v::SVector{N, Float64}) where N
    norm_square(m::Metric{N}, w::Wedge{N}) where N

Compute the inner product of a vector with itself with respect to the given metric. Note
that this value can be negative if the metric is not positive semi-definite.
"""
norm_square(m::Metric{N}, v::SVector{N, Float64}) where N = inner_product(m, v, v)

norm_square(m::Metric{N}, w::Wedge{N}) where N = inner_product(m, w, w)

import LinearAlgebra: norm
"""
    norm(m::Metric{N}, v::SVector{N, Float64}) where N
    norm(m::Metric{N}, w::Wedge{N}) where N

Compute the square root of the magnitude of the inner product of a vector with itself with
respect to the metric `m`.
"""
norm(m::Metric{N}, v::SVector{N, Float64}) where N = sqrt(abs(norm_square(m, v)))

norm(m::Metric{N}, w::Wedge{N}) where N = sqrt(abs(norm_square(m, w)))

export volume_square
"""
    volume_square(m::Metric{N}, s::Simplex{N, K}) where {N, K}

Compute the squared volume of a simplex with respect to the given metric. Note that this
value can be negative if the metric is not positive semi-definite.
"""
volume_square(m::Metric{N}, s::Simplex{N, K}) where {N, K} =
    norm_square(m, Wedge(s))/factorial(K-1)^2

export volume
"""
    volume(m::Metric{N}, s::Simplex{N, K}) where {N, K}

Compute the volume of a simplex with respect to the given metric.
"""
volume(m::Metric{N}, s::Simplex{N, K}) where {N, K} =
    sqrt(abs(norm_square(m, Wedge(s))))/factorial(K-1)


"""
Jarque–Bera Statistic as a normality measure for optimization
"""
function normality(x::AbstractVector{T}) where T<:Real

    P = Base.promote_op(/, T, T)

    # moments at origin
    n = P(length(x))
    m1 = mo2 = mo3 = mo4 = P(0)

    @inbounds for xi in x
        m1 += x1 = xi / n
        mo2 += x2 = x1 * x1 * n
        mo3 += x3 = x1 * x2 * n
        mo4 += x4 = x1 * x3 * n
    end

    # moments
    m2 = mo2 - m1^2
    m3 = mo3 - P(3)*m1*mo2 + P(2)*m1^3
    m4 = mo4 - P(4)*m1*mo3 + P(6)*m1^2*mo2 - P(3)*m1^2*m1^2

    S = m3 / (m2 * sqrt(m2))
    K = m4 / m2^2

    n/P(6) * (S^2 + P(1/4) * (K-P(3))^2)

end


"""
    volatility(returns; multiplier=1.0)

Calculates the volatility based on the standard deviation of the returns. The optional `multiplier` parameter allows for scaling the resulting volatility metric, i.e. for annualization.

# Formula

    Vol = std(returns) * sqrt(multiplier)

# Arguments
- `returns`:    Vector of asset returns (usually log-returns).
- `multiplier`: Optional scalar multiplier, i.e. use `12` to annualize monthly returns, and use `252` to annualize daily returns.
"""
function volatility(returns; multiplier=1.0)
    std(returns) * sqrt(multiplier)
end


module gauss_model

# Provide the functionality to model Gaussians as test cases
using DiskJockey.constants

export imageGauss, FTGauss

# Because of the flipped nature of the sky (but not flipped nature of the UV plane)
# there are some tricky conventions about how to pack the array.

# Given two arrays of l and m coordinates, fill an array of the Gaussian image
# following the MATLAB convention.
# p0 is a vector of [mu_RA, mu_DEC, sigma_x, sigma_y] in units of arcseconds
# mu_RA and mu_DEC are the locations of the centroid emission relative to the
# image origin (RA=0, DEC=0).
function imageGauss(ll::Vector{Float64}, mm::Vector{Float64}, p::Vector{Float64}, k::Int)

    # Both ll and mm increase with array index
    nx = length(ll)
    ny = length(mm)

    img = Array(Float64, ny, nx)
    mu = p[1:2] * arcsec #ll and mm shifts
    Sigma = Diagonal((p[3:4] * arcsec).^2) #Convert from arcsec to radians
    pre = 1. / (2pi * sqrt(det(Sigma))) * k
    for j=1:ny
        for i=1:nx
            R = Float64[ll[i] , mm[j]] - mu
            img[j, i] = pre * exp(-0.5 * (R' * (Sigma\R))[1])
        end
    end
    return img
end

# Given u and v coordinates in [kλ], evaluate the analytic FT of the
# aforementioned Gaussian
# N.B. Here Sigma refers to the (same) covariance matrix in the *image* domain
# always return a complex value
function FTGauss(uu::Float64, vv::Float64, p::Vector{Float64}, k::Int)
    uu = uu .* 1e3 #[λ]
    vv = vv .* 1e3 #[λ]
    mu_RA, mu_DEC = p[1:2]
    mu = Float64[mu_RA, mu_DEC] * arcsec #ll and mm shifts
    R = Float64[uu, vv]
    Sigma = Diagonal((p[3:4] * arcsec).^2) #Convert from arcsec to radians
    phase_shift = exp(-2pi * 1.0im * (R' * mu)[1]) # Not actually in polar phase form
    return k * exp(-2 * (pi^2) * (R' * Sigma * R)[1]) * phase_shift
    # in this case, Sigma serves as the inverse
end

# Given two arrays of u and v coordinates in [kλ], fill an array with the
# analytic FT of aforementioned Gaussian evaluated at every pairwise (u,v) pair
function FTGauss(uu::Vector{Float64}, vv::Vector{Float64}, p::Vector{Float64}, k::Int)
    nu = length(uu)
    nv = length(vv)
    # Both uu and vv increase with array index
    img = Array(Complex128, nv, nu)
    for j=1:nv
        for i=1:nu
            img[j, i] = FTGauss(uu[i], vv[j], p, k)
        end
    end
    return img
end

end # Module


import Base.cross, Base.dot, Base.-, Base.+, Base.*, Base./

type Vector
	x::Real
	y::Real
	z::Real
end

# Outer Constructors

Vector() = Vector(0, 0, 0)

# Methods on Vector

function toList(a::Vector)
	return [a.x, a.y, a.z]
end

function toVec(a::Array)
	return Vector(a[1], a[2], a[3])
end

function +(a::Vector, b::Vector)
	return toVec(toList(a) .+ toList(b))
end

function -(a::Vector, b::Vector)
	return toVec(toList(a) .- toList(b))
end

function *(a::Vector, b::Real)
	return toVec(toList(a) .* b)
end

function magnitude(a::Vector)
	return sqrt(a.x ^ 2 + a.y ^ 2 + a.z ^ 2)
end

function normal(a::Vector)
	mag = magnitude(a)
	return toVec(toList(a) ./ mag)
end

function dot(a::Vector, b::Vector)
	return dot(toList(a), toList(b))
end

function cross(a::Vector, b::Vector)
	return toVec(cross(toList(a), toList(b)))
end


export expMod, boundMod, fMod,identityMod

"""
	sigma,dsigma = expMod(model)

	maps model parameter to conductivity via

	sigma(m) = exp(m)

"""
function expMod(m)
	return exp.(m), sdiag(exp.(m))
end

"""
	sigma,dsigma = fMod(model;f::Function=identity,df::Function=m->sparse(1.0I,length(m),length(m)))

	maps model parameter to conductivity via

	sigma(m) = f(m) and dsigma(m) = sdiag(df(m))

"""
function fMod(m;f::Function=identity,df::Function=m->ones(eltype(m),length(m)))
	return f(m),sdiag(df(m))
end

function identityMod(m)
	return m, UniformScaling(one(eltype(m)))
end

export boundMod
"""
	sigma,dsigma = boundMod(m;boundLow=0.0,boundHigh=1.0)

	maps model parameter to conductivity via

		sigma = 0.5*(boundHigh-boundLow) * (tanh(m)+1.0 ) + boundLow

"""
function boundMod(m; boundLow = 0.0, boundHigh = 1.0)
	u        = tanh.(m)
	d        = 0.5 * (boundHigh - boundLow)
	sigma    = d * (u .+ 1.0) .+ boundLow
	dsigmadm = sdiag(d * (1.0 .- u .* u))
	return sigma, dsigmadm
end


using Measurements
include("/home/jefter66/Projects/Lablib/src/Stats.jl")

printstyled("Experimento 1 \n ", color =:green);
# densidade ρ da corda

## densidade pela geometria

m1 = 0.00552 ± 0.00001 # kg
l = 22.97 ± 0.01 # m

ρ1 = m1.val/l.val ± (m1.val*l.err + m1.err*l.val)/l.val^2
printstyled("Densidade linear calculada pela geometria \n"; color=:red);
println("ρ ± Δρ = (",ρ1, ") kg/m");

## densidade pela tração
m2 = 0.06067 ± 0.00001 #kg
v = [48.9, 49.5, 49.8, 49.7, 49.3]
vm = Statistics.mean(v);
v = round(vm,digits=3) ± round(standartDeviation(v),digits=2);
printstyled("Velocidade média e erro calculado pelo desvio padrão\n"; color=:red);
println("v_medio ± Δv = (",v, ") m/s")

# tração
g = 9.81
t = m2.val*g ± m2.err*g# Newtons
printstyled("Tração:\n"; color=:red);
println("T ± ΔT = (", t, ") N");

# densidade linear calculada a partir da tração
ρ2 = t.val/v.val^2 ± (2*t.val*v.val*v.err + t.err*v.val^2)/(v.val^4)
printstyled("Densidade linear \n "; color =:red);
println("ρ2 ± Δρ2= (", ρ2 , ") kg/m");

# equivalência

v1 = abs(ρ2.val - ρ1.val);
v2 = 2*(ρ1.err + ρ2.err);

printstyled("Checando equivalência", color=:red);
println("| x1 - x2 | < 2*(σ1 + σ2)");
println("| ",round(ρ1.val,digits=10) , " - ", round(ρ2.val,digits=10) , " | < 2*(", round(ρ1.err,digits=10), " + ", round(ρ2.err,digits=10), ")");
println(v1<v2);

# Ondas sonoras
printstyled("Experimento 2\n", color =:green);

## comprimento constante
printstyled("Comprimento constante \n ", color =:red);
L = (0.1100 ± 0.0001);
f = []
n = 5

## frequência constante
printstyled("Experimento 3\n", color =:green);

f = (2000±0.1);
L = [0.087,0.088,0.086,0.087];
ΔL = 0.1;

λ(L) = 2*L;

λs = Statistics.mean([λ(L[1]),λ(L[2]),λ(L[3]),λ(L[4])]) ± standartDeviation(L);

printstyled("Frequência constante \n ", color =:red);

println("Comprimento de onda médio: λ ± Δλ = ", λs);

v =  λs*f;
println("Velocidade média: v ± Δv = ",v);

## gás desconhecido
printstyled("Experimento 4 - Gás desconhecido 1° harmônico \n ", color =:red);
f = (2000±0.1);
n = 1
l = (0.23 ±0.0001);

v = 2*l*f;
println("Velocidade média: v ± Δv = ",v);


export orthog!,
       svd_recursive

function orthog!(M::AbstractMatrix{T};
                 npass::Int=2) where {T}
  nkeep = min(size(M)...)
  dots = zeros(T,nkeep)
  for i=1:nkeep
    coli = view(M,:,i)
    nrm = norm(coli)
    if nrm < 1E-10
      rand!(coli)
      nrm = norm(coli)
    end
    coli ./= nrm
    (i==1) && continue

    Mcols = view(M,:,1:i-1)
    dotsref = view(dots,1:i-1)
    for pass=1:npass
      mul!(dotsref,Mcols',coli)
      #BLAS.gemv!('N',1.0,Mcols,dotsref,-1.0,coli)
      coli .-= Mcols*dotsref
      nrm = norm(coli)
      if nrm < 1E-3 #orthog is suspect
        pass = pass-1
      end
      if nrm < 1E-10
        rand!(coli)
        nrm = norm(coli)
      end
      coli ./= nrm
    end
  end
end

function pos_sqrt(x::Float64)::Float64
  (x < 0.0) && return 0.0
  return sqrt(x)
end

function checkSVDDone(S::Vector,
                      thresh::Float64)
  N = length(S)
  (N <= 1 || thresh < 0.0) && return (true,1)
  S1t = S[1]*thresh
  start = 2
  while start <= N
    (S[start] < S1t) && break
    start += 1
  end
  if start >= N
    return (true,N)
  end
  return (false,start)
end

function svd_recursive(M::AbstractMatrix;
                      thresh::Float64=1E-3,
                      north_pass::Int=2)
  Mr,Mc = size(M)

  if Mr > Mc
    V,S,U = svd_recursive(transpose(M))
    conj!(U)
    conj!(V)
    return U,S,V
  end

  #rho = BLAS.gemm('N','T',-1.0,M,M) #negative to sort eigenvalues greatest to smallest
  rho = -M*M' #negative to sort eigenvalues in decreasing order
  D,U = eigen(Hermitian(rho),1:size(rho,1))

  Nd = length(D)
  for n=1:Nd
    D[n] = pos_sqrt(-D[n])
  end

  V = M'*U
  orthog!(V,npass=north_pass)

  (done,start) = checkSVDDone(D,thresh)

  done && return U,D,V

  u = view(U,:,start:Nd)
  v = view(V,:,start:Nd)

  b = u'*(M*v)
  bu,bd,bv = svd_recursive(b,
                          thresh=thresh,
                          north_pass=north_pass)

  u .= u*bu
  v .= v*bv
  view(D,start:Nd) .= bd

  return U,D,V
end

# TODO: maybe move to another location?
function polar(M::AbstractMatrix)
  U,S,V = svd(M) # calls LinearAlgebra.svd()
  return U*V',V*Diagonal(S)*V'
end


# Dirichlet
struct Dirichlet{P<:Tuple{Vararg{Real}}} <: Distribution{Tuple{Vararg{Real}},P}
    params :: P
    Dirichlet(; α) = new{typeof(α)}(α)
end

# Dirichlet(α::Real...) = Dirichlet(α)
# Dirichlet(α::Integer...) = Dirichlet(Float64.(α))
# Dirichlet(n::Integer, α::Real)= Dirichlet(ntuple(_->α, n))

function randDirichlet(α::Tuple{Vararg{T}}) where T<:Real
    p = Tuple(randGamma(αi, one(T)) for αi in α)
    p ./ sum(p)
end
randDirichlet(α::Tuple{Vararg{Integer}}) = randDirichlet(Float64.(α))
randDirichlet(n::Integer, α::Real) = randDirichlet(Tuple(α for _ in 1:n))
randDirichlet(n::Integer, α::Integer) = randDirichlet(n, Float64(α))

@cufunc function logpdfDirichlet(x::NTuple{N,Real}, α::NTuple{N,Real}) where N
    # a, b = sum(u -> SVector(u,lgamma(u)), α)
    # s = sum(((u,v) -> (u-one(u))log(v)).(α, x))
    # s - b + lgamma(a)
    T = promote_type(typeof.(x)..., typeof.(α)...)
    a = b = s = zero(T)
    for (xi, αi) in zip(x, α)
        a += T(αi)
        b += lgamma(T(αi))
        s += (T(αi) - one(T))log(T(xi))
    end
    s - b + lgamma(a)
end


using CompScienceMeshes
using BEAST
using LinearAlgebra

trias = CompScienceMeshes.meshrectangle(1.0,1.0,0.05)
trias = CompScienceMeshes.meshcircle(1.0,0.5)
tetrs = CompScienceMeshes.tetmeshcuboid(1,1,1,0.2)

f = BEAST.ScalarTrace(x -> sin(pi*x[1]*x[2])*sin(pi*x[2]))
f2 = BEAST.ScalarTrace(x -> (-x[2]*sin(2*pi*(x[1]^2+x[2]^2))*sin(atan(x[2],x[1])), x[1]*sin(2*pi*(x[1]^2+x[2]^2))*sin(atan(x[2],x[1])), 0) )
g = BEAST.ScalarTrace(x -> (sin(pi*x[1])*cos(pi*x[2]), -cos(pi*x[1])*sin(pi*x[2]), 0 ))
g = BEAST.ScalarTrace(x -> (sin(pi*x[1])*cos(pi*x[2]), sin(pi*x[2])*cos(pi*x[2]), sin(pi*x[3])*cos(pi*x[2])))
g2 = BEAST.ScalarTrace(x -> ( sin(pi*x[1])*cos(pi*x[3]),  0, -cos(pi*x[1])*sin(pi*x[3])))

N = 4

h = Vector(undef, N)
errorL = Vector(undef, N)
errorBDM = Vector(undef, N)
errorRT = Vector(undef, N)

for n in 1:N

    h[n] = 0.5^(n)

    trias2 = CompScienceMeshes.meshrectangle(1.0,1.0, h[n])

    trias = CompScienceMeshes.meshcircle(1.0, h[n], 3)

    Z = BEAST.lagrangec0d1(trias, dirichlet=false)
    Y = BEAST.brezzidouglasmarini(trias)
    X = BEAST.raviartthomas(trias)

    u_exact = DofInterpolate(Z,f)
    b_exact = DofInterpolate(Y,f2)
    r_exact = DofInterpolate(X,f2)

    identity = BEAST.Identity()

    M = assemble(identity, Z, Z)
    b = assemble(f,Z)
    u_n = M \ b

    errorL[n] = sqrt((u_n-u_exact)'*M*(u_n-u_exact))

    M = assemble(identity, Y, Y)
    b = assemble(f2,Y)
    b_n = M \ b

    errorBDM[n] = sqrt((b_n-b_exact)'*M*(b_n-b_exact))

    M = assemble(identity, X, X)
    b = assemble(f2,X)
    r_n = M \ b

    errorRT[n] = sqrt((r_n-r_exact)'*M*(r_n-r_exact))

end

N = 15

delta = Vector(undef, N)
error = Vector(undef, N)

for i in 1:N

    delta[i] = 0.5-0.025*i

    tetrs =  tetmeshcuboid(1.0,1.0,1.0, delta[i])

    bndry = boundary(tetrs)
    faces = skeleton(tetrs, 2)

    bndry_faces = [sort(c) for c in cells(skeleton(bndry, 2))]
    function is_interior(faces)
        !(sort(faces) in bndry_faces)
    end

    interior_faces = submesh(is_interior, faces)

    W = BEAST.brezzidouglasmarini3d(tetrs, interior_faces)

    u_exact = DofInterpolate(W,g2)

    identity = BEAST.Identity()

    M = assemble(identity, W, W)

    b = assemble(g2,W)
    u_n = M \ b

    error[i] = real(sqrt((u_n-u_exact)'*M*(u_n-u_exact)))
end

N=15

delta2 = Vector(undef, N)
error2 = Vector(undef, N)

for i in 1:N

    delta2[i] =0.5-0.025*i

    tetrs =  tetmeshcuboid(1.0,1.0,1.0, delta2[i])

    bndry = boundary(tetrs)
    faces = skeleton(tetrs, 2)

    bndry_faces = [sort(c) for c in cells(skeleton(bndry, 2))]
    function is_interior(faces)
        !(sort(faces) in bndry_faces)
    end

    interior_faces = submesh(is_interior, faces)

    V = BEAST.nedelecd3d(tetrs, interior_faces)

    q_exact = DofInterpolate(V,g)

    identity = BEAST.Identity()

    M2 = assemble(identity, V, V)

    b2 = assemble(g,V)

    q_n = M2 \ b2

    error2[i] = real(sqrt((q_n-q_exact)'*M2*(q_n-q_exact)))
end

using Plots

p2 = plot(delta,error, label="BDM3D", title="L2-Error on [0,1]^3",xlabel="h", ylabel="Error", legend=:bottomright, xaxis=:log, yaxis=:log)
plot!(p2, delta2, error2, label="Nedelec Div", xlims=[0.075,0.55])

p1 = plot(h,real(errorL), label="Lagrange", title="L2-Error on [0,1]^2",xlabel="h", ylabel="Error", legend=:bottomright, xaxis=:log, yaxis=:log, xlims=[0.025,0.5])
plot!(p1,h,real(errorBDM),label="BDM")
plot!(p1,h,real(errorRT),label="Raviart-Thomas")

plot(p1,p2, layout=2)


# ---
# title: 892. Surface Area of 3D Shapes
# id: problem892
# author: Tian Jun
# date: 2020-10-31
# difficulty: Easy
# categories: Math, Geometry
# link: <https://leetcode.com/problems/surface-area-of-3d-shapes/description/>
# hidden: true
# ---
#
# On a `N * N` grid, we place some `1 * 1 * 1 `cubes.
#
# Each value `v = grid[i][j]` represents a tower of `v` cubes placed on top of
# grid cell `(i, j)`.
#
# Return the total surface area of the resulting shapes.
#
#
#
# **Example 1:**
#
#
#
#     Input: [[2]]
#     Output: 10
#
#
# **Example 2:**
#
#
#
#     Input: [[1,2],[3,4]]
#     Output: 34
#
#
# **Example 3:**
#
#
#
#     Input: [[1,0],[0,2]]
#     Output: 16
#
#
# **Example 4:**
#
#
#
#     Input: [[1,1,1],[1,0,1],[1,1,1]]
#     Output: 32
#
#
# **Example 5:**
#
#
#
#     Input: [[2,2,2],[2,1,2],[2,2,2]]
#     Output: 46
#
#
#
#
# **Note:**
#
#   * `1 <= N <= 50`
#   * `0 <= grid[i][j] <= 50`
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


# f.jl
# F-tests
#
# Copyright (C) 2019   Raphael Saavedra
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

export VarianceFTest

struct VarianceFTest <: HypothesisTest
    n_x::Int   # size of sample 1
    n_y::Int   # size of sample 2
    df_x::Int  # degrees of freedom 1
    df_y::Int  # degrees of freedom 2
    F::Real    # test statistic
end

"""
    VarianceFTest(x::AbstractVector{<:Real}, y::AbstractVector{<:Real})

Perform an F-test of the null hypothesis that two real-valued vectors `x` and `y` have equal variances.

Implements: [`pvalue`](@ref)

# References

  * George E. P. Box, "Non-Normality and Tests on Variances", Biometrika 40 (3/4): 318–335, 1953.

# External links

  * [F-test of equality of variances on Wikipedia](https://en.wikipedia.org/wiki/F-test_of_equality_of_variances)
"""
function VarianceFTest(x::AbstractVector{<:Real}, y::AbstractVector{<:Real})
    n1, n2 = length(x), length(y)
    F = var(x) / var(y)
    return VarianceFTest(n1, n2, n1-1, n2-1, F)
end

testname(::VarianceFTest) = "Variance F-test"
population_param_of_interest(x::VarianceFTest) = ("variance ratio", 1.0, x.F)
default_tail(test::VarianceFTest) = :both

function show_params(io::IO, x::VarianceFTest, ident)
    println(io, ident, "number of observations: [$(x.n_x), $(x.n_y)]")
    println(io, ident, "F statistic:            $(x.F)")
    println(io, ident, "degrees of freedom:     [$(x.df_x), $(x.df_y)]")
end

function pvalue(x::VarianceFTest; tail=:both)
    dist = FDist(x.df_x, x.df_y)
    if tail == :both
        return 1 - 2*abs(cdf(dist, x.F) - 0.5)
    elseif tail == :right
        return 1 - cdf(dist, x.F)
    elseif tail == :left
        return cdf(dist, x.F)
    else
        throw(ArgumentError("tail=$(tail) is invalid"))
    end
end


"""
    capm(asset_returns, benchmark_returns; risk_free=0.0)

Calculates the CAPM α and β coefficients based on sample covariance statistics and a simple linear regression model.

The linear regression model looks as follows:

        r_a - r_f = α + β(r_b - r_f) + ϵ

The α coefficient in this model is also known as Jensen's alpha. β is the slope coefficient, and ϵ is an error term.

# Arguments
- `asset_returns`:      Vector of asset returns.
- `benchmark_returns`:  Vector of benchmark returns (e.g. market portfolio returns for CAPM beta).
- `risk_free`:          Optional vector or scalar value denoting the risk-free return (must have same frequency as the provided returns, e.g. daily).

# Returns
Tuple (α, β) with the estimated α and β coefficients of the CAPM model.
"""
function capm(asset_returns, benchmark_returns; risk_free=0.0)
    asset_returns_ex = asset_returns .- risk_free
    benchmark_returns_ex = benchmark_returns .- risk_free
    μ1 = mean(asset_returns_ex)
    μ2 = mean(benchmark_returns_ex)
    β = sum((asset_returns_ex .- μ1).*(benchmark_returns_ex .- μ2)) / sum((benchmark_returns_ex .- μ2).^2)
    α = μ1 - β*μ2
    (α, β)
end


function scalsq(A::AbstractArray{T,2}, b::AbstractVector{T}, w = nothing) where {T<:Number}
#
#  Solving the scaled least squares problem
#          W(Ax - b) = 0
#      with iterative refinement.
#
#  input  A --- the matrix
#         b --- the right-hand side vector
#         w --- the scaling vector (diagonal entries of W)
#
#
    m, n = size(A)
	E = real(one(T))

    if w == nothing
        w = ones(T, m)
        for j = 1:m
            abj = abs(b[j])
            if abj > E
                w[j] = E / abj
            end
        end
    end

    for j = 1:m
        A[j,1:end] = A[j,1:end] * w[j]
    end

    b = b .* w

    Q, S = qr(A)

    d = Q' * b
    x = backsub(S, d)

    # one step refinement
    bb = [b; zeros(T, n)]
	B = [I A; A' zeros(T, n, n)]
    r = b - A * x

    #for j = 1:3
    rr = bb - B * [r; x]; #disp(norm(rr));

    s = Q' * rr[1:m]
    c = forsub(S', rr[m+1:m+n])
	c2 = backsub(S, s[1:n] - c)
    #c1 = Q * [c; s[n+1:m]]
    #r = r + c1
    x = x + c2
    #%end;
    x
end


### A Pluto.jl notebook ###
# v0.14.5

using Markdown
using InteractiveUtils

# ╔═╡ d9cff120-aba1-11eb-0f83-db8e49ac6bab
md"# Chapter 03 from NNFS book"

# ╔═╡ 4a34dd2f-cbf5-4ac0-8f97-fd4050354547
md"### 2 Layers of Neurons (4 inputs, 3 hidden layers)"

# ╔═╡ ab14105e-a405-4d87-80d7-940ce87ef19a
md"""
**One observation:**
"""

# ╔═╡ fed9705c-b748-4cf9-827a-f10fd8073967
begin
	# first layer
	input1 = [1, 2, 3, 2.5]

	weight1 = [
		 0.2   0.8  -0.5   1.0
		 0.5  -0.91  0.26 -0.5
		-0.26 -0.27  0.17  0.87
	]

	bias1 = [2, 3, 0.5]

	# second layer
	weight2 = [
		 0.1   -0.14   0.5
		-0.5    0.12  -0.33
		-0.44   0.73  -0.13
	]

	bias2 = [-1, 2, -0.5]

	# calculating layers
	layer1 = weight1 * input1 + bias1
	layer2 = weight2 * layer1 + bias2
end

# ╔═╡ 7ec38404-9380-411d-8b0d-f073ba5c6cbc
md"""
**Multiple observations:**
"""

# ╔═╡ 773a22f6-99c3-4bf5-936b-0b22759038ed
begin
	# first layer
	inp01 = [
		[1, 2, 3, 2.5] [2, 5, -1, 2] [-1.5, 2.7, 3.3, -0.8]
	]

	w01 = [
		 0.2   0.8  -0.5   1.0
		 0.5  -0.91  0.26 -0.5
		-0.26 -0.27  0.17  0.87
	]

	b01 = [2, 3, 0.5]

	# second layer
	w02 = [
		 0.1   -0.14   0.5
		-0.5    0.12  -0.33
		-0.44   0.73  -0.13
	]

	b02 = [-1, 2, -0.5]

	# calculating layers
	layer01 = w01 * inp01 .+ b01
	layer02 = w02 * layer01 .+ b02
end

# ╔═╡ 6aabd18d-d0a1-4f7e-9019-92d7671e7265
md"### Creating the LayerDense Struct"

# ╔═╡ 50edc678-40c3-483c-8556-666f01553802
begin
	struct LayerDense
		w::Matrix{Float64}
		b::Vector{Float64}
		function LayerDense(n_in::Int, n_out::Int)
			w = 0.01 * randn(n_out, n_in)
			b = zeros(n_out)
			return new(w, b)
		end
	end

	(layer::LayerDense)(input) = layer.w * input .+ layer.b
end

# ╔═╡ da9d7fdb-8b5a-466c-9370-654c133e10a3
D = LayerDense(2, 3)

# ╔═╡ 173fe819-779e-4d8e-a2b3-e00fb9363cb2
imp01 = [3.5, 4.3]

# ╔═╡ c14fce98-4470-4153-ba62-75521c3e5ed8
imp02 = [[3.5, 4.3] [2.4, -2.8]]

# ╔═╡ f8915121-453b-49bc-83ed-07ea13541160
D(imp01)

# ╔═╡ ffe4c9ac-bbbf-49e0-b882-b6a8ed096622
D(imp02)

# ╔═╡ ae0e3918-c07e-40a5-880d-15cec3afce71
D(randn(2))

# ╔═╡ Cell order:
# ╟─d9cff120-aba1-11eb-0f83-db8e49ac6bab
# ╟─4a34dd2f-cbf5-4ac0-8f97-fd4050354547
# ╟─ab14105e-a405-4d87-80d7-940ce87ef19a
# ╠═fed9705c-b748-4cf9-827a-f10fd8073967
# ╟─7ec38404-9380-411d-8b0d-f073ba5c6cbc
# ╠═773a22f6-99c3-4bf5-936b-0b22759038ed
# ╟─6aabd18d-d0a1-4f7e-9019-92d7671e7265
# ╠═50edc678-40c3-483c-8556-666f01553802
# ╠═da9d7fdb-8b5a-466c-9370-654c133e10a3
# ╠═173fe819-779e-4d8e-a2b3-e00fb9363cb2
# ╠═c14fce98-4470-4153-ba62-75521c3e5ed8
# ╠═f8915121-453b-49bc-83ed-07ea13541160
# ╠═ffe4c9ac-bbbf-49e0-b882-b6a8ed096622
# ╠═ae0e3918-c07e-40a5-880d-15cec3afce71


# This file is pat of RLinearAlgebra.jl
# 1. Specifies type
# 2. Implements rsubsolve function
# 3. Exports Type

#using LinearAlgebra

"""
    LinSysVecColProjPO <: LinSysVecColProjection

A mutable structure that represents the standard column projection method
    with orthogonalization of the current projection against a set of `m` previous
    projection directions, where `m` is specified by the user.

See Patel, Vivak, Mohammad Jahangoshahi, and Daniel Adrian Maldonado. "An Implicit
    Representation and Iterative Solution of Randomly Sketched Linear Systems."
    SIAM Journal on Matrix Analysis and Applications (2021) 42:2, 800-831.
    https://doi.org/10.1137/19M1259481

# Fields

- `α::Float64`, the relaxation parameter (usually between `0.0` and `2.0`)
- `m::Int64`, the number of previous directions against which to orthogonalize
- `Z::Union{Vector{Vector{Float64}}, Nothing}`, stores the `m` vectors against which the
    current projection is orthogonalized against

Calling `LinSysVecColProjPO()` defaults the relaxation parameter to `1.0`, the memory
    parameter `m` to `5`, and `Z` to `nothing`.
"""
mutable struct LinSysVecColProjPO <: LinSysVecColProjection
    α::Float64
    m::Int64
    Z::Union{Vector{Vector{Float64}}, Nothing}
end
LinSysVecColProjPO() = LinSysVecColProjPO(1.0, 5, nothing)

# Common rsubsolve interface for linear systems
function rsubsolve!(
    type::LinSysVecColProjPO,
    x::AbstractVector,
    samp::Tuple{U,V,W} where {U<:AbstractVector,V<:AbstractArray,W<:Real},
    iter::Int64,
)
    # samp[1] is the search direction
    # samp[2] is the coefficient matrix, A
    # samp[3] is a projection of the residual of normal system, samp[1]' * A' * (A * x - b)

    # Allocate space for orthogonalization set Z
    if iter == 1
        d = length(x)
        type.Z = Vector{Float64}[zeros(Float64, d) for i in 1:type.m]
    end

    # Compute orthogonal complement of A' * A * samp[1] using modified Gram-Schmidt
    u = samp[2]' * (samp[2] * samp[1])
    for z in type.Z
        u = u - dot(z, u) * z
    end

    # Check whether vector u is nearly zero
    nrmUsq = dot(u, u)
    if nrmUsq < eps() * length(x); return nothing; end

    # Otherwise, update iterate
    x .= x - u * (type.α * samp[3] / nrmUsq )

    # Update orthonormal set
    z = u / sqrt(nrmUsq)
    type.Z .= push!(type.Z[2:end], z)

    return nothing
end


function bounded_log_grid(xMin::Float64, xMax::Float64, steps::Int)
  result = zeros(steps + 1)
  gridLogSpacing = (log(xMax) - log(xMin)) / steps
  edx = exp(gridLogSpacing)
  result[1] = xMin
  for j = 2:steps+1
    result[j] = result[j-1] * edx
  end

  return result
end


# This script provides an example of simulating changepoint data,
# using PELT to find these changepoints, and plotting the results
#
# Requires Plots package

using Distributions, Plots
using Changepoints

#########################
# Simulate changepoints #
#########################

n = 1000        # Length of sample
λ = 100         # Frequencey of changes

# Mu parameter sampled from a N(0,10) distribution for each changepoint
μ = Normal(0,5)
σ = 1.0 # Sigma parameter fixed

# Construct sample with changepoints where
# at each changepoint a new value is sample for μ
sample, cps = @changepoint_sampler n λ Normal(μ, σ)

########################
# Finding changepoints #
########################

# Find changepoints via PELT
# assuming Normal likelihood with changing mean
# parameter and fixed standard deviation of σ
pelt_cps, pelt_cost = @PELT sample Normal(:?, σ)

# Find changepoints via Binary Segmentation
bs_cps, bs_cost = @BS sample Normal(:?, σ)

# If plotting multiple sets of changepoints which need to labelled
# separately, it is best not to use `changepoint_plot` function
# but to instead first plot data
# and then add changepoints with `vline!` function
p = plot(sample, label="Simulated data")
vline!(p, cps, label="True changepoints")
vline!(p, pelt_cps, label="PELT changepoints")
vline!(p, bs_cps, label="BS changepoints")
display(p)


export  calculateMultiIndices,
        findUnivariateIndices

function calculateMultiIndices(d::Int, n::Int)
    # d denotes dimension of random variables/number of sources of uncertainty,
    # n the maximum degree of multivariate basis
    # function to calculate indices of multivariate basis following the algorithm
    # from "Spectral Methods for Uncertainty Quantiﬁcation; Le Maitre, Knio;2014" p.516-517
    # No::BigInt = factorial(BigInt(d+n))/(factorial(Bigint(d))*factorial(BigInt(n)));  #number of polynomials of multivariate basis
  n < 0 && throw(DomainError(n, "maximum degree must be non-negative"))
  d <= 0 && throw(DomainError(d, "number of uncertainties must be positive"))
  # catch case n == 0 --> No-d==0
  n == 0 && return zeros(Int64,1,d)
  # non-pathological cases begin here
  No = numberPolynomials(d,n)
  inds = vcat(zeros(Int64,1,d),Matrix(1I,d,d),zeros(Int64, No-d-1, d));  #initiate index matrix for basis
  pi=ones(Int64,No,d);

  for k in 2:No
    g = 0;
    for l in 1:d
      pi[k,l]=sum(pi[k-1,:])-g;
      g=g+pi[k-1,l];
    end
  end

  P = d + 1;
  for k in 2:n
    L = P;
    for j in 1:d, m in L-pi[k,j]+1:L
        P += 1;
        inds[P,:] = inds[m,:];
        inds[P,j] = inds[P,j]+1;
    end
  end

  return inds
end

"""
    computes the number of polynomials with a multivariate basis
    `(d+n)!/(d!+n!)`
"""

function numberPolynomials(d::Int64, n::Int64)
    x, y = max(d,n), min(d,n)
    return UInt128(prod(UInt128(x+1):UInt128(d+n)) ÷ factorial(UInt128(y)))
end

"""
    findUnivariateIndices(i::Int,ind::AbstractMatrix{Int64,2})
Given the multi-index `ind` this function returns all entries of the multivariate basis
that correspond to the `i`th univariate basis.
"""
function findUnivariateIndices(i::Int,ind::AbstractMatrix{Int})
  l,p = size(ind)
  i > p && throw(DomainError((i,p), "basis is $p-variate, you requested $i-variate"))
  deg = ind[end,end]
  deg < 0 && throw(DomainError(deg, "invalid degree"))
  col = ind[:,i]
  myind = zeros(Int64,deg)
  for deg_ in 1:deg
      myind[deg_] = findfirst(x->x==deg_,col)
  end
  pushfirst!(myind,1)
end


# Implicitly solve heat equation: ∂_t T = α * ∇²(T) + β*sin(γ*z)
# Boundary conditions: ∇T(z=a) = ∇T_bottom, T(z=b) = T_top
import Plots
using LinearAlgebra
using DiffEqBase
using OrdinaryDiffEq: SplitODEProblem, solve, IMEXEuler
import SciMLBase

a,b, n = 0, 1, 10               # zmin, zmax, number of cells
n̂_min, n̂_max = -1, 1            # Outward facing unit vectors
α = 100;                        # thermal diffusivity, larger means more stiff
β, γ = 10000, π;                # source term coefficients
Δt = 1000;                      # timestep
N_t = 10;                       # number of timesteps
FT = Float64;                   # float type
Δz = FT(b-a)/FT(n)
Δz² = Δz^2;
∇²_op = [1/Δz², -2/Δz², 1/Δz²]; # row or Laplacian operator
∇T_bottom = 10;                 # Temperature gradient at the top
T_top = 1;                      # Temperature at the bottom
S(z) = β*sin(γ*z)               # source term, (sin for easy integration)
zf = range(a, b, length=n+1);
zc = map(i-> zf[i]+Δz/2, 1:n);

function T_analytic(z) # Analytic steady state solution
    c1 = ∇T_bottom-β*cos(γ*a)/(γ*α)
    c2 = T_top-(β*sin(γ*b)/(γ^2*α)+c1*b)
    return β*sin(γ*z)/(γ^2*α)+c1*z+c2
end

T = zeros(FT, n+1);

# Equations: derivation to matrix form
# ∂_t T = α * ∇²(T) + S
# (T^{n+1}-T^n) = Δt (α * ∇²(T^{n+1}) + S)
# (T^{n+1} - Δt α * ∇²(T^{n+1})) = T^n + Δt*S
# (I - Δt α * ∇²) T^{n+1} = T^n + Δt*S

# Derive Dirichlet boundary stencil & source:
# ∂_t T = α * (T[i-1]+T[b]-2*T[i])/Δz² + S
# ∂_t T = α * (T[i-1]-2*T[i])/Δz² + S + α * T[b] / Δz²

# Derive Neumann boundary stencil & source:
# ∇T_bottom*n̂= (T[g] - T[i])/(2Δz),     n̂ = [-1,1] ∈ [zmin,zmax]
# T[i] + 2*Δz*∇T_bottom*n̂ = T[g]
# ∂_t T = α * (((T[i] + 2*Δz*∇T_bottom*n̂) - T[b])/Δz - (T[b] - T[i])/Δz)/Δz + S
# ∂_t T = α * (((T[i]) - T[b])/Δz - (T[b] - T[i])/Δz)/Δz + S + α/Δz²*2*Δz*∇T_bottom*n̂
# ∂_t T = α * (2*T[i] - 2*T[b])/Δz² + S + 2α/Δz*∇T_bottom*n̂
∇² = Tridiagonal(
    ones(FT, n) .* ∇²_op[1],
    ones(FT, n+1)   .* ∇²_op[2],
    ones(FT, n) .* ∇²_op[3]
);

# Modify boundary stencil to account for BCs
∇².d[1] = -2/Δz²
∇².du[1] = +2/Δz²

# Modify boundary stencil to account for BCs
∇².du[n] = 0  # modified stencil
∇².d[n+1] = 0 # to ensure `∂_t T = 0` at `z=zmax`
∇².dl[n] = 0  # to ensure `∂_t T = 0` at `z=zmax`

# Compute boundary source: α * T[b] / Δz²
AT_b = zeros(FT, n+1);
AT_b[1] = α*2/Δz*∇T_bottom*n̂_min;
AT_b[end-1] = α*T_top/Δz²;

# Set initial condition:
T .= 1;
T[n+1] = T_top # set top BC

function rhs!(dT, T, params, t)
    n = params.n
    i = 1:n # interior domain
    dT[i] .= S.(zf[i]) .+ AT_b[i]
    return dT
end

D = α .* ∇²

params = (;n)

tspan = (FT(0), N_t*FT(Δt))

prob = SplitODEProblem(
    SciMLBase.DiffEqArrayOperator(
        D,
    ),
    rhs!,
    T,
    tspan,
    params
)

println("Solving...")

alg = IMEXEuler(linsolve=LinSolveFactorize(lu!))

sol = solve(
    prob,
    alg,
    dt = Δt,
    saveat = range(FT(0), N_t*FT(Δt), length=5),
    progress = true,
    progress_message = (dt, u, p, t) -> t,
);

T_end = sol.u[end]

p1 = Plots.plot(zf, T_analytic.(zf), label="analytic", markershape=:circle, markersize=6)
p1 = Plots.plot!(p1, zf, T_end, label="numerical", markershape=:diamond)
p1 = Plots.plot!(p1, title="T ∈ cell faces")

p2 = Plots.plot(zf, abs.(T_end .- T_analytic.(zf)), label="error", markershape=:circle, markersize=6)
p2 = Plots.plot!(p2, title="T ∈ cell faces")

Plots.plot(p1, p2)


function pythagorean_triplets(N::Int)::Array{Tuple{Int,Int,Int},1}
    # Return all (a,b,c) such that:
    # a + b + c = N
    # a^2 + b^2 = c^2
    # a < b < c

    # Pythagorean triplets: Euclid did this a while ago
    # a = m^2 - n^2, b = 2mn, c = m^2 + n^2
    # so a + b + c = 2m^2 + 2mn = N
    # m(m + n) = N/2: not nice for generation, so let's give up and filter by it instead
    #
    # Need to take care of int multiples so also k.*(a,b,c) => km(m+n) = N/2 too
    # => N/2 / m(m+n) is an integer: divrem(N/2,m(m+n))

    # TODO / exercise for the reader: ensure bounds on N/2 are as tight as they can be
    sort!(triple.(Iterators.filter(x->conditions(x[1],x[2],N),Iterators.product(1:N/2,1:N/2)),N) |> unique; by=x->x[1])
end

function conditions(m,n,N)
    m > n > 0 &&
    rem(N/2,m*(m+n)) == 0
end

triple(t,N) = triple(t[1],t[2],N)

# We only make the triple of answers that we know will work so there's no point optimising this
triple(m,n,N) = div(N/2,m*(m+n)).*sort!((m^2 - n^2, 2m*n, m^2 + n^2) |> collect) |> Tuple{Int,Int,Int}

# This is vastly faster than mine, it's not fair
# https://exercism.io/tracks/julia/exercises/pythagorean-triplet/solutions/369f511a364d4ce0b9c054c55b49d8d8
function pythagorean_triplets(x::Int)

	triplets = Array{Tuple{Int64,Int64,Int64}}(undef, 0)
	for a in 1:(x ÷ 3)
		for b in (a+1):(x ÷ 2 + 1)

			c = x - a - b
			if (a*a + b*b == c*c)
				push!(triplets, (a, b, c))
			end
		end
	end
	return triplets
end


# This file investigates integration error with respect to free variables in 1D integration

using FractalTools
using Makie

# Construct interpolation data
f(x) = sin(2π * x) + 1
dx   = 0.1
xi   = 0.
xf   = 1.
x    = collect(xi : dx : xf)
y    = f.(x)
pts  = collect.(zip(x, y))

# Compute errors
fval = (cos(2π * xi) - cos(2π * xf)) / (2π) + (xf - xi)
freevars = 0.001 : 0.01 : 0.999
mse = map(freevars) do freevar
    ival = integrate(pts, Interp1D(freevar))
    abs(fval - ival)
end

# Plot mse
fig = Figure()
ax = fig[1, 1] = Axis(fig, xlabel="Free Variable", ylabel="Integration Error", title="1D Integration Error")
stem!(ax, freevars, mse, color=:black)
save(joinpath(@__DIR__, "integ1d_error.png"), fig)
display(fig)


using Revise
using AdFem
using LinearAlgebra
using PyPlot

m = 50
n = 50
h = 1/n

bdedge = bcedge("top", m, n, h)
bdnode = bcnode("left|right|bottom", m, n, h)

F1 = eval_f_on_gauss_pts((x,y)->3.0, m, n, h)
F2 = eval_f_on_gauss_pts((x,y)->-1.0, m, n, h)
F = compute_fem_source_term(F1, F2, m, n, h)

t1 = eval_f_on_boundary_edge((x,y)->-x-y, bdedge, m, n, h)
t2 = eval_f_on_boundary_edge((x,y)->2y, bdedge, m, n, h)
T = compute_fem_traction_term([t1 t2], bdedge, m, n, h)

D = constant(diagm(0=>[1,1,0.5]))
K = compute_fem_stiffness_matrix(D, m, n, h)
rhs = T - F
bdval = [eval_f_on_boundary_node((x,y)->x^2+y^2, bdnode, m, n, h);
        eval_f_on_boundary_node((x,y)->x^2-y^2, bdnode, m, n, h)]
DOF = [bdnode;bdnode .+ (m+1)*(n+1)]
K, rhs = impose_Dirichlet_boundary_conditions(K, rhs, DOF, bdval)
u = K\rhs
sess = Session(); init(sess)
S = run(sess, u)

figure(figsize=[10,4])
subplot(121)
visualize_scalar_on_fem_points(S[1:(m+1)*(n+1)], m, n, h)
subplot(122)
visualize_scalar_on_fem_points(S[(m+1)*(n+1)+1:2(m+1)*(n+1)], m, n, h)
savefig("numerical.png")

figure(figsize=[10,4])
x = LinRange(0, 1, 50)
X = zeros(50, 50)
Y = zeros(50, 50)
for i = 1:50
    for j = 1:50
        X[i,j] = x[i]
        Y[i,j] = x[j]
    end
end
subplot(121)
XY = fem_nodes(m, n, h)
X = XY[:,1]
Y = XY[:,2]
U = (@. X^2+Y^2)
V = (@. X^2-Y^2)
subplot(121)
visualize_scalar_on_fem_points(U, m, n, h)
subplot(122)
visualize_scalar_on_fem_points(V, m, n, h)
savefig("exact.png")

figure(figsize=[10,4])
subplot(121)
visualize_scalar_on_fem_points(U-S[1:(m+1)*(n+1)], m, n, h)
subplot(122)
visualize_scalar_on_fem_points(V-S[(m+1)*(n+1)+1:2(m+1)*(n+1)], m, n, h)
savefig("difference.png")


_tri(t, period) = (2 / π) * abs(asin(sin(π * (t - 1) / period)))
_sin(t, period) = abs(sin(π * (t - 1) / period))
_cycle(λ0, λ1, g) = abs(λ0 - λ1) * g + min(λ0, λ1)

"""
    Triangle{T, S<:Integer}(range0, range1, period)
    Triangle(;λ0, λ1, period)

A [triangle wave](https://en.wikipedia.org/wiki/Triangle_wave) schedule
with `period`.
The output conforms to
```text
abs(λ0 - λ1) * (2 / π) * abs(asin(sin(π * (t - 1) / period))) + min(λ0, λ1)
```

# Arguments
- `range0`/`λ0`: the first range endpoint
- `range1`/`λ1`: the second range endpoint
- `period::Integer`: the period
"""
struct Triangle{T, S<:Integer}
    range0::T
    range1::T
    period::S
end
Triangle(;λ0, λ1, period) = Triangle(λ0, λ1, period)

(schedule::Triangle)(t) = _cycle(schedule.range0, schedule.range1, _tri(t, schedule.period))

Base.eltype(::Type{<:Triangle{T}}) where T = T
Base.IteratorSize(::Type{<:Triangle}) = Base.IsInfinite()

Base.iterate(schedule::Triangle, t = 1) = schedule(t), t + 1

Base.axes(::Triangle) = (OneToInf(),)

"""
    TriangleDecay2{T, S<:Integer}(range0, range1, period)
    TriangleDecay2(;λ0, λ1, period)

A [triangle wave](https://en.wikipedia.org/wiki/Triangle_wave) schedule
with `period` and half the amplitude each cycle.
The output conforms to
```text
abs(λ0 - λ1) * Triangle(t) / (2^floor((t - 1) / period)) + min(λ0, λ1)
```
where `Triangle(t)` is `(2 / π) * abs(asin(sin(π * (t - 1) / schedule.period)))` (see [`Triangle`](#)).

# Arguments
- `range0`/`λ0`: the first range endpoint
- `range1`/`λ1`: the second range endpoint
- `period::Integer`: the period
"""
struct TriangleDecay2{T, S<:Integer}
    range0::T
    range1::T
    period::S
end
TriangleDecay2(;λ0, λ1, period) = TriangleDecay2(λ0, λ1, period)

(schedule::TriangleDecay2)(t) = _cycle(schedule.range0, schedule.range1,
                                       _tri(t, schedule.period) / (2^fld(t - 1, schedule.period)))

Base.eltype(::Type{<:TriangleDecay2{T}}) where T = T
Base.IteratorSize(::Type{<:TriangleDecay2}) = Base.IsInfinite()

Base.iterate(schedule::TriangleDecay2, t = 1) = schedule(t), t + 1

Base.axes(::TriangleDecay2) = (OneToInf(),)

"""
    TriangleExp{T, S<:Integer}(range0, range1, period, decay)
    TriangleExp(λ0, λ1, period, γ)
    TriangleExp(;λ0, λ1, period, γ)

A [triangle wave](https://en.wikipedia.org/wiki/Triangle_wave) schedule
with `period` and an exponentially decaying amplitude.
The output conforms to
```text
abs(λ0 - λ1) * Triangle(t) * γ^(t - 1) + min(λ0, λ1)
```
where `Triangle(t)` is `(2 / π) * abs(asin(sin(π * (t - 1) / schedule.period)))` (see [`Triangle`](#)).

# Arguments
- `range0`/`λ0`: the first range endpoint
- `range1`/`λ1`: the second range endpoint
- `period::Integer`: the period
- `decay`/`γ`: the decay rate
"""
struct TriangleExp{T, S<:Integer}
    range0::T
    range1::T
    period::S
    decay::T
end
TriangleExp(;λ0, λ1, period, γ) = TriangleExp(λ0, λ1, period, γ)

startvalue(schedule::TriangleExp) = schedule.tri.λ0
endvalue(schedule::TriangleExp) = schedule.tri.λ1
(schedule::TriangleExp)(t) = _cycle(schedule.range0, schedule.range1,
                                    _tri(t, schedule.period) * schedule.decay^(t - 1))

Base.eltype(::Type{<:TriangleExp{T}}) where T = T
Base.IteratorSize(::Type{<:TriangleExp}) = Base.IsInfinite()

Base.iterate(schedule::TriangleExp, t = 1) = schedule(t), t + 1

Base.axes(::TriangleExp) = (OneToInf(),)

"""
    Sin{T, S<:Integer}(range0, range1, period)
    Sin(;λ0, λ1, period)

A sine wave schedule with `period`.
The output conforms to
```text
abs(λ0 - λ1) * abs(sin(π * (t - 1) / period)) + min(λ0, λ1)
```

# Arguments
- `range0`/`λ0`: the first range endpoint
- `range1`/`λ1`: the second range endpoint
- `period::Integer`: the period
"""
struct Sin{T, S<:Integer}
    range0::T
    range1::T
    period::S
end
Sin(;λ0, λ1, period) = Sin(λ0, λ1, period)

(schedule::Sin)(t) = _cycle(schedule.range0, schedule.range1, _sin(t, schedule.period))

Base.eltype(::Type{<:Sin{T}}) where T = T
Base.IteratorSize(::Type{<:Sin}) = Base.IsInfinite()

Base.iterate(schedule::Sin, t = 1) = schedule(t), t + 1

Base.axes(::Sin) = (OneToInf(),)

"""
    SinDecay2{T, S<:Integer}(range0, range1, period)
    SinDecay2(;λ0, λ1, period)

A sine wave schedule with `period` and half the amplitude each cycle.
The output conforms to
```text
abs(λ0 - λ1) * Sin(t) / (2^floor((t - 1) / period)) + min(λ0, λ1)
```
where `Sin(t)` is `abs(sin(π * (t - 1) / period))` (see [`Sin`](#)).

# Arguments
- `range0`/`λ0`: the first range endpoint
- `range1`/`λ1`: the second range endpoint
- `period::Integer`: the period
"""
struct SinDecay2{T, S<:Integer}
    range0::T
    range1::T
    period::S
end
SinDecay2(;λ0, λ1, period) = SinDecay2(λ0, λ1, period)

(schedule::SinDecay2)(t) = _cycle(schedule.range0, schedule.range1,
                                  _sin(t, schedule.period) / (2^fld(t - 1, schedule.period)))

Base.eltype(::Type{<:SinDecay2{T}}) where T = T
Base.IteratorSize(::Type{<:SinDecay2}) = Base.IsInfinite()

Base.iterate(schedule::SinDecay2, t = 1) = schedule(t), t + 1

Base.axes(::SinDecay2) = (OneToInf(),)

"""
    SinExp{T, S<:Integer}(range0, range1, period, decay)
    SinDecay2(;λ0, λ1, period, γ)

A sine wave schedule with `period` and an exponentially decaying amplitude.
The output conforms to
```text
abs(λ0 - λ1) * Sin(t) * γ^(t - 1) + min(λ0, λ1)
```
where `Sin(t)` is `abs(sin(π * (t - 1) / period))` (see [`Sin`](#)).

# Arguments
- `range0`/`λ0`: the first range endpoint
- `range1`/`λ1`: the second range endpoint
- `period::Integer`: the period
- `decay`/`γ`: the decay rate
"""
struct SinExp{T, S<:Integer}
    range0::T
    range1::T
    period::S
    decay::T
end
SinExp(;λ0, λ1, period, γ) = SinExp(λ0, λ1, period, γ)

(schedule::SinExp)(t) = _cycle(schedule.range0, schedule.range1,
                               _sin(t, schedule.period) * schedule.decay^(t - 1))

Base.eltype(::Type{<:SinExp{T}}) where T = T
Base.IteratorSize(::Type{<:SinExp}) = Base.IsInfinite()

Base.iterate(schedule::SinExp, t = 1) = schedule(t), t + 1

Base.axes(::SinExp) = (OneToInf(),)

"""
    Cos{T, S<:Integer}(range0, range1, period)
    Cos(;λ0, λ1, period)

A cosine annealing schedule
(see ["SGDR: Stochastic Gradient Descent with Warm Restarts"](https://arxiv.org/abs/1608.03983v5))
The output conforms to
```text
abs(λ0 - λ1) * (1 + cos(π * mod(t - 1, period) / period)) / 2 + min(λ0, λ1)
```
This schedule is also referred to as "cosine annealing with warm restarts"
in machine learning literature.

# Arguments
- `range0`/`λ0`: the first range endpoint
- `range1`/`λ1`: the second range endpoint
- `period::Integer`: the period
"""
struct Cos{T, S<:Integer}
    range0::T
    range1::T
    period::S
end
Cos(;λ0, λ1, period) = Cos(λ0, λ1, period)

(schedule::Cos)(t) = _cycle(schedule.range0, schedule.range1,
                            (1 + cos(π * mod(t - 1, schedule.period) / schedule.period)) / 2)

Base.eltype(::Type{<:Cos{T}}) where T = T
Base.IteratorSize(::Type{<:Cos}) = Base.IsInfinite()

Base.iterate(schedule::Cos, t = 1) = schedule(t), t + 1

Base.axes(::Cos) = (OneToInf(),)


export dnorm, dgumbel, dgamma, dlognorm

"""
    dnorm(x::Real, μ::Real, σ::Real)

Normal distribution density function
"""
dnorm(x::Real, μ::Real, σ::Real) = exp(-0.5 * ((x - μ)/σ)^2)/(σ*sqrt2π)

"""
    dgumbel(x::Real, μ::Real, σ::Real)

Gumbel density, parameterized by mean (μ) and SD (σ) of x.
"""
function dgumbel(x::Real, μ::Real, σ::Real)
    β = (√6 * σ) / π
    α = μ - (β*γ)
    z = (x - α) / β
    exp(-(z + exp(-z)))
end

"""
    dgamma(x::Real, μ::Real, σ::Real)

Gamma density, parameterized by mean (μ) and sigma(σ) of x.
"""
function dgamma(x::Real, μ::Real, σ::Real)
    α = μ^2 / σ^2
    β = μ / σ^2
    (x^(α - 1) * exp(-β * x))  * (β^α / Γ(α))
end

"""
    dlognorm(x::Real, μ::Real, σ::Real)

Lognormal density, parameterized by mean (μ) and SD (σ) of x, NOT of log(x).
"""
function dlognorm(x::Real, μ::Real, σ::Real)
    lμ = log(μ^2 / sqrt(1 + σ^2 / μ^2))
    lσ = √(log(1 + σ^2 / μ^2))
    dnorm(log(x), lμ, lσ) / x
end


###################################
## Simple VI & GS for 2D Gaussian

using PyPlot
using Distributions

function calc_KL(mu1, lambda1, mu2, lambda2)
    D = size(mu1, 1)
    px_lnqx = 0.5 * logdet(lambda2) - 0.5 * ((mu1 - mu2)' * lambda2 * (mu1 - mu2) + trace(lambda2 * inv(lambda1)))
    px_lnpx = 0.5 * logdet(lambda1) - 0.5 * D
    KL = - (px_lnqx - px_lnpx)
    return KL[1]
end

function plot_results(result, truth)
    N = size(result, 1)
    H = Int(ceil(sqrt(N)))
    W = Int(ceil(N / H))
    for i in 1 : H
        for j in 1 : W
            n = (i - 1) * W + j
            if n <= N
                subplot(H, W, n)
                title("$n of $N")
                plot_gaussian(truth[1], truth[2], "b", "\$p(z)\$")
                plot_gaussian(result[n][1], result[n][2], "r", "\$p(z)\$")
            end
        end
    end
end

function plot_lines(X)
    D, N = size(X)
    X_d = zeros(D, 2*N + 1)
    X_d[:,1] = X[:,1]
    for i in 1 : N
        X_d[1, 2*i - 1] = X[1, i]
        X_d[1, 2*i] = X[1, i]
        X_d[2, 2*i] = X[2, i]
        X_d[2, 2*i + 1] = X[2, i]
    end
    plot(X[1,:], X[2,:], "oy")
    plot(X_d[1,1:2*N], X_d[2,1:2*N], "--y")
end

function plot_gaussian(Mu, Sigma, col, label)
    res = 100
    plot(Mu[1], Mu[2], "x", color=col)

    F = eigfact(Sigma)
    vec = F.vectors
    val = F.values
    dw = 2*pi/res
    w = dw * (0 : res)

    c = 1.0
    a = sqrt(c*val[1])
    b = sqrt(c*val[2])
    P1 = a*cos.(w)
    P2 = b*sin.(w)
    P = Mu .+ vec'*vcat(P1', P2')
    plot(P[1, :], P[2, :], "-", color=col, label=label)
end

"""
Variational inference for 2D Gauss.
"""
function main_VI()
    ## creat truth distribution
    D = 2 # dimension
    theta = 2.0*pi/12 # tilt
    A = reshape([cos.(theta), -sin.(theta),
                 sin.(theta), cos.(theta)],
                2, 2)
    mu = [0.0, 0.0]
    lambda = inv(A * inv(reshape([1,0,0,10], 2, 2)) * A')

    ## initialize
    #mu_h = randn(D)
    mu_h = [-0.5, 0.3]
    lambda_h = zeros(D,D)

    ## main iteration
    max_iter = 10
    KL = NaN * Array{Float64, 1}(max_iter)
    result = Array{Any, 1}(max_iter)
    for i in 1 : max_iter
        ## update
        mu_h[1] = mu[1] - inv(lambda[1,1])*lambda[1,2] * (mu_h[2] - mu[2])

        lambda_h[1,1] = lambda[1,1]
        mu_h[2] = mu[2] - inv(lambda[2,2])*lambda[2,1] * (mu_h[1] - mu[1])
        lambda_h[2,2] = lambda[2,2]

        ## calculate KL divergeince
        KL[i] = calc_KL(mu_h, lambda_h, mu, lambda)

        ## store the results
        result[i] = [deepcopy(mu_h), deepcopy(inv(lambda_h))]
    end

    ## visualize results
    figure("result per iteration (VI)")
    clf()
    plot_results(result, (mu, inv(lambda)))

    figure("result (VI)")
    clf()
    plot_gaussian(mu, inv(lambda), "b", "\$p(\\bf{z})\$")
    plot_gaussian(result[end][1], result[end][2], "r", "\$q(\\bf{z})\$")
    xlabel("\$z_1\$", fontsize=20)
    ylabel("\$z_2\$", fontsize=20)
    legend(fontsize=16)

    figure("KL divergence (VI)")
    clf()
    plot(1:max_iter, KL)
    ylabel("KL divergence", fontsize=16)
    xlabel("iteration", fontsize=16)
    show()
end

"""
Gibbs sampling for 2D Gauss.
"""
function main_GS()
    ## creat truth distribution
    D = 2 # dimension
    theta = 2.0*pi/12 # tilt
    A = reshape([cos.(theta), -sin.(theta),
                 sin.(theta), cos.(theta)],
                2, 2)
    mu = [0.0, 0.0]
    #lambda = inv(A * inv(reshape([1,0,0,10], 2, 2)) * A')
    lambda = inv(A * inv(reshape([1,0,0,100], 2, 2)) * A')

    ## initialize
    #max_iter = 1000
    max_iter = 50
    X = randn(D, max_iter)
    mu_h = randn(D)

    ## main iteration
    KL = NaN * Array{Float64, 1}(max_iter)
    for i in 2 : max_iter
        ## update
        mu_h[1] = mu[1] - inv(lambda[1,1])*lambda[1,2] * (X[2,i-1] - mu[2])
        X[1, i] = rand(Normal(mu_h[1], sqrt(inv(lambda[1,1]))))

        mu_h[2] = mu[2] - inv(lambda[2,2])*lambda[2,1] * (X[1,i] - mu[1])
        X[2, i] = rand(Normal(mu_h[2], sqrt(inv(lambda[2,2]))))

        if i > D
            KL[i] = calc_KL(mean(X[:,1:i], 2), inv(cov(X[:,1:i], 2)), mu, lambda)
        end
    end

    ## visualize results
    expt_mu = mean(X, 2)
    expt_Sigma = cov(X, 2)

    figure("samples (GS)")
    clf()
    plot_lines(X)
    plot_gaussian(mu, inv(lambda), "b", "\$p(\\bf{z})\$")
    plot_gaussian(expt_mu, expt_Sigma, "r", "\$q(\\bf{z})\$")
    xlabel("\$z_1\$", fontsize=20)
    ylabel("\$z_2\$", fontsize=20)
    legend(fontsize=16)

    figure("KL divergence (GS)")
    clf()
    plot(1:max_iter, KL)
    ylabel("KL divergence", fontsize=16)
    xlabel("sample size", fontsize=16)
    show()
end

main_VI()
main_GS()


using DEMC
using Random
using LinearAlgebra
using Distributions
Random.seed!(31953150)

# generate data
nobs = 100
npar = 3
Σ = ones(npar, npar)
for i = 1:npar
    for iprime = 1:i
        if i != iprime
            Σ[i, iprime] = 0.25
            Σ[iprime, i] = 0.25
        end
    end
end

DataDistribution = MvNormal(zeros(npar), Σ)
X = zeros(nobs, npar+1)
X[:,1] .= 1.
for iobs = 1:nobs
    X[iobs, 2:end] .= rand(DataDistribution)
end

β = rand(npar+1)
y = X*β + randn(nobs)

# log objective function - for correct std errors need to use optimal weights
log_obj(b) = -0.5 * sum((y.-X*b).^2)

# set up of DEMCz chain
Npar = length(β)
ndim = Npar
blockindex = [1:Npar] # parameter blocks: here choose all parameters to be updated simultaenously
Nblocks = length(blockindex)
eps_scale = 1e-5*ones(Npar) # scale of random error around DE update
γ = 2.38 # scale of DE update, 2.38 is the "optimal" number for a normal distribution
N = 3
K = 10
Z = randn((10*ndim, ndim))

# Number of iterations in Chain
Ngen = 2500
mc, Z = DEMC.demcz_sample(log_obj, Z, N, K, Ngen, Nblocks, blockindex, eps_scale, γ; verbose =false)

# drop first half of chain
keep = Int(round(Ngen/2))+1:Ngen
Ngen_burned = length(keep)
chain_burned = mc.chain[:,:,keep]
chainflat = DEMC.flatten_chain(chain_burned, N, Ngen_burned, Npar)'
bhat = mean(chainflat,dims=1)[:]
println("\n estimates: ", bhat, "\n dist to true: ", bhat - β)
# covariance of estimates
b, Σb = DEMC.mean_cov_chain(chain_burned, N, Ngen_burned, Npar)
# correct std errors for incorrect weighting matrix

# OLS
bols = (X'*X)\(X'*y)
ui = y .- X*bols
s2hat = (1/(nobs-npar)) * sum(ui.^2)
Σols = inv(X'X)*s2hat


# Firms' parameters
const A = 1
const N = 1
const α = 0.33
const β = 0.96
const δ = 0.05

"""
Compute wage rate given an interest rate, r
"""
function r_to_w(r::Real)
    return A * (1 - α) * (A * α / (r + δ)) ^ (α / (1 - α))
end

"""
Inverse demand curve for capital. The interest rate
associated with a given demand for capital K.
"""
function rd(K::Real)
    return A * α * (N / K) ^ (1 - α) - δ
end

"""
Map prices to the induced level of capital stock.

##### Arguments
- `am::Household` : Household instance for problem we want to solve
- `r::Real` : interest rate

##### Returns
- The implied level of aggregate capital
"""
function prices_to_capital_stock(am::Household, r::Real)

    # Set up problem
    w = r_to_w(r)
    setup_R!(am, r, w)
    aiyagari_ddp = DiscreteDP(am.R, am.Q, am.β)

    # Compute the optimal policy
    results = solve(aiyagari_ddp, PFI)

    # Compute the stationary distribution
    stationary_probs = stationary_distributions(results.mc)[:, 1][1]

    # Return K
    return dot(am.s_vals[:, 1], stationary_probs)
end

# Create an instance of Household
am = Household(β=β, a_max=20.0)

# Create a grid of r values at which to compute demand and supply of capital
num_points = 20
r_vals = linspace(0.005, 0.04, num_points)

# Compute supply of capital
k_vals = prices_to_capital_stock.(am, r_vals)

# Plot against demand for capital by firms
demand = rd.(k_vals)
labels =  ["demand for capital" "supply of capital"]
plot(k_vals, [demand r_vals], label=labels, lw=2, alpha=0.6)
plot!(xlabel="capital", ylabel="interest rate", xlim=(2, 14), ylim=(0.0, 0.1))


import Statistics:mean

function ParetoSub(X, perm)
    Z= X.-X[:,1]
    nondominated=reshape(any(Z.<0,dims=1),:)
    ispareto = all(any(Z[:,nondominated].>0, dims=1))
    X=X[:, nondominated]
    nondominated = perm[nondominated]
    return ispareto, nondominated, X
end

"""
    index = ParetoSet(X[, sense])
Get the Pareto set from a given set of oberservations `X`.
# Argument
- `X::AbstractMatrix{Real}` size(X)=(n_objectives, n_samples)
- `sense::AbstractVector{Union{Symbol, EGOSense}}=repeat(:Min, n_objectives)` sense of each objective, size(S)=(n_objectives,)
"""
function ParetoSet(X::AbstractMatrix)
    n,m = size(X)
    Xmin = minimum(X, dims=2)
    Xt = X.-Xmin
    Xm = mean(Xt, dims=2)
    Xn = maximum(Xt ./ (Xm.+maximum(Xm)), dims=1)
    perm = sortperm(reshape(Xn,:))
    Y = X[:, perm]
    membership = falses(m)
    while length(perm)>1
        k = perm[1]
        membership[k], perm, Y = ParetoSub(Y, perm)
    end
    membership[perm] .= true
    membership
end

ParetoSet(X, sense::AbstractVector{EGOSense}) = ParetoSet(X.*Int.(sense))

ParetoSet(X, sense::AbstractVector{Symbol}) = ParetoSet(X, sym2sense.(sense))


using LinearAlgebra
using Distributions
using Plots

include("./gen_data.jl")
include("./kalman_filters.jl")

"""Experimental parameters"""

# Number of sigma points
N = 10

# Time horizon
T = 100

# Experimental parameters
transition_coeffs = 1.0
emission_coeffs = 1.0

# Noises
process_noise = 0.5
measurement_noise = 0.1

# Prior state
state0 = ([0.], reshape([1e-12], 1,1))

"Generate data"

# Generate signal
observations, states = LGDS(transition_coeffs,
                            emission_coeffs,
                            process_noise,
                            measurement_noise,
                            state0;
                            time_horizon=T)

# Check signal visually
scatter(1:T, observations, color="blue", label="observations")
plot!(1:T, states[2:end], color="red", label="states")

"""Basic Kalman filter"""

# Call filter
include("./kalman_filters.jl")
mx, Px = kalman_filter(observations,
                       transition_coeffs,
                       emission_coeffs,
                       process_noise,
                       measurement_noise,
                       state0)

# Visualize estimates
scatter(1:T, observations[1,:], color="black", label="observations")
plot!(1:T, states[1,2:end], color="red", label="latent states")
plot!(1:T, mx[:], color="purple", label="inferred")
plot!(1:T, mx[:],
      ribbon=[sqrt.(Px[1,1,:]), sqrt.(Px[1,1,:])],
      color="purple", alpha=0.1, label="")
xlabel!("time (t)")
ylabel!("signal")
savefig("./viz/LGDS_kf.png")

"Generate data"

# Nonlinearities
transition_function(x) = .1*x.^3 + 0.5*x
emission_function(x) = x

# Noises
process_noise = reshape([0.1], 1,1)
measurement_noise = reshape([0.2], 1,1)

# Generate signal
observations, states = NLGDS(transition_function,
                             emission_function,
                             process_noise,
                             measurement_noise,
                             state0;
                             time_horizon=T)

# Check signal visually
scatter(1:T, observations, color="black", label="observations")
plot!(1:T, states[2:end], color="red", label="states")

"""Nonlinear Kalman filter"""

# Call basic filter
mx1, Px1 = kalman_filter(observations,
                         transition_coeffs,
                         emission_coeffs,
                         process_noise,
                         measurement_noise,
                         state0)

# Visualize estimates
p1 = scatter(1:T, observations[1,:], color="black", label="observations", size=(1200,500))
plot!(1:T, states[1,2:end], color="red", label="latent states")
plot!(1:T, mx1[:], color="purple", label="linear")
plot!(1:T, mx1[:],
      ribbon=[sqrt.(Px1[1,1,:]), sqrt.(Px1[1,1,:])],
      color="purple", alpha=0.1, label="")
xlabel!("time (t)")
ylabel!("signal")
savefig(p1, "./viz/NLGDS_KF.png")

# Call extended filter with first-order Taylor
mx2, Px2 = extended_kalman_filter(observations,
                                  transition_function,
                                  emission_function,
                                  process_noise,
                                  measurement_noise,
                                  state0)

p2 = scatter(1:T, observations[1,:], color="black", label="observations", size=(1200,500))
plot!(1:T, states[1,2:end], color="red", label="latent states")
plot!(1:T, mx2[:], color="blue", label="extended")
plot!(1:T, mx2[:],
      ribbon=[sqrt.(Px2[1,1,:]), sqrt.(Px2[1,1,:])],
      color="blue", alpha=0.1, label="")
xlabel!("time (t)")
ylabel!("signal")
savefig(p2, "./viz/NLGDS_E1KF.png")

# Call extended filter with second-order Taylor
mx3, Px3 = extended_kalman_filter(observations,
                                  transition_function,
                                  emission_function,
                                  process_noise,
                                  measurement_noise,
                                  state0,
                                  second_order=true)

p3 = scatter(1:T, observations[1,:], color="black", label="observations", size=(1200,500))
plot!(1:T, states[1,2:end], color="red", label="latent states")
plot!(1:T, mx3[:], color="cyan", label="extended")
plot!(1:T, mx3[:],
    ribbon=[sqrt.(Px3[1,1,:]), sqrt.(Px3[1,1,:])],
    color="cyan", alpha=0.1, label="")
xlabel!("time (t)")
ylabel!("signal")
savefig(p3, "./viz/NLGDS_E2KF.png")

# Call unscented filter
mx4, Px4 = unscented_kalman_filter(observations,
                                   transition_function,
                                   emission_function,
                                   process_noise,
                                   measurement_noise,
                                   state0,
                                   α=1., κ=0., β=2.)

p4 = scatter(1:T, observations[1,:], color="black", label="observations", size=(1200,500))
plot!(1:T, states[1,2:end], color="red", label="latent states")
plot!(1:T, mx4[:], color="green", label="unscented")
plot!(1:T, mx4[:],
      ribbon=[sqrt.(Px4[1,1,:]), sqrt.(Px4[1,1,:])],
      color="green", alpha=0.1, label="")
xlabel!("time (t)")
ylabel!("signal")
savefig(p4, "./viz/NLGDS_UKF.png")

plot(p1, p2, p3, p4, layout=(4,1), size=(2000,1600))
savefig("./viz/NLGDS_filters.png")


#!/usr/bin/julia

# Trizen
# 17 January 2017
# https://github.com/trizen

# Generates the Mandelbrot set.

# See also:
#   https://en.wikipedia.org/wiki/Mandelbrot_set
#   https://trizenx.blogspot.ro/2017/01/mandelbrot-set.html

using Images

@inline function hsv2rgb(h, s, v)
    c = v * s
    x = c * (1 - abs(((h/60) % 2) - 1))
    m = v - c

    if h < 60
        r,g,b = (c, x, 0)
    elseif h < 120
        r,g,b = (x, c, 0)
    elseif h < 180
        r,g,b = (0, c, x)
    elseif h < 240
        r,g,b = (0, x, c)
    elseif h < 300
        r,g,b = (x, 0, c)
    else
        r,g,b = (c, 0, x)
    end

    (r + m), (b + m), (g + m)
end

function mandelbrot()

    w, h = 1000, 1000

    zoom  = 0.5
    moveX = 0
    moveY = 0

    maxIter = 100
    img = zeros(RGB{Float64}, h, w)

    for x in 1:w, y in 1:h
        i = maxIter
        c = Complex(
            (2*x - w) / (w * zoom) + moveX,
            (2*y - h) / (h * zoom) + moveY
        )
        z = c
        while abs(z) < 2 && (i -= 1) > 0
            z = z^2 + c
        end
        r,g,b = hsv2rgb(i / maxIter * 360, 1, i / maxIter)
        img[y,x] = RGB{Float64}(r, g, b)
    end

    println("Generating image...")
    save("mandelbrot_set.png", img)
end

mandelbrot()


export mutualinfo, Kraskov1, Kraskov2

abstract type MutualInformationEstimator <: EntropyEstimator end

"""
    mutualinfo(x, y, est; base = 2, q = 1)

Estimate mutual information between `x` and `y`, ``I^{q}(x; y)``, using the provided
entropy/probability estimator `est` from Entropies.jl, and Rényi entropy of order `q`
(defaults to `q = 1`, which is the Shannon entropy), with logarithms to the given `base`.

Both `x` and `y` can be vectors or (potentially multivariate) [`Dataset`](@ref)s.

Worth highlighting here are the estimators that compute entropies _directly_, e.g.
nearest-neighbor based methhods. The choice is between naive
estimation using the [`KozachenkoLeonenko`](@ref) or [`Kraskov`](@ref) entropy estimators,
or the improved [`Kraskov1`](@ref) and [`Kraskov2`](@ref) dedicated ``I`` estimators. The
latter estimators reduce bias compared to the naive estimators.

**Note**: only Shannon entropy is possible to use for nearest neighbor estimators, so the
keyword `q` cannot be provided; it is hardcoded as `q = 1`.

## Description

Mutual information ``I`` between ``X`` and ``Y``
is defined as

```math
I(X; Y) = \\sum_{y \\in Y} \\sum_{x \\in X} p(x, y) \\log \\left( \\dfrac{p(x, y)}{p(x)p(y)} \\right)
```

Here, we rewrite this expression as the sum of the marginal entropies, and extend the
definition of ``I`` to use generalized Rényi entropies

```math
I^{q}(X; Y) = H^{q}(X) + H^{q}(Y) - H^{q}(X, Y),
```

where ``H^{q}(\\cdot)`` is the generalized Renyi entropy of order ``q``, i.e., the
`genentropy` function from Entropies.jl.
"""
function mutualinfo end

mutualinfo(x::Vector_or_Dataset, y::Vector_or_Dataset) =
    error("Estimator missing. Please provide a valid estimator as the third argument.")

function mutualinfo(x::Vector_or_Dataset, y::Vector_or_Dataset, est; base = 2, q = 1)
    X = genentropy(Dataset(x), est; base = base, q = q)
    Y = genentropy(Dataset(y), est; base = base, q = q)
    XY = genentropy(Dataset(x, y), est; base = base, q = q)
    MI = X + Y - XY
end

include("nearestneighbor.jl")


using LinearAlgebra: norm, inv, eigvals
import ForwardDiff
using StaticArrays: setindex
using DynamicalSystems

# This code is written for `SVector`s, or out-of-place dynamical systems.
# Notation z is used for the vector (x..., p), i.e. mixed space
"""
    bifurcation_rule_form(ds) → f, J
Produce dynamic rule `f` and Jacobian function `J` in a two-argument form.
"""
function bifurcation_rule_form(ds::ContinuousDynamicalSystem)
    f = (x, p) -> ds.f(x, p, 0.0)
    J = (x, p) -> ds.jacobian(x, p, 0.0)
    return f, J
end

function bifurcation_rule_form(ds::DiscreteDynamicalSystem)
    error("TODO: Convert `f` to `f - x`.")
    f = (x, p) -> ds.f(x, p, 0.0)
    J = (x, p) -> ds.jacobian(x, p, 0.0)
    return f, J
end

# Corrector: Newton's method in mixed-space with extra fixed condition
"""
    corrector(zpred, zprev, f, J; max_steps = 200, δ = 0.9, ε = 1e-3) → (z*, success)
Implement the Newton algorithm to converge to the fixed point for given predicted
mixed state `zpred`, previously found state `zprev`, function `f`, Jacobian `J`
and stepping factor `δ`.

If iteration occurs for more than `max_steps` without convergence better than `ε`,
then iteration stops and `success` will be `false`.
"""
function corrector(zpred, zprev, f, J; δ = 0.9, max_steps = 200, ε = 1e-6)
    Δz = zpred .- zprev
    # index of variable that changed the most
    # The `iszero` clause is just to keep the first iteration at given `p0`
    i = all(iszero, Δz) ? length(Δz) : argmax(Δz .^ 2)
    c = 0
    zⱼ = zpred
    zⱼ₊₁ = newton_step!(zⱼ, zpred, i, f, J, δ)
    while norm(zⱼ₊₁ - zⱼ) > ε
        zⱼ = zⱼ₊₁
        zⱼ₊₁ = newton_step!(zⱼ, zpred, i, f, J, δ)
        c += 1
        if c > max_steps
            @warn("Newton did not converge.")
            return (zⱼ₊₁, false)
        end
    end
    return zⱼ₊₁, true
end

function newton_step!(zⱼ, zpred, i, f, J, δ)
    Jfinal = mixed_jacobian(zⱼ, i, f, J)
    xⱼ = zⱼ[1:end-1]; pⱼ = zⱼ[end]
    g = f(xⱼ, pⱼ)
    gz = vcat(g, zⱼ[i] - zpred[i])
    zⱼ₊₁ = zⱼ - δ*(inv(Jfinal))*gz
    return zⱼ₊₁
end

function mixed_jacobian(z, i, f, J)
    x = z[1:end-1]; p = z[end]
    # start creating the mixed space jacobian
    j = J(x, p)
    # to the state space jacobian add one more column, derivative towards p
    pder = ForwardDiff.derivative(p -> f(x, p), p)
    Jmixed = hcat(j, pder)
    # add the last row, which is 1 for the `i` entry, 0 everywhere else
    last_row = setindex((@SVector zeros(length(z))), 1.0, i)
    Jfinal = vcat(Jmixed, last_row')
    return Jfinal
end

# Predictor: Secant (linear extrapolation of already found bifurcation curve)
function predictor(zs, dz0)
    if length(zs) == 1
        return zs[end]
    elseif length(zs) == 2 # 1 entry is z0, 2nd entry is 1st found fixed point
        return zs[end] .+ dz0
    else
        return 2zs[end] .- zs[end-1]
    end
end

# Continuation function: perform a step of predictor-corrector and save values
function continuation!(zs, f, J; dz0, pmin, pmax)
    zpred = predictor(zs, dz0)
    (pmin ≤ zpred[end] ≤ pmax) || return false
    zˣ, success = corrector(zpred, zs[end], f, J)
    push!(zs, zˣ)
    return success
end

# Continuation loop: do continuation for a given amount of steps
function continuation(f, J, x0, p0;
        pmin, pmax, dp0, dx0, N = 1000
    )

    z0 = vcat(x0, p0); zs = [z0]; dz0 = vcat(dx0, dp0)

    ps = [p0]
    xs = Dataset([x0])
    stability = Bool[]
    for i in 1:N
        success = continuation!(zs, f, J; dz0, pmin, pmax)
        # Stop iteration if we exceed given parameter margins
        success || break
        # Detect stability of found fixed point (needs `Array` coz of StaticArrays.jl)
        eigenvalues = eigvals(Array(J(zs[end][1:end-1], zs[end][end])))
        μ = maximum(real(v) for v ∈ eigenvalues)
        isstable = μ < 0
        push!(stability, isstable)
    end
    xs = Dataset([z[1:end-1] for z in zs])
    ps = [z[end] for z in zs]
    popfirst!(xs.data); popfirst!(ps) # remove initial guess
    return xs, ps, stability
end


### A Pluto.jl notebook ###
# v0.12.16

using Markdown
using InteractiveUtils

# ╔═╡ e077ab16-22e5-11eb-15ed-5f0654bf7328
using Pkg, DrWatson

# ╔═╡ e75a9a06-22e5-11eb-2e36-8d4f62b830ed
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using Random, Distributions, Plots
	Random.seed!(0)
end

# ╔═╡ ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
md"## Listing5.05"

# ╔═╡ fa68607e-22e5-11eb-0558-c9a4d9f77426
begin
	n1, n2 = 10, 15
	N = 10^6
	mu, sigma = 10, 4
	normDist = Normal(mu,sigma)
end;

# ╔═╡ 67a42f40-3a99-11eb-2199-cb72cdee58fa
begin
	fValues = Array{Float64}(undef, N)

	for i in 1:N
		data1 = rand(normDist,n1)
		data2 = rand(normDist,n2)
		fValues[i] = var(data1)/var(data2)
	end
end

# ╔═╡ 67a46e56-3a99-11eb-1254-3d0d490364f5
begin
	fRange = 0:0.1:5
	stephist(fValues, bins=400, c=:blue, label="Simulated", normed=true)
	plot!(fRange, pdf.(FDist(n1-1, n2-1), fRange),
		c=:red, label="Analytic", xlims=(0,5), ylims=(0,0.8),
			xlabel = "F", ylabel = "Density")
end

# ╔═╡ 475ff888-22e6-11eb-2354-09f8f40a8e12
md"## End of listing5.05"

# ╔═╡ Cell order:
# ╟─ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
# ╠═e077ab16-22e5-11eb-15ed-5f0654bf7328
# ╠═e75a9a06-22e5-11eb-2e36-8d4f62b830ed
# ╠═fa68607e-22e5-11eb-0558-c9a4d9f77426
# ╠═67a42f40-3a99-11eb-2199-cb72cdee58fa
# ╠═67a46e56-3a99-11eb-1254-3d0d490364f5
# ╟─475ff888-22e6-11eb-2354-09f8f40a8e12


function simpson_integrate(u, stepsize)
    numpts = length(u)
    @assert iseven(numpts - 1)
    mid = div(numpts,2)
    integral = 0.0
    for i = 1:mid
        integral += stepsize / 3.0 * (u[2i-1] + 4.0 * u[2i] + u[2i+1])
    end
    return integral
end


using SchumakerSpline
using Dates
tol = 10*eps()

StartDate = Date(2018, 7, 21)
x = Array{Date}(undef,1000)
for i in 1:1000
    x[i] = StartDate +Dates.Day(2* (i-1))
end

function f(x::Date)
    days_between = Dates.days(x - StartDate)
    return log(days_between+1) + sqrt(days_between)
end
y = f.(x)

spline = Schumaker(x,y)
for i in 1:length(x)
    abs(evaluate(spline, x[i]) - y[2]) < tol
end
# Evaluation with a Float64.
evaluate(spline, 11.5)

# Testing second derivatives
second_derivatives = evaluate.(spline, x,2)
maximum(second_derivatives) < tol

# Testing Integrals
function analytic_integral(lhs,rhs)
    lhs_in_days = Dates.days(lhs - StartDate)
    rhs_in_days = Dates.days(rhs - StartDate)
    return (rhs_in_days+1)*log(rhs_in_days+1)-rhs_in_days + (2/3)*rhs_in_days^(3/2) - ((lhs_in_days+1)*log(lhs_in_days+1) - lhs_in_days + (2/3)*lhs_in_days^(3/2))
end

lhs = StartDate
rhs = StartDate + Dates.Month(16)
numerical_integral = evaluate_integral(spline, lhs,rhs)
analytical = analytic_integral(lhs,rhs)
abs(  analytical - numerical_integral  ) < 1

## Testing with only one date provided.
x = Array{Date}(undef, 1)
x[1] = Date(2018, 7, 21)
y = Array{Float64}(undef, 1)
y[1] = 0.0
spline = Schumaker(x,y)
abs(evaluate(spline,  Date(2018, 7, 21))) < tol
abs(evaluate(spline,  Date(2019, 7, 21))) < tol
abs(evaluate(spline,  Date(2000, 7, 21))) < tol

## Testing with two dates provided.
x = Array{Date}(undef,2)
x[1] = Date(2018, 7, 21)
x[2] = Date(2018, 8, 21)
y = Array{Float64}(undef,2)
y[1] = 0.0
y[2] = 1.0
spline = Schumaker(x,y)
abs(evaluate(spline,  Date(2018, 7, 21))) < tol
abs(evaluate(spline,  Date(2018, 7, 30))) > tol
abs(evaluate(spline,  Date(2018, 8, 21)) - y[2]) < tol
abs(evaluate(spline,  Date(2019, 8, 21)) - y[2]) > tol

spline = Schumaker(x, y , extrapolation =  (Constant,Constant))
abs(evaluate(spline,  Date(2018, 8, 21)) - y[2]) < tol
abs(evaluate(spline,  Date(2019, 8, 21)) - y[2]) < tol

## Testing with three dates provided.
x = Array{Date}(undef,3)
x[1] = Date(2018, 7, 21)
x[2] = Date(2018, 8, 21)
x[3] = Date(2018, 9, 21)
y = Array{Float64}(undef,3)
y[1] = 0.0
y[2] = 1.0
y[3] = 1.3
spline = Schumaker(x,y)
abs(evaluate(spline,  x[2]) - y[2]) < tol


export Sech

"""
`Sech([domainType=Float64::Type,] dim_in::Tuple)`

Creates an hyperbolic secant non-linear operator with input dimensions `dim_in`:
```math
\\text{sech} ( \\mathbf{x} ).
```

"""
struct Sech{T,N} <: NonLinearOperator
	dim::NTuple{N,Int}
end

function Sech(DomainType::Type, DomainDim::NTuple{N,Int}) where {N}
	Sech{DomainType,N}(DomainDim)
end

Sech(DomainDim::NTuple{N,Int}) where {N} = Sech{Float64,N}(DomainDim)
Sech(DomainDim::Vararg{Int}) = Sech{Float64,length(DomainDim)}(DomainDim)

function mul!(y::AbstractArray{T,N}, L::Sech{T,N}, x::AbstractArray{T,N}) where {T,N}
	y .= sech.(x)
end

function mul!(y::AbstractArray,
              J::AdjointOperator{Jacobian{A,TT}},
              b::AbstractArray) where {T,N, A<: Sech{T,N}, TT <: AbstractArray{T,N}}
    L = J.A
    y .= -conj.( tanh.(L.x) .* sech.(L.x) ).*b
end

fun_name(L::Sech) = "sech"

size(L::Sech) = (L.dim, L.dim)

domainType(L::Sech{T,N}) where {T,N} = T
codomainType(L::Sech{T,N}) where {T,N} = T


### A Pluto.jl notebook ###
# v0.12.6

using Markdown
using InteractiveUtils

# ╔═╡ e77948a6-1ecc-11eb-3285-7f06dcdb7cf5
using Pkg, DrWatson

# ╔═╡ e7799b9e-1ecc-11eb-311f-035de72c3603
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using StatsBase, Combinatorics, Plots
end

# ╔═╡ 6c38d2ce-1ecc-11eb-281d-0d13a832adea
md"## Listing 2.3"

# ╔═╡ fe1ee6a6-1ecc-11eb-1930-0301940bc5f7
	N = 10^3

# ╔═╡ e77a2a28-1ecc-11eb-0acb-5167936362f5
begin
	matchExists1(n) = 1 - prod([k/365 for k in 365:-1:365-n+1])
	matchExists2(n) = 1- factorial(365,365-big(n))/365^big(n)

	function bdEvent(n)
		birthdays = rand(1:365,n)
		dayCounts = counts(birthdays, 1:365)
		return maximum(dayCounts) > 1
	end

	probEst(n) = sum([bdEvent(n) for _ in 1:N])/N
end

# ╔═╡ e7885e54-1ecc-11eb-03d1-2580aad82a3d
begin
	xGrid = 1:50
	analyticSolution1 = [matchExists1(n) for n in xGrid]
	analyticSolution2 = [matchExists2(n) for n in xGrid]
	("Maximum error: $(round(maximum(abs.(analyticSolution1 - analyticSolution2)), digits=25))")
end

# ╔═╡ e788edce-1ecc-11eb-3682-b1bfc98646a0
begin
	mcEstimates = [probEst(n) for n in xGrid]

	plot(xGrid, analyticSolution1, c=:blue, label="Analytic solution")
	scatter!(xGrid, mcEstimates, c=:red, ms=6, msw=0, shape=:xcross,
		label="MC estimate", xlims=(0,50), ylims=(0, 1),
		xlabel="Number of people in room",
		ylabel="Probability of birthday match",
		legend=:topleft)
end

# ╔═╡ e7965b58-1ecc-11eb-2308-093c9f5a167e
md"## End of listing 2.3"

# ╔═╡ Cell order:
# ╟─6c38d2ce-1ecc-11eb-281d-0d13a832adea
# ╠═e77948a6-1ecc-11eb-3285-7f06dcdb7cf5
# ╠═e7799b9e-1ecc-11eb-311f-035de72c3603
# ╠═fe1ee6a6-1ecc-11eb-1930-0301940bc5f7
# ╠═e77a2a28-1ecc-11eb-0acb-5167936362f5
# ╠═e7885e54-1ecc-11eb-03d1-2580aad82a3d
# ╠═e788edce-1ecc-11eb-3682-b1bfc98646a0
# ╟─e7965b58-1ecc-11eb-2308-093c9f5a167e


### A Pluto.jl notebook ###
# v0.14.6

using Markdown
using InteractiveUtils

# ╔═╡ 7ef4b0b3-9d0a-4356-bd16-14487f14e0ab
begin
	using PlutoUI
	PlutoUI.TableOfContents(aside=true)
end

# ╔═╡ 0cd92d78-1da6-435b-ad8e-60af29502e1d
begin
	# Necessary packages
	# If a package is missing, follow the instructions.
	using LightGraphs
	using GraphPlot
	using SparseArrays
	using LinearAlgebra
	using Plots
	using Distances
	using Arpack
	using Images
	using Random
end

# ╔═╡ cb61f761-b2ee-4818-9149-f85d95b76a1e
md"""
# Spectral Graph Bipartitioning

Many data clustering problems can be interpreted as clustering of vertices of graphs. __Graph bipartitioning problem__ is to partition vertices into subsets such that the connections within subsets are stronger than the connections between different subsets.

Partition of the vertices into two subsetts is done according to signs of the eigenvectors of the second smallest eigenvalue of the Laplacian matrix.

__Prerequisites__

The reader should be familiar with the basic graph theory, linear algebra and, in particular,  eigenvalues and eigenvectors.

__Competences__

The reader should be able to apply graph spectral bipartitioning and recursive bipartitioning to data clustering problems.

__Credits.__ The notebook was initially derived from M.Sc. Thesis of Ivančica Mirošević.
"""

# ╔═╡ 5926c79a-6038-4dbc-b49a-5b05d02250a0
md"""
# Graphs

For more details, see [W. H. Haemers, Matrices and Graphs, in L. Hogben, Ed., 'Handbook of Linear Algebra', pp. 39.1-39.14, CRC Press, Boca Raton, 2014.](https://www.routledge.com/Handbook-of-Linear-Algebra/Hogben/p/book/9781138199897) and [S. Butler and F. Chung, Spectral Graph Theory, ibid., pp. 47.1-47.6](https://www.routledge.com/Handbook-of-Linear-Algebra/Hogben/p/book/9781138199897) and the references therein.

## Definitions

A __weighted graph__ is an ordered triplet $G=(V,E,\omega)$, where $V=\{1,2,3,...,n\}$ is the set of __vertices__ , $E=\{(i,j)\}$ is a set of __edges__ connecting vertices, and $\omega$ is a set of __weights__ of edges. We assume $G$ is undirected.
"""

# ╔═╡ a66a6e80-1f80-11eb-15f8-71ffaa5050e7
md"
__Adjacency matrix__ of graph $G$ is the matrix $A$ defined as

$$A_{ij}=\begin{cases} 1 \quad \textrm{if}\ (i,j)\in E, \\
0\quad  \textrm{otherwise} \end{cases}.$$

__Weight matrix__ of graph $G$ is the matrix $W$ defined as

$W_{ij}=\begin{cases} \omega(e) \quad \textrm{if}\ e=(i,j)\in E, \\
0\quad  \textrm{otherwise} \end{cases}.$

__Laplacian matrix__ of graph $G$ is the matrix

$L=D-W,$

where
$D=\mathop{\mathrm{diag}}(d_1,d_2,\ldots,d_n)$ with $d_i=\sum_{k=1}^n W_{ik}$ for $i=1,\ldots,n$.

__Normalized Laplacian matrix__ is the matrix

$L_n=D^{-1/2} L D^{-1/2}\equiv D^{-1/2} (D-W) D^{-1/2}$

(__diagonally scaled $L$__).

__Incidence matrix__ of graph $G$ is the $|V|\times |E|$ matrix $I_G$. Each row of $I_G$ corresponds to a vertex of $G$ and each column corresponds to an edge of $G$.
In the column corresponding to en edge $e=(i,j)$, all elements are zero except the ones in the $i$-th and $j$-th row, which are equal to $\sqrt{\omega(e)}$ and $-\sqrt{\omega(e)}$, respectively.
"

# ╔═╡ d4cf12ce-12fb-4562-bbea-e9b85cba94c2
md"""
### Examples

Graph types and algorithms are implemented in the package [LightGraphs.jl](https://github.com/JuliaGraphs/LightGraphs.jl).

Plotting graphs is done by the packages
[GraphPlot.jl](https://github.com/JuliaGraphs/GraphPlot.jl).

As a small inconvenience, we can only plot unweighted graphs and plot weights as edge labels.
"""

# ╔═╡ 87cca1d3-b944-4d74-bf4c-1562f34cf91a
begin
	# Sources, targets and weights
	n=7
	sn=[1,1,1,2,2,3,3,3,5,5,6]
	tn=[2,3,4,4,5,4,6,7,6,7,7]
	wn=[2,3,4,7,1,3,2,1,7,3,5]
	[sn tn wn]
end

# ╔═╡ 3d7b4a01-9a7d-4111-a45d-1d31f1ef6278
begin
	# Create the graph
	G=Graph(n)
	for i=1:length(sn)
	    add_edge!(G,sn[i],tn[i])
	end
	G
end

# ╔═╡ 3b5ed1fa-75b0-43fb-a918-4c2e8f52d039
# What is the optimal bipartition?
gplot(G, nodelabel=1:n, edgelabel=wn)

# ╔═╡ 8f7d1f6c-ee4c-407f-80d9-d2abc2af949e
begin
	# We define some functions
	function WeightMatrix(src::Array,dst::Array,weights::Array)
	    n=nv(G)
	    sparse([src;dst],[dst;src],[weights;weights],n,n)
	end

	Laplacian(W::AbstractMatrix)=spdiagm(0=>vec(sum(W,dims=2)))-W

	function NormalizedLaplacian(L::AbstractMatrix)
	    D=1.0./sqrt.(diag(L))
		Diagonal(D)*L*Diagonal(D)
	end
end

# ╔═╡ b7f127d8-e8cd-4771-8091-c88f9ffe6b08
W=WeightMatrix(sn,tn,wn)

# ╔═╡ 3f5447bb-95cd-4520-b0b9-e85f33b323bd
Matrix(W)

# ╔═╡ ecb2a46e-f9dc-4e57-812f-0a2788afb202
begin
	L=Laplacian(W)
	Matrix(L)
end

# ╔═╡ 86e5f00f-13ee-4867-b0cd-187f2c782a46
Lₙ=NormalizedLaplacian(L)

# ╔═╡ cf26dbe9-3c4f-4c95-88ef-f197f360d759
Matrix(Lₙ)

# ╔═╡ 8fcd2440-4013-4b37-a747-2c0c7f6d4aeb
issymmetric(Lₙ)

# ╔═╡ 0c8d3572-3436-446f-952a-b19d0370ab38
# Let us compute the incidence matrix
function IncidenceMatrix(G::Graph, weights::Array)
    A=zeros(nv(G),ne(G))
    k=1
    for a in edges(G)
        A[a.dst,k]=sqrt.(weights[k])
        A[a.src,k]=-sqrt(weights[k])
        k+=1
    end
    A
end

# ╔═╡ 1437aadc-1c26-4eff-bb22-9cf9c15e1f5f
Iᵧ=IncidenceMatrix(G,wn)

# ╔═╡ 6529324b-49f7-4f01-b5cd-33668979b56b
md"""
## Facts

1.  $L=I_{G}I_{G}^{T}$.

2.  $L$ is symmetric PSD matrix.

3.  $L\mathbf{1}=0$ for $\mathbf{1}=[1,...,1]^{T}$, thus $0$ is an eigenvalue of $L$  and $\mathbf{1}$ is the corresponding eigenvector.

4. If $G$ has $c$ connected components, then $L$ has $c$ eigenvalues equal to $0$.

5. For every $x\in \mathbb{R}^{n}$, it holds
$x^{T}L x=\sum\limits_{i<j}W_{ij}(x_{i}-x_{j})^{2}$.

6. For every $x\in\mathbb{R}^{n}$ and $\alpha,\beta\in\mathbb{R}$, it holds
$(\alpha x+\beta \mathbf{1})^{T} L (\alpha x+\beta \mathbf{1})
=\alpha^{2} x^{T}L x$.

7. Assume that the eigenvalues of $L$ are increasingly ordered. Then,

$$
0=\lambda_1(L)\leq \lambda_2(L)\leq \cdots \leq\lambda_{n}(L)\leq
2\max\limits_{i=1,\cdots ,n}d_{i}.$$

8.  $\sigma(L_n) \subseteq [0,2]$.
"""

# ╔═╡ a9ba64a6-bd17-4b86-acb4-e7c7d8f87dc1
md"""
### Examples
"""

# ╔═╡ 6a6f5db5-7dd4-4a26-ac1a-ef3ca3a5d6be
# Fact 1
norm(L-Iᵧ*Iᵧ')

# ╔═╡ e11bc23d-03e2-4780-9488-93f0adfc2fcb
# Facts 2 and 7
issymmetric(L), eigs(L)[1], 2*maximum(diag(L))

# ╔═╡ 7192f7d0-28b9-11eb-28dd-8db0019504ca
eigen(Matrix(L))

# ╔═╡ e5890dca-368f-437e-bc45-6a395714dae2
# Fact 3
L*ones(n)

# ╔═╡ 61a978a7-b2d2-43e7-b3f7-95bf4c4f54b9
begin
	# Fact 5
	x=rand(n)
	x'*L*x, sum([W[i,j]*(x[i]-x[j])^2 for i=1:n, j=1:n])/2
end

# ╔═╡ da7f85ed-73b0-44b9-a97f-4a8ed5e9a1d1
begin
	# Fact 6
	α,β=rand(),rand()
	(α*x+β*ones(n))'*L*(α*x+β*ones(n)), α^2*x'*L*x
end

# ╔═╡ fed7f592-be03-4baf-8ace-ff9ee78fee71
# Fact 8
eigvals(Matrix(Lₙ))

# ╔═╡ 9f31928e-c933-43bb-9a5c-bf55f29737cd
md"""
# Bipartitioning

## Definitions

Let $\pi=\{V_{1},V_{2}\}$ be a partition of $V$ with $V_1,V_2\neq \emptyset$.

__Cut__ of partition $\pi$ is the sum of weights of all
edges between $V_1$ and $V_2$,

$$\mathop{\mathrm{cut}}(\pi)\equiv \mathop{\mathrm{cut}}(V_1,V_2)=\sum\limits_{{\displaystyle i\in V_{1} \atop \displaystyle j\in V_{2}}}W_{ij}.$$

__Weight__ of vertex $i\in V$ is the sum of the weights of all egdges emanating from $i$,
$\omega(i)=\sum\limits_{j=1}^{n}W_{ij}$.

__Weight__ of a subset $\bar V\subset V$ is the sum of the weights of all vertices in $\bar V$,

$\omega(\bar V)=\sum\limits_{\displaystyle i\in\bar V} \omega(i)$.

__Proportional cut__ of partition $\pi$ is

$$
\mathop{\mathrm{pcut}}(\pi)=\displaystyle\frac{\mathop{\mathrm{cut}}(\pi)}{|V_{1}|}+\frac{\mathop{\mathrm{cut}}(\pi)}{|V_{2}|}.$$

__Normalized cut__ of partition $\pi$ is

$$
\mathop{\mathrm{ncut}}(\pi)=\displaystyle\frac{\mathop{\mathrm{cut}}(\pi)}{\omega(V_{1})}+\frac{\mathop{\mathrm{cut}}(\pi)}{\omega(V_{2})}.$$
"""

# ╔═╡ 1e6d4c72-f25d-4bf5-99d6-ae0446be25dd
md"""
### Example

Consider the following partitions (all edges have unit weights):

 $(load(\"./files/cut2.png\"))

Left partition is $\pi$, right partition is $\pi'$.

|     Cut \ Partition  | $\pi$            |  $\pi'$     |
| ------- | ---------------- | ------------|
| $\mathop{\mathrm{cut}}$  |  $2$        |     $3$        |
| $\mathop{\mathrm{pcut}}$ | $\frac{2}{1}+\frac{2}{11}=2.18$|    $\frac{3}{6}+\frac{3}{6}=1$      |
| $\mathop{\mathrm{ncut}}$ | $\frac{2}{2}+\frac{2}{50}=1.04$ |  $\frac{3}{27}+\frac{3}{25}=0.23$|
"""

# ╔═╡ 02035eed-6d49-4253-a755-68a999e7e90e
md"""
## Facts

1. The informal description of the bipartitioning problem can be formulated as two problems,

$$
\mathop{\textrm{arg min}}\limits_{\pi} \mathop{\mathrm{pcut}}(\pi) \quad \textrm{or} \quad
\mathop{\textrm{arg min}}\limits_{\pi} \mathop{\mathrm{ncut}}(\pi).$$

The first problem favors partitions into subsets with similar numbers of vertices, while the second problem favors partitions into subsets with similar weights.

2. Both problems are NP-hard.

3. __Approximate solutions can be computed by suitable relaxations in $O(n^2)$ operations.__

4. The partition $\pi$ is defined by the vector $y$ such that

$$
y_{i}=
\begin{cases}
\frac{1}{2} & \text{for } i\in V_1 \\
-\frac{1}{2} & \text{for } i\in V_2
\end{cases}$$

The proportional cut problem can be formulated as the  __discrete proportional cut__ problem

$$
\underset{\displaystyle \big|\mathbf{y}^{T}\mathbf{1} \big|\leq \beta}
{\min\limits_{\displaystyle y_{i}\in \{-\frac{1}{2},\frac{1}{2}\}}}
\frac{1}{2}\sum_{i,j}(y_{i}-y_{j})^{2}W_{ij}.$$

Parameter $\beta$ controls the number of vertices in each subset.

5. The normalized cut problem can be formulated as the __discrete normalized cut__ problem

$$
\underset{\displaystyle \big|y^{T}D\mathbf{1} \big|\leq \beta}
{\min\limits_{\displaystyle y_{i}\in \{-\frac{1}{2},\frac{1}{2}\}}}
\frac{1}{2}\sum_{i,j}(y_{i}-y_{j})^{2}W_{ij}.$$

Parameter $\beta$ controls the weights of each subset.

6. Using the Fact 5 above, the discrete proportional cut problem can be formulated as the __relaxed proportional cut__ problem

$$
\underset{\displaystyle y^{T}y=1}{\underset{\displaystyle \big| y^{T}\mathbf{1} \big|
\leq 2\frac{\beta}{\sqrt{n}}}
{\min\limits_{\displaystyle y\in \mathbb{R}^{n}}}} y^{T}L y.$$

Similarly, the discrete normalized cut problem can be formulated as the __relaxed normalized cut__ problem

$$
\underset{\displaystyle y^{T}Dy=1}{\underset{\displaystyle \big| y^{T}D\mathbf{1}\big|
\leq \displaystyle \frac{\beta}{\sqrt{\theta n}}}{\min\limits_{\displaystyle y\in
\mathbb{R}^{n}}}}y^{T}L_n y.$$

7. __The Main Theorem.__ Let $A\in \mathbb{R}^{n\times n}$ be a symmetric matrix with eigenvalues $\lambda _{1}<\lambda _{2}<\lambda_{3}\leq \cdots \leq \lambda _{n}$ and let $v^{[1]},v^{[2]},\ldots,v^{[n]}$ be the corresponding eigenvectors. For the fixed $0\leq \alpha <1$, the solution of the problem

$$
\underset{\displaystyle y^{T}y=1}{\underset{\displaystyle \left|y^{T}v^{[1]}\right|\leq \alpha}
{\min\limits_{\displaystyle y\in \mathbb{R}^{n}}}} y^{T}Ay$$

is $y=\pm \alpha v^{[1]}\pm \sqrt{1-\alpha^{2}}v^{[2]}$.

For the proof see [D. J. Higham and M. Kibble, A Unified View of Spectral Clustering, Theorem 3.1, p. 7](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.115.1591&rep=rep1&type=pdf).

8. For $0\leq \beta <\frac{n}{2}$, the solution of the relaxed proportional cut problem is

$$
y=\pm \frac{2\beta}{n\sqrt{n}}\mathbf{1}\pm
\sqrt{1-4\frac{\beta ^{2}}{n^{2}}}v^{[2]},$$

where $v^{[2]}$ is an eigenvector corresponding to $\lambda_2(L)$. $v^{[2]}$ the __Fiedler vector__. Since the first summand carries no information, $V$ is partitioned according to the signs of the components of $v^{[2]}$:

$$
V_{1}=\{i:v^{[2]}_i <0\}, \quad V_{2}=\{i:v^{[2]}_i \geq 0\}.$$

_Notice that the value of $\beta$ is irrelevant for the solution._

9. For $0\leq \beta <\sqrt{\theta n}\left\Vert D^{\frac{1}{2}}\mathbf{1} \right\Vert _{2},$ the solution of the relaxed normalized cut problem is

$$
y=\pm \frac{\beta }{\sqrt{\theta n}\left\Vert
D^{\frac{1}{2}} \mathbf{1}\right\Vert _{2}^{2}}\mathbf{1}\pm
\sqrt{1-\frac{\beta ^{2}}{ \theta n\left\Vert
D^{\frac{1}{2}}\mathbf{1}\right\Vert _{2}^{2}}}D^{-\frac{1
}{2}} v_n^{[2]},$$

where $v_n^{[2]}$ is an eigenvector corresponding to $\lambda_2(L_n)$. $V_n$ is partitioned according to the signs of the components of $v_n^{[2]}$, as above.

10. Neither of the relaxed algorithms is guaranteed to solve exactly the true proportional / normalized cut problem. However, the computed solutions are in the right direction. Whether to use proportional or normalized cut formulation, depends upon the specific problem.
"""

# ╔═╡ c188b7cf-03e7-40b3-bf90-773978c7b410
# Voila!
eigs(L,nev=2,which=:SM, v0=ones(n))

# ╔═╡ f60b51b6-79c3-4a3f-8bfb-8f198cdb92a8
# For the normalized cut
eigs(Lₙ,nev=2,which=:SM, v0=ones(n))

# ╔═╡ bfac7a55-0d82-4549-b999-7db3272eaa20
md"""
### Concentric rings

A __complete graph__ has edges connecting each pair of vertices.

To a set of points $X=\{x_{1},x_{2},\cdots ,x_{m}\}$ , where $x_{i}\in\mathbb{R}^{n}$, we assign a weighted complete graph $G=(V,E)$ with $m$ vertices, where the vertex $j\in V$ corresponds to the point $x_j\in X$.

The main idea is to assign weight of an edge $e=(i,j)$ which reflects the distance between $x_i$ and $x_j$, something like $\omega(e)=\displaystyle\frac{1}{\mathop{\mathrm{dist}}(x_i,x_j)}$.

However, this has to be implemented with care. For example, using simple Euclidean distance yield the same results as the function `kmeans()`. In this example we use Gaussian kernel, that is

$$
\omega(e)=e^{\displaystyle -\|x_i-x_j\|_2^2/\sigma^2},$$

where the choice of $\sigma$ is based on experience.

The computation of various distances is implemented in the package [Distances.jl](https://github.com/JuliaStats/Distances.jl).

We will construct the Laplace matrix directly.
"""

# ╔═╡ 80ed91ad-6f9a-4f12-a1ee-bc8dc6b58168
begin
	# Two concentric circles
	k=2
	# Center
	Random.seed!(541)
	# center=[rand(-5:5),rand(-5:5)]
	center=[0,0]
	# Radii
	radii=randperm(10)[1:k]
	# Number of points in circles
	sizes=rand(1000:2000,k)
	center,radii,sizes
end

# ╔═╡ 2d8689c9-1b9e-4360-865c-23c2c8c5dcb9
begin
	# Generate points
	X=Array{Float64}(undef,2,sum(sizes))
	csizes=cumsum(sizes)
	# Random angles
	ϕ=2*π*rand(sum(sizes))
	for i=1:csizes[1]
		X[:,i]=center+radii[1]*[cos(ϕ[i]);sin(ϕ[i])] + (rand(2).-0.5)/50
	end
	for j=2:k
		for i=csizes[j-1]+1:csizes[j]
			X[:,i]=center+radii[j]*[cos(ϕ[i]);sin(ϕ[i])] + (rand(2).-0.5)/50
		end
	end
	scatter(X[1,:],X[2,:],title="Concentric rings", aspect_ratio=1,label="Points")
end

# ╔═╡ fa4ae713-a0b2-46ce-b5f7-558b26d82019
# Weight matrix
W₁=1 ./pairwise(SqEuclidean(),X)

# ╔═╡ 760e8ca2-596f-435f-befb-140b3d204f7f
begin
	# Laplacian matrix
	m=csizes[end]
	for i=1:m
	    W₁[i,i]=0
	end
	L₁=Diagonal(vec(sum(W₁,dims=2)))-W₁
	# Check Fact 3
	norm(L₁*ones(m))
end

# ╔═╡ 75956488-76ab-4262-a32f-0c53e1b3dc17
# Notice λ₁=0
E=eigs(L₁,nev=2,which=:SM, v0=ones(m))

# ╔═╡ a55260dd-f05d-4a65-9fdf-cb3fd4638857
begin
	# Define clusters
	C=ones(Int64,m)
	C[findall(E[2][:,2].>0)].=2
	C
end

# ╔═╡ ba164ff3-5ef4-4480-8af2-9dd9a99b1e8d
# Yet another plotting function
function plotKpartresult(C::Vector,X::Array)
	scatter(aspect_ratio=1)
    k=maximum(C)
    for j=1:k
        scatter!(X[1,findall(C.==j)],X[2,findall(C.==j)],label="Cluster $j")
    end
	scatter!(aspect_ratio=1)
end

# ╔═╡ 46de28bc-73ed-495c-a2e0-f565c1ce5651
plotKpartresult(C,X)

# ╔═╡ e004e4bf-ca04-4b41-947b-9f1ecd058abb
md"""
This is the same partitioning as obtained by `kmeans()`. Let us try Gaussian kernel. A rule of thumb is: if rings are close, use $\sigma<1$, if rings are apart, use $\sigma>1$.
"""

# ╔═╡ bb9d4d98-3aff-439e-b8a3-8347cf345839
begin
	σ=0.7 # 0.1
	W₂=exp.(-pairwise(SqEuclidean(),X)/σ^2)-I
	L₂=Diagonal(vec(sum(W₂,dims=2)))-W₂
	E₂=eigs(L₂,nev=2,which=:SM, v0=ones(m))
	C₂=ones(Int64,m)
	C₂[findall(E₂[2][:,2].>0)].=2
	plotKpartresult(C₂,X)
end

# ╔═╡ 012fa54a-5171-4980-a0ba-474a22c25981
md"""
# Recursive bipartitioning

## Definitions

Let $G=(V,E)$ be a weighted graph with weights $\omega$.

Let $\pi_k =\{V_{1},V_{2},...,V_{k}\}$ be a $k$-partition of $V$, with $V_i\neq \emptyset$ for $i=1,\ldots,k$.

The previous definition of $cut(\pi)\equiv cut(\pi_2)$ extends naturally to $k$-partition.

A __cut__ of a partition $\pi_k$ is

$$
\mathop{\mathrm{cut}}(\pi_k)=\sum\limits_{\displaystyle i<j} \mathop{\mathrm{cut}}(V_{i},V_{j}),$$

where $\mathop{\mathrm{cut}}(V_{i},V_{j})$ is interpreted as a cut of the bipartition of the subgraph of $G$ with vertices $V_1\cup V_2$.

__Proportional cut__ of a partition $\pi_k$ is

$$
\mathop{\mathrm{pcut}}(\pi_k)=\underset{i<j}{\sum\limits_{i,j=1}^{k}} \left(
\frac{\mathop{\mathrm{cut}}(V_{i},V_{j})}{|V_{i}|}+\frac{\mathop{\mathrm{cut}}(V_{i},V_{j})}{|V_{j}|}\right) =
\sum_{i=1}^{k}\frac{\mathop{\mathrm{cut}}(V_{i},V\backslash V_{i})}{|V_{i}|}.$$

__Normalized cut__ of a partition $\pi_k$ is

$$
\mathop{\mathrm{ncut}}(\pi_k)=\underset{i<j}{\sum\limits_{i,j=1}^{k}} \left(
\frac{\mathop{\mathrm{cut}}(V_{i},V_{j})}{\omega(V_{i})}+\frac{\mathop{\mathrm{cut}}(V_{i},V_{j})}{\omega(V_{j})}\right) =
\sum_{i=1}^{k}\frac{\mathop{\mathrm{cut}}(V_{i},V\backslash V_{i})}{ \omega(V_{i})}.$$

## Facts

If we want to cluster vertices of graph $G=(V,E)$ into $k$ clusters, we can apply the following recursive algorithm:

1. __Initialization.__ Compute the bipartition $\pi=\{V_{1},V_{2}\}$ of $V$. Set the counter $c=2$.

2. __Recursion.__ While $c<k$ repeat:

    1. Compute the bipartition of each subset of $V$.

    2. Among all $(c+1)$-partitions, choose the one with the smallest $\mathop{\mathrm{pcut}}(\pi_{c+1})$ or $\mathop{\mathrm{ncut}}(\pi_{c+1})$, respectively.

    3. Set $c=c+1$.

3. __Stop.__

There is no guarantee for optimality of this algorithm. Clearly, the optimal $k$-partiton may be a subpartition of one of the discarded partitions.
"""

# ╔═╡ f3eb8ad4-55a8-4e76-a581-07f9a7cc75f7

# ╔═╡ Cell order:
# ╟─7ef4b0b3-9d0a-4356-bd16-14487f14e0ab
# ╟─cb61f761-b2ee-4818-9149-f85d95b76a1e
# ╟─5926c79a-6038-4dbc-b49a-5b05d02250a0
# ╟─a66a6e80-1f80-11eb-15f8-71ffaa5050e7
# ╟─d4cf12ce-12fb-4562-bbea-e9b85cba94c2
# ╠═0cd92d78-1da6-435b-ad8e-60af29502e1d
# ╠═87cca1d3-b944-4d74-bf4c-1562f34cf91a
# ╠═3d7b4a01-9a7d-4111-a45d-1d31f1ef6278
# ╠═3b5ed1fa-75b0-43fb-a918-4c2e8f52d039
# ╠═8f7d1f6c-ee4c-407f-80d9-d2abc2af949e
# ╠═b7f127d8-e8cd-4771-8091-c88f9ffe6b08
# ╠═3f5447bb-95cd-4520-b0b9-e85f33b323bd
# ╠═ecb2a46e-f9dc-4e57-812f-0a2788afb202
# ╠═86e5f00f-13ee-4867-b0cd-187f2c782a46
# ╠═cf26dbe9-3c4f-4c95-88ef-f197f360d759
# ╠═8fcd2440-4013-4b37-a747-2c0c7f6d4aeb
# ╠═0c8d3572-3436-446f-952a-b19d0370ab38
# ╠═1437aadc-1c26-4eff-bb22-9cf9c15e1f5f
# ╟─6529324b-49f7-4f01-b5cd-33668979b56b
# ╟─a9ba64a6-bd17-4b86-acb4-e7c7d8f87dc1
# ╠═6a6f5db5-7dd4-4a26-ac1a-ef3ca3a5d6be
# ╠═e11bc23d-03e2-4780-9488-93f0adfc2fcb
# ╠═7192f7d0-28b9-11eb-28dd-8db0019504ca
# ╠═e5890dca-368f-437e-bc45-6a395714dae2
# ╠═61a978a7-b2d2-43e7-b3f7-95bf4c4f54b9
# ╠═da7f85ed-73b0-44b9-a97f-4a8ed5e9a1d1
# ╠═fed7f592-be03-4baf-8ace-ff9ee78fee71
# ╟─9f31928e-c933-43bb-9a5c-bf55f29737cd
# ╟─1e6d4c72-f25d-4bf5-99d6-ae0446be25dd
# ╟─02035eed-6d49-4253-a755-68a999e7e90e
# ╠═c188b7cf-03e7-40b3-bf90-773978c7b410
# ╠═f60b51b6-79c3-4a3f-8bfb-8f198cdb92a8
# ╟─bfac7a55-0d82-4549-b999-7db3272eaa20
# ╠═80ed91ad-6f9a-4f12-a1ee-bc8dc6b58168
# ╠═2d8689c9-1b9e-4360-865c-23c2c8c5dcb9
# ╠═fa4ae713-a0b2-46ce-b5f7-558b26d82019
# ╠═760e8ca2-596f-435f-befb-140b3d204f7f
# ╠═75956488-76ab-4262-a32f-0c53e1b3dc17
# ╠═a55260dd-f05d-4a65-9fdf-cb3fd4638857
# ╠═ba164ff3-5ef4-4480-8af2-9dd9a99b1e8d
# ╠═46de28bc-73ed-495c-a2e0-f565c1ce5651
# ╟─e004e4bf-ca04-4b41-947b-9f1ecd058abb
# ╠═bb9d4d98-3aff-439e-b8a3-8347cf345839
# ╟─012fa54a-5171-4980-a0ba-474a22c25981
# ╠═f3eb8ad4-55a8-4e76-a581-07f9a7cc75f7

src/Module C - Applications/L10 Spectral Graph Bipartitioning.jl


# News Boy Problem solved by L-Shaped Benders Decomposition
# Author: Edward J. Xu, edxu96@outlook.com
# Date: April 5th, 2019
# 0. Set working path, install packages and use module -----------------------------------------------------------------
push!(LOAD_PATH, "$(homedir())/Desktop/News Boy Problem, Stochastic Programming")
cd("$(homedir())/Desktop/News Boy Problem, Stochastic Programming")
using BendersLshapedStochasMilp_EDXU
using JuMP
using GLPKMathProgInterface
# 1. Date Input --------------------------------------------------------------------------------------------------------
demand = [12, 14, 16, 18, 20, 22, 24, 26, 28, 30] # Demand of newspapers in each scenario
lengthS = length(demand)
vec_prob = [0.05, 0.10, 0.10, 0.10, 0.15, 0.15, 0.10, 0.10, 0.10, 0.05 ] # probability of scenario
c = 20 # purchase price
p = 70 # selling price
h = 10 # scrap value
y = 20
# 2. Data Transfer -----------------------------------------------------------------------------------------------------
n_x = 10  # Number of all the x
n_y = 1
vec_min_y = hcat([0])
vec_max_y = hcat([30])
vec_f = hcat(c - 10)
vec_pi = hcat(vec_prob)
mat_c = zeros(10, 1, 1)
for i = 1:10
    mat_c[i, :, :] = hcat([- 70 + 10])
end
mat_h = zeros(10, 2, 1)
for i = 1: 10
    mat_h[i, 1, :] = hcat(- demand[i])
    mat_h[i, 2, :] = hcat(0)
end
mat3_t = zeros(10, 2, 1)
for i = 1:10
    mat3_t[i, 1, :] = hcat(0)
    mat3_t[i, 2, :] = hcat(1)
end
mat3_w = - ones(10, 2, 1)
epsilon = 0.001
timesIterationMax = 500
# 3. Begin Optimization ------------------------------------------------------------------------------------------------
BendersLshaped(; n_x = n_x, n_y = n_y, vec_min_y = vec_min_y, vec_max_y = vec_max_y, vec_f = vec_f,
    vec_pi = vec_pi, mat_c = mat_c, mat_h = mat_h, mat3_t = mat3_t, mat3_w = mat3_w,
    epsilon = epsilon, timesIterationMax = timesIterationMax)
# answer: obj = - 976