### A Pluto.jl notebook ###
# v0.12.16

using Markdown
using InteractiveUtils

# ╔═╡ e077ab16-22e5-11eb-15ed-5f0654bf7328
using Pkg, DrWatson

# ╔═╡ e75a9a06-22e5-11eb-2e36-8d4f62b830ed
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using Statistics
	using Random, Distributions, Plots
	Random.seed!(0)
end;

# ╔═╡ ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
md"## Listing6.15"

# ╔═╡ 1aa41898-3afa-11eb-2f49-570853ce6435
begin
	mu, sig = 50, 5
	dist = Normal(mu, sig)
	alpha = 0.01
	nMax = 40

	observations = rand(dist,1)
	piLarray, piUarray = [], []
end;

# ╔═╡ 0e5aed16-3b2f-11eb-2f7c-01f450cc5336
for _ in 2:nMax
    xNew = rand(dist)
    push!(observations,xNew)

    xbar, sd = mean(observations), std(observations)
    n = length(observations)
    tVal = quantile(TDist(n-1),1-alpha/2)
    delta = tVal * sd * sqrt(1+1/n)
    piL, piU = xbar - delta, xbar + delta

    push!(piLarray,piL); push!(piUarray,piU)
end

# ╔═╡ 0e5b22c2-3b2f-11eb-2895-7d6e7267e6b6
begin
	scatter(1:nMax, observations,
		c=:blue, msw=0, label="Observations")
	plot!(2:nMax, piUarray,
		c=:red, shape=:xcross, msw=0, label="Prediction Interval")
	plot!(2:nMax, piLarray,
		c=:red, shape=:xcross, msw=0, label="",
		ylims=(0,100), xlabel="Number of observations", ylabel="Value")
end

# ╔═╡ 475ff888-22e6-11eb-2354-09f8f40a8e12
md"## End of listing6.15"

# ╔═╡ Cell order:
# ╟─ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
# ╠═e077ab16-22e5-11eb-15ed-5f0654bf7328
# ╠═e75a9a06-22e5-11eb-2e36-8d4f62b830ed
# ╠═1aa41898-3afa-11eb-2f49-570853ce6435
# ╠═0e5aed16-3b2f-11eb-2f7c-01f450cc5336
# ╠═0e5b22c2-3b2f-11eb-2895-7d6e7267e6b6
# ╟─475ff888-22e6-11eb-2354-09f8f40a8e12


#Week 6
#A
eq1=@equ δ=4m*M/(m+M)^2 #maximum fraction of energy transferable
eq2=@equ Te-Tgas=2qe*E^2/(3δ*me*vm^2) #∝p^-2, steady state

#B
b1=@equ d*Ṅ=Ṅ*α*d*x
b2=@equ Ṅ=Ṅ0*exp(α*x)
b3=@equ i=Ṅ0*e*exp(α*d) # =i0*exp(α*d) #current leaving the anode #should e be qe?
b4=@equ i=i0*exp(α*d)/(1-γ*(exp(α*d)-1)) #sum of avalances
b5=@equ γ*exp(α*d)=γ+1 #1-γ*(exp(α*d)-1)=0 #breakdown criterion

#C
c1=@equ α=A*p*exp(-B*p/E)
c1b=@equ α=1/λ*exp(-ϵi/(e*E*λ))
c1c=@equ α=A*p*exp(-B*p*d/V)
c2=@equ E=V/d #for parallel plates
c3=@equ VB=B*p*d/(log(A*p*d)-log(log(1+1/γ))) #breakdown voltage
c3b=@equ VB=B*p*d/(log(C*p*d))
c3c=@equ C=A/log(1+1/γ)
c4=@equ Der(nj,t)+Dot(∇,Γj)=Sj
c5=@equ Sj=ne*α*ue
c6=@equ E=-∇V
c7=@equ ∇^2*V=0
#boundary conditions
c8=@equ n*Γi_anode=0 #ions
c9=@equ n*Γe_cathode=-γse*n*Γi_cathode

#D
d1=@equ Γi=ni*c̄i/4 #ion thermal flux
d1b=@equ c̄i=sqrt(8*e*Ti/(pi*mi))
d2=@equ Γe=ne*c̄e/4 #electron thermal flux
d2b=@equ c̄e=sqrt(8*e*Te/(pi*me))

#E
e1=@equ 1/2*M*u^2+e*V=1/2*M*us^2
e2=@equ ni*u=ns*us
e3=@equ ni=ns*(1-2*e*V/(M*us^2))^(-0.5) #ion density in sheath
e4=@equ ne=ns*exp(V/Te) #electron density in sheath
e5=@equ Der(Der(V,x),x)=-(ni-ne)*e/ϵ0 #Poisson's euqation in sheath
e6=@equ ub=sqrt(e*Te/M)#≤us #Bohm's criterion
e7=@equ ns*uB=0.61n0*sqrt(e*Te/M) #ion flux to a wall
e8=@equ e*ns*ub*A=0.61*e*A*n0*sqrt(e*Te/M) #ion current to a probe area A
e9=@equ Vs=Te/2*log(M/(2*pi*m))#~4.7Te for argon #sheath voltage drop
e10=@equ ϵi=e*Te/2*log(M/2.3m)#~5.2e*Te #Ion energy to a wall

#Week 7
#B
b1=@equ ΔEf=ΔEα+ΔEn
b2=@equ RDT*ΔEf=nD*nT*σDT*v*ΔEf	#fusion power density, depends on v
b3=@equ nD*nT*σv*ΔEf=1/4*n^2*σv*ΔEf #whole plasma, avg σv
b4=@equ Pb/volume=A*n^2*Zeff*sqrt(Te) #Bremsstrahlung
b4b=@equ A=5e-37 #Wm^-3
b4c=@equ Zeff=nj*Zj^2/n #sum over j
b5=@equ Pdl/volume=3n*T/τE #losses
b6=@equ Q=Pf/Pin
b7=@equ fα=Pα/(Pα+Pin)
b8=@equ QE=(Pout_E-Pin_E)/Pin_E #conversion efficiency
b8b=@equ Pin_E=Pin/ηe
b8c=@equ Pout_E=ηt*(Pf+Pin)

#C
c1=@equ El=1/((ϵl*ϵth)^3*ϵc^4)*(n0/n)^2	#driver energy

#D
d1=@equ thickness=1/(number_density*cross_section)	#required wall thickness
d2=@equ I=I0*exp(-x/λtot)

#E
e1=@equ Γr=D*Der(n,r)


################################################################################
## Linear Regression
##   y ~ N(b0 + b1 * x, s2)
##   b0, b1 ~ N(0, 1000)
##   s2 ~ invgamma(0.001, 0.001)
################################################################################

using Mamba

## Data
data = Dict(
  :x => [1, 2, 3, 4, 5],
  :y => [1, 3, 3, 3, 5]
)

## Log-transformed Posterior(b0, b1, log(s2)) + Constant
logf = function(x::DenseVector)
   b0 = x[1]
   b1 = x[2]
   logs2 = x[3]
   r = data[:y] - b0 - b1 * data[:x]
   (-0.5 * length(data[:y]) - 0.001) * logs2 -
     (0.5 * dot(r, r) + 0.001) / exp(logs2) -
     0.5 * b0^2 / 1000 - 0.5 * b1^2 / 1000
end

## MCMC Simulation with Slice Sampling
## With multivariate (1) and univariate (2) updating
n = 5000
sim1 = Chains(n, 3, names = ["b0", "b1", "s2"])
sim2 = Chains(n, 3, names = ["b0", "b1", "s2"])
width = [1.0, 1.0, 2.0]
theta1 = SliceUnivariate([0.0, 0.0, 0.0], width, logf)
theta2 = SliceMultivariate([0.0, 0.0, 0.0], width, logf)
for i in 1:n
  sample!(theta1)
  sample!(theta2)
  sim1[i, :, 1] = [theta1[1:2]; exp(theta1[3])]
  sim2[i, :, 1] = [theta2[1:2]; exp(theta2[3])]
end
describe(sim1)
describe(sim2)


# Hessian Eigenmaps (HLLE)
# ---------------------------
# Hessian eigenmaps: Locally linear embedding techniques for high-dimensional data,
# D. Donoho and C. Grimes, Proc Natl Acad Sci U S A. 2003 May 13; 100(10): 5591–5596

import Combinatorics: combinations

"""
    HLLE{NN <: AbstractNearestNeighbors, T <: Real} <: AbstractDimensionalityReduction

The `HLLE` type represents a Hessian eigenmaps model constructed for `T` type data with a help of the `NN` nearest neighbor algorithm.
"""
struct HLLE{NN <: AbstractNearestNeighbors, T <: Real} <: AbstractDimensionalityReduction
    λ::AbstractVector{T}
    proj::Projection{T}
    nearestneighbors::NN
    component::AbstractVector{Int}
end

## properties
outdim(R::HLLE) = size(R.proj, 1)
eigvals(R::HLLE) = R.λ
neighbors(R::HLLE) = R.nearestneighbors.k
vertices(R::HLLE) = R.component

## show
summary(io::IO, R::HLLE) = print(io, "Hessian Eigenmaps(outdim = $(outdim(R)), neighbors = $(neighbors(R)))")

## interface functions
"""
    fit(HLLE, data; k=12, maxoutdim=2, nntype=BruteForce)

Fit a Hessian eigenmaps model to `data`.

# Arguments
* `data`: a matrix of observations. Each column of `data` is an observation.

# Keyword arguments
* `k`: a number of nearest neighbors for construction of local subspace representation
* `maxoutdim`: a dimension of the reduced space.
* `nntype`: a nearest neighbor construction class (derived from `AbstractNearestNeighbors`)

# Examples
```julia
M = fit(HLLE, rand(3,100)) # construct Hessian eigenmaps model
R = transform(M)          # perform dimensionality reduction
```
"""
function fit(::Type{HLLE}, X::AbstractMatrix{T};
             k::Int=12, maxoutdim::Int=2, nntype=BruteForce) where {T<:Real}
    # Construct NN graph
    NN = fit(nntype, X, k)
    D, E = knn(NN, X)
    G, C = largest_component(SimpleWeightedGraph(adjmat(D,E)))
    XX = @view X[:, C]
    n = length(C)

    # Obtain tangent coordinates and develop Hessian estimator
    hs = (maxoutdim*(maxoutdim+1)) >> 1
    W = spzeros(T, hs*n, n)
    for i=1:n
        II = @view E[:,C[i]]
        # re-center points in neighborhood
        VX = view(XX, :, II)
        μ = mean(VX, dims=2)
        N = VX .- μ
        # calculate tangent coordinates
        tc = svd(N).V[:,1:maxoutdim]

        # Develop Hessian estimator
        Yi = [ones(T, k) tc zeros(T, k, hs)]
        for ii=1:maxoutdim
            Yi[:,maxoutdim+ii+1] = tc[:,ii].^2
        end
        yi = 2*(1+maxoutdim)
        for (ii,jj) in combinations(1:maxoutdim, 2)
            Yi[:, yi] = tc[:, ii] .* tc[:, jj]
            yi += 1
        end
        F = qr(Yi)
        H = transpose(F.Q[:,(end-(hs-1)):end])
        W[(1:hs).+(i-1)*hs, II] = H
    end

    # decomposition
    λ, V = decompose(transpose(W)*W, maxoutdim)
    return HLLE{nntype, T}(λ, transpose(V) .* convert(T, sqrt(n)), NN, C)
end

"""
    transform(R::LLE)

Transforms the data fitted to the Hessian eigenmaps model `R` into a reduced space representation.
"""
transform(R::HLLE) = R.proj


"""
    dirVector{FT}
Struct for spherical coordinate directions in θ (elevation angle) and ϕ (azimuth angle)
# Fields
$(DocStringExtensions.FIELDS)
"""
struct dirVector{FT}
    θ::FT
    ϕ::FT
end

"""
    dirVector_μ{FT}
Struct for spherical coordinate directions in θ (elevation angle) and ϕ (azimuth angle)"
# Fields
$(DocStringExtensions.FIELDS)
"""
struct dirVector_μ{FT}
    μ::FT
    ϕ::FT
end

# Define dot product for the directional vector in spherical coordinates:
LinearAlgebra.dot(Ω₁::dirVector, Ω₂::dirVector) = cos(Ω₁.θ) * cos(Ω₂.θ) + sin(Ω₁.θ) * sin(Ω₂.θ) * cos(Ω₂.ϕ - Ω₁.ϕ)

# Define dot product for the directional vector in spherical coordinates:
LinearAlgebra.dot(Ω₁::dirVector_μ, Ω₂::dirVector_μ) = Ω₁.μ * Ω₂.μ + sqrt(1-Ω₁.μ^2)  * sqrt(1-Ω₂.μ^2) * cos(Ω₂.ϕ - Ω₁.ϕ)


#
#   Kepler.jl
#
#   Solves Kepler's problem for TwoBody orbits.
#

"""
Solves Kepler's Problem for `orbit` and `Δtᵢ`.
"""
function kepler(r::AbstractVector, v::AbstractVector, μ::Number, Δt::Number = period(semimajor_axis(r,v,μ), μ); tol=1e-6, max_iter=100)

    e, a, i, Ω, ω, ν = keplerian(r, v, μ)
    T = period(a, μ)
    conic_section = conic(e)

    # Guess χ₀
    if conic_section == Hyperbolic
        χ₀ = sign(Δt) * √(-a) * log(ℯ, (-2 * μ / a * Δt) / (r ⋅ v + (sign(Δt) * √(-μ * a) * (1 - norm(r) / a))))
    elseif conic_section == Parabolic
        χ₀ = √(a) * tan(ν / 2)
    else
        Δt = mod(Δt, T)
        χ₀ = √(μ) * Δt / a
    end

    # Iteratively solve for χ
    # TODO: Compare loop vs. recursion performance here.
    # There shouldn't be too large of a difference, since this tends
    # to converge with only a few iterations.
    χₙ, rₙ, ψ, C₂, C₃ = χₖ(χ₀, Δt, r, v, a, μ, tol=tol, max_iter=max_iter)

    # Convert to a Orbit
    f = 1 - χₙ^2 / norm(r) * C₂
    ḟ = √(μ) / (norm(r) * rₙ) * χₙ * (ψ * C₃ - 1)
    g = Δt - (χₙ^3 / √(μ)) * C₃
    ġ = 1 - (χₙ^2 / rₙ) * C₂

    return ((f * r) .+ (g * v), (ḟ * r) .+ (ġ * v))
end

function χₖ(χₙ, Δt, rᵢ₀, vᵢ₀, a, μ; iter=1, tol=1e-14, max_iter=100)

    r₀ = norm(rᵢ₀)
    ψ = upreferred(χₙ^2 / a)

    if ψ > tol
        C₂ = (1 - cos(√(ψ))) /  ψ
        C₃ = (√(ψ) - sin(√(ψ))) / √(ψ^3)
    elseif ψ < -tol
        C₂ = (1 - cosh(√(-ψ))) / ψ
        C₃ = (sinh(√(-ψ)) - √(-ψ)) / √((-ψ)^3)
    else
        C₂ = 1.0 / 2.0
        C₃ = 1.0 / 6.0
    end

    r = χₙ^2 * C₂ + (rᵢ₀ ⋅ vᵢ₀) * χₙ / √(μ) * (1 - ψ*C₃) + r₀ * (1 - ψ * C₂)
    χₙ₊₁ = χₙ + ((√(μ) * Δt - χₙ^3 * C₃ - (rᵢ₀ ⋅ vᵢ₀) / √(μ) * χₙ^2 * C₂ - r₀ * χₙ * (1 - ψ * C₃)) / r)

    if iter > max_iter
        @error "Failed to converge!"
        return χₙ, r, ψ, C₂, C₃
    elseif abs(χₙ₊₁ - χₙ) < oneunit(χₙ) * tol
        return  χₙ, r, ψ, C₂, C₃
    else
        return χₖ(χₙ₊₁, Δt, rᵢ₀, vᵢ₀, a, μ; iter=iter+1, tol=tol, max_iter=max_iter)
    end

end


#=
Regularisation of network.
Contains L1 and L2 regularisation as well as sparsity constraint for sparse
encoder.
=#

################################################################################
#
# L1
#
################################################################################

########################################
# L1 Cost                                ∈ ℝ
########################################

# For each layer weight matrix
# Jℓ₁ = λ ∑||Wᵢⱼ||    ∀ Wᵢⱼ
function ℓ1_reg_cost(net)
    Jᵣ = zero(net.dtype)
    for ℓ in net.layers[2:end]

        # No parameters in pooling layers
        typeof(ℓ) <: PoolLayer && continue

        # ∑||Wᵢⱼ||
        Jᵣ += sumabs(ℓ.W)
    end

    # λ ∑||Wᵢⱼ||
    Jᵣ *= net.λ
end

########################################
# L1 Gradient                            ∈ ℝ^ num_feats
########################################

#=
Note that an option to make the L1 gradient numerically stable is to make the
function √(x²+ϵ). This would make the function smooth around x=0.
=#
# ∂Jℓ₂/∂Wᵢⱼ = λ sign(Wᵢⱼ)    ∀ Wᵢⱼ
# Added to the gradient in-place
function ℓ1_reg_grad!(net)
    for ℓ in net.layers[2:end]

        # No parameters in pooling layers
        typeof(ℓ) <: PoolLayer && continue

        # ∇W := λ sign(W) + ∇W
        BLAS.axpy!(net.λ, sign(ℓ.W), ℓ.∇W)
    end
end

################################################################################
#
# L2 (Weight Decay)
#
################################################################################

########################################
# L2 Cost                                ∈ ℝ
########################################

# Jℓ₂ = (λ/2) ∑||Wᵢⱼ||²    ∀ Wᵢⱼ
function ℓ2_reg_cost(net)
    Jᵣ = zero(net.dtype)
    for ℓ in net.layers[2:end]

        # No parameters in pooling layers
        typeof(ℓ) <: PoolLayer && continue

        # ∑||Wᵢⱼ||²
        Jᵣ += sumabs2(ℓ.W)
    end

    # (λ/2) ∑||Wᵢⱼ||²
    Jᵣ *= net.λ * net.dtype(0.5)
end

########################################
# L2 Gradient                            ∈ ℝ^ num_feats
########################################

# ∂Jℓ₂/∂Wᵢⱼ = λ Wᵢⱼ    ∀ Wᵢⱼ
# Added to the gradient in-place
function ℓ2_reg_grad!(net)
    for ℓ in net.layers[2:end]

        # No parameters in pooling layers
        typeof(ℓ) <: PoolLayer && continue

        # ∇W := λ W + ∇W
        BLAS.axpy!(net.λ, ℓ.W, ℓ.∇W)
    end
end

################################################################################
#
# L1 and L2 Dictionaries (for user selections through api)
#
################################################################################

reg_cost_dict = Dict{Symbol, Function}(:L1 => ℓ1_reg_cost, :L2 => ℓ2_reg_cost)
reg_grad_dict = Dict{Symbol, Function}(:L1 => ℓ1_reg_grad!, :L2 => ℓ2_reg_grad!)

################################################################################
#
# Sparsity (KL Divergence of Bernoulli means)
#
################################################################################

########################################
# KL Divergence                          ∈ ℝ
########################################

# Sum of KL Divergences between a Bernoulli random variable with mean p and a
# vector of Bernoulli random variables with mean p̂
function kl_div_sum{F<:AbstractFloat}(p̂::Vector{F}, p::F)
    s = F(0)
    for j in eachindex(p̂)
        @inbounds s += (p * log(p / p̂[j]) +
                        (F(1) - p) * log((F(1) - p) / (F(1) - p̂[j])))
    end
    return s
end

# Sparsity cost function for the net
function sparse_cost(net)
    J = zero(net.dtype)
    for ℓ in net.layers[2:end-1]

        # Only for sparse encoder layers
        !(typeof(ℓ) <: SparseEncoderLayer) && continue

        # β ∑ˢ KL(p || p̂)
        J += ℓ.β * kl_div_sum(ℓ.p̂, ℓ.p)
    end
    return J
end

########################################
# Sparsity Gradient                      ∈ ℝ^ num_feats
########################################

function delta_sparse!{F<:AbstractFloat}(Δ::Array{F}, spgrad::Vector{F},
                                         p̂::Vector{F}, p::F, β::F, m::F)
    for j in eachindex(p̂)
        @inbounds spgrad[j] = (- p / p̂[j] + (F(1) - p) / (F(1) - p̂[j])) * β / m
    end
    broadcast!(+, Δ, spgrad, Δ)
end


function corrp_test()
	x = [1.,2,3,5,10,22,0,1,-3,4,0,8,10,3,4,5,5,5,-10,0];
	y = [2.,3,4,5,7,10,10,11,-3,4,1,-1,0,1,2,3,4,4,4,-5];
	r1,p1 = corrp(x,y);
	# compare to Matlab results ([r,p] = corr(x',y'))
	@test r1 ≈ 0.2923 atol = .0001
	@test p1 ≈ 0.2111 atol = .0001

	x[2] = NaN;
	y[[2,4]] .= NaN;
	r2,p2 = corrp(x,y);
	@test r2 ≈ 0.2892 atol = .0001
	@test p2 ≈ 0.2444 atol = .0001
end

corrp_test();


import PolynomialApproximations: Remez
using SpecialFunctions: erfcx
using PyPlot

T = Float64

function g(y::T) where T <: AbstractFloat
    if -one(T) ≤ y ≤ -one(T) + eps(T)
        return one(T)
    elseif one(T) - eps(T) ≤ y ≤ one(T)
        return zero(T)
    else
        x = (1+y) / (1-y)
        return erfcx(x)
    end
end

function erfcx_approx(x)
    if x < one(T)
        y = (x-1) / (x+1)
    else
        rx = 1 / x
        y = (1-rx) / (1+rx)
    end
    return g_approx(y)
end

n = 15
iterations = 6
clamp = :both
ddp, pt, zmax, zmin = Remez.minimax(T, g, n, iterations, clamp)

g_approx(y) = Remez.Newton_poly(ddp, pt, y)

figure(1)
y = range(-one(T), one(T), length=201)

plot(y, g.(y))
grid(true)
xlabel(L"$y$")
ylabel(L"$g(y)$")

figure(2)
plot(y, g.(y)-g_approx.(y))
grid(true)
xlabel(L"$y$")
ylabel(L"error in $g$")

figure(3)
x = Float64[ 10 * (n/400)^2 for n = 0:400 ]
plot(x, erfcx_approx.(x) - erfcx.(x))
grid(true)
xlabel(L"$x$")
ylabel("error in erfcx")


@testset "TotalDegree" begin
    @test HC.bezout_number([2 1; 2 1; 1 1], [2, 1]) == 8
    @test HC.bezout_number([2 1 0; 1 1 1; 0 1 1], [1, 1, 1]) == 5
end


function compute_angle(origin,point)

    degree = atan((origin[2] - point[2]),(point[1] - origin[1])) * (180/pi)
    if degree > 90
        degree = 450 - degree
    else
        degree = 90 - degree
    end
    return degree
end

function compute_distance(origin,point)
    distance = sqrt((origin[1] - point[1])^2 + (origin[2] - point[2])^2)
    return distance
end

f = open("day10.txt");
ast_list = []
for (y, line) in enumerate(readlines(f))
    for (x, value) in enumerate(line)
        if value == '#'
            push!(ast_list,[x-1,y-1])
        end

    end
end
detected = -1
station = -1
for ast in ast_list
    detected_angles = Set()
    for other in ast_list
        if ast != other
            push!(detected_angles,compute_angle(ast,other))
        end
    end
    if length(detected_angles) > detected
        global detected
        global station
        detected = length(detected_angles)
        station = ast
    end
end

println(detected)
println(station)

filter!(x->x!=station,ast_list)

angle_dict = Dict()
for ast in ast_list
    angle = compute_angle(station,ast)

    if haskey(angle_dict, angle) == false
        push!(angle_dict,angle=>[ast])
    else
        value = angle_dict[angle]
        append!(value,[ast])
    end
end
iter = 0
for angle in sort([i for i in keys(angle_dict)],by=abs)
    remove = typemax(Float64)
    vaporized = nothing
    for i in angle_dict[angle]
        dist = compute_distance(station,i)
        if dist < remove
            remove = dist
            vaporized = i
        end
    end
    global iter
    iter += 1
    if iter == 200
        solution = vaporized[1]*100 + vaporized[2]
        println("Vaporize $vaporized, solution is $solution")
        break
    end
end


cd(@__DIR__)
using DifferentialEvolutionMCMC, Random, KernelDensity, Distributions
include("KDE.jl")

function loglike(data, μ, σ)
    simdata = rand(Normal(μ, σ), 10_000)
    kd = kernel(simdata)
    dist = InterpKDE(kd)
    like = max.(1e-10, pdf(dist, data))
    return sum(log.(like))
end

# returns prior log likelihood
function prior_loglike(μ, σ)
    LL = 0.0
    LL += logpdf(Normal(0, 1), μ)
    LL += logpdf(truncated(Cauchy(0, 1), 0, Inf), σ)
    return LL
end

# function for initial values
function sample_prior()
    μ = rand(Normal(0, 1))
    σ = rand(truncated(Cauchy(0, 1), 0, Inf))
    return [μ,σ]
end

# parameter names
names = (:μ,:σ)
# parameter bounds
bounds = ((-Inf,Inf),(0.0,Inf))

data = rand(Normal(0.0, 1.0), 50)

# model object
model = DEModel(;
    sample_prior,
    prior_loglike,
    loglike,
    data,
    names
)

# DEMCMC sampler object
de = DE(;sample_prior, bounds, burnin = 1000, Np = 6)
# number of interations per particle
n_iter = 2000
chains = sample(model, de, MCMCThreads(), n_iter, progress=true)


function sudokus_from_file(filename, sep = '\n')
    s = open(filename) do file
        read(file, String)
    end
    str_sudokus = split(strip(s), sep)
    grids = AbstractArray[]
    for str_sudoku in str_sudokus
        str_sudoku = strip(replace(str_sudoku, "." => "0"))
        if length(str_sudoku) != 81
            continue
        end
        one_line_grid = parse.(Int, split(str_sudoku, ""))
        grid = reshape(one_line_grid, 9, 9)
        push!(grids, grid)
    end
    return grids
end

function create_sudoku_grid!(com, grid)
    com_grid = Array{CS.Variable,2}(undef, 9, 9)
    for (ind, val) in enumerate(grid)
        if val == 0
            com_grid[ind] = CS.add_var!(com, 1, 9)
        else
            com_grid[ind] = CS.add_var!(com, 1, 9; fix = val)
        end
    end
    return com_grid
end

function add_sudoku_constr!(com, grid)
    for rc in 1:9
        #row
        variables = grid[CartesianIndices((rc:rc, 1:9))]
        CS.add_constraint!(com, CS.all_different([variables...]))
        #col
        variables = grid[CartesianIndices((1:9, rc:rc))]
        CS.add_constraint!(com, CS.all_different([variables...]))
    end

    for br in 0:2
        for bc in 0:2
            variables = grid[CartesianIndices((
                (br * 3 + 1):((br + 1) * 3),
                (bc * 3 + 1):((bc + 1) * 3),
            ))]
            CS.add_constraint!(com, CS.all_different([variables...]))
        end
    end
end

function jump_add_sudoku_constr!(m, x)
    for rc in 1:9
        @constraint(m, x[rc, :] in CS.AllDifferent())
        @constraint(m, x[:, rc] in CS.AllDifferent())
    end
    for br in 0:2
        for bc in 0:2
            @constraint(
                m,
                vec(x[(br * 3 + 1):((br + 1) * 3), (bc * 3 + 1):((bc + 1) * 3)]) in
                CS.AllDifferent()
            )
        end
    end
end

function moi_add_sudoku_constr!(m, x)
    for r in 1:9
        MOI.add_constraint(
            m,
            MOI.VectorOfVariables([x[r][c][1] for c in 1:9]),
            CS.CPE.AllDifferent(9),
        )
    end
    for c in 1:9
        MOI.add_constraint(
            m,
            MOI.VectorOfVariables([x[r][c][1] for r in 1:9]),
            CS.CPE.AllDifferent(9),
        )
    end
    variables = [MOI.VariableIndex(0) for _ in 1:9]
    for br in 0:2
        for bc in 0:2
            variables_i = 1
            for i in (br * 3 + 1):((br + 1) * 3), j in (bc * 3 + 1):((bc + 1) * 3)
                variables[variables_i] = x[i][j][1]
                variables_i += 1
            end
            MOI.add_constraint(m, variables, CS.CPE.AllDifferent(9))
        end
    end
end

function jump_fulfills_sudoku_constr(vals)
    for r in 1:9
        if !allunique(vals[r, :])
            println("row $r not unique")
            return false
        end
    end
    for c in 1:9
        if !allunique(vals[:, c])
            println("col $c not unique")
            return false
        end
    end
    for br in 0:2
        for bc in 0:2
            if !allunique(vec(vals[
                (br * 3 + 1):((br + 1) * 3),
                (bc * 3 + 1):((bc + 1) * 3),
            ]))
                println("block $br, $bc not unique")
                return false
            end
        end
    end
    return true
end

function fulfills_sudoku_constr(com_grid)
    correct = true
    for rc in 1:9
        row = com_grid[CartesianIndices((rc:rc, 1:9))]
        if any(v -> !CS.isfixed(v), row)
            return false
        end
        vals = unique([CS.value(v) for v in row])
        correct = length(vals) != 9 ? false : correct

        col = com_grid[CartesianIndices((1:9, rc:rc))]
        if any(v -> !CS.isfixed(v), col)
            return false
        end
        vals = unique([CS.value(v) for v in col])
        correct = length(vals) != 9 ? false : correct
    end

    for br in 0:2
        for bc in 0:2
            box = com_grid[CartesianIndices((
                (br * 3 + 1):((br + 1) * 3),
                (bc * 3 + 1):((bc + 1) * 3),
            ))]
            if any(v -> !CS.isfixed(v), box)
                return false
            end
            vals = unique([CS.value(v) for v in box])
            correct = length(vals) != 9 ? false : correct
        end
    end
    return correct
end

function print_search_space(com_grid; max_length = :default)
    if max_length == :default
        if all(v -> CS.isfixed(v), com_grid)
            max_length = 2
        else
            max_length = 20
        end
    end

    for y in 1:size(com_grid)[1]
        line = ""
        for x in 1:size(com_grid)[2]
            if !CS.isfixed(com_grid[y, x])
                possible = sort(values(com_grid[y, x]))
                pstr = join(possible, ",")
                space_left = floor(Int, (max_length - length(pstr)) / 2)
                space_right = ceil(Int, (max_length - length(pstr)) / 2)
                line *= repeat(" ", space_left) * pstr * repeat(" ", space_right)
            else
                pstr = string(CS.value(com_grid[y, x]))
                space_left = floor(Int, (max_length - length(pstr)) / 2)
                space_right = ceil(Int, (max_length - length(pstr)) / 2)
                line *= repeat(" ", space_left) * pstr * repeat(" ", space_right)
            end
        end
        println(line)
    end
end


using Flux, StochasticDiffEq
using NeuralPDE
using Plots
using CUDA

function phi(xi)
    y = Float64[]
    K = 100
    for x in eachcol(xi)
        val = max(K - maximum(x) , 0.00)
        y = push!(y , val)
    end
    y = reshape(y , 1 , size(y)[1] )
    return y
end

d = 1
r = 0.04
sigma = 0.2
xspan = (80.00 , 115.0)
tspan = (0.0 , 1.0)
σ(du , u , p , t) = du .= sigma.*u
μ(du , u , p , t) = du .= r.*u
prob = KolmogorovPDEProblem(μ , σ , phi , xspan , tspan, d)

sdealg = EM()
ensemblealg = EnsembleThreads()
dt = 0.01
dx = 0.01
trajectories = 100000

m = Chain(Dense(d, 64, elu),Dense(64, 128, elu),Dense(128 , 16 , elu) , Dense(16 , 1))
use_gpu = false
if CUDA.functional() == true
  m = fmap(CUDA.cu , m)
  use_gpu = true
end
opt = Flux.ADAM(0.0005)

@time sol = solve(prob, NeuralPDE.NNKolmogorov(m, opt, sdealg, ensemblealg), verbose = true, dt = dt,
            dx = dx , trajectories = trajectories , abstol=1e-6, maxiters = 1000 , use_gpu = use_gpu)

monte_carlo_sol = []
x_out = collect(85:2.00:110.00)
for x in x_out
  u₀= [x]
  g_val(du , u , p , t) = du .= 0.2.*u
  f_val(du , u , p , t) = du .= 0.04.*u
  dt = 0.01
  tspan = (0.0,1.0)
  prob = SDEProblem(f_val,g_val,u₀,tspan)
  output_func(sol,i) = (sol[end],false)
  ensembleprob_val = EnsembleProblem(prob , output_func = output_func )
  sim_val = solve(ensembleprob_val, EM(), EnsembleThreads() , dt=0.01, trajectories=100000,adaptive=false)
  s = reduce(hcat , sim_val.u)
  mean_phi = sum(phi(s))/length(phi(s))
  global monte_carlo_sol = push!(monte_carlo_sol , mean_phi)
end

x_model = reshape(x_out, 1 , size(x_out)[1])
if use_gpu == true
  m = fmap(cpu , m)
end
y_out = m(x_model)
y_out = reshape(y_out , 13 , 1)

plot(x_out , y_out , lw = 3 ,  xaxis="Initial Stock Price", yaxis="Payoff" , label = "NNKolmogorov")
plot!(x_out , monte_carlo_sol , lw = 3 ,  xaxis="Initial Stock Price", yaxis="Payoff" ,label = "Monte Carlo Solutions")

using SciMLTutorials
SciMLTutorials.tutorial_footer(WEAVE_ARGS[:folder],WEAVE_ARGS[:file])


export cohens_d

"""
    cohens_d(μ1, μ2, s)
    cohens_d(n1, μ1, s1, n2, μ2, s2)
    cohens_d(A::Array, B::Array)

Effect size according to Cohen's d for means `μ1` and `μ2`, number of samples `n1` and `n2`, and standard deviations `s1` and `s2` for respectively group 1 and 2.
"""
cohens_d(μ1, μ2, s) = (μ1 - μ2) / s
s_pooled(n1, s1, n2, s2) = sqrt(
    ( (n1 - 1) * s1^2 + (n2 - 1) * s2^2 ) /
    ( n1 + n2 - 2)
)
s_pooled(A::Array, B::Array) = s_pooled(length(A), std(A), length(B), std(B))
cohens_d(A::Array, B::Array) = cohens_d(mean(A), mean(B), s_pooled(A, B))
cohens_d(n1, μ1, s1, n2, μ2, s2) = cohens_d(μ1, μ2, s_pooled(n1, s1, n2, s2))

"""
    accuracy(trues, preds)::Number

The number of correct predictions in `pred` (by comparing `true` to `prediction`) divided by the total number of predictions.
"""
accuracy(trues, preds)::Number = count(trues .== preds) / length(preds)


export WLS, RandomCascade
using Wavelets, Statistics

"""
    WLS(shufflemethod::Surrogate = IAAFT();
        f::Union{Nothing, Function} = Statistics.cor,
        rescale::Bool = true,
        wt::Wavelets.WT.OrthoWaveletClass = Wavelets.WT.Daubechies{16}())

A wavelet surrogate generated by the following procedure:

1. Compute the wavelet transform of the signal. This results in a set of
    detail coefficients over a set of dyadic scales. As in Keylock (2006),
    we here use the maximal overlap discrete wavelet transform, or MODWT,
    so that the number of coefficients at each scale are the same.
2. Shuffle the detail coefficients at each dyadic scale using the
    provided `shufflemethod`. See "Shuffling methods" below for alternatives.
3. Apply the inverse wavelet transform to the shuffled detail coefficients
    to obtain a surrogate time series.

## Shuffling methods

You may choose to use any surrogate from this package to perform the
randomization of the detail coefficients at each dyadic scale.

The following methods have been discussed in the literature (more may exist):

- Random permutations of wavelet coefficients within each scale (Breakspear et al., 2003). To get this behaviour, use `WLS(x, RandomShuffle(), rescale = false, f = nothing)`.
- Cyclic rotation of wavelet coefficients within each scale (Breakspear et al., 2003).  To get this behaviour, use `WLS(x, Circshift(1:length(x)), rescale = false, f = nothing)`.
- Block resampling of wavelet coefficients within each scale (Breakspear et al., 2003). To get this behaviour, use `WLS(x, BlockShuffle(nblocks, randomize = true), rescale = false, f = nothing)`.
- IAAFT resampling of wavelet coefficients within each scale (Keylock, 2006). To get this behaviour, use `WLS(x, IAAFT(), rescale = true, f = Statistics.cor)`.
    This method preserves the local mean and variance structure of the signal, but
    randomises nonlinear properties of the signal (i.e. Hurst exponents)[^Keylock2006].
    These surrogates can therefore be used to test for changes in nonlinear properties
    of the original signal. In contrast to IAAFT surrogates, the IAAFT-wavelet surrogates
    also preserves nonstationarity. Using other `shufflemethod`s does not necessarily
    preserve nonstationarity. To deal with nonstationary signals, Keylock (2006) recommends
    using a wavelet with a high number of vanishing moments. Thus, our default is to
    use a Daubechies wavelet with 16 vanishing moments. *Note: The iterative procedure after
    the rank ordering step (step [v] in [^Keylock2006]) is not performed in
    this implementation.*

The default method and parameters replicate the behaviour of Keylock (2006)'s IAAFT
wavelet surrogates.

## Error minimization

For the [`IAAFT`](@ref) approach introduced in Keylock (2006), detail coefficients
at each level are circularly rotated to minimize an error function. The methods
introduced in Breakspear et al. (2003) do not apply this error minimization.

In our implementation, you can turn this option on/off using the `f` parameter of
the `WLS` constructor. If `f = nothing` turns off error minization. If `f` is set
to a two-argument function that computes some statistic, for example
`f = Statistics.cor`, then detail coefficients at each scale are circularly
rotated until that function is maximized (and hence the "error" minimized).
If you want to *minimize* some error function, then instead provide an appropriate
transform of your function. For example, if using the root mean squared deviation,
define `rmsd_inv(x, y) = 1 - StatsBase.rmsd(x, y)` and set `f = rmsd_inv`.

## Rescaling

If `rescale == true`, then surrogate values are mapped onto the
values of the original time series, as in the [`AAFT`](@ref) algorithm.
If `rescale == false`, surrogate values are not constrained to the
original time series values. If [`AAFT`](@ref) or [`IAAFT`](@ref) shuffling
is used, `rescale` should be set to `true`. For other methods, it does not
necessarily need to be.

[^Breakspear2003]: Breakspear, M., Brammer, M., & Robinson, P. A. (2003). Construction of multivariate surrogate sets from nonlinear data using the wavelet transform. Physica D: Nonlinear Phenomena, 182(1-2), 1-22.
[^Keylock2006]: C.J. Keylock (2006). "Constrained surrogate time series with preservation of the mean and variance structure". Phys. Rev. E. 73: 036707. doi:10.1103/PhysRevE.73.036707.
"""
struct WLS{WT <: Wavelets.WT.OrthoWaveletClass, S <: Surrogate, E <: Union{Nothing, Function}} <: Surrogate
    shufflemethod::Surrogate # should preserve values of the original series
    rescale::Bool
    f::E
    wt::WT

    function WLS(method::S = IAAFT(); rescale::Bool = true, wt::WT = Wavelets.WT.Daubechies{16}(), f::E = Statistics.cor) where {S <: Surrogate, WT <: Wavelets.WT.OrthoWaveletClass, E <: Union{Nothing, Function}}
        new{WT, S, E}(method, rescale, f, wt)
    end
end

# Initialize without error minimization.
function get_init_noerrorminimize(x, method, rng = Random.default_rng())
    wl = wavelet(method.wt)
    L = length(x)
    x_sorted = sort(x)

    # Wavelet coefficients (step [i] in Keylock)
    W = modwt(x, wl)
    T = eltype(W)
    R = zeros(size(W))

    Nscales = ndyadicscales(L)

    # Will contain surrogate realizations of the wavelet coefficients
    # at each scale (step [ii] in Keylock).
    sW = zeros(T, size(W))

    # Surrogate generators for each set of coefficients
    sgs = [surrogenerator(W[:, i], method.shufflemethod, rng) for i = 1:Nscales]

    init = (
        wl = wl,
        W = W,
        Nscales = Nscales,
        sW = sW,
        sgs = sgs,
        x_sorted = x_sorted,
        R = R,
    )

    return init
end

# Initialize error minimization, as in Keylock (2006).
function get_init_errorminimize(x, method::WLS, rng = Random.default_rng())
    wl = wavelet(method.wt)
    L = length(x)
    x_sorted = sort(x)

    # Wavelet coefficients (step [i] in Keylock)
    W = modwt(x, wl)
    T = eltype(W)
    Nscales = ndyadicscales(L)

    # Will contain surrogate realizations of the wavelet coefficients
    # at each scale (step [ii] in Keylock).
    sW = zeros(T, size(W))

    # We will also need a matrix to store the mirror images of the
    # surrogates (last part of step [ii])
    sWmirr = zeros(T, size(W))

    # Surrogate generators for each set of coefficients
    sgs = [surrogenerator(W[:, i], method.shufflemethod, rng) for i = 1:Nscales]

    # Temporary array for the circular shift error minimizing step
    circshifted_s = zeros(T, size(W))
    circshifted_smirr = zeros(T, size(W))
    R = zeros(size(W))

    s = similar(x)

    init = (wl = wl, W = W, Nscales = Nscales, L = L,
            sW = sW, sgs = sgs, sWmirr = sWmirr,
            circshifted_s = circshifted_s,
            circshifted_smirr = circshifted_smirr,
            x_sorted = x_sorted, R = R,
    )

    return init
end

function surrogenerator(x::AbstractVector{T}, method::WLS, rng = Random.default_rng()) where T
    init = isnothing(method.f) ?
        get_init_noerrorminimize(x, method, rng) :
        get_init_errorminimize(x, method, rng)

    s = similar(x)

    return SurrogateGenerator(method, x, s, init, rng)
end

function (sg::SurrogateGenerator{<:WLS})()
    if isnothing(sg.method.f)
        wls_noerrorminimize(sg)
    else
        wls_errorminimize(sg)
    end
end

function wls_noerrorminimize(sg::SurrogateGenerator{<:WLS})
    s = sg.s

    fds = (:wl, :W, :Nscales, :sW, :sgs, :x_sorted, :R)
    wl, W, Nscales, sW, sgs, x_sorted, R = getfield.(Ref(sg.init), fds)

    for λ in 1:Nscales
        sW[:, λ] .= sgs[λ]()
    end

    s .= imodwt(sW, wl)

    if sg.method.rescale
        s[sortperm(s)] .= x_sorted
    end

    return s
end

function wls_errorminimize(sg::SurrogateGenerator{<:WLS})
    s = sg.s

    # Error minimization function (if we reached this function,
    # f is never `Nothing`), so we can use it safely.
    f = sg.method.f

    fds = (:wl, :W, :Nscales, :L, :sW, :sgs, :sWmirr,
        :circshifted_s, :circshifted_smirr,
        :x_sorted, :R)

    wl, W, Nscales, L, sW, sgs, sWmirr,
        circshifted_s, circshifted_smirr,
        x_sorted, R = getfield.(Ref(sg.init), fds)

    # Create surrogate versions of detail coefficients at each dyadic scale [first part of step (ii) in Keylock]
    for λ in 1:Nscales
        sW[:, λ] .= sgs[λ]()
    end

    # Mirror the surrogate coefficients [last part of step (ii) in Keylock]
    sWmirr .= reverse(sW, dims = 1)

    # In the original paper, surrogates and mirror images are matched to original
    # detail coefficients in a circular manner until some error criterion is
    # minimized. Then, the surrogate or its mirror image, depending on which provides
    # the best fit to the original coefficients, is chosen as the representative
    # for a particular dyadic scale. Here, we instead use maximal correlation as
    # the criterion for matching.
    optimal_shifts = zeros(Int, Nscales)
    optimal_shifts_mirr = zeros(Int, Nscales)
    maxcorrs = zeros(Nscales)
    maxcorrs_mirr = zeros(Nscales)

    for i in 0:L-1
        circshift!(circshifted_s, sW, (i, 0))
        circshift!(circshifted_smirr, sWmirr, (i, 0))

        for λ in 1:Nscales
            origW = W[:, λ]
            c = f(origW, circshifted_s[:, λ])
            if c > maxcorrs[λ]
                maxcorrs[λ] = c
                optimal_shifts[λ] = i
            end

            c_mirr = f(origW, circshifted_smirr[:, λ])
            if c_mirr > maxcorrs_mirr[λ]
                maxcorrs_mirr[λ] = c_mirr
                optimal_shifts_mirr[λ] = i
            end
        end
    end

    # Decide which coefficients are retained (either surrogate or mirror surrogate coefficients)
    for λ in 1:Nscales
        if maxcorrs[λ] >= maxcorrs_mirr[λ]
            R[:, λ] .= circshift(sW[:, λ], optimal_shifts[λ])
        else
            R[:, λ] .= circshift(sWmirr[:, λ], optimal_shifts_mirr[λ])
        end
    end

    s .= imodwt(R, wl)

    if sg.method.rescale
        s[sortperm(s)] .= x_sorted
    end

    return s
end

"""
    RandomCascade(paddingmode::String = "zeros")

A random cascade multifractal wavelet surrogate (Paluš, 2008)[^Paluš2008].

If the input signal length is not a power of 2, the signal must be
padded before the surrogate is constructed. `paddingmode` determines
how the signal is padded. Currently supported padding modes: `"zeros"`.
The final surrogate (constructed from the padded signal) is subset
to match the length of the original signal.

Random cascade surrogate preserve multifractal properties of the input
time series, that is, interactions among dyadic scales and nonlinear
dependencies[^Paluš2008].

[^Paluš2008]: Paluš, Milan (2008). Bootstrapping Multifractals: Surrogate Data from Random Cascades on Wavelet Dyadic Trees. Physical Review Letters, 101(13), 134101–. doi:10.1103/PhysRevLett.101.134101
"""
struct RandomCascade{WT <: Wavelets.WT.OrthoWaveletClass} <: Surrogate
    wt::WT
    paddingmode::String

    function RandomCascade(; wt::WT = Wavelets.WT.Daubechies{16}(), paddingmode::String = "zeros") where {WT <: Wavelets.WT.OrthoWaveletClass}
        new{WT}(wt, paddingmode)
    end
end

function surrogenerator(x::AbstractVector{T}, method::RandomCascade, rng = Random.default_rng()) where T
    nlevels = ndyadicscales(length(x))
    mode = method.paddingmode

    # Pad input so that input to discrete wavelet transform has length which is a power of 2
    x̃ = zeros(2^(nlevels + 1))
    if mode == "zeros"
        copyto!(x̃, x)
    else
        throw(ArgumentError("""`paddingmode` must be one of ["zeros"]"""))
    end

    wl = wavelet(method.wt)

    # Wavelet coefficients (step [i] in Keylock)
    c = dwt(x̃, wl, nlevels)

    # Surrogate coefficients will be partly identical to original coefficients,
    # so we simply copy them and replace the necessary coefficients later.
    cₛ = copy(c)

    # Multiplication factors and index vectors can be pre-allocated for
    # levels 2:nlevels-1; they are overwritten for each new surrogate.
    Ms = [zeros(dyadicdetailn(j-1)) for j = 2:nlevels-1]
    ixs = [zeros(Int, dyadicdetailn(j-1)) for j = 2:nlevels-1]

    init = (
        wl = wl,
        c = c,
        cₛ = cₛ,
        nlevels = nlevels,
        s̃ = similar(x̃),
        Ms = Ms,
        ixs = ixs,
    )

    return SurrogateGenerator(method, x, similar(x), init, rng)
end

function (sg::SurrogateGenerator{<:RandomCascade})()
    s, rng = sg.s, sg.rng
    c, cₛ, s̃, wl, nlevels, Ms, ixs =
        sg.init.c, sg.init.cₛ, sg.init.s̃, sg.init.wl, sg.init.nlevels, sg.init.Ms, sg.init.ixs

    cₛ[dyadicdetailrange(0)] = @view c[dyadicdetailrange(0)]
    cₛ[dyadicdetailrange(1)] = @view c[dyadicdetailrange(1)]

    for (l, j) = enumerate(2:nlevels-1)
        cⱼ₋₁ = @view c[dyadicdetailrange(j - 1)]
        cⱼ = @view c[dyadicdetailrange(j)]

        M = Ms[l]
        ct = 1
        @inbounds for k = 1:length(cⱼ₋₁)
            if k % 2 == 0
                M[ct] = cⱼ[2*k] / cⱼ₋₁[k]
            else
                M[ct] = cⱼ[2*(k+1)] / cⱼ₋₁[k+1]
            end
            ct += 1
        end

        shuffle!(rng, M)
        new_coeffs!(M, cⱼ₋₁)
        ix = ixs[l]
        sortperm!(ix, M)
        cₛ[dyadicdetailrange(j-1)] .= @view cⱼ₋₁[ix]
    end
    s̃ .= idwt(cₛ, wl, nlevels)

    # Surrogate length must match length of original signal.
    s .= @view s̃[1:length(s)]
    return s
end

function new_coeffs!(M, cⱼ₋₁)
    @inbounds for k = 1:length(cⱼ₋₁)
        M[k] = M[k] * cⱼ₋₁[k]
    end
end


raw"""
Computes Cholesky decomposition of a Hermitian positive-definite matrix A, i.e.,

$$
A = L L^*
$$

where $L$ is a lower triangular matrix with real and positive diagonal entries, and $L^*$ denotes
the conjugate transpose of $L$.
"""
module CholeskyDecomposition

using LinearAlgebra

export
    cholesky_outer_product_decompose,
    cholesky_banachiewicz_decompose,
    cholesky_crout_decompose

function get_n(A::Matrix)::Int
    (n_r, n_c) = size(A)
    @assert n_r == n_c "Input matrix is not square"
    n_r
end

"""
The Cholesky (outer-product) algorithm

<https://en.wikipedia.org/wiki/Cholesky_decomposition#The_Cholesky_algorithm>
"""
function cholesky_outer_product_decompose(A::Matrix{T})::LowerTriangular{T} where T<:Number
    n = get_n(A)

    # Some edge case values:
    #
    #     A_1 = A
    #     A_{n + 1} = I(n)
    #     I_i ∈ ℜ_{i ^ 2}
    #     I_0 = ∅
    #     L_0 = I(n)
    #     B_0 = A
    #     B_i ∈ ℂ_{(n - i) ^ 2}
    #     B_n = ∅

    A_i = copy(A)
    A = nothing  # prevents accidental assignment
    L_0 = T.(I(n)) |> Matrix |> LowerTriangular
    L = copy(L_0)  # The product of L1...Ln
    for i in 1:n
        # Makes aliases (no need for `A_ii`)
        b_i = @view A_i[(i + 1):end, i]
        b_i_H = @view A_i[i, (i + 1):end]
        B_i = @view A_i[(i + 1):end, (i + 1):end]

        L_i = copy(L_0)
        sqrtAii = sqrt(A_i[i, i])
        L_i[i, i] = sqrtAii
        L_i[(i + 1):end, i] = b_i / sqrtAii

        L *= L_i

        # Mind the order
        B_i .-= b_i * transpose(b_i_H) / A_i[i, i]  # Ugly, can't do `b_i * b_i_H`
        b_i .= 0.0
        b_i_H .= 0.0
        A_i[i, i] = 1.0
    end
    L
end

"""
The Cholesky-Banachiewicz algorithm

<https://en.wikipedia.org/wiki/Cholesky_decomposition#The_Cholesky–Banachiewicz_and_Cholesky–Crout_algorithms>
"""
function cholesky_banachiewicz_decompose(A::Matrix{T})::LowerTriangular{T} where T<:Number
    n = get_n(A)
    # To write the in-place version, just set `L = A` and do symmetric updating.
    L = zeros(T, n, n) |> LowerTriangular

    # Note that as $A$ is positive-definite, **not positive semi-definite**, $L_{ii}$ will never be
    # zero, so there is no need to check.
    #
    # Also, do NOT use `dot` to compute `L[r, c]`!

    for r in 1:n
        for c in 1:(r - 1)
            L[r, c] = (
                A[r, c] - sum((@view L[r, 1:(c - 1)]) .* conj.(@view L[c, 1:(c - 1)]))
            ) / L[c, c]
        end
        l = @view L[r, 1:(r - 1)]
        L[r, r] = sqrt(A[r, r] - l ⋅ l)
    end
    L
end

"""
The Cholesky-Crout algorithm

It is similar to the Cholesky-Banachiewicz algorithm, but processes column by column instead of row
by row.
"""
function cholesky_crout_decompose(A::Matrix{T})::LowerTriangular{T} where T<:Number
    n = get_n(A)
    L = zeros(T, n, n) |> LowerTriangular
    # Deliberately write the index as `:` rather than `1:(c - 1)`
    for c in 1:n
        l = @view L[c, :]
        L[c, c] = sqrt(A[c, c] - l ⋅ l)
        for r in (c + 1):n
            L[r, c] = (A[r, c] - sum((@view L[r, :]) .* conj.(@view L[c, :]))) / L[c, c]
        end
     end
     L
end

end  # module


plan_clenshawcurtis(μ) = length(μ) > 1 ? FFTW.plan_r2r!(μ, FFTW.REDFT00) : ones(μ)'

"""
Compute nodes of the Clenshaw—Curtis quadrature rule.
"""
clenshawcurtisnodes(::Type{T}, N::Int) where T = T[cospi(k/(N-one(T))) for k=0:N-1]

"""
Compute weights of the Clenshaw—Curtis quadrature rule with modified Chebyshev moments of the first kind ``\\mu``.
"""
clenshawcurtisweights(μ::Vector) = clenshawcurtisweights!(copy(μ))
clenshawcurtisweights!(μ::Vector) = clenshawcurtisweights!(μ, plan_clenshawcurtis(μ))
function clenshawcurtisweights!(μ::Vector{T}, plan) where T
    N = length(μ)
    rmul!(μ, inv(N-one(T)))
    plan*μ
    μ[1] *= half(T); μ[N] *= half(T)
    return μ
end

# Chebyshev-T coefficients to values at Clenshaw-Curtis nodes

applyTN_plan(x) = length(x) > 1 ? FFTW.plan_r2r!(x, FFTW.REDFT00) : fill!(similar(x),1)'

applyTN!(x::Vector{T}) where {T<:AbstractFloat} = applyTN!(x,applyTN_plan(x))

function applyTN!(x::Vector{T},plan) where T<:AbstractFloat
    x[1] *= 2; x[end] *=2
    plan*x
    rmul!(x,half(T))
end
applyTN(x::Vector{T},plan) where {T<:AbstractFloat} = applyTN!(copy(x),plan)
applyTN(x::Vector{T}) where {T<:AbstractFloat} = applyTN!(copy(x))

# Values at Clenshaw-Curtis nodes to Chebyshev-T coefficients

applyTNinv_plan(x) = length(x) > 1 ? FFTW.plan_r2r!(x, FFTW.REDFT00) : ones(x)'

applyTNinv!(x::Vector{T}) where {T<:AbstractFloat} = applyTNinv!(x,applyTNinv_plan(x))

function applyTNinv!(x::Vector{T},plan) where T<:AbstractFloat
    plan*x
    x[1] /= 2;x[end] /= 2
    rmul!(x,inv(length(x)-one(T)))
end
applyTNinv(x::Vector{T},plan) where {T<:AbstractFloat} = applyTNinv!(copy(x),plan)
applyTNinv(x::Vector{T}) where {T<:AbstractFloat} = applyTNinv!(copy(x))

# sin(nθ) coefficients to values at Clenshaw-Curtis nodes except ±1

applyUN_plan(x) = length(x) > 0 ? FFTW.plan_r2r!(x, FFTW.RODFT00) : fill!(similar(x),1)'

applyUN!(x::AbstractVector{T}) where {T<:AbstractFloat} = applyUN!(x,applyUN_plan(x))

function applyUN!(x::AbstractVector{T},plan) where T<:AbstractFloat
    plan*x
    rmul!(x,half(T))
end
applyUN(x::AbstractVector{T},plan) where {T<:AbstractFloat} = applyUN!(copy(x),plan)
applyUN(x::AbstractVector{T}) where {T<:AbstractFloat} = applyUN!(copy(x))


# This implements src/snes/examples/tutorials/ex2.c from PETSc using the PETSc.jl package, using SNES
#
# This solves the equations sequentially
#
# Newton method to solve u'' + u^{2} = f, sequentially.

using PETSc, MPI, LinearAlgebra, SparseArrays, UnicodePlots

if ~MPI.Initialized()
    MPI.Init()
end

petsclib = PETSc.petsclibs[1]
PETSc.initialize(petsclib)

"""
    Computes initial guess
"""
function FormInitialGuess!(x)
    for i=1:length(x)
        x[i] = 0.50;
    end
end

"""
    Computes rhs forcing function
"""
function SetInitialArrays(n)
    h =  1.0/(n-1.0)
    F = zeros(n);
    xp = 0.0;
    for i=1:n
        v    = 6.0*xp + (xp+1.e-12)^6.0;
        F[i] = v;
        xp   = xp+h;
    end

    return F
end

"""
    Computes the residual f, given solution vector x
"""
function FormResidual!(cf,cx, args...)
    if typeof(cx) <: Ptr{Nothing}
        # When this routine is called from PETSc, cx is a pointer to a global vector
        # That's why we have to transfer it first to
        x   =   PETSc.unsafe_localarray(PETSc.scalartype(petsclib),cx)
    else
        x   = cx;
    end
    if typeof(cf) <: Ptr{Nothing}
        f   =   PETSc.unsafe_localarray(PETSc.scalartype(petsclib),cf)
    else
        f   = cf;
    end
    n       =   length(x);
    xp      =   LinRange(0.0,1.0, n);
    F       =   6.0.*xp .+ (xp .+1.e-12).^6.0;      # define source term function

    dx      =   1.0/(n-1.0);
    f[1]    =   x[1] - 0.0;
    for i=2:n-1
        f[i] = (x[i-1] - 2.0*x[i] + x[i+1])/dx^2 + x[i]*x[i] - F[i]
    end
    f[n]    =   x[n] - 1.0;
    Base.finalize(x)
    Base.finalize(f)

end

"""
    Computes the jacobian, given solution vector x
"""
function FormJacobian!(cx, args...)

    if typeof(cx) <: Ptr{Nothing}
        x   =   PETSc.unsafe_localarray(PETSc.scalartype(petsclib),cx)
    else
        x   =   cx;
    end

    J   =   args[1];        # preconditioner = args[2], in case we want it to be different from J
    n   =   length(x);
    dx  =   1.0/(n-1.0);

    # interior points (hand-coded jacobian)
    for i=2:n-1
        J[i,i-1] = 1.0/dx^2;
        J[i,i  ] = -2.0/dx^2 + 2.0*x[i];
        J[i,i+1] = 1.0/dx^2;
    end

    # boundary points
    J[1,1] = 1.0;
    J[n,n] = 1.0;

    if typeof(J) <: PETSc.AbstractMat
        PETSc.assemble(J);  # finalize assembly

    end

    Base.finalize(x)
end

# ==========================================
# Main code

# Compute initial solution
n   =   21;
F   =   SetInitialArrays(n);
x   =   zeros(n);

FormInitialGuess!(x);

# Compute initial jacobian using a julia structure to obtain the nonzero structure
# Note that we can also obtain this structure in a different manner
Jstruct  = zeros(n,n);
FormJacobian!(x, Jstruct);                              # jacobian in julia form
Jsp      =   sparse(Float64.(abs.(Jstruct) .> 0))       # sparse julia, with 1.0 in nonzero spots
PJ       =   PETSc.MatSeqAIJ(Jsp);                      # transfer to

# Setup snes
x_s = PETSc.VecSeq(x);                  # solution vector
res = PETSc.VecSeq(F);     # residual vector

S = PETSc.SNES{Float64}(PETSc.petsclibs[1],MPI.COMM_WORLD;
        snes_rtol=1e-12,
        snes_monitor=false,
        snes_converged_reason=false);
PETSc.setfunction!(S, FormResidual!, res)
PETSc.setjacobian!(S, FormJacobian!, PJ, PJ)

# solve
PETSc.solve!(x_s, S);

# Extract & plot solution
x_sol = x_s.array;                  # convert solution to julia format
FormResidual!(res.array,x_sol)      # just for checking, compute residual

@show norm(res.array)

PETSc.finalize(petsclib)

# plot solution in REPL
lineplot(LinRange(0,1,n),x_sol,xlabel="width",ylabel="solution")


# Test utililties.jl

# Tests could be more comprehensive (of course).  In some cases all
# that is done is to execute the function in some way just to make
# sure no exceptions are thrown as a result of something breaking with
# a new version of Julia.

println("testing utilites")

using PerceptualColourMaps
using Test, Statistics

# bbspline
P = [0 1 2 3
     0 1 2 3]
k = 3
N = 10
S = bbspline(P, k, N)
#dx = Base.LinAlg.gradient(S[1,:],1)
#dy = Base.LinAlg.gradient(S[2,:],1)
# There seems to be an awkward method ambiguity between Base.LinAlg.gradient and
# Calculus.gradient, hence the simple replacement code below.
dx = S[1, 2:end] - S[1, 1:(end-1)]
dy = S[2, 2:end] - S[2, 1:(end-1)]

slope = dy./dx
@test maximum(slope .- 1) < 1e-3

# pbspline
P = [0 1 2 3 0
     0 1 2 3 0]
k = 3
N = 100
S = pbspline(P, k, N)

# gaussfilt1d
a = zeros(20,1)
a[10] = 1
sigma = 2
sa = gaussfilt1d(a, sigma)
@test abs(sa[10] - 1/(sigma*sqrt(2pi))) < 1e-3
@test abs(sum(sa) - 1) < 1e-3

# interp1
x = [0, 2, 7]
y = 2*x
xi = [1.1, 0.5, 6]
yi = interp1(x,y,xi)
@test maximum(abs.(yi - 2*xi) .< eps())

# normalise
a = rand(5,3)
an = normalise(a)
@test isapprox(minimum(an), 0)
@test isapprox(maximum(an), 1)

reqmean = 5
reqvar = 7
an = normalise(a, reqmean, reqvar)
@test isapprox(mean(an), reqmean)
@test isapprox(var(an), reqvar)

# histtruncate
rows = 100;
cols = 200;
img = rand(rows, cols)
lHistCut = 2
uHistCut = 4
htimg = histtruncate(img, lHistCut, uHistCut)
minval = minimum(htimg)
maxval = maximum(htimg)
# Check that the number of saturated pixels at each extreme is
# approximately lHistCut and uHistCut. Note some integer rounding has
# to occur.
v = findall(abs.(htimg.-minval).<eps())
@test abs(length(v)/(rows*cols) * 100 - lHistCut) < 1

v = findall(abs.(htimg.-maxval).<eps())
@test abs(length(v)/(rows*cols) * 100 - uHistCut) < 1

# sineramp
sr = sineramp()
sr = sineramp((256, 512), 12.5, 8, 2)

# circlesineramp
(img, alpha) = circlesineramp()
(img, alpha) = circlesineramp(512, pi/10, 8, 2, false)


export angular_distance, along_line_angular_distance, angular_length
# export angular_distance, distance, along_line_angular_distance, angular_length
# """
#     Radius Earth [m]
#
# Source: en.wikipedia.org/wiki/Earth_radius
# """
# const Rₑ_m = 6371008.7714

"""
    angular_distance(point₁::Point, point₂::Point)

Return the `angular_distance` [deg] of the great circle line between the
positions `point₁` [deg] and `point₂` [deg] on a unit sphere.

Source: www.movable-type.co.uk/scripts/latlong.html
"""
function angular_distance(point₁::Point, point₂::Point)
    Δpos = point₂ - point₁
    a = sind(Δpos.ϕ/2.0)^2 + cosd(point₁.ϕ)*cosd(point₂.ϕ)*sind(Δpos.λ/2.0)^2
    c = 2.0 * atand(√a, √(1.0 - a))
    return c
end

"""
    angular_length(arc::Arc)

Return the `angular_length` [deg] of the great circle line section on a
unit sphere.
"""
angular_length(arc::Arc) =
angular_distance(arc.point₁, arc.point₂)

"""
    angular_length(points::Vector{Points{Float64}})

Return the `angular_length` [deg] along the subsequent points on a unit sphere.
"""
function angular_length(points::Vector{Point{Float64}})
    point₁ = points[1]
    dist_deg = 0.0
    for point₂ in points[2:end]
        dist_deg += angular_distance(point₁, point₂)
        point₁ = point₂
    end
    return dist_deg
end

"""
    angular_length(arcs::Arcs)

Return the `angular_length` [deg] along the arcs on a unit sphere.
"""
angular_length(arcs::Arcs) = angular_length(arcs.points)

"""
    angular_length(polygon::Polygon)

Return the `angular_length` [deg] (e.g. circumference) along the polygon on a unit
sphere.
"""
angular_length(polygon::Polygon) = angular_length(polygon.points)

# """
#     distance(angular_distance::Float64, radius::Float64=Rₑ_m)
#
# Convert the `angular_distance` [deg] into a distance on a sphere of given radius
#  [m]. The default radius is the radius of the Earth.
# """
# function distance(angular_distance::Float64, radius::Float64=Rₑ_m)
#     return deg2rad(angular_distance) * radius
# end

"""
    angular_distance(point₃::Point, point₁::Point, azimuth₁₂::Float64)

Return the `angular_distance` [deg] from point₃ [deg] to the closest point on a
great circle line starting in `point₁` [deg] with `azimuth₁₂` [deg] from
`point₁` to `point₂`. It is assumed that the great circle line does not stop in
`point₂`, but continuous around the unit sphere. A positive value indicates
being right of the line, and negative being left of the line.

Source: www.movable-type.co.uk/scripts/latlong.html
"""
function angular_distance(point₃::Point, point₁::Point, azimuth₁₂::Float64)
    angular_distance₁₃ = angular_distance(point₁, point₃)
    azimuth₁₃ = azimuth(point₁, point₃)
    return angular_distance(angular_distance₁₃, azimuth₁₃, azimuth₁₂)
end

"""
    angular_distance(point₃::Point, line₁::Line)

Return the `angular_distance` [deg] from point₃ [deg] to the closest point on a
great circle line starting in `point₁` [deg] with `azimuth₁₂` [deg]. A positive
value indicates being right of the line, and negative being left of the line.

Source: www.movable-type.co.uk/scripts/latlong.html
"""
angular_distance(point₃::Point, line₁::Line) = angular_distance(point₃,
line₁.point, line₁.azimuth)

"""
    angular_distance(angular_distance₁₃::Float64, azimuth₁₃::Float64,
    azimuth₁₂::Float64)

Return the `angular_distance` [deg] from point₃ [deg] to the closest point on a
great circle line starting in `point₁` [deg] with `azimuth₁₂` [deg]. For the
calculation we need the `angular_distance₁₃` and `azimuth₁₃` between `point₁`
and `point₃` [deg], and the `azimuth₁₂` from `point₁` to `point₂` along the
great circle line. It is assumed that the great circle line does not stop in
`point₂`, but continuous around the unit sphere.

Source: www.movable-type.co.uk/scripts/latlong.html
"""
function angular_distance(angular_distance₁₃::Float64, azimuth₁₃::Float64,
azimuth₁₂::Float64)
    return asind(sind(angular_distance₁₃) * sind(azimuth₁₃ - azimuth₁₂))
end

"""
    angular_distance(point₃::Point, point₁::Point, point₂::Point)

Return the `angular_distance` [deg] from point₃ [deg] to the closest point on a
great circle line section starting in `point₁` [deg] and ending in `point₂` [deg].
The great circle line section does not continue around the unit sphere.

The `angular_distance` does not change sign when being left or right of the arc.
"""
function angular_distance(point₃::Point, point₁::Point, point₂::Point, tolerance::Float64=tolerance_deg)
    angular_distance₁₂ = angular_distance(point₁, point₂)
    angular_distance₁₃ = angular_distance(point₁, point₃)
    angular_distance₂₃ = angular_distance(point₂, point₃)
    azimuth₁₂ = azimuth(point₁, point₂)
    angular_distance_line3 = angular_distance(point₃, point₁, azimuth₁₂)
    # along_line_angular_dist = along_line_angular_distance(angular_distance₁₃,
    # angular_distance_line3)
    along_line_pnt_3a = along_line_point(point₃, Line(point₁, azimuth₁₂))
    if abs(angular_distance(along_line_pnt_3a, point₁) +
        angular_distance(along_line_pnt_3a, point₂) -
        angular_distance₁₂) < tolerance
        return abs(angular_distance_line3)
    else
        return min(angular_distance₁₃, angular_distance₂₃)
    end
end

"""
    angular_distance(point₃::Point, points::Vector{Points{T}}) where T<:Float64

Return the `angular_distance` [deg] from point₃ [deg] to the closest point on a
an set of points representing an set of arcs.

The `angular_distance` does not change sign when being left or right of the arc.
"""
function angular_distance(point₃::Point, points::Vector{Point{T}}) where
    T<:Float64
    point₁ = points[1]
    point₂ = points[2]
    min_dist = angular_distance(point₃, point₁, point₂)
    point₁ = point₂
    for point₂ in points[2:end]
        min_dist = min(min_dist, angular_distance(point₃, point₁, point₂))
        point₁ = point₂
    end
    return min_dist
end

"""
    angular_distance(point₃::Point, arcs::Arcs)

Return the `angular_distance` [deg] from point₃ [deg] to the closest point on a
an `arcs`.

The `angular_distance` does not change sign when being left or right of the arc.
"""
angular_distance(point₃::Point, arcs::Arcs) = angular_distance(point₃, arcs.points)

"""
    angular_distance(point₃::Point, polygon::Polygon)

Return the `angular_distance` [deg] from point₃ [deg] to the closest point on
the border of a `polygon`.

The `angular_distance` does not change sign when being left or right of the arc.
"""
angular_distance(point₃::Point, polygon::Polygon) = angular_distance(point₃,
polygon.points)

"""
    angular_distance(point₃::Point, arc₁₂::Arc)

Return the `angular_distance` [deg] from point₃ [deg] to the closest point on a
great circle line section starting in `point₁` [deg] and ending in `point₂` [deg].
The great circle line section does not continue around the unit sphere.
"""
angular_distance(point₃::Point, arc₁₂::Arc) = angular_distance(point₃, arc₁₂.point₁, arc₁₂.point₂)

"""
    along_line_angular_distance(angular_distance₁₃::Float64,
    angular_distance_line3::Float64)

The `along_line_angular_distance` from the start `point₁` [deg] to the closest
point on the great circle line to `point₃`. As input we need the
`angular_distance₁₃` [deg] between `point₁` and `point₃` [deg] and the closest
distance `angular_distance_line3` [deg] between the great circle line and
`point3` [deg]. The distance is non-directional.

Source: www.movable-type.co.uk/scripts/latlong.html
"""
function along_line_angular_distance(angular_distance₁₃::Float64,
angular_distance_line3::Float64)
    return acosd(cosd(angular_distance₁₃) / cosd(angular_distance_line3))
end

"""
    along_line_angular_distance(point₃::Point, line₁::Line)

The `along_line_angular_distance` from the start `point₁` [deg] to the closest
point on the great circle line `line₁` to `point₃`. The distance is
non-directional.

Source: www.movable-type.co.uk/scripts/latlong.html
"""
function along_line_angular_distance(point₃::Point, line₁::Line)
    angular_distance₁₃ = angular_distance(line₁.point, point₃)
    angular_distance_line3 = angular_distance(point₃::Point, line₁::Line)
    return along_line_angular_distance(angular_distance₁₃,
    angular_distance_line3)
end


export Spindle

"""
`Spindle()` returns the Moser spindle graph. This is a seven-vertex
unit distance graph with chromatic number equal to 4.
"""
function Spindle()
    G = IntGraph(7)
    edges = [
        1 2
        1 3
        2 3
        2 4
        3 4
        1 5
        1 6
        5 6
        5 7
        6 7
        4 7 ]
    add_edges!(G,edges)

    d = Dict{Int,Vector}()
    a = sqrt(3)/2

    pts = [ 0 1/2 -1/2 0 ; 0 a a 2a ]

    theta = acos(5/6)/2
    R = [ cos(theta) -sin(theta); sin(theta) cos(theta) ]

    p1 = R*pts
    for k=1:4
        d[k] = p1[:,k]
    end

    p2 = R'*pts
    for k=5:7
        d[k] = p2[:,k-3]
    end
    embed(G,d)
    SimpleGraphs.name(G,"Moser Spindle")
    return G
end


"""
    isinside(chart, point) -> Bool

Returns true is the given point is in the image of the given chart, false otherwise.
"""
function isinside(cell, point)

  u = carttobary(cell, point)
  T = eltype(u)

  tol = eps(T) * 1e3
  w = one(T)
  for i in 1:dimension(cell)
    0+tol < u[i] < 1-tol || return false
    w -= u[i]
  end

  0+tol < w < 1-tol || return false
  return true

end

"""
	isinclosure(simplex, point) -> Bool

Determine whether point is in the closure of simplex. False positives are possible for points just outside of the simplex.
"""
function isinclosure(cell, point)

  u = carttobary(cell, point)
  T = eltype(u)
  dim = dimension(cell)
  udim = length(point)

  tol = eps(T) * 1e3
  w = one(T)
  for i in 1:dim
    0-tol < u[i] < 1+tol || return false
    w -= u[i]
  end

  0-tol < w < 1+tol || return false

  # finally, check that the point indeed is in the plane of the cell
  A = zeros(T, udim, dim+1)
  origin = cell[dim+1]
  for i in 1:dim
      for j in 1:udim
          A[j,i] = cell[i][j] - origin[j]
      end
  end
  for j in 1:udim; A[j,end] = point[j] - origin[j]; end
  rank(A) == dim+1 && return false

  return true

end


const GraphMatrix = Union{Matrix, BitMatrix}

function clique_size(seq::Union{Vector{Bool}, BitArray}, A::GraphMatrix)
  set = find(seq)
  clique = true
  for i in set, j in set
    if (i != j) && !(A[i,j] > 0)
      clique = false
      break
    end
  end
  return clique*length(set)
end

function clique_loss(seq::Union{Vector{Bool}, BitArray}, A::GraphMatrix, kappa::Float)
  set = find(seq)
  clique = true
  edges = Float(0)
  n = length(set)
  for i in set, j in set
    if i != j
      aij = (A[i,j] > 0)
      edges += aij
      clique = clique && aij
    end
  end

  return (Float(1) - Float(edges)/Float(max(n*(n-1+kappa), 1)), Float(-n*clique))
end

function clique_loss(seq::Vector{Int}, A::GraphMatrix, kappa::Float)
  n = size(A, 1)
  if length(seq) < n
    idx = falses(n)
    idx[seq] = true
  else
    idx = seq .> 1
  end
  return clique_loss(idx, A, kappa)
end

"""
### Clique finding using the cakewalk method
Minimize the soft-clique-loss function with cakewalk. Usage: \n
`opt_seq, opt_val = clique_cw(A, kappa)` \n
where `kappa` is a size booster parameter beteen 0.0 and 1.0.
For details about the loss, kappa, and the reasoning behind them see our paper.\n\n

When `opt_seq[i]` is 2, it means vertex `i` is selected, and 1 means it is not. \n
`opt_val` is a tuple of two floats. The first is the value being optimized, i.e.,
the soft-clique-loss, and the second is the actual loss that we are interested in,
the negative clique size.
"""
clique_cw(A::GraphMatrix, args...; kwargs...) = cakewalk(clique_loss, 2, size(A, 1), A, args...; kwargs...)

inc_clique_size(ind, A::GraphMatrix) = maximum(map(v -> (w = copy(ind); w[v] = true; clique_size(w, A)), 1:size(A,1)))

"""
### Validate that a clique is locally maximal
First, find some clique, \n
`opt_seq, opt_val = clique_cw(A, kappa)` \n
Then, validate the clique is locally maximal, \n
`is_local_max_clique(opt_seq .== 2, A)`\n
The returned value is binary.
"""
is_local_max_clique(ind, A::GraphMatrix) = (cs = clique_size(ind,A); (cs > 0) && (cs >= inc_clique_size(ind, A)))


# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    signarea(A, B, C)

Compute signed area of triangle formed
by points `A`, `B` and `C`.
"""
function signarea(A::Point{2}, B::Point{2}, C::Point{2})
  ((B - A) × (C - A)) / 2
end

"""
    sideof(point, segment)

Determines on which side of the oriented `segment`
the `point` lies. Possible results are `:LEFT`,
`:RIGHT` or `:ON` the segment.
"""
function sideof(p::Point{2,T}, s::Segment{2,T}) where {T}
  a, b = vertices(s)
  area = signarea(p, a, b)
  ifelse(area > atol(T), :LEFT, ifelse(area < -atol(T), :RIGHT, :ON))
end

"""
    sideof(point, chain)

Determines on which side of the closed `chain` the
`point` lies. Possible results are `:INSIDE` or
`:OUTSIDE` the chain.
"""
function sideof(p::Point{2,T}, c::Chain{2,T}) where {T}
  w = windingnumber(p, c)
  ifelse(isapprox(w, zero(T), atol=atol(T)), :OUTSIDE, :INSIDE)
end

"""
    mahalanobis(radii, angles, convention)

Return the Mahalanobis distance corresponding to an ellipsoid
with `radii` rotated by given `angles` according to given `convention`.

- For 2D ellipses, there are two radii and one rotation angle.
- For 3D ellipsoids, there are three radii and three rotation angles.

The list of available conventions can be found with:

```julia
julia> subtypes(RotationConvention)
```
"""
function mahalanobis(radii, angles, convention)
  ndims, nangles = length(radii), length(angles)
  valid = (ndims == 3 && nangles == 3) || (ndims == 2 && nangles == 1)
  @assert valid "invalid number of radii/angles"

  # invert radii if necessary
  invert = mainaxis(convention) == :Y
  ranges = invert ? [radii[i] for i in reverse(1:ndims,1,2)] : vec(radii)

  # scaling matrix
  λ = one(eltype(ranges)) ./ ranges.^2
  Λ = Diagonal(SVector{ndims}(λ))

  # rotation matrix
  R = rotmat(angles, convention)

  # ellipsoid metric
  W = Symmetric(R'*Λ*R)
  Mahalanobis(W)
end


# This file is part of the TaylorSeries.jl Julia package, MIT license
#
# Luis Benet & David P. Sanders
# UNAM
#
# MIT Expat license
#

## Differentiating ##
"""
    derivative(a)

Return the `Taylor1` polynomial of the differential of `a::Taylor1`.
The last coefficient is set to zero.

The function `differentiate` is an exact synonym of `derivative`.
"""
function derivative(a::Taylor1)
    res = zero(a)
    @inbounds for ord in eachindex(a)
        derivative!(res, a, ord)
        # ord == a.order-1 && break
    end
    return res
end

"""
    differentiate

An exact synonym of [`derivative`](@ref).
"""
const differentiate = derivative

"""
    derivative!(res, a) --> nothing

In-place version of `derivative`. Compute the `Taylor1` polynomial of the
differential of `a::Taylor1` and save it into `res`. The last coefficient is
set to zero.
"""
function derivative!(res::Taylor1, a::Taylor1)
    @inbounds for ord in eachindex(a)
        derivative!(res, a, ord)
        # ord == a.order-1 && break
    end
    res[a.order] = zero(a[0])
    nothing
end

"""
    derivative!(p, a, k) --> nothing

Update in-place the `k-th` expansion coefficient `p[k]` of `p = derivative(a)`
for both `p` and `a` `Taylor1`.

The coefficients are given by

```math
p_k = (k+1)a_{k+1}.
```

"""
derivative!(p::Taylor1, a::Taylor1, k::Int) = k < a.order ? p[k] = (k+1)*a[k+1] : nothing

"""
    derivative(a, n)

Compute recursively the `Taylor1` polynomial of the n-th derivative of
`a::Taylor1`.
"""
function derivative(a::Taylor1{T}, n::Int) where {T<:Number}
    @assert a.order ≥ n ≥ 0
    if n==0
        return a
    else
        res = deepcopy(a)
        for i in 1:n
            derivative!(res, res)
        end
        return res
    end
end

"""
    derivative(n, a)

Return the value of the `n`-th derivative of the polynomial `a`.
"""
function derivative(n::Int, a::Taylor1{T}) where {T<:Number}
    @assert a.order ≥ n ≥ 0
    factorial( widen(n) ) * a[n] :: T
end

## Integrating ##
"""
    integrate(a, [x])

Return the integral of `a::Taylor1`. The constant of integration
(0-th order coefficient) is set to `x`, which is zero if ommitted.
"""
function integrate(a::Taylor1{T}, x::S) where {T<:Number,S<:Number}
    order = get_order(a)
    aa = a[0]/1 + zero(x)
    R = typeof(aa)
    coeffs = Array{typeof(aa)}(undef, order+1)
    fill!(coeffs, zero(aa))
    @inbounds for i = 1:order
        coeffs[i+1] = a[i-1] / i
    end
    @inbounds coeffs[1] = convert(R, x)
    return Taylor1(coeffs, a.order)
end
integrate(a::Taylor1{T}) where {T<:Number} = integrate(a, zero(a[0]))

## Differentiation ##
"""
    derivative(a, r)

Partial differentiation of `a::HomogeneousPolynomial` series with respect
to the `r`-th variable.
"""
function derivative(a::HomogeneousPolynomial, r::Int)
    @assert 1 ≤ r ≤ get_numvars()
    # @show zero(a[1]) zero(eltype(a))
    T = eltype(a)
    a.order == 0 && return HomogeneousPolynomial(zero(a[1]), 0)
    @inbounds num_coeffs = size_table[a.order]
    coeffs = zeros(T,num_coeffs)
    @inbounds posTb = pos_table[a.order]
    @inbounds num_coeffs = size_table[a.order+1]

    @inbounds for i = 1:num_coeffs
        iind = coeff_table[a.order+1][i]
        n = iind[r]
        n == 0 && continue
        iind[r] -= 1
        kdic = in_base(get_order(),iind)
        pos = posTb[kdic]
        coeffs[pos] = n * a[i]
        iind[r] += 1
    end

    return HomogeneousPolynomial{T}(coeffs, a.order-1)
end
derivative(a::HomogeneousPolynomial, s::Symbol) = derivative(a, lookupvar(s))

"""
    derivative(a, r)

Partial differentiation of `a::TaylorN` series with respect
to the `r`-th variable. The `r`-th variable may be also
specified through its symbol.
"""
function derivative(a::TaylorN, r=1::Int)
    T = eltype(a)
    coeffs = Array{HomogeneousPolynomial{T}}(undef, a.order)

    @inbounds for ord = 1:a.order
        coeffs[ord] = derivative( a[ord], r)
    end
    return TaylorN{T}( coeffs, a.order )
end
derivative(a::TaylorN, s::Symbol) = derivative(a, lookupvar(s))

"""
    derivative(a::TaylorN{T}, ntup::NTuple{N,Int})

Return a `TaylorN` with the partial derivative of `a` defined
by `ntup::NTuple{N,Int}`, where the first entry is the number
of derivatives with respect to the first variable, the second is
the number of derivatives with respect to the second, and so on.
"""
function derivative(a::TaylorN, ntup::NTuple{N,Int}) where {N}

    @assert N == get_numvars() && all(ntup .>= 0)

    sum(ntup) > a.order && return zero(a)
    sum(ntup) == 0 && return copy(a)

    aa = copy(a)
    for nvar in 1:get_numvars()
        for numder in 1:ntup[nvar]
            aa = derivative(aa, nvar)
        end
    end

    return aa
end

"""
    derivative(ntup::NTuple{N,Int}, a::TaylorN{T})

Returns the value of the coefficient of `a` specified by
`ntup::NTuple{N,Int}`, multiplied by the corresponding
factorials.
"""
function derivative(ntup::NTuple{N,Int}, a::TaylorN) where {N}

    @assert N == get_numvars() && all(ntup .>= 0)

    c = getcoeff(a, [ntup...])
    for ind = 1:get_numvars()
        c *= factorial(ntup[ind])
    end

    return c
end

## Gradient, jacobian and hessian
"""
```
    gradient(f)
    ∇(f)
```

Compute the gradient of the polynomial `f::TaylorN`.
"""
function gradient(f::TaylorN)
    T = eltype(f)
    numVars = get_numvars()
    grad = Array{TaylorN{T}}(undef, numVars)
    @inbounds for nv = 1:numVars
        grad[nv] = derivative(f, nv)
    end
    return grad
end
const ∇ = TaylorSeries.gradient

"""
```
    jacobian(vf)
    jacobian(vf, [vals])
```

Compute the jacobian matrix of `vf`, a vector of `TaylorN` polynomials,
evaluated at the vector `vals`. If `vals` is ommited, it is evaluated at zero.
"""
function jacobian(vf::Array{TaylorN{T},1}) where {T<:Number}
    numVars = get_numvars()
    @assert length(vf) == numVars
    jac = Array{T}(undef, numVars, numVars)

    @inbounds for comp = 1:numVars
        jac[:,comp] = vf[comp][1][1:end]
    end

    return transpose(jac)
end
function jacobian(vf::Array{TaylorN{T},1}, vals::Array{S,1}) where {T<:Number,S<:Number}
    R = promote_type(T,S)
    numVars = get_numvars()
    @assert length(vf) == numVars == length(vals)
    jac = Array{R}(undef, numVars, numVars)

    for comp = 1:numVars
        @inbounds grad = gradient( vf[comp] )
        @inbounds for nv = 1:numVars
            jac[nv,comp] = evaluate(grad[nv], vals)
        end
    end

    return transpose(jac)
end

function jacobian(vf::Array{Taylor1{TaylorN{T}},1}) where {T<:Number}
    vv = convert(Array{TaylorN{Taylor1{T}},1}, vf)
    jacobian(vv)
end

"""
```
    jacobian!(jac, vf)
    jacobian!(jac, vf, [vals])
```

Compute the jacobian matrix of `vf`, a vector of `TaylorN` polynomials
evaluated at the vector `vals`, and write results to `jac`. If `vals` is ommited,
it is evaluated at zero.
"""
function jacobian!(jac::Array{T,2}, vf::Array{TaylorN{T},1}) where {T<:Number}
    numVars = get_numvars()
    @assert length(vf) == numVars
    @assert (numVars, numVars) == size(jac)
    for comp2 = 1:numVars
        for comp1 = 1:numVars
            @inbounds jac[comp1,comp2] = vf[comp1][1][comp2]
        end
    end
    nothing
end
function jacobian!(jac::Array{T,2}, vf::Array{TaylorN{T},1},
        vals::Array{T,1}) where {T<:Number}

    numVars = get_numvars()
    @assert length(vf) == numVars == length(vals)
    @assert (numVars, numVars) == size(jac)
    for comp = 1:numVars
        @inbounds for nv = 1:numVars
            jac[nv,comp] = evaluate(derivative(vf[nv], comp), vals)
        end
    end
    nothing
end

"""
```
    hessian(f)
    hessian(f, [vals])
```

Return the hessian matrix (jacobian of the gradient) of `f::TaylorN`,
evaluated at the vector `vals`. If `vals` is ommited, it is evaluated at
zero.
"""
hessian(f::TaylorN{T}, vals::Array{S,1}) where {T<:Number,S<:Number} =
    (R = promote_type(T,S); jacobian( gradient(f), vals::Array{R,1}) )

hessian(f::TaylorN{T}) where {T<:Number} = hessian( f, zeros(T, get_numvars()) )

"""
```
    hessian!(hes, f)
    hessian!(hes, f, [vals])
```

Return the hessian matrix (jacobian of the gradient) of `f::TaylorN`,
evaluated at the vector `vals`, and write results to `hes`. If `vals` is
ommited, it is evaluated at zero.
"""
hessian!(hes::Array{T,2}, f::TaylorN{T}, vals::Array{T,1}) where {T<:Number} =
    jacobian!(hes, gradient(f), vals)

hessian!(hes::Array{T,2}, f::TaylorN{T}) where {T<:Number} =
    jacobian!(hes, gradient(f))

##Integration
"""
    integrate(a, r)

Integrate the `a::HomogeneousPolynomial` with respect to the `r`-th
variable. The returned `HomogeneousPolynomial` has no added constant of
integration. If the order of a corresponds to `get_order()`, a zero
`HomogeneousPolynomial` of 0-th order is returned.

"""
function integrate(a::HomogeneousPolynomial, r::Int)
    @assert 1 ≤ r ≤ get_numvars()

    order_max = get_order()
    a.order == order_max && return HomogeneousPolynomial(zero(a[1]/1), 0)

    @inbounds posTb = pos_table[a.order+2]
    @inbounds num_coeffs = size_table[a.order+1]

    T = promote_type(eltype(a), eltype(a[1]/1))
    coeffs = zeros(T, size_table[a.order+2])

    @inbounds for i = 1:num_coeffs
        iind = coeff_table[a.order+1][i]
        n = iind[r]
        n == order_max && continue
        iind[r] += 1
        kdic = in_base(get_order(), iind)
        pos = posTb[kdic]
        coeffs[pos] = a[i] / (n+1)
        iind[r] -= 1
    end

    return HomogeneousPolynomial(coeffs, a.order+1)
end
integrate(a::HomogeneousPolynomial, s::Symbol) = integrate(a, lookupvar(s))

"""
    integrate(a, r, [x0])

Integrate the `a::TaylorN` series with respect to the `r`-th variable,
where `x0` the integration constant and must be independent
of the `r`-th variable; if `x0` is ommitted, it is taken as zero.
"""
function integrate(a::TaylorN, r::Int)
    T = promote_type(eltype(a), eltype(a[0]/1))
    order_max = min(get_order(), a.order+1)
    coeffs = zeros(HomogeneousPolynomial{T}, order_max)

    @inbounds for ord = 0:order_max-1
        coeffs[ord+1] = integrate( a[ord], r)
    end

    return TaylorN(coeffs)
end
function integrate(a::TaylorN, r::Int, x0::TaylorN)
    # Check constant of integration is independent of re
    @assert derivative(x0, r) == 0.0 """
    The integration constant ($x0) must be independent of the
    $(_params_TaylorN_.variable_names[r]) variable"""

    res = integrate(a, r)
    return x0+res
end
integrate(a::TaylorN, r::Int, x0) =
    integrate(a,r,TaylorN(HomogeneousPolynomial([convert(eltype(a),x0)], 0)))

integrate(a::TaylorN, s::Symbol) = integrate(a, lookupvar(s))
integrate(a::TaylorN, s::Symbol, x0::TaylorN) = integrate(a, lookupvar(s), x0)
integrate(a::TaylorN, s::Symbol, x0) = integrate(a, lookupvar(s), x0)


# Load Julia packages (libraries) needed  for the snippets in chapter 0

using StatisticalRethinking
import LogDensityProblems: logdensity, logdensity_and_gradient
import StatisticalRethinking: HMC2, generate_n_samples

# CmdStan uses a tmp directory to store the output of cmdstan

ProjDir = @__DIR__
cd(ProjDir)

# ### snippet 9.3

# Construct the logdensity problem

struct clip_9_3_model{TY <: AbstractVector, TX <: AbstractVector}
    "Observations."
    y::TY
    "Covariate"
    x::TX
end

# Make the type callable with the parameters *as a single argument*.

function (problem:: clip_9_3_model)(θ)
    @unpack y, x, = problem    # extract the data
    @unpack muy, mux = θ     # works on the named tuple too
    ll = 0.0
    ll += loglikelihood(Normal(mux, 1), x)
    ll += loglikelihood(Normal(muy, 1), y)
    ll += logpdf(Normal(0, 1), mux)
    ll += logpdf(Normal(0, 1), muy)
    ll
end

# Instantiate the model with data and inits.

Random.seed!(1234591)

N = 100
x = rand(Normal(0, 1), N)
y = rand(Normal(0, 1), N)

p = clip_9_3_model(y, x)
θ = (muy = 0.0, mux=0.0)
p(θ)

# Write a function to return properly dimensioned transformation.

problem_transformation(p::clip_9_3_model) =
    as((muy = asℝ, mux = asℝ))

# Wrap the problem with a transformation, then use Flux for the gradient.

P = TransformedLogDensity(problem_transformation(p), p)
∇P = ADgradient(:ForwardDiff, P);

# Tune and sample.

chain, NUTS_tuned = NUTS_init_tune_mcmc(∇P, 1000);

# We use the transformation to obtain the posterior from the chain.

posterior = TransformVariables.transform.(Ref(problem_transformation(p)), get_position.(chain));

# Extract the posterior means,

[mean(first, posterior), mean(last, posterior)]

# Draw 200 samples:

function draw_n_samples(model, grad;
  epsilon = 0.03, # Step size
  L = 11, # No of leapfrog steps
  n_samples = 1000, # No of samples
  q = [-0.1, 0.2]) # Initial position

  samples = zeros(n_samples, 2)
  for i in 1:n_samples
    q, ptraj, qtraj, accept, dH = HMC(model, grad, 0.03, 11, q)
    samples[i, :] = q
  end

  samples

end

samples = draw_n_samples(p, ∇P; n_samples=200);
mean(samples, dims=1)

# End of `09/clip-03.jl`


using ITensors
using Zygote

s = siteind("Qubit")

f(x) = op("Ry", s; θ=x)[1, 1]

x = 0.2
@show f(x), cos(x / 2)
@show f'(x), -sin(x / 2) / 2

# Simple gate optimization
ψ0 = state(s, "0")
ψp = state(s, "+")

function loss(x)
  U = op("Ry", s; θ=x)
  Uψ0 = replaceprime(U * ψ0, 1 => 0)
  return -(dag(ψp) * Uψ0)[]
end

# Extremely simple gradient descent implementation,
# where gradients are computing with automatic differentiation
# using Zygote.
function gradient_descent(f, x0; γ, nsteps, grad_tol)
  @show γ, nsteps
  x = x0
  f_x = f(x)
  ∇f_x = f'(x)
  step = 0
  @show step, x, f_x, ∇f_x
  for step in 1:nsteps
    x -= γ * ∇f_x
    f_x = f(x)
    ∇f_x = f'(x)
    @show step, x, f_x, ∇f_x
    if norm(∇f_x) ≤ grad_tol
      break
    end
  end
  return x, f_x, ∇f_x
end

x0 = 0
γ = 2.0 # Learning rate
nsteps = 30 # Number of steps of gradient descent
grad_tol = 1e-4 # Stop if gradient falls below this value
x, loss_x, ∇loss_x = gradient_descent(loss, x0; γ=γ, nsteps=nsteps, grad_tol=grad_tol)

@show x0, loss(x0)
@show x, loss(x)
@show π / 2, loss(π / 2)


using LinearAlgebra

"""
Performs a random hill climb solution for a discretized search space.

    Args:
      fn (Function): Objective function
      x0 (Array{Any}): Starting point
      h (Float64, optional): Discretization
      max_failed (Int64, optional): Max number of failed moves before exiting.
        Defaults to the dimensionality of the design space.
      verbose (Bool, optional): Optionally print results

    Returns:
      tuple: (x, fx, f_calls) where x is the minimizer, fx is the minimum, and
      f_calls is the number of function calls made.
"""
function random_hill(fn::Function, x0::Array{Float64};
                     h=.1, max_failed=nothing, verbose=false)
    x = copy(x0)
    fx = fn(x)
    n = length(x)
    failed = 0
    f_calls = 1

    # use an I matrix to randomly determine direction
    e_hat = Matrix(1I, n, n)

    if isnothing(max_failed)
        max_failed = n
    end

    while failed < max_failed
        d = e_hat[:, rand(1:n)] # randomly choose basis vec
        d *= rand([-1, 1]) # randomly choose pos/neg

        xnew = x + h*d
        fnew = fn(xnew)
        f_calls += 1

        if fnew < fx
            fx = fnew
            x = xnew
            failed = 0
        else
            failed += 1
        end
    end

    if verbose
        println("Iterations: $(f_calls)\nx*: $(x)\nf*: $(fx)\n")
    end

    return x, fx, f_calls
end


# Functions for STaylor1
@generated function exp(a::STaylor1{N,T}) where {N, T <: Number}
    ex_calc = quote end
    append!(ex_calc.args, Any[nothing for i in 1:N])
    syms = Symbol[Symbol("c$i") for i in 1:N]

    sym = syms[1]
    ex_line = :($(syms[1]) = exp(a[0]))
    ex_calc.args[1] = ex_line

    for k in 1:(N-1)
        kT = convert(T,k)
        sym = syms[k+1]
        ex_line = :($kT * a[$k] * $(syms[1]))
        @inbounds for i = 1:k-1
            ex_line = :($ex_line + $(kT-i) * a[$(k-i)] * $(syms[i+1]))
        end
        ex_line = :(($ex_line)/$kT)
        ex_line = :($sym = $ex_line)
        ex_calc.args[k+1] = ex_line
    end

    exout = :(($(syms[1]),))
    for i = 2:N
        push!(exout.args, syms[i])
    end
    return quote
               Base.@_inline_meta
               $ex_calc
               return STaylor1{N,T}($exout)
            end
end

@generated function log(a::STaylor1{N,T}) where {N, T <: Number}
    ex_calc = quote end
    append!(ex_calc.args, Any[nothing for i in 1:N])
    syms = Symbol[Symbol("c$i") for i in 1:N]

    (N >= 1) && (ex_calc.args[1] = :($(syms[1]) = log(constant_term(a))))
    (N >= 2) && (ex_calc.args[2] = :($(syms[2]) = a[1]/constant_term(a)))

    for k in 2:(N-1)
        ex_line = :($(k-1)*a[1]*$(syms[k]))
        @inbounds for i = 2:k-1
            ex_line = :($ex_line + $(k-i)*a[$i] * $(syms[k+1-i]))
        end
        ex_line = :((a[$k] - ($ex_line)/$(convert(T,k)))/constant_term(a))
        ex_line = :($(syms[k+1]) = $ex_line)
        ex_calc.args[k+1] = ex_line
    end

    exout = :(($(syms[1]),))
    for i = 2:N
        push!(exout.args, syms[i])
    end

    return quote
               Base.@_inline_meta
               iszero(constant_term(a)) && throw(ArgumentError("""
                       The 0-th order `STaylor1` coefficient must be non-zero
                       in order to expand `log` around 0.
                       """))
               $ex_calc
               return STaylor1{N,T}($exout)
            end
end

sin(a::STaylor1{N,T}) where {N, T <: Number} = sincos(a)[1]
cos(a::STaylor1{N,T}) where {N, T <: Number} = sincos(a)[2]
@generated function sincos(a::STaylor1{N,T}) where {N, T <: Number}

    ex_calc = quote end
    append!(ex_calc.args, Any[nothing for i in 1:(2*N)])

    syms_s = Symbol[Symbol("c$i") for i in 1:N]
    syms_c = Symbol[Symbol("c2$i") for i in 1:N]

    ex_line_s = :($(syms_s[1]) = sin(a[0]))
    ex_line_c = :($(syms_c[1]) = cos(a[0]))
    ex_calc.args[1] = ex_line_s
    ex_calc.args[2] = ex_line_c

    for k = 1:(N - 1)
        ex_line_s = :(a[1]*$(syms_c[k]))
        ex_line_c = :(-a[1]*$(syms_s[k]))
        for i = 2:k
            ex_line_s = :($ex_line_s + $i*a[$i]*$(syms_c[(k - i + 1)]))
            ex_line_c = :($ex_line_c - $i*a[$i]*$(syms_s[(k - i + 1)]))
        end
        ex_line_s = :($(syms_s[k + 1]) = ($ex_line_s)/$k)
        ex_line_c = :($(syms_c[k + 1]) = ($ex_line_c)/$k)
        ex_calc.args[2*k + 1] = ex_line_s
        ex_calc.args[2*k + 2] = ex_line_c
    end

    exout_s = :(($(syms_s[1]),))
    for i = 2:N
        push!(exout_s.args, syms_s[i])
    end

    exout_c = :(($(syms_c[1]),))
    for i = 2:N
        push!(exout_c.args, syms_c[i])
    end

    return quote
               Base.@_inline_meta
               $ex_calc
               return STaylor1{N,T}($exout_s), STaylor1{N,T}($exout_c)
            end
end

# Functions for STaylor1
#=
@generated function tan(a::STaylor1{N,T}) where {N, T <: Number}

    ex_calc = quote end
    append!(ex_calc.args, Any[nothing for i in 1:(4*N)])
    syms = Symbol[Symbol("c$i") for i in 1:N]
    syms2 = Symbol[Symbol("c2$i") for i in 1:N]

    for i = 1:N
        ex_calc.args[i] = :($(syms2[i]) = 0.0)
    end

    ex_line_c = :($(syms[1]) = tan(a[0]))
    ex_line_c2 = :($(syms2[1]) = ($(syms[1]))^2)
    ex_calc.args[N + 1] = ex_line_c
    ex_calc.args[N + 2] = ex_line_c2

    for k = 1:(N - 1)

        kodd = k%2
        kend = div(k - 2 + kodd, 2)
        kdiv2 = div(k, 2)

        ex_line_c = :($(k-1)*a[$(k-1)]*$(syms2[2]))
        for i = 1:(k - 1)
            q = k - i
            ex_line_c = :($ex_line_c + ($q)*a[$q]*$(syms2[i + 1]))
        end
        ex_line_c = :(a[$k] + ($ex_line_c)/$k)
        ex_line_c = :($(syms[k + 1]) = $ex_line_c)
        ex_calc.args[(3*k-2) + N + 2] = ex_line_c

        ex_line_c2 = :(a[0]*a[$k])
        for i = 1:kend
            ex_line_c2 = :($ex_line_c2 + a[$i]*a[$k - $i])
        end
        ex_line_c2 = :(2*$ex_line_c2)

        if kodd != 1
            ex_line_c2 = :($ex_line_c2 + a[$kdiv2]^2)
        end

        ex_line_c2 = :($(syms2[k + 1]) = $ex_line_c2)
        ex_calc.args[(3*k-1) + 2 + N] = ex_line_c2
        blar = k + 1
        blar_str = "c$(blar) "
        ex_calc.args[3*k + N + 2] = :(println($blar_str*string($(syms[k + 1]))))
    end

    exout = :(($(syms[1]),))
    for i = 2:N
        push!(exout.args, syms[i])
    end

    return quote
               Base.@_inline_meta
               $ex_calc
               return STaylor1{N,T}($exout)
            end
end
=#

# Functions for STaylor1
#=
@generated function asin(a::STaylor1{N,T}) where {N, T <: Number}
    ex_calc = quote end
    append!(ex_calc.args, Any[nothing for i in 1:N])
    syms = Symbol[Symbol("c$i") for i in 1:N]

    sym = syms[1]
    ex_line = :($(sym) = asin(a[0]))
    ex_calc.args[1] = ex_line

    #=
    for k in 1:(N-1)
        kT = convert(T,k)
        sym = syms[k+1]
        ex_line = :($kT * a[$k] * $(syms[1]))
        @inbounds for i = 1:k-1
            ex_line = :($ex_line + $(kT-i) * a[$(k-i)] * $(syms[i+1]))
        end
        ex_line = :(($ex_line)/$kT)
        ex_line = :($sym = $ex_line)
        ex_calc.args[k+1] = ex_line
    end
    =#

    exout = :(($(syms[1]),))
    for i = 2:N
        push!(exout.args, syms[i])
    end
    return quote
               Base.@_inline_meta
               $ex_calc
               return STaylor1{N,T}($exout)
            end
end
=#

# Functions for STaylor1
#=
@generated function acos(a::STaylor1{N,T}) where {N, T <: Number}
    ex_calc = quote end
    append!(ex_calc.args, Any[nothing for i in 1:N])
    syms = Symbol[Symbol("c$i") for i in 1:N]

    sym = syms[1]
    ex_line = :($(sym) = acos(a[0]))
    ex_calc.args[1] = ex_line

    #=
    for k in 1:(N-1)
        kT = convert(T,k)
        sym = syms[k+1]
        ex_line = :($kT * a[$k] * $(syms[1]))
        @inbounds for i = 1:k-1
            ex_line = :($ex_line + $(kT-i) * a[$(k-i)] * $(syms[i+1]))
        end
        ex_line = :(($ex_line)/$kT)
        ex_line = :($sym = $ex_line)
        ex_calc.args[k+1] = ex_line
    end
    =#

    exout = :(($(syms[1]),))
    for i = 2:N
        push!(exout.args, syms[i])
    end
    return quote
               Base.@_inline_meta
               $ex_calc
               return STaylor1{N,T}($exout)
            end
end
=#

# Functions for STaylor1
#=
@generated function atan(a::STaylor1{N,T}) where {N, T <: Number}
    ex_calc = quote end
    append!(ex_calc.args, Any[nothing for i in 1:N])
    syms = Symbol[Symbol("c$i") for i in 1:N]

    sym = syms[1]
    ex_line = :($(sym) = atan(a[0]))
    ex_calc.args[1] = ex_line

    #=
    for k in 1:(N-1)
        kT = convert(T,k)
        sym = syms[k+1]
        ex_line = :($kT * a[$k] * $(syms[1]))
        @inbounds for i = 1:k-1
            ex_line = :($ex_line + $(kT-i) * a[$(k-i)] * $(syms[i+1]))
        end
        ex_line = :(($ex_line)/$kT)
        ex_line = :($sym = $ex_line)
        ex_calc.args[k+1] = ex_line
    end
    =#

    exout = :(($(syms[1]),))
    for i = 2:N
        push!(exout.args, syms[i])
    end
    return quote
               Base.@_inline_meta
               $ex_calc
               return STaylor1{N,T}($exout)
            end
end
=#

sinh(a::STaylor1{N,T}) where {N, T <: Number} = sinhcosh(a)[1]
cosh(a::STaylor1{N,T}) where {N, T <: Number} = sinhcosh(a)[2]
@generated function sinhcosh(a::STaylor1{N,T}) where {N, T <: Number}

    ex_calc = quote end
    append!(ex_calc.args, Any[nothing for i in 1:(2*N)])

    syms_s = Symbol[Symbol("c$i") for i in 1:N]
    syms_c = Symbol[Symbol("c2$i") for i in 1:N]

    ex_line_s = :($(syms_s[1]) = sinh(a[0]))
    ex_line_c = :($(syms_c[1]) = cosh(a[0]))
    ex_calc.args[1] = ex_line_s
    ex_calc.args[2] = ex_line_c

    for k = 1:(N - 1)
        ex_line_s = :(a[1]*$(syms_c[k]))
        ex_line_c = :(a[1]*$(syms_s[k]))
        for i = 2:k
            ex_line_s = :($ex_line_s + $i*a[$i]*$(syms_c[(k - i + 1)]))
            ex_line_c = :($ex_line_c + $i*a[$i]*$(syms_s[(k - i + 1)]))
        end
        ex_line_s = :($(syms_s[k + 1]) = ($ex_line_s)/$k)
        ex_line_c = :($(syms_c[k + 1]) = ($ex_line_c)/$k)
        ex_calc.args[2*k + 1] = ex_line_s
        ex_calc.args[2*k + 2] = ex_line_c
    end

    exout_s = :(($(syms_s[1]),))
    for i = 2:N
        push!(exout_s.args, syms_s[i])
    end

    exout_c = :(($(syms_c[1]),))
    for i = 2:N
        push!(exout_c.args, syms_c[i])
    end

    return quote
               Base.@_inline_meta
               $ex_calc
               return STaylor1{N,T}($exout_s), STaylor1{N,T}($exout_c)
            end
end

# Functions for STaylor1
@generated function tanh(a::STaylor1{N,T}) where {N, T <: Number}

    ex_calc = quote end
    append!(ex_calc.args, Any[nothing for i in 1:(2*N)])
    syms = Symbol[Symbol("c$i") for i in 1:N]
    syms2 = Symbol[Symbol("c2$i") for i in 1:N]

    ex_line_c = :($(syms[1]) = tanh(a[0]))
    ex_line_c2 = :($(syms2[1]) = ($(syms[1]))^2)
    ex_calc.args[1] = ex_line_c
    ex_calc.args[2] = ex_line_c2

    for k = 1:(N - 1)

        kodd = k%2
        kend = div(k - 2 + kodd, 2)
        kdiv2 = div(k, 2)

        ex_line_c = :($k*a[$k]*$(syms2[1]))
        for i = 1:(k - 1)
            q = k - i
            ex_line_c = :($ex_line_c + ($q)*a[$q]*$(syms2[i+1]))
        end
        ex_line_c = :(a[$k] - $ex_line_c/$k)
        ex_line_c = :($(syms[k + 1]) = $ex_line_c)
        ex_calc.args[2*k + 1] = ex_line_c

        ex_line_c2 = :(a[1]*a[$k])
        @inbounds for i = 1:kend
            ex_line_c2 = :($ex_line_c2 + a[$i + 1]*a[$k - $i])
        end
        ex_line_c2 = :(2*$ex_line_c2)

        if kodd != 1
            ex_line_c2 = :($ex_line_c2 + a[$kdiv2 + 1]^2)
        end

        ex_line_c2 = :($(syms2[k + 1]) = $ex_line_c2)
        ex_calc.args[2*k + 2] = ex_line_c2
    end

    exout = :(($(syms[1]),))
    for i = 2:N
        push!(exout.args, syms[i])
    end

    return quote
               Base.@_inline_meta
               $ex_calc
               return STaylor1{N,T}($exout)
            end
end


using SpikeStats
using Test

testset = poissonSpikes(10,100,0.001)

println(names(SpikeStats))

@testset "SpikeStats.jl" begin
    @test typeof(testset) == Array{Float64,1}
    @test round(computeCV(diff(testset)), digits=2) == 0.93
    @test round(computeFano(testset,0.001), digits=2) == 0.90
end


# p24.jl - pseudospectra of Davies's complex harmonic oscillator
#         (For finer, slower plot, change 0:2 to 0:.5.)

# Eigenvalues:
N = 70; (D,x) = cheb(N); x = x[2:N];
L = 6; x = L*x; D = D/L;                   # rescale to [-L,L]
A = -D^2; A = A[2:N,2:N] + (1+3im)*diagm(x.^2);
lambda = eigvals(A);
clf(); plot(real(lambda),imag(lambda),"k.",markersize=6);
axis([0,50,0,40])

# Pseudospectra:
x = 0:1:50; y = 0:1:40;  zz = x' .+ 1im*y;
I = eye(N-1);
minsvd(z) = minimum(svdvals(z*I-A));
sigmin = [ minsvd(x[i]+1im*y[j]) for i=eachindex(x), j=eachindex(y) ];
contour(x,y,sigmin',levels=10.0.^(-4:.5:-.5));


function initial_velocity(N::Int64;T=298.15,R=8.3145,m=1.0)
  vel = [ [0.,0.] for i in 1:N]

  for i in 1:N
    angle = 2*pi*rand()
    px = rand()
    py = rand()
    vel[i][1] = sqrt(-(2*R*T/m)*log(1-px)) * angle       # velocity in the x axis
    vel[i][2] = sqrt(-(2*R*T/m)*log(1-py)) * angle       # velocity in the y axis
  end

  return vel
end

function norm(x::Vector{Float64})
  s = 0.
  for i in 1:length(x)
    s = s + x[i]^2
  end
  sqrt(s)
end

function normVEL(x)
  s = zeros(length(x))
  for i in 1:length(x)
    s[i] = norm(x[i])
  end
  return s
end

export initial_velocity, norm, normVEL

# p(v)dv
p(v;m=1.,R=8.3145,T=298.15) = m/(R*T) * v * exp(-m*v^2/(2R*T))

# probability v < x
pv(x;m = 1., T = 298.15, R = 8.3145) = -exp(-m*x^2/(2*R*T)) + 1


using jInv.Mesh
using jInv.LinearSolvers
using SparseArrays
using LinearAlgebra
using Test
println("===  Test Julia wrapper with symmetric (real) matrix ====");
domain = [0.0, 1.0, 0.0, 1.0];
n      = [33,17];
Mr     = getRegularMesh(domain,n)
G      = getNodalGradientMatrix(Mr);
m      = sparse(Diagonal(exp.(randn(size(G,1)))));
Ar     = G'*m*G;
Ar     = Ar + 1e-1*norm(Ar,1)*sparse(1.0I,size(Ar,2),size(Ar,2));
N      = size(Ar,2);
b      = Ar*rand(N,1);
bs     = Ar*rand(N,3);
Bs     = (b,sparse(b),bs,sparse(bs));

sNonSym = getJuliaSolver(sym = 0);
sSym    = getJuliaSolver(sym = 1);
sGenSym = getJuliaSolver(sym = 2);
sols    = (sNonSym,sSym,sGenSym)

for j=1:length(Bs)
	print("nrhs=$(size(Bs[j],2)), issparse(rhs)=$(issparse(Bs[j])) : ")
	for k=1:length(sols)
		print("sym=$(sols[k].sym), ")
		xtt1,  = solveLinearSystem(Ar,Bs[j],sols[k]);
		@test norm(Ar*xtt1-Bs[j],Inf)/norm(Bs[j],Inf) < 1e-10
	end
	print("\n")
end
println("\n")

println("===  Test Julia wrapper with shifted (complex) Laplacian ====");
Ar     = G'*m*G +  (1+1im)*sparse(1.0I,size(Ar,2),size(Ar,2));
b      = Ar*rand(N,1);
bs     = Ar*rand(N,3);
Bs     = (b,sparse(b),bs,sparse(bs));

sNonSym = getJuliaSolver(sym = 0);
sSym    = getJuliaSolver(sym = 1);
sGenSym = getJuliaSolver(sym = 2);
sols    = (sNonSym,sSym,sGenSym)

for j=1:length(Bs)
	print("nrhs=$(size(Bs[j],2)), issparse(rhs)=$(issparse(Bs[j])) : ")
	for k=1:length(sols)
		print("sym=$(sols[k].sym), ")
		xtt2,  = solveLinearSystem(Ar,Bs[j],sols[k]);
		@test norm(Ar*xtt2-Bs[j],Inf)/norm(Bs[j],Inf) < 1e-10
	end
	print("\n")
end
println("\n")

println("===  Test Julia Wrapper: nonsymmetric matrices ====");
n = 50
sNonSym  = getJuliaSolver(sym = 0);
sNonSym  = copySolver(sNonSym);
A = sprandn(n,n,5/n) + 10*sparse(1.0I,n,n)
B = randn(n)

X, = solveLinearSystem(A,B,sNonSym,0);
@test norm(A*X - B)/norm(B) < 1e-10

# multiple sparse rhs -
# Bs = sprandn(n,10,.3)
# X, = solveLinearSystem(A,Bs,sNonSym,0);
# @test norm(A*X - Bs,Inf)/norm(Bs,Inf) < 1e-10

X, = solveLinearSystem(A,B,sNonSym,1);
@test norm(A'*X - B)/norm(B) < 1e-10

X, = solveLinearSystem(A,B,sNonSym,1);
@test norm(A'*X - B)/norm(B) < 1e-10

X, = solveLinearSystem(A,B,sNonSym,0);
@test norm(A*X - B)/norm(B) < 1e-10

println("===  End Test Julia Wrapper ====");


module BVP1D

using LinearAlgebra
using OffsetArrays

export solve_bvp

function solve_bvp(L, f, γ0, γL, P)
    Δx = L / P
    c = 1 / Δx^2
    dv = fill(2c, P-1)
    ev = fill(-c, P-1)
    A = SymTridiagonal(dv, ev)
    x_ = range(0; length=P+1, stop=L)
    x = OffsetArray(x_, 0:P)
    b = f.(x[1:P-1])
    b[1]   += c * γ0
    b[P-1] += c * γL
    U = OffsetArray{Float64}(undef, 0:P)
    U[0] = γ0
    U[1:P-1] = A \ b
    U[P] = γL
    return U, x
end

end # module BVP1D


# Implementation of Blendenpik with Gaussian row mixing for the solution of least squares
# problem ||Ax - b||₂ where A has full column rank.
#
# This method for other row mixing strategies is described in
#
# Avron, Haim, Petar Maymounkov, and Sivan Toledo. "Blendenpik: Supercharging LAPACK's
# least-squares solver." SIAM Journal on Scientific Computing 32, no. 3 (2010): 1217-1236.
#
# January 2021

"""
    blendenpick_gauss(A, b; r)

Solves the least squares problem with coefficient `A` and constant `b`, where `A` has full
column rank, using the blendenpick method with Gaussian row mixing. The number of sampled
rows, `r`, defaults to the number of columns.
"""
function blendenpick_gauss!(
    x::Vector{T},
    A::Matrix{T},  # Coefficient matrix of system
    b::Vector{T};  # Constant vector of system
    r::Int=size(A, 2) + 0,  # Size of row sample
    verbose::Bool=false  #Show stats from lsqr solver
) where T <: Real
    m = size(A, 1)              # Number of rows in A

    # Mix rows of a A with Gaussians to generate r by size(A, 2) matrix
    A_mixed = randn(r, m) * A

    # Generate preconditioner using R⁻ factor of qr decomposition of mixed matrix
    _, R = qr(A_mixed)
    Rinv = R \ I

    # Run lsqr on transformed systems
    y, stats = lsqr(A * Rinv, b)

    verbose && show(stats)

    # Recover and return solution to original system
    x = Rinv * y
end


using Base.Test
using TimeIt
import NestedSampling: logaddexp, nball_vol_factor, sample

# Test logaddexp()
x = log(1.e-100)
y = log(2.e-100)
@test_approx_eq exp(logaddexp(x, y)) 3.e-100

# Test volume prefactor for an N-ball
@test_approx_eq nball_vol_factor(4) pi^2 / 2
@test_approx_eq nball_vol_factor(5) 8pi^2/15
@test_approx_eq nball_vol_factor(6) pi^3 / 6
@test_approx_eq nball_vol_factor(7) 16pi^3/105

# -----------------------------------------------------------------------------
# Simple likelihood function

    # gaussians centered at (1, 1) and (-1, -1) with a width of 0.1.
const mu1 = [1., 1.]
const mu2 = [-1., -1.]
const sigma = 0.1
const invvar = eye(2) / sigma^2

function logl(x)
    dx1 = x .- mu1
    dx2 = x .- mu2
    return logaddexp((dx1' * invvar * dx1)[1] / 2.0,
                     (dx2' * invvar * dx2)[1] / 2.0)
end

# Use a flat prior, over [-5, 5] in both dimensions
prior(x) = 10.0 .* x .- 5.0

srand(0)
res = sample(logl, prior, 2; npoints=100)
@printf "evidence = %6.3f +/- %6.3f\n" res["logz"] res["logzerr"]

#(Approximate) analytic evidence for two identical Gaussian blobs,
# over a uniform prior [-5:5][-5:5] with density 1/100 in this domain:
analytic_logz = log(2.0 * 2.0*pi*sigma*sigma / 100.)
@printf "analytic = %6.3f\n" analytic_logz

    # calculate evidence on fine grid.
    #dx = 0.1
    #xv = np.arange(-5.0 + dx/2., 5., dx)
    #yv = np.arange(-5.0 + dx/2., 5., dx)
    #grid_logz = -1.e300
    #for x in xv:
    #    for y in yv:
    #        grid_logz = np.logaddexp(grid_logz, logl(np.array([x, y])))
    #grid_logz += np.log(dx * dx / 100.)  # adjust for point density
    #print "grid_logz =", grid_logz

    #assert abs(res.logz - analytic_logz) < 2.0 * res.logzerr
    #assert abs(res.logz - grid_logz) < 2.0 * res.logzerr