function serial_pi_estimation(n_samples)
    n_inside = 0;

    x = zeros(2);

    for _ in 1:n_samples
        @. x = rand();
        if(norm(x,2)<1)
            n_inside+=1;
        end
    end

    pi_estimate = 4 * n_inside/n_samples;

    return pi_estimate

end

function pi_estimation(n_samples)
    n_inside = 0;

    x_inside = zeros(Bool, n_samples);

    x = zeros(2);
    Threads.@threads for j in 1:n_samples
        @. x = rand();
        if(norm(x,2)<1)
            x_inside[j] = 1;
        end
    end

    n_inside = sum(x_inside);

    pi_estimate = 4 * n_inside/n_samples;

    return pi_estimate

end

function atomic_pi_estimation(n_samples)

    n_inside = Atomic{Int}(0);

    x = zeros(2);

    Threads.@threads for _ in 1:n_samples
        @. x = rand();
        if(norm(x,2)<1)
            atomic_add!(n_inside,1);
        end
    end

    pi_estimate = 4 * n_inside[]/n_samples;

    return pi_estimate

end


using Pkg
Pkg.activate(pwd())

# # Functions

function plus(x,y)
    x + y
end

#+

plus(2, 3)
plus(2, -3)

#+

function plus(x,y)
    return x + y
end

#+

function plus(x, y)
    return x + y
    println("I am a useless line of code!!")
end

#+

plus(4, 5)
plus(3, -5)

#+

function powers(x)
    return x, x^2, x^3, x^4
end

#+

ps = powers(2)
typeof(ps)

#+

x1, x2, x3, x4 = powers(2)
x3

# ### Exercise:
# Write function `power(x::Real, p::Integer)` that for a number $x$ and a (possibly
# negative) integer $p$ computes $x^p$ without using the `^` operator. Use only basic
# arithmetic operators `+`, `-`, `*`, `/` and the `if` condition. The annotation
# `p::Integer` ensures that the input `p` is always an integer.
#
# **Hint:** use recursion.
#
# ---
# ### Solution:

# ---
#
# ## One-line functions

plus(x, y) = x + y

#+

plus(4, 5)
plus(3, -5)

#+

f(φ) = -4sin(φ - π/12)

#+

g(x) = (x -= 1; x *= 2; x)

#+

g(3)

# ### Exercise:
# Write a one-line function that returns `true` if the input argument is an even number and
# `false` otherwise.
#
# **Hint:** use modulo function and [ternary operator](@ref Ternary-operator) `?`.
#
# ---
# ### Solution:

# ---
#
# ## Optional arguments

hello(x = "world") = println("Hello $(x).")

#+

hello()
hello("people")

#+

powers(x, y = x*x, z = y*x, v = z*x) = x, y, z, v

#+

powers(2)
powers(2, 3)

#+

f(x = 1, y = x) = (x, y)
g(x = y, y = 1) = (x, y)

#+

f()
g()

# ### Exercise:
# Write a function which computes the value of the following quadratic form
# $$
# q_{a,b,c}(x,y) = ax^2 + bxy + cy^2,
# $$
# where $a, b, c, x \in \mathbb{R}$. Use optional arguments to set default values for
# parameters
# $$
# a = 1, \quad b = 2a, \quad c = 3(a + b).
# $$
# What is the function value at point $(4, 2)$ for default parameters? What is the
# function value at the same point if we use $c = 3$?
#
# ---
# ### Solution:

# ---
#
# ## Keyword arguments

linear(x; a = 1, b = 0) = a*x + b

#+

linear(2)
linear(2; a = 2)
linear(2; b = 4)
linear(2; a = 2, b = 4)

#+

linear(b = 4, 2, a = 2) # If you use this, you will burn in hell 🔥🔥🔥🔥

#+

a, b = 2, 4

linear(2; a = a, b = b)
linear(2; a, b)

# ### Exercise:
# Write a probability density function for the
# [Gaussian distribution](https://en.wikipedia.org/wiki/Normal_distribution)
# $$
# f_{\mu, \sigma}(x) = \frac{1}{\sigma \sqrt{ 2\pi }} \exp\left\{ -\frac{1}{2} \left(
# \frac{x - \mu}{\sigma} \right) ^2 \right\},
# $$
# where $\mu \in \mathbb{R}$ and $\sigma^2 > 0$. Use keyword arguments to obtain the
# standardized normal distribution ($\mu = 0$ and $\sigma = 1$). Check that the inputs
# are correct.
#
# Bonus: verify that this function is a probability density function, i.e., its integral
# equals 1.
#
# ---
# ### Solution:

# ---
#
# ## Variable number of arguments

nargs(x...) = println("Number of arguments: ", length(x))

#+

nargs()
nargs(1, 2, "a", :b, [1,2,3])

#+

args = (1, 2, 3)

nargs(args[1], args[2], args[3])
nargs(args...)
nargs(args)

#+

nargs(1:100)
nargs(1:100...)

#+

nargs([1,2,3,4,5])
nargs([1,2,3,4,5]...)

#+

roundmod(x, y; kwargs...) = round(mod(x, y); kwargs...)

#+

roundmod(12.529, 5)
roundmod(12.529, 5; digits = 2)
roundmod(12.529, 5; sigdigits = 2)

# ### Exercise:
# Write a function `wrapper`, that accepts a number and applies one of `round`, `ceil` or
# `floor` functions based on the keyword argument `type`. Use the function to solve the
# following tasks:
# - Round `1252.1518` to the nearest larger integer and convert the resulting value
# to `Int64`.
# - Round `1252.1518` to the nearest smaller integer and convert the resulting value
# to `Int16`.
# - Round `1252.1518` to `2` digits after the decimal point.
# - Round `1252.1518` to `3` significant digits.
#
# ---
# ### Solution:

# ---
#
# ## Anonymous functions

h1 = function (x)
    x^2 + 2x - 1
end

#+

h2 = x ->  x^2 + 2x - 1

#+

using Plots

f(x,a) = (x + a)^2
plot(-1:0.01:1, x -> f(x,0.5))

#+

map(x -> x^2 + 2x - 1, [1,3,-1])

#+

map([1,3,-1], [2,4,-2]) do x, y
    println("x = $(x), y = $(y)")
    return x + y
end

#+

function f(x, y)
    println("x = $(x), y = $(y)")
    return x + y
end

map(f, [1,3,-1], [2,4,-2])

# ## Dot syntax for vectorizing functions

x = [0, π/2, 3π/4];
A = zeros(length(x));

for (i, xi) in enumerate(x)
    A[i] = sin(xi)
end

A

#+

A = [sin(xi) for xi in x]

#+

A = sin.(x)

#+

plus(x::Real, y::Real) = x + y

#+

plus(1,3)
plus(1.4,2.7)

#+

x = [1,2,3,4]; # column vector

plus(x, x)
plus.(x, x)

#+

broadcast(plus, x, x)

#+

y = [1 2 3 4]; # row vector

plus.(x, y)
plus.(x, 1)

# ## Function composition and piping

(sqrt ∘ +)(3, 6) # equivalent to sqrt(3 + 6)
(sqrt ∘ abs ∘ sum)([-3, -6, -7])  # equivalent to sqrt(abs(sum([-3, -6, -7])))

#+

[-3, -6, -7] |> sum |> abs |> sqrt
[-4, 9, -16] .|> abs .|> sqrt

#+s

["a", "list", "of", "strings"] .|> [uppercase, reverse, titlecase, length]


"""

```
centerofcoordinates(coor::AbstractVector{T}) where T
```

Computes the center of coordinates of a vector.

"""
function centerofcoordinates(coor::AbstractVector{T}) where T
  cm = zeros(T)
  n = length(coor)
  for i in 1:n
    cm = cm + coor[i]
  end
  cm = cm / n
  return cm
end


# # 205: 2D Nonlinear Poisson equation
# ([source code](SOURCE_URL))

module Example205_NonlinearPoisson2D

using Printf
using VoronoiFVM
using ExtendableGrids
using ExtendableSparse
using GridVisualize

function main(;n=10,Plotter=nothing,verbose=false, unknown_storage=:sparse, max_lureuse=0,
              factorization=LUFactorization())
    h=1.0/convert(Float64,n)
    X=collect(0.0:h:1.0)
    Y=collect(0.0:h:1.0)

    grid=VoronoiFVM.Grid(X,Y)

    eps=1.0e-2

    physics=VoronoiFVM.Physics(
        num_species=1,
        reaction=function(f,u,node)
        f[1]=u[1]^2
        end,

        flux=function(f,u0,edge)
        u=unknowns(edge,u0)
        f[1]=eps*(u[1,1]^2-u[1,2]^2)
        end,

        source=function(f,node)
        x1=node[1]-0.5
        x2=node[2]-0.5
        f[1]=exp(-20.0*(x1^2+x2^2))
        end,

        storage=function(f,u,node)
        f[1]=u[1]
        end)
    sys=VoronoiFVM.System(grid,physics,unknown_storage=unknown_storage)
    enable_species!(sys,1,[1])

    boundary_dirichlet!(sys,1,2,0.1)
    boundary_dirichlet!(sys,1,4,0.1)

    inival=unknowns(sys)
    U=unknowns(sys)
    inival.=0.5

    control=VoronoiFVM.NewtonControl()
    control.verbose=verbose
    control.tol_linear=1.0e-5
    control.max_lureuse=max_lureuse
    control.factorization=factorization
    tstep=0.01
    time=0.0
    u15=0
    p=GridVisualizer(Plotter=Plotter)
    while time<1.0
        time=time+tstep
        solve!(U,inival,sys,control=control,tstep=tstep)
        u15=U[15]
        inival.=U

        if verbose
            @printf("time=%g\n",time)
        end

        scalarplot!(p[1,1],grid,U[1,:],Plotter=Plotter,clear=true,show=true)
        tstep*=1.0
    end
    return u15
end

function test()
    # test at once for iterative solution here
    testval=0.3554284760906605
    main(unknown_storage=:sparse,max_lureuse=0) ≈  testval &&
        main(unknown_storage=:dense,max_lureuse=0) ≈ testval &&
        main(unknown_storage=:sparse,max_lureuse=10) ≈ testval &&
        main(unknown_storage=:dense,max_lureuse=10) ≈ testval &&
        main(unknown_storage=:sparse,max_lureuse=0, factorization=ILU0Preconditioner()) ≈ testval &&
        main(unknown_storage=:dense,max_lureuse=0, factorization=ILU0Preconditioner()) ≈ testval
end
end


# See https://en.wikipedia.org/wiki/Derivative
# Examples from https://github.com/JuliaSymbolics/Symbolics.jl

using Symbolics

@variables x y z

expressions = [
    :(Symbolics.derivative(x + x^2, x))
    :(Symbolics.derivative(x + 2x^2 + 6x^3 - 5x^4, x))
    :(Symbolics.derivative(cos(x) * sin(x), x))
    :(Symbolics.derivative(exp(x), x))
    :(Symbolics.derivative(log(x), x))
    :(Symbolics.jacobian([x + x*y, x^4 + y], [x, y]))
    :(Symbolics.hessian(x^4 + z, [x, y, z]))
]

for expr in expressions
    println("[] :julia-answer \"\"\"", escape_string(string(expr)), " = ", eval(expr), "\"\"\".")
end


using CSV, DataFrames, StatsPlots, StanSample
using Distributions, Statistics, Random

ProjDir = @__DIR__

# Simulate the data

Random.seed!(123)

N = 15
obspairs = ((30.0, 53), (35.0, 45), (40.0, 28), (45.0, 26), (50.0, 25))

ppu = []; quantity = []
for i in 1:N in
	obs = rand(obspairs, 1)
	append!(ppu, [rand(Normal(obs[1][1], 1.0))])
	append!(quantity, [rand(Normal(obs[1][2], 1.0))])
end

df = DataFrame()
df[!, :ppu] = ppu
df[!, :quantity] = quantity
df[!, :ppu_l] = log.(ppu)
df[!, :quantity_l] = log.(quantity)

CSV.write(joinpath(ProjDir, "data", "data.csv"), df)

opt_01 = "
data {
	int<lower=0> N;
	vector[N] q; // Outcome
	vector[N] p; // Predictor
}
parameters {
	real a;
	real b;
	real<lower=0> sigma;
}
model {
	vector[N] mu;            		// mu is a vector
	a ~ cauchy(0.0, 5.0);			// intercept positive
	b ~ cauchy(0.0, 5.0);			// demand drops as price increases
	sigma ~ uniform(0, 15);
	mu = a + b * (p - mean(p));
	q ~ normal(mu, sigma);
}
";

mean_q_l = mean(df[:, :quantity_l])
mean_p_l = mean(df[:, :ppu_l])
data = Dict(
  :N => size(df, 1),
  :q => df[:, :quantity_l], # .- mean_q_l,
  :p => df[:, :ppu_l]		# .- mean_p_l
)

p1 = scatter(df[:, :ppu], df[:, :quantity],
	xlabel="ppu", ylabel="quantity sold", leg=false)
p2 = scatter(data[:p], data[:q],
	xlabel="log(ppu)", ylabel="log(quantity sold)", leg=false)
plot(p1, p2, layout=(2, 1))
savefig(joinpath(ProjDir, "plots", "data.png"))

sm = SampleModel("opt_01", opt_01)

rc = stan_sample(sm, data=data)

if success(rc)
	chns = read_samples(sm; output_format=:mcmcchains)
	show(chns)
	plot(chns)
	savefig(joinpath(ProjDir, "plots", "chains.png"))

	dfs = read_samples(sm; output_format=:dataframe)

	dfsa = DataFrame(chns)
	#density(dfsa[:, Symbol("mu_pred.1")], label="mu_pred.1")
	#for i in 2:3:N
	#	density!(dfsa[:, Symbol("mu_pred.$i")], label="mu_pred.$i")
	#end
	#savefig(joinpath(ProjDir, "plots", "density.png"))

	p = 3.4:0.01:4.0
	ps = collect(p)
	p1 = scatter(data[:p], data[:q],
		#xlims=(3.4, 4.0), ylims=(3.0, 4.0),
		xlabel="ppu_l", ylabel="quantity_l sold", leg=false)
	a_mean = mean(dfs[:, :a])
	b_mean = mean(dfs[:, :b])
	mu_q = a_mean .+ b_mean .* (p .- mean(p))
	plot!(p1, p, mu_q)
	savefig(joinpath(ProjDir, "plots", "regression.png"))

	price_points_l = data[:p]
	plot(xlabel="Price", ylabel="Quantity", leg=false)
	selected_rows = rand(1:size(dfs, 1), 75)
	for row in eachrow(dfs[selected_rows,:])
		qs = exp.(row[:a] .+ row[:b] .* (ps .- mean(ps)))
		plot!(exp.(p), qs, color=:gray)
	end
	qqs = exp.(a_mean .+ b_mean .* (price_points_l .- mean(ps)))
	scatter!(exp.(price_points_l), qqs)
	savefig(joinpath(ProjDir, "plots", "sampling.png"))

	scatter(dfs[:, :a], dfs[:, :b],
		#xlims = (-0.1, 1.0), ylims= (-10.0, 10.0),
		xlabel="a", ylabel="b", leg=false)
	savefig(joinpath(ProjDir, "plots", "correlations.png"))
end


__precompile__()

module mbInterpolate
export interpolateLanczos, interpolateTrigonometric

using mbUtil

"""
    interpolateLanczos(V, a::Integer)

simple periodic (!) Lanczos interpolation
"""
function interpolateLanczos(V,a::Integer)
	return mbUtil.vectorize() do y
		sum = zero(V[1])
		N = length(V)
		for i in floor(Integer,y)+(-a+1:a)
			local tmp = y-i
			sum += sinc(tmp)*sinc(tmp/a)*V[mod(i-1,N) + 1]
		end
		return sum
	end
end
interpolateLanczos(V, a::Integer, x) = interpolateLanczos(V,a)(x)

"""
    interpolateTrigonometric(a₀, a, b)
Returns trigonometric polynomial.
Use with 2a,-2b and divide by 2m+1 to use with rfft coefficients.
"""
function interpolateTrigonometric(a₀, a, b)
	return mbUtil.vectorize() do x
		a₀ + reduce(0, 1:length(a)) do I,i
			tmp = i*x
			I + a[i]*cos(tmp) + b[i]*sin(tmp)
		end
	end
end
interpolateTrigonometric(a₀, a, b, x) = interpolateTrigonometric(a₀, a, b)(x)

end #module


"""
    PoisLik <: Likelihood

Poisson likelihood
```math
p(yᵢ = k | fᵢ) = θᵏ\\exp(-θ)/k!
```
for ``k ∈ N₀``, where ``θ = \\exp(f)`` and ``f`` is the latent Gaussian process.
"""
struct PoisLik <: Likelihood end

#log of probability density
function log_dens(poisson::PoisLik, f::VecF64, y::Vector{Int})
    #where we exponentiate for positivity f = exp(fi)
    return y.*f - exp.(f) - lgamma.(1.0 .+ y)
end

#derivative of pdf wrt latent function
function dlog_dens_df(poisson::PoisLik, f::VecF64, y::Vector{Int})
    return y - exp.(f)
end

#mean and variance under likelihood
mean_lik(poisson::PoisLik, f::VecF64) = exp.(f)
var_lik(poisson::PoisLik, f::VecF64) = exp.(f)

get_params(poisson::PoisLik) = []
num_params(poisson::PoisLik) = 0


let xmin = (-1.0, -1.0, -1.0)
    xmax = ( 1.0,  1.0,  1.0)

    for n = 0:2:10
        @test nintegrate((x,y,z)->x^n, xmin, xmax)[1] ≈ 8/(n+1)
        @test nintegrate((x,y,z)->y^n, xmin, xmax)[1] ≈ 8/(n+1)
        @test nintegrate((x,y,z)->z^n, xmin, xmax)[1] ≈ 8/(n+1)
    end

    @test nintegrate((x,y,z)->x^2*y^2, xmin, xmax)[1] ≈ 8/9
    @test nintegrate((x,y,z)->z^2*y^2, xmin, xmax)[1] ≈ 8/9
    @test nintegrate((x,y,z)->x^2*z^2, xmin, xmax)[1] ≈ 8/9

    @test nintegrate((x,y,z)->x^2*y^2*z^2, xmin, xmax)[1] ≈ 8/27

    ɛ = eps()

    for n = 1:2:11
        @test nintegrate((x,y,z)->x^n, xmin, xmax)[1] < ɛ
    end

    @test nintegrate((x,y,z)->x^2*z^3, xmin, xmax)[1] < ɛ
    @test nintegrate((x,y,z)->x^3*z^2, xmin, xmax)[1] < ɛ

    @test nintegrate((x,y,z)->x^2*y*z, xmin, xmax)[1] < ɛ
    @test nintegrate((x,y,z)->x*y^2*z, xmin, xmax)[1] < ɛ
    @test nintegrate((x,y,z)->x*y*z^2, xmin, xmax)[1] < ɛ

end

let xmin = (0.0, 0.0, 0.0)
    xmax = (1.0, 1.0, 1.0)

    for n = 0:11
        @test nintegrate((x,y,z)->x^n, xmin, xmax)[1] ≈ 1/(n+1)
        @test nintegrate((x,y,z)->y^n, xmin, xmax)[1] ≈ 1/(n+1)
        @test nintegrate((x,y,z)->z^n, xmin, xmax)[1] ≈ 1/(n+1)
    end

    @test nintegrate((x,y,z)->x^2*y^2, xmin, xmax)[1] ≈ 1/9
    @test nintegrate((x,y,z)->z^2*y^2, xmin, xmax)[1] ≈ 1/9
    @test nintegrate((x,y,z)->x^2*z^2, xmin, xmax)[1] ≈ 1/9

    @test nintegrate((x,y,z)->x^2*z^3, xmin, xmax)[1] ≈ 1/12
    @test nintegrate((x,y,z)->x^3*z^2, xmin, xmax)[1] ≈ 1/12
    @test nintegrate((x,y,z)->x^2*y*z, xmin, xmax)[1] ≈ 1/12
    @test nintegrate((x,y,z)->x*y^2*z, xmin, xmax)[1] ≈ 1/12
    @test nintegrate((x,y,z)->x*y*z^2, xmin, xmax)[1] ≈ 1/12

    @test nintegrate((x,y,z)->x^3*y*z, xmin, xmax)[1] ≈ 1/16
    @test nintegrate((x,y,z)->x*y^3*z, xmin, xmax)[1] ≈ 1/16
    @test nintegrate((x,y,z)->x*y*z^3, xmin, xmax)[1] ≈ 1/16

    @test nintegrate((x,y,z)->x^2*y^2*z^2, xmin, xmax)[1] ≈ 1/27

end

let f = (x, y, z) -> x * sin(2y) * cos(3z)

    xmin = (0.0, 0.0, 0.0)
    xmax = (1.0, 1.0, 1.0)

    (I, E, n, R) = nintegrate(f, xmin, xmax)
    @test I == nintegrate(f, R)
    @test length(R) == 1

    W = weightedpoints(R)
    @test I ≈ nintegrate(f, W)

    W = weightedpoints(NIntegration.idem, R)
    @test I ≈ nintegrate(f, W)

    p = [0.5, 0.5, 0.5]
    io = IOBuffer()
    show(io, R)
    @test String(take!(io)) == "1 subregion"
    show(io, R[1])
    @test String(take!(io)) == string("(", p, ", ", p, ")")

end


using VoronoiCells
import VoronoiCells: LEFT, RIGHT, LOWER, UPPER
using VoronoiDelaunay
using Base.Test

# Area of square
S = [Point2D(LEFT,LOWER) ; Point2D(RIGHT,LOWER) ; Point2D(RIGHT,UPPER) ; Point2D(LEFT,UPPER)]
@test_approx_eq polyarea(S) 1.0

# Area of triangle
T1 = [Point2D(LEFT,LOWER) ; Point2D(RIGHT,LOWER) ; 0.5*(Point2D(RIGHT,UPPER) + Point2D(LEFT,UPPER))]
@test_approx_eq polyarea(T1) 0.5

T2 = [Point2D(LEFT,LOWER) ; Point2D(LEFT+0.5,LOWER) ;
Point2D(LEFT,LOWER+0.5)]
@test_approx_eq polyarea(T2) 0.5^3


## Barycentric formula

export bary,barysum

function bary(v::Vector{Float64},pts::Vector{Float64},x::Float64)
  n=length(v)
  @assert n == length(pts)

  retd = .5/(x-pts[1])
  retn = v[1]*retd

  for i = 2:2:n-1
    @inbounds cd = 1./(x-pts[i])
    retd -= cd
    @inbounds retn -= v[i]*cd
  end

  for i = 3:2:n-1
    @inbounds cd = 1./(x-pts[i])
    retd += cd
    @inbounds retn += v[i]*cd
  end

  cd = .5*(-1.)^(n-1)/(x-pts[n])
  retd += cd
  retn += v[n]*cd

  retn/retd
end

bary(v::Vector{Float64},x::Float64)=bary(v,chebyshevpoints(length(v);kind=2),x)

function randomadaptivebary(f::Function)
  r=rand()
  fr=f(r)
  err=1.
  logn=1

  tol=200eps()

  vals=Float64[]

  while err > tol
      n=2^logn + 1
      pts=chebyshevpoints(n;kind=2)
      vals=f(pts)
      err=abs(bary(vals,pts,r)-fr)/n
      logn+=1
  end

  vals
end


# By default, Julia/LLVM does not use fused multiply-add operations (FMAs).
# Since these FMAs can increase the performance of many numerical algorithms,
# we need to opt-in explicitly.
# See https://ranocha.de/blog/Optimizing_EC_Trixi for further details.
@muladd begin

"""
    ln_mean(x, y)

Compute the logarithmic mean

    ln_mean(x, y) = (y - x) / (log(y) - log(x)) = (y - x) / log(y / x)

Problem: The formula above has a removable singularity at `x == y`. Thus,
some care must be taken to implement it correctly without problems or loss
of accuracy when `x ≈ y`. Here, we use the approach proposed by
Ismail and Roe (2009).
Set ξ = y / x. Then, we have

    (y - x) / log(y / x) = (x + y) / log(ξ) * (ξ - 1) / (ξ + 1)

Set f = (ξ - 1) / (ξ + 1) = (y - x) / (x + y). Then, we use the expansion

    log(ξ) = 2 * f * (1 + f^2 / 3 + f^4 / 5 + f^6 / 7) + O(ξ^9)

Inserting the first few terms of this expansion yields

    (y - x) / log(ξ) ≈ (x + y) * f / (2 * f * (1 + f^2 / 3 + f^4 / 5 + f^6 / 7))
                     = (x + y) / (2 + 2/3 * f^2 + 2/5 * f^4 + 2/7 * f^6)

Since divisions are usually more expensive on modern hardware than
multiplications (Agner Fog), we try to avoid computing two divisions. Thus,
we use

    f^2 = (y - x)^2 / (x + y)^2
        = (x * (x - 2 * y) + y * y) / (x * (x + 2 * y) + y * y)

Given ε = 1.0e-4, we use the following algorithm.

    if f^2 < ε
      # use the expansion above
    else
      # use the direct formula (y - x) / log(y / x)
    end

# References
- Ismail, Roe (2009).
  Affordable, entropy-consistent Euler flux functions II: Entropy production at shocks.
  [DOI: 10.1016/j.jcp.2009.04.021](https://doi.org/10.1016/j.jcp.2009.04.021)
- Agner Fog.
  Lists of instruction latencies, throughputs and micro-operation breakdowns
  for Intel, AMD, and VIA CPUs.
  https://www.agner.org/optimize/instruction_tables.pdf
"""
@inline function ln_mean(x, y)
  epsilon_f2 = 1.0e-4
  f2 = (x * (x - 2 * y) + y * y) / (x * (x + 2 * y) + y * y) # f2 = f^2
  if f2 < epsilon_f2
    return (x + y) / @evalpoly(f2, 2, 2/3, 2/5, 2/7)
  else
    return (y - x) / log(y / x)
  end
end

"""
    inv_ln_mean(x, y)

Compute the inverse `1 / ln_mean(x, y)` of the logarithmic mean
[`ln_mean`](@ref).

This function may be used to increase performance where the inverse of the
logarithmic mean is needed, by replacing a (slow) division by a (fast)
multiplication.
"""
@inline function inv_ln_mean(x, y)
  epsilon_f2 = 1.0e-4
  f2 = (x * (x - 2 * y) + y * y) / (x * (x + 2 * y) + y * y) # f2 = f^2
  if f2 < epsilon_f2
    return @evalpoly(f2, 2, 2/3, 2/5, 2/7) / (x + y)
  else
    return log(y / x) / (y - x)
  end
end

end # @muladd


# Question
# Find the sum of all the multiples of 3 or 5 below 1000.

# Time
# O(1)

function ap(n, d, a)
    return n*(2*a + (n-1)*d)/2
end
function main()
    limit = 1000
    n3 =  floor(1000/3)
    n5 = floor((1000-1)/5)
    n15 = floor(1000/15)

    sum = ap(n3, 3, 3) + ap(n5, 5, 5) - ap(n15, 15, 15)
    println(convert(Int, trunc(sum)))
end

@time main()


"""
Convert a vector `w` to a symmetric matrix `P` as
P = [
    w[1] ⋅ ⋅ ⋅ ⋅ ...;
    w[2] w[n+1]  ...;
    w[3] w[n+2]  ...;
    ...  ...        ;
    w[n] ...        ;
]
# Notes
It should be compatible with `convert_quadratic_to_linear_basis`.
"""
function convert_to_matrix(w::AbstractVector)::AbstractMatrix
    # n(n+1)/2 x 1 -> n x n
    _l = length(w)
    l = Int((-1 + sqrt(1+4*2*_l)) / 2)  # P is l x l lower triangular matrix with elements of w
    P = zeros(l, l)
    idx = 1
    for i in 1:l
        for j in i:l
            if i == j
                P[j, i] = w[idx]
            else
                P[j, i] = 0.5 * w[idx]
                P[i, j] = 0.5 * w[idx]
            end
            idx += 1
        end
    end
    P
    # # n^2 x 1 -> n x n for convenience...
    # l = Int(sqrt(length(w)))
    # P = zeros(l, l)
    # idx = 1
    # for i in 1:l
    #     for j in 1:l
    #         P[j, i] = w[idx]
    #         idx += 1
    #     end
    # end
    # P
end

"""
Basis transformation for linear ADP and IRL;
xᵀ * P * x = wᵀ * ϕ(x)
where ϕ(x) = [x1^2, x2*x1, x3*x1, ..., xn*x1, x2^2, x3*x2, x4*x2, ..., xn*x2, x3^2, x4*x3, ...]
# Notes
It should be compatible with `convert_to_matrix`.
"""
function convert_quadratic_to_linear_basis(x::AbstractVector)
    ## Notes
    ## ϕ(x) = (I_m ⊗ x) x;
    ## Px = (I_m ⊗ xᵀ) w
    ## where P ∈ ℝ^{m×n}, w ∈ ℝ^{nm×1}, I_m: m×m identity matrix, ⊗: Kronecker product
    # n = length(x)
    # kron(Matrix(I, n, n), x) * x
    ## for dimension reduction
    n = length(x)
    ϕ = zeros(Int(n*(n+1)/2))
    idx = 1
    for i in 1:n
        for j in i:n
            ϕ[idx] = x[j] * x[i]
            idx += 1
        end
    end
    ϕ
end


######
# This example shows how to do function approximation in 2D using Padua points
# and in 3D by hand
######

using ApproxFun
    import ApproxFun: transform
f = (x,y) -> exp(x*y)

S = Chebyshev(0..1)^2

N = 10^2; p = points(S, N) # 105 padua points
f̃ = (xy) -> f(xy[1], xy[2])
F = f̃.(p)
F̌ = transform(S, F)  # 105 Chebyshev^2 coefficients ordered by polynomial degree
f̌ = Fun(S, F̌)
f̌(0.1,0.2) ≈ f(0.1,0.2)

S = Chebyshev(0..1)
p_x = points(S, 10)
p_y = points(S, 10)'
p_z = reshape(points(S, 10), 1, 1, 10)

f = (x,y,z) -> exp(x*y+z)

F = f.(p_x, p_y, p_z)

for k = 1:size(F,2),j=1:size(F,3)
    F[:,k,j] = transform(S,F[:,k,j])
end
for k = 1:size(F,1),j=1:size(F,3)
    F[k,:,j] = transform(S,F[k,:,j])
end
for k = 1:size(F,1),j=1:size(F,2)
    F[k,j,:] = transform(S,F[k,j,:])
end

f̌ = function(x,y,z)
    ret1 = Array{Float64}(undef, size(F,2), size(F,3))
    for j=1:size(F,2),l=1:size(F,3)
        ret1[j,l] = Fun(S,F[:,j,l])(x)
    end
    ret2 = Array{Float64}(undef, size(F,3))
    for l=1:size(F,3)
        ret2[l] = Fun(S,ret1[:,l])(y)
    end
    Fun(S, ret2)(z)
end

f̌(0.1,0.2,0.3) ≈ f(0.1,0.2,0.3)


#=

Solves the price function for the Lucas tree in a continuous state
setting, using piecewise linear approximation for the sequence of
candidate price functions.  The consumption endownment follows the
log linear AR(1) process

    log y' = alpha log y + sigma epsilon

where y' is a next period y and epsilon is an iid standard normal
shock. Hence

    y' = y^alpha * xi   where xi = e^(sigma * epsilon)

The distribution phi of xi is

    phi = LN(0, sigma^2) where LN means lognormal

@authors : Spencer Lyon <spencer.lyon@nyu.edu>, John Stachurski

References
----------

http://quant-econ.net/jl/markov_asset.html

=#

using QuantEcon
using Distributions

"""
A function that takes two arrays and returns a function that approximates the
data using continuous piecewise linear interpolation.

"""
function lin_interp(x_vals::Vector{Float64}, y_vals::Vector{Float64})
    # == linear interpolation inside grid, constant values outside grid == #
    w = LinInterp(x_vals, y_vals)
    return w
end

"""
The Lucas asset pricing model --- parameters and grid data
"""
type LucasTree
    gamma::Real       # coefficient of risk aversion
    beta::Real        # Discount factor in (0, 1)
    alpha::Real       # Correlation coefficient in the shock process
    sigma::Real       # Volatility of shock process
    phi::Distribution # Distribution for shock process
    grid::Vector      # Grid of points on which to evaluate prices.
    shocks::Vector    # Draws of the shock
    h::Vector         # The h function represented as a vector
end

"""
Constructor for the Lucas asset pricing model
"""
function LucasTree(;gamma=2.0,
                beta=0.95,
                alpha=0.9,
                sigma=0.1,
                grid_size=100)

    phi = LogNormal(0.0, sigma)
    shocks = rand(phi, 500)

    # == build a grid with mass around stationary distribution == #
    ssd = sigma / sqrt(1 - alpha^2)
    grid_min, grid_max = exp(-4 * ssd), exp(4 * ssd)
    grid = collect(linspace(grid_min, grid_max, grid_size))

    # == set h(y) = beta * int u'(G(y,z)) G(y,z) phi(dz) == #
    h = similar(grid)
    for (i, y) in enumerate(grid)
        h[i] = beta * mean((y^alpha .* shocks).^(1 - gamma))
    end

    return LucasTree(gamma,
                    beta,
                    alpha,
                    sigma,
                    phi,
                    grid,
                    shocks,
                    h)
end

"""
The approximate Lucas operator, which computes and returns updated function
Tf on the grid points.
"""
function lucas_operator(lt::LucasTree, f::Vector{Float64})

    # == unpack names == #
    grid, alpha, beta, h = lt.grid, lt.alpha, lt.beta, lt.h
    z = lt.shocks

    Tf = similar(f)
    Af = lin_interp(grid, f)

    for (i, y) in enumerate(grid)
        Tf[i] = h[i] + beta * mean(Af.(y^alpha .* z))
    end
    return Tf
end

"""
Compute the equilibrium price function associated with Lucas tree `lt`
"""
function compute_lt_price(lt::LucasTree, max_iter=500)

    # == Simplify names == #
    grid = lt.grid
    alpha, beta, gamma = lt.alpha, lt.beta, lt.gamma

    # == Create suitable initial vector to iterate from == #
    f_init = zeros(grid)

    func(f_vec) = lucas_operator(lt, f_vec)
    f = compute_fixed_point(func, f_init;
                                    max_iter=max_iter,
                                    err_tol=1e-4,
                                    verbose=false)

    # p(y) = f(y) * y^gamma
    price = f .* grid.^(gamma)

    return price
end


# memory_function.jl

"""
χ(Ω::Float64, α::Float64, v::Float64, w::Float64)

    Calculate the memory function χ(Ω) of the polaron at finite temperatures (equation (35a) in FHIP 1962) for a given frequency Ω. v and w are the variational polaron parameters that minimise the free energy, for the supplied α Frohlich coupling.
"""
function χ(Ω, β, α, v, w)

    # FHIP1962, page 1011, eqn (47c).
    R = (v^2 - w^2) / (w^2 * v)

    # FHIP1962, page 1009, eqn (35c).
    D(x) = w^2 / v^2 * (R * (1 - cos(v * x)) * coth(β * v / 2) + x^2 / β - 1im * (R * sin(v * x) + x))

    # FHIP1962, page 1009, eqn (36).
    S(x) = 2 * α / (3 * √π) * (exp(1im * x) + 2 * cos(x) / (exp(β) - 1)) / (D(x))^(3 / 2)

    # FHIP1962, page 1009, eqn (35a).
    integrand(x) = (1 - exp(-1im * Ω * x)) * imag(S(x))
    QuadGK.quadgk(x -> integrand(x), 0.0, Inf)[1]
end

"""
χ(Ω::Float64, α::Float64, v::Float64, w::Float64)

    Calculate the memory function χ(Ω) of the polaron at zero-temperatures (equation (35a) in FHIP 1962) for a given frequency Ω. v and w are the variational polaron parameters that minimise the free energy, for the supplied α Frohlich coupling.
"""
function χ(Ω, α, v, w)

    # FHIP1962, page 1011, eqn (47c).
    R = (v^2 - w^2) / (w^2 * v)

    # FHIP1962, page 1009, eqn (35c) with β → ∞.
    D(x) = w^2 / v^2 * (R * (1 - cos(v * x)) - 1im * (R * sin(v * x) + x))

    # FHIP1962, page 1009, eqn (36) with β → ∞.
    S(x) = 2 * α / (3 * √π) * exp(1im * x) / (D(x))^(3 / 2)

    # FHIP1962, page 1009, eqn (35a). Set upper limit < ∞ but very large so cos argument finite.
    integrand(x) = (1 - exp(-1im * Ω * x)) * imag(S(x))
    QuadGK.quadgk(x -> integrand(x), 0.0, 1e200)[1]
end

"""
χ_dc(β::Float64, α::Float64, v::Float64, w::Float64)

    Calculate the memory function lim(Ω → 0){χ(Ω) / Ω} of the polaron at finite temperatures (equation (35a) in FHIP 1962) at zero frequency. v and w are the variational polaron parameters that minimise the free energy, for the supplied α Frohlich coupling.
"""
function χ_dc(β, α, v, w)

    # FHIP1962, page 1011, eqn (47c).
    R = (v^2 - w^2) / (w^2 * v)

    # FHIP1962, page 1009, eqn (35c).
    D(x) = w^2 / v^2 * (R * (1 - cos(v * x)) * coth(β * v / 2) + x^2 / β - 1im * (R * sin(v * x) + x))

    # FHIP1962, page 1009, eqn (36).
    S(x) = 2 * α / (3 * √π) * (exp(1im * x) + 2 * cos(x) / (exp(β) - 1)) / (D(x))^(3 / 2)

    # Set frequency small enough to mimic Ω = 0 without generating numerical instabilities in integral.
    Ω = 1e-200

    # FHIP1962, page 1009, eqn (35a). Readily divided by Ω to get sinc function in integrand rather than sine which would give 0 all the time.
    integrand(x) = (1 - exp(-1im * Ω * x)) * imag(S(x)) / Ω
    QuadGK.quadgk(x -> integrand(x), 0.0, Inf)[1]
end


using FastTransforms, Test

@testset "Padua transform and its inverse" begin
    n=200
    N=div((n+1)*(n+2),2)
    v=rand(N)  #Length of v is the no. of Padua points
    Pl=plan_paduatransform!(v)
    IPl=plan_ipaduatransform!(v)
    @test Pl*(IPl*copy(v)) ≈ v
    @test IPl*(Pl*copy(v)) ≈ v
    @test Pl*copy(v) ≈ paduatransform(v)
    @test IPl*copy(v) ≈ ipaduatransform(v)

    # check that the return vector is NOT reused
    Pl=plan_paduatransform!(v)
    x=Pl*v
    y=Pl*rand(N)
    @test x ≠ y

    IPl=plan_ipaduatransform!(v)
    x=IPl*v
    y=IPl*rand(N)
    @test x ≠ y

    # Accuracy of 2d function interpolation at a point

    """
    Interpolates a 2d function at a given point using 2d Chebyshev series.
    """
    function paduaeval(f::Function,x::AbstractFloat,y::AbstractFloat,m::Integer,lex)
        T=promote_type(typeof(x),typeof(y))
        M=div((m+1)*(m+2),2)
        pvals=Vector{T}(undef,M)
        p=paduapoints(T,m)
        map!(f,pvals,p[:,1],p[:,2])
        coeffs=paduatransform(pvals,lex)
        plan=plan_ipaduatransform!(pvals,lex)
        cfs_mat=FastTransforms.trianglecfsmat(plan,coeffs)
        f_x=sum([cfs_mat[k,j]*cos((j-1)*acos(x))*cos((k-1)*acos(y)) for k=1:m+1, j=1:m+1])
        return f_x
    end
    f_xy = (x,y) ->x^2*y+x^3
    g_xy = (x,y) ->cos(exp(2*x+y))*sin(y)
    x=0.1;y=0.2
    m=130
    l=80
    f_m=paduaeval(f_xy,x,y,m,Val{true})
    g_l=paduaeval(g_xy,x,y,l,Val{true})
    @test f_xy(x,y) ≈ f_m
    @test g_xy(x,y) ≈ g_l

    f_m=paduaeval(f_xy,x,y,m,Val{false})
    g_l=paduaeval(g_xy,x,y,l,Val{false})
    @test f_xy(x,y) ≈ f_m
    @test g_xy(x,y) ≈ g_l
end


@testset "Certification" begin
    @testset "Simple 1: Input = Vector{Expression}" begin
        @var x y
        # define the polynomials
        f₁ = (x^4 + y^4 - 1) * (x^2 + y^2 - 2) + x^5 * y
        f₂ = x^2 + 2x * y^2 - 2 * y^2 - 1 / 2
        F = [f₁, f₂]
        result = solve(F)

        # Input: System(F)
        cert = certify(System(F), result)
        @test nnonsingular(result) == 18
        @test ncertified(cert) == 18
        @test ndistinct_certified(cert) == 18
        @test nreal_certified(cert) == 4
        @test ncomplex_certified(cert) == 14
        @test ndistinct_real_certified(cert) == 4
        @test ndistinct_complex_certified(cert) == 14
        save("tmp_cert.txt", cert)
        @test !isempty(read("tmp_cert.txt", String))

        # Input: F
        cert = certify(F, result)
        @test nnonsingular(result) == 18
        @test ncertified(cert) == 18
        @test ndistinct_certified(cert) == 18
        @test nreal_certified(cert) == 4
        @test ndistinct_real_certified(cert) == 4

        # Control Display
        cert = certify(F, result, show_progress = false)

        # Double solutions
        S = solutions(result)
        cert = certify(F, [S; S])
        @test ncertified(cert) == 36
        @test ndistinct_certified(cert) == 18
        @test nreal_certified(cert) == 8
        @test ndistinct_real_certified(cert) == 4
    end

    @testset "Simple 2: Input = Vector{MP.Polynomial}" begin
        @polyvar x y
        # define the polynomials
        f₁ = (x^4 + y^4 - 1) * (x^2 + y^2 - 2) + x^5 * y
        f₂ = x^2 + 2x * y^2 - 2 * y^2 - 1 / 2
        F = [f₁, f₂]
        result = solve(F)

        # Input: System(F)
        cert = certify(System(F), result)
        @test nnonsingular(result) == 18
        @test ncertified(cert) == 18
        @test ndistinct_certified(cert) == 18
        @test nreal_certified(cert) == 4
        @test ndistinct_real_certified(cert) == 4

        # Input: F
        cert = certify(F, result)
        @test nnonsingular(result) == 18
        @test ncertified(cert) == 18
        @test ndistinct_certified(cert) == 18
        @test nreal_certified(cert) == 4
        @test ndistinct_real_certified(cert) == 4

        # Control Display
        cert = certify(F, result, show_progress = false)

        # Double solutions
        S = solutions(result)
        cert = certify(System(F), [S; S])
        @test ncertified(cert) == 36
        @test ndistinct_certified(cert) == 18
        @test nreal_certified(cert) == 8
        @test ndistinct_real_certified(cert) == 4
    end

    @testset "Parameters: Input = Vector{Expression}" begin
        @var x y
        f = (x^4 + y^4 - 1) * (x^2 + y^2 - 2) + x^5 * y
        # define new variables u₁, u₂ and λ₁
        @var λ[1:1] u[1:2]
        # define the jacobian of F
        J = differentiate([f], [x, y])
        # J' defines the transpose of J
        F = [[x, y] - u - J' * λ; f]
        u₀ = [-0.32, -0.1]
        res = solve(F, parameters = u, target_parameters = u₀)

        # Input: System(F)
        C = System(F, parameters = u)
        cert = certify(C, res; target_parameters = u₀)
        @test ncertified(cert) == 36
        @test ndistinct_certified(cert) == 36
        @test nreal_certified(cert) == 8
        @test ndistinct_real_certified(cert) == 8

        # Input: F
        cert = certify(F, res, u₀; parameters = u)
        @test ncertified(cert) == 36
        @test ndistinct_certified(cert) == 36
        @test nreal_certified(cert) == 8
        @test ndistinct_real_certified(cert) == 8

        # Invalid solutions
        cert = certify(C, [100 .* randn(ComplexF64, 3) for i = 1:10], u₀)
        @test ncertified(cert) < 10
    end

    @testset "Parameters: Input = Vector{MP.Polynomial}" begin
        @polyvar x y
        f = (x^4 + y^4 - 1) * (x^2 + y^2 - 2) + x^5 * y
        # define new variables u₁, u₂ and λ₁
        @polyvar λ[1:1] u[1:2]
        # define the jacobian of F
        J = differentiate([f], [x, y])
        # J' defines the transpose of J
        F = [[x, y] - u - J' * λ; f]
        u₀ = [-0.32, -0.1]
        res = solve(F, parameters = u, target_parameters = u₀)

        # Input: System(F)
        C = System(F, parameters = u)
        cert = certify(C, res; target_parameters = u₀)
        @test ncertified(cert) == 36
        @test ndistinct_certified(cert) == 36
        @test nreal_certified(cert) == 8
        @test ndistinct_real_certified(cert) == 8

        # Input: F
        cert = certify(F, res; parameters = u, target_parameters = u₀)
        @test ncertified(cert) == 36
        @test ndistinct_certified(cert) == 36
        @test nreal_certified(cert) == 8
        @test ndistinct_real_certified(cert) == 8
    end

    @testset "positive" begin
        @var x y
        f = System([x^2 + y^2 - 1, x - y])
        res = solve(f; compile = false, start_system = :total_degree)
        cert = certify(f, res, compile = false)
        @test count(is_positive, certificates(cert)) == 1
        @test count(s -> is_positive(s, 1), certificates(cert)) == 1
        @test count(is_real, certificates(cert)) == 2
    end

    @testset "3264" begin
        F = steiner()
        real_conics = [
            10124547 // 662488724,
            8554609 // 755781377,
            5860508 // 2798943247,
            -251402893 // 1016797750,
            -25443962 // 277938473,
            1 // 1,
            520811 // 1788018449,
            2183697 // 542440933,
            9030222 // 652429049,
            -12680955 // 370629407,
            -24872323 // 105706890,
            1 // 1,
            6537193 // 241535591,
            -7424602 // 363844915,
            6264373 // 1630169777,
            13097677 // 39806827,
            -29825861 // 240478169,
            1 // 1,
            13173269 // 2284890206,
            4510030 // 483147459,
            2224435 // 588965799,
            33318719 // 219393000,
            92891037 // 755709662,
            1 // 1,
            8275097 // 452566634,
            -19174153 // 408565940,
            5184916 // 172253855,
            -23713234 // 87670601,
            28246737 // 81404569,
            1 // 1,
        ]
        real_sols = read_solutions(joinpath(@__DIR__, "data/3264_real_sols.txt"))
        cert = certify(F, real_sols, real_conics; compile = true)
        @test ndistinct_real_certified(cert) == 3264
        cert = certify(F, real_sols, real_conics; compile = false)
        @test ndistinct_real_certified(cert) == 3264
        @test_throws ArgumentError certify(F, real_sols; compile = false)
        @test_throws ArgumentError certify(F, real_sols; compile = true)
    end

    @testset "certify uses complex inversion" begin
        @var x y
        F = System([y / x + x^2 - 3], parameters = [y])
        monres = monodromy_solve(F)
        cert = certify(F, monres)
        @test ndistinct_certified(cert) >= 1
    end
end


include("Kernals.jl")
include("gaussian_process.jl")

function uniform(a,b,N)
    rand(N)*(b-a)+a
end

using Distributions
n=50 #number of test points
N=10;#Number of training points
Xtest=linspace(-5,5,n); #Xtest are all of available points to check on the axis
X=uniform(-5,5,N); #X is the values we will sample at and train
noise_dist=Normal(0,10.0^(-6)) #Distribution of Noise
Y=sin(X)+rand(noise_dist) #Y is our function values with noise
K=cov_gen(std_exp_square_ker,Xtest,Xtest)+eye(length(Xtest))*1e-6
L=ctranspose(chol(K));
dist=MvNormal(zeros(n),eye(K)) # We draw f from N(0,I)L which is the same as N(0,K)
fprior=L*rand(dist,3); #Each column is a function

using PyPlot
fig = figure("pyplot_plot",figsize=(5,5))
ax = axes()
fill_between(Xtest,-2*diag(K),2*diag(K),facecolor="#a6a6a6")#This fills confindence interval for two standard
#deviations, currently the variance is one (as we take off diagonal)
plot(Xtest,fprior,alpha=0.75)
title("N=3 Multivariate Gaussians")
ylabel("f(x)")
xlabel("x")
grid("off")
show()

D=[(X[i],Y[i]) for i=1:length(X)];
mu,sigma,D = gaussian_process(std_exp_square_ker,D,1e-6,Xtest);
mu=reshape(mu,length(mu));
sigma=reshape(sigma,length(sigma));
y=map(x->x[2],D);   # these are our y noisy function
x=map(x->x[1],D); #These are our x training points
using PyPlot
fig = figure("pyplot_plot",figsize=(5,5))
ax = axes()
fill_between(Xtest,mu-2*sigma,mu+2*sigma,facecolor="#a6a6a6")#This fills confindence interval for two standard
#deviations, currently the variance is one (as we take off diagonal)
plot(x,y,linewidth=0,marker="o")

title("N=3 Multivariate Gaussians")
ylabel("f(x)")
xlabel("x")
grid("off")
show()


function circle_len1(θ; n_pts=1000000)
    xmin, xmax = cos(θ), 1
    ts = sort(rand(n_pts))
    xs = @. (xmax-xmin)/(1. - 0.) * ts + xmin
    ys = @. sqrt(1 - xs * xs)
    d = zero(eltype(xs))
    for i in 1:(length(xs)-1)
        d += sqrt((xs[i+1]-xs[i])^2+(ys[i+1]-ys[i])^2)
    end
    println(d)
    println(θ)
end

function circle_len2(θ; n_pts=1000000)
    xmin, xmax = cos(θ), 1
    t = sort(rand(n_pts))
    x = @. (xmax-xmin)/(1. - 0.) * t + xmin
    y = @. sqrt(1 - x * x)
    ds = @. sqrt((x[2:end] - x[1:end-1])^2+(y[2:end] - y[1:end-1])^2)
    d=sum(ds)
    println(d)
    println(θ)
end

function circle_len3(θ; n_pts=1000000)
    ts = θ * sort(rand(n_pts))
    xs = sin.(ts)
    ys = cos.(ts)
    d = zero(eltype(xs))
    for i in 1:(length(xs)-1)
        d += sqrt((xs[i+1]-xs[i])^2+(ys[i+1]-ys[i])^2)
    end
    println(d)
    println(θ)
end

circle_len3(2π)


#############################
##  Numerical experiments associated with section 5.1.2
#############################
## While the paper does not explicitly treat third kind Volterra equations, an extension to these
## cases is relatively straightforward. Note that while in general there won't be exponential convergence
## unless the involved powers are favorably smooth, our method still achieves higher accuracy than many competitor methods.
##

using ApproxFun, MultivariateOrthogonalPolynomials, BandedMatrices, BlockBandedMatrices, SpecialFunctions, Plots
using SparseVolterraExamples

#############################
## Problem in Equation (17)
#############################
####
## The following block computes the coefficient vector of the approximation for a given polynomial order of approximation n.
## The accuracy obtained depends on both the polynomial order for the multiplication as well as the solution.
## Choosing the same order is fine in most cases but better accuracy can be obtained by adjusting for a given problem.
function solveSec512A(n,multin)
    gf(x) = x^(2/3)*(10/3*x^(7/3)-3/16*x^(14/3));
    Kfun(x,y) = y;
    V = triVolterraFullKernelOpP01(Kfun,n,true);
        V = reflectPabtoPba(n)*WLoweringP01P00(n)*V;
        V = Conversion(Jacobi(0,0,0..1),Jacobi(1,2,0..1))[1:n,1:n]*V;
        V = Multiplication(Fun(x->x^(2/3),Jacobi(1,2,0..1),multin),Jacobi(1,2,0..1))[1:n,1:n]*Derivative(Jacobi(0,1,0..1),1)[1:n,1:n]-V
    coeff = [DirectEvalLHSP10at0(n);V[1:n-1,1:n]] \ [0;pad(Fun(x->gf(x),Jacobi(1,2, 0..1)).coefficients,n-1)]
    return coeff
end
####
## Plot the analytic solution against the computed solution
coeff = solveSec512A(50,40)
plot(Fun(Jacobi(0,1,0..1),coeff),grid=false,xlabel="x",ylabel="u(x)",label="sparse method")
plot!(x->(x)^(10/3),0,1,grid=false,xlabel="x",ylabel="u(x)",label="analytic solution")
####
## Check the numerical error.
plot(x->(Fun(Jacobi(0,1,0..1),coeff)(x)-(x)^(10/3)),0,1,grid=false,xlabel="x",ylabel="error",label=false)

#############################
## Problem in Equation (18)
#############################
####
## The following block computes the coefficient vector of the approximation for a given polynomial order of approximation n.
## The accuracy obtained depends on both the polynomial order for the multiplication as well as the solution.
## Choosing the same order is fine in most cases but better accuracy can be obtained by adjusting for a given problem.
function solveSec512B(n,multin)
    gf(x) = 9/2*x^4-1/20*x^(11/2)-1/6*x^6;
    Kfun(x,y) = sqrt(y);
    V = triVolterraFullKernelOpP01(Kfun,n,true,n);
        V = reflectPabtoPba(n)*WLoweringP01P00(n)*V;
        V = Conversion(Jacobi(0,0,0..1),Jacobi(1,2,0..1))[1:n,1:n]*V;
        V = Multiplication(Fun(x->sqrt(x),Jacobi(1,2,0..1),multin),Jacobi(1,2,0..1))[1:n,1:n]*Derivative(Jacobi(0,1,0..1),1)[1:n,1:n]-(1/20)*Conversion(Jacobi(0,1,0..1),Jacobi(1,2,0..1))[1:n,1:n]*Multiplication(Fun(x->x,Jacobi(0,1,0..1),n),Jacobi(0,1,0..1))[1:n,1:n]-V
    coeff = [DirectEvalLHSP10at0(n);V[1:n-1,1:n]] \ [0;pad(Fun(x->gf(x),Jacobi(1,2, 0..1)).coefficients,n-1)]
    return coeff
end
####
## Plot the analytic solution against the computed solution
coeff = solveSec512B(80,30)
plot(Fun(Jacobi(0,1,0..1),coeff),grid=false,xlabel="x",ylabel="u(x)",label="sparse method")
plot!(x->(x)^(9/2),0,1,grid=false,xlabel="x",ylabel="u(x)",label="analytic solution")
####
## Check the numerical error.
plot(x->(Fun(Jacobi(0,1,0..1),coeff)(x)-(x)^(9/2)),0,1,grid=false,xlabel="x",ylabel="error",label=false)

examples/Section 5-1-2 - Methods for third kind VIDEs.jl


module Flatland

using LinearAlgebra

abstract type Shape end

abstract type AbstractRectangle <: Shape end

mutable struct Rectangle <: AbstractRectangle
    x::Float64
    y::Float64
    l::Float64
    w::Float64
    function Rectangle((x, y); l=1.0, w=1.0)
        return new(x, y, l, w)
    end
end

function Rectangle((xmin, xmax), (ymin, ymax))
    @assert xmin < xmax && ymin < ymax "Corners have to be ordered: `xmin < xmax && ymin < ymax `"
    x = (xmin + xmax) / 2
    y = (ymin + ymax) / 2
    l = xmax - xmin
    w = ymax - ymin
    return Rectangle((x, y), l=l, w=w)
end

mutable struct Square <: AbstractRectangle
    x::Float64
    y::Float64
    l::Float64
    function Square((x, y); l=1.0)
        return new(x, y, l)
    end
end

lw(shape::Rectangle) = shape.l, shape.w
lw(shape::Square) = shape.l, shape.l

mutable struct RegularPolygon{N} <: Shape
    x::Float64
    y::Float64
    R::Float64
    θ::Float64  # angle
    function RegularPolygon((x, y), n::Int; R=1.0, θ=0.0)
        @assert n ≥ 3 "polygons need a minimum of three corners"
        return new{n}(x, y, R, θ)
    end
end

#=
`Circle`s are pretty straightforward, having a center and a radius.
=#

mutable struct Circle <: Shape
    x::Float64
    y::Float64
    R::Float64
    function Circle((x, y); R=1.0)
        return new(x, y, R)
    end
end

abstract type AbstractTriangle <: Shape end

mutable struct Triangle <: AbstractTriangle
    x1::Float64
    x2::Float64
    x3::Float64
    y1::Float64
    y2::Float64
    y3::Float64
    Triangle((x1, y1), (x2, y2), (x3, y3)) = new(x1, x2, x3, y1, y2, y3)
end

ncorners(::AbstractRectangle) = 4
ncorners(::AbstractTriangle) = 3
ncorners(::Circle) = 0
ncorners(::RegularPolygon{N}) where {N} = N

function corners(shape::AbstractRectangle)
    x, y = center(shape)
    l, w = lw(shape)
    return [(x+l/2, y+w/2), (x-l/2, y+w/2), (x-l/2, y-w/2), (x+l/2, y-w/2)]
end

function corners(shape::RegularPolygon)
    x, y = center(shape)
    θ = shape.θ
    R = shape.R
    n = ncorners(shape)
    return [(x+R*cos(t+θ), y+R*sin(t+θ)) for t in range(0, step=2π/n, length=n)]
end

function corners(shape::Triangle)
    return [(shape.x1, shape.y1), (shape.x2, shape.y2), (shape.x3, shape.y3)]
end

center(shape::Shape) = shape.x, shape.y

function center(shape::Triangle)
    (x1, y1), (x2, y2), (x3, y3) = corners(shape)
    return ((x1 + x2 + x3) / 3, (y1 + y2 + y3) / 3)
end

# xycoords returns two of the outline vectors: ycoords and ycoords, for `Circle`, you can specify the number of points to take
# (50 by default).

xycoords(s::Shape) = [first(p) for p in corners(s)], [last(p) for p in corners(s)]

function xycoords(shape::Circle; n=50)
    ts = range(0, 2π, length=n)
    return shape.x .+ shape.R * cos.(ts), shape.y .+ shape.R * sin.(ts)
end

# x,y-bounding

#=
The fuctions below yield the outer limits of the x and y axes of your shape. Can you complete the method as a oneliner?

Hint: The function `extrema` could be useful here...
=#

xlim(shape::Shape) = extrema(xycoords(shape)[1])
xlim(shape::Circle) = (shape.x - shape.R, shape.x + shape.R)
xlim(shape::AbstractRectangle) = (shape.x - 0.5shape.l, shape.x + 0.5shape.l)

ylim(shape::Shape) = extrema(xycoords(shape)[2])
ylim(shape::Circle) = (shape.y - shape.R, shape.y + shape.R)
ylim(shape::AbstractRectangle) = (shape.y - 0.5lw(shape)[2], shape.y + 0.5lw(shape)[2])

boundingbox(shape::Shape) = Rectangle(xlim(shape), ylim(shape))

# # Area

#=
Next, we compute the area of our shapes.
=#

area(shape::AbstractRectangle) = prod(lw(shape))

function area(shape::RegularPolygon)
    R = shape.R
    n = ncorners(shape)
    dθ = 2π / n  # angle of a triangle
    h = R * cos(dθ/2)
    b = R * sin(dθ/2)
    return b * h * n
end

function area(shape::Triangle)
    (x1, y1), (x2, y2), (x3, y3) = corners(shape)
    return abs(x1*y2 + x2*y3 + x3*y1 - y1*x2 - y2*x3 - y3*x1) / 2
end

area(shape::Circle) = shape.R^2 * π

# move, rotate and scale

#=
Moving, rotating and scaling should also work.
Important, the functions work in-place, meaning that the modify your structure (that is why use use `mutable` structures).

For `Circle` and `AbstractRectangle` types, `rotate!` leaves them unchanged.

Hint: Rotations are in radials, so between $0$ and $2\pi$.
=#

function move!(shape::Shape, (dx, dy))
    shape.x += dx
    shape.y += dy
end

function move!(shape::Triangle, (dx, dy))
    shape.x1 += dx
    shape.x2 += dx
    shape.x3 += dx
    shape.y1 += dy
    shape.y2 += dy
    shape.y3 += dy
    shape
end

rotate!(shape::Union{Circle,AbstractRectangle}, dθ) = shape

function rotate!(shape::RegularPolygon, dθ)
    shape.θ += dθ
    shape
end

function rotate!(shape::Triangle, dθ)
    xc, yc = center(shape)
    # center triangle
    move!(shape, (-xc, -yc))
    (x1, y1), (x2, y2), (x3, y3) = corners(shape)
    cosdθ = cos(dθ)
    sindθ = sin(dθ)
    shape.x1 = x1 * cos(dθ) - y1 * sin(dθ)
    shape.x2 = x2 * cos(dθ) - y2 * sin(dθ)
    shape.x3 = x3 * cos(dθ) - y3 * sin(dθ)
    shape.y1 = y1 * cos(dθ) + x1 * sin(dθ)
    shape.y2 = y2 * cos(dθ) + x2 * sin(dθ)
    shape.y3 = y3 * cos(dθ) + x3 * sin(dθ)
    # set to original position
    move!(shape, (xc, yc))
    shape
end

function scale!(shape::Union{Circle,RegularPolygon}, a)
    @assert a > 0 "scaling has to be a positive number"
    shape.R *= a
end

function scale!(shape::Rectangle, a)
    @assert a > 0 "scaling has to be a positive number"
    shape.w *= a
    shape.l *= a
end

function scale!(shape::Square, a)
    @assert a > 0 "scaling has to be a positive number"
    shape.w *= a
end

function scale!(shape::Triangle, a)
    @assert a > 0 "scaling has to be a positive number"
    xc, yc = center(shape)
    move!(shape, (-xc, -yc))
    shape.x1 *= a
    shape.x2 *= a
    shape.x3 *= a
    shape.y1 *= a
    shape.y2 *= a
    shape.y3 *= a
    move!(shape, (xc, yc))
    return shape
end

# plotting utilities

using RecipesBase

#=
OK, let's take a look at our shapes! We use `RecipesBase` to allow plotting.
This falls back on `xycoords` (can you see how it works?), so make sure this method is operational.
=#

@recipe function f(s::Shape)
    xguide --> "x"
    yguide --> "y"
    label --> ""
    aspect_ratio := :equal
    seriestype := :shape
    x, y = xycoords(s)
    return x, y
end

Base.in((x, y), s::Circle) = (s.x - x)^2 + (s.y - y)^2 ≤ s.R^2

function Base.in((x, y), shape::AbstractRectangle)
    xc, yc = center(shape)
    l, w = lw(shape)
    return (xc - 0.5l ≤ x ≤ xc + 0.5l) && (yc - 0.5w ≤ y ≤ yc + 0.5w)
end

crossprod((x1, y1), (x2, y2)) = x1 * y2 - x2 * y1

"""
    same_side((a, b), p, q)

Given a line described by two points, `a` and `b`, check whether two points
`p` and `q` are on the same side.
"""
function same_side((a, b), p, q)
    # normal vector on the line
    n = (a[2] - b[2], b[1] - a[1])
    # check if they are on both sides by projection
	return sign(n ⋅ (p .- a)) == sign(n ⋅ (q .-a ))
end

"""
    linecross((p1, p2), (q1, q2))

Check whether line segments `(p1, p2)` and `(q1, q2)` intersect.
"""
function linecross((p1, p2), (q1, q2))
    v = p2 .- p1
    w = q2 .- q1
    vw = crossprod(v, w)
    t = crossprod(q1 .- p1, w) / vw
    s = crossprod(q1 .- p1, v) / vw
    return 0.0 ≤ t ≤ 1.0 &&  0.0 ≤ s ≤ 1.0
end

function Base.in(q, s::Triangle)
    p1, p2, p3 = corners(s)
    return same_side((p1, p2), p3, q) &&
            same_side((p2, p3), p1, q) &&
            same_side((p3, p1), p2, q)
end

function Base.in(q, shape::Shape)
    corns = corners(shape)
    n = ncorners(shape)
    c = center(shape)
    # check if q is always on the same side as the center
    for i in 1:n-1
        !same_side((corns[i], corns[i+1]), c, q) && return false
    end
    !same_side((corns[end], corns[1]), c, q) && return false
    return true
end

@inline function boundboxes_overlap(shape1::Shape, shape2::Shape)
    (xmin1, xmax1), (xmin2, xmax2) = xlim(shape1), xlim(shape2)
    (ymin1, ymax1), (ymin2, ymax2)  = ylim(shape1), ylim(shape2)
    # check for x and y overlap
    return (xmin1 ≤ xmin2 ≤ xmax1 || xmin1 ≤ xmax2 ≤ xmax1 || xmin2 ≤ xmin1 ≤ xmax2) &&
            (ymin1 ≤ ymin2 ≤ ymax1 || ymin1 ≤ ymax2 ≤ ymax1 ||ymin2 ≤ ymin1 ≤ ymax2)
end

Base.intersect(shape1::AbstractRectangle, shape2::AbstractRectangle) = boundboxes_overlap(shape1, shape2)

function Base.intersect(shape1::T, shape2::T) where {T<:Shape}
    (shape1.x - shape2.x)^2 + (shape1.y - shape2.y)^2 > (shape1.R + shape2.R)^2 && return false
    return center(shape1) ∈ shape2 ||
            center(shape2) ∈ shape1 ||
            any(c->c ∈ shape2, corners(shape1)) ||
            any(c->c ∈ shape1, corners(shape2))
end

function Base.intersect(shape1::Triangle, shape2::Triangle)
    # first check if the bounding boxes overlap
    boundboxes_overlap(shape1, shape2) || return false
    # yes?
    # now check if one shape is within the other one
    p1, p2, p3 = corners(shape1)
    q1, q2, q3 = corners(shape2)
    (p1 ∈ shape2 || q1 ∈ shape1) && return true
    # if not, we have to check whether two lines intersect
    return linecross((p1, p2), (q1, q2)) ||
            linecross((p1, p2), (q2, q3)) ||
            linecross((p1, p2), (q1, q3)) ||
            linecross((p3, p2), (q1, q2)) ||
            linecross((p3, p2), (q2, q3)) ||
            linecross((p3, p2), (q1, q3))
end

function Base.intersect(shape1::Circle, shape2::Circle)
    c1 = center(shape1)
    c2 = center(shape2)
    d = sum(abs2, c1 .- c2) |> sqrt
    return d < shape1.R + shape2.R
end

# random generation

#=
Finally, `randplace!` takes a shape, rotates it randomly and moves it randomly within the bounds of the limits
`(xmin, xmax)` and `(ymin, ymax)`.
=#

function randplace!(shape::Shape, (xmin, xmax), (ymin, ymax); rotate=true)
    # random rotation
    rotate && rotate!(shape, 2π * rand())
    # random tranlation within bound
    dxmin, dxmax = (xmin, xmax) .- xlim(shape)
    dymin, dymax = (ymin, ymax) .- ylim(shape)
    dx = (dxmax - dxmin) * rand() + dxmin
    dy = (dymax - dymin) * rand() + dymin
    move!(shape, (dx, dy))
    return shape
end

function rejection_sampling!(shapes::Vector{<:Shape}, xlims, ylims)
    trials = 0
    n = length(shapes)
    while true
        trials += 1
        for (i, shape) in enumerate(shapes)
            randplace!(shape, xlims, ylims)
            # any intersection with previous shapes: start again
            overlap = false
            for j in 1:i-1
                if intersect(shape, shapes[j])
                    overlap = true
                    break
                end
            end
            overlap && break
            i==n && return shapes, trials
        end
    end
end

# function to place `n` copies of a given `shape`.

function rejection_sampling(shape, n, xlims, ylims)
    shapes = [deepcopy(shape) for i in 1:n]
    trials = rejection_sampling!(shapes, xlims, ylims)
    return shapes, trials
end

end


using COSMO, JuMP, LinearAlgebra
using Statistics
using JuMP
import Plots as Plots
import ProgressMeter as Pm
MOI = JuMP.MathOptInterface

include("utils.jl")

function MakeLassoOptimizationProblem(A::Matrix, y::Matrix, λ::Float64)
    """
        Phrase the quadratic programming problem for Lasso regularization
        problem.

    """
    m, n = size(A)
    @assert size(y, 1) == m && size(y, 2) == 1 "the label and"*
    " the data matrix doesn't have the maching dimension X:" *
    string(m, "×",  n) * string(";y: ", size(y))
    @assert λ ≤ 1 && λ ≥ 0 "Regularization λ should be between (0, 1)"

    Pb = Pm.ProgressUnknown("Building Optimization Model: ", spinner=true)

    model = Model(with_optimizer(COSMO.Optimizer); bridge_constraints = false)

    set_optimizer_attribute(model, MOI.Silent(), true)
    set_optimizer_attribute(model, "max_iter", 5000*10)

    @variable(model, x[1:n]); Pm.next!(Pb)
    setvalue.(x, A\y); Pm.next!(Pb)
    @variable(model, η[1:n]); Pm.next!(Pb)
    @constraint(model, -η .<= x); Pm.next!(Pb)
    @constraint(model, x .<= η); Pm.next!(Pb)
    @objective(model, Min, λ*sum(η) + sum((A*x - y).^2)); Pm.finish!(Pb)

    return model

end

function MakeQuantileOptimizationProblem(A::Matrix, y::Matrix, λ::Float64)
    """
        Phrase the Quantile regression problem into a Quadratic Programming
        problem

    """
end

mutable struct LassoSCOP
    ## It's just a collection of data.

    A::Matrix # Feature matrix
    y::Matrix # label vector
    μ::Matrix # feature mean
    u::Number # label mean
    Z::Matrix # Starndardized Matrix
    l::Matrix # Zero mean label

    OptModel::Model  # The JuMP model for getting it right.
    λ::Float64       # the regularization parameter.

    LassoPath::Union{Matrix, Nothing} # going along a fixed row is the fixing
    # the feature while varying the lambda quantity.
    λs::Union{Vector, Nothing} # the lambda values.

    function LassoSCOP(A::Matrix, y::Union{Matrix, Vector}, λ::Float64=0.0)
        A = copy(A)
        m, _ = size(A)
        @assert size(y, 1) == m ""*
        "The rows of X should match of the columns of y, but the size of"*
        string("X, Y is: ", size(A), " ", size(y))

        y = copy(y)
        if ndims(y) == 1
            y = reshape(y, (length(y), 1))
        end
        μ = mean(A, dims=1)::Matrix
        u = mean(y)
        Z = A .- μ
        l = y .- u
        OptModel = MakeLassoOptimizationProblem(Z, l, λ)

        return new(A, y, μ, u, Z, l, OptModel, λ)
    end
end

function LassoPath(this::LassoSCOP, tol::Float64=1e-8)
    """
        Analyze the Lasso Problem by drawing a lasso path. It will start with
        a parameter that will make all predictors zero and then solve it
        iterative by chopping the regularization λ by half each iteration.

        **this**:
            An instance of the LassoSCOP
        **tol**:
            If the infinity norm of vector of the change in the weights is
            less than this quantity, then it stops and return all the results.

    """

    u = this.u
    A = this.Z
    y = this.l
    Results = Vector{Vector}()

    function λMax(A, y)
        ToMax = A'*(y .- u)
        ToMax *= 2
        return maximum(abs.(ToMax))
    end

    λ = λMax(A, y)
    λs = Vector{Float64}()
    Changeλ(this, λ)
    x = SolveForx(this)
    dx = Inf
    push!(Results, x)
    push!(λs, λ)
    MaxItr = 100
    pb = Pm.ProgressThresh(tol, "inf norm of δx: ")
    while dx >= tol && MaxItr >= 0
        push!(λs, λ)
        λ /= 2
        MaxItr -= 1
        setvalue.(this.OptModel[:x], x) # warm start!
        Changeλ(this, λ)
        x = SolveForx(this)
        push!(Results, x)
        dx = norm(Results[end - 1] - Results[end], Inf)
        Pm.update!(pb, dx)
    end

    ResultsMatrix = zeros(length(x), length(Results))
    for II ∈ 1:length(Results)
        ResultsMatrix[:, II] = Results[II]
    end

    # Store it.
    this.LassoPath = ResultsMatrix
    this.λs = λs
    return ResultsMatrix, λs
end

function VisualizeLassoPath(
                            this::LassoSCOP,
                            fname::Union{String, Nothing}=nothing,
                            title::Union{String, Nothing}=nothing
                            )
    """
        Make a plots for the lasso path and save it in the pwd.

    """
    @assert isdefined(this, :LassoPath) "Lasso Path not defined for the object"*
    "yet".
    error("Haven't implemented it yet.")
    λs = this.λs
    Paths = this.LassoPath
    Loggedλ = log2.(λs)
    p1 = Plots.heatmap(
        Paths[end:-1:begin, :],
        title=title===nothing ? "Lasso Path" : title
        )
    p2 = plot(Loggedλ, Paths', label=nothing)
    Plots.xlabel!(p2, "log_2(λ)")
    p = Plots.plot(p1, p2, layout=(2, 1))
    Plots.plot!(size=(600, 800))
    Plots.plot!(dpi=400)
    Plots.savefig(p, fname===nothing ? "plot.png" : fname)
    return
end

function CaptureImportantWeights(
                                this::LassoSCOP,
                                top_k::Union{Float64, Int64} = 0.5,
                                threshold::Float64=1e-10
                                )
    """
        Caputre the indices for the most important predictors from the
        regression. Returns the indices of the important weights.

        ---
        **this::LassoSCOP**:
            An instance of the type LassoSCOP

        ---
        **top_k::Union{Float64, Int64} = 0.5**
            When it's in between 0, 1, it's specifying the portion of weights
            that are non-zero, when it's not in in this range and it's positive,
            then it's specifying the number of top weights, measured by the
            absolute value of the weights.

        ---
        **threshold::Float64=1e-10**
            Weigthts less than threshold is viewed as unimportant and it's
            discarded as zero.

    """
    # TODO: test this
    @assert isdefined(this, :LassoPath) "Lasso Path not defined for this"*
    "Object yet. "
    @assert top_k >= 0 "this parameters, should be a positive number"
    top_k = min(top_k, size(this.LassoPath, 1))
    @assert threshold >= 0 "This parameters shouold be a positive number"

    Paths = this.LassoPath
    top_k::Int64 = top_k <= 1 ? ceil(size(Paths, 1)*0.5) : round(top_k)
    for JJ in size(Paths, 2)
        Col = view(Paths, :, JJ)
        NonNegative = sum(abs.(Col) .>= threshold)
        # rank them by abs and returns the indices for top k weights
        if NonNegative >= top_k
            Indices = sortperm(abs.(Col), rev=true)
            return Indices[begin:top_k]
        end
    end
    return Vector{Int64}()
end

function Changeλ(this::LassoSCOP, λ)
    """
        Change the Lasso regularizer of the current model
    """
    model = this.OptModel
    x = model[:x]  # objects can be indexed with symbols!
    η = model[:η]
    y = this.l
    A = this.Z
    @objective(model, Min, λ*sum(η) + sum((A*x - y).^2))

end

function SolveForx(this::LassoSCOP)::Vector{Float64}
    """
        Solve for the weights of the current model,
        given the current configuration of the model.
    """
    optimize!(this.OptModel)
    TernimationStatus = termination_status(this.OptModel)
    # @assert TernimationStatus == MOI.OPTIMAL "Terminated with non-optimal value when solving for x. "*
    # string("The status is: ", TernimationStatus)*"\n this is the results \n $(OptResults)"

    if !(TernimationStatus == MOI.OPTIMAL)
        Warn("\nWarning: convergence status for solver: $(TernimationStatus)")
        Warn("Current Value λ: $(this.λ)")
        PrintTitle("Here is the summary for the solution: ")
        display(solution_summary(this.OptModel))
    end
    return value.(this.OptModel[:x])
end

function Getαβ(this::LassoSCOP, lambda::Float64)
    """
        Get the weights and biases, for the original model
        (The model trained is normalized),
        for a particular regularization value.
    """
    error("Not yet implemented. ")

end

# TODO: Override Base.show for this LASSOPath TYPE.

# TODO: Iterate, Use Base.Iterate, override it.
function Base.iterate(this::LassoSCOP)
    """
        Iterate through the parameters for the Lasso program:
            * λ
            * solutions
    """
    u = this.u
    A = this.Z
    y = this.l

    function λMax(A, y)
        ToMax = A'*(y .- u)
        ToMax *= 2
        return maximum(abs.(ToMax))
    end
    λ = λMax(A, y)
    x = SolveForx(this)
    return λ, x

end

function Base.Iterate(this::LassoSCOP, state::Int64)

end


module OperationsTests

using Test
using TensorValues

a = VectorValue(1,2,3)
b = VectorValue(2,1,6)

# Comparison

@test a==a
@test a ≈ a
@test a!=b
@test [a,a] == [a,a]
@test [a,a] ≈ [a,a]

# Addition / subtraction

c = +a
r = VectorValue(1,2,3)
@test c == r

c = -a
r = VectorValue(-1,-2,-3)
@test c == r

c = a + b
r = VectorValue(3,3,9)
@test c == r

c = a - b
r = VectorValue(-1,1,-3)
@test c == r

# Matrix Division

t = one(TensorValue{3,Int,9})

c = t\a

@test c == a

# Operations by a scalar

a = VectorValue(1,2,3)
r = VectorValue(2,4,6)

c = 2 * a
@test isa(c,VectorValue{3,Int})
@test c == r

c = a * 2
@test isa(c,VectorValue{3,Int})
@test c == r

r = VectorValue(3,4,5)

c = 2 + a
@test isa(c,VectorValue{3,Int})
@test c == r

c = a + 2
@test isa(c,VectorValue{3,Int})
@test c == r

r = VectorValue(1/2,1.0,3/2)

c = a / 2
@test isa(c,VectorValue{3,Float64})
@test c == r

# Dot product (simple contraction)

a = VectorValue(1,2,3)
b = VectorValue(2,1,6)

t = TensorValue(1,2,3,4,5,6,7,8,9)
s = TensorValue(9,8,3,4,5,6,7,2,1)

c = a * b
@test isa(c,Int)
@test c == 2+2+18

c = t * a
@test isa(c,VectorValue{3,Int})
r = VectorValue(30,36,42)
@test c == r

c = s * t
@test isa(c,TensorValue{3,Int})
r = TensorValue(38,24,18,98,69,48,158,114,78)
@test c == r

# Inner product (full contraction)

c = inner(2,3)
@test c == 6

c = inner(a,b)
@test isa(c,Int)
@test c == 2+2+18

c = inner(t,s)
@test isa(c,Int)
@test c == 185

# Reductions

a = VectorValue(1,2,3)

c = sum(a)
@test isa(c,Int)
@test c == 1+2+3

c = maximum(a)
@test isa(c,Int)
@test c == 3

c = minimum(a)
@test isa(c,Int)
@test c == 1

# Outer product (aka dyadic product)

a = VectorValue(1,2,3)
e = VectorValue(2,5)

c = outer(2,3)
@test c == 6

r = VectorValue(2,4,6)
c = outer(2,a)
@test isa(c,VectorValue{3,Int})
@test c == r

c = outer(a,2)
@test isa(c,VectorValue{3,Int})
@test c == r

c = outer(a,e)
@test isa(c,MultiValue{Tuple{3,2},Int})
r = MultiValue{Tuple{3,2},Int}(2,4,6,5,10,15)
@test c == r

# Linear Algebra

t = TensorValue(10,2,30,4,5,6,70,8,9)

c = det(t)
@test c ≈ -8802.0

c = inv(t)
@test isa(c,TensorValue{3})

# Measure

a = VectorValue(1,2,3)
c = meas(a)
@test c ≈ 3.7416573867739413

t = TensorValue(10,2,30,4,5,6,70,8,9)
c = meas(t)
@test c ≈ 8802.0

# Broadcasted operations

a = VectorValue(1,2,3)
b = VectorValue(2,1,6)

A = Array{VectorValue{3,Int},2}(undef,(4,5))
A .= a
@test A == fill(a,(4,5))

C = .- A

r = VectorValue(-1,-2,-3)
R = fill(r,(4,5))
@test C == R

B = fill(b,(4,1))

C = A .+ B

r = VectorValue(3,3,9)
R = fill(r,(4,5))
@test C == R

C = A .- B

r = VectorValue(-1,1,-3)
R = fill(r,(4,5))
@test C == R

# conj

v = VectorValue(1,0)
@test v == v'
@test conj(v) == v'

t = TensorValue(1,2,3,4)
@test trace(t) == 5
@test tr(t) == 5

t = TensorValue(1,2,3,4,5,6,7,8,9)
@test trace(t) == 15
@test tr(t) == 15

@test symmetic_part(t) == TensorValue(1.0, 3.0, 5.0, 3.0, 5.0, 7.0, 5.0, 7.0, 9.0)

a = TensorValue(1,2,3,4)
b = a'
@test adjoint(a) == b
@test b == TensorValue(1,3,2,4)
@test a*b == TensorValue(10,14,14,20)

u = VectorValue(1.0,2.0)
v = VectorValue(2.0,3.0)
@test dot(u,v) ≈ inner(u,v)
@test norm(u) ≈ sqrt(inner(u,u))

end # module OperationsTests


using Distributions
using StatsBase: var

########## GDEMO ##################
# Example partially based on https://stats.stackexchange.com/a/266672/234110 and
# http://www2.stat.duke.edu/~rcs46/modern_bayes17/lecturesModernBayes17/lecture-7/07-gibbs.pdf.
# We follow the model used in https://github.com/TuringLang/Turing.jl/blob/60724e22a9831066fc2e0e82d428bd4922bda6e8/test/test_utils/models.jl#L2,
# with normally distributed observations and conjugate priors for both parameters:
# λ ~ Gamma(α₀, θ₀)
# m ~ Normal(0, √(1 / λ))
# Xᵢ ~ Normal(m, √(1 / λ)) (iid)

function gdemo_forward(N, α₀, θ₀)
    λ = rand(Gamma(α₀, θ₀))
    σ = √(1 / λ)
    m = rand(Normal(0, σ))
    x = rand(Normal(m, σ), N)
    return (m=m, λ=λ), x
end

function gdemo_logjoint(α₀, θ₀, x, m, λ)
    σ = √(1 / λ)
    return logpdf(Gamma(α₀, θ₀), λ) +
        logpdf(Normal(0, σ), m) +
        sum(logpdf.(Normal(m, σ), x))
end

function gdemo_statistics(x)
    # The conditionals and posterior can be formulated in terms of the following statistics:
    N = length(x) # number of samples
    x̄ = mean(x) # sample mean
    s² = var(x; mean=x̄, corrected=false) # sample variance
    return N, x̄, s²
end

function gdemo_cond_m(α₀, θ₀, x, λ)
    N, x̄, s² = gdemo_statistics(x)
    mₙ = N * x̄ / (N + 1)
    λₙ = λ * (N + 1)
    σₙ = √(1 / λₙ)
    return Normal(mₙ, σₙ)
    # return Normal(7/6, 0.00001)
end

function gdemo_cond_λ(α₀, θ₀, x, m)
    N, x̄, s² = gdemo_statistics(x)
    αₙ = α₀ + (N - 1) / 2 + 1
    βₙ = (s² * N / 2 + m^2 / 2 + inv(θ₀))
    return Gamma(αₙ, inv(βₙ))
end

function gdemo(x; α₀=2.0, θ₀=inv(3.0))
    cond_m((λ,)) = gdemo_cond_m(α₀, θ₀, x, λ)
    cond_λ((m,)) = gdemo_cond_λ(α₀, θ₀, x, m)
    conditionals = (m=cond_m, λ=cond_λ)
    return conditionals
end

############### Mixture of two Gaussians ####################
# see http://www.cs.columbia.edu/~blei/fogm/2015F/notes/mixtures-and-gibbs.pdf
# μ ~ arraydist(Normal.(fill(m, K), fill(λ, K)))
# z ~ arraydist(Categorical.(fill(π, N)))
# x ~ arraydist(Normal.(μ[z], σ))

function mixture_cond_z(π, K, m, λ, σ, x, μ)
    function mixtureweight(x)
        p = π .* pdf.(Normal.(μ, σ), Ref(x))
        return p ./ sum(p)
    end
    return Product(Categorical.(mixtureweight.(x)))
end

function mixture_cond_μ(π, K, m, λ, σ, x, z)
    n = [count(z .== k) for k = 1:K]

    # If there were no observations assigned to center `k`, `n[k] == 0`, and
    # we use the prior instead.
    x_bar = [(n[k] != 0) ? (sum(x[z .== k]) / n[k]) : m for k = 1:K]
    λ_hat = [(n[k] != 0) ? inv(n[k] / σ^2 + 1/λ^2) : λ for k = 1:K]
    μ_hat = [(n[k] != 0) ? x_bar[k] * (n[k] / σ^2) * λ_hat[k] : m for k = 1:K]

    return Product(Normal.(μ_hat, λ_hat))
end

function mixture(π, K, m, λ, σ, x)
    cond_z((μ,)) = mixture_cond_z(π, K, m, λ, σ, x, μ)
    cond_μ((z,)) = mixture_cond_μ(π, K, m, λ, σ, x, z)
    return (z=cond_z, μ=cond_μ)
end

########### 1D ISING MODEL #########################
function ising_conditional(β, i, M)
    return function (neighbours)
        x_prev = neighbours[mod1(i - 1, M - 1)]
        x_next = neighbours[mod1(i, M - 1)]
        p₁ = exp(-β * (x_prev + x_next))
        p₂ = exp(-β * (abs(x_prev - 1) + abs(x_next - 1)))
        return Bernoulli(p₁ / (p₁ + p₂))
    end
end

function isingdemo(β, M)
    # One dimensional Ising model on a circle, without external field;
    # see https://stats.stackexchange.com/a/312044/234110
    conditional_dists = [Symbol(:x, i) => ising_conditional(β, i, M) for i = 1:M]
    conditionals = merge(NamedTuple(), conditional_dists)
end

function ising_conditional(β, i, j, M, N)
    return function (neighbours)
        x_prev = neighbours[mod1(i - 1, M - 1)]
        x_next = neighbours[mod1(i, M - 1)]
        x_down = neighbours[mod1(j - 1, N - 1)]
        x_up = neighbours[mod1(j, N - 1)]
        p₁ = exp(-β * (x_prev + x_next + x_up + x_down))
        p₂ = exp(-β * (abs(x_prev - 1) + abs(x_next - 1) + abs(x_up - 1) + abs(x_down - 1)))
        return Bernoulli(p₁ / (p₁ + p₂))
    end
end

function isingdemo(β, M, N)
    # One dimensional Ising model on a circle, without external field;
    # see https://stats.stackexchange.com/a/312044/234110
    conditional_dists = [Symbol(:x, i, j) => ising_conditional(β, i, j, M, N) for i = 1:M, j = 1:N]
    conditionals = merge(NamedTuple(), conditional_dists)
end

function visualize_ising(θ::AbstractVector{<:Real})
    join(v == 1 ? '\u2588' : ' ' for v in values(θ))
end

function visualize_ising(θ::AbstractMatrix{<:Real})
    join((join(v == 1 ? '\u2588' : ' ' for v in row) for row in eachrow(θ)), '\n')
end


#=
Author: Stephan Scholz
Year: 2021

This file contains a machine learning approach (SciML) for the van der Pol oscillator
=#

# ODE function of van der Pol oscillator
function vdp(dx, x, p, t)

    μ = p[1]

    dx[1] = μ * (x[1] - 1/3 * x[1]^3 - x[2] )
    dx[2] = 1/μ * x[1]
end

tspan = (0.0, 100.0) # time span
x0    = [0.5, 0.5]   # initial values
p     = [1.0]        # parameter: μ
tsteps= 0.1          # saved time steps

using DifferentialEquations
prob = ODEProblem( vdp, x0, tspan, p )      # Build ODEProblem
sol = solve(prob, Tsit5(), saveat = tsteps) # Solve ODEProblem

# using Plots
# plot(sol, title="Solution of the van der Pol oscillator")

# MACHINE LEARNING
function loss(p)
    sol_pred = solve(prob, Tsit5(), p=p, saveat = tsteps)
    loss = sum(abs2, sol - sol_pred)
    return loss, sol_pred
end

pred_param = [] # Stores all found parameters

callback = function (p, l, pred)
    display(l)
    append!(pred_param, p)

    # Tell sciml_train to not halt the optimization. If return true, then
    # optimization stops.
    return false
end

using DiffEqFlux

p_ml = [0.1] # Initial parameter
loss(p_ml)   # Test whether loss function works

# Run ML training
result_ode = DiffEqFlux.sciml_train(loss, p_ml, ADAM(0.1), cb = callback, maxiters=100)

# Build and plot Cost function
x_min = 0.1
x_max = 2
x_step = 0.01
x_range = range(x_min,x_max,step=x_step)
Nlength = length(x_range)

loss_results = zeros(Nlength)
idx = 1;
for i in x_range
   loss_results[idx],_ = loss([i])
   idx += 1;
end

# Cost function
plot(x_range, loss_results, label="loss", title="Cost function", xaxis=("Parameter μ"))

# Save Machine Learning results as gif
@gif for pp in pred_param
    sol_remade = solve(remake(prob, p=[pp]), Tsit5(), saveat=tsteps)

    # Phase portrait
    pp_text = string( "μ = ", string(round(pp; digits=3)) )
    plot(sol_remade[1,:], sol_remade[2,:], annotations=(0.1, 0, Plots.text(pp_text, :left)), title="Phase portrait of van der Pol oscillator", label=false)

    # Oscillation
    # plot(sol_remade, title="Machine Learning of van der Pol oscillator", label=["x1" "x2"], legend=:bottomright)

end


#Demo 2: Reflection coefficients
#-------------------------------------------------------------------------------
using InspectDR
using Colors
import Graphics: width, height

#==Input
===============================================================================#

#Constants
#-------------------------------------------------------------------------------
black = RGB24(0, 0, 0)
white = RGB24(1, 1, 1)
red = RGB24(1, 0, 0)
green = RGB24(0, 1, 0)
blue = RGB24(0, 0, 1)
μ0 = 4pi*1e-7 #F/m
ϵ0 = 8.854e-12 #H/m

#Input data
#-------------------------------------------------------------------------------
fmax = 10e9
fstep = 50e6
f = collect(0:fstep:fmax)
ℓ = 50e-3 #Meters
ZL = Float64[1e6, 377, 60.0]
_colors = [blue, red, green]

#==Equations
===============================================================================#
Γ(Z; Zref::Real=50.0) = (Z - Zref) ./ (Z + Zref)
#ZC: Characteristic impedance
#ZL: Load impendance (termination)
function Zline(ℓ::Real, f::Vector, ZL::Number; ZC::Number=50.0, α::Real=0, μ::Real=μ0, ϵ::Real=ϵ0)
	j = im
	β = f*(2pi*sqrt(μ*ϵ))
	γ = α+j*β
	tanh_γℓ = tanh(γ*ℓ)
	return ZC*(ZL+ZC*tanh_γℓ)./(ZC+ZL*tanh_γℓ)
end
function Γline(ℓ::Real, f::Vector, ZL::Number; ZC::Number=50.0, Zref::Number=50.0, α::Real=0, μ::Real=μ0, ϵ::Real=ϵ0)
	return Γ(Zline(ℓ, f, ZL; ZC=ZC, α=α, μ=μ, ϵ=ϵ), Zref=Zref)
end

#Calculations
#-------------------------------------------------------------------------------
Γload = []
for ZLi in ZL
	_Γ = Γline(ℓ, f, ZLi, ZC=40)
	push!(Γload, _Γ)
end

#==Generate plot
===============================================================================#
mplot = InspectDR.Multiplot(title="Transmission Line Example")
mplot.ncolumns = 2

plot_linf = InspectDR.Plot2D()
	plot_linf.axes = InspectDR.axes(:lin, :dB20)
	plot_linf.ext_full = InspectDR.PExtents2D(ymax=5)
#	plot_linf.layout.legend.enabled=true
plot_logf = InspectDR.Plot2D()
	plot_logf.axes = InspectDR.axes(:log10, :dB20)
	plot_logf.ext_full = InspectDR.PExtents2D(xmin=10e6,ymax=5)
	plot_logf.layout.grid = grid(vmajor=true, vminor=true, hmajor=true)
plot_ysmith = InspectDR.Plot2D()
	plot_ysmith.axes = InspectDR.axes(:smith, :Y)
	plot_ysmith.ext_full = InspectDR.PExtents2D(xmin=-1.2,xmax=1.2,ymin=-1.2,ymax=1.2)
	plot_ysmith.layout.legend.enabled=true
plot_smith = InspectDR.Plot2D()
	plot_smith.axes = InspectDR.axes(:smith, :Z, ref=50)
	plot_smith.ext_full = InspectDR.PExtents2D(xmin=-1.2,xmax=1.2,ymin=-1.2,ymax=1.2)
	plot_smith.layout.legend.enabled=true

for plot in [plot_linf, plot_logf]
	a = plot.annotation
	a.title = "Reflection Coefficient (Γ)"
	a.xlabel = "Frequency (Hz)"
	a.ylabel = "Magnitude (dB)"
end

a = plot_ysmith.annotation
	a.title = "Y-Smith Chart"
	a.xlabel = "Real(Γ)"
	a.ylabel = "Imaginary(Γ)"

a = plot_smith.annotation
	a.title = "Z-Smith Chart"
	a.xlabel = "Real(Γ)"
	a.ylabel = "Imaginary(Γ)"

#Select which plots to actually display:
plotlist = [plot_linf, plot_logf, plot_ysmith, plot_smith]
#plotlist = [plot_smith]

for plot in plotlist
	for i in 1:length(Γload)
		wfrm = add(plot, f, Γload[i], id="ZL=$(ZL[i])")
		wfrm.line = line(color=_colors[i], width=2)
	end

	add(mplot, plot)
end

gplot = display(InspectDR.GtkDisplay(), mplot)

#==Save multi-plot to file
===============================================================================#

maximize_square = true
if maximize_square
	#Target plot size to get square Smith plots without gaps:
	lyt = plot_smith.layout
		lyt.wdata = 500
		lyt.hdata = 500
	bb = InspectDR.plotbounds(lyt, plot_smith.axes) #Required
		mplot.wplot = width(bb)
		mplot.hplot = height(bb)
end

InspectDR.write_png("export_multiplot.png", mplot)
InspectDR.write_svg("export_multiplot.svg", mplot)
InspectDR.write_eps("export_multiplot.eps", mplot)
InspectDR.write_pdf("export_multiplot.pdf", mplot)

:DONE


"""
    eom_mediated_link(u, p, t) -> SVector{9}

Equations of motion for a three-subsystem dynamical system where `X` and `Y` are
driven by `Z`. At the default value of the coupling constant `c = 0`, the
responses `X` and `Y` are already synchronized to the driver `Z`.
The frequencies `ωx`, `ωy` and ``ωz` have near unit ratios by default.

# References
Krakovská, Anna, et al. "Comparison of six methods for the detection of causality in a bivariate time series." Physical Review E 97.4 (2018): 042207
"""
function eom_mediated_link(u, p, t)
    ωx, ωy, ωz, k, l, m, c = (p...,)
    x₁, x₂, x₃, y₁, y₂, y₃, z₁, z₂, z₃ = (u...,)

    dx₁ = -ωx*x₂ - x₃ + c*(z₁ - x₁)
	dx₂ = ωx*x₁ + k*x₂
	dx₃ = l + x₃*(x₁ - m)

	dy₁ = -ωy*y₂ - y₃ + c*(z₁ - y₁)
	dy₂ = ωy*y₁ + k*y₂
	dy₃ = l + y₃*(y₁ - m)

	dz₁ = -ωz*z₂ - z₃
	dz₂ = ωz*z₁ + k*z₂
	dz₃ = l + z₃*(z₁ - m)

    SVector{9}(dx₁, dx₂, dx₃, dy₁, dy₂, dy₃, dz₁, dz₂, dz₃)
end

function mediated_link(u₀, ωx, ωy, ωz, k, l, m, c)
    p = [ωx, ωy, ωz, k, l, m, c]
    ContinuousDynamicalSystem(eom_mediated_link, u₀, p)
end

"""
    mediated_link(;u₀ = rand(9), ωx = 1, ωy = 1.015, ωz = 0.985,
        k = 0.15, l = 0.2, m = 10.0, c = 0.06) -> ContinuousDynamicalSystem

Equations of motion for a three-subsystem dynamical system where `X` and `Y` are
driven by `Z`. At the default value of the coupling constant `c = 0`, the
responses `X` and `Y` are already synchronized to the driver `Z`.

The dynamics is generated by the vector field:

```math
\\begin{aligned}
dx_1 &= -\\omega_x x_2 - x_3 + c*(z_1 - x_1) \\
dx_2 &= \\omega_x x_1 + k*x_2 \\
dx_3 &= l + x_3(x_1 - m) \\
dy_1 &= -\\omega_y y_2 - y_3 + c*(z_1 - y_1) \\
dy_2 &= \\omega_y y_1 + k*y_2 \\
dy_3 &= l + y_3(y_1 - m) \\
dz_1 &= -\\omega_z z_2 - z_3 \\
dz_2 &= \\omega_z z_1 + k*z_2 \\
dz_3 &= l + z_3(z_1 - m) \\
\end{aligned}
```

# References
Krakovská, Anna, et al. "Comparison of six methods for the detection of causality in a bivariate time series." Physical Review E 97.4 (2018): 042207
"""
mediated_link(;u₀ = rand(9), ωx = 1, ωy = 1.015, ωz = 0.985,
            k = 0.15, l = 0.2, m = 10.0, c = 0.06) =
    mediated_link(u₀, ωx, ωy, ωz, k, l, m, c)


"""
Log of the InverseGamma PDF. Equivalent to using Distributions; logpdf(InverseGamma(α, β), x)
https://en.wikipedia.org/wiki/Inverse-gamma_distribution
"""
function log_inverse_gamma(x::Real, α::Real=1., β::Real=1.; d::Integer=0)
    @assert 0 <= d <= 2
    if d == 0
        x > 0 ? val = -(β / x) - (1 + α) * log(x) + α * log(β) - loggamma(α) : val = -Inf
    elseif d == 1
        x > 0 ? val = (β / x - (1 + α)) / x : val = 0
    else
        x > 0 ? val = (-2 * β / x + (1 + α)) / (x * x) : val = 0
    end
    return val
end

function gamma_mode_std_2_alpha_theta(m::Real, s::Real)
    θ = (sqrt(m ^ 2 + 4 * s ^ 2) - m) / 2
    α = m / θ + 1
    return [α, θ]
end

"""
Log of the Gamma PDF. Equivalent to using Distributions; logpdf(Gamma(α, β), x)
https://en.wikipedia.org/wiki/Gamma_distribution
"""
function log_gamma(x::Real, parameters::Vector{<:Real}; d::Integer=0, passed_mode_std::Bool=false)
    @assert 0 <= d <= 2
    @assert length(parameters) == 2
    assert_positive(parameters)
    if passed_mode_std
        parameters = gamma_mode_std_2_alpha_theta(parameters[1], parameters[2])
    end
    α = parameters[1]
    θ = parameters[2]
    if d == 0
        x > 0 ? val = -(x / θ) + (α - 1) * log(x) - α * log(θ) - loggamma(α) : val = -Inf
    elseif d == 1
        x > 0 ? val = (α - 1) / x - 1 / θ : val = 0
    else
        x > 0 ? val = -(α - 1) / (x * x) : val = 0
    end
    return val
end

gauss_cdf(x::Real) = (1 + erf(x))/2

"log of the Gaussian PDF. Equivalent to using Distributions; logpdf(Gaussian(μ, σ), x)"
function log_gaussian(x::Real, parameters::Vector{<:Real}; d::Integer=0, min::Real=-Inf, max::Real=Inf)
    @assert 0 <= d <= 2
    @assert length(parameters) == 2
    μ = parameters[1]
    σ = parameters[2]
    assert_positive(σ)
    @assert min < max
    normalization = 1 - gauss_cdf(min - μ) - gauss_cdf(μ - max)
    if d == 0
        (min < x < max) ? val = -((x - μ)^2/(2 * σ * σ)) - log(sqrt(2 * π) * σ) - log(normalization) : val = -Inf
    elseif d == 1
        (min < x < max) ? val = -(x - μ)/(σ * σ) : val = 0
    else
        (min < x < max) ? val = -1 / (σ * σ) : val = 0
    end
    return val
end

"Log of the Uniform PDF."
function log_uniform(x::Real, min_max::Vector{<:Real}=[0,1]; d::Integer=0)
    @assert 0 <= d <= 2
    @assert length(min_max) == 2
    min, max = min_max
    @assert min < max
    if d == 0
        min <= x <= max ? -log(max - min) : -Inf
    else
        return 0
    end
end

"""
Log of the log-Uniform PDF.
Flattens out in log space starting at shift
Also known as a (modified in shifted case) Jeffrey's prior
"""
function log_loguniform(x::Real, min_max::Vector{<:Real}; d::Integer=0, shift::Real=0)
    @assert 0 <= d <= 2
    @assert length(min_max) == 2
    min, max = min_max
    @assert 0 < min + shift < max + shift
    xpshift = x + shift
    if d == 0
        min <= x <= max ? val = -log(xpshift) - log(log((max + shift)/(min + shift))) : val = -Inf
    elseif d == 1
        min <= x <= max ? val = -1 / xpshift : val = 0
    elseif d == 2
        min <= x <= max ? val = 1 / (xpshift * xpshift) : val = 0
    end
    return val
end

"""
Log of the Rayleigh PDF.
"""
function log_Rayleigh(x::Real, σ::Real; d::Integer=0)
    @assert 0 <= d <= 2
    if d == 0
        0 <= x <= 1 ? val = -(x * x / (2 * σ * σ)) + log(x / σ / σ) -  log(1 - exp(-1 / (2 * σ * σ))) : val = -Inf
    elseif d == 1
        0 <= x <= 1 ? val = 1 / x - x / σ / σ : val = 0
    elseif d == 2
        0 <= x <= 1 ? val = -1 / x / x - 1 / σ / σ : val = 0
    end
    return val
end

"""
Log of the 2D circle PDF
"""
function log_circle(x::Vector{<:Real}, min_max_r::Vector{<:Real}; d::Vector{<:Integer}=[0,0])
    @assert minimum(d) == 0
    @assert maximum(d) <= 2
    @assert sum(d) <= 2

    @assert length(x) == length(min_max_r) == length(d) == 2
    min_r, max_r = min_max_r

    # @assert min_r < sqrt(dot(x, x)) < max_r
    # @assert min_θ < atan(x[1], x[2]) < max_θ
    if all(d .== 0)
        min_r < sqrt(dot(x, x)) < max_r ? val = -log(2 * π * (max_r ^ 2 - min_r ^ 2)) : val = -Inf
    else
        return 0
    end
    return val
end

"""
Log of the 2D unit cone PDF
"""
function log_cone(x::Vector{<:Real}; d::Vector{<:Integer}=[0,0])
    @assert minimum(d) >= 0
    @assert maximum(d) <= 2
    @assert sum(d) <= 2

    @assert length(x) == length(d) == 2

    r_sq = dot(x, x)  # x^2 + y^2
    r = sqrt(r_sq)
    if d == [0,0]
        0 <= r < 1 ? val = log(3 / π * (1 - r)) : val = -Inf
    elseif d == [0,1]
        0 <= r < 1 ? val = x[2] / (r_sq - r) : val = 0
    elseif d == [0,2]
        0 <= r < 1 ? val = (-x[2] ^ 2 * r + x[1] ^ 2 * (r - 1)) /
            (r ^ 3 * (1 - 2 * r + r_sq)) : val = 0
    elseif d == [1,0]
        0 <= r < 1 ? val = x[1] / (r_sq - r) : val = 0
    elseif d == [1,1]
        0 <= r < 1 ? val = (x[1] * x[2] * (1 - 2 * r)) /
            (r ^ 3 * (1 - 2 * r + r_sq)) : val = 0
    elseif d == [2,0]
        0 <= r < 1 ? val = (-x[1] ^ 2 * r + x[2] ^ 2 * (r - 1)) /
            (r ^ 3 * (1 - 2 * r + r_sq)) : val = 0
    end
    return val
end

"""
Log of the 2D unit quadratic cone PDF
"""
function log_quad_cone(x::Vector{<:Real}; d::Vector{<:Integer}=[0,0])
    @assert minimum(d) >= 0
    @assert maximum(d) <= 2
    @assert sum(d) <= 2

    @assert length(x) == length(d) == 2

    r_sq = dot(x, x)  # x^2 + y^2
    r = sqrt(r_sq)
    if d == [0,0]
        0 <= r < 1 ? val = log(6 / π * (1 - 2 * r + r_sq)) : val = -Inf
    elseif d == [0,1]
        0 <= r < 1 ? val = 2 * x[2] / (r_sq - r) : val = 0
    elseif d == [0,2]
        0 <= r < 1 ? val = 2 * (-x[2] ^ 2 * r + x[1] ^ 2 * (r - 1)) /
            (r ^ 3 * (1 - 2 * r + r_sq)) : val = 0
    elseif d == [1,0]
        0 <= r < 1 ? val = 2 * x[1] / (r_sq - r) : val = 0
    elseif d == [1,1]
        0 <= r < 1 ? val = (2 * x[1] * x[2] * (1 - 2 * r)) /
            (r ^ 3 * (1 - 2 * r + r_sq)) : val = 0
    elseif d == [2,0]
        0 <= r < 1 ? val = 2 * (-x[1] ^ 2 * r + x[2] ^ 2 * (r - 1)) /
            (r ^ 3 * (1 - 2 * r + r_sq)) : val = 0
    end
    return val
end

"""
Log of the 2D unit cubic cone PDF
"""
function log_cubic_cone(x::Vector{<:Real}; d::Vector{<:Integer}=[0,0])
    @assert minimum(d) >= 0
    @assert maximum(d) <= 2
    @assert sum(d) <= 2

    @assert length(x) == length(d) == 2

    r_sq = dot(x, x)  # x^2 + y^2
    r = sqrt(r_sq)
    if d == [0,0]
        0 <= r < 1 ? val = log(10 / π * (1 - r)^3) : val = -Inf
    elseif d == [0,1]
        0 <= r < 1 ? val = 3 * x[2] / (r_sq - r) : val = 0
    elseif d == [0,2]
        0 <= r < 1 ? val = 3 * (-x[2] ^ 2 * r + x[1] ^ 2 * (r - 1)) /
            (r ^ 3 * (1 - 2 * r + r_sq)) : val = 0
    elseif d == [1,0]
        0 <= r < 1 ? val = 3 * x[1] / (r_sq - r) : val = 0
    elseif d == [1,1]
        0 <= r < 1 ? val = (3 * x[1] * x[2] * (1 - 2 * r)) /
            (r ^ 3 * (1 - 2 * r + r_sq)) : val = 0
    elseif d == [2,0]
        0 <= r < 1 ? val = 3 * (-x[1] ^ 2 * r + x[2] ^ 2 * (r - 1)) /
            (r ^ 3 * (1 - 2 * r + r_sq)) : val = 0
    end
    return val
end

"""
Log of the 2D rotated Rayleigh PDF that is cutoff at r=1
ONLY ROUGHLY NORMALIZED according to σ = 1/5
"""
function log_rot_Rayleigh(x::Vector{<:Real}; d::Vector{<:Integer}=[0,0], σ=1/5)
    @assert minimum(d) >= 0
    @assert maximum(d) <= 2
    @assert sum(d) <= 2

    @assert length(x) == length(d) == 2
    r_sq = dot(x, x)  # x^2 + y^2
    r = sqrt(r_sq)
    σ_sq = σ ^ 2
    log_norm = -2 * log(σ) - 0.454215
    if d == [0,0]
        0 <= r < 1 ? val = -r_sq / (2 * σ_sq) + log(r) + log_norm : val = -Inf
    elseif d == [0,1]
        0 <= r < 1 ? val = -x[2] * (r_sq - σ_sq) / (r_sq * σ_sq) : val = 0
    elseif d == [0,2]
        0 <= r < 1 ? val = -(x[1] ^ 4 + x[1] ^ 2 * (2 * x[2] ^ 2 - σ_sq) + x[2] ^ 2 * (x[2] ^ 2 + σ_sq)) / (r_sq ^ 2 * σ_sq) : val = 0
    elseif d == [1,0]
        0 <= r < 1 ? val = -x[1] * (r_sq - σ_sq) / (r_sq * σ_sq) : val = 0
    elseif d == [1,1]
        0 <= r < 1 ? val = -2 * x[1] * x[2] / r_sq ^ 2 : val = 0
    elseif d == [2,0]
        0 <= r < 1 ? val = -(x[2] ^ 4 + x[2] ^ 2 * (2 * x[1] ^ 2 - σ_sq) + x[1] ^ 2 * (x[1] ^ 2 + σ_sq)) / (r_sq ^ 2 * σ_sq) : val = 0
    end
    return val
end

"""
Log of the bivariate normal PDF
NOTE THAT THAT WHEN USING lows!=[-∞,...], THIS IS NOT PROPERLY NORMALIZED
"""
function log_bvnormal(x::Vector{T}, Σ::Cholesky{T,Matrix{T}}; μ::Vector{T}=zeros(T, length(x)), d::Vector{<:Integer}=[0,0], lows::Vector{T}=zeros(T, length(x)) .- Inf) where {T<:Real}
    @assert minimum(d) >= 0
    @assert maximum(d) <= 2
    @assert sum(d) <= 2

    @assert length(x) == length(d) == length(μ) == 2
    y = x - μ

    if sum(d) == 0
        all(x .>= lows) ? val = -nlogL(Σ, y) : val = -Inf
    elseif sum(d) == 1
        y1 = Float64.(d)
        all(x .>= lows) ? val = -dnlogLdθ(y1, Σ \ y) : val = 0
    elseif d == [0,2]
        y1 = y2 = [0, 1.]
        all(x .>= lows) ? val = -d2nlogLdθ(y2, [0,0.], Σ \ y, Σ \ y1) : val = 0
    elseif d == [1,1]
        y1 = [1, 0.]
        y2 = [0, 1.]
        all(x .>= lows) ? val = -d2nlogLdθ(y2, [0,0.], Σ \ y, Σ \ y1) : val = 0
    elseif d == [2,0]
        y1 = y2 = [1, 0.]
        all(x .>= lows) ? val = -d2nlogLdθ(y2, [0,0.], Σ \ y, Σ \ y1) : val = 0
    end
    return val
end
function bvnormal_covariance(σ11::Real, σ22::Real, ρ::Real)
    assert_positive(σ11, σ22)
    @assert 0 <= ρ <= 1
    v12 = σ11*σ22*ρ
    return ridge_chol([σ11^2 v12;v12 σ22^2])
end


import Distributions
import SpecialFunctions

"""    gaussianMechConstant(ϵ::Real,δ::Real)
Compute the proportionality constant κ(ϵ,δ) for the Gaussian mechanism.
"""
function gaussianMechConstant(ϵ::Real, δ::Real)
    K = sqrt(2) * SpecialFunctions.erfinv(1-2δ)
    return (K+sqrt(K^2+2ϵ))/(2ϵ)
end

"""    (k₁, k₂) = gaussianMechConstant2(ϵ::Real,δ::Real)
Compute the proportionality constant κ(ϵ,δ) for the Gaussian mechanism,
both with the formula using the inverse Q-function (k₁), and according to the
approximation of Theorem A.1 in Dwork and Roth's book (k₂, higher value).
"""
function gaussianMechConstant2(ϵ::Real, δ::Real)
    K = sqrt(2) * SpecialFunctions.erfinv(1-2δ)
    return ((K+sqrt(K^2+2ϵ))/(2ϵ), sqrt(2log(1.25/δ))/ϵ)
end

"""    mean_dp(x::Vector,epsilon::Float64,delta::Float64=0.0)
Compute the average of a set of numbers contained in the vector x,
in an (ϵ,δ)-differentially private way. It is assumed that each entry
x_i of the vector is in [0,1], and the adjacency relation looks at
arbitrary variations in one entry x_i, within this interval.
"""
function mean_dp(x::Vector, ϵ::Real, δ::Real=0.0)
    if δ == 0.0
        d = Distributions.Laplace(0, 1/(ϵ*length(x)))
    else
        d = Distributions.Normal(0, gaussianMechConstant(ϵ,δ)/length(x))
    end
    return Distributions.mean(x) + Distributions.rand(d)
end

"""    laplaceMech(x,f,l1sens::Real,ϵ::Real)
Computes a randomized version to f(x) according to the
Laplace mechanism, for ϵ-differential privacy. l1sens is
the l1-sensitivity of f, which must be computed and provided by the user.
f must take values in R^k, for some k, i.e., return an array of k real values.
"""
function laplaceMech(x, f, l1sens::Real, ϵ::Real)
    t = f(x)
    d = Distributions.Laplace(0, l1sens/ϵ)
    return (t .+ Distributions.rand(d, length(t)))
end

"""    gaussianMech(x,f,l2sens::Real,ϵ::Real,δ::Real)
Computes a randomized version to f(x) according to the
Gaussian mechanism, for (ϵ,δ)-differential privacy. l2sens is
the l2-sensitivity of f, which must be computed by the user.
f must take values in R^k, from some k, i.e., return an array
of k real values.
"""
function gaussianMech(x, f, l2sens::Real, ϵ::Real, δ::Real)
    t = f(x)
    d = Distributions.Normal(0, gaussianMechConstant(ϵ, δ) * l2sens)
    return (t .+ Distributions.rand(d, length(t)))
end

"""    truncatedLaplaceMech(x,f,l1sens::Real,ϵ::Real,δ::Real)
Computes a randomized version to f(x) according to the
truncated Laplace mechanism, for (ϵ,δ)-differential privacy.
This scheme adds bounded noise.
l1sens is the l1-sensitivity of f, which must be computed and provided by the
user. f must take values in R^k, for some k, i.e., return an array of k real
values.

Returns: (r, a) where r is a noisy version of f(x), and a defines the
support of the noise distribution (in the interval [f(x)-a, f(x)+a])
"""
function truncatedLaplaceMech(x, f, l1sens::Real, ϵ::Real, δ::Real)
    t = f(x)
    k = length(t)
    λ = l1sens/ϵ
    a = λ * log( 1 + exp(ϵ) * (k*(1-exp(-ϵ/k)))/(2*δ) )
    # alternative, more conservative but indpt of k
    # a = λ * log(1 + ϵ * exp(ϵ) / (2*δ))
    d1 = Distributions.Laplace(0, λ)
    d = Distributions.truncated(d1, -a, a)
    return (t .+ Distributions.rand(d, length(t)), a)
end


function test_softlabel_softmax_loss_layer(backend::Backend, tensor_dim, T, eps)
  println("-- Testing SoftlabelSoftmaxLossLayer on $(typeof(backend)){$T}...")

  dims = abs(rand(Int,tensor_dim)) % 6 + 6
  dims = tuple(dims...)
  op_dim = max(abs(rand(Int)) % tensor_dim, 1)
  println("    > $dims (operate on dimension $op_dim)")

  input = rand(T, dims) + 0.01
  input_blob = make_blob(backend, input)
  diff_blob = make_blob(backend, T, size(input))

  labels = abs(rand(T, dims)) + 0.01
  labels = labels ./ sum(labels, op_dim)
  label_blob = make_blob(backend, labels)

  inputs = Blob[input_blob, label_blob]

  loss_weight = abs(rand(T))
  layer = SoftlabelSoftmaxLossLayer(bottoms=[:pred, :labels], dim=op_dim, weight=loss_weight)
  state = setup(backend, layer, inputs, Blob[diff_blob])

  println("    > Forward")
  forward(backend, state, inputs)

  expected_loss = 0
  expected_grad = zeros(T, size(input))

  dim_pre, dim_prob, dim_post = split_dims(input, op_dim)
  canonical_input = reshape(input, dim_pre, dim_prob, dim_post)
  canonical_label = reshape(labels, dim_pre, dim_prob, dim_post)
  canonical_grad = reshape(expected_grad, dim_pre, dim_prob, dim_post)

  for i = 1:dim_pre
    for j = 1:dim_post
      pred = exp(canonical_input[i,:,j])
      pred /= sum(pred)

      expected_loss += sum(-log(pred) .* canonical_label[i,:,j])
      canonical_grad[i,:,j] = repmat(vec(pred), 1, dim_prob) * vec(canonical_label[i,:,j])
      canonical_grad[i,:,j] -= canonical_label[i,:,j]
    end
  end
  scale = dims[op_dim] / prod(dims)
  expected_loss *= scale * loss_weight
  expected_grad *= scale * loss_weight
  expected_grad = reshape(expected_grad, size(input))

  @test -eps < state.loss - expected_loss < eps

  println("    > Backward")
  backward(backend, state, inputs, Blob[diff_blob])
  grad = to_array(diff_blob)

  @test all(-eps .< grad - expected_grad .< eps)

  shutdown(backend, state)
end

function test_softlabel_softmax_loss_layer(backend::Backend, T, eps)
  for i in [2,4,5]
    test_softlabel_softmax_loss_layer(backend, i, T, eps)
  end
end
function test_softlabel_softmax_loss_layer(backend::Backend)
  test_softlabel_softmax_loss_layer(backend, Float64, 1e-5)
  test_softlabel_softmax_loss_layer(backend, Float32, 1e-3)
end
if test_cpu
  test_softlabel_softmax_loss_layer(backend_cpu)
end
if test_gpu
  test_softlabel_softmax_loss_layer(backend_gpu)
end