# logreg.jl
#
# Code for logistic regression, but this version uses mini-batches.
#
# In this case, for a triple (a, b, x), where a, x are n-vectors and b
# is {-/+ 1} valued, the objective is
#
#  F(x, a, b) = log(1 + exp(-b * a' * x)).

using Distributions;

# (A, b, x) = GenerateLogisticData(m::Int64 = 200, n::Int64 = 20;
#                                  condition_number::Float64 = 1.0,
#                                  snr::Float64 = 1.0)
#
# Generates logistic regression data where the data matrix A is m-by-n,
# the label (target) vector b is length m, and the distribution of b is
#
#  p(b | a, x) = exp(b * a' * x) / (1 + exp(b * a' * x)),
#
# for b in {-1, 1}. The parameter snr governs the mean magnitude of
# the inner products A * x, that is, the margins. (High snr means less
# noise in the labels).
function GenerateLogisticData(m::Int64 = 200, n::Int64 = 20;
                              condition_number::Float64 = 1.0,
                              snr::Float64 = 1.0)
  A = randn(m, n);
  if (condition_number > 1.0)
    # Make A have condition number given
    condition_vec = Vector(LinRange(1.0, condition_number, n));
    condition_vec = condition_vec / norm(condition_vec);
    A = A .* repeat(condition_vec', outer = (m, 1));
  end
  x = randn(n);
  mean_inner_product = mean(abs.(A * x));
  # Make it so that the mean inner product between data and x is equal
  # to snr (the signal to noise ratio).
  x = snr * x / mean_inner_product;
  margins = A * x;
  probs = 1 ./ (1 .+ exp.(-margins));
  b = sign.(rand(m) .- 1 + probs);
  return (A, b, x);
end

# o = LogisticRegressionObj(A::Matrix{Float64}, b::Vector{Float64},
#                           x::Vector{Float64})
#
# Computes and returns logistic regression objective (mean log loss).
function LogisticRegressionObj(A::Matrix{Float64}, b::Vector{Float64},
                               x::Vector{Float64})
  margins = b .* (A * x);
  pos_inds = (margins .> 0);
  neg_inds = (margins .<= 0);
  obj = (sum(log.(1 .+ exp.(-margins[pos_inds])))
         + sum(log.(1 .+ exp.(margins[neg_inds])) - margins[neg_inds]));
  return obj / size(A, 1);
end

# g = LogisticRegressionGrad(A::Matrix{Float64}, b::Vector{Float64},
#                            x::Vector{Float64})
#
# Computes and returns the gradient of the logistic regression loss.
function LogisticRegressionGrad(A::Matrix{Float64}, b::Vector{Float64},
                                x::Vector{Float64})
  margins = b .* (A * x);
  probs = b ./ (1 .+ exp.(margins));
  grad = -A' * probs;
  return grad / size(A, 1);
end

# x = SGUpdateLogistic(A::SubArray{Float64, 2}, b::SubArray{Float64, 1},
#                        x_init::Vector{Float64}, stepsize::Float64)
#
# Computes and applies a minibatch stochastic gradient update to the
# vector x_init for the logistic regression loss. The entire minibatch is
# in the SubArrays (A, b), which can be treated like matrices.
function SGUpdateLogistic(A::SubArray{Float64, 2}, b::SubArray{Float64, 1},
                          x_init::Vector{Float64}, stepsize::Float64)
  # Replace this code to return the correct update.
  return x_init;
end

# x = TruncatedUpdateLogistic(A::SubArray{Float64, 2},
#                             b::SubArray{Float64, 1},
#                             x_init::Vector{Float64}, stepsize::Float64)
#
# Performs a mini-batch update using the truncated model for logistic
# regression. In particular, letting a_i be the rows of the matrix A and
# b_i the associated labels, uses that the logistic loss has lower-bound 0
# to make the approximation
#
#  F_approx(x) = max{ F_0 + g' * (x - x_init), 0 }
#
# where F_0 = (1/m) sum_{i=1}^m log(1 + exp(-b_i * a_i' * x)) and
#
#   g = (d / dx) (1/m) sum_{i = 1}^m log(1 + exp(-b_i * a_i' * x))
#
# is the gradient on the mini-batch.
function TruncatedUpdateLogistic(A::SubArray{Float64, 2},
                                 b::SubArray{Float64, 1},
                                 x_init::Vector{Float64}, stepsize::Float64)
  # Replace this code to return the correct update.
  return x_init;
end

function TruncatedUpdateLogistic(a::Vector{Float64},
                                 b::Float64,
                                 x_init::Vector{Float64}, stepsize::Float64)
  # Replace this code to return the correct update.
  return x_init;
end

function SGUpdateLogistic(a::Vector{Float64}, b::Float64,
                          x_init::Vector{Float64}, stepsize::Float64)
  # Replace this code to return the correct update.
  return x_init;
end

Machine Learning Summer School 2019 (London, UK)/tutorials/optimization/logreg.jl


using JuMP
model = Model();

@variable(model, free_x)

@variable(model, keyword_x, lower_bound = 1, upper_bound = 2)

has_upper_bound(keyword_x)

upper_bound(keyword_x)

@variable(model, a[1:2, 1:2])

n = 10
l = [1; 2; 3; 4; 5; 6; 7; 8; 9; 10]
u = [10; 11; 12; 13; 14; 15; 16; 17; 18; 19]

@variable(model, l[i] <= x[i = 1:n] <= u[i])

@variable(model, y[i = 1:2, j = 1:2] >= 2i + j)

@variable(model, z[i = 2:3, j = 1:2:3] >= 0)

@variable(model, w[1:5,["red", "blue"]] <= 1)

@variable(model, u[i = 1:3, j = i:5])

@variable(model, v[i = 1:9; mod(i, 3) == 0])

@variable(model, integer_z, integer = true)

@variable(model, binary_z, binary = true)

@variable(model, psd_x[1:2, 1:2], PSD)

@variable(model, sym_x[1:2, 1:2], Symmetric)

model = Model()
@variable(model, x)
@variable(model, y)
@variable(model, z[1:10]);

@constraint(model, con, x <= 4)

@constraint(model, [i = 1:3], i * x <= i + 1)

@constraint(model, [i = 1:2, j = 2:3], i * x <= j + 1)

@constraint(model, [i = 1:2, j = 1:2; i != j], i * x <= j + 1)

for i in 1:3
    @constraint(model, 6x + 4y >= 5i)
end

@constraint(model, [i in 1:3], 6x + 4y >= 5i)

@constraint(model, sum(z[i] for i in 1:10) <= 1)

using GLPK

model = Model(GLPK.Optimizer)
@variable(model, x >= 0)
@variable(model, y >= 0)
set_objective_sense(model, MOI.MIN_SENSE)
set_objective_function(model, x + y)

optimize!(model)

@show objective_value(model);

objective_sense(model)

objective_function(model)

objective_function_type(model)

vector_model = Model(GLPK.Optimizer)

A= [ 1 1 9  5;
     3 5 0  8;
     2 0 6 13]

b = [7; 3; 5]

c = [1; 3; 5; 2]

@variable(vector_model, x[1:4] >= 0)
@constraint(vector_model, A * x .== b)
@objective(vector_model, Min, c' * x)

optimize!(vector_model)

@show objective_value(vector_model);


using Random,Distributions,StatsBase,Plots,HypothesisTests,Measures; pyplot()
Random.seed!(3)

dist = Gamma(2, 2.5)
distH0 = Exponential(5)
n = 100
data = rand(dist,n)

Fhat = ecdf(data)
diffF(dist, x) = sqrt(n)*(Fhat(x) - cdf.(dist,x))
xGrid = 0:0.001:30
ksStat = maximum(abs.(diffF(distH0, xGrid)))

M = 10^5
KScdf(x) = sqrt(2pi)/x*sum([exp(-(2k-1)^2*pi^2 ./(8x.^2)) for k in 1:M])

println("p-value calculated via series: ",
	1-KScdf(ksStat))
println("p-value calculated via Kolmogorov distribution: ",
	1-cdf(Kolmogorov(),ksStat),"\n")

println(ApproximateOneSampleKSTest(data,distH0))

p1 = plot(xGrid, Fhat(xGrid),
	c=:black, lw=1, label="ECDF from data")
p1 = plot!(xGrid, cdf.(dist,xGrid),
	c=:blue, ls=:dot, label="CDF under \n actual distribution")
p1 = plot!(xGrid, cdf.(distH0,xGrid),
	c=:red, ls=:dot, label="CDF under \n postulated H0",
	xlims=(0,20), ylims=(0,1), xlabel = "x", ylabel = "Probability")

p2= plot(cdf.(dist,xGrid), diffF(dist, xGrid),lw=0.5,
	c=:blue,	label="KS Process under \n actual distribution")
p2 = plot!(cdf.(distH0,xGrid), diffF(distH0, xGrid), lw=0.5,
	c=:red, xlims=(0,1), label="KS Process under \n postulated H0",
    xlabel = "t", ylabel = "K-S Process")

plot(p1, p2, legend=:bottomright, size=(800, 400), margin = 5mm)


function counting_sort_letters(A, pos) # A:Array, p:position
    n = length(A)
    k = 26 # number of letters in alphabet
    start_ascii = Int('a'[1]) # Ascii decimal of a
    C = fill(0,k) # initialize counter array to 0
    # Storing count for each letter
    for i = 1:n
        if (length(A[i]) < pos) return A end
        counterIdx = Int(A[i][pos])-start_ascii+1 # -a_ascii+1 to start from 1
        C[counterIdx] = C[counterIdx] + 1
    end
    # Changing C[j] so it contains actual position of all each element
    for j = 2:(k)
        C[j] = C[j] + C[j-1]
    end
    B = fill("z",n) # initialize return array
    for i = n:-1:1
        counterIdx = Int(A[i][pos])-start_ascii+1
        B[C[counterIdx]] = A[i] # Always adds next string to first element of B
        C[counterIdx] = C[counterIdx] - 1 # Reduce the count of current element by 1
    end
    return B
end

function counting_sort_length(A)
    n = length(A)
    k = 0 # Max string length in array A
    for i = 1:n # find the longest string
        if (length(A[i]) > k) k = length(A[i]) end
    end
    C = fill(0,k) # initialize counter array to 0
    for i = 1:n # counter
        counterIdx = length(A[i])
        C[counterIdx] = C[counterIdx] + 1
    end
    # Changing C[j] so it contains actual position of all each element
    for j = 2:(k)
        C[j] = C[j] + C[j-1]
    end
    B = fill("z",n) # initialize return array
    for i = n:-1:1
        counterIdx = length(A[i])
        B[C[counterIdx]] = A[i]  # Always adds next string to first element of B
        C[counterIdx] = C[counterIdx] - 1 # Reduce the count of current element by 1
    end
    return B
end

function flexradix(A, max_length)
    A = counting_sort_length(A)

    # SPLIT UP ARRAY AND RETURN ONLY THE SIDE WHERE EVERY ELEMENT HAS
    # STRING SIZE BIGGER THAN P??

    for p = 1:max_length # p: position
        A = counting_sort_letters(A, p)
    end
    return A

    #=
    n = length(A)
    C = fill(0,max_length) # initialize counter array to 0
    for i = 1:n # counter
        counterIdx = length(A[i])
        C[counterIdx] = C[counterIdx] + 1
    end
    # Now we have a sorted list by length, and the number of lengths. We can then
    # iterate through the lengths and ommit the previous smaller arrays as
    # the problem developes
    for i = 1:max_length
        A =
        C = fill(0,n) # Counter for strings with different sizes
        A = counting_sort_letters(A, i)
        C[counterIdx] = C[counterIdx] - 1 # Reduce the count of current element by 1
    end
    =#
end


##
using IFGF
using LinearAlgebra
using StaticArrays
using Random
using Plots
plotlyjs()
Random.seed!(1)

includet(joinpath(IFGF.PROJECT_ROOT,"test","testutils.jl"))

k    = 8π
λ    = 2π/k
ppw  = 10
dx   = λ/ppw

Xpts = Ypts = sphere_uniform_surface_mesh(dx)
npts = length(Xpts)

K    = HelmholtzKernel(k)
T    = ComplexF64

I   = rand(1:npts,100)
B   = randn(T,npts)
exa = [sum(K( Xpts[i],Ypts[j])*B[j] for j in 1:npts) for i in I]

# trees
splitter   = Trees.CardinalitySplitter(;nmax=200)

# cone size function
ds_func = IFGF.cone_domain_size_func(k,(1.0,π/2,π/2))

# loop over interpolation order
pvec = 2:8
er   = []
for p in pvec
    ptuple  = (p,p,p)
    A       = IFGFOp(K,Xpts,Ypts;splitter,p=ptuple,ds_func,threads=true)
    C       = zeros(T,npts)
    mul!(C,A,B,1,0;threads=true)
    ee = norm(C[I]-exa,2) / norm(exa,2)
    push!(er,ee)
end

print(
    "Uniform points on surfaces of sphere with parameters:
        r   = 1
        k   = $k
        n   = $npts
        ppw = $ppw"
)

fig = plot(pvec,er,yscale=:log10,m=:circle,
    ylabel="approximate ℓ₂ relative error",
    xlabel="p",
    label="sphere")

dir = @__DIR__
savefig(fig,joinpath(dir,"sphere_p_convergence.png"))


# module Angles

export
    wrap_to_2pi,
    wrap_to_pi,
    angular_difference

wrap_to_2pi(θ) = mod(θ,2π)

"""
    `wrap_to_pi(θ₀)`

    wraps the angle θ₀ to a value in (-π,π]
"""
function wrap_to_pi(θ₀)
    # θ = θ₀
    θ = wrap_to_2pi(θ₀)
    # while θ > π
    if θ > π
        θ = θ - 2π
    # end
    # while θ <= -π
    elseif θ <= -π
        θ += 2π
    end
    return θ
end

function angular_difference(θ1,θ2)
    t1 = wrap_to_2pi(θ1)
    t2 = wrap_to_2pi(θ2)
    if t2 < t1
        return -angular_difference(t2,t1)
    end
    wrap_to_pi(t2 - t1)
end

"""
    intersections_between_circles(v1,v2,r1,r2)
"""
function intersections_between_circles(v1,v2,r1,r2)
    d = norm(v1-v2)
    if d > r1+r2
        return nothing
    elseif d < 1e-4 # almost collocated
        return nothing
    elseif d + r1 < r2 || d + r2 < r1 # completely contained
        return nothing
    else
        x1,y1 = v1[1], v1[2]
        x2,y2 = v2[1], v2[2]
        l = (r1^2 - r2^2 + d^2)/(2*d)
        h = sqrt(r1^2-l^2)
        v0 = (l/d) * [(x2-x1), (y2-y1)] .+ v1
        dv = (h/d) * [(y2-y1), -(x2-x1)]

        return v0 .+ dv, v0 .- dv
    end
end

function nearest_points_between_circles(v1,v2,r1,r2)
    d = norm(v1-v2)
    vec = v2 .- v1
    if d > r1+r2
        return v1 + normalize(vec) * r1,  v2 - normalize(vec) * r2
    elseif d + r1 < r2 # circle 1 contained in circle 2
        return v1 - normalize(vec) * r1,  v2 - normalize(vec) * r2
    elseif d + r2 < r1 # circle 2 contained in circle 1
        return v1 + normalize(vec) * r1,  v2 + normalize(vec) * r2
    elseif d < 1e-4 # almost collocated
        return nothing
    else
        return intersections_between_circles(v1,v2,r1,r2)
    end
end


using IterativeSolvers

A = Float64[
  16 4 8;
  4 5 -4;
  8 -4 22;
]
a = Float64[16; 18; -22]
r = vcat(collect(1:6), collect(10:10:100))
println()

x_j = [(i=i, j=jacobi(A, a; maxiter=i)) for i in r]
A * x_j[11].j |> display
println()

x_gs = [(i=i, gs=gauss_seidel(A, a; maxiter=i)) for i in r]
A * x_gs[11].gs |> display
println()

ω = 1.0
x_sor = [(i=i, sor=sor(A, a, ω; maxiter=i)) for i in r]
A * x_sor[11].sor |> display
println()

x_ssor = [(i=i, ssor=ssor(A, a, ω; maxiter=i)) for i in r]
x_ssor |> display
println()
A * x_ssor[11].ssor |> display
println()


@doc raw"""
    LinearMixingModelKernel(k::Kernel, H::AbstractMatrix)
    LinearMixingModelKernel(Tk::AbstractVector{<:Kernel},Th::AbstractMatrix)

Kernel associated with the linear mixing model, taking a vector of `m` kernels and a `m × p` matrix H for a function with `p` outputs. Also accepts a single kernel `k` for use across all `m` basis vectors.

# Definition

For inputs ``x, x'`` and output dimensions ``p_x, p_{x'}'``, the kernel is defined as[^BPTHST]
```math
k\big((x, p_x), (x, p_{x'})\big) = H_{:,p_{x}}K(x, x')H_{:,p_{x'}}
```
where ``K(x, x') = Diag(k_1(x, x'), ..., k_m(x, x'))`` with zero off-diagonal entries.
``H_{:,p_{x}}`` is the ``p_x``-th column (`p_x`-th output) of ``H \in \mathbb{R}^{m \times p}``
representing ``m`` basis vectors for the ``p`` dimensional output space of ``f``.
``k_1, \ldots, k_m`` are ``m`` kernels, one for each latent process, ``H`` is a
mixing matrix of ``m`` basis vectors spanning the output space.

[^BPTHST]: Wessel P. Bruinsma, Eric Perim, Will Tebbutt, J. Scott Hosking, Arno Solin, Richard E. Turner (2020). [Scalable Exact Inference in Multi-Output Gaussian Processes](https://arxiv.org/pdf/1911.06287.pdf).
"""
struct LinearMixingModelKernel{Tk<:AbstractVector{<:Kernel},Th<:AbstractMatrix} <: MOKernel
    K::Tk
    H::Th
    function LinearMixingModelKernel(Tk::AbstractVector{<:Kernel}, H::AbstractMatrix)
        @assert length(Tk) == size(H, 1) "Number of kernels and number of rows in H must match"
        return new{typeof(Tk),typeof(H)}(Tk, H)
    end
end

function LinearMixingModelKernel(k::Kernel, H::AbstractMatrix)
    return LinearMixingModelKernel(Fill(k, size(H, 1)), H)
end

function (κ::LinearMixingModelKernel)((x, px)::Tuple{Any,Int}, (y, py)::Tuple{Any,Int})
    (px > size(κ.H, 2) || py > size(κ.H, 2) || px < 1 || py < 1) &&
        error("`px` and `py` must be within the range of the number of outputs")
    return sum(κ.H[i, px] * κ.K[i](x, y) * κ.H[i, py] for i in 1:length(κ.K))
end

function Base.show(io::IO, k::LinearMixingModelKernel)
    return print(io, "Linear Mixing Model Multi-Output Kernel")
end

function Base.show(io::IO, mime::MIME"text/plain", k::LinearMixingModelKernel)
    print(io, "Linear Mixing Model Multi-Output Kernel. Kernels:")
    for k in k.K
        print(io, "\n\t")
        show(io, mime, k)
    end
end


function num_conbination(n::Int64, d::Int64)
    div(factorial(n+d), factorial(n)*factorial(d)) - 1
end

"""
    make_design_matrix(x, dims)
This function return the design matrix.

# Parameters
- `x`: input data
- `dims`: degree of design matrix

# Example
```jldoctest regression
julia> make_design_matrix(x, dims = 2) |> size
(20, 5)
```
"""
function make_design_matrix(x; dims = 2)
    n_datas, n_features = size(x)
    out_features = num_conbination(n_features, dims)
    y = Array{Float64}(undef, n_datas, out_features)
    current = 1
    y[:, current : current+n_features-1] = x
    index = collect(current : current+n_features-1)
    current += n_features
    push!(index, current)
    for _ in 2 : dims
        new = []
        stop = index[end]
        for feature_idx in 1 : n_features
            start = index[feature_idx]
            push!(new, current)
            next= current + stop - start
            next <= current && break
            @. y[:, current : next-1] = y[:, start:stop-1] * x[:, feature_idx]
            current = next
        end
        push!(new, current)
        index = new
    end
    return y
end


fib(n) = n < 2 ? n : fib(n-1) + fib(n-2)

fibTailRecursive(a, b=0, c=1) = a === 0 ? b : fibTailRecursive(a - 1, c, b + c)

fibFormula(n) = round(((5 ^ .5 + 1) / 2) ^ n / (5 ^ .5))

function fibLinear(n)
  x,y = (0,1)
  for i = 1:n
    x,y = (y, x+y)
  end
  return x
end

function fibMatrix(n)
  prevMatrix = BigInt[1 1; 1 0]
  matrix = prevMatrix ^ n
  matrix[2, 1]
end

input = length(ARGS) === 0 ? 29 : parse(Int, ARGS[1])
println(fibMatrix(input))


# Compute F-statistic, given vector of vectors
#ANOVA
function f(R)
    X = vcat(R...)
    N = length(X)
    nj = length.(R)
    X̅ = mean(X)
    X̅j = mean.(R)
    k = length(R)
    top = sum(nj .*(X̅j .- X̅) .^2) /(k-1)
    bottom = sum([sum((R[i] .- X̅j[i]) .^2) for i in 1:k])/(N-k)
    return top/bottom
end
function f_slow(R,N,nj,X̅,k)
    X̅j = mean.(R)
    top = sum(nj .*(X̅j .- X̅) .^2) /(k-1)
    bottom = sum([sum((R[i] .- X̅j[i]) .^2) for i in 1:k])/(N-k)
    return top/bottom
end

function f(R,N,nj,X̅,k)
    X̅j = mean.(R)
    top = 0.0
    bottom = 0.0
    @inbounds for i in 1:k
        top += nj[i] * (X̅j[i] - X̅)^2
        for j in 1:nj[i]
            bottom += (R[i][j] - X̅j[i])^2
        end
    end
    top/= (k-1)
    bottom /= (N-k)
    return top/bottom
end

#Pairwise ANOVA
function f_pairs(R)
    n = length(R)
    F = zeros(n,n)
    for j in 1:n-1
        for i in j+1:n
            F[i,j] = f([R[i],R[j]])
        end
    end
    return F
end
function f_pairs(R ::Vector,N ::Array{Int},nj ::Matrix{Tuple},X̅ ::Matrix{Float64})
    n = length(R)
    F = zeros(n,n)
    for j in 1:n-1
        for i in j+1:n
            F[i,j] = f([R[i],R[j]],N[i,j],nj[i,j],X̅[i,j],2)
        end
    end
    return F
end
# get parameter values corresponding to each pairwise combination of groups
function get_pars(R)
    n = length(R)
    N_p = Array{Int}(undef,n,n)
    nj_p= Array{Tuple}(undef,n,n)
    X̅_p = Array{Float64}(undef,n,n)

    for j in 1:n-1
        for i in j+1:n
            N_p[i,j] = length(R[i]) + length(R[j])
            nj_p[i,j] = (length(R[i]) , length(R[j]))
            X̅_p[i,j] = mean(vcat(R[i]...,R[j]...))
        end
    end
    return N_p,nj_p,X̅_p
end


using SparseArrays, LinearAlgebra
"""
    as_to_bs(a1, a2)

Calculate reciprocal lattice vectors, `b1` and `b2`, from the real space lattice
vectors, `a1` and `a2`.
"""
function as_to_bs(a1, a2)
    b1 = 2pi * [+a2[2],-a2[1]] / (+a1[1]*a2[2]-a1[2]*a2[1])
    b2 = 2pi * [-a1[2],+a1[1]] / (-a2[1]*a1[2]+a2[2]*a1[1])
    return b1, b2
end

"""
    bs_to_as(b1, b2)

Calculate real space lattice vectors, `a1` and `a2`, from reciprocal lattice
vectors, `b1` and `b2`.

This is actually the same as `as_to_bs`, but I think having both functions makes
the intention of the code more obvious.
"""
function bs_to_as(b1, b2)
    return as_to_bs(b1, b2)
end

"""
    DEX, DEY, DHX, DHY = diff_yee2(NGRID,RES,BC,kinc=nothing)
=================
- `NGRID`: [Nx Ny] grid size.
- `RES`: [dx dy] grid resolution of the 1X grid.
- `BC`: [xbc ybc] boundary conditions, `-2`: periodic (requires kinc), `0`: Dirichlet.
- `kinc`: [kx ky] incident wave vector. This argument is only needed for periodic boundaries.
Position on the grid: m = (ny - 1)*Nx + nx
"""
function diff_yee2(NGRID::Matrix{Float64}, RES::Matrix{Float64}, BC::Matrix{Int64}, a1::Array{<:Real,1}, a2::Array{<:Real,1}, kinc=nothing)
    if kinc != nothing
        kinc0 = kinc
    end
    p(nx,ny,NX)::Int64 = nx+NX*(ny-1)
    dx = RES[1]; dy = RES[2]
    NX = NGRID[1]; NY = NGRID[2]; N = NX*NY;
    Lamx = NX * dx
    Lamy = NY * dy
    N = convert(Int64,N);NX = convert(Int64,NX);NY = convert(Int64,NY)
    if NX == 1
        if BC[1] == 0
            DEX = spzeros(N, N)
        else
            DEX = spdiagm(0 => im*kinc0[2]*ones(N))
        end
    else
        DEX = spdiagm(0 => -1 ./ dx .* ones(ComplexF64, N), 1 => 1 ./ dx .* ones(ComplexF64, N - 1))
        for ny=1:(NY-1)
            DEX[p(NX,ny,NX),p(NX+1,ny,NX)] = 0  # Dirichlet BC
        end
        if BC[1]==-2
            for ny=1:NY
                DEX[p(NX,ny,NX),p(1,ny,NX)]=exp(-im*transpose(kinc0)*a1*Lamx)/dx # periodic BC
            end

        end
    end

    if NY==1
        if BC[2] == 0
            DEY = spzeros(N,N)
        else
            DEY = spdiagm(0 => im*kinc0[1]*ones(N))
        end
    else
        DEY = spdiagm(0 => -1 ./dy.*ones(N), NX => 1 ./dy.*ones(N - NX)) .+ 0*im
        if BC[2]==-2
            for nx=1:NX
                DEY[p(nx,NY,NX),p(nx,1,NX)]=exp(-im*transpose(kinc0)*a2*Lamy)/dy # periodic BC
            end
        end
    end

    DHX=-DEX'
    DHY=-DEY'
    return DEX,DEY,DHX,DHY
end

# """
#     DiagonalMatrix(diag::AbstractVector{ComplexF64})

# A sparse diagonal matrix that can be used in left division (D \\ X)
# """
# struct DiagonalMatrix <: AbstractMatrix{ComplexF64}
#     diag::AbstractVector{ComplexF64}
# end
# Base.size(A::DiagonalMatrix) = (length(A.diag), length(A.diag))
# Base.getindex(A::DiagonalMatrix, I::Vararg{Int,2}) = I[1]==I[2] ? A.diag[I[1]] : 0

eye(M) = Matrix{ComplexF64}(I, M, M)
diag_R(mat) = mat[diagind(mat)]

function mask(M)
    mat = zeros(M,M)
    for i = -(M-1):M-1
        mat = mat + diagm(i=>(-1)^(i)*ones(M-abs(i)))
    end
    return mat
end


"""
mutable struct InverseDistanceSurrogate{X,Y,P,L,U} <: AbstractSurrogate

The square polynomial model can be expressed by 𝐲 = 𝐗β + ϵ, with β = 𝐗ᵗ𝐗⁻¹𝐗ᵗ𝐲
"""
mutable struct SecondOrderPolynomialSurrogate{X,Y,B,L,U} <: AbstractSurrogate
    x::X
    y::Y
    β::B
    lb::L
    ub::U
end

function SecondOrderPolynomialSurrogate(x,y,lb,ub)
    X = _construct_2nd_order_interp_matrix(x, first(x))
    Y = _construct_y_matrix(y, first(y))
    β = X\Y
    return SecondOrderPolynomialSurrogate(x, y, β, lb, ub)
end

function _construct_2nd_order_interp_matrix(x, x_el)
    n = length(x)
    d = length(x_el)
    D = 1 + 2*d + d*(d-1)÷2
    X = ones(eltype(x_el), n, D)
    for i = 1:n, j = 1:d
        X[i, j+1] = x[i][j]
    end
    idx = d + 1
    for j = 1:d, k = j+1:d
        idx += 1
        for i = 1:n
            X[i, idx] = x[i][j]*x[i][k]
        end
    end
    for i = 1:n, j = 1:d
        X[i, j+1+d+d*(d-1)÷2] = x[i][j]^2
    end
    return X
end

_construct_y_matrix(y, y_el::Number) = y
_construct_y_matrix(y, y_el) = [y[i][j] for i=1:length(y), j=1:length(y_el)]

function (my_second_ord::SecondOrderPolynomialSurrogate)(val)
    #just create the val vector as X and multiply
    d = length(val)

    y = my_second_ord.β[1, :]
    for j = 1:d
        y += val[j]*my_second_ord.β[j+1, :]
    end
    idx = d + 1
    for j = 1:d, k = j+1:d
        idx += 1
        y += val[j] * val[k] * my_second_ord.β[idx, :]
    end
    for j = 1:d
        y += val[j]^2 * my_second_ord.β[j+1+d+d*(d-1)÷2, :]
    end
    return _match_container(y, first(my_second_ord.y))
end

function add_point!(my_second::SecondOrderPolynomialSurrogate, x_new, y_new)
    if eltype(x_new) == eltype(my_second.x)
        append!(my_second.x, x_new)
        append!(my_second.y, y_new)
    else
        push!(my_second.x, x_new)
        push!(my_second.y, y_new)
    end
    X = _construct_2nd_order_interp_matrix(my_second.x, first(my_second.x))
    Y = _construct_y_matrix(my_second.y, first(my_second.y))
    β = X\Y
    my_second.β = β
    nothing
end


# Copyright (c) 2021
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

using LinearAlgebra;

"""
    SL0(A::Matrix{Float64}, b::Vector{Float64}; maxiter=100, epsilon=0.01, sigma_decrease_factor=0.85)

Find the solution to Ax=b using Iterative Recursive Least Squares.

### Input

- `A`       -- Matrix: Ax=b
- `b`       -- Vector: Ax=b
- `sigma_decrease_factor` -- (optional) number of optmization iterations
- `epsilon` -- (optional) threshold to stop optimizing
- `maxiter` -- (optional) max number of iterations

### Output

Solution to Ax=b (Vector{Float64})

### Algorithm

Smoothed L0 (http://ee.sharif.edu/~SLzero/)

"""
function SL0(A::Matrix{Float64},
             b::Vector{Float64};
             sigma_decrease_factor=.85,
             maxiter=150,
             epsilon=.001)

    # set up the locals
    local mu_0, L, A_pinv, s, x;

    # assign constants
    mu_0 = 2;          # The  value  of  mu_0  scales  the sequence of mu
    L = 3;             # number  of  iterations of the internal (steepest ascent) loop
    A_pinv = pinv(A);  # pseudo-inverse of matrix A defined by A_pinv=A'*inv(A*A')

    # initialize the solution
    s = A_pinv*b;
    sigma = 2*maximum(abs.(s));

    for j = 1:maxiter

        for i = 1:L
            delta = s.*exp.(-abs.(s).^2/sigma^2);
            s -= mu_0*delta;
            s -= A_pinv*(A*s - b);
        end
        sigma *= sigma_decrease_factor;
    end
    x = s;
    i = abs.(x) .< epsilon;
    x[i] = zeros(sum(i));
    return x
end

# A = randn(10, 50); b = randn(10);
# x = SL0(A, b);
# println(x)


using Plots,Statistics,LaTeXStrings,JLD,LsqFit
function NeighborReturner(network_, i, j)
    neighbors=[]
    if j!=1 && network_[i,j-1]!=0
        push!(neighbors, network_[i,j-1]) # the left neighbor--> neighbor[1]
    end
    if i!=1 && network_[i-1,j]!=0 #if true --> the neighbors[2] is the upper neighbor
        push!(neighbors,network_[i-1,j])
    end
    return neighbors
end
function LabelFinder(a, L)
    while L[a] != L[L[a]]
        a = L[a]
    end
    return L[a]
end
function percolation_check(network_,L,dim)
    first_col=[]
    last_col=[]
    for i in 1:dim
        if network_[i,1]!=0
            push!(first_col,LabelFinder(network_[i,1],L))
        end
    end
    for i in 1:dim
        if network_[i,dim]!=0
            push!(last_col,LabelFinder(network_[i,dim],L))
        end
    end
    return intersect(first_col,last_col)
end
function percolation(dim,p)
    network_=zeros(Int, dim, dim)
    L=[]
    S=[]
    counter=1
    for col in 1:dim
        for row in 1:dim
            if p>rand()
                neighbors=NeighborReturner(network_, row, col)
                if length(neighbors)==0
                    network_[row,col]=counter
                    push!(L, counter)
                    push!(S, 1)
                    #println(S)
                    counter+=1
                elseif length(neighbors)==1
                    network_[row,col]=LabelFinder(neighbors[1], L)
                    S[LabelFinder(neighbors[1], L)]+=1
                elseif length(neighbors)==2 && LabelFinder(neighbors[1],L)==LabelFinder(neighbors[2],L)
                    network_[row,col]=LabelFinder(neighbors[1], L)
                    S[LabelFinder(neighbors[1], L)]+=1
                else
                    S[LabelFinder(neighbors[1], L)]+=1+S[LabelFinder(neighbors[2], L)]
                    S[LabelFinder(neighbors[2], L)]=0
                    network_[row,col]=LabelFinder(neighbors[1], L)
                    L[neighbors[2]]=LabelFinder(neighbors[1], L)
                end
            end
        end
    end
    return network_,L,S
end
function RadiusOfGyration(network_,L,S,dim)
    if length(Set(S)) < 3
        return 0.0
    end
    if findall(x->x==maximum(S),S)[1] ∈ percolation_check(network_,L,dim)
        S[findall(x->x==maximum(S),S)[1]]=0 #the infinit cluster is omitted from S in this way.
    end
    cluster_size=maximum(S)
    S_max=findall(x->x==maximum(S),S)[1]
    numerator=0
    i_nums=[]
    j_nums=[]
    for coordinate_num in findall(x->x==S_max,network_)
        push!(i_nums, coordinate_num[1])
        push!(j_nums, coordinate_num[2])
    end
    i_com=mean(i_nums)
    j_com=mean(j_nums)
    TheFraction_list=[]
    for i in 1:dim
        for j in 1:dim
             if network_[i, j]==S_max
                push!(TheFraction_list,((i-i_com)^2 + (j-j_com)^2))
            end
        end
    end
    RadiusOfGyration=sqrt(mean(TheFraction_list))
    return RadiusOfGyration
end
dim_list=[10,20,40,80,160]
SavedData_std=[]
SavedData_mean=[]
for n in 1:5
    dim=dim_list[n]
    Meanlist = []
    STDlist = []
    probability=[hcat(0:0.03:0.25)...,hcat(0.25:0.02:0.75)...,hcat(0.75:0.03:1)...]
    for p in probability
        xi=[]
        for run_num in 1:1000
            network_,L,S=percolation(dim,p)
            push!(xi,RadiusOfGyration(network_,L,S,dim))
        end
        push!(STDlist, std(xi))
        push!(Meanlist, mean(xi))
    end
    push!(SavedData_std, STDlist)
    push!(SavedData_mean, Meanlist)
end
save("C:\\Users\\Narges\\Documents\\GitHub\\computational_physics\\chapter4\\4.5\\AllData_HK.jld",
"std_data", SavedData_std,
"mean_data", SavedData_mean)

#creating the figures:
load("C:\\Users\\Narges\\Documents\\GitHub\\computational_physics\\chapter4\\4.5\\AllData_HK.jld")
#plot withough error bars:
plot(dpi=400)
for i in 1:5
    dim=dim_list[i]
    scatter!(probability, SavedData_mean[i][1:43],label=nothing,markersize=3,c=:black,alpha=0.3)
    plot!(probability, SavedData_mean[i][1:43],label=L"L=%$dim")

end
scatter!(xlabel="P", ylabel=L"\xi", title=L"\xi\_ P")
savefig("C:\\Users\\Narges\\Documents\\GitHub\\computational_physics\\chapter4\\Fig\\4.5_plot.png")
#scatter plot including error bars:
plot(dpi=400)
for i in 1:5
    dim=dim_list[i]
    scatter!(probability, SavedData_mean[i][1:43],yerr=SavedData_std[i][1:43],markersize=3,
    alpha=0.7,label=L"L=%$dim")

end
scatter!(xlabel="P", ylabel=L"\xi", title=L"\xi\_ P")
savefig("C:\\Users\\Narges\\Documents\\GitHub\\computational_physics\\chapter4\\Fig\\4.5_scatter.png")
#getting Pc(∞) and ν:
xdata=[]
ydata=dim_list
for i in 1:5
    Pc_L= probability[findall(x->x==maximum(SavedData_mean[i]),SavedData_mean[i])[1]]
    push!(xdata, Pc_L)

end
println(xdata)
println(ydata)
@.model(x,p)=abs(x-p[1])^(-p[2])
p0=[1.3, 0.59]
fit=curve_fit(model, xdata, ydata, p0)
#result: ν-->0.61 , Pc(∞)-->1.02


##### OVERRIDES FOR EFFICIENCY / CORRECTNESS

function add_vertices!(g::AbstractSimpleWeightedGraph, n::Integer)
    T = eltype(g)
    U = weighttype(g)
    (nv(g) + one(T) <= nv(g)) && return false       # test for overflow
    emptycols = spzeros(U, nv(g) + n, n)
    g.weights = hcat(g.weights, emptycols[1:nv(g), :])
    g.weights = vcat(g.weights, emptycols')
    return true
end

function degree_matrix(g::AbstractSimpleWeightedGraph, T::DataType=weighttype(g); dir::Symbol=:out)
    if is_directed(g)
        if dir == :out
            d = vec(sum(g.weights, dims=1))
        elseif dir == :in
            d = vec(sum(g.weights, dims=2))
        elseif dir == :both
            d = vec(sum(g.weights, dims=1)) + vec(sum(g.weights, dims=2))
        else
            throw(DomainError(dir, "invalid argument, only accept :in, :out and :both"))
        end
    else
        d = vec(sum(g.weights, dims=1))
    end
    return SparseMatrixCSC(T.(diagm(0=>d)))
end

function adjacency_matrix(g::AbstractSimpleWeightedGraph, T::DataType=weighttype(g); dir::Symbol=:out)
    if dir == :out
        return SparseMatrixCSC(T.(copy(g.weights))')
    else
        return T.(copy(g.weights))
    end
end

function laplacian_matrix(g::AbstractSimpleWeightedGraph, T::DataType=weighttype(g); dir::Symbol=:out)
    degree_matrix(g, T; dir=dir) - adjacency_matrix(g, T; dir=dir)
end

function pagerank(g::SimpleWeightedDiGraph, α=0.85, n::Integer=100, ϵ=1.0e-6)
    A = weights(g)
    S = vec(sum(A, dims=1))
    S = 1 ./ S
    S[findall(S .== Inf)] .= 0.0
    M = A'  # need a separate line due to bug #17456 in julia
    # scaling the adjmat to stochastic adjacency matrix
    M = (Diagonal(S) * M)'
    N = Int(nv(g))
    # solution vector
    x = fill(1.0 / N, N)
    # personalization vector
    p = fill(1.0 / N, N)
    # temporary to hold the results of SpMV
    y = zeros(Float64, N)
    # adjustment for leaf nodes in digraph
    dangling_weights = p
    is_dangling = findall(S .== 0)
    # save some flops by precomputing this
    pscaled = (1 .- α) .* p
    for _ in 1:n
        xlast = x
        # in place SpMV to conserve memory
        mul!(y, M, x)
        # using broadcast to avoid temporaries
        x = α .* (y .+ sum(x[is_dangling]) .* dangling_weights) .+ pscaled
        # l1 change in solution convergence criterion
        err = sum(abs, (x .- xlast))
        if (err < N * ϵ)
            return x
        end
    end
    error("Pagerank did not converge after $n iterations.")
end

savegraph(fn::AbstractString, g::AbstractSimpleWeightedGraph, gname::AbstractString="graph"; compress=true) =
    savegraph(fn, g, gname, SWGFormat(), compress=compress)

savegraph(fn::AbstractString, d::Dict{T, U}; compress=true) where T <: AbstractString where U <: AbstractSimpleWeightedGraph =
    savegraph(fn, d, SWGFormat(), compress=compress)

# It is possible that this is suboptimal, but it is the most trivial extension of the implementation used in Graphs.jl
function cartesian_product(g::G, h::G) where G <: AbstractSimpleWeightedGraph
    z = G(nv(g) * nv(h))
    id(i, j) = (i - 1) * nv(h) + j
    for e in edges(g)
        i1, i2 = Tuple(e)
        for j = 1:nv(h)
            add_edge!(z, id(i1, j), id(i2, j), weight(e))
        end
    end

    for e in edges(h)
        j1, j2 = Tuple(e)
        for i in vertices(g)
            add_edge!(z, id(i, j1), id(i, j2), weight(e))
        end
    end
    return z
end

# Connected Components on a Sparse Matrix

function _cc(g::SimpleWeightedGraph{T,U}) where T where U
    a = weights(g)
    comp = 0
    n = size(a, 1)
    marks = zeros(T, n)
    queue = Vector{T}()
    for i = 1:n
        if marks[i] == 0
            comp += 1
            push!(queue, i)
            while !isempty(queue)
                v = pop!(queue)
                marks[v] = comp
                for index in nzrange(a,v)
                    n = a.rowval[index]
                    if marks[n] == 0
                        push!(queue, n)
                    end
                end
            end
        end
    end
    marks, comp
end

function connected_components(g::SimpleWeightedGraph{T,U}) where T where U
    marks, num_cc = _cc(g)
    cc = [Vector{T}() for i = 1:num_cc]
    for (i,v) in enumerate(marks)
        push!(cc[v], i)
    end
    cc
end

function induced_subgraph(g::T, vlist::AbstractVector{U}) where T <: AbstractSimpleWeightedGraph where U <: Integer
    E = eltype(g)
    allunique(vlist) || throw(ArgumentError("Vertices in subgraph list must be unique"))
    new_weights = g.weights[E.(vlist), E.(vlist)]
    newg = zero(g)
    newg.weights = new_weights
    return newg, Vector{E}(vlist)
end


# Matern 5/2 isotropic covariance function

"""
    Mat52Iso <: MaternIso

Isotropic Matern 5/2 kernel (covariance)
```math
k(x,x') = σ²(1+√5|x-x'|/ℓ + 5|x-x'|²/(3ℓ²))\\exp(- √5|x-x'|/ℓ)
```
with length scale ``ℓ`` and signal standard deviation ``σ``.
"""
mutable struct Mat52Iso{T<:Real} <: MaternIso
    "Length scale"
    ℓ::T
    "Signal variance"
    σ2::T
    "Priors for kernel parameters"
    priors::Array
end

"""
Matern 5/2 isotropic covariance function

    Mat52Iso(ll::Real, lσ::Real)

# Arguments
  - `ll::Real`: length scale (given on log scale)
  - `lσ::Real`: signal standard deviation (given on log scale)
"""
Mat52Iso(ll::T, lσ::T) where T = Mat52Iso{T}(exp(ll), exp(2 * lσ), [])

function set_params!(mat::Mat52Iso, hyp::AbstractVector)
    length(hyp) == 2 || throw(ArgumentError("Matern 5/2 has two parameters, received $(length(hyp))."))
    mat.ℓ, mat.σ2 = exp(hyp[1]), exp(2 * hyp[2])
end
get_params(mat::Mat52Iso{T}) where T = T[log(mat.ℓ), log(mat.σ2) / 2]
get_param_names(mat::Mat52Iso) = [:ll, :lσ]
num_params(mat::Mat52Iso) = 2

cov(mat::Mat52Iso, r::Number) =
    (s = √5 * r / mat.ℓ; mat.σ2 * (1 + s + s^2 / 3) * exp(-s))

@inline dk_dll(mat::Mat52Iso, r::Real) =
    (s = √5 * r / mat.ℓ; mat.σ2 / 3 * s^2 * (1 + s) * exp(-s))


# This file was generated, do not modify it.

using MLJ, RDatasets, Random

data = dataset("datasets", "USArrests")
names(data)

describe(data, :mean, :std)

X = select(data, Not(:State))
X = coerce(X, :UrbanPop=>Continuous);

@load PCA pkg=MultivariateStats

pca_mdl = PCA(pratio=1)
pca = machine(pca_mdl, X)
fit!(pca)

W = transform(pca, X);

schema(W).names

r = report(pca)
cumsum(r.principalvars ./ r.tvar)

data = dataset("ISLR", "OJ")

X = select(data, [:PriceCH, :PriceMM, :DiscCH, :DiscMM, :SalePriceMM,
                  :SalePriceCH, :PriceDiff, :PctDiscMM, :PctDiscCH]);

Random.seed!(1515)

@pipeline SPCA(std = Standardizer(),
               pca = PCA(pratio=1-1e-4))
spca_mdl = SPCA()
spca = machine(spca_mdl, X)
fit!(spca)
W = transform(spca, X)
names(W)

r  = report(spca).reports[1]
cs = cumsum(r.principalvars ./ r.tvar)

using PyPlot

figure(figsize=(8,6))

bar(1:length(cs), cs)
plot(1:length(cs), cs, color="red", marker="o")

xlabel("Number of PCA features", fontsize=14)
ylabel("Ratio of explained variance", fontsize=14)

savefig("assets/literate/ISL-lab-10-g1.svg") # hide

Random.seed!(1515)

@load KMeans
@pipeline SPCA2(std = Standardizer(),
                pca = PCA(),
                km = KMeans(k=3))

spca2_mdl = SPCA2()
spca2 = machine(spca2_mdl, X)
fit!(spca2)

assignments = report(spca2).reports[1].assignments
mask1 = assignments .== 1
mask2 = assignments .== 2
mask3 = assignments .== 3;

using PyPlot

figure(figsize=(8, 6))
for (m, c) in zip((mask1, mask2, mask3), ("red", "green", "blue"))
    plot(W[m, 1], W[m, 2], ls="none", marker=".", markersize=10, color=c)
end

xlabel("PCA-1", fontsize=13)
ylabel("PCA-2", fontsize=13)
legend(["Group 1", "Group 2", "Group 3"], fontsize=13)

savefig("assets/literate/ISL-lab-10-cluster.svg") # hide


## Canonical Form of Normal distribution

immutable NormalCanon <: ContinuousUnivariateDistribution
    η::Float64       # σ^(-2) * μ
    λ::Float64    # σ^(-2)
    μ::Float64       # μ

    function NormalCanon(η::Real, λ::Real)
        @check_args(NormalCanon, λ > zero(λ))
    	new(η, λ, η / λ)
    end
    NormalCanon() = new(0.0, 1.0, 0.0)
end

@distr_support NormalCanon -Inf Inf

## conversion between Normal and NormalCanon

Base.convert(::Type{Normal}, d::NormalCanon) = Normal(d.μ, 1.0 / sqrt(d.λ))
Base.convert(::Type{NormalCanon}, d::Normal) = (λ = 1.0 / σ^2; NormalCanon(λ * d.μ, λ))
canonform(d::Normal) = convert(NormalCanon, d)

#### Parameters

params(d::NormalCanon) = (d.η, d.λ)

#### Statistics

mean(d::NormalCanon) = d.μ
median(d::NormalCanon) = mean(d)
mode(d::NormalCanon) = mean(d)

skewness(d::NormalCanon) = 0.0
kurtosis(d::NormalCanon) = 0.0

var(d::NormalCanon) = 1.0 / d.λ
std(d::NormalCanon) = sqrt(var(d))

entropy(d::NormalCanon) = 0.5 * (log2π + 1.0 - log(d.λ))

#### Evaluation

pdf(d::NormalCanon, x::Float64) = (sqrt(d.λ) / sqrt2π) * exp(-0.5 * d.λ * abs2(x - d.μ))
logpdf(d::NormalCanon, x::Float64) = 0.5 * (log(d.λ) - log2π - d.λ * abs2(x - d.μ))

zval(d::NormalCanon, x::Float64) = (x - d.μ) * sqrt(d.λ)
xval(d::NormalCanon, z::Float64) = d.μ + z / sqrt(d.λ)

cdf(d::NormalCanon, x::Float64) = normcdf(zval(d,x))
ccdf(d::NormalCanon, x::Float64) = normccdf(zval(d,x))
logcdf(d::NormalCanon, x::Float64) = normlogcdf(zval(d,x))
logccdf(d::NormalCanon, x::Float64) = normlogccdf(zval(d,x))

quantile(d::NormalCanon, p::Float64) = xval(d, norminvcdf(p))
cquantile(d::NormalCanon, p::Float64) = xval(d, norminvccdf(p))
invlogcdf(d::NormalCanon, lp::Float64) = xval(d, norminvlogcdf(lp))
invlogccdf(d::NormalCanon, lp::Float64) = xval(d, norminvlogccdf(lp))

#### Sampling

rand(cf::NormalCanon) = cf.μ + randn() / sqrt(cf.λ)
rand!{T<:Real}(cf::NormalCanon, r::AbstractArray{T}) = rand!(convert(Normal, cf), r)


# Map-MCMC for mixtures.

# This code assumes the following functions have been defined:
#   likelihood(x[j],theta)
#   log_v_prior(v)
#   log_theta_prior(theta)
#   theta_prop(theta)
#   log_theta_prop(theta,thetap)
#   new_thetas(m)

function sampler(x,n_samples,m,c,sigma,zeta)
    # x = data (array of datapoints)
    # n_samples = # of MCMC iterations
    # m = maximum # of mixture components
    # c = cutoff point for weight map
    # sigma = scale of weight proposals
    # zeta = power to raise likelihood to

    n = length(x)

    # initialize state
    v = zeros(m); v[1] = 2*c; v[2:end] = c/2 # latent weights
    q = max(v-c,0) # mapped weights (unnormalized)
    s = sum(q)
    theta = new_thetas(m)

    # initialize vars for computing likelihood
    L = [likelihood(x[j],theta[i]) for i=1:m, j=1:n]
    M = vec(q'*L)  # mixture density with unnormalized weights
    ll = sum(log(M)) - n*log(s)  # log-lik
    Mp = zeros(n)  # Mp and Lp will hold proposed values
    Lp = zeros(n)

    # record-keeping
    k_r = zeros(n_samples)
    m_r = zeros(n_samples,m)
    l_r = zeros(n_samples,m)
    v_r = zeros(n_samples,m)
    nta = 0  # number of theta proposals accepted
    nva = 0  # number of v proposals accepted

    # draw samples
    for iter = 1:n_samples
        # update parameters with Metropolis-Hastings moves
        for i = 1:m
            thetap = theta_prop(theta[i])
            llp = -n*log(s)
            for j = 1:n
                Lp[j] = likelihood(x[j],thetap)
                Mp[j] = max(M[j] + q[i]*(Lp[j] - L[i,j]), 0)
                # Note: max(.,0) prevents negative values due to roundoff error.
                llp += log(Mp[j])
            end
            # compute acceptance probability
            top = log_theta_prior(thetap) + zeta*llp + log_theta_prop(thetap,theta[i])
            bot = log_theta_prior(theta[i]) + zeta*ll + log_theta_prop(theta[i],thetap)
            p_accept = min(1, exp(top-bot))
            # accept or reject
            if rand() < p_accept
                theta[i] = copy(thetap)
                for j = 1:n; L[i,j] = Lp[j]; end
                M,Mp = Mp,M
                ll = llp
                nta += 1
            end
        end

        # update weights with Metropolis-Hastings moves
        for i = 1:m
            vp = v[i]*exp(randn()*sigma)
            qp = max(vp-c,0)
            sp = s + qp - q[i]
            if sp > 0
                llp = -n*log(sp)
                for j = 1:n
                    Mp[j] = max(M[j] + (qp - q[i])*L[i,j], 0)
                    llp += log(Mp[j])
                end
                # compute acceptance probability
                top = log_v_prior(vp) + zeta*llp - log(v[i])
                bot = log_v_prior(v[i]) + zeta*ll - log(vp)
                p_accept = min(1, exp(top-bot))
                # accept or reject
                if rand() < p_accept
                    v[i] = vp
                    q[i] = qp
                    s = sp
                    M,Mp = Mp,M
                    ll = llp
                    nva += 1
                end
            end
        end

        # record
        k_r[iter] = sum(q.>0)  # number of active components
        for i = 1:m
            m_r[iter,i] = theta[i][1]  # means
            l_r[iter,i] = theta[i][2]  # precisions
            v_r[iter,i] = v[i]  # latent weights
        end

        # visualize mixture components during sampling
        if false
            figure(10); clf(); hold(true)
            xs = linspace(-5,5,1000)
            for i = 1:m
                if q[i]>0
                    plot(xs,(q[i]/s)*normpdf(xs,theta[i][1],theta[i][2]))
                end
            end
            draw()
        end
    end
    art = nta/(m*n_samples)  # MH acceptance rate for theta proposals
    arv = nva/(m*n_samples)  # MH acceptance rate for v proposals
    return (q/s),theta,k_r,v_r,art,arv,m_r,l_r
end


using Test, DiscreteExteriorCalculus
const DEC = DiscreteExteriorCalculus
using LinearAlgebra: norm
using Statistics: mean

@testset "Point and Simplex" begin
    p = Point(1,2,3)
    @test typeof(p) == Point{3}
    s = Simplex(Point(1,2,3), Point(4,5,6), Point(1,4,7))
    @test typeof(s) == Simplex{3, 3}
end

@testset "Barycentric and barycentric_projection" begin
    m = Metric(3)
    s = Simplex(Point(0,0,0), Point(2,0,0), Point(1,.5,0))
    for p in [s.points; Point(3,5,7); Point(-12,25,-100); Point(24.3,-3.5,-2.2)]
        b = Barycentric(m, s, p)
        @test norm(Point(b).coords - [p.coords[1:end-1]; 0]) < 1e-14
    end
end

@testset "distance" begin
    m = Metric(3)
    s = Simplex(Point(0,0,0), Point(2,0,0), Point(1.3,.7,0), Point(0,0,1))
    ps = [s.points; Point(3,5,7); Point(-12,25,-100); Point(24.3,-3.5,-2.2)]
    for p1 in ps
        for p2 in ps
            d1 = sqrt(abs(distance_square(m, Barycentric(m, s, p1), p2)))
            d2 = norm(m, p1.coords - p2.coords)
            d3 = norm(p1.coords - p2.coords) # using Euclidean metric
            @test d2 ≈ d3
            @test abs(d1 - d2) < 1e-5
        end
    end
end

@testset "circumsphere" begin
    function is_circumcenter(m::Metric{N}, s::Simplex{N, K}, p::Point{N}) where {N, K}
        if K == 1
            return s.points[1] == p
        else
            dists = [norm(m, p.coords - q.coords) for q in s.points]
            return all([d ≈ dists[1] for d in dists])
        end
    end
    function circumradius_square(m::Metric{N}, s::Simplex{N}) where N
        center, _ = circumsphere(m, s)
        return mean([norm_square(m, center.coords - v.coords) for v in s.points])
    end
    m = Metric(3)
    s = Simplex(Point(1,2,3), Point(-12,3,5), Point(8,3,4), Point(9,25,-2))
    for k in 1:4
        for sub_s in subsimplices(s, k)
            center, radius_square = circumsphere(m, sub_s)
            @test is_circumcenter(m, sub_s, center)
            @test radius_square ≈ circumradius_square(m, sub_s)
        end
    end
end

@testset "barycentric_subspace" begin
    m = Metric(2)
    s = Simplex(Point(0,0), Point(1,0), Point(0,1))
    f = Simplex(s.points[1:2]...)
    p = Point(.5, 3)
    M, v = DEC.barycentric_subspace(m, s, f, p)
    @test M * Barycentric(m, s, p).coords == v
    @test M * [.5, .5, 0] == v
    @test M * Barycentric(m, s, Point(.5, -2)).coords == v
end

@testset "rotate_about_face" begin
    m = Metric(3)
    s = Simplex(Point(-1,0,0), Point(0,0,0), Point(0,1,0))
    r, θ = 3, π/6
    p = Point(r * cos(θ), 0, r * sin(θ))
    b = DEC.rotate_about_face(m, s, p, 1)
    @test norm(Point(b).coords - [r, 0, 0]) < 1e-15

    m = Metric(2)
    s = Simplex(Point(0,0),Point(1,0))
    p = Point(1,1)
    b = DEC.rotate_about_face(m, s, p, 2)
    @test Point(b).coords ≈ [-sqrt(2), 0]
    b = DEC.rotate_about_face(m, s, p, 1)
    @test Point(b).coords ≈ [2, 0]
end

@testset "pairwise_delaunay" begin
    m = Metric(3)
    s1 = Simplex(Point(-.1,.5,0), Point(0,0,0), Point(0,1,0))
    center, _ = circumsphere(m, s1)
    θ = π/6
    for r in [.1, 1, 5, 10]
        p = Point(r * cos(θ), 0, r * sin(θ))
        proj_p = Point(r, 0, 0)
        # using Euclidean metric
        delaunay = norm(center.coords - proj_p.coords)^2 - norm(center.coords - s1.points[1].coords)^2
        s2 = Simplex(s1.points[2:end]..., p)
        @test pairwise_delaunay(m, s1, s2) ≈ delaunay
    end
end

# TODO test with other metrics


using Plots

using Algopt.TestFunctions: Rosenbrock

using Algopt.FirstOrder

ros = Rosenbrock()

function solve_with_method(method)
    x_0 = [rand(-2:.1:2), rand(-2:.1:2)]
    res = FirstOrder.search(method, ros.f, ros.∇f, x_0, trace = true)
    println(res[:, end])
    println(size(res))

    x = y = range(-2, 2, step = .1)
    plot(contour(x, y, (x, y)->ros.f([x, y]); levels = range(0, 5, step = .1)))
    plot!(res[1,:], res[2,:], linewidth = 3)
end

function max_grad_descent()
    mgd = FirstOrder.MaximumGradientDescent()
    solve_with_method(mgd)
end

function grad_descent(α)
    grd = FirstOrder.GradientDescent(α = α)
    solve_with_method(grd)
end

function conjugate_grad_descent()
    cgd = FirstOrder.ConjugateGradientDescent()
    solve_with_method(cgd)
end


# # Symmetry reducting a strong relaxation of the quadratic assigment problem
# Here, we are going to show how to load a QAP from QABLib, formulate
# a strong semidefinite relaxation of it, symmetry reduce it, and finally solve it.

# ## Quadratic assigment problems
# QAPs are given by two quadratic matrices $A$ and $B$. The objective is to permute
# the rows and columns of $B$, such that the inner product between the matrices is
# minimized.
#
# ``\mathrm{QAP}(A,B) = \min_{\phi\in S_n} \sum_{i,j=1}^n a_{ij}b_{\phi(i)\phi(j)}``
#
# QAPs are notoriously hard to solve exactly, but there exist strong polynomial time
# relaxations, such as the following semidefinite programming relaxation:
#
# ```math
# \begin{aligned}
# \min\enspace & \langle B\otimes A ,Y\rangle\\
# \mathrm{s.t.}\enspace & \langle I_n\otimes E_{jj},Y\rangle=1 \text{ for }j\in [n],\\
# & \langle E_{jj}\otimes I_n,Y\rangle=1 \text{ for }j\in [n],\\
# & \langle I_n\otimes (J_n-I_n)+(J_n-I_n)\otimes I_n,Y\rangle =0, \\
# & \langle J_{n^2},Y\rangle = n^2,\\
# & Y\in D^{n^2},
# \end{aligned}
# ```
#
# But in practice this relaxation is often too big to be solved directly.

# ## Loading the data of a QAP
using SparseArrays, LinearAlgebra
using Test #src

file = joinpath(@__DIR__, "esc16j.dat")
data = open(file) do file
    read(file, String)
end
data = split(data, [' ', '\n', '\r'], keepempty = false)

n = parse(Int64, data[1])
A = zeros(Int64, n, n)
B = zeros(Int64, n, n)

pos = 2
for x = 1:n
    for y = 1:n
        A[x, y] = parse(Int64, data[pos])
        global pos += 1
    end
end
for x = 1:n
    for y = 1:n
        B[x, y] = parse(Int64, data[pos])
        global pos += 1
    end
end

# ## Building the SDP (in vectorized standard form)
n = size(A, 1)

## Objective
CPrg = sparse(kron(B, A))

In = sparse(I, n, n)
Jn = ones(n, n)

## Vectorised constraint matrices as rows of large matrix APrg
APrg = spzeros(2n + 1, n^4)
bPrg = zeros(2n + 1)
currentRow = 1

for j = 1:n
    Ejj = spzeros(n, n)
    Ejj[j, j] = 1.0
    APrg[currentRow, :] = vec(kron(In, Ejj))
    bPrg[currentRow] = 1
    global currentRow += 1
    ## Last constraint is linearly dependent on others
    if (j < n)
        APrg[currentRow, :] = vec(kron(Ejj, In))
        bPrg[currentRow] = 1
        global currentRow += 1
    end
end

APrg[currentRow, :] = vec(kron(In, Jn - In) + kron(Jn - In, In))
bPrg[currentRow] = 0
currentRow += 1
APrg[currentRow, :] = vec(ones(n^2, n^2))
bPrg[currentRow] = n^2

CPrg = sparse(vec(0.5 * (CPrg + CPrg')));

# ## Symmetry reducing the SDP

# We first determine an optimal admissible partition subspace
using SDPSymmetryReduction
P = admPartSubspace(CPrg, APrg, bPrg, true)
P.n

@test P.n == 150 #src
# And then we block-diagonalize it
blkD = blockDiagonalize(P, true);
@test sort(blkD.blkSizes) == sort([7, 7, 7, 7, 7, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]) #src

# ## Determining the coefficients of the reduced SDP
PMat = hcat([sparse(vec(P.P .== i)) for i = 1:P.n]...)
newA = APrg * PMat
newB = bPrg
newC = CPrg' * PMat;

## Removing linearly dependent constraints #src
# using RowEchelon  #src
# T = rref!(Matrix(hcat(newA,newB))) #src
# r = rank(T) #src
# newA = T[1:r,1:end-1] #src
# newB = T[1:r,end] #src
# length(newB) #src

# ## Solving the reduced SDP with JuMP and CSDP

using JuMP, CSDP
m = Model(CSDP.Optimizer)

## Initialize variables corresponding parts of the partition P
## >= 0 because the original SDP-matrices are entry-wise nonnegative
x = @variable(m, x[1:P.n] >= 0)

@constraint(m, newA * x .== newB)
@objective(m, Max, newC * x)

psdBlocks = sum(blkD.blks[i] .* x[i] for i = 1:P.n)
for blk in psdBlocks
    if size(blk, 1) > 1
        @constraint(m, blk in PSDCone())
    else
        @constraint(m, blk .>= 0)
    end
end

optimize!(m)
#
termination_status(m)
#
objective_value(m)

@test objective_value(m) ≈ 48.3042 atol = 5#src


using OffsetArrays
using SpecialFunctions: gamma, erfcx
using PyPlot
using Polynomials
using Printf
using LinearAlgebra

Γ(z) = gamma(z)

function pade(α::T, β::T, m::Integer, n::Integer) where T <: AbstractFloat
    if mod(m+n,2) == 0
        throw(ArgumentError("m+n must be odd"))
    end
    r = div(m+n-1,2)
    a = OffsetArray{T}(undef, 0:m-1)
    b = OffsetArray{T}(undef, 0:n-1)
    pow = one(T)
    for k = 0:m-1
        a[k] = pow / Γ(β+k*α)
        pow = -pow
    end
    b[0] = zero(T)
    pow = one(T)
    for k = 1:n-1
        pow = -pow
        b[k] = pow * sinpi(k*α-β) * Γ(1+k*α-β) / π
    end
    A = zeros(T, 2r-1, 2r-1)
    rhs = zeros(T, 2r-1)
    if m ≥ r+1
        for k = 1:r-1
            A[k,k] = one(T)
            for j = 1:k
                A[k,r-1+j] = -a[k-j]
            end
            rhs[k] = a[k]
        end
        for k = r:m-1
            for j = 1:r
                A[k,r-1+j] = -a[k-j]
            end
            rhs[k] = a[k]
        end
        for k = r-n+1:r-1
            row = k+r
            A[row,k] = one(T)
            for j = k+1:r
                col = j + r - 1
                A[row,col] = -b[j-k]
            end
        end
    else
        for k = 1:m-1
            A[k,k] = one(T)
            for j = 1:k
                col = r - 1 + j
                A[k,col] = -a[k-j]
            end
            rhs[k] = a[k]
        end
        for k = 1:n-r-1
            row = k + m - 1
            for j = 1:r
                col = r - 1 + j
                A[row,col] = -b[j+k]
            end
            rhs[row] = b[k]
        end
        for j = 1:r
            col = r - 1 + j
            A[r,col] = -b[j]
        end
        rhs[r] = -a[0]
        for k = 1:r-1
            row = r + k
            A[row,k] = one(T)
            for j = k+1:r
                col = r - 1 + j
                A[row,col] = -b[j-k]
            end
        end
    end
    coeffs = A \ rhs
    p = OffsetArray{T}(undef, 0:r-1)
    q = OffsetArray{T}(undef, 0:r)
    p[0] = a[0]
    for k = 1:r-1
        p[k] = coeffs[k]
    end
    q[0] = one(T)
    for k = 1:r
        q[k] = coeffs[k+r-1]
    end
    return p, q, A, rhs
end

#T = BigFloat
T = Float64
r = 3
α = 1 / parse(T, "2")
β = parse(T, "1")
x = range(zero(T), parse(T, "10"), length=201)
y = erfcx.(x)
figure(1)
for m = r:5
    n = 2r + 1 - m
    @printf("%3d  %3d", m, n)
    p, q, A, rhs = pade(α, β, m, n)
    @printf("  %5.2f\n", cond(A))
    P = Polynomial(p[0:r-1])
    Q = Polynomial(q[0:r])
    err = P.(x) ./ Q.(x) - y
    plot(x, err, label="m, n = $m, $n")
end
legend()
grid(true)

figure(2)
#for (r, m, n) in [(4, 5, 4), (8, 9, 8), (12, 13, 12), (16, 17, 16)]
for (r, m, n) in [(4, 5, 4), (8, 9, 8), (12, 13, 12)]
    p, q, A, rhs = pade(α, β, m, n)
    println("cond(A) = ", cond(A))
    display(p)
    display(q)
    P = Polynomial(p[0:r-1])
    Q = Polynomial(q[0:r])
    err = P.(x) ./ Q.(x) - y
    semilogy(x, abs.(err), label="m, n = $m, $n")
end
legend()
grid(true)


function can_cut(c, k, as)
    sum([ceil(Int, a/c) - 1 for a in as]) ≤ k
end

function solve()
    n, k = [parse(Int, x) for x in split(readline())]
    as = [parse(Int, x) for x in split(readline())]

    lb, ub = 0, maximum(as)
    while lb + 1 < ub
        c = ceil(Int, (ub + lb) / 2)
        if can_cut(c, k, as)
            ub = c
        else
            lb = c
        end
    end
    ceil(Int, ub)
end

println(solve())


function read_Species_From_Text_File(file_name)
    return reaction_matrix
end

function calculate_concentration(c_0, λ, Δt)
    Λ = Diagonal(λ)
    R = assemble_transformation_matrix_inverse(λ)
    S = assemble_transformation_matrix(λ)
    A = S * Λ * R * Δt
    return exp(A) * c_0
end

function transformation_matrix_inverse_ij(i, j, λ)
    s_ij::eltype(λ) = 1.0
    # Loop over species ancestor index l for species i with (i-j) generations
    for l = j:(i-1)
        s_ij *= λ[l] / (λ[l] - λ[i])
    end
    return s_ij
end

function transformation_matrix_ij(i, j, λ)
    s_ij = λ[j] / (λ[i] - λ[j])
    # Loop over species ancestor index l for species i with (i-j) generations
    for l = (j+1):(i-1)
        s_ij *= λ[l] / (λ[l] - λ[j])
    end
    return s_ij
end

function assemble_transformation_matrix(λ)
    n = length(λ)
    S = zeros(eltype(λ), length(λ), length(λ))
    S = UnitLowerTriangular(S)
    for i = 2:n
        for j = 1:(i-1)
            S[i, j] = transformation_matrix_ij(i, j, λ)
        end
    end
    return S
end

function assemble_transformation_matrix_inverse(λ)
    n = length(λ)
    R = zeros(eltype(λ), length(λ), length(λ))
    R = UnitLowerTriangular(R)
    for i = 2:n
        for j = 1:i
            R[i, j] = transformation_matrix_inverse_ij(i, j, λ)
        end
    end
    return R
end

function retrieve_generation_number(i, j)
    return (i - j)
end

function transformation_matrix_5x5(λ = [1.0; 2.0; 3.0; 4.0; 5.0])

    s = zeros(5, 5)
    s[1, 1] = 1

    s[2, 1] = λ[1] / (λ[2] - λ[1])
    s[3, 1] = λ[1] / (λ[3] - λ[1]) * λ[2] / (λ[2] - λ[1])
    s[4, 1] = λ[1] / (λ[4] - λ[1]) * λ[2] / (λ[2] - λ[1]) * λ[3] / (λ[3] - λ[1])
    s[5, 1] =
        λ[1] / (λ[5] - λ[1]) * λ[2] / (λ[2] - λ[1]) *
        λ[3] / (λ[3] - λ[1]) * λ[4] / (λ[4] - λ[1])

    s[2, 2] = 1
    s[3, 2] = λ[2] / (λ[3] - λ[2])
    s[4, 2] = λ[2] / (λ[4] - λ[2]) * λ[3] / (λ[3] - λ[2])
    s[5, 2] = λ[2] / (λ[5] - λ[2]) * λ[3] / (λ[3] - λ[2]) * λ[4] / (λ[4] - λ[2])

    s[3, 3] = 1
    s[4, 3] = λ[3] / (λ[4] - λ[3])
    s[5, 3] = λ[3] / (λ[5] - λ[3]) * λ[4] / (λ[4] - λ[3])

    s[4, 4] = 1
    s[5, 4] = λ[4] / (λ[5] - λ[4])

    s[5, 5] = 1

    return s
end

function transformation_matrix_inverse_5x5(λ = [1.0; 2.0; 3.0; 4.0; 5.0])

    s = zeros(5, 5)
    s[1, 1] = 1

    s[2, 1] = λ[1] / (λ[1] - λ[2])
    s[3, 1] = λ[1] / (λ[1] - λ[3]) * λ[2] / (λ[2] - λ[3])
    s[4, 1] = λ[1] / (λ[1] - λ[4]) * λ[2] / (λ[2] - λ[4]) * λ[3] / (λ[3] - λ[4])
    s[5, 1] =
        λ[1] / (λ[1] - λ[5]) * λ[2] / (λ[2] - λ[5]) * λ[3] / (λ[3] - λ[5]) *
        λ[4] / (λ[4] - λ[5])

    s[2, 2] = 1
    s[3, 2] = λ[2] / (λ[2] - λ[3])
    s[4, 2] = λ[2] / (λ[2] - λ[4]) * λ[3] / (λ[3] - λ[4])
    s[5, 2] = λ[2] / (λ[2] - λ[5]) * λ[3] / (λ[3] - λ[5]) * λ[4] / (λ[4] - λ[5])

    s[3, 3] = 1
    s[4, 3] = λ[3] / (λ[3] - λ[4])
    s[5, 3] = λ[3] / (λ[3] - λ[5]) * λ[4] / (λ[4] - λ[5])

    s[4, 4] = 1
    s[5, 4] = λ[4] / (λ[4] - λ[5])

    s[5, 5] = 1

    return s
end


#=
Small illustation on the capacitated vehicle routing problem (CVRP)
Just run the script for an illustration

Mathematical model based on: https://www.researchgate.net/publication/323173028_Two_models_of_the_capacitated_vehicle_routing_problem
using modified indices to match Julia's 1-based indexing

In this model we eliminate all subtour explicitely, keep in mind that this is very expensive
as the number of subtours grows exponentially with the number of cities! A modified approach is to solve the
relaxed problem (i.e. without subtour constraints) and only add them when they occur (thus solving the model again),
adding the constraint that prohibits the subtour that was found during the previous iteration cf.
https://how-to.aimms.com/Articles/332/332-Implicit-Dantzig-Fulkerson-Johnson.html
for a clear explanation.

=#
using JuMP
using GLPK
using Plots
using Combinatorics:powerset # may require additional install

"""
Multiple Vehicle Routing problem (using the entire set of possible subtours for subtour elimination)
"""
function MVRP()
    # general problem setup #
    # --------------------- #
    # number of cities
    n = 6
    # pick random locations
    coord_x = rand(n+1)
    coord_y = rand(n+1)
    dist(x_1, x_2, y_1, y_2) = sqrt((x_1 - x_2)^2 + (y_1 - y_2)^2)
    # cost matrix (symmetrical, distances, depot is city 1)
    c = zeros(n+1, n+1)
    for i in 1:n+1
        for j in i+1:n+1
            c[i,j] = dist(coord_x[i], coord_x[j], coord_y[i], coord_y[j])
        end
    end
    c .+= c'
    # demand for each city (number of e.g. number of pallets)
    d = rand(1:10, n+1)
    d[1] = 0 # no demand @ depot
    # truck capacity (equal for all trucks, can easily be extended to different size trucks)
    Q = 20
    # minimum number of trucks for order to be feasible:
    p = ceil(Int,sum(d) / Q)
    @info "$(p) trucks required"

    # Linear programming setup #
    # ------------------------ #
    model = Model(with_optimizer(GLPK.Optimizer))
    # Add variables (x_{i,j,p}: edge from i > j by truck p, Boolean valued)
    @variable(model, x[1:n+1,1:n+1,1:p], Bin)
    # Add objective
    @objective(model, Min, sum(sum(x[:,:,i] .* c) for i in 1:p))
    # Add constraints
    for r in 1:p
        # each vehicle can leave the depot only once
        @constraint(model, sum(x[1, j, p] for j in 2:n+1) == 1)
        # sum of the demands of the customers visited in a route is less than or equal to the capacity of the vehicle
        @constraint(model, sum(d[j]*x[i,j,r] for i in 1:n+1 for j in 2:n+1 if i≠j) <= Q)
        # the number of the vehicles arriving at every customer and entering the depot is equal to the number of the vehicles leaving
        for j in 1:n+1
            @constraint(model, sum(x[i,j,r] for i in 1:n+1 if i≠j) == sum(x[j,i,r] for i in 1:n+1))
        end
    end
    # ensure that each customer is visited by exactly one vehicle
    for j in 2:n+1
        @constraint(model, sum(x[i,j,r] for r in 1:p for i in 1:n+1 if i≠j) == 1)
    end

    #=
    # eliminate sub-tours (essential but costly)
    for subset in powerset(2:n+1, 2, n)
        @constraint(model, sum(x[i,j,r] for r in 1:p for i in subset for j in subset if i≠j) <= length(subset) - 1 )
    end
    =#
    # Solve problem
    optimize!(model)
    @info "termination status: $(termination_status(model))"
    @info "objective_value $(objective_value(model))"

    #     Nice illustration    #
    # ------------------------ #
    marks = [:circle for _ in 1:n+1]
    marks[1] = :cross
    fig = plot()
    scatter!(fig, coord_x[1:1], coord_y[1:1], marker=:cross, label="Depot", color=:black,markersize=12)
    scatter!(fig, coord_x[2:end], coord_y[2:end], marker=:circle, label="Stores", color=:gray )
    for r in 1:p
        inds = findall(x-> !iszero(x), value.(x[:,:,r]))
        for ind in inds
            xx = [coord_x[ind[1]]; coord_x[ind[2]]]
            yy = [coord_y[ind[1]]; coord_y[ind[2]]]
            plot!(fig, xx, yy, color=r, label="")
        end
    end
    title!(fig, "lay-out for $(p) trucks with $(n) cities")
    display(fig)
end

"""
Multiple Trip Vehicle Routing problem

Implementation according to the 4-index formulation, also uses the entire set of subtours (similar to the other example)

Remark: since we are mimimising the total time of all trips, it can happen that a single truck gets two trips, while the
other one gets nothing at all. In a simulation context, you should take this into account.
"""
function MTVRP()
    # general problem setup #
    # --------------------- #
    # number of cities
    n = 7
    # pick random locations
    coord_x = rand(n+1)
    coord_y = rand(n+1)
    dist(x_1, x_2, y_1, y_2) = sqrt((x_1 - x_2)^2 + (y_1 - y_2)^2)
    # cost matrix (symmetrical, distances, depot is city 1)
    c = zeros(n+1, n+1)
    for i in 1:n+1
        for j in i+1:n+1
            c[i,j] = dist(coord_x[i], coord_x[j], coord_y[i], coord_y[j])
        end
    end
    c .+= c'
    # demand for each city (number of e.g. number of pallets)
    d = rand(1:10, n+1)
    d[1] = 0 # no demand @ depot
    # truck capacity (equal for all trucks, can easily be extended to different size trucks)
    Q = 20
    V = 2 # number of vehicles
    R = 3 # number of trips/vehicle
    TH = 5 # max trip length => results in "infeasible"
    # minimum number of trucks for order to be feasible:
    @info "$(ceil(Int,sum(d) / Q)) trips required for a solution"
    @info "$(V) trucks used, max $(R) trips per truck, total work time $(TH)"
    # Linear programming setup #
    # ------------------------ #
    model = Model(with_optimizer(GLPK.Optimizer))
    # Add variables
    # x_{i,j,v, r}: edge from i > j by truck v on trip r, Boolean valued
    @variable(model, x[1:n+1, 1:n+1, 1:V, 1:R], Bin)
    # y_{i, v, r}: 1 if trip r of vehicle v visits vertex i, Boolean valued
    @variable(model, y[1:n+1, 1:V, 1:R], Bin)

    # Add objective (slices)
    @objective(model, Min, sum(sum(x[:,:,v,r] .* c) for v in 1:V for r in 1:R)) # (1)

    # Add constraints
    for i in 2:n+1
        @constraint(model, sum(y[i,v,r] for v in 1:V for r in 1:R) == 1) # (2)
    end

    for v in 1:V
        for r in 1:R
            for i in 1:n+1
                @constraint(model, sum(x[i,j,v,r] for j in 1:n+1 if j≠i) == y[i,v,r]) # (3)
                @constraint(model, sum(x[j,i,v,r] for j in 1:n+1 if j≠i) == y[i,v,r]) # (3)
                @constraint(model, sum(x[j,i,v,r] for j in 1:n+1 if j≠i) == sum(x[i,j,v,r] for j in 1:n+1 if j≠i)) # (3)
            end

            @constraint(model, sum(d[i] * y[i,v,r] for i in 2:n+1) <= Q) # (4)
            for subset in powerset(2:n+1, 2, n)
                @constraint(model, sum(x[i,j,v,r] for i in subset for j in subset if i≠j) <= length(subset) - 1 ) # (5)
            end

        end
    end

    for v in 1:V
        @constraint(model, sum(sum(x[:,:,v,r] .* c) for r in 1:R) <= TH) # (6)
    end

    optimize!(model)
    @info "termination status: $(termination_status(model))"
    @info "objective_value $(objective_value(model))"

    #     Nice illustration    #
    # ------------------------ #
    marks = [:circle for _ in 1:n+1]
    marks[1] = :cross
    fig = plot()
    scatter!(fig, coord_x[1:1], coord_y[1:1], marker=:cross, label="Depot", color=:black,markersize=12)
    scatter!(fig, coord_x[2:end], coord_y[2:end], marker=:circle, label="Stores", color=:gray )
    for v in 1:V
        for r in 1:R
            inds = findall(x-> !iszero(x), value.(x[:,:,v,r]))
            for ind in inds
                xx = [coord_x[ind[1]]; coord_x[ind[2]]]
                yy = [coord_y[ind[1]]; coord_y[ind[2]]]
                plot!(fig, xx, yy, color=v, label="", line=r)
            end
            @info "truck $(v), route $(r): $(round(sum(c[inds]), digits=2)) kms"
        end
    end
    title!(fig, "lay-out for $(V) trucks with $(n) cities, max $(R) routes per truck\ncolor= truck, line thickness=route")
    display(fig)
end

MVRP()
#MTVRP()


# ------------------------------------------------------------
# Truncated cosine and its Fourier transform

# Support = [-1/2,1/2]
function tcos(x)
	if 0.0 <= x <= 0.5
		return cos(2*pi*x)
	else
		return 0.0
	end
end

function ftcos(xi)
	if abs(xi) == 1.0
		return 0.25 + 0.0*im
	else
		return im*xi*( 1 + exp(-pi*xi*im) ) / (2*pi*(1 - xi^2))
	end
end

# Overload ftcos to vector input
@vectorize_1arg Float64 ftcos

# ------------------------------------------------------------
# Reconstruct truncated cosine in Haar basis

using GeneralizedSampling

J = 5
M = 2^(J+2)
# Both GeneralizedSampling and Plots (below) have a grid function
xi = GeneralizedSampling.grid(M, 0.5)
f = ftcos(xi)

T = Freq2Wave(xi, "haar", J)
wcoef = T \ f

# ------------------------------------------------------------
# Plot reconstruction

using IntervalWavelets

x, yw = weval( real(wcoef), "haar", 10 )

using Plots

p = plot(x, tcos,
         linecolor=:red,
         linewidth=2,
         label="true"
        )

plot!(p, x, yw,
      linecolor=:black,
      linewidth=2,
      label="approximation"
     )

display(p)


using DifferentialEquations

function sirmodel(du, u, p, t)
    S = u[1]
    I = u[2]
    R = u[3]
    dS = -p[1]*S*I
    dI = p[1]*S*I-p[2]*I
    dR = p[2]*I
    du[1] = dS
    du[2] = dI
    du[3] = dR
end

function solveSIR(N, β, γ)

    tspan = (0.0, 160.0)
    u0 = [N-1.0, 1.0, 0.0]
    p = [β, γ]

    prob = ODEProblem(sirmodel, u0, tspan, p)

    sol = solve(prob)

    return sol
end

function calcEffectiveReproduction(sol)

    v = Array{Float64,1}()
    t = Array{Float64,1}()
    for index in 2:length(sol)
        push!(v, sol[index][2]/sol[index-1][2])
        push!(t, sol.t[index])
    end

    return [v,t]
end


function jaccard_similarity(set1::Array, set2::Array)

    intersec = 0.0
    set1 = unique(set1)
    set2 = unique(set2)

    for i in set1
        for j in set2
            if i == j
                intersec += 1.0
            end
        end
    end
    return intersec / (length(set1) + length(set2) - intersec)
end


"""
Elliptic Curve
==============

References
----------

- https://en.wikipedia.org/wiki/Elliptic_curve
- "An Introduction to the Theory of Elliptic Curves", Joseph H. Silverman
- https://andrea.corbellini.name/2015/05/17/elliptic-curve-cryptography-a-gentle-introduction/
"""
module EllipticCurveCrypto

include("field.jl")

export
    Point, EllipticCurve, is_on_curve,
    ec_zero, ec_add, ec_neg, ec_scalar_mul, ec_scalar_mul_ml

struct Point{T}
    x::T
    y::T
end

# `T` might be mutable (e.g. `BigInt`) and compare using object identity by default.
function Base.:(==)(a::Point{T}, b::Point{T}) where T
    a.x == b.x && a.y == b.y
end

function Base.:(≈)(a::Point{T}, b::Point{T}) where T
    a.x ≈ b.x && a.y ≈ b.y
end

"""
Elliptic curve (with characteristic >= 5) `y ^ 2 = x ^ 3 + a * x + b`.
"""
struct EllipticCurve{T}
    field::Field{T}

    a::T
    b::T

    function EllipticCurve(f::F, a::T, b::T) where {T, F<:Field{T}}
        a = f_from(f, a)
        b = f_from(f, b)
        if f_add(f,
            f_mul(f, f_pow(f, a, 3), f_from(f, T(4))),
            f_mul(f, f_pow(f, b, 2), f_from(f, T(27)))) == f_zero(f)
            error("The curve is singular")
        end
        new{T}(f, a, b)
    end
end

function is_on_curve(ec::EllipticCurve{T}, p::Point{T})::Bool where T
    if p == ec_zero(ec)
        return true
    end

    f = ec.field
    a = ec.a
    b = ec.b
    x = p.x
    y = p.y
    lhs = f_pow(f, y, 2)
    rhs = f_add(f,
        f_add(f, f_pow(f, x, 3), f_mul(f, x, a)),
        b)
    lhs == rhs
end

# Zero, also the infinity.
function ec_zero(ec::EllipticCurve{T})::Point{T} where T
    z = f_zero(ec.field)
    Point(z, z)
end

function ec_add(ec::EllipticCurve{T}, p1::Point{T}, p2::Point{T})::Point{T} where T
    f = ec.field
    z = ec_zero(ec)

    s = if p1 == z
        return p2
    elseif p2 == z
        return p1
    elseif p1.x == p2.x
        if f_add(f, p1.y, p2.y) == f_zero(f)
            return z
        else
            # Doubling
            (x, y) = (p1.x, p1.y)
            numer = f_add(f, f_mul(f, f_from(f, T(3)), f_pow(f, x, 2)), ec.a)
            denom = f_mul(f, f_from(f, T(2)), y)
            f_div(f, numer, denom)
        end
    else
        f_div(f, f_sub(f, p2.y, p1.y), f_sub(f, p2.x, p1.x))
    end

    x = f_sub(f, f_mul(f, s, s), f_add(f, p1.x, p2.x))
    y = f_sub(f, f_mul(f, s, f_sub(f, p1.x, x)), p1.y)
    Point(x, y)
end

function ec_neg(ec::EllipticCurve{T}, p::Point{T})::Point{T} where T
    Point(p.x, f_neg(ec.field, p.y))
end

# Double-and-add algorithm
function ec_scalar_mul(ec::EllipticCurve{T}, p::Point{T}, s::Int)::Point{T} where T
    acc = ec_zero(ec)
    while s != 0
        if s & 1 != 0
            acc = ec_add(ec, acc, p)
        end
        p = ec_add(ec, p, p)
        s >>= 1
    end
    acc
end

# Side-channel attack resistent multiplication.
"Point multiplication using Montgomery ladder."
function ec_scalar_mul_ml(ec::EllipticCurve{T}, p::Point{T}, s::Int)::Point{T} where T
    # p * d = \sum_{i = 0..m} p * (d_i 2^i)          ...0-based
    #       = \sum_{i = 1..m} p * (d_i 2^(i - 1))    ...1-based
    d = digits(s, base=2)
    m = length(d)

    # Invariants:
    #
    #     r0 = p * t0 = p * (s >> (i - 1))
    #     r1 = p * t1 = p * (t0 + 1)
    r0 = ec_zero(ec)
    r1 = p
    for i in m:-1:1
        if d[i] == 0
            r1 = ec_add(ec, r0, r1)  # t0 + (t0 + 1) => (2 * t0) + 1
            r0 = ec_add(ec, r0, r0)  # 2 * t0
        else
            r0 = ec_add(ec, r0, r1)  # t0 + (t0 + 1) => 2 * t0 + 1
            r1 = ec_add(ec, r1, r1)  # 2 * (t0 + 1) => (2 * t0 + 1) + 1
        end
    end
    r0
end

end # module


module TestDiagrammaticPrograms
using Test

using Catlab.CategoricalAlgebra, Catlab.Programs.DiagrammaticPrograms
using Catlab.Programs.DiagrammaticPrograms: NamedGraph, MaybeNamedGraph
using Catlab.Graphs.BasicGraphs: TheoryGraph

# Graphs
########

para_parsed = @graph begin
  s
  t
  s → t
  s → t
end
para = @acset MaybeNamedGraph{Symbol} begin
  V = 2
  E = 2
  src = [1,1]
  tgt = [2,2]
  vname = [:s, :t]
  ename = [nothing, nothing]
end
@test para_parsed == para

para_parsed = @graph NamedGraph{Symbol} begin
  x, y
  (f, g): x → y
end
para = @acset NamedGraph{Symbol} begin
  V = 2
  E = 2
  src = [1,1]
  tgt = [2,2]
  vname = [:x, :y]
  ename = [:f, :g]
end
@test para_parsed == para

tri_parsed = @graph NamedGraph{Symbol} begin
  v0, v1, v2
  fst: v0 → v1
  snd: v1 → v2
  comp: v0 → v2
end
tri = @acset NamedGraph{Symbol} begin
  V = 3
  E = 3
  src = [1,2,1]
  tgt = [2,3,3]
  vname = [:v0, :v1, :v2]
  ename = [:fst, :snd, :comp]
end
@test tri_parsed == tri

# Categories
############

Δ¹_parsed = @category begin
  V, E
  (δ₀, δ₁): V → E
  σ₀: E → V

  σ₀ ∘ δ₀ == id(V)
  σ₀ ∘ δ₁ == id(V)
end
Δ¹_graph = @acset NamedGraph{Symbol} begin
  V = 2
  E = 3
  src = [1,1,2]
  tgt = [2,2,1]
  vname = [:V, :E]
  ename = [:δ₀, :δ₁, :σ₀]
end
Δ¹ = FinCat(Δ¹_graph, [ [1,3] => empty(Path, Δ¹_graph, 1),
                        [2,3] => empty(Path, Δ¹_graph, 1) ])
@test Δ¹_parsed == Δ¹

# Diagrams
##########

C = FinCat(TheoryGraph)
F_parsed = @diagram C begin
  v::V
  (e1, e2)::E
  t::tgt : e1 → v
  s::src : e2 → v
end
J = FinCat(@acset NamedGraph{Symbol} begin
  V = 3
  E = 2
  src = [2,3]
  tgt = [1,1]
  vname = [:v, :e1, :e2]
  ename = [:t, :s]
end)
@test dom(F_parsed) == J
F = FinFunctor([:V,:E,:E], [:tgt, :src], J, C)
@test F_parsed == F

F_parsed = @diagram C begin
  v => V
  (e1, e2) => E
  t: e1 → v => tgt
  s: e2 → v => src
end
@test F_parsed == F

end


function tensorproduct(x::Array{Float64,2},porder::Int)::Tuple{Array{Float64,2},Array{Float64,2},Array{Float64,2},Array{Float64,2}}
# #TENSORPRODUCT Vandermonde matrices for tensor product polynomials in [0,1]^d
# #
# #   [F,FX,FY,FZ] = TENSORPRODUCT(X,P)
# #
# #      X:         Coordinates of the points wherethe polynomials
# #                 are to be evaluated (npoints,dim)
# #      PORDER:    Maximum order of the polynomials consider. That
# #                 is all polynomials of complete degree up to p,
# #                 npoly = (PORDER+1)*(PORDER+1)*(PORDER+1)
# #      F:         Vandermonde matrix (npoints,npoly)
# #      FX:        Vandermonde matrix for the derivative of the Koornwinder
# #                 polynomials w.r.t. x (npoints,npoly)
# #      FY:        Vandermonde matrix for the derivative of the Koornwinder
# #                 polynomials w.r.t. y (npoints,npoly)
# #      FZ:        Vandermonde matrix for the derivative of the Koornwinder
# #                 polynomials w.r.t. z (npoints,npoly)
# #

n,dim = size(x);

if dim== 1 # 1D
        f,fx = legendrepolynomial(x,porder);  # Legendre basis
        fy=[0.0 0.0];
        fz=[0.0 0.0];
elseif dim == 2 # 2D
        g1,gx = legendrepolynomial(x[:,1],porder); # Legendre basis in x direction
        g2,gy = legendrepolynomial(x[:,2],porder); # Legendre basis in y direction
        f  = zeros(Float64,n,(porder+1)*(porder+1));
        fx = 0*f;
        fy = 0*f;
        fz=[0.0 0.0];
        # perform tensor product to obtain the shape functions and their
        # derivatives on the unit square
        for ii=1:n
            tm = kron(g2[ii,:],g1[ii,:]);
            f[ii,:] =  kron(g2[ii,:],g1[ii,:]);
            fx[ii,:] = kron(g2[ii,:],gx[ii,:]);
            fy[ii,:] = kron(gy[ii,:],g1[ii,:]);
        end
elseif dim==3
        g1,gx=legendrepolynomial(x[:,1],porder); # Legendre basis in x direction
        g2,gy=legendrepolynomial(x[:,2],porder); # Legendre basis in y direction
        g3,gz=legendrepolynomial(x[:,3],porder); # Legendre basis in z direction
        f  = zeros(Float64,n,(porder+1)*(porder+1)*(porder+1));
        fx = 0*f;
        fy = 0*f;
        fz = 0*f;
        # perform tensor product to obtain the shape functions and their
        # derivatives on the unit cube
        for ii=1:n
            f[ii,:] =  kron(g3[ii,:],kron(g2[ii,:],g1[ii,:]));
            fx[ii,:] = kron(g3[ii,:],kron(g2[ii,:],gx[ii,:]));
            fy[ii,:] = kron(g3[ii,:],kron(gy[ii,:],g1[ii,:]));
            fz[ii,:] = kron(gz[ii,:],kron(g2[ii,:],g1[ii,:]));
        end
else
        error("Only can handle dim=1, dim=2 or dim=3");
end

return f, fx, fy, fz;

end


include("getdigit_n.jl")
include("myprintf.jl")
using Printf
using .GetDigit_n
using .MyPrintf

const DIGIT = 100
const MAXN = 100

function factorial(m)
    return m < 2 ? BigFloat(1) : reduce(*, [BigFloat(i) for i in 2:m])
end

function ramanujan(n)
    res = BigFloat(0)
    for k = 0:n
        numerator = factorial(4 * k) * (BigFloat(1103) + BigFloat(26390 * k))
        denominator = (factorial(k) * (BigFloat(396))^(BigFloat(k)))^(BigFloat(4))

        res += numerator / denominator
    end

    return BigFloat(9801) / (res * BigFloat(2) * sqrt(BigFloat(2)))
end

function main()
    n = GetDigit_n.getn(ramanujan, MAXN, DIGIT, true)
    precision = GetDigit_n.getdigit(ramanujan, n, DIGIT, true)
    precision = GetDigit_n.getdigit_2(ramanujan, n, DIGIT, precision)

    setprecision(precision)
    MyPrintf.myprintf("π（計算値） = ", DIGIT, ramanujan(n))
    MyPrintf.myprintf("π（厳密値） = ", DIGIT, BigFloat(π))
    @printf("n = %d, 計算%d桁で、%d桁求まりました\n", n, precision, DIGIT)
end

main()


export ScaledBeta, TrGeometric, TrBinomial, StringCategorical, likelihood, log_likelihood
export normalising_const
## Scaled beta distribution
"""
Scaled Beta distribution. Instantiate with ScaledBeta() e.g.
```
d = ScaledBeta(α, β, 0.0, 10.0)
```
would make a ScaledBeta type d which is bounded by 0.0 and 10.0. Note
- (α,β) are usual parameters;
- lb = lower bound;
- ub = upper bound.
"""
struct ScaledBeta<:ContinuousUnivariateDistribution
    α::Float64
    β::Float64
    lb::Float64
    ub::Float64
end

Distributions.rand(d::ScaledBeta) = d.lb + (d.ub-d.lb) * rand(Beta(d.α, d.β))
Distributions.mean(d::ScaledBeta) = d.lb + (d.ub-d.lb) * ( d.α / (d.α + d.β) )
Distributions.pdf(d::ScaledBeta, x::Real) = (d.lb ≤ x ≤ d.ub) ? pdf(Beta(d.α, d.β), (x - d.lb)/(d.ub-d.lb))/(d.ub-d.lb) : 0
Distributions.minimum(d::ScaledBeta) = d.lb
Distributions.maximum(d::ScaledBeta) = d.ub

## Trucated geometric
"""
`TrGeomtric(p::Float64, lb::Int, ub::Int)`

Truncated Geometric distribution. Note:
- lb = lower bound = samllest possible value (default=0)
- ub = upper bound = largest possible value.
Instantiation:

1. `TrGeometric(p, ub)` -  here `lb` defaults to 0
2. `TrGeometric(p, lb, ub)` here `lb` is specified

"""
struct TrGeometric<:DiscreteUnivariateDistribution
    p::Float64 # Prob param
    lb::Int # Lower bound (smallest value)
    ub::Int  # Upper bound
end

function normalising_const(d::TrGeometric)
    return (1 - (1-d.p)^(d.ub-d.lb+1))
end

TrGeometric(p::Float64, ub::Int) = TrGeometric(p, 0, ub)

function Distributions.pdf(d::TrGeometric, x::Integer)
    _insupport = insupport(d, x)
    return _insupport ? d.p*(1-d.p)^(x-d.lb) / normalising_const(d) : 0.0
end

function Distributions.logpdf(d::TrGeometric, x::Integer)
    _insupport = insupport(d, x)
    return _insupport ? log(d.p) + (x-d.lb) * log(1-d.p) -  log(normalising_const(d)) : -Inf
end

function Distributions.cdf(d::TrGeometric, x::Number)
    if d.lb ≤ x ≤ d.ub
        return ( 1 - (1 - d.p)^(trunc(Int,x)-d.lb+1) )/normalising_const(d)
    else
        return float(x > d.ub)
    end
end

# Here I make use of closed form of generalised inverse of the TrGeom dist.
# Not sure if this is the fastest implementation
function Distributions.quantile(d::TrGeometric, q::Real)
    return 0.0<q<1.0 ? ceil(Int, log(1-q*normalising_const(d))/log(1-d.p) - 1 )+d.lb : d.ub*(p≥1.0) + d.lb*(q≤0.0)
end

Distributions.minimum(d::TrGeometric) = d.lb
Distributions.maximum(d::TrGeometric) = d.ub

#Truncated Binomial wrappper - solves the problem of when we have n=0

struct TrBinomial <: DiscreteUnivariateDistribution
    n::Int
    p::Real
    lb::Int
    ub::Int
    dist::Distribution
    function TrBinomial(n::Int, p::Real, lb::Int, ub::Int)
        @assert (lb ≤ ub) "Lower bound must be ≤ upper bound."
        if lb < ub
            tmp_dist = Truncated(Binomial(n, p), lb, ub)
        else
            tmp_dist = DiscreteUniform(lb, ub)
        end
        new(n, p, lb, ub, tmp_dist)
    end
end

function Distributions.rand(d::TrBinomial)
    return rand(d.dist)
end
function Distributions.rand(d::TrBinomial, n::Int)
    return rand(d.dist, n)
end

struct StringCategorical <: DiscreteUnivariateDistribution
    x::Vector{String}
    p::Vector{T} where T<:Real
    latent_dist::Distribution
    probability_map::Dict{String, Float64}
    function StringCategorical(x::Vector{String}, p::Vector{T} where T<: Real)
        latent_dist = Categorical(p)
        probability_map = Dict(val => prob for (val, prob) in zip(x, p))
        new(x, p, latent_dist, probability_map)
    end
end

Distributions.pdf(d::StringCategorical, val::String) = d.probability_map[val]
function likelihood(d::StringCategorical, val::Vector{String})
    z = 1.0
    for s in val
        z *= pdf(d, s)
    end
    return z
end

Distributions.logpdf(d::StringCategorical, val::String) = log(d.probability_map[val])
function log_likelihood(d::StringCategorical, val::Vector{String})
    z = 0.0
    for s in val
        z += logpdf(d, s)
    end
    return z
end

function Distributions.rand(d::StringCategorical)
    return d.x[rand(d.latent_dist)]
end

function Distributions.rand(d::StringCategorical, n::Int)
    return d.x[rand(d.latent_dist, n)]
end


# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    PlanePartition(normal; tol=1e-6)

A method for partitioning spatial data into a family of hyperplanes defined
by a `normal` direction. Two points `x` and `y` belong to the same
hyperplane when `(x - y) ⋅ normal < tol`.
"""
struct PlanePartition{T,N} <: SPredicatePartitionMethod
  normal::SVector{N,T}
  tol::Float64

  function PlanePartition{T,N}(normal, tol) where {N,T}
    new(normalize(normal), tol)
  end
end

PlanePartition(normal::SVector{N,T}; tol=1e-6) where {T,N} =
  PlanePartition{T,N}(normal, tol)

PlanePartition(normal::NTuple{N,T},; tol=1e-6) where {T,N} =
  PlanePartition(SVector(normal), tol=tol)

(p::PlanePartition)(x, y) = abs((x - y) ⋅ p.normal) < p.tol


using SymPy

@syms px1 py1 lambda1 vx1 vy1 px2 py2 lambda2 vx2 vy2

lx1 = px1 + lambda1*vx1
ly1 = py1 + lambda1*vy1

lx2 = px2 + lambda2*vx2
ly2 = py2 + lambda2*vy2

lx12 = solve(lx1-lx2, lambda2)[1]
l2i = subs(ly2, lambda2, lx12)

sol = solve(ly1-l2i, lambda1)[1]

xi = simplify(subs(lx1, lambda1, sol))
yi = simplify(subs(ly1, lambda1, sol))


using FinancePlus
using Test

@testset "FinancePlus.jl" begin
    # Write your tests here.
    @test FinancePlus.simple_interest(100.00 , 0.1 ,2) == 20.0
    @test FinancePlus.compound_interest(100.00 , 0.1 ,2) == 21.000000000000014
    @test FinancePlus.future_value(100.00 , 0.1 ,2) == 121.00000000000001
    @test FinancePlus.present_value(121.00000000000001 , 0.1 ,2) == 100.00
    @test FinancePlus.future_value(100.00 , 0.1 ,2, 1) == 121.00000000000001
    @test FinancePlus.future_value(100.00 , 0.1 ,2, 2) == 121.55062500000003
    @test FinancePlus.present_value(121.00000000000001 , 0.1 ,2, 2) == 99.54699944981772
    @test FinancePlus.present_value(121.00000000000001 , 0.1 ,2, 1) == 100.00
    @test FinancePlus.present_annuity(50000.00 , 0.06 ,25) == 639167.8079134206
    @test FinancePlus.present_annuity_due(50000.00 , 0.06 ,25) == 677517.8763882258
    @test FinancePlus.present_value_interest_factor(0.05 ,5) == 0.7835261664684589
    @test FinancePlus.future_value_interest_factor(0.05 ,5) == 1.2762815625000004

end


abstract type PrimeField <: Number end

infield(x::Number,y::Number) = x >= 0 && x < y

"Represents FieldElement type in which 𝑛 ∈ 𝐹𝑝 and 𝑝 ∈ ℙ"
struct FieldElement <: PrimeField
    𝑛::Integer
    𝑝::Integer
    FieldElement(𝑛,𝑝) = !infield(𝑛,𝑝) ? throw(DomainError("𝑛 is not in field range")) : new(𝑛,𝑝)
end

"Formats PrimeField as 𝑛 : 𝐹ₚ"
function show(io::IO, z::PrimeField)
    print(io, z.𝑛, " : 𝐹", z.𝑝)
end

"Returns true if both 𝑛 and 𝑝 are the same"
==(𝑋₁::PrimeField,𝑋₂::PrimeField) = 𝑋₁.𝑝 == 𝑋₂.𝑝 && 𝑋₁.𝑛 == 𝑋₂.𝑛
==(::PrimeField,::Integer) = false

"Adds two numbers of the same field"
function +(𝑋₁::PrimeField,𝑋₂::PrimeField)
    if 𝑋₁.𝑝 != 𝑋₂.𝑝
        throw(DomainError("Cannot operate on two numbers in different Fields"))
    else
        𝑛 = mod(𝑋₁.𝑛 + 𝑋₂.𝑛, 𝑋₁.𝑝)
        return typeof(𝑋₁)(𝑛, 𝑋₁.𝑝)
    end
end

"Substracts two numbers of the same field"
function -(𝑋₁::PrimeField,𝑋₂::PrimeField)
    if 𝑋₁.𝑝 != 𝑋₂.𝑝
        throw(DomainError("Cannot operate on two numbers in different Fields"))
    else
        𝑛 = mod(𝑋₁.𝑛 - 𝑋₂.𝑛, 𝑋₁.𝑝)
        return typeof(𝑋₁)(𝑛, 𝑋₁.𝑝)
    end
end

"Multiplies two numbers of the same field"
function *(𝑋₁::PrimeField,𝑋₂::PrimeField)
    if 𝑋₁.𝑝 != 𝑋₂.𝑝
        throw(DomainError("Cannot operate on two numbers in different Fields"))
    else
        𝑛 = mod(𝑋₁.𝑛 * 𝑋₂.𝑛, 𝑋₁.𝑝)
        return typeof(𝑋₁)(𝑛, 𝑋₁.𝑝)
    end
end

"Multiplies a PrimeField by an Integer"
function *(𝑐::Integer,𝑋::PrimeField)
    𝑛 = mod(𝑐 * 𝑋.𝑛, 𝑋.𝑝)
    return typeof(𝑋)(𝑛, 𝑋.𝑝)
end

"Returns 𝑋ᵏ using Fermat's Little Theorem"
function ^(𝑋::PrimeField,𝑘::Int)
    𝑛 = powermod(𝑋.𝑛, mod(𝑘, (𝑋.𝑝 - 1)), 𝑋.𝑝)
    return typeof(𝑋)(𝑛, 𝑋.𝑝)
end

"Returns 1/𝑋 as a special case of exponentiation where 𝑘 = -1"
function inv(𝑋::PrimeField)
    𝑛 = powermod(𝑋.𝑛, mod(-1, (𝑋.𝑝 - 1)), 𝑋.𝑝)
    return typeof(𝑋)(𝑛, 𝑋.𝑝)
end

function div(𝑋₁::PrimeField,𝑋₂::PrimeField)
    return 𝑋₁ / 𝑋₂
end

"Returns 𝑋₁/𝑋₂ using Fermat's Little Theorem"
function /(𝑋₁::PrimeField,𝑋₂::PrimeField)
    if 𝑋₁.𝑝 != 𝑋₂.𝑝
        throw(DomainError("Cannot operate on two numbers in different Fields"))
    else
        𝑛 = mod(𝑋₁.𝑛 * powermod(𝑋₂.𝑛, 𝑋₁.𝑝 - 2, 𝑋₁.𝑝), 𝑋₁.𝑝)
        return typeof(𝑋₁)(𝑛, 𝑋₁.𝑝)
    end
end


# p28b.jl - eigenmodes of Laplacian on the disk

# r coordinate, ranging from -1 to 1 (N must be odd)
N = 25; N2 = Int((N-1)/2);
(D,r) = cheb(N); D2 = D^2;
D1 = D2[2:N2+1,2:N2+1]; D2 = D2[2:N2+1,N:-1:N2+2];
E1 =  D[2:N2+1,2:N2+1]; E2 =  D[2:N2+1,N:-1:N2+2];

# t = theta coordinate, ranging from 0 to 2*pi (M must be even):
M = 20; dt = 2*pi/M; t = dt*(1:M); M2 = Int(M/2);
D2t = toeplitz([-pi^2/(3*dt^2)-1/6; @. .5*(-1)^(2:M)/sin(dt*(1:M-1)/2)^2]);

# Laplacian in polar coordinates:
R = diagm(1./r[2:N2+1]);
Z = zeros(M2,M2); I = eye(M2);
L = kron(D1+R*E1,eye(M)) + kron(D2+R*E2,[Z I;I Z]) + kron(R^2,D2t);

# Compute 25 eigenmodes:
index = 1:25;
(Lam,V) = eig(-L); ii = sortperm(abs.(Lam))[index];
Lam = Lam[ii]; V = V[:,ii];
Lam = sqrt.(real(Lam/Lam[1]));

# Plot nodal lines:
(rr,tt) = (r[1:N2+1],[0;t]);
(xx,yy) = @. (cos(tt)*rr',sin(tt)*rr');
z = exp.(1im*pi*(-100:100)/100);  clf();
for i = 1:25
    subplot(5,5,i);
    u = reshape(real(V[:,i]),M,N2);
    u = [zeros(M+1) u[[M;1:M],:]];
    u = u/norm(u[:],Inf);
    plot(real(z),imag(z));
    xlim(-1.07,1.07); ylim(-1.07,1.07); axis("off"); axis("equal");
    contour(xx,yy,u,levels=[0]);
    title("$(signif(Lam[i],5))",fontsize=8);
end


using FEMfunctions

using LinearAlgebra
using SparseArrays

include("twodElliptic.jl")

x,eConn,u = twodElliptic(129)

function uExact(x)
  C = 0.2/π^2
  return C*sin(π*x[1])*sin(2*π*x[2])
end

nNodes = size(x,1)
uEx = zeros(Float64,nNodes)
for i=1:nNodes
  uEx[i] = uExact(x[i,:])
end

M = twodMassMatrix(x,eConn)

diff = u-uEx
@test sqrt( dot(diff,(M*diff)) ) < 1e-8


using IntervalArithmetic
using Test

using LinearAlgebra

@testset "Complex interval operations" begin
    a = @interval 1im
    b = @interval 4im + 3
    c = (@interval -1 4) + (@interval 0 2)*im

    @test typeof(a) == Complex{IntervalArithmetic.Interval{Float64}}
    @test a ==  Interval(0) + Interval(1)*im
    @test a * a == Interval(-1)
    @test a + a == Interval(2)*im
    @test a - a == 0
    @test a / a == 1

    @test 3+2im ∈ c
    @test a ∪ b == (@interval 0 3) + (@interval 1 4)*im
    @test c ∩ (a ∪ b) == (@interval 0 3) + (@interval 1 2)*im
    @test a ∩ b == ∅ + ∅*im
    @test isdisjoint(a,b) == true
end

@testset "Complex functions" begin
    Z = (3 ± 1e-7) + (4 ± 1e-7)*im
    @test sin(Z) == complex(sin(real(Z)) * cosh(imag(Z)), sinh(imag(Z)) * cos(real(Z)))
    @test exp(-im * Interval(π)) == Interval(-1.0, -0.9999999999999999) - Interval(1.224646799147353e-16, 1.2246467991473532e-16)*im

    sZ = sqrt(Z)
    @test sZ == Interval(1.99999996999999951619,2.00000003000000070585) + Interval(0.99999996999999984926,1.00000003000000048381)*im
    @test sqrt(-Z) == imag(sZ) - real(sZ)*im

    @test sqrt((@interval -1 0) + (@interval 0)*im) .== (@interval 0 1)*im
    @test sqrt((@interval -1 1) + (@interval 0)*im) .== (@interval 0 1) + (@interval 0 1)*im
    @test sqrt((@interval -9//32 Inf)*im + (@interval 0)) .== (@interval 0 Inf) + (@interval -3//8 Inf)*im
end

@testset "Complex powers" begin
    x = (3..3) + 4im
    @test x^2 == -7 + 24im
    @test sqrt(x) ⊆ x^0.5
    @test x^-2 == inv(x)^2

    a = -3.1
    @test x ⊆ (x^a)^(1/a)
end

@testset "abs2 and abs" begin
    x = (0..3) + (0..4)*im
    @test abs2(x) == 0..25
    @test abs(x) == norm(x) == 0..5

    y = (-1..1) + (-2..2)*im
    @test abs(y).lo == 0.0
    @test abs2(y).lo == 0.0
end


#=
Starting with 1 and spiralling anticlockwise in the following way, a square spiral with side length 7 is formed.

37 36 35 34 33 32 31
38 17 16 15 14 13 30
39 18  5  4  3 12 29
40 19  6  1  2 11 28
41 20  7  8  9 10 27
42 21 22 23 24 25 26
43 44 45 46 47 48 49

It is interesting to note that the odd squares lie along the bottom right diagonal, but what is more interesting is that 8 out of the 13 numbers lying along both diagonals are prime; that is, a ratio of 8/13 ≈ 62%.

If one complete new layer is wrapped around the spiral above, a square spiral with side length 9 will be formed. If this process is continued, what is the side length of the square spiral for which the ratio of primes along both diagonals first falls below 10%?
=#
using Primes

function calc()
  start, step = 1,2
  iterations, currentsquaresize = 3,3
  cprimes, call = 0,1
  while true
    start += step
    for i in start:step:(start+(iterations*step))
      if isprime(i)
        cprimes = cprimes + 1
      end
      call = call + 1
    end
    ratio = cprimes / call
    if ratio < 0.1
      return currentsquaresize
    end
    start += (iterations*step)
    currentsquaresize += 2
    step += 2
  end
end
@time println(calc())


"""
    sphericalhankel2(nu, x)

Spherical Hankel function of the second kind, order `nu`
and argument `x`.
"""
function sphericalhankel2(nu, x)
    jn = SpecialFunctions.sphericalbesselj(nu, x)
    yn = SpecialFunctions.sphericalbessely(nu, x)
    return jn - im*yn
end

"""
    sphericalhankel1(nu, x)

Spherical Hankel function of the first kind, order `nu`
and argument `x`.
"""
function sphericalhankel1(nu, x)
    return conj(sphericalhankel2(nu, x))
end

"""
    sphericalhankel2_and_derivatives(nu::UnitRange, x)
    sphericalhankel2_and_derivatives(nu::Number, x)

Evaluates the spherical Hankel function of the second kind, its first and
second derivative, for orders `nu::UnitRange` and argument `x`.
"""
function sphericalhankel2_and_derivatives(nu::UnitRange, x)
    start = nu.start
    stop = nu.stop
    len = stop-start+1   # number of terms to compute
    sph = sphericalhankel2.(start:stop+2, x)  # compute two more terms
    # compute first derivative using recurrence
    # h_n'(x) = -h_{n+1}(x) + n/x*h_n(z)
    sph_d = similar(sph, len+1)   # compute one more term
    n = stop+1   # order
    for index in len+1:-1:1
        sph_d[index] = -sph[index+1] + n/x*sph[index]
        n -= 1
    end
    # compute second derivative using recurrence
    # h_n''(x) = n / x^2 * (x * h_n'(x) - h_n(x)) - h_{n+1}'(x)
    sph_dd = similar(sph, len)
    n = stop   # order
    for index in len:-1:1
        sph_dd[index] = n / x^2 * (x*sph_d[index] - sph[index]) - sph_d[index+1]
        n -= 1
    end
    return sph[1:len], sph_d[1:len], sph_dd
end
function sphericalhankel2_and_derivatives(nu::Number, x)
    sph, sph_d, sph_dd = sphericalhankel2_and_derivatives(nu:nu, x)
    return first(sph), first(sph_d), first(sph_dd)
end

"""
    sphericalhankel1_and_derivatives(nu::UnitRange, x)
    sphericalhankel1_and_derivatives(nu::Number, x)

Evaluates the spherical Hankel function of the first kind, its first and
second derivative, for orders `nu` and argument `x`.
"""
function sphericalhankel1_and_derivatives(nu, x)
    return conj.(sphericalhankel2_and_derivatives(nu, x))
end

"""
    riccatihankel2_and_derivatives(nu::UnitRange, x)

Evaluates the spherical Riccati-Bessel Hankel function of the second kind, its first and
second derivative, for orders `nu::UnitRange` and argument `x`.
"""
function riccatihankel2_and_derivatives(nu::UnitRange, x)
    sph, sph_d, sph_dd = sphericalhankel2_and_derivatives(nu, x)
    rh = x.*sph
    rh_d = sph + x.*sph_d
    rh_dd = 2*sph_d + x.*sph_dd
    return rh, rh_d, rh_dd
end


using SparseGrids
using Base.Test

# Moments of exp(-x^2)
function gaussmoment(m::Int)
	@assert m >= 0

	if isodd(m)
		return 0.0
	elseif iseven(m)
		m2 = div(m,2)
		return prod( 1:2:m-1 ) * sqrt(pi) / 2^m2
	end
end

# Moments of exp(-|x|^2)
function gaussmoment( P::Vector{Int} )
	I = 1.0
	for d = 1:length(P)
		I *= gaussmoment(P[d])
	end

	return I
end

# Like gaussmoment, but with quadrature rule
function gaussquad( P::Vector, nodes::Matrix, weights::Vector )
	# Evaluate integrand
	F = broadcast( ^, nodes, P )
	I = vec(prod(F,1))

	# Evaluate quadrature sum
	Q = dot( I, weights )

	return Q
end

# ------------------------------------------------------------
# Test quadrature for various dimensions and orders
# Note: It is not interesting to test uneven moments for Gauss-Hermite

# Dimension and order
D = 3
order = 4

# Moments to test
vecs = Vector{Vector{Int}}(D)
for d in 1:D
	vecs[d] = [0:order;]
end
P = combvec( vecs )

M = size(P, 2)

for generator = [FastGaussQuadrature.gausshermite, kpn]
	# Quadrature points and weights
	N, W = sparsegrid( D, order, generator )

	# Maximum degree for which the generator gives correct results
	generator == FastGaussQuadrature.gausshermite ?  max_degree = 2*order-1 : max_degree = D*order

	for m = 1:M
		# Test if all moments are even
		curP = P[:,m]

		if all(map(iseven, curP))
			I = gaussmoment(curP)
			Q = gaussquad(curP, N, W)

			# Expected test result depends on the total degree
			if sum(curP) <= max_degree
				@test_approx_eq I Q
			else
				@test abs(I - Q) > 1e-3
			end
		end
	end
end