# Deterministic Value Function Iteration
# Greg Kaplan 2017
# Translated by Tom Sweeney Dec 2020

using Random, Interpolations, Plots

# PARAMETERS

## preferences
risk_aver = 2
beta = 0.95

## returns
r = 0.03
R = 1+r

## income
y = 1

## asset grids
na = 1000
amax = 20
borrow_lim = 0
agrid_par = 1 # 1 for linear, 0 for L-shaped

## computation
max_iter = 1000
tol_iter = 1.0e-6
Nsim = 100
Tsim = 500

# OPTIONS
Display = 1
DoSimulate = 1
MakePlots = 1

# DRAW RANDOM NUMBERS
Random.seed!(2020)
arand = rand(Nsim)

# SET UP GRIDS

## assets
agrid = range(0,1,length=na)
agrid = agrid.^(1 ./ agrid_par)
agrid = borrow_lim .+ (amax.-borrow_lim).*agrid

# UTILITY FUNCTION

if risk_aver==1
    u(c) = log.(c)
else
    u(c) = (c.^(1-risk_aver).-1)./(1-risk_aver)
end

u1(c) = c.^(-risk_aver)

# INITIALIZE VALUE FUNCTION

Vguess = u(r.*agrid.+y)./(1-beta)

# ITERATE ON VALUE FUNCTION

V = copy(Vguess)

Vdiff = 1
iter = 0

while iter <= max_iter && Vdiff > tol_iter
    iter = iter + 1
    Vlast = copy(V)
    V = zeros(na)
    global sav = zeros(na)
    global savind = zeros(Int,na)
    global con = zeros(na)

    ## loop over assets
    for ia = 1:na

        cash = R.*agrid[ia] + y
        Vchoice = u(max.(cash.-agrid,1.0e-10)) + beta.*Vlast
        V[ia] = maximum(Vchoice)
        savind[ia] = argmax(Vchoice)[1]
        sav[ia] = agrid[savind[ia]]
        con[ia] = cash .- sav[ia]
    end

    Vdiff = maximum(abs.(V-Vlast))
    if Display>=1
        println("Iteration no. " * string(iter), " max val fn diff is " * string(Vdiff))
    end
end

# SIMULATE
if DoSimulate==1
    yindsim = zeros(Int,Nsim,Tsim)
    aindsim = zeros(Int,Nsim,Tsim)

    ## initial assets: uniform on [borrow_lim, amax]
    ainitial = borrow_lim .+ arand.*(amax-borrow_lim)

    ## allocate to nearest point on agrid;
    aindsim[:,1] = interpolate((agrid,), 1:na, Gridded(Constant())).(ainitial)

    ## loop over time periods
    for it = 1:Tsim
        if Display >= 1 && mod(it,100)==0
            println(" Simulating, time period " * string(it))
        end
        ## asset choice
        if it<Tsim
            aindsim[:,it+1] = savind[aindsim[:,it]]
        end
    end

    ## assign actual asset and income values;
    asim = agrid[aindsim]
    csim = R.*asim[:,1:Tsim-1] .+ y .- asim[:,2:Tsim]
end

# MAKE PLOTS
if MakePlots==1

    ## consumption policy function
    p1 = plot(agrid, con, xlims=(0,amax), title="Consumption", color=:blue, legend=false)
    display(p1)

    ## savings policy function
    p2 = plot(agrid, sav.-agrid, xlims=(0,amax), title="Savings", color=:blue, legend=false)
    plot!(agrid, zeros(na,1), color=:black, lw=0.5)
    display(p2)

    ## nice zoom
    xlimits = (0,1)
    xlimind = trues(na)
    if minimum(agrid) < xlimits[1]
        xlimind = xlimind .& (agrid.>=maximum(agrid[agrid<xlimits[1]]))
    elseif minimum(agrid) > xlimits[2]
        xlimind .= 0
    end
    if maximum(agrid) > xlimits[2]
        xlimind = xlimind .& (agrid.<=minimum(agrid[agrid.>xlimits[2]]))
    elseif maximum(agrid) < xlimits[1]
        xlimind .= 0
    end

    ## consumption policy function: zoomed in
    p3 = plot(agrid[xlimind], con[xlimind], xlims=xlimits, title="Consumption: Zoomed", marker=:circle, color=:blue, linewidth=2, legend=false)
    plot!(show=true)
    display(p3)

    ## savings policy function: zoomed in
    p4 = plot(agrid[xlimind], sav[xlimind].-agrid[xlimind], xlims=xlimits, title="Savings: Zoomed (a'-a)", marker=:circle, color=:blue, linewidth=2, legend=false)
    plot!(agrid, zeros(na), color=:black, lw=0.5)
    display(p4)

    ## asset dynamics distribution
    p5 = plot(repeat((1:Tsim),1,Nsim), asim', title="Asset Dynamics", legend=false)
    display(p5)

    ## consumption dynamics distribution
    p6 = plot(repeat((1:Tsim-1),1,Nsim), csim', title="Consumption Dynamics", legend=false)
    display(p6)

end


function  poisson(theta, y, x)
	lambda = exp.(x*theta)
    n = size(y,1)
    c = zeros(n)
    for i = 1:n
        c[i] = logabsgamma(y[i])[1]
    end
    logdensity = y .* (x*theta) .- lambda .- c
end


module DMatTest
using Base.Test
using Quantum

xb = basis(:X,[1:3])
xbplus = basis(:X,[1:4])
xbminus = basis(:X,[0:3])

d = dmat(eye(3), xb)
@test d==d''
@test d[:,:]==d.coeffs
@test d*ket(:X,2) == dvec([0.0,1.0,0.0], xb)

@test d+ket(:X,4)*bra(:X,4) == dmat(eye(4),xbplus)
@test d+ket(:X,4)*bra(:X,3) == dmat([eye(3), [0 0 1]],xbplus, xb')
@test d+ket(:X,3)*bra(:X,4) == dmat([eye(3) [0,0,1]],xb, xbplus')
@test d+ket(:X,3)*bra(:X,3) == dmat((coeff=eye(3); coeff[3,3]=2.0; coeff),xb)

@test ket(:X,0)*bra(:X,0)+d == dmat(eye(4),xbminus)
@test ket(:X,0)*bra(:X,1)+d == dmat([[1 0 0], eye(3)],xbminus, xb')
@test ket(:X,1)*bra(:X,0)+d == dmat([[1,0,0] eye(3)],xb, xbminus')
@test ket(:X,1)*bra(:X,1)+d == dmat((coeff=eye(3); coeff[1,1]=2.0; coeff),xb)

@test d+d==2*d
@test isdual(d,d')
@test map(x->x*2, d)==d+d
@test commutator(d,d)==dmat(zeros(3,3), xb)

end


"""
    xxhat, Mhat = kalmanEM(yy, M1::LinearHomogSystem, maxit = 500, tol = 0.001)

EM procedure for the state space model (Shumway, Stoffer)

- yy -- data
- M1 -- initial model
- maxit -- maximum number of iterations
- tol -- rel. tolerance for test for convergence of marginal likelihood

- xxhat, Mhat -- smoothed process with estimated model
"""
function kalmanEM(yy, M::LinearHomogSystem, maxit = 500, tol = 0.001)

    H, Phi, b, Q, R = M.H, M.Phi, M.b, M.Q, M.R
    d2, d = size(M.H)
    assert(ndims(yy) == 2)
    assert(norm(b) == 0)
    n = size(yy,2)

    x0 = M.x0
    P0 = M.P0
    xxs = xxf = zeros(d, n) # #x(f/s)[k] predited and corrected (and smoothed)

    PPf = zeros(d, d, n)  #P[k,k]
    PPpred = zeros(d, d, n)     #P[k, k-1]
    PPs = zeros(d, d, n)  #P[k,k]
    PPcs = zeros(d, d, n)  #P[k-1,k]

    K = zeros(d, d)

    cvg = 1.+tol
    nll = NaN
    for k in 1:maxit
        xf = x0
        Pf = P0

        # E step, forward pass
        nllold = nll
        nll = 0.
        for i in 1:n
            xf, Pf, Ppred, l, K  = kalman_kernel(xf, yy[:, i], Pf, H, Phi, b, Q, R)
            xxf[:, i], PPf[:, :, i], PPpred[:, :, i] = xf, Pf, Ppred
            l += d*log(2pi)/2         # constant messes up convergence test
            nll -= l
        end

        if k > 1
            cvg = (nllold-nll)/abs(nllold)
            cvg < 0 && warn("Likelihood decreasing")
            if abs(cvg) < tol
#                println([nll, cvg])
                break
            end
        end

        # E step, backwards pass including PPcs

        #start with xf, Pf from forward pass
        xs = xf
        Ps = Pf

        PPcs[:, :, n] = (I - K*H)*Phi*PPf[:, :, n-1]
        J = PPf[:, :, n]*Phi'/(Phi*Ps*Phi' + Q)
        PPs[:, :, n] = Ps
        for i in n-1:-1:1
            Jprev = J
            xs, Ps, J = smoother_kernel(xs, Ps, xxf[:, i],  PPf[:, :, i], PPpred[:, :, i+1], Phi, b)
            xxs[:, i], PPs[:, :, i] = xs, Ps

            j = i + 2 #i = j - 2 as J = J(i)
            if 2 < j <= n

                 Jjm2 = J
                 Jjm1 = Jprev
                 PPcs[:, :, j-1] = PPf[:, :, j-1]*Jjm2' +  Jjm1*(PPcs[:, :, j]  - Phi*PPf[:, :, j-1])*Jjm2'
                 #Pcs[,,j-1]=Pf[,,j-1]%*%t(J[,,j-2])+ J[,,j-1]%*%(Pcs[,,j]-Phi%*%Pf[,,j-1])%*%t(J[,,j-2])}

            end
        end
        x0, P0, J0 = smoother_kernel(xs, Ps, x0, P0, PPpred[:, :, 1], Phi, b)
        PPcs[:, :, 1] = PPf[:, :, 1]*J0' +  J*(PPcs[:, :, 2]  - Phi*PPf[:, :, 1])*J0'

        # M step

        A11 = xxs[:, 1]*xxs[:, 1]' + PPs[:, :, 1]
        A10 = xxs[:, 1]*x0' + PPcs[:, :, 1]
        A00 = x0*x0' + P0

        u = yy[:,1]-H*xxs[:,1]
        R = u*u' + H*PPs[:, :, 1]*H'

        for i in 1:n-1
            A11 = A11 + xxs[:, i+1]*xxs[:, i+1]' + PPs[:, :, i+1]
            A10 = A10 + xxs[:, i+1]*xxs[:, i]' + PPcs[:, :, i+1]
            A00 = A00 + xxs[:, i]*xxs[:, i]' + PPs[:, :, i]

            u = yy[:,i+1]-H*xxs[:,i+1]
            R = R + u * u' + H*PPs[:, :, i+1]*H'
        end

        Phi = A10/A00
        Q = (A11 - Phi*A10')/n
        Q = (Q+Q')/2
        R = R/n

#        println("PARAM $k: nll", [nll, cvg], "\n x0 $x0\n P0 $P0\n R $R\n Phi $Phi\n Q $Q")

    end

    xxs, LinearHomogSystem(x0, P0, Phi, b, Q, H, R)
end


using DrWatson
@quickactivate "StatReth"

# %%
using StatsBase
using Distributions
using Plots
using StatsPlots
using Turing  # for MCMCChains

# %% 3.2
n = 1000
p_grid = range(0, 1, length = n)
prob_p = ones(n)
prob_data = @. pdf(Binomial(9, p_grid), 6)
posterior = prob_data .* prob_p
posterior ./= sum(posterior)

plot(p_grid, posterior)

# %% 3.3 - 3.5
weights = pweights(posterior)
samples = sample(p_grid, weights, 10_000)

scatter(samples)
density(samples)
# plot!(p_grid, posterior * n)

# %% 3.6, 3.7
sum(posterior[p_grid .< 0.5])

sum(samples .< 0.5) / 10_000    # either
mean(samples .< 0.5)            # or

# %% 3.8
mean(0.5 .< samples .< 0.75)    # either
mean(samples) do s              # or (this will be faster than above)
    0.5 < s < 0.75
end
mean(samples) do s              # or (can do multi-line; return variable on the last line)
    condition1 = 0.5 < s
    condition2 = s < 0.75
    both_conditions = condition1 & condition2
    both_conditions
end

# %% 3.9, 3.10
quantile(samples, 0.8)
quantile(samples, (0.1, 0.9))

# %% 3.11
p_grid = range(0, 1, length = n)
prior = ones(n)
likelihood = @. pdf(Binomial(3, p_grid), 3)
posterior = likelihood .* prior
posterior ./= sum(posterior)
samples = sample(p_grid, pweights(posterior), 10_000)

# %% 3.12, 3.13
quantile(samples, (0.25, 0.75))

chn = Chains(samples)
hpd(chn; alpha = 0.5)

# %% 3.14, 3.16, TODO: not sure about 3.15
p_grid[argmax(posterior)]

mean(samples)
median(samples)

# %% 3.17
sum(posterior .* abs.(0.5 .- p_grid))

# %% 3.18, 3.19
loss = map(p_grid) do d
    sum(posterior .* abs.(d .- p_grid))
end
plot(loss)

p_grid[argmin(loss)]


using DataFrames
using CSV
using Plots
#%%
data_rk = open("rk_error.dat")
data_rk4 = open("rk4_error.dat")
data_eu = open("eu_error.dat")

df_rk=CSV.read(data_rk, header=false, delim=" ")
df_rk4=CSV.read(data_rk4, header=false, delim=" ")
df_eu=CSV.read(data_eu, header=false, delim=" ")
#%%
x_=4:2048
eu=x_.^-1
rk=x_.^-2
rk4=x_.^-4
#%%
plot(df_rk[1],df_rk[2], label="Runge-Kutta", color="black", dpi=600)
plot!(x_,rk, label="y=x^-2", color="black", line=(:dash))

plot!(df_rk4[1],df_rk4[2], label="RK 4th order", color="red")
plot!(x_,rk4, label="y=x^-4", color="red" , line=(:dash))

plot!(df_eu[1],df_eu[2], label="Euler", color="blue")
plot!(x_,eu, label="y=x^-1",color="blue", line=(:dash))

xaxis!(:log10, xlims=(4,2048))
yaxis!(:log10, ylims=(1e-15,Inf))
title!("error as a function of n_steps")
xlabel!("n_steps")
ylabel!("error")
plot!(legend=:bottom)
savefig("errors.png")


# Principal Component Analysis

"""
Linear Principal Component Analysis
"""
struct PCA{T<:Real} <: LinearDimensionalityReduction
    mean::AbstractVector{T}     # sample mean: of length d (mean can be empty, which indicates zero mean)
    proj::AbstractMatrix{T}     # projection matrix: of size d x p
    prinvars::AbstractVector{T} # principal variances: of length p
    tprinvar::T                 # total principal variance, i.e. sum(prinvars)
    tvar::T                     # total input variance
end

## constructor

function PCA(mean::AbstractVector{T}, proj::AbstractMatrix{T}, pvars::AbstractVector{T}, tvar::T) where {T<:Real}
    d, p = size(proj)
    (isempty(mean) || length(mean) == d) ||
        throw(DimensionMismatch("Dimensions of mean and projection matrix are inconsistent."))
    length(pvars) == p ||
        throw(DimensionMismatch("Dimensions of projection matrix and principal variables are inconsistent."))
    tpvar = sum(pvars)
    tpvar <= tvar || isapprox(tpvar,tvar) || throw(ArgumentError("principal variance cannot exceed total variance."))
    PCA(mean, proj, pvars, tpvar, tvar)
end

## properties
"""
    size(M)

Returns a tuple with the dimensions of input (the dimension of the observation space)
and output (the dimension of the principal subspace).
"""
size(M::PCA) = size(M.proj)

"""
    mean(M::PCA)

Returns the mean vector (of length `d`).
"""
mean(M::PCA) = fullmean(size(M.proj,1), M.mean)

"""
    projection(M::PCA)

Returns the projection matrix (of size `(d, p)`). Each column of the projection matrix corresponds to a principal component.
The principal components are arranged in descending order of the corresponding variances.
"""
projection(M::PCA) = M.proj

"""
    eigvecs(M::PCA)

Get the eigenvalues of the PCA model `M`.
"""
eigvecs(M::PCA) = projection(M)

"""
    principalvars(M::PCA)

Returns the variances of principal components.
"""
principalvars(M::PCA) = M.prinvars
principalvar(M::PCA, i::Int) = M.prinvars[i]

"""
    eigvals(M::PCA)

Get the eigenvalues of the PCA model `M`.
"""
eigvals(M::PCA) = principalvars(M)

"""
    tprincipalvar(M::PCA)

Returns the total variance of principal components, which is equal to `sum(principalvars(M))`.
"""
tprincipalvar(M::PCA) = M.tprinvar

"""
    tresidualvar(M::PCA)

Returns the total residual variance.
"""
tresidualvar(M::PCA) = M.tvar - M.tprinvar

"""
    var(M::PCA)

Returns the total observation variance, which is equal to `tprincipalvar(M) + tresidualvar(M)`.
"""
var(M::PCA) = M.tvar

"""
    r2(M::PCA)
    principalratio(M::PCA)

Returns the ratio of variance preserved in the principal subspace, which is equal to `tprincipalvar(M) / var(M)`.
"""
r2(M::PCA) = M.tprinvar / M.tvar
const principalratio = r2

"""
    loadings(M::PCA)

Returns model loadings, i.e. the weights for each original variable when calculating the principal component.
"""
loadings(M::PCA) = sqrt.(principalvars(M))' .* projection(M)

## use

"""
    predict(M::PCA, x::AbstractVecOrMat{<:Real})

Given a PCA model `M`, retur transform observations `x` into principal components space, as

\$\\mathbf{y} = \\mathbf{P}^T (\\mathbf{x} - \\boldsymbol{\\mu})\$

Here, `x` can be either a vector of length `d` or a matrix where each column is an observation,
and `\\mathbf{P}` is the projection matrix.
"""
predict(M::PCA, x::AbstractVecOrMat{T}) where {T<:Real} = transpose(M.proj) * centralize(x, M.mean)

"""
    reconstruct(M::PCA, y::AbstractVecOrMat{<:Real})

Given a PCA model `M`, returns a (approximately) reconstructed observations
from principal components space, as

\$\\tilde{\\mathbf{x}} = \\mathbf{P} \\mathbf{y} + \\boldsymbol{\\mu}\$

Here, `y` can be either a vector of length `p` or a matrix where each column
gives the principal components for an observation, and \$\\mathbf{P}\$ is the projection matrix.
"""
reconstruct(M::PCA, y::AbstractVecOrMat{T}) where {T<:Real} = decentralize(M.proj * y, M.mean)

## show & dump

function show(io::IO, M::PCA)
    idim, odim = size(M)
    print(io, "PCA(indim = $idim, outdim = $odim, principalratio = $(r2(M)))")
end

#### PCA Training

## auxiliary

const default_pca_pratio = 0.99

function check_pcaparams(d::Int, mean::AbstractVector, md::Int, pr::Real)
    isempty(mean) || length(mean) == d ||
        throw(DimensionMismatch("Incorrect length of mean."))
    md >= 1 || error("`maxoutdim` parameter must be a positive integer.")
    0.0 < pr <= 1.0 || throw(ArgumentError("principal ratio must be a positive real value ≤ 1.0."))
end

function choose_pcadim(v::AbstractVector{T}, ord::Vector{Int}, vsum::T, md::Int,
                       pr::Real) where {T<:Real}
    md = min(length(v), md)
    k = 1
    a = v[ord[1]]
    thres = vsum * convert(T, pr)
    while k < md && a < thres
        a += v[ord[k += 1]]
    end
    return k
end

## core algorithms
"""
    pcacov(C, mean; ...)

Compute and return a PCA model based on eigenvalue decomposition of a given covariance matrix `C`.

**Parameters:**
- `C`: The covariance matrix of the samples.
- `mean`: The mean vector of original samples, which can be a vector of length `d`,
           or an empty vector `Float64[]` indicating a zero mean.

*Note:* This function accepts two keyword arguments: `maxoutdim` and `pratio`.
"""
function pcacov(C::AbstractMatrix{T}, mean::AbstractVector{T};
                maxoutdim::Int=size(C,1),
                pratio::Real=default_pca_pratio) where {T<:Real}

    check_pcaparams(size(C,1), mean, maxoutdim, pratio)
    Eg = eigen(Symmetric(C))
    ev = Eg.values
    ord = sortperm(ev; rev=true)
    vsum = sum(ev)
    k = choose_pcadim(ev, ord, vsum, maxoutdim, pratio)
    v, P = extract_kv(Eg, ord, k)
    PCA(mean, P, v, vsum)
end

"""
    pcasvd(Z, mean, tw; ...)

Compute and return a PCA model based on singular value decomposition of a centralized sample matrix `Z`.

**Parameters:**
- `Z`: a matrix of centralized samples.
- `mean`: The mean vector of the **original** samples, which can be a vector of length `d`,
          or an empty vector `Float64[]` indicating a zero mean.
- `n`: a number of samples.

*Note:* This function accepts two keyword arguments: `maxoutdim` and `pratio`.
"""
function pcasvd(Z::AbstractMatrix{T}, mean::AbstractVector{T}, n::Real;
                maxoutdim::Int=min(size(Z)...),
                pratio::Real=default_pca_pratio) where {T<:Real}

    check_pcaparams(size(Z,1), mean, maxoutdim, pratio)
    Svd = svd(Z)
    v = Svd.S::Vector{T}
    U = Svd.U::Matrix{T}
    for i = 1:length(v)
        @inbounds v[i] = abs2(v[i]) / n
    end
    ord = sortperm(v; rev=true)
    vsum = sum(v)
    k = choose_pcadim(v, ord, vsum, maxoutdim, pratio)
    si = ord[1:k]
    PCA(mean, U[:,si], v[si], vsum)
end

## interface functions
"""
    fit(PCA, X; ...)

Perform PCA over the data given in a matrix `X`. Each column of `X` is an **observation**.

**Keyword arguments**

- `method`: The choice of methods:
    - `:auto`: use `:cov` when `d < n` or `:svd` otherwise (*default*).
    - `:cov`: based on covariance matrix decomposition.
    - `:svd`: based on SVD of the input data.
- `maxoutdim`: The output dimension, i.e. dimension of the transformed space (*min(d, nc-1)*)
- `pratio`: The ratio of variances preserved in the principal subspace (*0.99*)
- `mean`: The mean vector, which can be either of
    - `0`: the input data has already been centralized
    - `nothing`: this function will compute the mean (*default*)
    - a pre-computed mean vector

**Notes:**

- The output dimension `p` depends on both `maxoutdim` and `pratio`, as follows. Suppose
  the first `k` principal components preserve at least `pratio` of the total variance, while the
  first `k-1` preserves less than `pratio`, then the actual output dimension will be \$\\min(k, maxoutdim)\$.

- This function calls [`pcacov`](@ref) or [`pcasvd`](@ref) internally, depending on the choice of method.
"""
function fit(::Type{PCA}, X::AbstractMatrix{T};
             method::Symbol=:auto,
             maxoutdim::Int=size(X,1),
             pratio::Real=default_pca_pratio,
             mean=nothing) where {T<:Real}

    @assert !SparseArrays.issparse(X) "Use Kernel PCA for sparse arrays"

    d, n = size(X)

    # choose method
    if method == :auto
        method = d < n ? :cov : :svd
    end

    # process mean
    mv = preprocess_mean(X, mean)

    # delegate to core
    if method == :cov
        C = covm(X, isempty(mv) ? 0 : mv, 2)
        M = pcacov(C, mv; maxoutdim=maxoutdim, pratio=pratio)
    elseif method == :svd
        Z = centralize(X, mv)
        M = pcasvd(Z, mv, n; maxoutdim=maxoutdim, pratio=pratio)
    else
        throw(ArgumentError("Invalid method name $(method)"))
    end

    return M::PCA
end


struct Camera{T <: AbstractFloat}
	origin::Vec3{T}
	lower_left_corner::Vec3{T}
	horizontal::Vec3{T}
	vertical::Vec3{T}
	u::Vec3{T}
	v::Vec3{T}
	w::Vec3{T}
	lens_radius::T
end

"""
	Args:
		vfov: vertical field-of-view in degrees
		aspect_ratio: horizontal/vertical ratio of pixels
      aperture: if 0 - no depth-of-field
"""
function default_camera(lookfrom::Vec3{T}=(SA{T}[0,0,0]),
						lookat::Vec3{T}=(SA{T}[0,0,-1]),
						vup::Vec3{T}=(SA{T}[0,1,0]),
						vfov::T=T(90), aspect_ratio::T=T(16/9),
						aperture::T=T(0), focus_dist::T=T(1)) where T
	viewport_height = T(2) * tand(vfov/T(2))
	viewport_width = aspect_ratio * viewport_height

	w = normalize(lookfrom - lookat)
	u = normalize(vup × w)
	v = w × u

	origin = lookfrom
	horizontal = focus_dist * viewport_width * u
	vertical = focus_dist * viewport_height * v
	lower_left_corner = origin - horizontal/T(2) - vertical/T(2) - focus_dist*w
	lens_radius = aperture/T(2)
	Camera{T}(origin, lower_left_corner, horizontal, vertical, u, v, w, lens_radius)
end

default_camera(lookfrom, lookat, vup, vfov, aspect_ratio, aperture, focus_dist; elem_type::Type{T}) where T =
	default_camera(Vec3{T}(lookfrom), Vec3{T}(lookat), Vec3{T}(vup),
		T(vfov), T(aspect_ratio), T(aperture), T(focus_dist)
	)

@inline @fastmath function get_ray(c::Camera{T}, s::T, t::T) where T
	rd = SVector{2,T}(c.lens_radius * random_vec2_in_disk(T))
	offset = c.u * rd.x + c.v * rd.y #offset = c.u * rd.x + c.v * rd.y
    Ray(c.origin + offset, normalize(c.lower_left_corner + s*c.horizontal +
									 t*c.vertical - c.origin - offset))
end


#=
The arithmetic sequence, 1487, 4817, 8147, in which each of the terms increases by 3330, is unusual in two ways: (i) each of the three terms are prime, and, (ii) each of the 4-digit numbers are permutations of one another.

There are no arithmetic sequences made up of three 1-, 2-, or 3-digit primes, exhibiting this property, but there is one other 4-digit increasing sequence.

What 12-digit number do you form by concatenating the three terms in this sequence?
=#
include("projecteulerutils.jl")
using Primes

function calc()
  b = c = 0
  for a in 1000:10000-3330-3330
    if a == 1487
      continue
    end
    b = a+3330
    c = b+3330
    if isprime(a) && isprime(b) && isprime(c) && ispermutation(digits(a),digits(b)) && ispermutation(digits(b), digits(c))
      return "$a$b$c"
    end
  end
end
@time println(calc())


module TrajOpt

    using FixedSizeArrays
    using Polynomials

    export HermiteBasis, QuadratureRule, BCTypes
    export compute_interpolation_matrices
    export assemble_stiffness_matrix
    export apply_bc!

    immutable Block{T,N} <: MutableFixedMatrix{T,N,N}

    end

    """
        hb = HermiteBasis(degree=N)

    Return left and right Hermite interpolation polynomials of specified degree.

    # Description
    Hermite basis function are used to interpolate a function over the interval
    τ = [0, 1] when the value of the function and it's first k derviatives are
    known at endpoints of the interval. The basis functions are simple τ-poly-
    nomials with degree N = 2k-1. To compute the value of the interpolant at a
    point within the interval, the basis functions are used to construct a
    weighted sum of the function data as follows (note: this is pseudo code):

        x_left  = [ x(0.0), x'(0.0) ... ]
        x_right = [ x(1.0), x'(1.0) ... ]
        x(τ) = sum( h.left(τ).*x_left ) + sum( h.right(τ).*x_right )

    # Fields
    * left::Vector{Poly}   Basis functions for left-endpoint function data
    * right::Vector{Poly}  Basis functions for right-endpoint function data
    """
    immutable HermiteBasis

        left  :: Vector{Poly{Rational{Int64}}}
        right :: Vector{Poly{Rational{Int64}}}

        function HermiteBasis(; degree=3)
            if degree == 1
                left  = [ Poly([ 1,  -1 ]//1, :τ) ]
                right = [ Poly([ 0,   1 ]//1, :τ) ]
            elseif degree == 3
                left  = [
                    Poly([ 1,   0,  -3,   2 ]//1, :τ)
                    Poly([ 0,   1,  -2,   1 ]//1, :τ)
                ]
                right = [
                    Poly([ 0,   0,   3,  -2 ]//1, :τ)
                    Poly([ 0,   0,  -1,   1 ]//1, :τ)
                ]
            elseif degree == 5
                left  = [
                    Poly([ 1,   0,   0, -10,  15,  -6 ]//1, :τ)
                    Poly([ 0,   1,   0,  -6,   8,  -3 ]//1, :τ)
                    Poly([ 0,   0,   1,  -3,   3,  -1 ]//2, :τ)
                ]
                right = [
                    Poly([ 0,   0,   0,  10, -15,   6 ]//1, :τ)
                    Poly([ 0,   0,   0,  -4,   7,  -3 ]//1, :τ)
                    Poly([ 0,   0,   0,   1,  -2,   1 ]//2, :τ)
                ]
            else
                throw(ArgumentError("degree=$degree: must be 1, 3, or 5."))
            end
            return new(left, right)
        end

    end

    """
        qr = QuadratureRule(num_points=N)

    Returns the N-point Guassian quadrature rule.

    # Description
    Numerical quadrature approximates the integral of a continuous function
    by sampling the integrand at discrete points in the domiain of integration
    and then taking a weighted sum of the sample values. Guassian quadrature
    rules are optimal for a 1D domain of integration: an N-point rule can
    exactly integrate a polynomial of degree 2*N-1. This type returns the
    sample locations and weights for a Gaussian Quadrature Rule with a
    specified number of sample locations.

    # Fields
    * locations::Vector{Real}  Quad point locations on bi-unit interval
    * weights::Vector{Real}    Summation weights for function samples

    """
    immutable QuadratureRule

        locations :: Vector{Float64}
        weights   :: Vector{Float64}

        function QuadratureRule(; num_points=2)
            if num_points == 1
                locations = [  0.0 ]
                weights   = [  2.0 ]
            elseif num_points == 2
                locations = [ -1.0,  1.0 ] * sqrt(1/3)
                weights   = [  1.0,  1.0 ]
            elseif num_points == 3
                locations = [ -1.0,  0.0,  1.0 ] * sqrt(3/5)
                weights   = [  5.0,  8.0,  5.0 ] / 9.0
            else
                throw(ArgumentError(
                    "num_points=$num_points: must be integer between 1 and 3."
                ))
            end
            return new(locations, weights)
        end
    end

    function compute_interpolation_matrices(hb, qr, time_points)

        # Preliminaries
        n_interval = length(time_points)-1
        n_quad = length(qr.locations)
        n_dof = length(hb.left)

        # Allocate interpolation matrices
        zero_block = MutableFixedMatrix{n_dof, n_quad, Float64}(0.0)
        HL = repmat([zero_block], n_interval)
        HR = repmat([zero_block], n_interval)

        # Map quadrature locations to unit interval
        τ = (qr.locations + 1)/2  # Maps [-1,1] => [0,1]
        τ_scaling = 1/2

        # Compute unit-interval interp matrices
        HL_unit = τ_scaling * polyval.( hb.left,  τ' )
        HR_unit = τ_scaling * polyval.( hb.right, τ' )

        # Apply interval scaling to interp matrices
        Δt = diff(time_points)
        scale_factors(Δt) = [ Δt^(q+1) for q = 0:n_dof-1 ]
        for i = 1:n_interval
            S = scale_factors(Δt[i]);
            HL[i] = S .* HL_unit
            HR[i] = S .* HR_unit
        end

        return (HL, HR)

    end

    """
        K = assemble_stiffness_matrix(hb, time)

    Compute stiffness matrix for time integration of 2nd order ODEs.

    # Description
    The Continuous Galerkin method can be used to discretize and solve systems
    of differential equations of the form ``\ddot{x} = f(t,x,\dot{x})``. The
    discretization of this system using Hermite basis functions results in a
    system of simultaneous non-linear equations ``Kx + f(x) = 0``, where ``K``
    is the stiffness matrix. This matrix is a block tri-diagonal matrix that
    is independent of the solution, x, and depends only on the distribution of
    time points used to discretize the solution domain. See the Jupyter note-
    book 'Theory.ipynb' for details.

    # Inputs
    * hb::HermiteBasis  Basis functions used to represent solution
    * time::AbstractVector{Float64}  Time points discretizing solution domain

    # Outputs
    * K::Tridiagonal{FixedSizeArrays.Mat}  Block tridiagonal stiffness matrix
    """
    function assemble_stiffness_matrix(hb, time_points)

        # Preliminaries
        n_point = length(time_points)
        n_interval = n_point-1
        n_dof = length(hb.left)

        # Error checking
        @assert n_dof>1   "Must use HermiteBasis with order >= 3"
        @assert n_point>2 "Must be at least 3 time points"

        # Allocate stiffness matrix
        zero_block = MutableFixedMatrix{Float64, n_dof, n_dof}(0.0)
        K = Tridiagonal(
            repmat( [zero_block], n_point-1 ), # Lower Diagonal
            repmat( [zero_block], n_point   ), # Main  Diagonal
            repmat( [zero_block], n_point-1 ), # Upper Diagonal
        )

        # Compute unit-interval stiffness matrices
        dhb_left  = polyder(hb.left)
        dhb_right = polyder(hb.right)
        integrate(x) = polyval( polyint(x), 1.0 )
        KLL = [ integrate(bi*bj) for bi=dhb_left,  bj=dhb_left  ]
        KLR = [ integrate(bi*bj) for bi=dhb_left,  bj=dhb_right ]
        KRL = [ integrate(bi*bj) for bi=dhb_right, bj=dhb_left  ]
        KRR = [ integrate(bi*bj) for bi=dhb_right, bj=dhb_right ]

        # Assemble global matrix from scaled unit-interval matrices
        Δt = diff(time_points)
        scale_factors(Δt) = [ Δt^-(p*q+1) for p=0:n_dof-1, q=0:n_dof-1 ]
        for i = 1:n_interval
            S = scale_factors(Δt[i])
            K[ i , i ] += S.*KLL
            K[ i ,i+1] += S.*KLR
            K[i+1, i ] += S.*KRL
            K[i+1,i+1] += S.*KRR
        end

        # Add boundary operators
        BL = [ polyval(bi*bj, 0.0) for bi=hb.left,  bj=dhb_left  ]
        BR = [ polyval(bi*bj, 1.0) for bi=hb.right, bj=dhb_right ]
        K[ 1 , 1 ] += scale_factors(Δt[ 1 ]) .* BL
        K[end,end] -= scale_factors(Δt[end]) .* BR

        return K

    end

    """
        F = assemble_load_vector(HL, HR, accel)

    Constructs the discrete load vector (RHS) for a uniform acceleration.

    # Description
    This function is a specicialized version of the assemble_load_vector that
    only works for constant accelerations that are independent of the trajectory
    solution (e.g. a constant gravitational acceleration). This is mainly used
    to verify correctness and accuracy of the Continuous Galerkin stiffness
    matrix and the associated boundary conditions.

    # Inputs
    HL::Vector{Mat}  Left-hand Hermite interpolation matrices
    HR::Vector{Mat}  Right-hand Hermite interpolation matrices
    accel::Float64   Constant acceleration applied to trajectory

    # Outputs
    F::Vector{Vec}   Load vector for discrete system of equations
    """
    function assemble_load_vector(HL, HR, weights, accel)

        # Preliminaries
        n_interval = length(HL)
        n_point = n_interval+1
        n_dof = size(HL[1],1)

        # Error checking
        @assert n_dof>1   "Must use HermiteBasis with order >= 3"
        @assert n_point>2 "Must be at least 3 time points"

        # Allocate load vector
        F = repmat(MutableFixedVector{Float64, n_dof}(0.0), n_point)

        # Evaluate load integrals
        aw = accel * weights
        for i = 1:n_interval
            F[ i ] += HL[i] * aw
            F[i+1] += HR[i] * aw
        end

    end

    """
    Defines named constants for different boundary condition types
    """
    module BCTypes
        @enum BCType dirichlet=1 neumann=2
    end

    """
        apply_bc!(K, bc, i)

    Modifies stiffness matrix to enforce Dirichlet/Neumman boundary conditions.

    # Description:
    Modifies stiffness matrix to enforce Dirichlet/Neumman boundary conditions
    at a specified time-point in the solution. Note that this doesn't actually
    define what the *value* at the boundary will be; the value is specified by
    calling apply_bc! with the load vector as the first argument, e.g.

        apply_bc!(f, bc, i, value)

    This version of apply_bc! simply modifies the stiffness matrix so that the
    required degree of freedom is set equal to the value specified in the load
    vector.
    """
    function apply_bc!{T <: MutableFixedMatrix}(K::Tridiagonal{T}, bc, i)
        # How this works:
        # To embed a boundary condition in the K matrix, we need to replace the
        # existing discrete equation for the "p"-th degree of freedom such that
        #   (K*x)[p] = x[p] = f[p]
        # Since K block tri-diagonal, this boils down to zero-ing out the
        # "q"-th row in of the three blocks coupled to the i-th node and then
        # setting the diagonal element to unity.
        #
        # Since we are using Hermite shape functions, the value of "q" depends
        # on the type of BC. If we want a Dirichlet BC we modify the 1st nodal
        # degree of freedom; if we want a Neumann BC, we modifiy the 2nd nodal
        # degree of freedom. This mapping from BC type to row index is codified
        # using the BCType enumeration.
        N = size(K,2)
        ibc = Int(bc)
        K[i,i][ibc,:]  = 0.0
        K[i,i][ibc,ibc] = 1.0
        if i > 1;  K[i,i-1][ibc,:] = 0.0;  end
        if i < N;  K[i,i+1][ibc,:] = 0.0;  end
        return nothing
    end

    """
        apply_bc!(F, bc, i, value)

    Modifies load vector to apply a Dirichlet/Neumman boundary conditions.

    # Description:
    Modifies assembled load vector to apply a Dirichlet or Neumman boundary
    conditions at a specified time-point in the solution. Note that for this
    to be effective, the stiffness matrix must also be modified at the specified
    node to accept a boundary condition. This is achieved by calling apply_bc!
    with the stiffness matrix as the first argument.
    """
    function apply_bc!{T <: MutableFixedVector}(F::Vector{T}, bc, i, value)
        F[i][Int(bc)] = value
        return nothing
    end

end


using Distributions
using DataFrames
using Random

"""
    generateFormants(nTokens; [cats=["iy", "aa", "uw"], gender=["w", "m"]])

Generate synthetic formants from multivariate normal distributions based on observations from  Hillenbrand et al. (1995, Acoustic characteristics of American English vowels, DOI: 10.1121/1.411872). Currently supports generating vowels for /i/ as `"iy"`, /ɑ/ as `"aa"`, and /u/ as `"uw"`, using values for men as `"m"` and women as `"w"`. One observation was dropped from the women /ɑ/ token because the F2 value could not be measured when Hillenbrand et al. collected the data. Values in the mean vectors and covariance matrices were rounded to two decimal places.

Args
======

* `nTokens` The number of tokens to generate for each category and gender pairing
* `cats` (keyword argument) A vector of vowel categories to generate tokens for
* `gender` (keyword argument) A vector of gender categories to generate tokens for
* `seed` A seed value for a `MersenneTwister` random number generator; allows for reproducible results; using the default value of `nothing` will use the system-generated random seed.
* `rng` An `AbstractRNG` object to use for random number generation; if the default value of `nothing` is used, a `MersenneTwister` object will be created
"""
function generateFormants(nTokens; cats=["iy", "aa", "uw"], gender=["m", "w"], seed=nothing, rng::T=nothing) where T <: Union{AbstractRNG, Nothing}

  if isnothing(seed) && isnothing(rng)
    rng = MersenneTwister()
  elseif isnothing(rng)
    rng = MersenneTwister(seed)
  end

  mappings = Dict()

  mappings[("iy", "w")] = MultivariateNormal([437.25, 2761.31], [1650.06 1277.86; 1277.86 21738.56])
  mappings[("aa", "w")] = MultivariateNormal([916.36, 1525.83], [8449.85 4354.50; 4354.50 15615.80])
  mappings[("uw", "w")] = MultivariateNormal([459.67, 1105.52], [1496.06 -417.93; -417.93 42130.34])

  mappings[("iy", "m")] = MultivariateNormal([342.69, 2322.78], [796.99 493.36; 493.36 18580.77])
  mappings[("aa", "m")] = MultivariateNormal([756.49, 1308.93], [3879.39 1751.56; 1751.56 12324.24])
  mappings[("uw", "m")] = MultivariateNormal([379.67, 992.24], [1106.32 1482.52; 1482.52 12833.73])

  generated = []
  vowel = []
  gend = []

  for c in cats
    for g in gender

      if ! haskey(mappings, (c, g))
        error("Unsupported vowel and gender combination: ($c, $g)")
      end

      dist = mappings[(c, g)]
      data = rand(rng, dist, nTokens)
      push!(generated, data)
      vowel = [vowel; repeat([c], nTokens)]
      gend = [gend; repeat([g], nTokens)]
    end
  end

  gen = reduce(hcat, generated)
  return DataFrame(f1=gen[1,:], f2=gen[2,:], vowel=vowel, gender=gend)
end


using Gen, Genify
using Distributions: Normal

##  Define model of noisy weighing scale in Julia

function scale(n::Int)
    mass = rand(Normal(5, 1))
    obs = measure(mass, n)
end

function measure(mass::Real, n::Int)
    obs = Float64[]
    for i in 1:n
        m = rand(Normal(mass, 2))
        push!(obs, m)
    end
    return obs
end

## Genify the model

genscale = genify(scale, Int)

## Construct choicemap of noisy measurements

observations = choicemap(((:obs => :m => i, 4) for i in 1:10)...)

## Perform importance sampling to estimate the true mass

trs, ws, _ = importance_sampling(genscale, (10,), observations, 100)
mass_est = sum(tr[:mass] * exp(w) for (tr, w) in zip(trs, ws))


"""
    F = JopTanh(spc)

where `F` is the hyperbolic tangent operator with domain and range given by `spc::JetSpace`.
we use 'tanh'(c*x) = (\exp{c*x}-\exp{-c*x})/(\exp{c*x}+\exp{-c*x}),
the derivative of which is 1-tanh(c*x)^2.
We expect the domain and range to be real.
"""
JopTanh(spc::JetSpace{T}, c = 1) where {T} = JopNl(dom = spc, rng = spc, f! = JopTanh_f!, df! = JopTanh_df!, df′! = JopTanh_df′!, s = (c=T(c),))
export JopTanh

JopTanh_f!(d::AbstractArray, m::AbstractArray; c) = d .= tanh.(c .* m)

JopTanh_df!(δd::AbstractArray, δm::AbstractArray; mₒ, c) = δd .= c .* (1 .- tanh.(c .* mₒ).^2) .* δm

JopTanh_df′!(δd::AbstractArray, δm::AbstractArray; mₒ, c) = δd .= conj.(c .* (1 .- tanh.(c .* mₒ).^2)) .* δm


#=
We can determine how "out of order" an array A is by counting the number of inversions it has. Two elements A[i] and A[j] form an inversion if A[i] > A[j] but i < j. That is, a smaller element appears after a larger element.

Given an array, count the number of inversions it has. Do this faster than O(N^2) time.

You may assume each element in the array is distinct.

For example, a sorted list has zero inversions. The array [2, 4, 1, 3, 5] has three inversions: (2, 1), (4, 1), and (4, 3). The array [5, 4, 3, 2, 1] has ten inversions: every distinct pair forms an inversion.
=#

function inversions(A::Array{Int, 1})
    B = copy(A)
    number_of_inversions = 0
    while length(B) != 1
        if findmin(B)[2] != 1
            number_of_inversions = number_of_inversions + sum(B[1] .> B[2:end])
        end
        B = B[2:end]
    end
    return number_of_inversions
end


immutable Rayleigh <: ContinuousUnivariateDistribution
    σ::Float64

    function Rayleigh(σ::Real)
        σ > zero(σ) || error("Rayleigh: σ must be positive")
        @compat new(Float64(σ))
    end

    Rayleigh() = new(1.0)
end

@distr_support Rayleigh 0.0 Inf

#### Parameters

scale(d::Rayleigh) = d.σ
params(d::Rayleigh) = (d.σ,)

#### Statistics

mean(d::Rayleigh) = sqrthalfπ * d.σ
median(d::Rayleigh) = 1.177410022515474691 * d.σ   # sqrt(log(4.0)) = 1.177410022515474691
mode(d::Rayleigh) = d.σ

var(d::Rayleigh) = 0.429203673205103381 * d.σ^2   # (2.0 - π / 2) = 0.429203673205103381
std(d::Rayleigh) = 0.655136377562033553 * d.σ

skewness(d::Rayleigh) = 0.631110657818937138
kurtosis(d::Rayleigh) = 0.245089300687638063

entropy(d::Rayleigh) = 0.942034242170793776 + log(d.σ)

#### Evaluation

function pdf(d::Rayleigh, x::Float64)
	σ2 = d.σ^2
	x > 0.0 ? (x / σ2) * exp(- (x^2) / (2.0 * σ2)) : 0.0
end

function logpdf(d::Rayleigh, x::Float64)
	σ2 = d.σ^2
	x > 0.0 ? log(x / σ2) - (x^2) / (2.0 * σ2) : -Inf
end

logccdf(d::Rayleigh, x::Float64) = - (x^2) / (2.0 * d.σ^2)
ccdf(d::Rayleigh, x::Float64) = exp(logccdf(d, x))

cdf(d::Rayleigh, x::Float64) = 1.0 - ccdf(d, x)
logcdf(d::Rayleigh, x::Float64) = log1mexp(logccdf(d, x))

quantile(d::Rayleigh, p::Float64) = sqrt(-2.0 * d.σ^2 * log1p(-p))

#### Sampling

rand(d::Rayleigh) = d.σ * sqrt(2.0 * randexp())


println("\n\tEstimate autocorrelations");
println("\n\t...for N1...");
xcorr = [0:10:400];
ycorr1 = zeros( length( xcorr ), 5 );
for a in [1:5]
  # Calculate autocorrelations
  ycorr1[:,a] = autocorrelation(N1, v1, a, xcorr);
  # Normalize by dividing through by Ĉ[1,a]
  ycorr1[:,a] /= ycorr1[1,a];
end
println("\n\t...for N2 (See plots below)");
ycorr2 = zeros( length( xcorr ), 5 );
for a in [1:5]
  # Calculate autocorrelations
  ycorr2[:,a] = autocorrelation(N2, v2, a, xcorr);
  # Normalize by dividing through by Ĉ[1,a]
  ycorr2[:,a] /= ycorr2[1,a];
end

# Make plots for each alpha
println("\n\tPlacing estimated autocorrelation plots 'plots1' and 'plots2':");
using Gadfly
plots1 = Plot[];
plots2 = Plot[];
for i in [1:5]
  # Plot
  push!(plots1, Gadfly.plot ( x = xcorr, y = ycorr1[:,i], Geom.line, Geom.point,
      Guide.xlabel("Separation"), Guide.ylabel("Normalized Autocorrelation"),
      Guide.title(join(["Estimated Autocorrelation for Dataset",string(i),
                        "with Sample Size N1 = $N1"]," "," "))))
  push!(plots2, Gadfly.plot ( x = xcorr, y = ycorr2[:,i], Geom.line, Geom.point,
      Guide.xlabel("Separation"), Guide.ylabel("Normalized Autocorrelation"),
      Guide.title(join(["Estimated Autocorrelation for Dataset",string(i),
                        "with Sample Size N2 = $N2"]," "," "))))
end

# Estimate autocorrelation times
println("\n\tEstimating autocorrelation times τ1 and τ2");
τ1 = Float64[];
τ2 = Float64[];
for a in [1:5]
  corra = autocorrelation(N1, v1, a, ns);
  corra /= corra[1];                          # Normalize
  push!(τ1,sum(corra) - 0.5);                 # Get autocorrelation time
end
for a in [1:5]
  corra = autocorrelation(N2, v2, a, ns);
  corra /= corra[1];                          # Normalize
  push!(τ2,sum(corra) - 0.5);                 # Get autocorrelation time
end

println("\n\tEstimated Autocorrelation Times for N1 = $N1:\n");
for i in [1:5]
  println("\t\tτ̂1v$i:\t$(τ1[i])");
end
println("\n\tEstimated Autocorrelation Times for N2 = $N2:\n");
for i in [1:5]
  println("\t\tτ̂2v$i:\t$(τ2[i])");
end
println("\n\tThe estimated autocorrelation times are clearly far better for the larger sample.");


"""
## sample!

This subroutine returns the local coordinates and weighting coefficients
of the integrating points.

### Function
```julia
sample!(fin_el, s, wt)
```

### Arguments
```julia
* fin_el::FiniteElement      : Finite element type
* s::Matrix{Float64}        : Local coordinates (Updated)
* wt::Vector{Float64}       : Weighting coefficients (Updated)
```
"""
function sample!(fin_el::Line, s::Matrix{Float64} , wt::Vector{Float64})
  #
  # This subroutine returns the local coordinates and weighting coefficients
  # of the integrating points.
  #
  nip = size(s,1)
  (s[:,1], wt[:]) = QuadGK.gauss(Float64, nip)
end


using Distributions
using Base.Test

d = Kolmogorov()
# compare to Smirnov's tabulated values
@test round(cdf(d,0.28),6) == .000_001
@test round(cdf(d,0.50),6) == .036_055
@test round(cdf(d,0.99),6) == .719_126
@test round(cdf(d,1.00),6) == .730_000
@test round(cdf(d,1.01),6) == .740_566
# @test round(cdf(d,1.04),6) == .770_434 # table value appears to be wrong
@test round(cdf(d,1.50),6) == .977_782
@test round(cdf(d,2.00),6) == .999_329
@test round(cdf(d,2.50),7) == .999_992_5
@test round(cdf(d,3.00),8) == .999_999_97


# Soft thresholding function
@inline soft_threshold(x, ϵ) = max(x - ϵ, 0) + min(x + ϵ, 0)

# RPCA using the inexact Augumented Lagrange Multiplier (ALM).
# Given a observation matrix D, find row-rank matrix A and sparse matrix E
# so that D = A + E.
function inexact_alm_rpca(D::AbstractMatrix;
                          sparseness::Float64=1.0/sqrt(maximum(size(D))),
                          max_iter::Int=1000,
                          error_tol::Float64=1.0e-7,
                          ρ::Float64=1.5,
                          verbose::Bool=false,
                          nonnegativeA::Bool=false,
                          nonnegativeE::Bool=false)
    M, N = size(D)
    λ = sparseness

    A⁰, E⁰ = zeros(M, N), zeros(M, N)

    # initialize
    Y⁰ = copy(D)
    norm² = svdvals(Y⁰)[1] # can be tuned
    norm∞ = norm(vec(Y⁰), Inf) / λ
    dual_norm = max(norm², norm∞)
    d_norm = norm(D)
    Y⁰ /= dual_norm

    μ⁰ = 1.25 / norm²
    μ̄ = μ⁰ * 1.0e+7

    converged::Bool = false
    k = 0
    sv⁰ = 10

    Yᵏ, Aᵏ, Eᵏ, μᵏ, svᵏ = Y⁰, A⁰, E⁰, μ⁰, sv⁰
    while !converged
        # update sparse matrix E
        Eᵏ = soft_threshold(D - Aᵏ + 1/μᵏ * Yᵏ, λ * 1/μᵏ)
        # force non-negative (heuristic)
        if nonnegativeE
            Eᵏ = max(Eᵏ, 0)
        end

        U, S, V = svd(D - Eᵏ + μᵏ^-1 * Yᵏ)

        # trancate dimention
        svpᵏ = trunc(Int, sum(S .> μᵏ^-1))
        if svpᵏ < svᵏ
            svᵏ = min(svpᵏ + 1, N)
        else
            svᵏ = min(svpᵏ + round(0.05 * N), N)
        end

        # update row-rank matrix A
        Aᵏ = U[:,1:svpᵏ] * diagm(S[1:svpᵏ] - 1/μᵏ) * V[:,1:svpᵏ]'
        # force non-negative (heuristic)
        if nonnegativeA
            Aᵏ = max(Aᵏ, 0)
        end

        Z = D - Aᵏ - Eᵏ

        Yᵏ = Yᵏ + μᵏ * Z
        μᵏ = min(μᵏ * ρ, μ̄)

        objective = norm(Z) / d_norm
        verbose && println("#$(k) objective: $(objective)")

        if objective < error_tol
            if verbose
                println("converged")
            end
            converged = true
        end

        k = k + 1
        k >= max_iter && break
    end

    Aᵏ, Eᵏ
end


function η_karnik(u::Matrix{FuzzyNumber}, d2::Matrix{FuzzyNumber}; m::Real=2.0)
	m > 1 || error("fuzzifier m ∈ (1, ∞), got $m")
	levels = u[1, 1].levels
	num_levels = length(levels)
	N, c = size(u)

	η = Vector{FuzzyNumber}(undef, c)
	for i = 1:c
		grades = Vector{Interval}(undef, num_levels)
		for (lvl, α) in enumerate(levels)
			u_cut = cut(FuzzyVector(u[:, i]), α)
			d2_cut = cut(FuzzyVector(d2[:, i]), α)
			c_left = km_iwa(d2_cut, u_cut; bound="lower", m=m)
			c_right = km_iwa(d2_cut, u_cut; bound="upper", m=m)
			grades[lvl] = Interval(c_left, c_right)
		end
		η[i] = FuzzyNumber(levels, grades)
	end
	η
end

function u_lpcm(X⃗::FuzzyVector, C⃗::FuzzyVector, η::Real; m::Real=2.0)
	m > 1 || error("fuzzifier m ∈ (1, ∞), got $m")
	η > 0 || error("η must be suitable positive number")
	levels = X⃗[1].levels
	num_levels = length(levels)

	grades = Vector{Interval}(undef, num_levels)
	D2 = FuzzySets.d_interval(X⃗, C⃗, squared=true)
	for (lvl, α) in enumerate(levels)
		D2_cut = cut(D2, α)
		grades[lvl] = u_lpcm(D2_cut, η; m=m)
	end
	u = FuzzyNumber(levels, grades)
	u
end

function u_lpcm(d2::Interval, η::Real; m::Real=2.0)
	m > 1 || error("fuzzifier m ∈ (1, ∞)")
	η > 0 || error("η must be suitable positive number")
	power = 1 / (m - 1)

	u = Interval(1) / (Interval(1) + (d2 / η)^power)
	u
end


"""
    hahn(x, p::Integer, α, β, N::Integer)

Evaluate the Hahn polynomial with parameters `α`, `β`, `N` of degree `p` at `x`
using the three term recursion [Öffner, Zweidimensionale klassische und diskrete
orthogonale Polynome, Chapter 5].
"""
function hahn(x, p::Integer, α, β, N)
    T = typeof( (α+β+2)*x/(N*(α+1)) )
    a = one(T)
    b = 1 - (α+β+2)*x/(N*(α+1))

    if p <= 0
        return a
    elseif p == 1
        return b
    elseif typeof(N) <: Integer && p > N
        throw(ArgumentError("p==$p must not be larger than N==$N."))
    end

    for n in 2:p
        An = T( (n+α+β)*(n+α)*(N-n+1) ) / ( (2n-1+α+β)*(2n+α+β) )
        Cn = T( (n-1)*(n+α+β+N)*(n-1+β) ) / ( (2n-2+α+β)*(2n-1+α+β) )
        a, b = b, ( (An+Cn-x)*b - Cn*a ) / An
    end

    b
end


"""
    Logistic(μ,θ)

The *Logistic distribution* with location `μ` and scale `θ` has probability density function

```math
f(x; \\mu, \\theta) = \\frac{1}{4 \\theta} \\mathrm{sech}^2
\\left( \\frac{x - \\mu}{2 \\theta} \\right)
```

```julia
Logistic()       # Logistic distribution with zero location and unit scale, i.e. Logistic(0, 1)
Logistic(u)      # Logistic distribution with location u and unit scale, i.e. Logistic(u, 1)
Logistic(u, b)   # Logistic distribution with location u ans scale b

params(d)       # Get the parameters, i.e. (u, b)
location(d)     # Get the location parameter, i.e. u
scale(d)        # Get the scale parameter, i.e. b
```

External links

* [Logistic distribution on Wikipedia](http://en.wikipedia.org/wiki/Logistic_distribution)

"""
struct Logistic{T<:Real} <: ContinuousUnivariateDistribution
    μ::T
    θ::T
    Logistic{T}(µ::T, θ::T) where {T} = new{T}(µ, θ)
end

function Logistic(μ::T, θ::T; check_args=true) where {T <: Real}
    check_args && @check_args(Logistic, θ > zero(θ))
    return Logistic{T}(μ, θ)
end

Logistic(μ::Real, θ::Real) = Logistic(promote(μ, θ)...)
Logistic(μ::Integer, θ::Integer) = Logistic(float(μ), float(θ))
Logistic(μ::T) where {T <: Real} = Logistic(μ, one(T))
Logistic() = Logistic(0.0, 1.0, check_args=false)

@distr_support Logistic -Inf Inf

#### Conversions
function convert(::Type{Logistic{T}}, μ::S, θ::S) where {T <: Real, S <: Real}
    Logistic(T(μ), T(θ))
end
function convert(::Type{Logistic{T}}, d::Logistic{S}) where {T <: Real, S <: Real}
    Logistic(T(d.μ), T(d.θ), check_args=false)
end

#### Parameters

location(d::Logistic) = d.μ
scale(d::Logistic) = d.θ

params(d::Logistic) = (d.μ, d.θ)
@inline partype(d::Logistic{T}) where {T<:Real} = T

#### Statistics

mean(d::Logistic) = d.μ
median(d::Logistic) = d.μ
mode(d::Logistic) = d.μ

std(d::Logistic) = π * d.θ / sqrt3
var(d::Logistic) = (π * d.θ)^2 / 3
skewness(d::Logistic{T}) where {T<:Real} = zero(T)
kurtosis(d::Logistic{T}) where {T<:Real} = T(6)/5

entropy(d::Logistic) = log(d.θ) + 2

#### Evaluation

zval(d::Logistic, x::Real) = (x - d.μ) / d.θ
xval(d::Logistic, z::Real) = d.μ + z * d.θ

pdf(d::Logistic, x::Real) = (lz = logistic(-abs(zval(d, x))); lz*(1-lz)/d.θ)
logpdf(d::Logistic, x::Real) = (u = -abs(zval(d, x)); u - 2*log1pexp(u) - log(d.θ))

cdf(d::Logistic, x::Real) = logistic(zval(d, x))
ccdf(d::Logistic, x::Real) = logistic(-zval(d, x))
logcdf(d::Logistic, x::Real) = -log1pexp(-zval(d, x))
logccdf(d::Logistic, x::Real) = -log1pexp(zval(d, x))

quantile(d::Logistic, p::Real) = xval(d, logit(p))
cquantile(d::Logistic, p::Real) = xval(d, -logit(p))
invlogcdf(d::Logistic, lp::Real) = xval(d, -logexpm1(-lp))
invlogccdf(d::Logistic, lp::Real) = xval(d, logexpm1(-lp))

function gradlogpdf(d::Logistic, x::Real)
    e = exp(-zval(d, x))
    ((2e) / (1 + e) - 1) / d.θ
end

mgf(d::Logistic, t::Real) = exp(t * d.μ) / sinc(d.θ * t)

function cf(d::Logistic, t::Real)
    a = (π * t) * d.θ
    a == zero(a) ? complex(one(a)) : cis(t * d.μ) * (a / sinh(a))
end


"""
    function dwz_ranking(
        players::Vector{Player},
        ranking_scheme_to_update::String,
        last_tournament_id::String
    )

Update the players rankings using the Deutsche Wertungszahl Algorithm

Arguments
=========
- `players`:
    an array of players in the simulation.
- `ranking_scheme_to_update`:
    the name of the ranking scheme to update for each Player.
- `last_tournament_id`:
    the last tournament that was completed.
"""

function dwz_ranking(
    players::Vector{Player},
    ranking_scheme_to_update::String,
    last_tournament_id::String
)
    updatedRankings = Dict{Int64, Int64}();
    for player in players
        if haskey(player.doubles_results, last_tournament_id)
            player_ranking = player.rankings["Dwz"]
            teammate_ranking = search_dwz_ranking(player.doubles_results.partner_player_id, players)
            achieved_points = 0
            expected_points = 0
            n = length(player.doubles_results[last_tournament_id])
            for match in player.doubles_results[last_tournament_id].matches
                zA = (player_ranking + teammate_ranking) / 2
                zG = search_dwz_ranking(match.opponent1_player_id, players) + search_dwz_ranking(match.opponent2_player_id, players)
                wE = 1/(1+(10^((zG-zA)/400)))
                wA = match.result == "Won" ? 1 : 0
                achieved_points += wA
                expected_points += wE
            end
            if player.age < 20
                J = 5
            elseif player.age <= 25
                J = 10
            else
                J = 15
            end
            E0 = (player_ranking/1000)^4 + J
            a = player.age <= 20 && achieved_points > expected_points ? player_ranking/2000 : 1
            B = player_ranking < 1300 && achieved_points <= expected_points ? exp((1300-player_ranking)/150)-1 : 0
            E = a * E0 + B
            Zn = player_ranking + (800/(E+n))(achieved_points-expected_points)
            updatedRankings[player.id] = Zn
        else
            # If player didn't participate in tournament, their ranking doesn't change
            updatedRankings[player.id] = player.rankings["Dwz"];
        end
    end
    map(player -> player.rankings[ranking_scheme_to_update] = updatedRankings[player.id], players)
    return nothing
end

"""
    function search_dwz_ranking(
        player_id::Int64,
        players::Vector{Player}
    )::Float64

Search for a player's current Dwz Ranking

Arguments
=========
- `player_id`:
    the player whose ranking is under search.
- `players`:
    an array of players in the simulation.
"""

function search_dwz_ranking(
    player_id::Int64,
    players::Vector{Player}
)::Float64
    player_ranking = filter(p -> p.id == player_id, players)[1].rankings["Dwz"]
    return player_ranking
end


using FFTW
import LinearAlgebra: svd, diag, diagm, eigen
import QuantumInfo: eye

function hilbert(signal)
	# construct the Hilbert transform of the signal via the FFT
	# in essense, we just want to set negative frequency components to zero
	spectrum = fft(signal)
	n = length(signal)
	midpoint = ceil(Int, n/2)

	kernel = zeros(n)
	kernel[1] = 1
	if iseven(n)
		kernel[midpoint + 1] = 1
	end
	kernel[2:midpoint] .= 2
	ifft(kernel .* spectrum)
end

function KT_estimation(data, timeStep, order)
	#Perform a modified KT estimation to obtain the estimates of the parameters
	#from a set of data.
	#
	# function [freqs, Tcs, amps] = KT_estimation(data, timeStep, order)
	#
	# See ? Van Huffel, S. (1993). Enhanced resolution based on minimum variance estimation and exponential data modeling.
	#       Signal Processing, 33(3), 333-355. doi:10.1016/0165-1684(93)90130-3

	analyticSig = hilbert(data)

	#Create the raw Hankel matrix
	N = length(analyticSig)
	K = order
	M = trunc(Integer, N/2)-1
	L = N-M+1
	H = zeros(ComplexF64, L, M)
	for ct = 1:M
	    H[:,ct] = analyticSig[ct:ct+L-1]
	end

	#Try and seperate the signal and noise subspace via the svd
	U,S,V = svd(H)

	#Reconstruct the approximate Hankel matrix with the first K singular values
	#Here we can iterate and modify the singular values
	S_k = diagm(0=>S[1:K])
	#Estimate the variance from the rest of the singular values
	varEst = (1/((M-K)*L)) * sum(S[K+1:end].^2)
	Sfilt = (S_k.^2 - L*varEst*eye(K)) / S_k
	Hbar = U[:,1:K] * Sfilt * V[:,1:K]'

	#Reconstruct the data from the averaged anti-diagonals
	cleanedData = zeros(ComplexF64, N)
	tmpMat = reverse(Hbar,dims=2)
	idx = -L+1
	for ct = N:-1:1
	    cleanedData[ct] = mean(diag(tmpMat,idx))
	    idx += 1
	end

	#Create a cleaned Hankel matrix
	cleanedH = similar(H)
	cleanedAnalyticSig = hilbert(cleanedData)
	for ct = 1:M
	    cleanedH[:,ct] = cleanedAnalyticSig[ct:ct+L-1]
	end

	#Compute Q with total least squares
	#U_K1*Q = U_K2
	U,_,_ = svd(cleanedH)
	U_K = U[:,1:K]
	tmpMat = [U_K[1:end-1,:] U_K[2:end,:]]
	_,_,V = svd(tmpMat)
	n = size(U_K,2)
	V_AB = V[1:n,1+n:end]
	V_BB = V[1+n:end,1+n:end]
	Q = -V_AB/V_BB

	#Now poles are eigenvalues of Q
	poles, _ = eigen(Q)

	#Take the log and return the decay constant and frequency
	freqs = zeros(K)
	Tcs = zeros(K)
	for ct = 1:K
	    sk = log(poles[ct])
	    freqs[ct] = imag(sk)/2/pi/timeStep
	    Tcs[ct] = -1/real(sk)*timeStep
	end

	#Refit the data to get the amplitude
	A = zeros(ComplexF64, N, K)
	for ct = 1:K
	    A[:,ct] = poles[ct].^(0:N-1)
	end

	amps = A\cleanedData

	return freqs, Tcs, amps

end


# Test Sherman-Morrison inverse
A = rand(2, 2)
Ainv = inv(A)
u = rand(2, 1)
B = A + u * u'
@test NonlinearBandits.sherman_morrison_inv(Ainv, u, u) ≈ inv(B)
@test_throws(
    DimensionMismatch("u and v have mismatched dimensions"),
    NonlinearBandits.sherman_morrison_inv(Ainv, u, zeros(3, 1))
)
@test_throws(
    ArgumentError("u and v should have exactly one column"),
    NonlinearBandits.sherman_morrison_inv(Ainv, zeros(2, 2), zeros(2, 2))
)
@test_throws(
    DimensionMismatch("size of A does not match u/v"),
    NonlinearBandits.sherman_morrison_inv(Ainv, zeros(3, 1), zeros(3, 1))
)

## Test Bayesian linear model fitting procedure
d, n, σ = 2, 10000, 2
β = rand(Normal(0, 10), d, 1)
f(x) = β' * x
X, y = NonlinearBandits.gaussian_data(d, n, f; σ=σ)
shape0, scale0 = 0.001, 0.001

lm = BayesLM(d; shape0=shape0, scale0=scale0)
@test_throws(
    ArgumentError("normal-inverse-gamma has undefined mean for shape ≤ 1"), std(lm)
)
fit!(lm, X, y)

prediction_mae = mae(lm(X), mapslices(f, X; dims=1))
β_mae = mae(lm.β, β)
@test β_mae < 0.5
@test prediction_mae < 0.5
@test lm.shape == shape0 + n / 2
@test isapprox(std(lm), σ; atol=0.5)

# Check that sequential updates lead to the same results
sequential_lm = BayesLM(d; shape0=shape0, scale0=scale0)
for i in 1:n
    fit!(sequential_lm, X[:, i:i], y[:, i:i])
end

@test sequential_lm.β ≈ lm.β
@test sequential_lm.Σ ≈ lm.Σ
@test sequential_lm.shape == shape0 + (n / 2)
@test sequential_lm.scale ≈ lm.scale

# Check errors
@test_throws(
    ArgumentError("d, shape0 and scale0 must be strictly positive"),
    BayesLM(0; shape0=0.0, scale0=0.0)
)
@test_throws(ArgumentError("y should have exactly 1 row"), fit!(lm, X, y'))
@test_throws(
    DimensionMismatch("X and y don't match in dimension 2"), fit!(lm, X[:, 1:2], y)
)
@test_throws(
    DimensionMismatch("X does not have the same number of predictors as the model"),
    fit!(lm, X[1:1, :], y)
)
@test_throws(
    DimensionMismatch("X does not have the same number of predictors as the model"),
    lm(X[1:1, :])
)


# Create confusion matrix for any number of classes
function confusion_matrix(y_cond::Vector{Int}, y_pred::Vector{Int})

    # Generate empty confusion matrix
    classes = sort(unique([y_cond; y_pred]))
    conf_matrix = zeros(Int64, length(classes), length(classes))

    # Add counts
    for i in 1:length(y_cond)
        # translate label to index
        cond_class_index = findfirst(classes, y_cond[i])
        pred_class_index = findfirst(classes, y_pred[i])
        # predicted class is the row, condition class is the column
        conf_matrix[pred_class_index, cond_class_index] += 1
  end

  return conf_matrix
end

# plot confusion matrix
function plot_confusion_matrix(y_cond::Vector{Int},
                               y_pred::Vector{Int},
                               ylabel::String = "Var1",
                               xlabel::String = "Var2",
                               classes::Vector{String} = string.(sort(unique([y_cond; y_pred]))))
    # Number of classes
    num = length(classes)

    # Create DataFrame with data from confusion matrix
    axes = vcat([[classes[i] classes[j]] for i in 1:num, j in 1:num]...)
    df = DataFrame([axes confusion_matrix(y_cond, y_pred)[:]])

    # Colormap
    cmap = vcat(RGB(1,1,1), sequential_palette(255, 99, c=0.88, s=0.6, b=0.75, w=0.3, d=0.25,
                            wcolor=RGB(1,1,1), dcolor=RGB(0,0,1), logscale=false))

    # Plot
    plot(df, x="x2", y="x1", color="x3", Geom.rectbin,
         Coord.cartesian(yflip=true, xflip=false, fixed=true),
         Guide.YLabel(ylabel), Guide.XLabel(xlabel), Guide.colorkey("Count"),
         Scale.color_continuous(colormap = p->cmap[round(Int, p*99+1)], minvalue=0))
end

# Print accuracy stats
# Optional to print short, medium or long version
function print_binary_accuracy(y_cond::Vector{Int}, y_pred::Vector{Int}; size="short")

    # Check only 2 classes
    length(unique([y_cond; y_pred])) > 2 && throw("function is only for 2 classes")

    # Create confusion matrix
    CM = confusion_matrix(y_cond, y_pred)

    # Calculate metrics

    # 1st level metrics
    Population = sum(CM)          # Total number in population

    TP = CM[1, 1]                 # "True Positive" / "Hit"
    TN = CM[2, 2]                 # "True Negative" / "Correct Reject"
    FP = CM[1, 2]                 # "False Positive" / "False Alarm" / Type I Error
    FN = CM[2, 1]                 # "False Negative" / "Miss" / Type II Error

    CondPos = TP + FN             # Sum of Condition = Positive
    CondNeg = FP + TN             # Sum of Condition = Negative
    PredPos = TP + FP             # Sum of Prediction = Positive
    PredNeg = FN + TN             # Sum of Prediction = Negative

    # 2nd level metrics

    TPR = TP / CondPos            # True Positive Rate / Sensitivity / Hit Rate / Recall
    TNR = TN / CondNeg            # True Negative Rate / Specificity
    PPV = TP / PredPos            # Positive Predictive Value / Precision
    NPV = TN / PredNeg            # Negative Predictive Value
    FPR = FP / CondNeg            # False Positive Rate / Fall-out
    FDR = FP / PredNeg            # False Discovery Rate
    FNR = FN / CondNeg            # False Negative Rate
    FOR = FN / PredNeg            # False Omission Rate

    # 3rd level metrics

    Accuracy = (TP + TN) / Population                  # Accuracy Rate
    PosLR = TPR / FPR                                  # Positive Likelihood Ratio
    NegLR = FNR / TNR                                  # Negative Likelihood Ratio
    DiagOR = PosLR / NegLR                             # Diagnostic Odds Ratio

    # F Scores i.e. F_β = ((1 + β^2) * PPV * TPR)/ (β^2 * PPV + TPR)
    F_1 = (2 * PPV * TPR) / (PPV + TPR)                # F1 Score / Harmonic Mean of Precision and Recall
    F_2 = (5 * PPV * TPR) / (4 * PPV + TPR)            # F2 Score - weights recall higher than precision
    F_05 = (1.25 * PPV * TPR) / (0.25 * PPV + TPR)     # F0.5 Score - weights precision higher than recall

    # Matthews's Correlation Coefficient
    # Guideline:
    # +1 = perfect prediction, 0 = no better than random, -1 = total disagreement
    MCC = (TP*TN - FP*FN) / sqrt((TP+FP)*(TP+FN)*(TN+FP)*(TN+FN))

    # Kappa Coefficient - compares Observed Accuracy with Expected Accuracy (random chance)
    # Guidelines:
    # L&K: <0 = no agreement, 0-0.20 = slight, 0.21-0.40 = fair, 0.41-0.60 = moderate, 0.61-0.80 = substantial, 0.81-1 = almost perfect
    # Fleiss: <0.4 = poor, 0.40-0.75 = fair to good, >0.75 = excellent
    ObsAcc = (TP + TN) / Population                                    # Observed Accuracy
    ExpAcc = (CondPos * PredPos + CondNeg * PredNeg) / Population^2    # Expected Accuracy
    Kappa = (ObsAcc - ExpAcc) / (1 - ExpAcc)                           # Kappa

    # Display metrics
    if size == "short"
        @printf "Accuracy:    %f\n" Accuracy
        @printf "F_1:         %f\n" F_1
    elseif size == "medium"
        @printf "Accuracy:    %f\n" Accuracy
        @printf "F_1:         %f\n" F_1
        @printf "F_2:         %f\n" F_2
        @printf "F_05:        %f\n" F_05
        @printf "MCC:         %f\n" MCC
        @printf "Kappa:       %f\n" Kappa
    elseif size == "long"
        @printf "Accuracy:    %f\n" Accuracy
        @printf "F_1:         %f\n" F_1
        @printf "F_2:         %f\n" F_2
        @printf "F_05:        %f\n" F_05
        @printf "MCC:         %f\n" MCC
        @printf "Kappa:       %f\n" Kappa
        @printf "Precision:   %f\n" PPV
        @printf "Recall:      %f\n" TPR
        @printf "Specificity: %f\n" TNR
    end
end

# ROC Curve


function distributeCandies(nums::Array{Int64})
    a = Set(nums)
    if length(a) > length(nums)/2
        length(nums) / 2
    else
        length(a)
    end
end

using Test
@test distributeCandies([1,1,2,2,3,3]) == 3
@test distributeCandies([1,1,2,3]) == 2


wall = Lar.svg2lar("./test/svg/assembly/wall.svg", flag=false);
openings = Lar.svg2lar("./test/svg/assembly/svg/openings.svg", flag=false);
rectangle = Lar.svg2lar("./test/svg/assembly/rectangle.svg", flag=false);
box = Lar.svg2lar("./test/svg/assembly/box.svg", flag=false);

assembly = Lar.Struct([wall, openings, rectangle, box]);
V,EV = Lar.struct2lar(assembly);
if todisplay  GL.VIEW([ GL.GLGrid(V,EV, GL.COLORS[1],1), GL.GLFrame2 ]);  end

model = CAGD.Model(V);
CAGD.addModelCells!(model, 1, EV);

A_model = CAGD.planar_arrangement(model);

A_model.G
Matrix(A_model.T[1])
Matrix(A_model.T[2])

# Display Models

if todisplay
    VV = [[k] for k=1:size(A_model.G, 2)];
    triangulated_faces = CAGD.triangulate2D(A_model);
    FVs = convert(Array{Lar.Cells}, triangulated_faces);

    GL.VIEW(GL.GLExplode(A_model.G, FVs, 1.0, 1.0, 1.0, 99, 1));
    GL.VIEW(GL.GLExplode(A_model.G, FVs, 4.0, 4.0, 4.0, 99, 1));
end

innerpoints = Lar.internalpoints2d(A_model.G', A_model.T[1], A_model.T[2][1:end, :])
# associate internal points to 2-cells
listOfModels = Lar.evalStruct(assembly)
inputfacenumbers = [length(listOfModels[k][2]) for k=1:length(listOfModels)];
# test input data for containment of reference points
boolmatrix = sparse(BitArray(undef, length(innerpoints), length(listOfModels)));
containmenttest = Lar.testinternalpoint2d(listOfModels);
for (k,point) in enumerate(innerpoints) # k runs on columns
    cells = containmenttest(point); # contents of columns
    #println(k," ",faces)
    for l in cells
        boolmatrix[k,l] = 1;
    end
end

boolmatrix = sparse(Bool[
    true  false false true
    true  false true  true
    false false false true
    true  true  false true
    false true  false true
    false false false true
    true  true  false true
    false false false true
    false true  false false
    false false true  true
    true  false false true
    false true  false true
    true  false false true
    true  false false true
    false true  false true
    false false  true  true
    false false true  false
]);

Matrix(boolmatrix)

wall = boolmatrix[:, 1]
Lshape = boolmatrix[:, 2]
rect = boolmatrix[:, 3]
box = boolmatrix[:, 4]

GL.VIEW(GL.GLExplode(A_model.G, FVs[wall.nzind],   1.0, 1.0, 1.0, 99, 1));
GL.VIEW(GL.GLExplode(A_model.G, FVs[Lshape.nzind], 1.0, 1.0, 1.0, 99, 1));
GL.VIEW(GL.GLExplode(A_model.G, FVs[rect.nzind],   1.0, 1.0, 1.0, 99, 1));
GL.VIEW(GL.GLExplode(A_model.G, FVs[box.nzind],    1.0, 1.0, 1.0, 99, 1));

shells = Array{SparseVector{Bool,Int64},1}()
push!(shells, .|(wall, Lshape, rect));
push!(shells, .&(wall, .!Lshape, .!rect));
push!(shells, .&(box, .!shells[2]))

if todisplay
    for i = 1 : length(shells)
    GL.VIEW(GL.GLExplode(A_model.G, FVs[shells[i].nzind], 1.0, 1.0, 1.0, 1, 1));
    GL.VIEW(GL.GLExplode(A_model.G, FVs[shells[i].nzind], 1.0, 1.0, 1.0, 99, 1));
    end
end


### A Pluto.jl notebook ###
# v0.12.10

using Markdown
using InteractiveUtils

# ╔═╡ e077ab16-22e5-11eb-15ed-5f0654bf7328
using Pkg, DrWatson

# ╔═╡ e75a9a06-22e5-11eb-2e36-8d4f62b830ed
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using DataFrames, CSV, Dates, StatsBase, Plots, TimeSeries
end

# ╔═╡ ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
md"## Listing4.20"

# ╔═╡ fa68607e-22e5-11eb-0558-c9a4d9f77426
begin
	data = CSV.read(datadir("temperatures.csv"),copycols = true)
	brisbane = data.Brisbane
	dates = [Date(
				Year(data.Year[i]),
				Month(data.Month[i]),
				Day(data.Day[i])
			) for i in 1:nrow(data)]

	window1, window2 = 7, 14
	d1 = values(moving(mean,TimeArray(dates,brisbane),window1))
	d2 = values(moving(mean,TimeArray(dates,brisbane),window2))
end;

# ╔═╡ 56bc1334-250c-11eb-3c15-f98b6097c3a6
begin
	grid = (2pi:-2pi/365:0) .+ pi/2
	monthsNames = Dates.monthname.(dates[1:31:365])

	plot(grid, d1[1:366],
		c=:blue, proj=:polar, label="Brisbane weekly average temp.")
	plot!(grid, d2[1:366],
		xticks=([mod.((11pi/6:-pi/6:0) .+ pi/2,2pi) ;], monthsNames),
		c=:red, proj=:polar,
		label="Brisbane fortnightly average temp.", legend=:outerbottom)
end

# ╔═╡ 475ff888-22e6-11eb-2354-09f8f40a8e12
md"## End of listing4.20"

# ╔═╡ Cell order:
# ╟─ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
# ╠═e077ab16-22e5-11eb-15ed-5f0654bf7328
# ╠═e75a9a06-22e5-11eb-2e36-8d4f62b830ed
# ╠═fa68607e-22e5-11eb-0558-c9a4d9f77426
# ╠═56bc1334-250c-11eb-3c15-f98b6097c3a6
# ╟─475ff888-22e6-11eb-2354-09f8f40a8e12


import RowEchelon

#=
Homework 3
Section 1.7
Problem 41
=#

import RowEchelon

#=
Homework 3
Section 1.7
Problem 5
=#

# Create the matrix
A = [3 -4 10 7 -4 ;
	 -5 -3 -7 -11 15 ;
	 4 3 5 2 1 ;
	 8 -7 23 4 15]

# Row Reduce the matrix
A_rref = RowEchelon.rref(A)

# Store the rows and columns of A
rows = size(A)[1]
cols = size(A)[2]

# Find the pivot columns
pivotCols = zeros(0)
for i = 1:rows
	for j = 1:cols
		if A_rref[i,j] != 0
			append!(pivotCols,j)
			break
		end
	end
end

# The columns we want are the pivot columns
B = Array{Float64}(undef, 4, 0)
for col in pivotCols
	global B
	B = hcat(B, A[:,Int64(col)])
end

# Row Reduce
B_rref = RowEchelon.rref(B)

# Print results:
println("Original Matrix:")
println(A)
print("Matrix made from columns ")
print(pivotCols)
println(":")
println(B)
println("Row Reduced:")
println(B_rref)


"""
    PoissonProcess{T<:Real} <: AbstractMarkovChain

##### Fields
- `rate::T` : the rate; i.e., the parameter "lambda".
"""
struct PoissonProcess{T<:Real} <: AbstractMarkovChain
    rate::T
end

"""
    Base.show(io::IO, pp::PoissonProcess) -> nothing

Show a `PoissonProcess` in a human-friendly manner.
"""
function Base.show(io::IO, pp::PoissonProcess)
    println(io, typeof(pp))
    println(io, " Rate (i.e. lambda):")
    println(io, pp.rate)
    return nothing
end

"""
    Base.eltype(pp::PoissonProcess) -> DataType

Return the type associated with `pp`'s `rate` parameter.
"""
function Base.eltype(pp::PoissonProcess)
    return typeof(pp.rate)
end

"""
    nexpectedarrivals(pp::PoissonProcess, intervallength::Real) -> Int

Return the expected number of arrivals `E[N(intervallength)]` over the time interval
`[0, intervallength]` for `pp`.

##### Complexity
-Time complexity: `O(1)`.
"""
function nexpectedarrivals(pp::PoissonProcess, intervallength::Real)
    return pp.rate*intervallength
end


using Test
using JuliaEuler
using Lazy

@testset "helper" begin
    # Write your tests here.
    @testset "Fib" begin
        @test fib(;under=90) |> collect |> Vector{Int} == [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89]
        @test fib(;terms=11) |> collect |> Vector{Int} == [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89]
        @test take(11, fibs) |> collect |> Vector{Int} == [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89]
        @test takewhile(x-> x < 90,fibs) |> collect |> Vector{Int} == [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89]
        @test take(11,fibs) |> collect |> Vector{Int} == [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89]
    end
    @testset "Prime" begin
        @test primes(20) == [2,3,5,7,11,13,17,19]
        @test primes(2) == [2]
        @test primes(5) == [2,3,5]
        @test factors(2309125) == [(5,3), (7,2), (13,1), (29,1)]
        @test factors(2) == [(2,1)]
        @test factors(5) == [(5,1)]
        @test primes_of_num(13195) == [5,7,13,29]
        @test primes_of_num(2) == [2]
        @test primes_of_num(5) == [5]

    end

    @testset "ProperDivisors" begin
        @test proper_divisors(220) == [1, 2, 4, 5, 10, 11, 20, 22, 44, 55, 110]
        @test proper_divisors(284) == [1, 2, 4, 71, 142]
    end

    @testset "Permutations" begin
        @test Set(permutations([1,2,3], 2)) == Set([(1,2), (1,3), (2,1), (2,3), (3,1),(3,2)])
        @test Set(permutations([1,2,3], 3)) == Set([(1,2,3), (1,3,2), (2,1,3), (2,3,1), (3,1,2), (3,2,1)
            ])

    end

    @testset "Product" begin
        @test Set(product([1,2,3], 2)) == Set([(1,1), (1,2), (1,3), (2,1), (2,2), (2,3), (3,1), (3,2), (3,3)])
        @test Set(product([1,2,3], 3)) == Set([
            (1,1,1), (1,1,2), (1,1,3), (1,2,1), (1,2,2), (1,2,3), (1,3,1), (1,3,2), (1,3,3),
            (2,1,1), (2,1,2), (2,1,3), (2,2,1), (2,2,2), (2,2,3), (2,3,1), (2,3,2), (2,3,3),
            (3,1,1), (3,1,2), (3,1,3), (3,2,1), (3,2,2), (3,2,3), (3,3,1), (3,3,2), (3,3,3)
            ])
    end

    @testset "combinations" begin
        @test Set(combinations([1,2,3], 2)) == Set([(1,2), (1,3), (2,3)])
        @test Set(combinations([1,2,3], 2;with_replacement=true)) == Set([(1,1), (1,2), (1,3), (2,2), (2,3), (3,3)])
        @test Set(combinations([1,2,3], 3)) == Set([(1,2,3)])
        @test Set(combinations([1,2,3], 3;with_replacement=true)) == Set([
            (1,1,1), (1,1,2), (1,1,3), (1,2,2), (1,2,3), (1,3,3),
            (2,2,2), (2,2,3), (2,3,3),
            (3,3,3)
    ])
    end

    @testset "unique_sorted" begin
        @test JuliaEuler.unique_sorted([1,2,3]) == true
        @test JuliaEuler.unique_sorted([1,2, 2,3]) == false
    end
end


using SparseDiffTools
using DiffEqDiffTools: finite_difference_jacobian, finite_difference_jacobian!
using SparsityDetection

using LinearAlgebra, SparseArrays, Test

fcalls = 0

function f(x)
  global fcalls += 1
  dx = similar(x)
  for i in 2:length(x)-1
    dx[i] = x[i-1] - 2x[i] + x[i+1]
  end
  dx[1] = -2x[1] + x[2]
  dx[end] = x[end-1] - 2x[end]
  dx
end

function f(dx,x)
  global fcalls += 1
  for i in 2:length(x)-1
    dx[i] = x[i-1] - 2x[i] + x[i+1]
  end
  dx[1] = -2x[1] + x[2]
  dx[end] = x[end-1] - 2x[end]
  nothing
end

function second_derivative_stencil(N)
  A = zeros(N,N)
  for i in 1:N, j in 1:N
      (j-i==-1 || j-i==1) && (A[i,j]=1)
      j-i==0 && (A[i,j]=-2)
  end
  A
end

output = ones(30); input = ones(30)
sparsity_pattern = sparsity!(f,output,input)
true_jac = Float64.(sparse(sparsity_pattern))
colors = matrix_colors(true_jac)
@test colors == repeat(1:3,10)

#Jacobian computed without coloring vector
fcalls = 0
J = finite_difference_jacobian(f, rand(30))
@test J ≈ second_derivative_stencil(30)
@test fcalls == 31

#Jacobian computed with coloring vectors
fcalls = 0
_J = similar(true_jac)
finite_difference_jacobian!(_J, f, rand(30), colorvec = colors)
@test fcalls == 4
@test _J ≈ J

fcalls = 0
_J = similar(true_jac)
_denseJ = collect(_J)
finite_difference_jacobian!(_denseJ, f, rand(30), colorvec = colors, sparsity=_J)
@test fcalls == 4
@test _denseJ ≈ J


import LinearAlgebra

function close_to_zero(value::T) where {T<:AbstractFloat}
    return abs(value) < 1e-2
end

function se3_inv(T::Matrix{<:AbstractFloat})
    C = T[1:3, 1:3]
    d = T[1:3, 4]
    return [C' -C'*d; 0 0 0 1]
end

# function skew(x::Vector{<:AbstractFloat})
#     return [
#         0 -x[3] x[2]
#         x[3] 0 -x[1]
#         -x[2] x[1] 0
#     ]
# end

function so3_wedge(ϕ::Vector{<:AbstractFloat})
    return [
        0 -ϕ[3] ϕ[2]
        ϕ[3] 0 -ϕ[1]
        -ϕ[2] ϕ[1] 0
    ]
end

function so3_exp(psi::Vector{<:AbstractFloat})
    angle = norm(psi, 2)
    axis = psi / angle
    s = sin(angle)
    c = cos(angle)
    return c * I(3) + (1 - c) * axis .* axis' - s * so3_wedge(axis)
end

function so3_vee(C::Matrix{<:AbstractFloat})
    return [C[3, 2], C[1, 3], C[2, 1]]
end

function so3_log(C::Matrix{<:AbstractFloat})

    cos_angle = 0.5 * tr(C) - 0.5
    cos_angle = clamp(cos_angle, -1.0, 1.0)
    angle = acos(cos_angle)

    if close_to_zero(angle)
        tmp = C - I
    else
        tmp = (angle / (2 * sin(angle))) * (C - C')
    end

    return so3_vee(tmp)
end

function so3_jacobian(psi::Vector{<:AbstractFloat})
    angle = norm(psi)

    if close_to_zero(angle)
        return I + 0.5 * so3_wedge(psi)
    end

    axis = psi / angle
    s = sin(angle)
    c = cos(angle)

    return (s / angle) * I(3) + (1 - s / angle) * axis * axis' + (1 - c) / angle * so3_wedge(axis)
end

function so3_inv_left_jacobian(phi::Vector{<:AbstractFloat})
    dof = 3
    angle = norm(phi)

    # Near phi==0, use first order Taylor expansion
    if close_to_zero(angle)
        return I(3) - 0.5 * so3_wedge(phi)
    end

    axis = phi / angle
    half_angle = 0.5 * angle
    cot_half_angle = 1.0 / tan(half_angle)

    return half_angle * cot_half_angle * I(dof) + (1 - half_angle * cot_half_angle) * (axis * axis') - half_angle * so3_wedge(axis)

end

# Might have to update this
function se3_exp(d::Vector{<:AbstractFloat}, psi::Vector{<:AbstractFloat})
    J = so3_jacobian(psi)
    C = so3_exp(psi)
    return [C J*d; 0 0 0 1]
end

function se3_log(T::Matrix{<:AbstractFloat})
    psi = so3_log(T[1:3, 1:3])

    J = so3_inv_left_jacobian(psi)
    r = inv(J) * T[1:3, 4]

    return [r; psi]
end

# Functions taken from the python liegroups packages

function se3_wedge(ξ::Vector{<:AbstractFloat})
    dim = 4
    Ξ = zeors(size(ξ, 1), dim, dim)
    Ξ[:, 1:3, 1:3] = so3_wedge(ξ[:, 4:6])
    Ξ[:, 0:3, 3] = ξ[:, 1:3]
    return Ξ
end

function se3_odot(p::Vector{<:AbstractFloat})
    dim = 4
    dof = 6

    result = zeros(size(p, 1), dof)

    if size(p, 1) == dim - 1
        result[1:3, 1:3] = I(3)
        result[1:3, 4:6] = so3_wedge(-p)
    else
        result[1:3, 1:3] = I(3)
        result[1:3, 4:6] = so3_wedge(-p)
    end

    return result
end

function se3_adjoint(T::Matrix{<:AbstractFloat})
    C = T[1:3, 1:3]
    r = T[1:3, 4]
    return [
        C so3_wedge(r)*C
        zeros(3, 3) C
    ]
end


"""
    entropy(P)

Computes the entropy of a distribution.
"""
function entropy(P; norm=false)
    if norm
        P /= sum(P)
    end
    return - sum(P[P.>0] .* log.(P[P.>0]))
end

"""
    relative_entropy(P, Q)

Computes the relative entropy of P to Q.
"""
relative_entropy(P, Q) = - sum(P .* log.(Q))

"""
    KL(P, Q)

Computes the Kullback-Leibler divergence of P to Q
"""
KL(P, Q) = relative_entropy(P, Q) - entropy(P)

"""
    utility(P, M)

Computes the average utility.
"""
function utility(P, M)
    return sum(P .* M)
end

function report_utility(P, M, bottom_species=nothing, top_species=nothing)
    @assert size(P) == size(M) "P and M should be of the same size"
    n, m = size(P)
    bottom_species == nothing && bottom_species == 1:n
    top_species == nothing && top_species == 1:m
    U = P .* M
    a, b = marginals(P)
    println("Bottom species")
    for i in 1:n
        println("\t$(bottom_species[i]): utility = $(sum(U[i,:]) / a[i]), entropy = $(
                                entropy(P[i,:] |> x -> x / sum(x)))")
    end
    println()
    println("Top species")
    for j in 1:m
        println("\t$(top_species[j]): utility = $(sum(U[:,j] / b[j])), entropy = $(
                                entropy(P[:,j] |> x -> x / sum(x)))")
    end
    println()
    println("Average utility = $(sum(U))")
    println("Average entropy = $(entropy(P))")
end


using Distributions
import Base.size, Base.length, Base.rand, Base.mean, Base.convert

doc"""
    MatrixNormal(M, U, V)
The *Matrix Normal distribution* with mean `M` and row and column covariances `U` and `V` has probability density function
$f(\mathbf{X}; \mathbf{M}, \mathbf{U}, \mathbf{V}) = \frac{\exp\left(
    -\frac{1}{2} \mathrm{tr}\left[
        \mathbf{V}^{-1} (\mathbf{X} - \mathbf{M})^\top \mathbf{U}^{-1}
        (\mathbf{X} - \mathbf{M})
    \right]
    \right)}{(2\pi)^{np/2} |\mathbf{V}|^{n/2} |\mathbf{U}|^{p/2}}$
when $\mathbf{X}$ is $n \times p$.
```julia
MatrixNormal(U, V)      # Matrix Normal with mean 0 and covariances U and V
MatrixNormal(M, U, V)      # Matrix Normal with mean M and covariances U and V
params(d)        # Get the parameters, i.e. (M, U, V)
mean(d)          # Get the mean, i.e., M
cov(d)           # Get the covariance, i.e., V ⊗ U
var(d)           # Get the marginal variances for individual entries
invcov(d)        # inverse of covariance
logdetcov(d)     # log determinant of covariance
logpdf(d, X)     # log of the pdf at point X
```
External links
* [Matrix Normal distribution on Wikipedia](http://en.wikipedia.org/wiki/Matrix_normal_distribution)
"""
immutable MatrixNormal{Mean<:AbstractMatrix, CovU<:AbstractPDMat, CovV<:AbstractPDMat} <: ContinuousMatrixDistribution
    M::Mean
    U::CovU
    V::CovV
end

### Constructors
typealias CovLike Union{AbstractMatrix, AbstractVector, AbstractPDMat}
normalize(σ::AbstractVector) = PDiagMat(abs2(σ))
normalize(Σ::AbstractMatrix) = PDMat(Σ)
normalize(Σ::AbstractPDMat) = Σ
normalize(d::Int, σ::Real) = PDScalMat(d, abs2(σ))

function MatrixNormal{Mean <: AbstractMatrix, CovU<:AbstractPDMat, CovV<:AbstractPDMat}(M::Mean, U::CovU, V::CovV)
    size(M) == (dim(U), dim(V)) || throw(DimensionMismatch("The dimensions of M, U, and V are inconsistent."))
    MatrixNormal{Mean, CovU, CovV}(M, U, V)
end
function MatrixNormal{Mean<:AbstractMatrix, CovU<:CovLike, CovV<:CovLike}(M::Mean, U::CovU, V::CovV)
    MatrixNormal(M, normalize(U), normalize(V))
end

MatrixNormal{CovU<:AbstractPDMat, CovV<:AbstractPDMat}(U::CovU, V::CovV) = MatrixNormal(zeros(dim(U), dim(V)), U, V)
MatrixNormal{CovU<:CovLike, CovV<:CovLike}(U::CovU, V::CovV) = MatrixNormal(normalize(U), normalize(V))

### Conversions
convert(::Type{MvNormal}, d::MatrixNormal) = MvNormal(vec(mean(d)), cov(d))

### interface
size(d::MatrixNormal) = (dim(d.U), dim(d.V))
params(d::MatrixNormal) = (d.M, d.U, d.V)
mean(d::MatrixNormal) = d.M
cov(d::MatrixNormal) = kron(full(d.V), full(d.U))
var(d::MatrixNormal) = diag(d.U) * diag(d.V)'
invcov(d::MatrixNormal) = kron(full(inv(d.V)), full(inv(d.U)))
logdetcov(d::MatrixNormal) = ((n, p) = size(d); n * logdet(d.V) + p * logdet(d.U))

rand(d::MatrixNormal) = reshape(rand(convert(MvNormal, d)), size(d))
entropy(d::MatrixNormal) = entropy(convert(MvNormal, d))

function logpdf(d::MatrixNormal, X::AbstractMatrix)
    size(X) == size(d) || error("Input matrix x has wrong size.")
    m = X - d.M
    n, p = size(d)
    invU = full(inv(d.U))
    invV = full(inv(d.V))
    lpdf = -0.5 * trace(invV * m' * invU * m)
    lpdf -= (n * p/2) * Float64(Distributions.log2π)
    lpdf -= 0.5 * logdetcov(d)
    lpdf
end