# using OrdinaryDiffEq
using ModelingToolkit
using DataDrivenDiffEq
using LinearAlgebra, DiffEqSensitivity, Optim
using DiffEqFlux, Flux
using Plots
using DifferentialEquations
gr()

function lotka(du, u, p, t)
    α, β, γ, δ = p
    du[1] = α*u[1] - β*u[2]*u[1]
    du[2] = γ*u[1]*u[2]  - δ*u[2]
end

# Define the experimental parameter
tspan = (0.0f0,3.0f0)
u0 = Float32[0.44249296,4.6280594]
p_ = Float32[1.3, 0.9, 0.8, 1.8]
prob = ODEProblem(lotka, u0,tspan, p_)
solution = solve(prob, Vern7(), abstol=1e-12, reltol=1e-12, saveat = 0.1)

scatter(solution, alpha = 0.25)
plot!(solution, alpha = 0.5)

# Ideal data
X = Array(solution)
# Add noise to the data
println("Generate noisy data")
Xₙ = X + Float32(1e-3)*randn(eltype(X), size(X))

# Define the neueral network which learns L(x, y, y(t-τ))
# Actually, we do not care about overfitting right now, since we want to
# extract the derivative information without numerical differentiation.
L = FastChain(FastDense(2, 32, tanh),FastDense(32, 32, tanh), FastDense(32, 2))
p = initial_params(L)

function dudt_(u, p,t)
    x, y = u
    z = L(u,p)
    [p_[1]*x + z[1],
    -p_[4]*y + z[2]]
end

prob_nn = ODEProblem(dudt_,u0, tspan, p)
sol_nn = concrete_solve(prob_nn, Tsit5(), u0, p, saveat = solution.t)

plot(solution)
plot!(sol_nn)

function predict(θ)
    Array(concrete_solve(prob_nn, Tsit5(), u0, θ, saveat = solution.t,
                         abstol=1e-6, reltol=1e-6,
                         sensealg = InterpolatingAdjoint(autojacvec=ReverseDiffVJP())))
    # @show Array(concrete_solve(prob_nn, Vern7(), u0, θ, saveat = solution.t,
    #                                           abstol=1e-6, reltol=1e-6,
    #                                           sensealg = InterpolatingAdjoint(autojacvec=ReverseDiffVJP())))
  #   tmp_prob = remake(prob_nn,u0=u0,p=θ)
  # tmp_sol =  solve(tmp_prob, Rodas4(), saveat = solution.t,
  #                                           abstol=1e-6, reltol=1e-6,
  #                                           sensealg = InterpolatingAdjoint(autojacvec=ReverseDiffVJP()))
  # # @show size(Array(tmp_sol))
  # pred =  Array(tmp_sol)
end

# No regularisation right now
function loss(θ)
    pred = predict(θ)
    sum(abs2, Xₙ .- pred), pred
end

# Test
loss(p)

const losses = []

callback(θ,l,pred) = begin
    @show l
    push!(losses, l)
    if length(losses)%50==0
        println("Current loss after $(length(losses)) iterations: $(losses[end])")
    end
    false
end

# First train with ADAM for better convergence
res1 = DiffEqFlux.sciml_train(loss, p, ADAM(0.5), cb=callback, maxiters = 100)

# Train with BFGS
res2 = DiffEqFlux.sciml_train(loss, res1.minimizer, BFGS(initial_stepnorm=0.01), cb=callback, maxiters = 10)

println("Final training loss after $(length(losses)) iterations: $(losses[end])")

# Plot the losses
plot(losses, yaxis = :log, xaxis = :log, xlabel = "Iterations", ylabel = "Loss")

# Plot the data and the approximation
NNsolution = predict(res2.minimizer)
# Trained on noisy data vs real solution
plot(solution.t, NNsolution')
plot!(solution.t, X')

# Ideal derivatives
DX = Array(solution(solution.t, Val{1}))

prob_nn2 = ODEProblem(dudt_,u0, tspan, res2.minimizer)
_sol = solve(prob_nn2, Tsit5())
DX_ = Array(_sol(solution.t, Val{1}))

# The learned derivatives
plot(DX')
plot!(DX_')

# Ideal data
L̄ = [-p_[2]*(X[1,:].*X[2,:])';p_[3]*(X[1,:].*X[2,:])']
# Neural network guess
L̂ = L(Xₙ,res2.minimizer)

scatter(L̄')
plot!(L̂')

# Plot the error
scatter(abs.(L̄-L̂)', yaxis = :log)

## Sparse Identification

# Create a Basis
@variables u[1:2]
# Lots of polynomials
polys = Operation[1]

for i ∈ 1:5
    push!(polys, u[1]^i)
    push!(polys, u[2]^i)
    for j ∈ i:5
        if i != j
            push!(polys, (u[1]^i)*(u[2]^j))
            push!(polys, u[2]^i*u[1]^i)
        end
    end
end

# And some other stuff
h = [cos.(u)...; sin.(u)...; polys...]
basis = Basis(h, u)

# Create an optimizer for the SINDy problem
opt = SR3()
# Create the thresholds which should be used in the search process
λ = exp10.(-7:0.1:3)
# Target function to choose the results from; x = L0 of coefficients and L2-Error of the model
g(x) = x[1] < 1 ? Inf : norm(x, 2)
# Test on original data and without further knowledge
println("SINDy on full ideal, unavailable data")
Ψ = SINDy(Xₙ[:, :], DX[:, :], basis, λ, opt, g = g, maxiter = 10000) # Fail
println(Ψ)
print_equations(Ψ)

# Test on ideal derivative data for unknown function ( not available )
println("SINDy on partial ideal, unavailable data")
Ψ = SINDy(Xₙ[:, 1:end], L̄[:, 1:end], basis, λ,opt, g = g, maxiter = 10000) # Succeed
println(Ψ)
print_equations(Ψ)

# Test on uode derivative data
println("SINDy on learned, partial, available data")
Ψ = SINDy(Xₙ[:, 2:end], L̂[:, 2:end], basis, λ,  opt, g = g, maxiter = 10000, normalize = true, denoise = true) # Succeed
println(Ψ)
print_equations(Ψ)

# Extract the parameter
p̂ = parameters(Ψ)
println("First parameter guess : $(p̂)")

# The parameters are a bit off, but the equations are recovered
# Start another SINDy run to get closer to the ground truth
# Create function
unknown_sys = ODESystem(Ψ)
unknown_eq = ODEFunction(unknown_sys)

# Just the equations
b = Basis((u, p, t)->unknown_eq(u, [1.; 1.], t), u)

# Retune for better parameters -> we could also use DiffEqFlux or other parameter estimation tools here.
Ψf = SINDy(Xₙ[:, 2:end], L̂[:, 2:end], b, STRRidge(0.01), maxiter = 100, convergence_error = 1e-18) # Succeed
println(Ψf)
p̂ = parameters(Ψf)
println("Second parameter guess : $(p̂)")

# Create function
recovered_sys = ODESystem(Ψf)
recovered_eq = ODEFunction(recovered_sys)

# Build a ODE for the estimated system
function dudt(du, u, p, t)
    # Add SINDy Term
    α, δ, β, γ = p
    z = recovered_eq(u, [β; γ], t)
    du[1] = α*u[1] + z[1]
    du[2] = -δ*u[2] + z[2]
end

# Create the approximated problem and solution
ps = [p_[[1,4]]; p̂]
approximate_prob = ODEProblem(dudt, u0, tspan, ps)
approximate_solution = solve(approximate_prob, Tsit5(), saveat = 0.01)

# Plot
plot(solution)
plot!(approximate_solution)

## Simulation

# Look at long term prediction
t_long = (0.0, 50.0)
approximate_prob = ODEProblem(dudt, u0, t_long, ps)
approximate_solution_long = solve(approximate_prob, Tsit5(), saveat = 0.1) # Using higher tolerances here results in exit of julia
plot(approximate_solution_long)

true_prob = ODEProblem(lotka, u0, t_long, p_)
true_solution_long = solve(true_prob, Tsit5(), saveat = approximate_solution_long.t)
plot!(true_solution_long)

using JLD2
@save "knowledge_enhanced_NN.jld2" solution recovered_sys approximate_solution_long NNsolution L true_solution_long Xₙ L̄ L̂
@load "knowledge_enhanced_NN.jld2" solution recovered_sys approximate_solution_long NNsolution L true_solution_long Xₙ L̄ L̂

c1 = 3 # RGBA(174/255,192/255,201/255,1) # Maroon
c2 = :orange # RGBA(132/255,159/255,173/255,1) # Red
c3 = :blue # RGBA(255/255,90/255,0,1) # Orange
c4 = :purple # RGBA(153/255,50/255,204/255,1) # Purple

p1 = plot(0.1:0.1:tspan[end],abs.(Array(solution)[:,2:end] .- NNsolution[:,2:end])' .+ eps(Float32),
          lw = 3, yaxis = :log, title = "Timeseries of UODE Error",
          color = [3 :orange], xlabel = "t",
          label = ["x(t)" "y(t)"],
          titlefont = "Helvetica", legendfont = "Helvetica",
          legend = :topright)

# Plot L₂
p2 = plot(X[1,:], X[2,:], L̂[2,:], lw = 3,
     title = "Neural Network Fit of U2(t)", color = c1,
     label = "Neural Network", xaxis = "x", yaxis="y",
     titlefont = "Helvetica", legendfont = "Helvetica",
     legend = :bottomright)
plot!(X[1,:], X[2,:], L̄[2,:], lw = 3, label = "True Missing Term", color=c2)

p3 = scatter(solution, color = [c1 c2], label = ["x data" "y data"],
             title = "Extrapolated Fit From Short Training Data",
             titlefont = "Helvetica", legendfont = "Helvetica",
             markersize = 5)

plot!(p3,true_solution_long, color = [c1 c2], linestyle = :dot, lw=5, label = ["True x(t)" "True y(t)"])
plot!(p3,approximate_solution_long, color = [c3 c4], lw=1, label = ["Estimated x(t)" "Estimated y(t)"])
plot!(p3,[2.99,3.01],[0.0,maximum(hcat(Array(solution),Array(approximate_solution_long)))],lw=2,color=:black)
annotate!([(1.5,9,text("Training \nData", 10, :center, :top, :black, "Helvetica"))])
l = @layout [grid(1,2)
             grid(1,1)]
plot(p1,p2,p3,layout = l)

savefig("sindy_extrapolation.pdf")


using Random
Random.seed!(1)

N = 10^5
prob1 = 0.7
eps0, eps1 = 0.1, 0.05

flipWithProb(bit,prob) = rand() < prob ? xor(bit,1) : bit

TxData = rand(N) .< prob1
RxData = [x == 0 ? flipWithProb(x,eps0) : flipWithProb(x,eps1) for x in TxData]

numTx1 = 0
totalRx1 = 0
for i in 1:N
   if RxData[i] == 1
        global totalRx1 += 1
        global numTx1 += TxData[i]
    end
end

monteCarlo = numTx1/totalRx1
analytic = ((1-eps1)*0.7)/((1-eps1)*0.7+0.3*eps0)

println("Monte Carlo: ", monteCarlo, "\t\tAnalytic: ", analytic)


import Statistics.cor

# Autocorrelation

#TODO: move to another file to maintain a more logical structure

function lagindex(x::Vector{Float64}, n::Int=1)::UnitRange{Int}
    @assert n < size(x,1)
    N = size(x, 1)
    idx = 1:N
    if n == 0
        return idx
    elseif n > 0
        return idx[n+1:N]
    elseif n < 0
        return idx[1:N+n]
    end
end

function lag(x::Vector{Float64}, n::Int=1; pad::Bool=true, padval::Float64=NaN)
    if n==0
        return x
    elseif n>0
        if pad
            return [ones(n)*padval; x[n+1:end]]
        else
            return x[n+1:end]
        end
    elseif n<0
        n *= -1
        if pad
            return [x[1:end-n]; ones(n)*padval]
        else
            return x[1:end-n]
        end
    end
end

function corlag(x::AbstractArray{T,1}, n::Int=1) where {T<:Number}
    if n == 0
        return 1.0
    end
    @assert n > 0
    @assert n < size(x,1) - 2
    idx = collect(lagindex(x, n))
    return cor(x[idx.-n], x[idx])
end

function acf(x::Vector{T}, maxlag::Int=15; lags::AbstractArray{Int,1}=0:maxlag)::Vector{Float64} where {T<:Number}
    @assert all(lags .< size(x,1)-2)
    return map((n) -> corlag(x, n), lags)
end

function acf(x::Matrix{T}, maxlag::Int=15; lags::AbstractVector{Int}=0:maxlag)::Matrix{Float64} where {T<:Number}
    k = size(x,2)
    out = zeros(Float64, (length(lags), k))
    @inbounds for j in 1:k
        out[:,j] = acf(x[:,j], maxlag, lags=lags)
    end
    return out
end

"""
    acf(x::TS,maxlag::Int=15;lags::AbstractArray{Int,1}=0:maxlag)
    acf(x::Vector{T},maxlag::Int=15;lags::AbstractArray{Int,1}=0:maxlag)::Vector{Float64}where{T<:Number}
    acf(x::Matrix{T},maxlag::Int=15;lags::AbstractVector{Int}=0:maxlag)::Matrix{Float64}where{T<:Number}

Compute the autocorrelation function of a time series.

The output will be a matrix with the same number of columns as the input time series `x`
and number of rows equal to the number of lags used as inputs to the autocorrelation function.

...
# Arguments
- `x::TS`: Time series object array containing columns on which to compute autocorrelation.
- `maxlag::Int=15`: Maximum lag of the autocorrelation series.
Optional args:
- `lags::AbstractArray{Int,1}=0:maxlag`: Explicitly specified list of lags to use (overrides use of `maxlag`).
...
"""
acf(x::TS, maxlag::Int=15; lags::AbstractArray{Int,1}=0:maxlag) = acf(x.values, maxlag; lags=lags)


"""
    norm1(v)

Normalize `v` to sum 1.
"""
function norm1(v)
    vs = sum(v)
    if vs ≈ 1.0
        return v
    else
        return v ./ vs
    end
end

"""
    infovalue(p, q)

Compute the symmetric Relative Entropy (Kullback-Liebler Divergence) between `p` and `q`,
two probability vectors or two frequency count vectors.
Value is bounded within [0, ∞].
"""
function infovalue(p, q)
    pn = norm1(p)
    qn = norm1(q)

    kldivergence(pn, qn) + kldivergence(qn, pn)
end
infovalue(f::AbstractMatrix) = infovalue(f[:, 1], f[:, 2])
infovalue(f::NamedArray) = infovalue(f.array)

"""
    cramerv(f::AbstractMatrix{T}) where T <: Real

Compute Cramer's V of a contingency table `f`.\\
ϕ coefficient is √(χ²/n), bounded within [0, √min(row-1,col-1)],
Cramer's V is ϕ/√min(row-1,col-1), bounded within [0,1].
"""
function cramerv(f::AbstractMatrix{T}) where {T<:Real}
    minimum(size(f)) >= 2 || throw(ArgumentError("Matrix needed, not single row or column"))

    p = norm1(f)
    rp = sum(p; dims = 2)               # row marginal
    cp = sum(p; dims = 1)               # column marginal
    ep = rp .* cp                       # expected probabilities
    ϕ² = sum((p .- ep) .^ 2 ./ ep)      # ϕ²

    sqrt(ϕ² / (minimum(size(f)) - 1))   # Cramer's V
end
cramerv(f::NamedArray) = cramerv(f.array)

"""
    cramerv(x::Vector{T} where T<:Integer, y::Vector{T} where T<:Integer)

Compute Cramer's V of two presumed discrete vectors `x` and `y`.\\
"""
cramerv(x::Vector{T} where {T<:Integer}, y::Vector{T} where {T<:Integer}) =
    cramerv(freqtable(x, y).array)

"""
    cramerv(x::Vector, y::Vector; groups = 10)

Compute Cramer's V of two presumed continuous vectors `x` and `y`,
vectors are binned into `groups` first.
"""
function cramerv(x::AbstractVector, y::AbstractVector; groups = 10)
    xr = ranks(x; groups = groups)
    yr = ranks(y, groups = groups)

    cramerv(freqtable(xr, yr).array)
end

"""
    mutualinfo(f::AbstractMatrix{T}) where T <: Real

Compute the mutual information of frequency matrix `f`;
value is bounded within [0, log(min(size(`f`)))].
"""
function mutualinfo(f::AbstractMatrix{T} where {T<:Real})
    minimum(size(f)) >= 2 || throw(ArgumentError("Matrix needed, not single row or column"))

    ps = norm1(f)
    px = sum(ps; dims = 2)
    py = sum(ps; dims = 1)

    entropy(px) + entropy(py) - entropy(ps)
end
mutualinfo(f::NamedArray) = mutualinfo(f.array)

"""
    mutualinfo(x::Vector{T} where T<:Integer, y::Vector{T} where T<:Integer)

Compute mutual information of two presumed discrete vectors `x` and `y`.\\
"""
mutualinfo(x::Vector{T} where {T<:Integer}, y::Vector{T} where {T<:Integer}) =
    mutualinfo(freqtable(x, y).array)

"""
    mutualinfo(x::Vector, y::Vector)

Compute the mutual information of two presumed continuous vectors `x` and `y`.
"""
function mutualinfo(x::AbstractVector, y::AbstractVector; groups = 10)
    xr = ranks(x; groups = groups)
    yr = ranks(y, groups = groups)

    mutualinfo(freqtable(xr, yr).array)
end

"""
    eda(df::AbstractDataFrame, target::Symbol)

Return a dataframe of Mutual Information and Cramer's V between `target` and other
variables in `df`. If `target` is binary, Information Value is also returned.

The dataframe is sorted by descending Mutual Information.
"""
function eda(df::AbstractDataFrame, target::Symbol; groups = 20)::AbstractDataFrame
    t = df[!, target]
    tnlvl = length(unique(t))
    tnlvl <= 1 && throw(ArgumentError("Target is single valued"))

    if tnlvl > groups
        t = ranks(t, groups = groups)
        tnlvl = length(unique(t))
    end

    println("Target: $target   type: $(typeof(t))   Levels: $tnlvl")

    lk = ReentrantLock()

    if tnlvl == 2
        out = DataFrame(
            Variable   = Symbol[],
            Vartype    = String[],
            Varlvls    = Int[],
            MutualInfo = Float64[],
            CramerV    = Float64[],
            InfoValue  = Float64[],
        )

        Threads.@threads for v in propertynames(df)
            v == target && continue

            vtype = eltype(df[!, v])
            if vtype <: Union{Missing,Real}
                vb = ranks(df[!, v], groups = groups)
            else
                vb = df[!, v]
            end

            frq = proptable(vb, t).array
            vnlvl = size(frq, 1)
            if vnlvl == 1
                println("Warning: [$v] is singled valued, skipped.")
                continue
            elseif vnlvl > 50
                println("Note: [$v] has more than 100 levels, consider binning")
            end

            mutin = mutualinfo(frq)
            cramv = cramerv(frq)
            iv = infovalue(frq[:, 1], frq[:, 2])

            lock(lk)
            push!(out, (v, string(vtype), vnlvl, mutin, cramv, iv))
            unlock(lk)
        end
    else
        out = DataFrame(
            Variable   = Symbol[],
            Vartype    = String[],
            Varlvls    = Int[],
            MutualInfo = Float64[],
            CramerV    = Float64[],
        )

        Threads.@threads for v in propertynames(df)
            v == target && continue

            vtype = eltype(df[!, v])
            if vtype <: Union{Missing,Real}
                vb = ranks(df[!, v], groups = groups)
            else
                vb = df[!, v]
            end

            frq = proptable(vb, t).array
            vnlvl = size(frq, 1)
            if vnlvl == 1
                println("Warning: [$v] is singled valued, skipped.")
                continue
            elseif vnlvl > 50
                println("Note: [$v] has more than 100 levels, consider binning")
            end

            mutin = mutualinfo(frq)
            cramv = cramerv(frq)

            lock(lk)
            push!(out, (v, string(vtype), vnlvl, mutin, cramv))
            unlock(lk)
        end
    end

    sort!(out, [:MutualInfo, :CramerV], rev = true)
end


"""
    Multivector2{T <: Real} <: AbstractMultivector{T}

An immutable 4-tuple of real numbers that represents a member
of a 2-dimensional multivector algebra.

Each `Multivector2` has the form
```math
    p+qB
```
where ``p`` and ``q`` are 1-dimensional multivectors, or
```math
    a+bA+cB+dAB
```
where ``a``, ``b``, ``c``, and ``d`` are real (and of the same type),
and ``A * A = 0``, ``B * B = 0``, and ``AB = A * B = -B * A``.
Here ``*`` is the wedge product.
"""
struct Multivector2{T <: Real} <: AbstractMultivector{T}
    l::Multivector1{T}
    r::Multivector1{T}

    Multivector2{U}(l::Multivector1{U}, r::Multivector1{U}) where {U <: Real} = new(l, r)
end

function Multivector2(x::Multivector1{T}, y::Multivector1{T}) where {T <: Real}
    Multivector2{T}(x, y)
end

function Multivector2(z::Multivector1)
    Multivector2(z, zero(z))
end

function Multivector2(a::T, b::T, c::T, d::T) where {T <: Real}
    Multivector2{T}(Multivector1{T}(a, b), Multivector1{T}(c, d))
end

function Multivector2(a::T, b::T, c::T) where {T <: Real}
    Multivector2{T}(Multivector1{T}(a, b), Multivector1{T}(c, zero(T)))
end

function Multivector2(a::T, b::T) where {T <: Real}
    Multivector2{T}(Multivector1{T}(a, b), zero(Multivector1{T}))
end

function Multivector2(a::T) where {T <: Real}
    Multivector2{T}(Multivector1{T}(a, zero(T)), zero(Multivector1{T}))
end

function Multivector2(a::Real, b::Real, c::Real, d::Real)
    Multivector2(promote(a, b, c, d)...)
end

function Multivector2(a::Real, b::Real, c::Real)
    Multivector2(promote(a, b, c)...)
end

function Multivector2(a::Real, b::Real)
    Multivector2(promote(a, b)...)
end

function Multivector2(x::Multivector1, y::Multivector1)
    Multivector2(promote(x.l, x.r, y.l, y.r)...)
end

function show(io::IO, z::Multivector2)
    print(io, "[1: ")
    print(io, z.l.l)
    print(io, ", A: ")
    print(io, z.l.r)
    print(io, ", B: ")
    print(io, z.r.l)
    print(io, ", AB: ")
    print(io, z.r.r)
    print(io, "]")
end

function real(z::Multivector2)
    real(z.l)
end

function unreal(z::Multivector2)
    vcat(unreal(z.l), asarray(z.r))
end

function zero(z::Multivector2{T}) where {T <: Real}
    Multivector2{T}(zero(z.l), zero(z.r))
end

function zero(::Type{Multivector2{T}}) where {T <: Real}
    Multivector2(zero(T), zero(T), zero(T), zero(T))
end

function one(z::Multivector2{T}) where {T <: Real}
    Multivector2{T}(one(z.l), zero(z.r))
end

function one(::Type{Multivector2{T}}) where {T <: Real}
    Multivector2(one(T), zero(T), zero(T), zero(T))
end

"""
    conj{T <: Real}(z::Multivector2{T})

The `Multivector2` conjugate. If ``z=a+bA+cB+dAB``, then `conj(z)` gives
```math
    a-bA-cB-dAB
```
This operation is an involution.
"""
function conj(z::Multivector2{T}) where {T <: Real}
    Multivector2{T}(conj(z.l), -z.r)
end

"""
    cloak{T <: Real}(z::Multivector2{T})

The cloak conjugate changes the sign of even blades. If ``z=a+bA+cB+dAB``, then `cloak(z)` gives
```math
    -a+bA+cB-dAB
```
This operation is equivalent to `-dagger(z)` and thus is also an involution.
"""
function cloak(z::Multivector2{T}) where {T <: Real}
    Multivector2{T}(cloak(z.l), dagger(z.r))
end

"""
    dagger{T <: Real}(z::Multivector2{T})

The dagger conjugate changes the sign of odd blades. If ``z=a+bA+cB+dAB``, then `dagger(z)` gives
```math
    a-bA-cB+dAB
```
This operation is an involution.
"""
function dagger(z::Multivector2{T}) where {T <: Real}
    Multivector2{T}(dagger(z.l), cloak(z.r))
end

"""
    star{T <: Real}(z::Multivector2{T})

Returns the Hodge star conjugate. If ``z=a+bA+cB+dAB``, then `star(z)` gives
```math
    d-cA+bB+aAB
```
This operation is not an involution, but `star(star(z)) = dagger(z)`.
"""
function star(z::Multivector2{T}) where {T <: Real}
    Multivector2{T}(dagger(star(z.r)), star(z.l))
end

"""
    selfstar(z::Multivector2)

The self-star-conjugate part.
This operation is idempotent.
"""
function selfstar(z::Multivector2)
    (z + star(z) + dagger(z) + star(dagger(z))) / 4
end

"""
    antiselfstar(z::Multivector2)

The anti-self-star-conjugate part.
This operation is idempotent.
"""
function antiselfstar(z::Multivector2)
    (z - star(z) + dagger(z) - star(dagger(z))) / 4
end

function (+)(x::Multivector2, y::Multivector2)
    Multivector2(x.l + y.l, x.r + y.r)
end

function (+)(x::Multivector2, y::Multivector1)
    Multivector2(x.l + y, x.r)
end

function (+)(x::Multivector1, y::Multivector2)
    Multivector2(x + y.l, y.r)
end

function (+)(z::Multivector2, a::Real)
    Multivector2(z.l + a, z.r)
end

function (+)(a::Real, z::Multivector2)
    Multivector2(z.l + a, z.r)
end

function (-)(z::Multivector2{T}) where {T <: Real}
    Multivector2{T}(-z.l, -z.r)
end

function (-)(x::Multivector2, y::Multivector2)
    Multivector2(x.l - y.l, x.r - y.r)
end

function (-)(x::Multivector2, y::Multivector1)
    Multivector2(x.l - y, x.r)
end

function (-)(x::Multivector1, y::Multivector2)
    Multivector2(x - y.l, -y.r)
end

function (-)(z::Multivector2, a::Real)
    Multivector2(z.l - a, z.r)
end

function (-)(a::Real, z::Multivector2)
    Multivector2(a - z.l, -z.r)
end

"""
    (*)(x::Multivector2, y::Multivector2)

Wedge product of 2-dimensional multivectors.
This operation is non-commutative but associative.
"""
function (*)(x::Multivector2, y::Multivector2)
    Multivector2(x.l * y.l,  (y.r * x.l) + (x.r * conj(y.l)))
end

function (*)(x::Multivector2, y::Multivector1)
    Multivector2(x.l * y, x.r * conj(y))
end

function (*)(x::Multivector1, y::Multivector2)
    Multivector2(x * y.l,  y.r * x)
end

"""
    (*)(z::Multivector2, a::Real)
    (*)(a::Real, z::Multivector2)

Scaling and/or reflection of a `Multivector2` by a real number.
"""
function (*)(z::Multivector2, a::Real)
    Multivector2(z.l * a, z.r * a)
end

function (*)(a::Real, z::Multivector2)
    Multivector2(a * z.l, a * z.r)
end

function (/)(x::Multivector2, y::Multivector2)
    if iszerodivisor(y)
        error(ZeroDivisorDenominator)
    end

    x * inv(y)
end

function (\)(y::Multivector2, x::Multivector2)
    if iszerodivisor(y)
        error(ZeroDivisorDenominator)
    end

    inv(y) * x
end

function (/)(z::Multivector2, a::Real)
    Multivector2(z.l / a, z.r / a)
end

function (\)(a::Real, z::Multivector2)
    Multivector2(a \ z.l, a \ z.r)
end

function random(::Type{Multivector2{T}}) where T <: Real
    Multivector2{T}(random(Multivector1{T}), random(Multivector1{T}))
end


"""
    eom_henon2(dx, x, p, n)

Equations of motion for a 2D Henon system consisting of two identical
Henon maps with unidirectional forcing ``X \\to Y `` [1].

## Equations of motion

The equations of motion are

```math
\\begin{aligned}
x_1(t+1) &= 1.4 - x_1^2(t) + 0.3x_2(t) \\\\
x_2(t+1) &= x_1(t) \\\\
y_1(t+1) &= 1.4 - [c_{xy} x_1(t) y_1(t) + (1-c_{xy}) y_1^2(t)] + 0.3 y_2(t) \\\\
y_2(t+1) &= y_1(t)
\\end{aligned}
```

## References

1. Krakovská, A., Jakubík, J., Chvosteková, M., Coufal, D., Jajcay, N., & Paluš, M. (2018).
    Comparison of six methods for the detection of causality in a bivariate time series.
    Physical Review E, 97(4), 042207.
"""
function eom_henon2(x, p, n)
    c_xy = p[1]
    x₁, x₂, y₁, y₂ = (x...,)
    dx₁ = 1.4 - x₁^2 + 0.3*x₂
    dx₂ = x₁
    dy₁ = 1.4 - (c_xy * x₁ * y₁  +  (1 - c_xy)*y₁^2) + 0.3*y₂
    dy₂ = y₁
    return SVector{4}(dx₁, dx₂, dy₁, dy₂)
end

function henon2(u₀, c_xy)
    p = [c_xy]
    DiscreteDynamicalSystem(eom_henon2, u₀, p)
end

"""
    henon2(;u₀ = rand(4), c_xy = 2.0) -> DiscreteDynamicalSystem

Initialize a 2D Henon system consisting of two identical Henon maps with
unidirectional forcing ``X \\to Y `` [1].

## Equations of motion

The equations of motion are

```math
\\begin{aligned}
x_1(t+1) &= 1.4 - x_1^2(t) + 0.3x_2(t) \\\\
x_2(t+1) &= x_1(t) \\\\
y_1(t+1) &= 1.4 - [c_{xy} x_1(t) y_1(t) + (1-c_{xy}) y_1^2(t)] + 0.3 y_2(t) \\\\
y_2(t+1) &= y_1(t)
\\end{aligned}
```

## References

1. Krakovská, A., Jakubík, J., Chvosteková, M., Coufal, D., Jajcay, N., & Paluš, M. (2018).
    Comparison of six methods for the detection of causality in a bivariate time series.
    Physical Review E, 97(4), 042207.
"""
henon2(;u₀ = rand(4), c_xy = 2.0) = henon2(u₀, c_xy)


using LifeContingencies
using MortalityTables
using Dates
using DataFrames

mort_table_name = "2012 IAM Period Table – Male, ANB"
improv_table = "Projection Scale G2 – Male, ANB"
iss_date = Date(2017,09,01)
val_date = Date(2017,12,31)
issue_age = 65
annual_income = 12000
deferral_period = 0
benefit_period = 120
interest_rate = 0.04

# Not used because table name provides
# sex = :M
# projection_scale = :G2
# age_rule = :ANB

mort = MortalityTables.table(mort_table_name)
imp = MortalityTables.table(improv_table)

table_end = omega(mort)

# the projection scale table only goes to age 105, wheras we want  through age 120
function imp_rate(table,age)
    if age > lastindex(table)
        return 0.0
    else
        return table[age]
    end
end

SRA = map(0:(table_end - issue_age)) do time

    att_age = issue_age + time

    if time > 1
        imp_factor = prod(map(age -> 1 - imp_rate(imp,age),issue_age:(att_age-1)))
    else
        imp_factor = 1.0
    end

    # returned values for each time

    (
        att_age = att_age,
        t=time,
        q=mort[att_age],
        imp_factor = imp_factor,
        q_imp = mort[att_age] * imp_factor,

    )
end

### A type to handle life contingent maths via the LifeContingencies package
ins = LifeContingency(
    SingleLife(
        mort = UltimateMortality([x.q_imp for x in SRA],start_age = issue_age), # mort is indexed by attained age, not starting at 1
        issue_age = issue_age
    ),
    InterestRate(interest_rate)
)

ä(ins,55) * annual_income
sum(LifeContingencies.APV.(ins,1:55) * 12000)

ℓ₁ = [43302,42854,42081,41351,40050]
ℓ₂ = [47260,47040,46755,46500,46227]

ps₁ =  ℓ₁ ./ ℓ₁[1]
qs₁ = [1 - ps₁[t] / ps₁[t - 1] for t in 2:5 ]

ps₂ =  ℓ₂ ./ ℓ₂[1]
qs₂ = [1 - ps₂[t] / ps₂[t - 1] for t in 2:5 ]

m1 = UltimateMortality(qs₁, start_age=65)
m2 = UltimateMortality(qs₂, start_age=60)

l1 = SingleLife(mort = m1, issue_age = 65)
l2 = SingleLife(mort = m2, issue_age = 60)

jl = JointLife(lives=(l1, l2), contingency = LastSurvivor(), joint_assumption=Frasier())

ins = LifeContingency(jl,InterestRate(0.05))
ins_l1 = LifeContingency(jl.lives[1],InterestRate(0.05))
ins_l2 = LifeContingency(jl.lives[2],InterestRate(0.05))
# problem 9.1.f


# Log sum exp.
function lse(arr)
    max = maximum(arr)
    max == -Inf ? -Inf : max + log(sum(exp.(arr .- max)))
end

function lse(x1::Real, x2::Real)
    m = max(x1, x2)
    m == -Inf ? m : m + log(exp(x1 - m) + exp(x2 - m))
end

# Effective sample size.
function ess(lnw::Vector{Float64})
    log_ess = -lse(2. * lnw)
    return exp(log_ess)
end

# Normalize log weights.
function nw(lw::Vector{Float64})
    lt = lse(lw)
    lnw = lw .- lt
    return (lt, lnw)
end


#=
  From the WebPPL model:https://github.com/probmods/ppaml2016/blob/gh-pages/chapters/5-data.md
  """
  Here, we explore the result of an experiment with 15 trials and binary outcomes (e.g., flipping a coin with an
  uncertain weight, asking people if they'll vote for Hillary Clinton or Donald Trump, ...)
  """
=#

using Turing
include("jl_utils.jl")

@model function bda(k=1,n=15)
    p ~ Uniform(0, 1)

    # Observed k number of successes, assuming a binomial
    k ~ Binomial(n, p)

    # sample from binomial with updated p
    posteriorPredictive ~ Binomial(n,p)

    # sample fresh p
    prior_p ~ Uniform(0,1)
    # sample from binomial with fresh p
    priorPredictive ~ Binomial(n,prior_p)

end

# data
k = 1  # number of successes
n = 15 # number of attempts

model = bda(k,n)

# chns = sample(model, MH(), 10_000)
# chns = sample(model, PG(15), 10_000)
# chns = sample(model, IS(), 10_000)
chns = sample(model, SMC(), 10_000)
# chns = sample(model, SMC(), MCMCThreads(), 10_000, 4)

display(chns)

show_var_dist_pct(chns, :priorPredictive)
show_var_dist_pct(chns, :posteriorPredictive)


"""
A recursive implementation of divided differences.
"""
mutable struct DividedDifference{T}
    memo::Dict{Tuple{Int64,Int64},T}
    DividedDifference{T}() where {T <: Real} = new(Dict{Tuple{Int64,Int64},T}())
end

"""
Calculate divided differences of a function f at points τ. This calculates the divided difference
from τ_i to τ_j. The `dd` is a memoization.
"""
function divided_difference(i::Integer, j::Integer, τ::Vector, f::Vector, dd::DividedDifference)
    if (i, j) ∈ keys(dd.memo)
        return dd.memo[(i, j)]
    end

    val = zero(Float64)
    if i == j
        val = f[i]
    elseif i < j
        val = (divided_difference(i + 1, j, τ, f, dd) - divided_difference(i, j - 1, τ, f, dd)) /
            (τ[j] - τ[i])
    else
        throw(ArgumentError("divided_difference needs i <= j, found i = {i}, j = {j}"))
    end
    dd.memo[(i, j)] = val
    val
end

function divided_difference_explicit(
    i::Integer, k::Integer, τ::Vector, f::Vector{T}) where {T <: Real}

    total = zero(T)
    for j = i:k
        denom = one(T)
        for l = i:k
            if l != j
                denom *= τ[j] -τ[l]
            end
        end
        total += f[j] / denom
    end
    total
end


# using JuMP, VariationalInequality
# using Base.Test

using JuMP, VariationalInequality
# include("../src/model.jl")
# include("../src/algorithms.jl")

# https://cdr.lib.unc.edu/indexablecontent/uuid:778ca632-74ca-4858-8c3c-6dcfc7e6e703
# Example 4.2. This example is adapted from the example in (Fukushima, 1986). Let
m = VIPModel()
@variable(m, x[1:3])

@NLconstraint(m, x[1]^2 + 0.4x[2]^2 + 0.6x[3]^2 <= 1)
@NLconstraint(m, 0.6x[1]^2 + 0.4x[2]^2 + x[3]^2 <= 1)
@NLconstraint(m, x[1] + x[2] + x[3] >= sqrt(3))

@NLexpression(m, F1, 2x[1] + 0.2x[1]^3 - 0.5x[2] + 0.1x[3] - 4)
@NLexpression(m, F2, -0.5x[1] + x[2] + 0.1x[2]^3 + 0.5)
@NLexpression(m, F3, 0.5x[1] - 0.2x[2] + 2x[3] - 0.5)

correspond(m, F1, x[1])
correspond(m, F2, x[2])
correspond(m, F3, x[3])
# or
# correspond(m, [F1, F2, F3], x)

solveVIP(m, algorithm=:fixed_point, max_iter=1000, step_size=0.1)
sol1, Fval1, gap1 = saveSolution(m)
@assert 0<= gap1 < 1e-6

# The above `solveVIP` sets the value of variables at the solution
clearValues(m)

solveVIP(m, algorithm=:extra_gradient, max_iter=1000, step_size=0.1)
sol2, Fval2, gap2 = saveSolution(m)
@assert 0<= gap2 < 1e-6

# sol2, Fval2, gap2 = solveVIP(m, algorithm=:hyperplane, max_iter=1000, step_size=0.1)

@show sol1
@show sol2
@show gap1
@show gap2

# OKAY.
# x∗ = (0.9168, 0.4850, 0.3303)


factorial_ij(i::Integer, j::Integer) = factorial(big(min(i,j))) / factorial(big(max(i,j)))

z(x::Real, p::Real) = sqrt(2.)*(x + p*im)
z(x::Vector{<:Real}, p::Vector{<:Real}) = z.(x, p')

α(m::Integer, n::Integer) = n - m
α(m::Vector{<:Integer}, n::Vector{<:Integer}) = α.(min.(m, n'), max.(m, n'))

gaussian_function(x::Real, p::Real) = exp(-0.5 * abs2(z(x,p))) / π
gaussian_function(x::Vector{<:Real}, p::Vector{<:Real}) = gaussian_function.(x,p')

neg_one_to_power_of(i::Integer) = (i % 2 == 0) ? 1 : -1

function coefficient_of_wave_function(m::Integer, n::Integer)
    if n ≥ m
        return neg_one_to_power_of(m) * sqrt(factorial_ij(m, n))
    else
        return neg_one_to_power_of(n) * sqrt(factorial_ij(m, n))
    end
end

coefficient_of_wave_function(m::Vector{<:Integer}, n::Vector{<:Integer}) = coefficient_of_wave_function.(m, n')

function z_to_power(m::Integer, n::Integer, x::Real, p::Real)
    if n ≥ m
        return conj(z(x, p'))^(n - m)
    else
        return z(x, p')^(m - n)
    end
end

function z_to_power(m::Vector{<:Integer}, n::Vector{<:Integer}, x::Vector{<:Real}, p::Vector{<:Real})
    x = reshape(x, 1, 1, length(x))
    p = reshape(p, 1, 1, 1, length(p))
    z_to_power.(m, n', x, p)
end

function z_to_power(m::Vector{<:Integer}, n::Vector{<:Integer})
    function z_to_power_xp(x::Vector{<:Real}, p::Vector{<:Real})
        x = reshape(x, 1, 1, length(x))
        p = reshape(p, 1, 1, 1, length(p))
        z_to_power.(m, n', x, p)
    end
    return z_to_power_xp
end

function z_to_power(x::Vector{<:Real}, p::Vector{<:Real})
    x = reshape(x, 1, 1, length(x))
    p = reshape(p, 1, 1, 1, length(p))
    function z_to_power_mn(m::Vector{<:Integer}, n::Vector{<:Integer})
        z_to_power.(m, n', x, p)
    end
    return z_to_power_mn
end


#############################################################################
# JuMP
# An algebraic modelling langauge for Julia
# See http://github.com/JuliaOpt/JuMP.jl
#############################################################################
# minellipse.jl
#
# This example is from the Boyd & Vandenberghe book "Convex Optimization".
# If PyPlot is installed, passing the argument `plot` will plot
# the solution, e.g. julia minellipse.jl plot
#
# Given a set of ellipses centered on the origin
# E(A) = { u | u^T inv(A) u <= 1 }
# find a "minimal" ellipse that contains the provided ellipses
#
# We can formulate this as an SDP:
#     minimize  trace(WX)
#   subject to  X >= A_i,    i = 1,...,m
#               X PSD
# where W is a PD matrix of weights to choose between different solutions
#############################################################################

using JuMP
using Mosek
using PyPlot  # Comment out if not installed

# We will use three ellipses
m = 3
# Two "simple" ones
As = Any[ [2.0  0.0;
           0.0  1.0],
          [1.0  0.0;
           0.0  3.0]]
# and a random one
randA = rand(2,2)
push!(As, (randA' * randA) * (rand()*2+1))

# We change the weights to see different solutions, if they exist
W = [1.0 0.0
     0.0 1.0];

mod = Model()
@variable(mod, X[1:2,1:2], SDP)
@objective(mod, Min, trace(W*X))
for i = 1:m
    @SDconstraint(mod, X >= As[i])
end
solve(mod)

X_val = getvalue(X)
println(X_val)

# Plot it, if desired (e.g. julia minellipse.jl plot)
if length(ARGS) > 0 && Pkg.installed("PyPlot") !== nothing
    # Setup the figure
    fig = figure()
    ax = fig[:gca]()
    ax[:set_xticks]([-4:+4])
    ax[:set_yticks]([-4:+4])

    # Draw provided ellipses
    for i = 1:m
        xs = Float64[]
        ys = Float64[]
        for angle in linspace(0, 2*pi, 100)
            u = [cos(angle),sin(angle)]
            x = As[i] * u
            push!(xs, x[1])
            push!(ys, x[2])
        end
        plot(xs, ys, "b", linewidth=2.0)
    end

    # Draw bounding ellipse
    xs = Float64[]
    ys = Float64[]
    for angle in linspace(0, 2*pi, 100)
        u = [cos(angle),sin(angle)]
        x = X_val * u
        push!(xs, x[1])
        push!(ys, x[2])
    end
    plot(xs, ys, "r", linewidth=2.0)

    fig[:canvas][:draw]()
    readline()
end


# Examples borrowed from: http://arxiv.org/pdf/1211.3759.pdf

# 2/3 N(x|0,1) + 1/3 N(x|0,1/10)
function kurtotic(theta)
    if theta[3] <= 0 || theta[6] <= 0
        return -Inf
    end
    llk = 0
    for i in 1:length(y)
      llk += log(theta[1] * pdf(Normal(theta[2], theta[3]), y[i]) +
                 theta[4] * pdf(Normal(theta[5], theta[6]), y[i]))
    end
    return llk
end

function bimodal_density(x)
    lik = (1/2) * pdf(Normal(-1,2/3),x) + (1/2) * pdf(Normal(1,2/3),x)
    return log(lik)
end

# 1/2 N(x|-1,2/3) + 1/2 N(x|1,2/3)
function bimodal_loglikelihood(theta)
    if theta[3] < 0 || theta[6] < 0
        return -Inf
    end
    llk = 0
    for i in 1:length(y)
      llk += log(theta[1] * pdf(Normal(theta[2], theta[3]), y[i]) +
                 theta[4] * pdf(Normal(theta[5], theta[6]), y[i]))
    end
    return -llk
end

# requires y and x
function logistic_regression(theta)
    llk = 0
    eta = [1 / (1 + exp(- x[i,:] * theta))[1] for i in 1:length(y)]
    for i = 1:length(y)
        llk += y[i] * log(eta[i]) + (1-y[i]) * log(1-eta[i])
    end
    return llk
end


using FastTransforms, LinearAlgebra

function oprec!(n::Integer, v::AbstractVector, alpha::Real, delta2::Real)
    if n > 0
        v[1] = 1
    end
    if n > 1
        v[2] = (4*alpha+8-(alpha+4)*delta2)/4
    end
    for i = 1:n-2
        v[i+2] = (((2*i+alpha+2)*(2*i+alpha+4)+alpha*(alpha+2))/(2*(i+1)*(2*i+alpha+2))*(2*i+alpha+3)/(i+alpha+3) - delta2/4*(2*i+alpha+3)/(i+1)*(2*i+alpha+4)/(i+alpha+3))*v[i+1] - (i+alpha+1)/(i+alpha+3)*(2*i+alpha+4)/(2*i+alpha+2)*v[i]
    end
    return v
end

"""
This example calculates the spectrum of the nonlocal diffusion operator:

```math
ℒ_δ u = ∫_𝕊² ρ_δ(|𝐱-𝐲|)[u(𝐱) - u(𝐲)] dΩ(𝐲),
```

defined in Eq. (2) of

    R. M. Slevinsky, H. Montanelli, and Q. Du, A spectral method for nonlocal diffusion operators on the sphere, J. Comp. Phys., 372:893--911, 2018.

available at https://doi.org/10.1016/j.jcp.2018.06.024
"""
function evaluate_lambda(n::Integer, alpha::T, delta::T) where T
    delta2 = delta*delta
    scl = (1+alpha)*(2-delta2/2)

    lambda = Vector{T}(undef, n)

    if n > 0
        lambda[1] = 0
    end
    if n > 1
        lambda[2] = -2
    end

    oprec!(n-2, view(lambda, 3:n), alpha, delta2)

    for i = 2:n-1
        lambda[i+1] *= -scl/(i-1)
    end

    p = plan_jac2jac(T, n-1, zero(T), zero(T), alpha, zero(T))

    lambda[2:end] .= p'lambda[2:end]

    for i = 2:n-1
        lambda[i+1] = ((2i-1)*lambda[i+1] + (i-1)*lambda[i])/i
    end

    for i = 2:n-1
        lambda[i+1] += lambda[i]
    end

    return lambda
end

lambda = evaluate_lambda(1024, -0.5, 1.0)
lambdabf = evaluate_lambda(1024, parse(BigFloat, "-0.5"), parse(BigFloat, "1.0"))

norm(lambda-lambdabf, Inf)/norm(lambda, Inf)


using DifferentialEquations, Flux, DiffEqFlux
using Plots

u0 = [1.0,1.0]
tspan = (0.0,10.0)
tsteps = 0.0:0.1:10.0

my_nn = Chain(
  Dense(2, 32, relu),
  Dense(32, 2)) |> f64

p, re = Flux.destructure(my_nn)

function right_hand_side(du,u,p,t)
  m = re(p)
  nn_output = m(u)
  du[1] = nn_output[1]
  du[2] = nn_output[2]
end

prob = ODEProblem(right_hand_side,u0,tspan,p)
sol = solve(prob, Tsit5())

# Plot the solution
plot(sol)
savefig("Initial_NN_ODE.png")

function predict_ode(p)
  tmp_prob = remake(prob, p = p)
  Array(solve(tmp_prob, Tsit5(), saveat = tsteps))
end

function loss_ode(p)
    pred = predict_ode(p)
    loss = sum(abs2, 1.0 .- pred)
    return loss, pred
end

callback = function (p, l, pred)
  display(l)
  plt = plot(pred, ylim = (0, 6))
  display(plt)
  # Tell sciml_train to not halt the optimization. If return true, then
  # optimization stops.
  return false
end

result_ode = DiffEqFlux.sciml_train(loss_ode, p,
                                    cb = callback,
                                    maxiters = 100)

prob_final = ODEProblem(right_hand_side, u0, tspan, result_ode)
sol_final = solve(prob_final,Tsit5())
plot(sol_final)
savefig("NN_ode_optimized.png") # Both state components close to 1


"""
# Algorithms with numbers

- Basic arithmetic
    - Addition
    - Bases and logs
    - Multiplication and division
- Modular arithmetic
    - Substitution rule
    - Modular addition and multiplication
    - Two's complement
    - Modular exponentiation
    - Euclid's algorithm for greatest common divisor
    - Euclid's rule
    - An extension of Euclid's algorithm
    - Modular division
    - Is your social security number a prime?
- Primality testing
    - Fermat's little theorem
    - Hey, that was group theory!
    - Generating random primes
    - Lagrange's prime number theorem
    - Carmichael numbers
    - Randomized algorithms: a virtual chapter
- Cryptography
    - An application of number theory?
    - Private-key schemes: one-time pad and AES
    - RSA
- Universal Hashing
    - Hash tables
    - Families of hash functions
"""
module Chapter1

export
    multiply,
    divide,
    modexp,
    euclid,
    extendedeuclid,
    primality,
    primality2

"""
    multiply(x, y)

Compute the product of two integers `x` and `y`.

# Examples
```jldoctest
julia> multiply(7, 8)
56
```
"""
function multiply(x::Integer, y::Integer)
    if y == 0
        return 0
    end

    z = multiply(x, y ÷ 2)
    if iseven(y)
        return 2z
    else
        return x + 2z
    end
end

"""
    divide(x, y)

Compute the quotient and remainder of `x ÷ y`.

# Examples
```jldoctest
julia> divide(5, 3)
(1, 2)
```
"""
function divide(x::Integer, y::Integer)
    if x == 0
        return (q,r) = (0,0)
    end

    (q,r) = divide(x ÷ 2, y)
    q = 2q
    r = 2r
    if isodd(x)
        r = r + 1
    end
    if r >= y
        r = r - y
        q = q + 1
    end
    return (q,r)
end

"""
    modexp(x, y, N)

Compute the modular exponentation `x^y mod N`.

# Examples
```jldoctest
julia> modexp(2, 3, 5)
3
```
"""
function modexp(x::Integer, y::Integer, N::Integer)
    if y == 0
        return 1
    end

    z = modexp(x, y ÷ 2, N)
    if iseven(y)
        return mod(z^2, N)
    else
        return mod(x * z^2, N)
    end
end

"""
    euclid(a, b)

Using Euclid's algorithm, compute the greatest common divisor
between two integers `a` and `b` where `a >= b >= 0`.

# Examples
```jldoctest
julia> euclid(25, 20)
5
```
"""
function euclid(a::Integer, b::Integer)
    @assert a >= b >= 0
    if b == 0
        return a
    end

    return euclid(b, mod(a, b))
end

"""
    extendedeuclid(a, b)

A simple extension of Euclid's algorithm.

- Input: Two positive integers `a` and `b` with `a >= b >= 0`.
- Ouput: Integers `x`, `y`, `d` such that `d = gcd(a,b)` and `ax + by = d`.

# Examples
```jldoctest
julia> extendedeuclid(9, 6)
(1, -1, 3)
```
"""
function extendedeuclid(a::Integer, b::Integer)
    @assert a >= b && b >= 0
    if b == 0
        return (1, 0, a)
    end

    (x′, y′, d) = extendedeuclid(b, mod(a, b))
    return (y′, x′ - a ÷ b * y′, d)
end

"""
    isprime(N)

Determine whether an integer `N` is prime using Fermat's little theorem.

# Examples
```jldoctest
julia> primality(5)
true
```
"""
function primality(N::Integer)
    if N < 2
        return false
    end

    a = rand(1:(N - 1))
    return powermod(a, N - 1, N) == 1
end

"""
    primality2(N)

Determine whether an integer `N` is prime with low error probability.
"""
function primality2(N::Integer)
    if N < 2
        return false
    end

    a = rand(1:(N - 1), N - 1)
    if all([powermod(a[i], N - 1, N) == 1 for i ∈ 1:(N - 1)])
        return true
    else
        return false
    end
end

end # module


using Distributions, StatsBase, Random
Random.seed!(1)

dist1 = TriangularDist(0,10,5)
dist2 = DiscreteUniform(1,5)
theorMean1, theorMean2 = mean(dist1), mean(dist2)

N = 10^6
data1 = rand(dist1,N)
data2 = rand(dist2,N)
estMean1, estMean2 = mean(data1), mean(data2)

println("Symmetric Triangular Distiribution on [0,10] has mean $theorMean1
	(estimated: $estMean1)")
println("Discrete Uniform Distiribution on {1,2,3,4,5} has mean $theorMean2
	(estimated: $estMean2)")


"""
    MeanSquareError(NeuralNetwork)

Sum of square error loss function.

    Usage:
```julia-repl
MeanSquareError(NeuralNetwork)
```
Arguments:
- `NeuralNetwork` : The NeuralNetwork structure.
"""
function MeanSquareError(NeuralNetwork)
	ΣSE = 0.0

	for i in 1:size(NeuralNetwork.y, 1)
		ΣSE += (NeuralNetwork.y[i] - NeuralNetwork.output[i]) ^ 2.0
    end

    loss = 1.0 / size(NeuralNetwork.y) * ΣSE
    return loss
end


# Generates an AR(p) process with coefficients `ϕ`.
# `ϕ` should be provided as a vector and it represents the coefficients of the AR model.
# Hence the order of the generated process is equal to the length of `ϕ`.
# `s` indicates the total length of the series to be generated.
function generate_process(ϕ::AbstractVector{Float32}, s::Int)
    @assert s > 0 "s must be positive"
    # Generate white noise
    ϵ = randn(Float32, s)
    # Initialize time series
    X = zeros(Float32, s)
    p = length(ϕ)
    X[1] = ϵ[1]
    # Reverse the order of the coefficients for multiplication later on
    ϕ = reverse(ϕ)
    # Fill first p observations
    for t ∈ 1:p-1
        X[t+1] = X[1:t]'ϕ[1:t] + ϵ[t+1]
    end
    # Compute values iteratively
    for t ∈ p+1:s
        X[t] = X[t-p:t-1]'ϕ + ϵ[t]
    end
    X
end

# Create batches of a time series `X` by splitting the series into
# sequences of length `s`. Each new sequence is shifted by `r` steps.
# When s == r,  the series is split into non-overlapping batches.
function batch_timeseries(X, s::Int, r::Int)
    @assert r > 0 "r must be positive"
    # If X is passed in format T×1, reshape it
    if isa(X, AbstractVector)
        X = permutedims(X)
    end
    T = size(X, 2)
    @assert s ≤ T "s cannot be longer than the total series"
    # Ensure uniform sequence lengths by dropping the first observations until
    # the total sequence length matches a multiple of the batchsize
    X = X[:, ((T - s) % r)+1:end]
    [X[:, t:r:end-s+t] for t ∈ 1:s] # Output
end


using Catlab, Catlab.Theories
using Catlab.CategoricalAlgebra
using Catlab.Graphs
using Catlab.Graphics
using Catlab.Graphics.Graphviz
#import Catlab.Graphics.Graphviz: to_graphviz, to_graphviz_property_graph

using Colors
draw(g) = to_graphviz(g, node_labels=true, edge_labels=true)

GraphvizGraphs.to_graphviz(f::ACSetTransformation; kw...) =
  to_graphviz(GraphvizGraphs.to_graphviz_property_graph(f; kw...))

function GraphvizGraphs.to_graphviz_property_graph(f::ACSetTransformation; kw...)
  pg = GraphvizGraphs.to_graphviz_property_graph(dom(f); kw...)
  vcolors = hex.(range(colorant"#0021A5", stop=colorant"#FA4616", length=nparts(codom(f), :V)))
  ecolors = hex.(range(colorant"#6C9AC3", stop=colorant"#E28F41", length=nparts(codom(f), :E)))
  hex.(colormap("Oranges", nparts(codom(f), :V)))
  for v in vertices(dom(f))
    fv = f[:V](v)
    set_vprops!(pg, v, Dict(:color => "#$(vcolors[fv])"))
  end
  for e in edges(dom(f))
    fe = f[:E](e)
    set_eprops!(pg, e, Dict(:color => "#$(ecolors[fe])"))
  end
  pg
end

to_graphviz(Catlab.Graphs.BasicGraphs.TheoryGraph)

# # The Category of Graphs
#
# The Theory of Graphs is given by the following Schema:
# ```julia
# @present TheoryGraph(FreeSchema) begin
#   V::Ob
#   E::Ob
#   src::Hom(E,V)
#   tgt::Hom(E,V)
# end

# """ Abstract type for graphs, aka directed multigraphs.
# """
# @abstract_acset_type AbstractGraph <: HasGraph

# """ A graph, also known as a directed multigraph.
# """
# @acset_type Graph(TheoryGraph, index=[:src,:tgt]) <: AbstractGraph
# ```
#
# That is all we need to do to generate the functor category [TheoryGraph, FinSet].
# Catlab knows how to take a finitely presented category and generate all the data structures
# that you need to represent functors into FinSet and natural transformations between those functors.
# Note: the index=[:src, :tgt] keyword argument tells Catlab that you want to have an efficient index
# the preimages of those morphisms. in this example, we want to be able to find the incoming and
# outgoing edges of a vertex in O(1) time.

# ## Creating some Graphs

# Once you have fixed the schema (aka indexing category or theory), you can make some instances.
# Catlab has a DSL for specifying instances of any schema. It is called `@acset`.
# In order to specify a Functor F=(F₀, F₁) into FinSet, you need to provide some data.
#     1. For every A:Ob(C), you need F₀(A):FinSet
#     2. For every f:A→B, you need to specify a FinFunction F₁(f):F₀(A)→F₀(B)
# If the theory C has some equations, the data you provide would have to also satisfy those equations.
# The theory of graphs has no equations, so there are no constraints on the data you provide,
# except for those that come from functoriality.

e = @acset Graphs.Graph begin
    V = 2
    E = 1
    src = [1]
    tgt = [2]
end

draw(e)

# a wedge is two edges that share a target
w = @acset Graphs.Graph begin
    V = 3
    E = 2
    src=[1,3]
    tgt=[2,2]
end

draw(w)

# The CSet API generalizes the traditional Graph API

parts(w, :V)  # vertex set

parts(w,:E) # edge set

w[:src] # source map

w[:tgt] # target map

incident(w, 1, :src) # edges out of vertex 1

incident(w, 2, :tgt) # edges into vertex 2

w[incident(w, 2, :tgt), :src] # vertices that are the source of edges whose target is vertex 2

w[incident(w, 1, :src), :tgt] # vertices that are the target of edges whose src is vertex 1

# ### Exercise:
# 1. Use the @acset macro to make a graph with at least 5 vertices
# 2. Draw the graph
# 3. Compute in neighbors and out neighbors and make sure they match your expectations.
# 4. Write a function that computes the 2-hop out-neighbors of a vertex.

# # Graph Homomorphisms
# We can construct some graph homomorphisms between our two graphs.
# What data do we need to specify?

ϕ = ACSetTransformation(e,w,E=[1], V=[1,2])

is_natural(ϕ)

# The ACSetTransformation constructor does not automatically validate that the naturality squares commute!

ϕᵦ = ACSetTransformation(e,w,E=[1], V=[3,2])
is_natural(ϕᵦ)

# Our ϕᵦ in't natural because the edge map e₁ ↦ e₁ is not consistent with our vertex map, which sends v₁ ↦ v₃ and v₂ ↦ v₂. We can fix this by sending e₁ to e₂

ϕᵦ′ = ACSetTransformation(e,w,E=[2], V=[3,2])
is_natural(ϕᵦ′)

# So how does Catlab store the data of the natural transformation?
# the domain
ϕ.dom
# the codomain
ϕ.codom
# the components
ϕ.components
# you can see the components using standard indexing notation with the object names. Notice that while CSets are indexed by morphisms, natural transformations are indexed by objects.
ϕ[:V]
ϕ[:E]
# We can check the  naturality squares ourselves
# The sources are preserved: `src ⋅ ϕᵥ == ϕₑ ⋅ src`
ϕ[:V](dom(ϕ)[:,:src]) == codom(ϕ)[collect(ϕ[:E]), :src]
# The targets are preserved: `tgt ⋅ ϕᵥ == ϕₑ ⋅ tgt`
ϕ[:V](dom(ϕ)[:,:tgt]) == codom(ϕ)[collect(ϕ[:E]), :tgt]

# This approach generalizes to the following:
#
# ```julia
# function is_natural(α::ACSetTransformation{S}) where {S}
#    X, Y = dom(α), codom(α)
#    for (f, c, d) in zip(hom(S), dom(S), codom(S))
#      Xf, Yf, α_c, α_d = subpart(X,f), subpart(Y,f), α[c], α[d]
#      all(Yf[α_c(i)] == α_d(Xf[i]) for i in eachindex(Xf)) || return false
#    end
#    return true
# end
# ```
#
# Notice how we iterate over the homs in the schema category S `(f, c, d) in zip(hom(S), dom(S), codom(S))` We get one universally quantified equation `all(Yf[α_c(i)] == α_d(Xf[i]) for i in eachindex(Xf))` for each morphism in the indexing category
#
# ### Exercise:
# 1. Take your graph from the previous exercise and construct a graph homomorphism from the wedge (w) into it.
# 2. Check that the naturality equations are satisfied.
# 3. Explain why we don't need to specify any data for the source and target morphisms in TheoryGraph when definining a graph homomorphism

# ## Finding Homomorphisms Automatically
# As you saw in the previous exercise, constructing a natural transformation can be quite tedious. We want computers to automate tedious things for us. So we use an algorithm to enumerate all the homomorphisms between two CSets.

# CSet homomorphisms f:A→B are ways of finding a part of B that is shaped like A. You can view this as pattern matching. The graph A is the pattern and the graph B is the data. A morphism f:A→B is a collection of vertices and edges in B that is shaped like A. Note that you can ask Catlab to enforce constraints on the homomorphisms it will find including computing monic (injective) morphisms by passing the keyword `monic=true`. A monic morphism into B is a subobject of B.  You can pass `iso=true` to get isomorphisms.

t = @acset Graphs.Graph begin
    V = 3
    E = 3
    src = [1,2,1]
    tgt = [2,3,3]
end

draw(t)

T = @acset Graphs.Graph begin
    V = 6
    E = 9
    src = [1,2,1, 3, 1,5,2,2,4]
    tgt = [2,3,3, 4, 4,6,5,6,6]
end

draw(T)

# The simplest pattern in a graph is just a single edge and each homomorphism ϕ:e→G is a single edge in G.
length(homomorphisms(e, T, monic=true)) == nparts(T,:E) # number of edges

length(homomorphisms(t, T))

# We can define this helper function to print out all the homomorphisms between graphs. Because all our graphs are simple, we only need to look at the vertex components.
graphhoms(g,h) = begin
    map(homomorphisms(g,h)) do ϕ
        collect(ϕ[:V])
    end
end

graphhoms(t, T)

# Homs ϕ:t→T are little triangles in T, but homs ϕ:T→t are colorings of T with 3 colors. The fact that there are edges in t that are missing, means that it provides constraints on what graphs have morphisms into it. For example, there are no morphisms T→t.

graphhoms(T, t)

# The reason we don't have a morphism into t is vertices 2,3,4,5 aren't arranged into a triangle. We can relax those constraints by adding loops to the codomain. Loops in the codomain allow you to merge adjacent vertices when you construct the homomorphism.

add_loops!(g) = add_parts!(g, :E, nparts(g,:V), src=parts(g,:V), tgt=parts(g,:V))
add_loops(g) = begin
    h = copy(g)
    add_loops!(h)
    return h
end

draw(add_loops(t))

# Once we add loops, then we have morphisms.

length(homomorphisms(T,add_loops(t)))

# ## Bipartite Graphs
# Many computer science problems involve graphs that have two types of vertices. For example, when matching students to classes, you might represent the students as one type of vertex and the classes as another type of vertex. Then the edges (s,c) represent "student s is enrolled in class c". In this scenario there can be no edges from a class vertex to another class vertex, or from a student vertex to a student vertex. Graphs for which there exists such a classification are called bipartite graphs. In Category Theory, we shift from thinking about graphs with properties to graph homomorphisms that witness that property and think of bipartitioned graphs.

# First we construct a bipartite graph:
sq = apex(product(add_loops(e), add_loops(e)))
rem_parts!(sq, :E, [1,5,6,8,9])
draw(sq)

# We will use the symmetric edge graph to identify the bipartitions of this graph.
esym = @acset Graphs.Graph begin
    V = 2
    E = 2
    src = [1,2]
    tgt = [2,1]
end

draw(id(esym))
# There are two ways to bipartition sq.
graphhoms(sq, esym)

# This comes from the fact that esym has 2 automorphisms!
graphhoms(esym, esym)

# the first coloring
draw(homomorphisms(sq, esym)[1])

# but we can also swap the roles of the colors
draw(homomorphisms(sq, esym)[2])

# ### Exercise:
# 1. Construct a graph representation of a checkerboard
# 2. Draw the two bipartitions of the checkerboard

# We can generalize the notion of Bipartite graph to any number of parts. I like to call Kₖ the k-coloring classifier because homomorphims into α:G → Kₖ are k-colorings of G.

clique(k::Int) = begin
    Kₖ = Graphs.Graph(k)
    for i in 1:k
        for j in 1:k
            if j ≠ i
                add_parts!(Kₖ, :E, 1, src=i, tgt=j)
            end
        end
    end
    return Kₖ
end

K₃ = clique(3)
draw(id(K₃))

# Our graph T is not 2-colorable,
length(homomorphisms(T, esym))

# but we can use 3 colors to color T.
draw(homomorphism(T, K₃))

# ### Exercise:
# 1. Find a graph that is not 3-colorable
# 2. Find a graph that is not 4-colorable

# ## Homomorphisms in [C, Set] are like Types
# Any graph can play the role of the codomain. If you pick a graph that is incomplete, you get a more constrained notion of coloring where there are color combinations that are forbidden.
triloop = @acset Graphs.Graph begin
    V = 3
    E = 3
    src = [1,2,3]
    tgt = [2,3,1]
end

draw(id(triloop))

# With this graph, we can pick out directed 3-cycles in a graph like T2,
T2 = @acset Graphs.Graph begin
    V = 6
    E = 6
    src = [1,2,3,4,5,6]
    tgt = [2,3,1,5,6,4]
end
graphhoms(T2, triloop)

# and we can draw those cyclic roles with colors
draw(homomorphisms(T2, triloop)[1])

T3 = @acset Graphs.Graph begin
    V = 6
    E = 7
    src = [1,2,3,4,5,6, 2]
    tgt = [2,3,1,5,6,4, 4]
end
graphhoms(T3, triloop)

# Using the colors as shown:
draw(id(triloop))

# We can see our coloring of T3:
draw(homomorphisms(T3, triloop)[1])


"""Calculte the Continuous Ranked Probability Score (CRPS) of an ensemble array wrt
a true observation o. In-place version that will sort the array."""
function CRPS!(x::Array{T,1},o::T) where T
    sort!(x)
    n = length(x)
    dp = 1/n                                # probability increment per xi in x

    # find index m to split integration over x into heaviside=0 or 1
    m = findfirst(xi -> xi >= o,x)

    # integration from o to x if o is outside of the range of x
    s = m==1 ? x[1]-o : 0.0

    # integrate over the heaviside=0 part
    for i in 1:(m == nothing ? n-1 : m-1)
        s += (i*dp)^2*(x[i+1]-x[i])
    end

    # integrate over the heaviside=1 part
    for i in (m == nothing ? n : m):n-1
        s += (i*dp-1)^2*(x[i+1]-x[i])
    end

    # integration from x to o if o is outside of the range of x
    s += (m == nothing) ? o-x[end] : 0.0

    return s
end

"""Calculte the Continuous Ranked Probability Score (CRPS) of an ensemble array wrt
a true observation o."""
CRPS(x::Array{T,1},o::T) where T = CRPS!(copy(x),o)
CRPS(x::Array{T,1},o::Real) where T = CRPS!(copy(x),T(o))


using Memoize, IterTools, LinearAlgebra

@memoize function mem_det(A::Array{Int,2})
    return round(Int, det(A))
end

"""
`istum(A)` determines if the integer matrix `A`
is totally unimodular. This may be called with an
optional second argument `istum(A,true)` that,
in case the matrix is not totally unimodular, will
report the first submatrix found whose determinant
is not in `{-1,0,1}`.

**Warning**: This is not an efficient function.
"""
function istum(A::Array{Int,2}, verbose::Bool = false)
    r, c = size(A)
    n = min(r, c)

    for k = 1:n
        rows = subsets(collect(1:r), k)
        cols = subsets(collect(1:c), k)
        for x in rows
            for y in cols
                B = A[x, y]
                d = mem_det(B)
                if abs(d) > 1
                    if verbose
                        println("Rows: ", x)
                        println("Cols: ", y)
                        println("$B has determinant $d")
                    end
                    return false
                end
            end
        end
    end
    return true
end


#SIR-X Model

using DifferentialEquations

beta = 0.215; gamma = 0.07; N = 7000;
eta0 = 0.003; eta = 0.003

"""SIR-X Model
Extension of the basic SIR model

"""
function SIR_X!(du,u,p,t)
    du[1] = -beta* u[1]/N*u[2] - eta0*u[1]                          #S
    du[2] = beta*u[1]/N*u[2] - gamma*u[2] - eta0*u[2] - eta*u[2]    #I
    du[3] = gamma*u[2] + eta0*u[1]                                  #R
    du[4] = (eta0 + eta)*u[2]                                       #X
end

   u0 = [N-15.0;15.0;0.0;0.0]
   tspan = (0.0,100.0)
   prob = ODEProblem(SIR_X!,u0,tspan)
   sol = solve(prob)

   plot(sol)


# Functions related to the counting bound.

module CountingBound

export countingBound, approxNumMaximalCliques1

"""
Counting bound, based on Shannon's argument.
  m: number of edges
  w: number of 'wires' -- that is, log2(number of functions),
    which is the number of bits needed to specify a function
  Returns: average number of NAND gates (with unbounded fan-in)
    required to compute any of those functions.
    (This may not be an integer).
"""
function countingBound(m, w)
  m = BigFloat(m)
  w = BigFloat(w)
  b = m - 0.5
  # the "-1" here is because this is the average, not the max.
  sqrt(2*w + b*b) - b - 1
end

"""
  Approximate number of maximal hypercliques of some size.
  Note that k < r < n .
  Also, the precision of what's returned can be set by setprecision().
  k: number of vertices per hyperedge
  r: number of vertices in the clique
  n: vertices in the larger graph
  Returns: expected number of maximal hypercliques. (This is
		approximate, but presumably it's more accurate for larger
    numbers).
"""
function approxNumMaximalCliques1(k, r, n)
  k = BigInt(k)
  r = BigInt(r)
  n = BigInt(n)
  one = BigInt(1)
  two = BigInt(2)

  # probability that one of those is not covered by a larger clique
  a = one << binomial(r, k-one)
  # print("computed a\n")
  pNumerator = (a-one) ^ (n-r)
  # print("computed numerator\n")
  pDenominator = a ^ (n-r)
  # print("computed denominator\n")

  # expected number of r-cliques should be equivalent to:
  # numRCliques = Rational(binomial(n, r), two ^ binomial(r, k))
  # # result is number of cliques, * prob. they're maximal
  # numRCliques * (pNumerator / pDenominator)
  r = (pNumerator * binomial(n, r)) /
    (pDenominator * (one << binomial(r, k)))
  # ??? is this off by two? I don't think so.
  r
end

"""
  Approximate number of maximal hypercliques of some size
(alternate take).
  Note that k < r < n .
  Also, the precision of what's returned can be set by setprecision().
  k: number of vertices per hyperedge
  r: number of vertices in the clique
  n: vertices in the larger graph
  Returns: number of maximal hypercliques. This is approximate, because
    it's the expected number. (Presumably it's more accurate for larger
    numbers).
"""
function approxNumMaximalCliques2(k, r, n)
  k = BigInt(k)
  r = BigInt(r)
  n = BigInt(n)
  one = BigInt(1)
  two = BigInt(2)

  # probability that one of those is not covered by a larger clique
  a = one << binomial(r, k-one)
  print("computed a\n")
  pNumerator = (a-one) ^ (n-r)
	# ??? how is this implemented for BigInts?
	# also, if a = 1111111... in binary, is there a cheaper way to
	# compute this?
  print("computed numerator\n")
  pDenominator = a ^ (n-r)
	# FIXME is denominator all powers of two? If so, presumably could
	# replace the division with a bit shift
  print("computed denominator\n")

  # expected number of r-cliques should be equivalent to:
  # numRCliques = Rational(binomial(n, r), two ^ binomial(r, k))
  # # result is number of cliques, * prob. they're maximal
  # numRCliques * (pNumerator / pDenominator)
  r = (pNumerator * binomial(n, r)) /
    (pDenominator * (one << binomial(r, k)))

  # ??? is this off by two? (doesn't seem to be, for small k, r, n)
  r
end

end


### Edge and Gradient Related Image Operations ###

# Phase (angle of steepest gradient ascent), calculated from X and Y gradient images
"""
    phase(grad_x, grad_y) -> p

Calculate the rotation angle of the gradient given by `grad_x` and
`grad_y`. Equivalent to `atan(-grad_y, grad_x)`, except that when both `grad_x` and
`grad_y` are effectively zero, the corresponding angle is set to zero.
"""
function phase(grad_x::T, grad_y::T, tol=sqrt(eps(T))) where T<:Number
    atan(-grad_y, grad_x) * ((abs(grad_x) > tol) | (abs(grad_y) > tol))
end
phase(grad_x::Number,   grad_y::Number)   = phase(promote(grad_x, grad_y)...)
phase(grad_x::NumberLike, grad_y::NumberLike) = phase(gray(grad_x), gray(grad_y))

phase(grad_x::AbstractRGB, grad_y::AbstractRGB) = phase(vecsum(grad_x), vecsum(grad_y))

magnitude_phase(grad_x::NumberLike, grad_y::NumberLike) =
    hypot(grad_x, grad_y), phase(grad_x, grad_y)

function magnitude_phase(grad_x::AbstractRGB, grad_y::AbstractRGB)
    gx, gy = vecsum(grad_x), vecsum(grad_y)
    magnitude_phase(gx, gy)
end

vecsum(c::AbstractRGB) = float(red(c)) + float(green(c)) + float(blue(c))

## TODO? orientation seems nearly redundant with phase, deprecate?

"""
    orientation(grad_x, grad_y) -> orient

Calculate the orientation angle of the strongest edge from gradient images
given by `grad_x` and `grad_y`.  Equivalent to `atan(grad_x, grad_y)`.  When
both `grad_x` and `grad_y` are effectively zero, the corresponding angle is set to
zero.
"""
function orientation(grad_x::T, grad_y::T, tol=sqrt(eps(T))) where T<:Number
    atan(grad_x, grad_y) * ((abs(grad_x) > tol) | (abs(grad_y) > tol))
end
orientation(grad_x::Number,   grad_y::Number)   = orientation(promote(grad_x, grad_y)...)
orientation(grad_x::NumberLike, grad_y::NumberLike) = orientation(gray(grad_x), gray(grad_y))

orientation(grad_x::AbstractRGB, grad_y::AbstractRGB) = orientation(vecsum(grad_x), vecsum(grad_y))

# Magnitude of gradient, calculated from X and Y image gradients
"""
```
m = magnitude(grad_x, grad_y)
```

Calculates the magnitude of the gradient images given by `grad_x` and `grad_y`.
Equivalent to `sqrt(grad_x.^2 + grad_y.^2)`.

Returns a magnitude image the same size as `grad_x` and `grad_y`.
"""
magnitude(grad_x::AbstractArray, grad_y::AbstractArray) = hypot.(grad_x, grad_y)

Base.hypot(x::AbstractRGB, y::AbstractRGB) = hypot(vecsum(x), vecsum(y))

phase(grad_x::AbstractArray, grad_y::AbstractArray) = phase.(grad_x, grad_y)

# Orientation of the strongest edge at a point, calculated from X and Y gradient images
# Note that this is perpendicular to the phase at that point, except where
# both gradients are close to zero.

orientation(grad_x::AbstractArray{T}, grad_y::AbstractArray{T}) where {T} = orientation.(grad_x, grad_y)

# Return both the magnitude and phase in one call
"""
    magnitude_phase(grad_x, grad_y) -> m, p

Convenience function for calculating the magnitude and phase of the gradient
images given in `grad_x` and `grad_y`.  Returns a tuple containing the magnitude
and phase images.  See `magnitude` and `phase` for details.
"""
function magnitude_phase(grad_x::AbstractArray{T}, grad_y::AbstractArray{T}) where T
    m = similar(grad_x, eltype(T))
    p = similar(m)
    for I in eachindex(grad_x, grad_y)
        m[I], p[I] = magnitude_phase(grad_x[I], grad_y[I])
    end
    m, p
end

# Return the magnitude and phase of the gradients in an image
function magnitude_phase(img::AbstractArray, method::Function=KernelFactors.ando3, border::AbstractString="replicate")
    grad_x, grad_y = imgradients(img, method, border)
    return magnitude_phase(grad_x, grad_y)
end

# Return the x-y gradients and magnitude and phase of gradients in an image
"""
```
grad_y, grad_x, mag, orient = imedge(img, kernelfun=KernelFactors.ando3, border="replicate")
```

Edge-detection filtering. `kernelfun` is a valid kernel function for
[`imgradients`](@ref), defaulting to [`KernelFactors.ando3`](@ref).
`border` is any of the boundary conditions specified in `padarray`.

Returns a tuple `(grad_y, grad_x, mag, orient)`, which are the horizontal
gradient, vertical gradient, and the magnitude and orientation of the strongest
edge, respectively.
"""
function imedge(img::AbstractArray, kernelfun=KernelFactors.ando3, border::AbstractString="replicate")
    grad_y, grad_x = imgradients(img, kernelfun, border)
    mag = magnitude(grad_y, grad_x)
    orient = orientation(grad_y, grad_x)
    return (grad_y, grad_x, mag, orient)
end

# Thin edges
"""
```
thinned = thin_edges(img, gradientangle, [border])
thinned, subpix = thin_edges_subpix(img, gradientangle, [border])
thinned, subpix = thin_edges_nonmaxsup(img, gradientangle, [border]; [radius::Float64=1.35], [theta=pi/180])
thinned, subpix = thin_edges_nonmaxsup_subpix(img, gradientangle, [border]; [radius::Float64=1.35], [theta=pi/180])
```

Edge thinning for 2D edge images.  Currently the only algorithm available is
non-maximal suppression, which takes an edge image and its gradient angle, and
checks each edge point for local maximality in the direction of the gradient.
The returned image is non-zero only at maximal edge locations.

`border` is any of the boundary conditions specified in `padarray`.

In addition to the maximal edge image, the `_subpix` versions of these functions
also return an estimate of the subpixel location of each local maxima, as a 2D
array or image of `Graphics.Point` objects.  Additionally, each local maxima is
adjusted to the estimated value at the subpixel location.

Currently, the `_nonmaxsup` functions are identical to the first two function
calls, except that they also accept additional keyword arguments.  `radius`
indicates the step size to use when searching in the direction of the gradient;
values between 1.2 and 1.5 are suggested (default 1.35).  `theta` indicates the
step size to use when discretizing angles in the `gradientangle` image, in
radians (default: 1 degree in radians = pi/180).

Example:

```
g = rgb2gray(rgb_image)
gx, gy = imgradients(g)
mag, grad_angle = magnitude_phase(gx,gy)
mag[mag .< 0.5] = 0.0  # Threshold magnitude image
thinned, subpix =  thin_edges_subpix(mag, grad_angle)
```
"""
thin_edges(img::AbstractArray{T,2}, gradientangles::AbstractArray, border::AbstractString="replicate") where {T} =
    thin_edges_nonmaxsup(img, gradientangles, border)
thin_edges_subpix(img::AbstractArray{T,2}, gradientangles::AbstractArray, border::AbstractString="replicate") where {T} =
    thin_edges_nonmaxsup_subpix(img, gradientangles, border)

# Code below is related to non-maximal suppression, and was ported to Julia from
# http://www.csse.uwa.edu.au/~pk/research/matlabfns/Spatial/nonmaxsup.m
# (Please conserve the original copyright below.)

# NONMAXSUP - Non-maxima suppression
#
# Usage:
#          (im,location) = nonmaxsup(img, gradientangles, radius);
#
# Function for performing non-maxima suppression on an image using
# gradient angles.  Gradient angles are assumed to be in radians.
#
# Input:
#   img - image to be non-maxima suppressed.
#
#   gradientangles - image containing gradient angles around each pixel in radians
#                    (-pi,pi)
#
#   radius  - Distance in pixel units to be looked at on each side of each
#             pixel when determining whether it is a local maxima or not.
#             This value cannot be less than 1.
#             (Suggested value about 1.2 - 1.5)
#
# Returns:
#   im        - Non maximally suppressed image.
#   location  - `Graphics.Point` image holding subpixel locations of edge
#               points.
#
# Notes:
#
# This function uses bilinear interpolation to estimate
# intensity values at ideal, real-valued pixel locations on each side of
# pixels to determine if they are local maxima.

# Copyright (c) 1996-2013 Peter Kovesi
# Centre for Exploration Targeting
# The University of Western Australia
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# The Software is provided "as is", without warranty of any kind.

# December  1996 - Original version
# September 2004 - Subpixel localization added
# August    2005 - Made Octave compatible
# October   2013 - Final thinning applied to binary image for Octave
#                  compatbility (Thanks to Chris Pudney)
# June      2014 - Ported (and modified significantly) to Julia (Kevin Squire)

import .Point

if !applicable(zero, Point)
    import Base.zero
    zero(::Type{Point}) = Point(0.0,0.0)
end

# Used to encode the sign, integral, and fractional components of
# an offset from a coordinate
struct CoordOffset
    s::Int      # sign
    i::Int      # integer part
    f::Float64  # fractional part
end

CoordOffset(x::Float64) = ((frac,i) = modf(x); CoordOffset(sign(frac), round(Int, i), abs(frac)))
(-)(off::CoordOffset) = CoordOffset(-off.s,-off.i, off.f)
(*)(x::Number, off::CoordOffset) = x*(off.i + off.s*off.f)
(*)(off::CoordOffset, x::Number) = x*(off.i + off.s*off.f)
(+)(x::Number, off::CoordOffset) = x + off.i + off.s*off.f
(+)(off::CoordOffset, x::Number) = x + off.i + off.s*off.f

# Precalculate x and y offsets relative to centre pixel for each orientation angle
function _calc_discrete_offsets(θ, radius)

    θ_count = round(Int, 2π/θ)
    θ = 2π/θ_count
    angles = (0:θ_count)*θ

    # x and y offset of points at specified radius and angles
    # from each reference position.

    xoffs = [CoordOffset( x) for x in  radius * cos.(angles)]
    yoffs = [CoordOffset(-y) for y in  radius * sin.(angles)]

    return θ, xoffs, yoffs
end

_discretize_angle(angle::AbstractFloat, invθ) =
    angle < 0 ? round(Int, (angle + 2π)*invθ)+1 : round(Int, angle*invθ)+1

# Interpolate the value of an offset from a particular pixel
#
# Returns (interpolated value, min_value of adjacent pixels in direction of offset)
#
# The second value is made available to eliminate double edges; if the value at
# (x,y) is less than the value or values adjacent to it in the direction of the
# gradient (xoff,yoff), then it is not a local maximum

function _interp_offset(img::AbstractArray, x::Integer, y::Integer, xoff::CoordOffset, yoff::CoordOffset, Ix, Iy, pad)
    fx = Ix[x + xoff.i + pad]
    fy = Iy[y + yoff.i + pad]
    cx = Ix[x + xoff.i + xoff.s + pad]
    cy = Iy[y + yoff.i + yoff.s + pad]

    tl = img[fy,fx]    # Value at bottom left integer pixel location.
    tr = img[fy,cx]    # bottom right
    bl = img[cy,fx]    # top left
    br = img[cy,cx]    # top right

    upperavg = tl + xoff.f * (tr - tl)  # Now use bilinear interpolation to
    loweravg = bl + xoff.f * (br - bl)  # estimate value at x,y

    min_adjacent = (fx == x) & (fy == y) ? min(tr,bl) : tl

    return (upperavg + yoff.f * (loweravg - upperavg), min_adjacent)
end

# Core edge thinning algorithm using nonmaximal suppression
function thin_edges_nonmaxsup_core!(out::AbstractArray{T,2}, location::AbstractArray{Point,2},
                                    img::AbstractArray{T,2}, gradientangles::AbstractMatrix, radius, border, theta) where T
    calc_subpixel = !isempty(location)

    # Error checking
    size(img) == size(gradientangles) == size(out) || error("image, gradient angle, and output image must all be the same size")
    calc_subpixel && size(location) != size(img) && error("subpixel location has a different size than the input image")
    radius < 1.0 && error("radius must be >= 1")

    # Precalculate x and y offsets relative to centre pixel for each orientation angle
    θ, xoffs, yoffs = _calc_discrete_offsets(theta, radius)
    iθ = 1/θ

    # Indexes to use for border handling
    pad = ceil(Int, radius)
    Ix = Images.padindexes(img, 2, pad, pad, border)
    Iy = Images.padindexes(img, 1, pad, pad, border)

    # Now run through the image interpolating grey values on each side
    # of the centre pixel to be used for the non-maximal suppression.

    (height,width) = size(img)

    for x = 1:width, y = 1:height
        (c = img[y,x]) == 0 && continue  # For thresholded images

        or = _discretize_angle(gradientangles[y,x],iθ)   # Disretized orientation
        v1, n1 = _interp_offset(img, x, y, xoffs[or], yoffs[or], Ix, Iy, pad)

        if (c > v1) & (c >= n1) # We need to check the value on the other side...
            v2, n2 = _interp_offset(img, x, y, -xoffs[or], -yoffs[or], Ix, Iy, pad)

            if (c > v2) & (c >= n2)  # This is a local maximum.
                                     # Record value in the output image.
                if calc_subpixel
                    # Solve for coefficients of parabola that passes through
                    # [-1, v2]  [0, img] and [1, v1].
                    # v = a*r^2 + b*r + c

                    # c = img[y,x]
                    a = (v1 + v2)/2 - c
                    b = a + c - v2

                    # location where maxima of fitted parabola occurs
                    r = -b/2a
                    location[y,x] = Point(x + r*xoffs[or], y + r*yoffs[or])

                    if T<:AbstractFloat
                        # Store the interpolated value
                        out[y,x] = a*r^2 + b*r + c
                    else
                        out[y,x] = c
                    end
                else
                    out[y,x] = c
                end
            end
        end
    end

    out
end

# Main function call when subpixel location of edges is not needed
function thin_edges_nonmaxsup!(out, img, gradientangles, border::AbstractString="replicate";
                               radius::Float64=1.35, theta=pi/180)
    thin_edges_nonmaxsup_core!(out, Matrix{Point}(0,0), img, gradientangles, radius, border, theta)
end

function thin_edges_nonmaxsup(img, gradientangles, border::AbstractString="replicate";
                                 radius::Float64=1.35, theta=pi/180)
    (height,width) = size(img)
    out = zeros(eltype(img), height, width)
    thin_edges_nonmaxsup_core!(out, Matrix{Point}(undef,0,0), img, gradientangles, radius, border, theta)
end

# Main function call when subpixel location of edges is desired
function thin_edges_nonmaxsup_subpix!(out, location, img, gradientangles,
                                      border::AbstractString="replicate";
                                      radius::Float64=1.35, theta=pi/180)
    eltype(location) != Point && error("Preallocated subpixel location array/image must have element type Graphics.Point")

    thin_edges_nonmaxsup_core!(out, location, img, gradientangles, radius, border, theta)
    img, location
end

function thin_edges_nonmaxsup_subpix(img, gradientangles,
                                     border::AbstractString="replicate";
                                     radius::Float64=1.35, theta=pi/180)
    (height,width) = size(img)
    out = zeros(eltype(img), height, width)
    location = zeros(Point, height, width)
    thin_edges_nonmaxsup_core!(out, location, img, gradientangles, radius, border, theta)
    out, location
end

"""
```
canny_edges = canny(img, (upper, lower), sigma=1.4)
```

Performs Canny Edge Detection on the input image.

Parameters :

  (upper, lower) :  Bounds for hysteresis thresholding
  sigma :           Specifies the standard deviation of the gaussian filter

# Example

```julia
imgedg = canny(img, (Percentile(80), Percentile(20)))
```
"""
function canny(img_gray::AbstractMatrix{T}, threshold::Tuple{N,N}, sigma::Number = 1.4) where {T<:NumberLike, N<:Union{NumberLike,Percentile{NumberLike}}}
    img_grayf = imfilter(img_gray, KernelFactors.IIRGaussian((sigma,sigma)), NA())
    img_grad_y, img_grad_x = imgradients(img_grayf, KernelFactors.sobel)
    img_mag, img_phase = magnitude_phase(img_grad_x, img_grad_y)
    img_nonMaxSup = thin_edges_nonmaxsup(img_mag, img_phase)
    if N<:Percentile{}
        upperThreshold ,lowerThreshold = StatsBase.percentile(img_nonMaxSup[:], [threshold[i].p for i=1:2])
    else
        upperThreshold, lowerThreshold = threshold
    end
    img_thresholded = hysteresis_thresholding(img_nonMaxSup, upperThreshold, lowerThreshold)
    edges = map(i -> i >= 0.9, img_thresholded)
    edges
end

canny(img::AbstractMatrix, threshold::Tuple{N,N}, args...) where {N<:Union{NumberLike,Percentile{NumberLike}}} =
    canny(convert(Array{Gray}, img), args...)

function hysteresis_thresholding(img_nonMaxSup::AbstractArray{T, 2}, upperThreshold::Number, lowerThreshold::Number) where T
    img_thresholded = map(i -> i > lowerThreshold ? i > upperThreshold ? 1.0 : 0.5 : 0.0, img_nonMaxSup)
    queue = CartesianIndex{2}[]
    R = CartesianIndices(size(img_thresholded))

    I1, Iend = first(R), last(R)
    for I in R
      if img_thresholded[I] == 1.0
        img_thresholded[I] = 0.9
        push!(queue, I)
        while !isempty(queue)
          q_top = popfirst!(queue)
          for J in CartesianIndices(map((f,l)->f:l,(max(I1, q_top - I1)).I, (min(Iend, q_top + I1)).I))
            if img_thresholded[J] == 1.0 || img_thresholded[J] == 0.5
              img_thresholded[J] = 0.9
              push!(queue, J)
            end
          end
        end
      end
    end
    img_thresholded
end

function padindexes(img::AbstractArray{T,n}, dim, prepad, postpad, border::AbstractString) where {T,n}
    M = size(img, dim)
    I = Vector{Int}(undef, M + prepad + postpad)
    I = [(1 - prepad):(M + postpad);]
    if border == "replicate"
        I = min.(max.(I, 1), M)
    elseif border == "circular"
        I = 1 .+ mod.(I .- 1, M)
    elseif border == "symmetric"
        I = [1:M; M:-1:1][1 .+ mod.(I .- 1, 2 * M)]
    elseif border == "reflect"
        I = [1:M; M-1:-1:2][1 .+ mod.(I .- 1, 2 * M - 2)]
    else
        error("unknown border condition")
    end
    I
end


"""
    CalculusWithJulia

A package to accompany [notes](calculuswithjulia.github.io) on using Julia with calculus.

This package does two things: 1) it loads several other packages making it easier to use (and install) the functionality provided by them and 2) defines a handful of functions and plot recipes for convenience.

## Packages loaded by `CalculusWithJulia`

### Built in packages.

The `MathConstants` package is reexported, allowing `e` to be used instead of `\euler[tab]` for a value of Euler's constant.

The `SpecialFunctions` is loaded giving access to a few special functions used in these notes, e.g., `airyai`.

The `LinearAlgebra` package is loaded for access to several of its functions for working with vectors `norm`, `cdot` (`⋅`), `cross` (`×`), `det`.

### Plotting

The `Plots` package is loaded giving access to `plot`, `scatter`, `annotate`, etc. The backends  used in the notes are `plotly`, `gr`, and `pyplot`.

The `ImplicitEquations` package is loaded for plotting implicitly defined functions.

In addition, several plot recipes are provided to ease the creation of plots:
`plotif`, `trimplot`, and `signchart` are used for plotting univariate functions;
`plot_polar` and `plot_parametric_curve` are used to plot curves in 2 or 3 dimensions;
`plot_parametric_surface` makes the plotting os parameterically defined surfaces easier;
`vectorfieldplot` and `vectorfieldplot3d` can be used to plot vector fields; and
`arrow` is a simplified interface to `quiver` that also indicates 3D vectors.

### Symbolic math

The `Sympy` package is loaded for symbolic mathematics

### Zeros of functions

The `Roots` package is loaded for its `fzero` and `fzeros` functions.

### Derivatives

The `ForwardDiff` package is loaded giving access to its  `derivative`,  `gradient`, `jacobian`, and `hessian` functions for finding automatic derivatives of functions. In addition, this package defines `'` (for functions) to return a derivative (which commits [type piracy](https://docs.julialang.org/en/v1/manual/style-guide/index.html#Avoid-type-piracy-1)), `∇` to find the gradient (`∇(f)`), the divergence (`∇⋅F`). and the curl (`∇×F`), along with `divergence` and `curl`.

### Integration

This package reexports `QuadGK`, for one-dimensional integrals; `HCubature`, for multidimensional integrals; and provides `riemann`, for Riemann sums, and `fubini`, for iterated integrals over non-rectangular regions based on `QuadGK`.

"""
module CalculusWithJulia

using Reexport
@reexport using Plots
using RecipesBase
import ImplicitEquations # handle conflict with SymPy in plot-utils.jl
import ImplicitEquations: Pred
@reexport using LinearAlgebra
@reexport using Base.MathConstants
@reexport using SpecialFunctions
@reexport using Roots
@reexport using SymPy
import ForwardDiff
export ForwardDiff
@reexport using QuadGK
@reexport using HCubature

include("multidimensional.jl")
include("derivatives.jl")
include("integration.jl")
include("plot-utils.jl")

export unzip, divergence, gradient, curl, ∇
export tangent, secant, D
export riemann, fubini
export plotif, trimplot, signchart,
       plot_polar, plot_polar!,
       plot_parametric_curve,   plot_parametric_curve!,
       plot_parametric_surface, plot_parametric_surface!,
       vectorfieldplot,   vectorfieldplot!,
       vectorfieldplot3d, vectorfieldplot3d,
       arrow, arrow!

end # module


function recursive_factorial(n)
    if n == 0
        return 1
    else
        return n*recursive_factorial(n-1)
    end
end

recursive_factorial(7)

function recursive_factorial_info(n)
    if n == 0
        println("n == 0, returning 1")
        return 1
    else
        println("n == ", n, ", calling itself with parameter ", n-1)
        output = n*recursive_factorial_info(n-1)
        println("n == ", n, ", finished calling itself, multiplying and returning ", n, "! = ", output)
        return output
    end
end

recursive_factorial_info(7)

function ack(m,n)
    if m == 0
        return n + 1
    elseif m > 0 && n == 0
        return ack(m-1,1)
    else
        return ack(m-1, ack(m,n-1))
    end
end

ack(3,4)

function ack_info(m,n)
    function printme()
        print("ack(", m, ",", n, "): ")
    end
    if m == 0
        output = n + 1
        printme()
        println("Case 1: returning n + 1 = ", output)
        return output
    elseif m > 0 && n == 0
        printme()
        println("Case 2: calling itself with parameters m-1,1 == ", m-1, ",", 1)
        output = ack_info(m-1,1)
        printme()
        println("Case 2: finished calling itself, returning with output ", output)
        return output
    else
        printme()
        println("Case 3: calling itself for new n-value with parameters m,n-1 == ", m, ",", n-1)
        newn = ack_info(m,n-1)
        printme()
        println("Case 3: finished calling itself for new n-value == ", newn)
        printme()
        println("Case 3: calling itself with parameters m-1, A(m,n-1) == ", m-1, ",", newn)
        output = ack_info(m-1,newn)
        printme()
        println("Case 3: finished calling itself, returning ", output)
        return output
    end
end

ack_info(2,1)

function my_gcd(a,b)
    if a == 0
        return b
    elseif b == 0
        return a
    else
        return my_gcd(b, a % b)
    end
end

factor = 123_456_789
prime1 = 67_867_979
prime2 = 86_028_121
my_gcd(prime1*factor, prime2*factor)

using PyPlot

function drawTriangle(x, y, level)
    # Draw recursively colored triangles.
    # x,y are 3-vectors that define the vertices of a triangle.

    if level == 0
        # Recursion limit (depth) reached
        fill(x, y, "y") # Color whole triangle yellow
    else
        # Draw the triangle...
        plot(x[[1,2,3,1]], y[[1,2,3,1]], "k", linewidth=0.5)
        # Determine the midpoints...
        a = (x + x[[2,3,1]]) / 2
        b = (y + y[[2,3,1]]) / 2
        # Draw and color the interior triangle mauve
        fill(a, b, "m")
        # Apply the process to the three "corner" triangles...
        newx = [x a a[[3,1,2]]]
        newy = [y b b[[3,1,2]]]
        for i = 1:3
            drawTriangle(newx[i,:], newy[i,:], level - 1)
        end
    end
end

# Equilateral triangle
x = [0, 1, 0.5]
y = [0, 0, 1/sqrt(2)]

drawTriangle(x, y, 5)

function mergeLR!(L, R, x)
    # Merge the *already sorted arrays* L and R into a sorted array x
    i = j = k = 1

    # Merge L and R into x
    while i <= length(L) && j <= length(R)
        if L[i] < R[j]
            x[k] = L[i]
            i += 1
        else
            x[k] = R[j]
            j += 1
        end
        k += 1
    end

    # Copy remaining elements
    while i <= length(L)
        x[k] = L[i]
        i += 1
        k += 1
    end
    while j <= length(R)
        x[k] = R[j]
        j += 1
        k += 1
    end
end

function mergesort!(x)
    # Sort the elements of the array x using the Mergesort algorithm
    if length(x) <= 1
        return x
    else
        mid = length(x) ÷ 2   # Find the midpoint of the array
        L = x[1:mid]          # Divide array into 2 halves
        R = x[mid+1:end]

        mergesort!(L)         # Sort first half
        mergesort!(R)         # Sort second half

        mergeLR!(L, R, x)
    end
end

# Example: Sort random integers
x = rand(1:1000, 10)
println(x)
mergesort!(x)
println(x)

for array_size = Int64[1e3, 1e4, 1e5, 1e6, 1e7]
    x = rand(array_size)   # Random floating point numbers
    println("n = ", array_size)
    @time mergesort!(x)
end

function Mvalues(n)
    returned_values = Int64[]

    function M(n)
        if n > 100
            newval = n - 10
        else
            newval = M(M(n + 11))
        end
        push!(returned_values, newval)
        return newval
    end

    M(n)
    return returned_values
end

Mvalues(105)     # Easy - terminates immediately, M(105) = 95

Mvalues(97)      # More complex - finally returns M(97) = 91


function mk_score(x)
    s = 0
    n = length(x)

    for i = 1:n-1
        for j = i : n
            s = s + sign(x[j] - x[i])
        end
    end

    return s
end

function variance_s(x, n)
    # calculate the unique data
    unique_x = unique(x)
    g = length(unique_x)

    # calculate the var(s)
    if n == g            # there is no tie
        var_s = (n*(n-1)*(2*n+5))/18

    else                 # there are some ties in data
        tp = zeros(length(unique_x))
        demo = ones(n)

        for i in 1:g
            tp[i] = sum(demo[x == unique_x[i]])
        end
        var_s = (n*(n-1)*(2*n+5) - sum(tp*(tp-1)*(2*tp+5)))/18
    end

    return var_s
end

function z_score(s, var_s)
    if s > 0
        z = (s - 1)/sqrt(var_s)
    elseif s == 0
        z = 0
    elseif s < 0
        z = (s + 1)/sqrt(var_s)
    end

    return z
end

function p_value(z, α)
    # two tail test
    p = 2*(1-normcdf(abs(z)))
    h = abs(z) > norminvcdf(1-α/2)

    if (z < 0) && h
        trend = "decreasing"
    elseif (z > 0) && h
        trend = "increasing"
    else
        trend = "no trend"
    end

    return p, h, trend
end


using Plots
#using LsqFit
import Distributions as di
using Random
using Turing
using Statistics
using StatsPlots
using FillArrays

n = di.Normal(1,2)
sample_n = rand(n,20)

@model function normal_fit(data)
	μ ~ Uniform(-10,10)
	σ ~ Uniform(0,20)
    data ~ MvNormal(Fill(μ,length(data)),σ)
end

model1 = normal_fit(sample_n)
chain = Turing.sample(model1,NUTS(0.65),1000)
plot(chain)

rwd = Normal(0,1)
rw_steps = rand(rwd,999)

x = [0.0]
for step in rw_steps
    push!(x,x[end]+step)
end
plot(x)

steps = []
for i in 1:length(x)-1
    push!(steps,x[i+1]-x[i])
end

rw_steps
steps

@model function diffusion_fit(data)
	μ = 0
	σ ~ Uniform(0,20)
	for i in 1:length(data)
		data[i] ~ Normal(μ,σ)
	end
end

model2 = diffusion_fit(steps)
chain2 = Turing.sample(model2,NUTS(0.65),1000)


#### TriDiagSolverFuncs

# A module with functions to solve tridiagonal
# systems of equations using the Thomas algorithm.

export solve_tridiag!
export solve_tridiag_stored!
export init_β_γ!

"""
    solve_tridiag!(x, B, a, b, c, n, xtemp, γ, β)

Solves for `x` in the equation
          `Ax = B`
where `A` is a tridiagonal matrix:
```
           _                                           _ -1
          |  b[1] c[1]                                   |
          |  a[1] b[2]  c[2]                             |
          |       a[2]  b[3]  c[3]                       |
 x    =   |           *     *     *                      |   B
          |                 *     *     *                |
          |                    a[n-2] b[n-1]  c[n-1]     |
          |_                           a[n-1]  b[n]     _|

          |______________________________________________|
                                 A
```
and given arguments:
--------------------------------------------
| x[1:n]       | the result                |
| B[1:n]       | right hand side           |
| a[1:n-1]     | sub-diagonal              |
| b[1:n]       | main diagonal             |
| c[1:n-1]     | super-diagonal            |
| n            | system size               |
| xtemp[1:n]   | temporary                 |
| γ[1:n-1]     | temporary                 |
| β[1:n]       | temporary                 |
--------------------------------------------
"""
function solve_tridiag!(x, B, a, b, c, n, xtemp, γ, β)
  # Define coefficients:
  β[1] = b[1]
  γ[1] = c[1]/β[1]
  for i in 2:n-1
    β[i] = b[i]-a[i-1]*γ[i-1]
    γ[i] = c[i]/β[i]
  end
  β[n] = b[n]-a[n-1]*γ[n-1]

  # Forward substitution:
  xtemp[1] = B[1]/β[1]
  for i = 2:n
    m = B[i] - a[i-1]*xtemp[i-1]
    xtemp[i] = m/β[i]
  end

  # Backward substitution:
  x[n] = xtemp[n]
  for i in n-1:-1:1
    x[i] = xtemp[i]-γ[i]*x[i+1]
  end
end

"""
    solve_tridiag_stored!(x, B, a, β, γ, n, xtemp)

Solves for `x` in the equation
          `Ax = B`
where `A` is a tridiagonal matrix.

Coefficients in solve_tridiag! can be pre-computed,
by applying LU factorization to A (shown below).
The coefficients, β and γ, can be computed in init_β_γ!.
```
 _                                           _
|  b[1]  c[1]                                 |
|  a[1]  b[2]  c[2]                           |
|         a[2]  b[3]  c[3]                    |
|           *     *     *                     |
|                 *     *     *               |
|                    a[n-2]  b[n-1]  c[n-1]   |
|_                            a[n-1]  b[n]   _|

=
 _                                        _   _                                      _ -1
|  β[1]                                    | |  1  γ[1]                               |
|  α[1]  β[2]                              | |        1  γ[2]                         |
|        α[2]  β[3]                        | |              1  γ[3]                   |
|           *     *     *                  | |                 *     *                |
|                 *     *                  | |                       *     *          |
|                    α[n-2]  β[n-1]        | |                             1   γ[n-1] |
|_                           α[n-1]  β[n] _| |_                                1     _|
```

and given arguments:
--------------------------------------------
| x[1:n]       | the result                |
| B[1:n]       | right hand side           |
| a[1:n-1]     | sub-diagonal              |
| β[1:n]       | temporary                 |
| γ[1:n-1]     | temporary                 |
| n            | system size               |
| xtemp[1:n]   | temporary                 |
--------------------------------------------
"""
function solve_tridiag_stored!(x, B, a, β, γ, n, xtemp)
  # Forward substitution:
  xtemp[1] = B[1]/β[1]
  for i = 2:n
    m = B[i] - a[i-1]*xtemp[i-1]
    xtemp[i] = m/β[i]
  end

  # Backward substitution:
  x[n] = xtemp[n]
  for i = n-1:-1:1
    x[i] = xtemp[i]-γ[i]*x[i+1]
  end
end

"""
    init_β_γ!(β, γ, a, b, c, n)

Returns the pre-computed coefficients, from applying
LU factorization, for the tridiagonal system. These
coefficients can be passed as arguments to solve_tridiag_stored!.
--------------------------------------------
| β[1:n]       | temporary                 |
| γ[1:n-1]     | temporary                 |
| a[1:n-1]     | sub-diagonal              |
| b[1:n]       | main diagonal             |
| c[1:n-1]     | super-diagonal            |
| n            | system size               |
--------------------------------------------
"""
function init_β_γ!(β, γ, a, b, c, n)
  β[1] = b[1]
  γ[1] = c[1]/β[1]
  for i = 2:n-1
    β[i] = b[i]-a[i-1]*γ[i-1]
    γ[i] = c[i]/β[i]
  end
  β[n] = b[n]-a[n-1]*γ[n-1]
end


function EuropeanSpreadOptionMC(K, T, S1, S2, sig1, sig2, div1, div2, rho, r, N, M)

## setting parameters

dt = T/N
nu1dt = (r - div1 - 0.5 * sig1^2) * dt
nu2dt = (r - div2 - 0.5 * sig2^2) * dt
sig1sdt = sig1 * sqrt(dt)
sig2sdt = sig2 * sqrt(dt)
srho = sqrt(1 - rho^2)

## variables to store the sums of the option values

sum_CT = 0
sum_CT2 = 0

## outer loop is the number of simulations bigger is better

for j = 1:M

	St1 = S1
	St2 = S2

	## inner loop is simulating each price path of length N
	for i = 1:N

		error1 = randn()
		error2 = randn()

		z1 = error1
		z2 = rho * error1 + srho * error2

		St1 = St1 * exp(nu1dt + sig1sdt * z1)
		St2 = St2 * exp(nu2dt + sig2sdt * z2)

	end

    	## wraps up and stores all the values of each path
	CT = max(0, St1 - St2 - K)
	sum_CT = sum_CT + CT
	sum_CT2 = sum_CT2 + CT^2

end

## discounting the average call value
call_value = sum_CT / M * exp(-r*T)

SD = (sqrt(sum_CT2 - 1/M * (sum_CT^2)) * exp(-2*r*T)) / (M-1)
SE = SD / sqrt(M)

println("The Call Value is $call_value")
println("The SD is $SD")
println("The SE is $SE")

MC = [call_value, SD, SE]

end


include("Lanczos_Ritz_Utilities.jl")
include("../../src/iterative_lanczos.jl")

# ------------------------------------------------------------------------------
# Demonstrating the loss of orthogonality of the lanczos vectors.
# And the linear dependence of eigenvalues of the exterior of the spectrum.
# ------------------------------------------------------------------------------
n = 64
A = Diagonal(LinRange(-1, 1, n).^3)
il = IterativeLanczos(A, rand(n))
for _ in 1: n - 1
    il()
end
Q = GetQMatrix(il)
T = GetTMatrix(il)

fig = heatmap(Q'*Q, size=(722, 512))
fig2 = heatmap(Q'*A*Q, size=(722, 512))
fig3 = heatmap(
    (A*Q[:, 1:end - 1] - Q*T[:, 1:end - 1]).|> abs,
    size=(1200, 768), dpi=250
)
savefig(fig, "$(@__DIR__)/plots/fig3.png")
savefig(fig2, "$(@__DIR__)/plots/fig4.png")

# ------------------------------------------------------------------------------
# All the ritz values during the computations process. and plotting it.
# ------------------------------------------------------------------------------

function RiztTrajectoryPlot(filename, n=64; itr_offset=20, full_ortho=false, title="")
    A = Diagonal(LinRange(-1, 1, n).^3)
    il = IterativeLanczos(A, ones(n))
    il.reorthogonalize = full_ortho
    FoundRitzValues = Vector{Vector{Float64}}()
    # push!(FoundRitzValues, [GetTMatrix(il)])
    TrueEigenValues = diag(A)

    for II in 1: n - 1
        il()
        T = GetTMatrix(il)
        λs, _ = eigen(T)
        if II >= itr_offset
            push!(FoundRitzValues, λs)
        end
    end

    for RitzValue in FoundRitzValues
        sort!(RitzValue, rev=true)
    end

    fig4 = scatter(title=title, legend=false)
    for Idx in 1: div(itr_offset,2)
        RitzTrajectory = Vector{Float64}()
        for RitzValues in FoundRitzValues
            if Idx <= length(RitzValues)
                push!(RitzTrajectory, RitzValues[Idx])
            end
        end
        plot!(
            fig4,
            # Idx: length(RitzTrajectory) + Idx - 1,
            itr_offset:(length(RitzTrajectory) + itr_offset - 1),
            RitzTrajectory, size=(750,750),
            dpi=250,
            markershape=:cross,
            linestyle=:solid
        )
    end
    for Idx in 1:div(itr_offset, 2)
        RitzTrajectory = Vector{Float64}()
        for RitzValues in FoundRitzValues
            if Idx <= length(RitzValues)
                push!(RitzTrajectory, RitzValues[end - Idx + 1])
            end
        end
        plot!(
            fig4,
            # Idx: length(RitzTrajectory) + Idx - 1,
            itr_offset:(length(RitzTrajectory) + itr_offset - 1),
            RitzTrajectory,
            dpi=250,
            markershape=:xcross,
            linestyle=:solid
        )
    end
    xlabel!(fig4, "iterations")
    ylabel!(fig4, "ritz values")
    display(fig4)
    savefig(fig4, "$(@__DIR__)/plots/$(filename).png")

return end

RiztTrajectoryPlot("ritz_trajectory_plot_floats", title="ritz trajectory no-reorthgonalization")
RiztTrajectoryPlot("ritz_trajectory_plot_exact", full_ortho=true, title="ritz trajectory with full-reorthogonalizatoin")