using SemiDiscretizationMethod

function createMathieuProblem(δ,ε,b0,a1;T=2π)
    AMx =  ProportionalMX(t->@SMatrix [0. 1.; -δ-ε*cos(2π/T*t) -a1]);
    τ1=2π # if function is needed, the use τ1 = t->foo(t)
    BMx1 = DelayMX(τ1,t->@SMatrix [0. 0.; b0 0.]);
    cVec = Additive(t->@SVector [0.,sin(4π/T*t)])
    LDDEProblem(AMx,[BMx1],cVec)
end;

τmax=2π # the largest τ of the system
T=2π #Principle period of the system (sin(t)=sin(t+P))
mathieu_lddep=createMathieuProblem(3.,2.,-0.15,0.1,T=T); # LDDE problem for Hayes equation
method=SemiDiscretization(1,0.01) # 3rd order semi discretization with Δt=0.1
# if P = τmax, then n_steps is automatically calculated
mapping=DiscreteMapping_1step(mathieu_lddep,method,τmax,
    n_steps=Int((T+100eps(T))÷method.Δt),calculate_additive=true); #The discrete mapping of the system

@show spectralRadiusOfMapping(mapping); # spectral radius ρ of the mapping matrix (ρ>1 unstable, ρ<1 stable)
fp=fixPointOfMapping(mapping); # stationary solution of the hayes equation (equilibrium position)

plot(0.0:method.Δt:P,fp[1:2:end],
    xlabel=L"-s",title=L"t \in [nP,(n+1)P],\quad n \to \infty",guidefontsize=14,linewidth=3,
    label=L"x(t-s)",legendfontsize=11,tickfont = font(10))

plot!(0.0:method.Δt:P,fp[2:2:end],
    xlabel=L"-s",linewidth=3,
    label=L"\dot{x}(t-s)")

plot!(0.0:method.Δt:P,sin.(2*(0.0:method.Δt:P)),linewidth=3,label=L"\sin(2t)")

using MDBM

using Plots
gr();
using LaTeXStrings

a1=0.1;
ε=1;
τmax=2π;
T=1π;
method=SemiDiscretization(2,T/40);

foo(δ,b0) = log(spectralRadiusOfMapping(DiscreteMapping_1step(createMathieuProblem(δ,ε,b0,a1,T=T),method,τmax,
    n_steps=Int((T+100eps(T))÷method.Δt)))); # No additive term calculated

axis=[Axis(-1:0.2:5.,:δ),
    Axis(-2:0.2:1.5,:b0)]

iteration=3;
stab_border_points=getinterpolatedsolution(solve!(MDBM_Problem(foo,axis),iteration));

scatter(stab_border_points...,xlim=(-1.,5),ylim=(-2.,1.5),
    label="",title="Stability border of the delay Mathieu equation",xlabel=L"\delta",ylabel=L"b_0",
    guidefontsize=14,tickfont = font(10),markersize=2,markerstrokewidth=0)


module LogisticRegression2
using Revise
using Flux
using Flux: binarycrossentropy, throttle, @epochs
using Flux: Descent, TrackedArray, param, data
using Distributions
using LinearAlgebra
using Random
using Plots
using Base.Iterators: repeated

theme(:juno)

# Generate data
D = 2;   # Data dimension
n = 100; # Number of observations
x1 = rand(MvNormal([1., -5.], I), n);
x2 = rand(MvNormal([-2., -1.5], I), n);
y1 = repeat([1], n);
y2 = repeat([0], n);

classOneColor = :orange
classTwoColor = RGBA(0,.2,.7,1);
p = scatter(x1[1, :], x1[2, :], label="Class 1", color=classOneColor)
scatter!(p, x2[1, :], x2[2, :], label="Class 2", color=classTwoColor)
X = transpose([x1 x2]);
y = [y1; y2];

# Create model
struct LogisticRegression
    α :: TrackedArray # bias
    β :: TrackedArray # weights
    LogisticRegression(α :: Array, β :: Array) = new(param(α), param(β))
end

function (m :: LogisticRegression)(X)
    σ.(m.α .+ X*m.β)
end

function plot_decision_surface(m, xlim :: Tuple{Int, Int}, ylim :: Tuple{Int, Int})
    x = xlim[1]:0.5:xlim[2]
    y = ylim[1]:0.5:ylim[2]
    f = (x, y) -> (first ∘ data ∘ m)([x y])
    contour(x, y, f, fill = true, color = :pu_or)
end

function plot_fit(m)
    xlim = ylim = (-8, 8)
    p = plot_decision_surface(m, xlim, ylim)
    scatter!(p, x1[1, :], x1[2, :], label="Class 1", color=classOneColor)
    scatter!(p, x2[1, :], x2[2, :], label="Class 2", color=classTwoColor)
    p
end

function plot_loss(losses)
    plot(1:length(losses), losses)
end

function create_animation(αs :: Array, βs :: Array)
    anim = @animate for (α, β) in zip(αs, βs)
        m = LogisticRegression(α, β)
        display(plot_fit(m))
    end

    gif(anim, "plots/transition.gif", fps = 60)
end

m = LogisticRegression(rand(1), rand(D, 1)); display(plot_fit(m));
θ = Flux.params(m.α, m.β);
numEpochs = 100;
loss = (x, y) -> mean(binarycrossentropy.(m(x), y));

losses, αs, βs = [], [], []
cb = () -> begin
    l = Flux.data(loss(X, y));
    push!(losses, l);
    push!(αs, Flux.data(m.α));
    push!(βs, Flux.data(m.β));
end

dataset = repeated((X, y), numEpochs)
Flux.train!(loss, θ, dataset, Descent(0.1), cb = cb)

plot_loss(losses)
plot_fit(m)

create_animation(αs, βs)

end


# Polyharmonic functions used below.
function polyharmonic_phi1(r) return r; end
function polyharmonic_phi2(r) return r>1e-16 ? r^2 * log(r) : r; end
function polyharmonic_phi3(r) return r^3; end
function polyharmonic_phi4(r) return r>1e-16 ? r^4 * log(r) : r^3; end

# Uses polyharmonic interpolation to interpolate at x using values u at centers c.
# sizes are:
# x is d x nx
# c is d x nc
# u is nc
# (optional) ord determines the type of basis function. default is 3. 1,2,3,4 are available
function polyharmonic_interp(x, c, u, ord=3)
    dim = size(x,1);
    nx = size(x,2); # number of places to evaluate

    # If no centers are provided, this is meaningless, but return zero.
    if length(c)==0
        return zeros(nx);
    end

    nc = size(c,2); # number of centers

    # Constant if only one center is given
    if nc == 1
        return u[1] .* ones(nx);
    end

    if ord == 1
        phi = polyharmonic_phi1;
    elseif ord == 2
        phi = polyharmonic_phi2;
    elseif ord == 3
        phi = polyharmonic_phi3;
    else
        phi = polyharmonic_phi4;
    end

    # build A matrix
    A = zeros(nc + dim+1, nc + dim+1);
    for i=1:nc
        for j=1:nc
            A[i,j] = phi(norm(c[:,j] - c[:,i]));
        end
    end
    # build B matrix (within A matrix)
    A[1:nc, nc+1] .= 1;
    A[nc+1, 1:nc] .= 1;
    A[1:nc, (nc+2):(nc+dim+1)] = c';
    A[(nc+2):(nc+dim+1), 1:nc] = c;

    # build RHS vector
    b = vcat(u, zeros(dim+1));

    wv = A\b; # coefficients for the polyharmonic

    sol = zeros(nx);
    for i=1:nx
        # evaluate at x[i]
        sol[i] = wv[nc+1];
        for k=1:dim
            sol[i] += wv[nc+k+1] * x[k,i];
        end
        for j=1:nc
            sol[i] += wv[j]*phi(norm(c[:,j] - x[:,i]));
        end
    end

    return sol;
end


#
# Cubic fit
#

struct Cubic
  a :: Float64
  b :: Float64
  c :: Float64
  d :: Float64
  R :: Float64
  x :: Vector{Float64}
  y :: Vector{Float64}
  ypred :: Vector{Float64}
  residues :: Vector{Float64}
end

function fitcubic(X :: AbstractVector, Y :: AbstractVector, options :: Options)
  X, Y = checkdata(X,Y)
  @. model(x,p) = p[1]*x^3 + p[2]*x^2 + p[3]*x + p[4]
  fit = find_best_fit(model, X, Y, 4, options)
  R = pearson(X,Y,model,fit)
  x, y, ypred = finexy(X,options.fine,model,fit)
  return Cubic(fit.param...,R,x,y,ypred,fit.resid)
end
fitcubic(X :: AbstractVector, Y :: AbstractVector) = fitcubic(X,Y,Options())

function Base.show( io :: IO, fit :: Cubic )
  println("")
  println(" ------------------- Cubic Fit ----------------- ")
  println("")
  println(" Equation: y = ax^3 + bx^2 + cx + d ")
  println("")
  println(" With: a = ", fit.a)
  println("       b = ", fit.b)
  println("       c = ", fit.c)
  println("       d = ", fit.d)
  println("")
  println(" Pearson correlation coefficient, R = ", fit.R)
  println("")
  println(" Predicted Y: ypred = [",fit.ypred[1],", ",fit.ypred[2],"...")
  println(" residues = [", fit.residues[1],", ",fit.residues[2],"...")
  println("")
  println(" ----------------------------------------------- ")
end

export fitcubic


using Printf
import Flux: σ
using ModelingToolkit
using GalacticOptim
using Optim
using DiffEqFlux
using NeuralPDE
using Quadrature, Cubature, Cuba
using Plots

@parameters t,x,y
@variables c(..)
@derivatives Dt'~t
@derivatives Dxx''~x
@derivatives Dyy''~y
@derivatives Dx'~x
@derivatives Dy'~y

# Parameters

u = 1.0
v = 1.0
v_vector = (u, v)
R = 0
D = 0 # 0.1 # diffusion
t_max = 2.0
x_min = -1.0
x_max = 1.0
y_min = -1.0
y_max = 1.0

# exp(-(x^2+y^2)/0.1)
# div(v_vector * c) = dx(uc) + dy(vc)
# Equations, initial and boundary conditions
eqs = [ Dt(c(t,x,y)) ~ D * (Dxx(c(t,x,y)) + Dyy(c(t,x,y))) - (u*Dx(c(t,x,y)) + v*Dy(c(t,x,y))) + R]

bcs = [
        c(0, x, y) ~ exp(-(x^2+y^2)/0.1), #cos(π*x) * cos(π*y) + 1.0,
        c(t, x_min, y) ~ c(t, x_max, y),
        c(t, x, y_min) ~ c(t, x, y_max)
]

# Space and time domains
domains = [t ∈ IntervalDomain(0.0,t_max),
        x ∈ IntervalDomain(x_min,x_max),
        y ∈ IntervalDomain(y_min,y_max)
]

# Discretization
nx = 32
# ny = 32
dx = (x_max-x_min) / (nx - 1)
# dy = (y_max-y_min) / (ny -1)
dt = 0.01

# Neural network
dim = length(domains)
output = length(eqs)
hidden = 32

chain = FastChain( FastDense(dim, hidden, tanh),
                    FastDense(hidden, hidden, tanh),
                    FastDense(hidden, 1))

# strategy = GridTraining(dx=[dt,dx,dy])
strategy = StochasticTraining(900)

discretization = PhysicsInformedNN(chain, strategy=strategy)

pde_system = PDESystem(eqs, bcs, domains, [t,x,y], [c])
prob = discretize(pde_system,discretization)

cb = function (p,l)
    println("Current loss is: $l")
    return false
end

res = GalacticOptim.solve(prob,ADAM();cb=cb,maxiters=100)

# Plots

phi = discretization.phi

initθ = discretization.initθ

acum =  [0;accumulate(+, length.(initθ))]
sep = [acum[i]+1 : acum[i+1] for i in 1:length(acum)-1]
minimizers = [res.minimizer[s] for s in sep]
ts,xs,ys = [domain.domain.lower:dx:domain.domain.upper for domain in domains]

# Plot initial condition
c_predict = reshape([ phi([0, x, y], res.minimizer)[1] for x in xs for y in ys], length(xs), length(ys))
plot(xs, ys, c_predict)

# Animate
anim = @animate for (i, t) in enumerate(0:dt:t_max)
    @info "Animating frame $i..."
    c_predict = reshape([phi([t, x, y], res.minimizer)[1] for x in xs for y in ys], length(xs), length(ys))
    title = @sprintf("Advection-diffusion t = %.3f", t)
    heatmap(xs, ys, c_predict, label="", title=title , xlims=(-1, 1), ylims=(-1, 1), color=:thermal, clims=(0, 1))
end

gif(anim, "advection_diffusion_2d_pinn.gif", fps=15)

# Plot correct solution
using JLD2
file = jldopen("advection_diffusion/simulation/cosine_advection_diffusion.jld2")
iterations = parse.(Int, keys(file["timeseries/t"]))

anim = @animate for (i, iter) in enumerate(iterations)
    @info "Animating frame $i..."
    Hx = file["grid/Hx"]
    x = file["grid/xC"][1+Hx:end-Hx]
    t = file["timeseries/t/$iter"]
    c = file["timeseries/c/$iter"][:]

    title = @sprintf("Advection-diffusion t = %.3f", t)
    p = plot(x, c .+ 1, linewidth=2, title=title, label="Oceananigans",
             xlabel="x", ylabel="Tracer", xlims=(-1, 1), ylims=(0, 2))

    c_predict = reshape([phi([t, x], res.minimizer)[1] for x in xs], length(xs))

    plot!(p, x, c_predict, linewidth=2, label="Neural PDE")
end

gif(anim, "tanh_hidden32_it100_comparison.gif", fps=15)


# p11.jl - Chebyshev differentation of a smooth function

xx = -1:.01:1; uu = @. exp(xx)*sin(5*xx); clf();
for N = [10 20]
    (D,x) = cheb(N); u = @. exp(x)*sin(5*x);
    axes([.15,.66-.4*(N==20),.31,.28]);
    plot(x,u,".",markersize=6); grid(true);
    plot(xx,uu);
    title("u(x),  N=$N")
    error = D*u - @. exp(x)*(sin(5*x)+5*cos(5*x));
    axes([.55,.66-.4*(N==20),.31,.28]);
    plot(x,error,".-",markersize=6); grid(true);
    title("error in u'(x),  N=$N");
end


"""
    rotation_matrix_zyx(α1, α2, α3)

Calculate the rotation matrix ``R_{ab}`` that transforms a vector in b frame to a frame
(``v_{a} = R_{ab} v_{b}``) given the rotations (α1, α2, α3) (rad).

Frame b is obtained from frame a performing three intrinsic rotations of magnitude
α1, α2 and α3 in ZYX order.
"""
function rotation_matrix_zyx(α1, α2, α3)

    sα1, cα1 = sin(α1), cos(α1)
    sα2, cα2 = sin(α2), cos(α2)
    sα3, cα3 = sin(α3), cos(α3)

    R = @SMatrix [
        cα2 * cα1   (sα3 * sα2 * cα1 - cα3 * sα1)   (cα3 * sα2 * cα1 + sα3 * sα1)
        cα2 * sα1   (sα3 * sα2 * sα1 + cα3 * cα1)   (cα3 * sα2 * sα1 - sα3 * cα1)
        -sα2        sα3 * cα2                       cα3 * cα2
    ]

    return R
end

"""
    rotation_matrix_zyx(q0, q1, q2, q3)

Calculate the rotation matrix ``R_{ab}`` that transforms a vector in b frame to a frame
(``v_{a} = R_{ab} v_{b}``) given the quaternions ``q_0, q_1, q_2, q_3``.
"""
function rotation_matrix_zyx(q0, q1, q2, q3)

    q02, q12, q22, q32 = q0*q0, q1*q1, q2*q2, q3*q3

    R = @SMatrix [
        (q02+q12-q22-q32)    2*(q1*q2 - q0*q3)    2*(q1*q3 + q0*q2)
        2*(q1*q2 + q0*q3)    (q02-q12+q22-q32)    2*(q2*q3 - q0*q1)
        2*(q1*q3 - q0*q2)    2*(q2*q3 + q0*q1)    (q02-q12-q22+q32)
    ]

    return R
end

"""
    quaternions(ψ, θ, ϕ)

Calculate quaternion representation given the Euler angles (ψ, θ, ϕ) (rad).
"""
function quaternions(ψ, θ, ϕ)

    s_ψ2, c_ψ2 = sin(ψ/2), cos(ψ/2)
    s_θ2, c_θ2 = sin(θ/2), cos(θ/2)
    s_ϕ2, c_ϕ2 = sin(ϕ/2), cos(ϕ/2)

    q0 = c_ψ2*c_θ2*c_ϕ2 + s_ψ2*s_θ2*s_ϕ2
    q1 = c_ψ2*c_θ2*s_ϕ2 - s_ψ2*s_θ2*c_ϕ2
    q2 = c_ψ2*s_θ2*c_ϕ2 + s_ψ2*c_θ2*s_ϕ2
    q3 = s_ψ2*c_θ2*c_ϕ2 - c_ψ2*s_θ2*s_ϕ2

    return @SVector [q0, q1, q2, q3]
end

"""
    euler_angles(q0, q1, q2, q3)

Calculate Euler angles (ψ, θ, ϕ) (rad) given the quaternions ``q_0, q_1, q_2, q_3``.
"""
function euler_angles(q0, q1, q2, q3)

    ψ = atan(2 * (q1*q2 + q0*q3), q0*q0 + q1*q1 - q2*q2 - q3*q3)
    θ = asin(-2 * (q1*q3 - q0*q2))
    ϕ = atan(2 * (q2*q3 + q0*q1), q0*q0 - q1*q1 - q2*q2 + q3*q3)

    return @SVector [mod2pi(ψ), θ, ϕ]
end

"""
    body2horizon(x, y, z, ψ, θ, ϕ)

Transform the vector coordintes (x, y, z) given in body axis to local horizon given the
Euler angles (ψ, θ, ϕ) (rad).
"""
function body2horizon(x, y, z, ψ, θ, ϕ)
    v = @SVector [x, y, z]
    rv = rotation_matrix_zyx(ψ, θ, ϕ) * v
    return rv
end

"""
    horizon2body(x, y, z, ψ, θ, ϕ)

Transform the vector coordintes (x, y, z) given in local horizon axis to body given the
Euler angles (ψ, θ, ϕ) (rad).
"""
function horizon2body(x, y, z, ψ, θ, ϕ)
    v = @SVector [x, y, z]
    rv = transpose(rotation_matrix_zyx(ψ, θ, ϕ)) * v
    return rv
end

"""
    body2horizon(x, y, z, q0, q1, q2, q3)

Transform the vector coordintes (x, y, z) given in body axis to local horizon given the
quaternions ``q_0, q_1, q_2, q_3``.
"""
function body2horizon(x, y, z, q0, q1, q2, q3)
    v = @SVector [x, y, z]
    rv = rotation_matrix_zyx(q0, q1, q2, q3) * v
    return rv
end

"""
    horizon2body(x, y, z, q0, q1, q2, q3)

Transform the vector coordintes (x, y, z) given in local horizon axis to body given the
quaternions ``q_0, q_1, q_2, q_3``.
"""
function horizon2body(x, y, z, q0, q1, q2, q3)
    v = @SVector [x, y, z]
    rv = transpose(rotation_matrix_zyx(q0, q1, q2, q3)) * v
    return rv
end

"""
    wind2body(x, y, z, α, β)

Transform the vector coordintes (x, y, z) given in wind axis to body given the angle of
attack (α) and the angle of sideslip (β) (rad).
"""
function wind2body(x, y, z, α, β)
    v = @SVector [x, y, z]
    rv = transpose(rotation_matrix_zyx(-β, α, 0)) * v
    return rv
end

"""
    body2wind(x, y, z, α, β)

Transform the vector coordintes (x, y, z) given in body axis to wind given the angle of
attack (α) and the angle of sideslip (β) (rad).
"""
function body2wind(x, y, z, α, β)
    v = @SVector [x, y, z]
    rv = rotation_matrix_zyx(-β, α, 0) * v
    return rv
end

"""
    wind2horizon(x, y, z, χ, γ, μ)

Transform the vector coordintes (x, y, z) given in wind axis to local horizon given the
velocity angles (χ, γ, μ) (rad).
"""
function wind2horizon(x, y, z, χ, γ, μ)
    v = @SVector [x, y, z]
    rv = rotation_matrix_zyx(χ, γ, μ) * v
    return rv
end

"""
    horizon2wind(x, y, z, χ, γ, μ)

Transform the vector coordintes (x, y, z) given in local horizon axis to wind given the
velocity angles (χ, γ, μ) (rad).
"""
function horizon2wind(x, y, z, χ, γ, μ)
    v = @SVector [x, y, z]
    rv = transpose(rotation_matrix_zyx(χ, γ, μ)) * v
    return rv
end

"""
    ecef2horizon(x, y, z, lat, lon)

Transform the vector coordintes (x, y, z) given in ECEF (Earth Fixed Earth Centered)
coordinates to local horizon coordinates using geodetic latitude and longitude (rad).
"""
function ecef2horizon(x, y, z, lat, lon)
    v = @SVector [x, y, z]
    rv = transpose(rotation_matrix_zyx(lon, -lat - π/2, 0)) * v
end

"""
    horizon2ecef(x, y, z, lat, lon)

Transform the vector coordintes (x, y, z) given in local horizon axis to ECEF (Earth Fixed
Earth Centered) using geodetic latitude and longitude (rad).
"""
function horizon2ecef(x, y, z, lat, lon)
    v = @SVector [x, y, z]
    rv = rotation_matrix_zyx(lon, -lat - π/2, 0) * v
end


# chemical problems
# 22
function haverlys_pooling_problem(x)
    ## Haverly's Pooling Problem
    x1 = x[1]; x2 = x[2]; x3 = x[3];
    x4 = x[4]; x5 = x[5]; x6 = x[6];
    x7 = x[7]; x8 = x[8]; x9 = x[9];
    ## objective function
    f = zeros(2);
    f[1] = -9*x1-15*x2+6*x3+16*x4;
    f[2] = 10 .* (x5+x6);
    ## constraints
    g = zeros(2);
    h = zeros(4);
    g[1] = x9 .* x7+2*x5-2.5*x1;
    g[2] = x9 .* x8+2*x6-1.5*x2;
    h[1] = x7+x8-x4-x3;
    h[2] = x1-x5-x7;
    h[3] = x2-x6-x8;
    h[4] = x9 .* x7+x9 .* x8-3 .* x3-x4;

    return f, g, h
end

# 23
function reactor_network_design(x)
    ## Reactor Network Design
    k1 = 0.09755988; k2 = 0.99*k1;
    k3 = 0.0391908; k4 = 0.9*k3;
    x1 = x[1]; x2 = x[2]; x3 = x[3];
    x4 = x[4]; x5 = x[5]; x6 = x[6];
    ## objective function
    f = zeros(2);
    f[1] = -x4;
    f[2] = x5 .^ (0.5)+x6 .^ (0.5);
    ## constraints
    g = zeros(1);
    h = zeros(4);
    g[1] = f[2]-4;
    h[1] = k1 .* x5 .* x2 + x1 -1;
    h[2] = k3 .* x5 .* x3+x3+x1-1;
    h[3] = k2 .* x6 .* x2 - x1 + x2;
    h[4] = k4 .* x6 .* x4 + x2-x1+x4-x3;

    return f, g, h
end

# 24
function heat_exchanger_network_design(x)
    ## Heat Exchanger Network Design
    x1 = x[1]; x2 = x[2]; x3 = x[3]; x4 = x[4]; x5 = x[5];
    x6 = x[6]; x7 = x[7]; x8 = x[8]; x9 = x[9];
    ## objective function
    f = zeros(3);
    f[1] = 35 .* x1 .^ (0.6)+ 35 .* x2 .^ (0.6);
    f[2] = 200 .* x1 .* x4-x3;
    f[3] = 200 .* x2 .* x6-x5;
    ## constraints
    g = zeros(1);
    h = zeros(6);
    h[1] = x3 - 1e4 .* (x7-100);
    h[2] = x5 - 1e4 .* (300-x7);
    h[3] = x3 - 1e4 .* (600-x8);
    h[4] = x5 - 1e4 .* (900-x9);
    h[5] = x4 .* log(abs(x8-100)+1e-6)-x4 .* log(abs(600-x7)+1e-6)-x8+x7+500;
    h[6] = x6 .* log(abs(x9-x7)+1e-6)-x6 .* log(600)-x9+x7+600;

    return f, g, h
end


module FermiDiracIntegrals

using Polylogarithms

export F

"""
Complete Fermi-Dirac-integral

Arguments:
* j
* x

Formula:
``{F_j(x) = \\frac{1}{\\Gamma(j+1)} \\int_0^{\\infty}{\\frac{t^j}{\\exp(t-x)+1}dt}}``

Implementation:
Using the polylogarithm
"""
function F(j,x)
    -polylog(j+1,-exp(x))
end

"""
Approximation of the complete Fermi-Dirac-integral for j = 1/2

Checked for a relative tolerance of 3% in the range x = -100:0.1:100

Speed: 100 times faster than the polylog version

Source:
J. S. Blakemore: Approximations for Fermi-Dirac Integrals. Solid-State Electronics, 25(11):1067-1076, 1982.
"""
function F(::Val{1/2},x)
    if x < 1.3
        1/(exp(-x)+0.27)
    else
        4/3/sqrt(pi)*(x^2+pi^2/6)^(3/4)
    end
end

end


const DT = Integer

function find_amicable(limit::DT)
  T = typeof(limit)
  v = Vector{Tuple{Pair{T, Vector{T}}, Pair{T, Vector{T}}}}()
  t = Set{T}()

  for n ∈ one(T):limit
    d₁ = divisors(n)
    p = sum(d₁)
    d₂ = divisors(p)
    s = sum(d₂)

    if s == n && d₁ ≠ d₂ && s ∉ t
      push!(t, s)
      push!(t, p)
      push!(v, (n => d₁, p => d₂))
    end
  end

  (t, v)
end

# (Set([2924, 5564, 6232, 5020, 220, 284, 1184, 2620, 1210, 6368]),
#  [(220 => [1, 2, 4, 5, 10, 11, 20, 22, 44, 55, 110], 284 => [1, 2, 4, 71, 142]), (1184 => [1, 2, 4, 8, 16, 32, 37, 74, 148, 296, 592], 1210 => [1, 2, 5, 10, 11, 22, 55, 110, 121, 242, 605]), (2620 => [1, 2, 4, 5, 10, 20, 131, 262, 524, 655, 1310], 2924 => [1, 2, 4, 17, 34, 43, 68, 86, 172, 731, 1462]), (5020 => [1, 2, 4, 5, 10, 20, 251, 502, 1004, 1255, 2510], 5564 => [1, 2, 4, 13, 26, 52, 107, 214, 428, 1391, 2782]), (6232 => [1, 2, 4, 8, 19, 38, 41, 76, 82, 152, 164, 328, 779, 1558, 3116], 6368 => [1, 2, 4, 8, 16, 32, 199, 398, 796, 1592, 3184])])
#
# julia> s |> sum
# 31626

function is_amicable(n₁::Any, n₂::Any)
  throw(ArgumentError("n₁ and n₂ should be Integers"))
end

function is_amicable(n₁::DT, n₂::DT)::Bool
  d₁ = divisors(n₁)
  d₂ = divisors(n₂)
  n₂ == sum(d₁) && n₁ == sum(d₂)
end

function divisors(n::DT)::Vector{<: DT}
  T = typeof(n)
  ary::Vector{T} = [one(T), ]

  d = T(2)
  while d * d ≤ n
    q, r = divrem(n, d)
    if r == 0
      q == d ? push!(ary, d) : push!(ary, d, q)
    end
    d += 1
  end

  return ary |> sort
end


# # Singular Integrals

# This example shows how to use the `Integration` module for computing
# quadrature rules for functions with point singularities (such as those
# appearing in the numerical discretization of boundary integral equations).

# ## Change of variables

# The first set of *tricks* revolve a round a simple change of variables. We
# focus first on the one-dimensional case, where we wish to integrate
```math
    \int_0^1 f(x) dx,
```
# and where the function ``f`` can have an integrable singularity at

using WaveProp.Geometry
using WaveProp.Integration
using WaveProp.Integration
using QuadGK

f        = (x) -> x==0 ? 0.0 : log(abs(x))*cos(x)
I,_        = quadgk(f,0,1,rtol=1e-16)

rows = GaussLegendre.([5,10,20,40,80])
cols = [identity,IMT{1,2}(),Kress{8}(), Window{1,1,7}()]
ee   = []
for qstd in rows
    for shandler in cols
        q        = SingularQuadratureRule(qstd,shandler)
        Is       = integrate(f,q)
        er       = abs(I-Is)
        push!(ee,er)
    end
end

using NamedArrays
ee = reshape(ee,length(cols),length(rows)) |> transpose |> NamedArray
setnames!(ee,string.(rows),1)
setnames!(ee,string.(cols),2)
setdimnames!(ee,["Base quadrature","Singularity handler"])
show(ee)

# ##
# using Plots
# qstd = GaussLegendre(10)
# x̂,ŵ  = qstd()
# cols = [IMT{1,2}(),Kress{8}(), Window{0.5,1,7}()]
# fig = plot()
# for shandler in cols
#     phi       = shandler.(x̂)
#     phip      = [jacobian(shandler,x)[1] for x in x̂]
#     plot!(fig,x̂,f.(phi) .* phip,label=string(shandler),m=:x)
# end
# display(fig)


immutable HessenbergQR{T}
    cosθ::Vector{T}
    sinθ::Vector{T}
end

## QR decomposition on an upper Hessenberg matrix
## H will be overwritten by Q'HQ=RQ
function hessenqr!{T}(H::Matrix{T})
    ## Size of matrix
    n = size(H, 1)
    if n != size(H, 2)
        error("matrix must be square")
    end

    ## Rotation factors
    ## Each pair (cosθ[i], sinθ[i]) forms a rotation matrix
    ## Gi = [cosθ[i]  -sinθ[i]]
    ##      [sinθ[i]   cosθ[i]]
    cosθ = ones(T, n)
    sinθ = zeros(T, n)

    for i = 1:(n - 1)
        ## Make sure H is upper Hessenberg
        ## Zero the elements below H[i+1, i]
        H[(i + 2):end, i] = zero(T)
        ## Calculate cosθ and sinθ
        x = H[i, i]
        y = H[i + 1, i]
        r = hypot(x, y)
        ## If r is too small, (cosθ, sinθ) stores the original values (1, 0)
        if r < eps(T)
            r = zero(T)
        else
            cosθ[i] = x / r
            sinθ[i] = -y / r
        end
        ## Apply the rotation on the left H -> Gi * H
        ## H[i, :]     <- cosθ[i] * H[i, :] - sinθ[i] * H[i +1, :]
        ## H[i + 1, :] <- sinθ[i] * H[i, :] + cosθ[i] * H[i +1, :]
        H[i, i] = r
        H[i + 1, i] = zero(T)
        c = cosθ[i]
        s = sinθ[i]
        for j = (i + 1):n
            tmp = H[i, j]
            H[i, j]     = c * tmp - s * H[i + 1, j]
            H[i + 1, j] = s * tmp + c * H[i + 1, j]
        end
    end

    ## Apply the rotations on the right H -> H * Gi'
    ## H[:, i]     <- cosθ[i] * H[:, i] - sinθ[i] * H[:, i + 1]
    ## H[:, i + 1] <- sinθ[i] * H[:, i] + cosθ[i] * H[:, i + 1]
    for i = 1:(n - 1)
        for j = 1:(i + 1)
            tmp = H[j, i]
            H[j, i]     = cosθ[i] * tmp - sinθ[i] * H[j, i + 1]
            H[j, i + 1] = sinθ[i] * tmp + cosθ[i] * H[j, i + 1]
        end
    end

    return HessenbergQR(cosθ, sinθ)
end

## QR decomposition on a tridiagonal matrix
## H will be overwritten by Q'HQ=RQ
function tridiagqr!{T}(H::Matrix{T})
    ## Size of matrix
    n = size(H, 1)
    if n != size(H, 2)
        error("matrix must be square")
    end

    ## Force H to be tridiagonal
    maindiag = diag(H)
    subdiag = diag(H, -1)
    fill!(H, zero(T))
    for i = 1:(n - 1)
        H[i, i] = maindiag[i]
        H[i + 1, i] = subdiag[i]
        H[i, i + 1] = subdiag[i]
    end
    H[n, n] = maindiag[n]

    ## Rotation factors
    ## Each pair (cosθ[i], sinθ[i]) forms a rotation matrix
    ## Gi = [cosθ[i]  -sinθ[i]]
    ##      [sinθ[i]   cosθ[i]]
    cosθ = ones(T, n)
    sinθ = zeros(T, n)

    for i = 1:(n - 1)
        ## Calculate cosθ and sinθ
        x = H[i, i]
        y = H[i + 1, i]
        r = hypot(x, y)
        ## If r is too small, (cosθ, sinθ) stores the original values (1, 0)
        if r < eps(T)
            r = zero(T)
        else
            cosθ[i] = x / r
            sinθ[i] = -y / r
        end
        ## Apply the rotation on the left H -> Gi * H
        ## Update H[i, i] and H[i + 1, i]
        H[i, i] = r
        H[i + 1, i] = zero(T)
        ## Update H[i, i + 1] and H[i + 1, i + 1]
        c = cosθ[i]
        s = sinθ[i]
        tmp = H[i, i + 1]
        H[i,     i + 1] = c * tmp - s * H[i + 1, i + 1]
        H[i + 1, i + 1] = s * tmp + c * H[i + 1, i + 1]
        ## Update H[i, i + 2] and H[i + 1, i + 2]
        if i < n - 1
            H[i,     i + 2] = -s * H[i + 1, i + 2]
            H[i + 1, i + 2] *= c
        end
    end

    ## Apply the rotations on the right H -> H * Gi'
    ## H[:, i]     <- cosθ[i] * H[:, i] - sinθ[i] * H[:, i + 1]
    ## H[:, i + 1] <- sinθ[i] * H[:, i] + cosθ[i] * H[:, i + 1]
    for i = 1:(n - 1)
        c = cosθ[i]
        s = sinθ[i]

        tmp = H[i + 1, i]
        H[i, i]         = c * H[i, i] - s * H[i, i + 1]
        H[i + 1, i]     = c * tmp     - s * H[i + 1, i + 1]
        H[i + 1, i + 1] = s * tmp     + c * H[i + 1, i + 1]

        H[i, i + 1] = H[i + 1, i]
        if i < n - 1
            H[i, i + 2] = zero(T)
        end
    end

    return HessenbergQR(cosθ, sinθ)
end

## Apply the QR factorization to the right of a matrix A
## A -> A * Q'
function applyright!{T}(qr::HessenbergQR{T}, A::Matrix{T})
    cosθ = qr.cosθ
    sinθ = qr.sinθ
    n = length(cosθ)
    ## A[:, i]     <- cosθ[i] * A[:, i] - sinθ[i] * A[:, i + 1]
    ## A[:, i + 1] <- sinθ[i] * A[:, i] + cosθ[i] * A[:, i + 1]
    for i = 1:(n - 1)
        for j = 1:size(A, 1)
            tmp = A[j, i]
            A[j, i]     = cosθ[i] * tmp - sinθ[i] * A[j, i + 1]
            A[j, i + 1] = sinθ[i] * tmp + cosθ[i] * A[j, i + 1]
        end
    end
end


"""
    FrankCopula{d,T}

Fields:
  - θ::Real - parameter

Constructor

    FrankCopula(d, θ)

The [Frank](https://en.wikipedia.org/wiki/Copula_(probability_theory)#Most_important_Archimedean_copulas) copula in dimension ``d`` is parameterized by ``\\theta \\in [0,\\infty)``. It is an Archimedean copula with generator :

```math
\\phi(t) = -\\frac{\\log\\left(1+e^{-t}(e^{-\\theta-1})\\right)}{\theta}
```
"""
struct FrankCopula{d,T} <: ArchimedeanCopula{d}
    θ::T
end
FrankCopula(d,θ) = θ >= 0 ? FrankCopula{d,typeof(θ)}(θ) : @error "Theta must be positive"
ϕ(  C::FrankCopula,       t) = -log(1+exp(-t)*(exp(-C.θ)-1))/C.θ
ϕ⁻¹(C::FrankCopula,       t) = -log((exp(-t*C.θ)-1)/(exp(-C.θ)-1))

D₁ = GSL.sf_debye_1 # sadly, this is C code.
# could be replaced by :
# using QuadGK
# D₁(x) = quadgk(t -> t/(exp(t)-1), 0, x)[1]/x
# to make it more general. but once gain, it requires changing the integrator at each evlauation,
# which is problematic.
# Better option is to try to include this function into SpecialFunctions.jl.

τ(C::FrankCopula) = 1+4(D₁(C.θ)-1)/C.θ
function τ⁻¹(::Type{FrankCopula},τ)
    if τ == zero(τ)
        return τ
    end
    x₀ = (1-τ)/4
    return Roots.fzero(x -> (1-D₁(x))/x - x₀, 10)
end

radial_dist(C::FrankCopula) = Logarithmic(1-exp(-C.θ))


using EngEconomics

# Given
i = 0.1
xInit = 12500
yInit = 8900
xAnnualBenefit = 6800
yAnnualBenefit = 2000
xSalvage = 5000
ySalvage = 8900
xLife = 2
yLife = 3

# Find: Present Worth Analysis of Both

# Determine the least common multiple of xLife and yLife to figure out how many
# times we have to repeat the PW Analysis
years = 6
xTimes = years / xLife
yTimes = years / yLife

# Determine Product X's PW
xInitPW = -xInit * (1 + presentWorthFactor(i, xLife) + presentWorthFactor(i, 2 * xLife))
xAnnualBenefitPW = (xAnnualBenefit * seriesPresentAmountFactor(i, xLife)) * (1 + presentWorthFactor(i, xLife) + presentWorthFactor(i, 2 * xLife))
xSalvagePW = xSalvage * (presentWorthFactor(i, xLife) + presentWorthFactor(i, 2 * xLife) + presentWorthFactor(i, 3 * xLife))
xPW = xInitPW + xAnnualBenefitPW + xSalvagePW

# Determine Product Y's PW
yInitPW = -yInit * (1 + presentWorthFactor(i, yLife))
yAnnualBenefitPW = (yAnnualBenefit * seriesPresentAmountFactor(i, yLife)) * (1 + presentWorthFactor(i, yLife))
ySalvagePW = ySalvage * (presentWorthFactor(i, yLife) + presentWorthFactor(i, 2 * yLife))
yPW = yInitPW + yAnnualBenefitPW + ySalvagePW

# Determine which product is better by comparing they're present value
if xPW > yPW
	println("Pick Product X")
else
	println("Pick Product Y")
end


import Pkg; Pkg.add(Pkg.PackageSpec(url="https://github.com/JuliaComputing/JuliaAcademyData.jl"))
using JuliaAcademyData; activate("Deep learning with Flux")

#

# ## Going deep: Deep neural networks
#
# So far, we've learned that if we want to classify more than two fruits, we'll need to go beyond using a single neuron and use *multiple* neurons to get multiple outputs. We can think of stacking these multiple neurons together in a single neural layer.
#
# Even so, we found that using a single neural layer was not enough to fully distinguish between bananas, grapes, **and** apples. To do this properly, we'll need to add more complexity to our model. We need not just a neural network, but a *deep neural network*.
#
# There is one step remaining to build a deep neural network. We have been saying that a neural network takes in data and then spits out `0` or `1` predictions that together declare what kind of fruit the picture is. However, what if we instead put the output of one neural network layer into another neural network layer?
#
# This gets pictured like this below:
#
#
#
# On the left we have 3 data points in blue. Those 3 data points each get fed into 4 neurons in purple. Each of those 4 neurons produces a single output, but those output are each fed into three neurons (the second layer of purple). Each of those 3 neurons spits out a single value, and those values are fed as inputs into the last layer of 6 neurons. The 6 values that those final neurons produce are the output of the neural network. This is a deep neural network.

#-

# ### Why would a deep neural network be better?
#
# This is a little perplexing when you first see it. We used neurons to train the model before: why would sticking the output from neurons into other neurons help us fit the data better? The answer can be understood by drawing pictures. Geometrically, the matrix multiplication inside of a layer of neurons is streching and rotating the axis that we can vary:

## using Plots, Random; Random.seed!(0); xs = range(0, stop=10, length=200); ys = randn.() .+ xs .+ 0.5.*xs.^1.5 .+ 6.0.*sin.(clamp.(xs .- 5, 0, Inf)) .+ 4.5.*clamp.(.-abs.(xs .- 2), -2, 0); scatter(xs, ys, label="", ticks = false, title="Original data", ylim=extrema(ys).+[-.5,.5]); savefig("data/17-raw.png"); fit1 = [xs ones(size(xs))] \ ys; linear1(x) = fit1[1]*x + fit1[2]; plot!(linear1, label="Linear transform", linewidth=4, legend=:topleft); savefig("data/17-raw-withfit.png");
HTML("""""")

# A nonlinear transformation—such as the sigmoid function or a piecewise linear function—then adds a bump to this linearly-transformed data:

## scatter(xs, ys .- linear1.(xs), ticks = false, label = "", title="data with linear transform", ylim=extrema(ys).+[-.5,.5]); savefig("data/17-linear1.png"); nonlinearity(x) = clamp.(4.0.*x .- 4.5, -4, 4); plot!(nonlinearity, label = "nonlinearity", linewidth=4); savefig("data/17-linear1-withfit.png");
HTML("""""")

# Resulting in a bit more of the data accounted for:

## ys2 = ys .- linear1.(xs) .- nonlinearity.(xs); scatter(xs, ys2, ticks = false, label = "", title="data after first 'layer'", ylim=extrema(ys).+[-.5,.5]); savefig("data/17-nonlinear1.png"); fit2 = [xs ones(size(xs))] \ ys2; linear2(x) = fit2[1]*x + fit2[2]; plot!(linear2, label="second linear tranform", linewidth=4); savefig("data/17-nonlinear1-withfit.png");
HTML("""""")

# Now let's repeat this process. When we send the data through another layer of neurons, we get another rotation and another "bump":

## ys3 = ys2 .- linear2.(xs); scatter(xs .- 4.5, ys3, ticks = false, label = "", title="data after first layer + second linear transform", ylim=extrema(ys).+[-.5,.5]); savefig("data/17-linear2.png"); plot!(nonlinearity, label = "nonlinearity", linewidth=4); savefig("data/17-linear2-withfit.png");
HTML("""""")

#-

## ys4 = ys3 .- nonlinearity.(xs .- 4.5); scatter(xs, ys4, ticks = false, label = "", title="data after second 'layer'", ylim=extrema(ys).+[-.5,.5]); savefig("data/17-nonlinear2.png");
HTML("""""")

#
# Visually, we see that if we keep doing this process we can make the axis line up with any data. What this means is that **if we have enough layers, then our neural network can approximate any model**.
#
# The trade-off is that with more layers we have more parameters, so it may be harder (i.e. computationally intensive) to train the neural network. But we have the guarantee that the model has enough freedom such that there are parameters that will give the correct output.
#
# Because this model is so flexible, the problem is reduced to that of learning: do the same gradient descent method on this much larger model (but more efficiently!) and we can make it classify our data correctly. This is the power of deep learning.

#-

# **Important caveat**: The above is a _cartoon_ that loosely describes the concepts behind a deep neural network. In reality, remember that we're not successively fitting "the best" linear transformation and then subsequently fitting a nonlinearity; the two are happening simultaneously through gradient descent. This allows the linear transform to — in effect — rotate and stretch the data such that the nonlinearity "falls" in the appropriate place to best minimize the loss function.

#-

# # Deep learning with Flux

#-

# Let's load the same datasets from the previous lecture and pre-process them in the same way:

using CSV, DataFrames, Flux, Plots
apples1 = DataFrame(CSV.File("data/Apple_Golden_1.dat", delim='\t', allowmissing=:none, normalizenames=true))
apples2 = DataFrame(CSV.File("data/Apple_Golden_2.dat", delim='\t', allowmissing=:none, normalizenames=true))
apples3 = DataFrame(CSV.File("data/Apple_Golden_3.dat", delim='\t', allowmissing=:none, normalizenames=true))
apples = vcat(apples1, apples2, apples3)
bananas = DataFrame(CSV.File("data/Banana.dat", delim='\t', allowmissing=:none, normalizenames=true))
grapes1 = DataFrame(CSV.File("data/Grape_White.dat", delim='\t', allowmissing=:none, normalizenames=true))
grapes2 = DataFrame(CSV.File("data/Grape_White_2.dat", delim='\t', allowmissing=:none, normalizenames=true))
grapes = vcat(grapes1, grapes2)
## Extract out the features and construct the corresponding labels
x_apples  = [ [apples[i, :red], apples[i, :blue]] for i in 1:size(apples, 1) ]
x_bananas  = [ [bananas[i, :red], bananas[i, :blue]] for i in 1:size(bananas, 1) ]
x_grapes = [ [grapes[i, :red], grapes[i, :blue]] for i in 1:size(grapes, 1) ]
xs = vcat(x_apples, x_bananas, x_grapes)
ys = vcat(fill(Flux.onehot(1, 1:3), size(x_apples)),
          fill(Flux.onehot(2, 1:3), size(x_bananas)),
          fill(Flux.onehot(3, 1:3), size(x_grapes)));

# In the previous lecture, we used a `Dense(2, 3, σ)` as our model. Now we want to construct multiple layers and chain them together:

layer1 = Dense(2, 4, σ)
layer2 = Dense(4, 3, σ)

#-

layer2(layer1(xs[1]))

#-

#nb ?Chain
#jl @doc Chain

#-

m = Chain(layer1, layer2)
m(xs[1])

#-

xs[1] |> layer1 |> layer2

# ### The core algorithm from the last lecture

model = Chain(Dense(2, 3, σ)) # Update this!
L(x,y) = Flux.mse(model(x), y)
opt = SGD(params(model))
Flux.train!(L, zip(xs, ys), opt)

#-

## Recall his is how we repeatedly walked down our gradient previously...
for _ in 1:1000
    Flux.train!(L, zip(xs, ys), opt)
end
## But our model is now more complicated and this will take more time!

#-

data = zip(xs, ys)
@time Flux.train!(L, data, opt)
@time Flux.train!(L, data, opt)

# ### Improving efficiency by batching

length(data)

#-

first(data)

# Recall our matrix-vector multiplication from the previous lecture:

W = [10 1;
     20 2;
     30 3]
x = [3;
     2]
W*x

#-

Flux.batch(xs)

#-

model(Flux.batch(xs))

#-

databatch = (Flux.batch(xs), Flux.batch(ys))
@time Flux.train!(L, (databatch,), opt)
@time Flux.train!(L, (databatch,), opt)

#-

Flux.train!(L, Iterators.repeated(databatch, 10000), opt)

#-

L(databatch[1], databatch[2])

# ### Visualization

using Plots
function plot_decision_boundaries(model, x_apples, x_bananas, x_grapes)
    plot()

    contour!(0:0.01:1, 0:0.01:1, (x,y)->model([x,y]).data[1], levels=[0.5, 0.501], color = cgrad([:blue, :blue]), colorbar=:none)
    contour!(0:0.01:1, 0:0.01:1, (x,y)->model([x,y]).data[2], levels=[0.5,0.501], color = cgrad([:green, :green]), colorbar=:none)
    contour!(0:0.01:1, 0:0.01:1, (x,y)->model([x,y]).data[3], levels=[0.5,0.501], color = cgrad([:red, :red]), colorbar=:none)

    scatter!(first.(x_apples), last.(x_apples), m=:cross, label="apples", color = :blue)
    scatter!(first.(x_bananas), last.(x_bananas), m=:circle, label="bananas", color = :green)
    scatter!(first.(x_grapes), last.(x_grapes), m=:square, label="grapes", color = :red)
end
plot_decision_boundaries(model, x_apples, x_bananas, x_grapes)

# ### Further improvements with a better loss function and normalization of outputs

scatter([0],[0], label="correct answer", xlabel="model output: [1-x,x]", ylabel="loss against [1, 0]", legend=:topleft, title="Loss function behavior")
plot!(x->Flux.mse([1-x, x/2], [1,0]), -1.5, 1.5, label="mse")
## plot!(x->Flux.crossentropy([1-x, x/2], [1,0]), 0, 1, label="crossentropy")

#-

sum(model(xs[1]))

#-

Flux.mse([0.01,0.98,0.01], [1.0,0,0])

#-

softmax([1.0,-3,0])

# ### The new algorithm
#
# Use `softmax` as a final normalization and change the loss function to `crossentropy`:

model = Chain(Dense(2, 4, σ), Dense(4, 3, identity), softmax)
L(x,y) = Flux.crossentropy(model(x), y)
opt = SGD(params(model))

#-

Flux.train!(L, Iterators.repeated(databatch,5000), opt)

#-

plot_decision_boundaries(model, x_apples, x_bananas, x_grapes)

Courses/Deep learning with Flux/1700.Introduction-to-deep-learning.jl


# Faster than Polynomials.jl with less memory allocation + single-point type
# stability; adapted from Octave and
# https://github.com/JuliaMath/Polynomials.jl/blob/master/src/Polynomials.jl
function poly(x::Array{T,1}) where T <: Number
  n = length(x)
  y = zeros(T, n+1)
  y[1] = one(T)
  for j = 1:n
    y[2:j+1] .-= x[j].*y[1:j]
  end
  return y
end

function polyval(p::Array{T1,1}, x::T2) where {T1 <: Number, T2 <: Number}
  y = T2(p[1])
  for i = 2:lastindex(p)
    y = p[i] .+ x*y
  end
  return y
end

function polyval(p::Array{T1,1}, x::Array{T2,1}) where {T1 <: Number, T2 <: Number}
  y = ones(T2, length(x)) .* p[1]
  for i = 2:length(p)
    broadcast!(*, y, y, x)
    broadcast!(+, y, y, p[i])
  end
  return y
end

function polyfit(x::Array{T1,1}, y::Array{T2,1}, n::Integer=1) where {T1 <: Real, T2 <: Real}
  nx = length(x)
  nx == length(y) || error("SeisIO.polyfit requires length(t) == length(x)")
  -1 < n < nx || throw(DomainError)
  A = Array{T2, 2}(undef, length(x), n+1)
  A[:,n+1] .= one(T2)
  for i = n:-1:1
      A[:,i] .= A[:,i+1] .* x
  end
  return A \ y
end

# Convert to Float64 or use improved sum
function linreg(t::Array{Float64,1}, x::AbstractArray{T,1}) where T
  n = length(t)
  st = sum(t)
  sx = sum(x)
  stt = dot(t,t)
  stx = dot(t,x)
  sxx = dot(x,x)
  d = n*stt - st*st
  b = (stt*sx - st*stx)/d
  a = (n*stx - st*sx)/d
  return T[a,b]
end

# p = linreg(x, fs)
function linreg(x::AbstractArray{T,1}, dt::Float64) where T
  n = length(x)
  t = (1:n)*dt
  st = sum(t)
  sx = sum(x)
  stt = dot(t,t)
  stx = dot(t,x)
  sxx = dot(x,x)
  d = n*stt - st*st
  b = (stt*sx - st*stx)/d
  a = (n*stx - st*sx)/d
  return T[a,b]
end

#= CHANGELOG for this file
2019-08-08
* poly, polyval, and polyfit should now always output powers in descending
order, i.e., p^n ... p^0
* BUG fixed: polyval(p::Array{T,1}, x::T) used power ordering of Polynomials.jl;
corrected to be consistent with other routines
* polyfit now allows order n=0 and takes any Integer for n; n=0 returns the mean
2019-08-19
* added linreg for low-memory linear regression; identical to SAC detrend
2019-09-03
* bug fix for rare situation where float precision led to length(t) != length(x)
=#


#######################
## Prepare libraries ##
#######################

# export JULIA_NUM_THREADS=4

BASE_FOLDER = pwd()
(basename(BASE_FOLDER) == "examples") && (BASE_FOLDER = dirname(BASE_FOLDER))
using Pkg
Pkg.activate(BASE_FOLDER)
# Install all required packages if not present
isfile(joinpath(BASE_FOLDER,"Manifest.toml")) || Pkg.instantiate()

using Combinatorics, Distributions, LightGraphs, LinearAlgebra, QuantEcon
using Roots, SymPy, Manifolds
using Nash

## Create game from provided payoff matrices
# Player 1 payoff matrix is
    # 1  0
    # 0  1
# Player 2 payoff matrix is
    # 1  0
    # 1  1
generate_game([1 0; 0 1], [1 0; 1 1])

## Generate random game for 2 players.
# First player has 2 actions and second player has 3 actions
# Payoffs are sampled from standard normal Distributions

random_2players_game(Normal(0,1),2,3)

## Generate random game for 3 players.
# First player has 2 actions, second player has 3 actions, third player has 3 actions
# Payoffs are sampled from Binomial distribution with 5 trials and p = 0.5

random_nplayers_game(Binomial(5,0.5),[2,3,3])

## Get payoffs for each player from given mixed strategy profiles
# Player 1 plays first and second action with p = 0.5
# Player 2 plays first action with p = 0.75 and second action with p = 0.25

game = random_nplayers_game(Binomial(5,0.5),[2,2]);
s = [[0.5,0.5],[0.75,0.25]]
s[1]'*game["player2"]*s[2]
get_payoff(game,s)

## Return best reply of first (k = 1) player
# with perturbation epsil=0.5
best_reply(generate_game([1 0; 0 1], [1 0; 0 1]), [[1.0, 0.], [0, 1]], 1, 0.5)
# no perturbation, with convex hull
a2 = best_reply(generate_game(Matrix(I,3,3), Matrix(I,3,3)), [[1/2,1/2,0],[1/3,1/3,1/3]], 1, return_val = "chull")
# plot best responses
plot_br(a2)

# Check if passed strategy is best response for each player and is it a Nash equilibrium
is_nash_q(generate_game([1 0 ; 0 1], [1 0; 0 1]), [[1, 0], [1, 0]])

# Iteratively find best reply with perturbation (epsil=1/3) and limit of 10 iterations
game = generate_game(Matrix(I, 3, 3), Matrix(I, 3, 3))
s = [[1/2,1/2, 0], [1/3,1/3,1/3]]
game_history = iterate_best_reply(game, s, 1/3, 10)

# Transform game into Markov Chain
game = generate_game(Matrix(1I, 3, 3), Matrix(I, 3, 3))
s = [[1, 0, 0],[0, 1, 0]]
mc = game2markov(game, s)
simulate(mc, 9, init = 3)
simulate(mc, 9) # random initial condition

is_irreducible(mc) # can we reach any state from any other state?
communication_classes(mc)
period(mc)
is_aperiodic(mc)
stationary_distributions(mc)

plot_markov(10, mc)

# Generate random symmetric game
game = random_symmetric_2players_game(Binomial(10,1/2),2)

# Find necessary values to find symmetric nash equilibrium
x = symbols("x", real=true)
find_symmetric_nash_equilibrium_2players_game(game, [x, 1-x])

# Create equations for replicator
create_replicator_eqs(game, [x, 1-x])

# Generate graph of relations that need to hold to make a game symmetric
actions_no_vector = [2, 2, 2];
graph = create_symmetries_graph(actions_no_vector)

# Check single relation
check_equality_condition(graph,((2,2,1),1), ((2,1,2),1))

# Find all needed relations
find_all_equalities(graph)

# counting vFunction - three players coordination game
pay = reshape([1 0 0 0 0 0 0 1], (2,2,2))
g1 = generate_game(pay, pay, pay)
vFunction(g1, [[1, 0], [1, 0], [1, 0]])
vFunction(g1, [[1, 0], [1, 0], [0, 1]])

# Creates cartersian product of simplices
s1 = [[1, 0], [1, 0], [0, 1]]
v1 = cart_prod_simplices(s1)

s2 = [[1, 0], [1, 0], [1, 2, 3]]
v2 = cart_prod_simplices(s2)


### A Pluto.jl notebook ###
# v0.12.20

using Markdown
using InteractiveUtils

# ╔═╡ 0b868b30-6420-11eb-18f9-1592f7542fe6
begin
    using PlutoUI
    using Plots
    using Printf
	using SpecialFunctions
end

# ╔═╡ 4c4fff40-64c8-11eb-2c45-57f4793ee997
md"_Parabolic Partial Differential Equation, version 1_"

# ╔═╡ 56d1f810-64c8-11eb-3e0d-81c0bbafdc62
md"""
## **Parabolic Partial Differential Equation**
#### **Description:**
This notebook computes the Parabolic Partial Differential Equation with 4 different discretization methods (2 explicit - 2 implicit)

		FTCS(Δt,Δy,ν)
		DuFortFrankel(Δt,Δy,ν)
		Laasonen(Δt,Δy,ν)
		CrankNicolson(Δt,Δy,ν)

If the arguments are missing, it will use the default values.

		FTCS(Δt=0.002,Δy=0.001,ν=0.000217)
		DuFortFrankel(Δt=0.002,Δy=0.001,ν=0.000217)
		Laasonen(Δt=0.002,Δy=0.001,ν=0.000217)
		CrankNicolson(Δt=0.002,Δy=0.001,ν=0.000217)

#### **Parabolic Partial Differential Equation**
In this problem, velocity profile between 2 infinite plates will be investigated. The bottom plates is moving with U₀ and the upper plate is stationary. The equation for the system can be represented by:

$$\frac{∂u}{∂t}=α\frac{∂²u}{∂x²}$$

Boundary conditions for the system:

$$u=U₀=1 \hspace{12pt} y=0$$
$$u=0    \hspace{36pt} y=h$$

where h is the distance between two plates. In this problem, it is taken as 0.04m. The partial system of equation is solved using following discritizations:

#### **Forward Time Central Space (FTCS) Method**

It is a first-order in time and second-order in space. The discritization for this partial equation:

$$u_{i}^{n+1}=u_{i}^{n}+d(u_{i+1}^{n}-2u_{i}^{n}+u_{i-1}^{n})$$

where

$$d=\frac{α(Δt)}{Δx^{2}}$$

For the stability condition:

$$\frac{α(Δt)}{Δx^{2}}=d\leqslant0.5$$

#### **DuFort-Frankel Method**

It is a second-order in time and second-order in space. The discritization for this partial equation:

$$(1+2d)u_{i}^{n+1}=(1-2d)u_{i}^{n-1}+2d(u_{i+1}^{n}+u_{i-1}^{n})$$

where

$$d=\frac{α(Δt)}{Δx^{2}}$$

For the stability condition:

$$\frac{α(Δt)}{Δx^{2}}\leqslant0.5$$

This method requires 2 sets of initial data so in this notebook, FTCS is used to obtain data set at t=Δt.

#### **Laasonen Method**

It is a first-order in time and second-order in space. The discritization for this partial equation:

$$du_{i-1}^{n+1}-(1+2d)u_{i}^{n+1}+du_{i+1}^{n+1}=-u_{i}^{n}$$

where

$$d=\frac{α(Δt)}{Δx^{2}}$$

This method is unconditionally stable.

#### **Crank Nicholson Method**

It is a second-order in time and second-order in space. The discritization for this partial equation:

$$-\frac{1}{2}du_{i-1}^{n+1}-(1+d)u_{i}^{n+1}-\frac{1}{2}du_{i+1}^{n+1}=\frac{1}{2}du_{i-1}^{n}+(1-d)u_{i}^{n}+\frac{1}{2}du_{i+1}^{n}$$

where

$$d=\frac{α(Δt)}{Δx^{2}}$$

This method is unconditionally stable.

Details of schemes:
Computational Fluid Dynamics, Volume I, Hoffmann - Chiang

Feel free to ask questions!

*Furkan Oz*
foz@okstate.edu
"""

# ╔═╡ 29683fb0-6419-11eb-05d0-2d7392bc8afa
"""
Computes the Forward Time Central Space for the Parabolic Partial Differential Equation
This notebook computes the Parabolic Partial Differential Equation
		FTCS(Δt,Δy,ν)
If the arguments are missing, it will use the default values.

		FTCS(Δt=0.002,Δy=0.001,ν=0.000217)
Furkan Oz,
foz@okstate.edu,

"""
function FTCS(Δt=0.002,Δy=0.001,ν=0.000217)
	#ν=0.000217 # viscosity [m²/s]
	h = 0.04 # Distance between 2 plate [m]
	T = 1.25 # Total time [s]
	#Δt = 0.002 # Time increment
	Nt = T/Δt+1 # Number of time step
	Nt = round(Int32,Nt)

	#Δy = 0.001 # Spacing between node points
	y = 0:Δy:h # y-coordinates
	Nx = h/Δy+1 # Number of Node points
	Nx = round(Int32,Nx)

	U₀ = 1 # Plate speed
	d = ν*Δt/(Δy)^2
	#Initializing
	u = zeros(Nx,Nt)

	# Boundary Conditions
	u[1,:] = [U₀ for i=1:Nt]
	u[Nx,:] = [0 for i=1:Nt]

	# Initial Conditions
	u[:,1] = [0 for i=1:Nx]
	u[1,1] = U₀
	for ii=1:Nt-1
		for i=2:Nx-1
			u[i,ii+1] = u[i,ii]+d*(u[i+1,ii]-2*u[i,ii]+u[i-1,ii])
		end
	end

	return u,y

end

# ╔═╡ 36be1c60-6442-11eb-204a-b37677c1acfd
"""
Computes the DuFort Frankel method for the Parabolic Partial Differential Equation
This notebook computes the Parabolic Partial Differential Equation
		DuFortFrankel(Δt,Δy,ν)
If the arguments are missing, it will use the default values.

		DuFortFrankel(Δt=0.002,Δy=0.001,ν=0.000217)
Furkan Oz,
foz@okstate.edu,

"""
function DuFortFrankel(Δt=0.002,Δy=0.001,ν=0.000217)
	#ν=0.000217 # viscosity [m²/s]
	h = 0.04 # Distance between 2 plate [m]
	T = 1.25 # Total time [s]
	#Δt = 0.002 # Time increment
	Nt = T/Δt+1 # Number of time step
	Nt = round(Int32,Nt)

	#Δy = 0.001 # Spacing between node points
	y = 0:Δy:h # y-coordinates
	Nx = h/Δy+1 # Number of Node points
	Nx = round(Int32,Nx)

	U₀ = 1 # Plate speed
	d = ν*Δt/(Δy)^2
	#Initializing
	u = zeros(Nx,Nt)

	# Boundary Conditions
	u[1,:] = [U₀ for i=1:Nt]
	u[Nx,:] = [0 for i=1:Nt]

	# Initial Conditions
	u[:,1] = [0 for i=1:Nx]
	u[1,1] = U₀
	for ii=1:1
		for i=2:Nx-1
			u[i,ii+1] = u[i,ii]+d*(u[i+1,ii]-2*u[i,ii]+u[i-1,ii])
		end
	end
	for ii=2:Nt-1
		for i=2:Nx-1
			u[i,ii+1] = ((1-2*d)*u[i,ii-1]+2*d*(u[i+1,ii]+u[i-1,ii]))/(1+2*d)
		end
	end
	return u,y

end

# ╔═╡ 899d6a70-6448-11eb-2870-692c6e93c421
"""
Computes the Laasonen method for the Parabolic Partial Differential Equation
This notebook computes the Parabolic Partial Differential Equation
		Laasonen(Δt,Δy,ν)
If the arguments are missing, it will use the default values.

		Laasonen(Δt=0.002,Δy=0.001,ν=0.000217)
Furkan Oz,
foz@okstate.edu,

"""
function Laasonen(Δt=0.002,Δy=0.001,ν=0.000217)
	#ν=0.000217 # viscosity [m²/s]
	h = 0.04 # Distance between 2 plate [m]
	T = 1.25 # Total time [s]
	#Δt = 0.002 # Time increment
	Nt = T/Δt+1 # Number of time step
	Nt = round(Int32,Nt)

	#Δy = 0.001 # Spacing between node points
	y = 0:Δy:h # y-coordinates
	Nx = h/Δy+1 # Number of Node points
	Nx = round(Int32,Nx)

	U₀ = 1 # Plate speed
	d = ν*Δt/(Δy)^2

	#Initializing
	A = zeros(Nx,Nx)
	b = zeros(Nx)
	u = zeros(Nx,Nt)

	# Boundary Conditions
	A[1,1] = 1
	b[1] = U₀
	A[Nx,Nx] = 1
	b[Nx] = 0

	# Initial Conditions
	u[:,1] = [0 for i=1:Nx]
	u[1,1] = U₀

	for ii=2:Nt-1
		# Boundary Conditions
		A[1,1] = 1
		b[1] = U₀
		A[Nx,Nx] = 1
		b[Nx] = 0
		for i=2:Nx-1
			A[i,i-1] = d
			A[i,i+0] =-1-2*d
			A[i,i+1] = d
			b[i] =-u[i,ii]
		end
		ulin = A\b
		u[1,ii+1] = b[1]
		u[Nx,ii+1] = b[Nx]
		u[2:Nx-1,ii+1] = [ulin[i] for i=2:Nx-1]
	end
	return u,y
end

# ╔═╡ 9c410110-644c-11eb-3bd9-93935242ac98
"""
Computes the Crank Nicolson method for the Parabolic Partial Differential Equation
This notebook computes the Parabolic Partial Differential Equation
		CrankNicolson(Δt,Δy,ν)
If the arguments are missing, it will use the default values.

		CrankNicolson(Δt=0.002,Δy=0.001,ν=0.000217)
Furkan Oz,
foz@okstate.edu,

"""
function CrankNicolson(Δt=0.002,Δy=0.001,ν=0.000217)
	#ν=0.000217 # viscosity [m²/s]
	h = 0.04 # Distance between 2 plate [m]
	T = 1.25 # Total time [s]
	#Δt = 0.002 # Time increment
	Nt = T/Δt+1 # Number of time step
	Nt = round(Int32,Nt)

	#Δy = 0.001 # Spacing between node points
	y = 0:Δy:h # y-coordinates
	Nx = h/Δy+1 # Number of Node points
	Nx = round(Int32,Nx)

	U₀ = 1 # Plate speed
	d = ν*Δt/(Δy)^2

	#Initializing
	A = zeros(Nx,Nx)
	b = zeros(Nx)
	u = zeros(Nx,Nt)

	# Boundary Conditions
	A[1,1] = 1
	b[1] = U₀
	A[Nx,Nx] = 1
	b[Nx] = 0

	# Initial Conditions
	u[:,1] = [0 for i=1:Nx]
	u[1,1] = U₀

	for ii=3:Nt-1
		# Boundary Conditions
		A[1,1] = 1
		b[1] = U₀
		A[Nx,Nx] = 1
		b[Nx] = 0
		for i=2:Nx-1
			A[i,i-1] =-0.5*d
			A[i,i+0] = 1+d
			A[i,i+1] =-0.5*d
			b[i] = 0.5*d*u[i+1,ii]+(1-d)*u[i,ii]+0.5*d*u[i-1,ii]
		end
		ulin = A\b
		u[1,ii+1] = b[1]
		u[Nx,ii+1] = b[Nx]
		u[2:Nx-1,ii+1] = [ulin[i] for i=2:Nx-1]
	end
	return u,y
end

# ╔═╡ 8939c23e-642a-11eb-2de6-ade5ae2f4b4d
function Analytical(t,Δy=0.001,ν=0.000217)
	#ν=0.000217 # viscosity [m²/s]
	h = 0.04 # Distance between 2 plate [m]
	y = 0:Δy:h
	Nx = h/Δy+1 # Number of Node points
	Nx = round(Int32,Nx)
	U₀ = 1 # Plate speed
	η = [y[i]/(2*sqrt(ν*t)) for i=1:Nx]
	η₁= h/(2*sqrt(ν*t))
	u = [U₀*(erfc(η[i])-erfc(2*η₁-η[i])+erfc(2*η₁+η[i])-erfc(4*η₁-η[i])+erfc(4*η₁+η[i])-erfc(8*η₁-η[i])+erfc(8*η₁+η[i])-erfc(16*η₁-η[i])) for i=1:Nx]
	return u,y
end

# ╔═╡ 70d8c3ee-641f-11eb-1d57-a943a361c397
begin
	uFTCS,yFTCS = FTCS()
	plot([uFTCS[:,1],uFTCS[:,90],uFTCS[:,180],uFTCS[:,270],uFTCS[:,360],uFTCS[:,450],uFTCS[:,540]],yFTCS,
			title = "FTCS - \\Deltat = 0.002 - \\Deltay = 0.001",
			label = ["t=0s" "t=0.18s" "t=0.36s" "t=0.54s" "t=0.72s" "t=0.90s" "t=1.08s"],
			legend = :topright,
			ylims = (0,0.04),
			xlabel = "U",
			ylabel = "y",
			linewidth = 2,
			linecolor = :black,
			markershape = :auto,
			markercolor = :red,
			)
end

# ╔═╡ 95697270-6427-11eb-02be-bf3b75fbbd33
begin
	u2,y2 = FTCS(0.005,0.001)

	plot([u2[:,1],u2[:,50],u2[:,100],u2[:,150],u2[:,200]],y2,
			title = "FTCS - \\Deltat = 0.005 - \\Deltay = 0.001",
			label = ["t=0s" "t=0.25s" "t=0.50s" "t=0.75s" "t=1.00s"],
			legend = :topright,
			ylims = (0,0.04),
			xlabel = "U",
			ylabel = "y",
			linewidth = 2,
			linecolor = :black,
			markershape = :auto,
			markercolor = :red,
			)
end

# ╔═╡ b1e5b270-643f-11eb-1a37-e3227c598177
begin
	ua1,ya1 = Analytical(0.18,0.001)
	uDF,yDF = DuFortFrankel()
	uLaasonen,yLaasonen = Laasonen()
	uCrankNicolson,yCrankNicolson = CrankNicolson()
	plot([ua1-uFTCS[:,90],ua1-uDF[:,90],ua1-uLaasonen[:,90],ua1-uCrankNicolson[:,90]],[ya1],
				title = "Error - t = 0.18 - \\Deltay = 0.001",
				label = ["FTCS" "D/F" "Laasonen" "CrankNicolson"],
				legend = :topright,
				xlims = (-0.005,0.01),
				ylims = (0,0.04),
				xlabel = "U",
				ylabel = "y",
				linewidth = 2,
				linecolor = :black,
				markershape = :auto,
				markercolor = :red,
				)
end

# ╔═╡ 8c11e680-6440-11eb-076a-0b1e169ef37a
begin
	ua2,ya2 = Analytical(1.08,0.001)
	plot([ua2-uFTCS[:,540],ua2-uDF[:,540],ua2-uLaasonen[:,540],ua2-uCrankNicolson[:,540]],[ya2],
				title = "Error - t = 1.08 -\\Deltay = 0.001",
				label = ["FTCS" "D/F" "Laasonen" "CrankNicolson"],
				legend = :topright,
				xlims = (-0.0005,0.0015),
				ylims = (0,0.04),
				xlabel = "U",
				ylabel = "y",
				linewidth = 2,
				linecolor = :black,
				markershape = :auto,
				markercolor = :red,
				)
end

# ╔═╡ 33a79720-644e-11eb-0eca-e794b9b87ccf
begin
	ua3,ya3 = Analytical(1.00,0.001)
	uLaasonen005,yLaasonen005 = Laasonen(0.005)
	uLaasonen01,yLaasonen01 = Laasonen(0.01)
	uLaasonen1,yLaasonen1 = Laasonen(0.1)
	uLaasonen2,yLaasonen2 = Laasonen(0.2)
	plot([ua3-uLaasonen005[:,200],ua3-uLaasonen01[:,100],ua3-uLaasonen1[:,10],ua3-uLaasonen2[:,5]],[ya2],
					title = "Error - Laasonen - t = 1.00 -\\Deltay = 0.001",
					label = ["\\Deltat=0.005" "\\Deltat=0.01" "\\Deltat=0.1" "\\Deltat=0.2"],
					legend = :topright,
					xlims = (0,0.2),
					ylims = (0,0.04),
					xlabel = "U",
					ylabel = "y",
					linewidth = 2,
					linecolor = :black,
					markershape = :auto,
					markercolor = :red,
					)
end

# ╔═╡ Cell order:
# ╟─0b868b30-6420-11eb-18f9-1592f7542fe6
# ╟─4c4fff40-64c8-11eb-2c45-57f4793ee997
# ╟─56d1f810-64c8-11eb-3e0d-81c0bbafdc62
# ╟─29683fb0-6419-11eb-05d0-2d7392bc8afa
# ╟─36be1c60-6442-11eb-204a-b37677c1acfd
# ╟─899d6a70-6448-11eb-2870-692c6e93c421
# ╟─9c410110-644c-11eb-3bd9-93935242ac98
# ╟─8939c23e-642a-11eb-2de6-ade5ae2f4b4d
# ╟─70d8c3ee-641f-11eb-1d57-a943a361c397
# ╟─95697270-6427-11eb-02be-bf3b75fbbd33
# ╟─b1e5b270-643f-11eb-1a37-e3227c598177
# ╟─8c11e680-6440-11eb-076a-0b1e169ef37a
# ╟─33a79720-644e-11eb-0eca-e794b9b87ccf


#=
Root finding and ODE solver.
See Numerical Recipes 16.1 - 16.2
=#

function fdjac{T<:AbstractFloat}(f::Function, x::Array{T, 1}; epsilon=1e-6)
    #=
    Forward difference Jacobian
    =#
    n = length(x)
    y = f(x)
    jacobian = Array{Float64}(n, n)
    for i = 1:n
        temp = x[i]
        h = abs(x[i]) * epsilon
        if h == 0.0
            h = epsilon
        end # if
        x[i] = temp + h
        h = x[i] - temp
        ystep = f(x)
        x[i] = temp
        dy = (ystep - y) ./ h
        jacobian[1:end, i] = dy
    end # for
    return jacobian
end # fdjac

function newton(f, x0; epsilon=1e-2, max_count=1000)
    #=
    Multidimensional Newton-Raphson root finding.
    TODO: figure out scaled stopping critereon
    =#
    x = x0
    count = 1
    @info("newton: Starting Newton method with x0 = ", x0)
    damping = 0.001
    while true
        @info("newton: count = ", count)
        if count > max_count
            @warn("newton: Number of iterations has exceeded ", max_count)
            break
        end # if
        # approximate Jacobian
        J = fdjac(f, x)
        # invert for dx
        invJ = inv(J)
        dx = -invJ * f(x)
        frac = dx ./ x
        @info("newton: dx / x = ", frac)
        # adaptive damping
        if all(abs(frac) .< 0.1)
            damping = 1
        elseif any(abs(frac) .> 1)
            # restrict to 1% change
            damping = 0.01 / maximum(abs(frac))
        else
            # dampen to move no more than 10% change, and no more than 2
            damping = minimum([0.1 / maximum(abs(frac)), 2 * damping])
        end
        @info("newton: damping = ", damping)
        close_enough = all(abs(frac) .< epsilon)
        x = x + dx .* damping
        @info("newton: x = ", x)
        if close_enough
            @info("newton: Zero-point found at x = ", x)
            return x
        end # if
        count += 1
    end # while
end # newton

function rkstep{T<:AbstractFloat}(f::Function,
                                  y::Array{T, 1},
                                  x::T,
                                  h::T)
    k1 = f(x, y)
    k2 = f(x + h / 2, y + h / 2 .* k1)
    k3 = f(x + h / 2, y + h / 2 .* k2)
    k4 = f(x + h, y + h .* k3)
    dy = h / 6 * (k1 + 2 * k2 + 2 * k3 + k4)
    return y + dy
end # rkstep

function rk{T<:AbstractFloat}(f::Function,
                              y0::Array{T, 1},
                              xrange::Array{T, 1})
    #=
    Classic fourth-order Runge-Kutta method
    f(x, y) = dy / dx
    Integrate from x0 to xf, y0 = y(x0)
    =#
    # @debug("rk: starting Runge-Kutta, y0 = ", y0, ", h = ", h)
    x0 = xrange[1]
    xf = xrange[end]
    n = length(xrange)
    m = length(y0)
    x = Array{Float64}(n)
    y = Array{Float64}(n, m)
    x[1] = x0
    y[1, 1:end] = y0
    for (i, xstep) in enumerate(xrange[2:end])
        j = i + 1
        h = xstep - x[j - 1]
        yy = rkstep(f, y[j - 1, 1:end], x[j - 1], h)
        x[j] = xstep
        y[j, 1:end] = yy
    end # while
    return x, y
end # rk

# TODO: finish adaptive step size
function rkck{T<:AbstractFloat}(f::Array{Function, 1},
                                x0::AbstractFloat,
                                y0::Array{T, 1},
                                h0::AbstractFloat,
                                xf::AbstractFloat;
                                epsilon=1e-3)
    #=
    Cash-Karp embedded Runge-Kutta method with adaptive step size.
    f(x, y) = dy / dx
    Integrate from x0 to xf, y0 = y(x0)
    =#
    # Butcher tableau for Cash-Karp
    a = [0, 1 / 5, 3 / 10, 3 / 5, 1, 7 / 8]
    b = zeros(6, 5)
    b[2, 1] = 1/5
    b[3, 1:2] = [3 / 40, 9 / 40]
    b[4, 1:3] = [3 / 10, -9 / 10, 6 / 5]
    b[5, 1:4] = [-11 / 54, 5 / 2, -70 / 27, 35 / 27]
    b[6, :] = [1631 / 55296, 175 / 512, 575 / 13824, 44275 / 110592, 253 / 4096]
    c = [37 / 378, 0, 250 / 621, 125 / 594, 0, 512 / 1771]
    c_embedded = [2825 / 27648, 0, 18575 / 48384, 13525 / 55296, 277 / 14336, 1 / 4]

    # number of equations
    m = length(y0)

    x_list = []
    y_list = []
    h_list = []
    push!(x_list, x0)
    push!(y_list, y0)
    push!(h_list, h0)
    while x_list[end] < xf
        xx = x_list[end]
        yy = y_list[end]
        hh = h_list[end]
        k = zeros(6, m) # todo, speed up by putting column loop inside
        k[1, :] = map(g -> hh * g(xx, yy), f)
        for i = 2:6
            for j = 1:m
                k[i, j] = hh * f[j](xx + a[i] * hh, yy + dot(vec(b[i, 1:i - 1]), k[1:i - 1, j]))
            end # for
        end # for
        delta = abs(map(i -> dot(c - c_embedded, k[:, i]), 1:m))
        delta_target = epsilon * (abs(yy) + abs(vec(k[1, :])))
        if any(delta .> delta_target)
            # reject step, try a smaller step size
            h_list[end] = 0.9 * hh * minimum(delta_target ./ delta) ^ 0.25
            continue
        else
            # accept step
            push!(h_list, 0.9 * hh * minimum(delta_target ./ delta) ^ 0.2)
        end # if
        # println("accepted")
        x_next = xx + hh
        y_next = map(i -> yy[i] + dot(c, k[:, i]), 1:m)
        push!(x_list, x_next)
        push!(y_list, y_next)
    end # while
    x = Array{Float64}(x_list)
    h = Array{Float64}(h_list)
    y = Array{Float64}(length(y_list), length(y_list[1]))
    for i in 1:length(y_list)
        y[i, 1:end] = y_list[i]
    end #for
    return x, y, h
end # rkck


## This file implements Example H071 using the MathProgBase Interface.
## This is the example code from the MathProgBase test file from https://github.com/JuliaOpt/MathProgBase.jl/blob/master/test/nlp.jl , slightly modified to remove the tests.
## This interface is solver independent, but will use Ipopt as the solver here.

using MathProgBase
import MathProgBase.MathProgSolverInterface   ##In Julia, importing means that the prefix 'MathProgSolverInterface.' must be included before calling any functions within this module
using Ipopt

tic()  #Start the clock

# Here the type represents the complete instance, but it
# could also store instance data.
# The abstract type AbstractNLPEvaluator is used by solvers for accessing the objective function f and constraints g.
# Solvers may query the value, gradients, Hessian-vector products, and the Hessian of the Lagrangian.
type HS071 <: MathProgSolverInterface.AbstractNLPEvaluator
end

# hs071
# min x1 * x4 * (x1 + x2 + x3) + x3
# st  x1 * x2 * x3 * x4 >= 25
#     x1^2 + x2^2 + x3^2 + x4^2 = 40
#     1 <= x1, x2, x3, x4 <= 5
# Start at (1,5,5,1)
# End at (1.000..., 4.743..., 3.821..., 1.379...)

## Initialize must be called before any other methods. The vector requested_features lists features requested by the solver.
function MathProgSolverInterface.initialize(d::HS071, requested_features::Vector{Symbol})
    for feat in requested_features
        if !(feat in [:Grad, :Jac, :Hess])
            error("Unsupported feature $feat")
        end
    end
end

## Returns which features are available for the model HS071
MathProgSolverInterface.features_available(d::HS071) = [:Grad, :Jac, :Hess]

################################
# Define the Objective Function
################################
MathProgSolverInterface.eval_f(d::HS071, x) = x[1] * x[4] * (x[1] + x[2] + x[3]) + x[3]

###################
# Constraint set
##################
function MathProgSolverInterface.eval_g(d::HS071, g, x)
    g[1] = x[1]   * x[2]   * x[3]   * x[4]
    g[2] = x[1]^2 + x[2]^2 + x[3]^2 + x[4]^2
end

#####################################
# Gradient of the Objective Function
#####################################
function MathProgSolverInterface.eval_grad_f(d::HS071, grad_f, x)
    grad_f[1] = x[1] * x[4] + x[4] * (x[1] + x[2] + x[3])
    grad_f[2] = x[1] * x[4]
    grad_f[3] = x[1] * x[4] + 1
    grad_f[4] = x[1] * (x[1] + x[2] + x[3])
end

###################################################################################################################################
##These functions collect the sparsity structures of the Jacobian matrix and the Hessian of the Lagrangian (row first then columns)
MathProgSolverInterface.jac_structure(d::HS071) = [1,1,1,1,2,2,2,2],[1,2,3,4,1,2,3,4]
# lower triangle only
MathProgSolverInterface.hesslag_structure(d::HS071) = [1,2,2,3,3,3,4,4,4,4],[1,1,2,1,2,3,1,2,3,4]

#########################################################################################################################
##OR you could define the values of the non-zero elements. The function below does this for the Jacobian matrix and saves
## them to the matrix J. There is no need to create the matrix J.
####################################################################
function MathProgSolverInterface.eval_jac_g(d::HS071, J, x)
    # Constraint (row) 1
    J[1] = x[2]*x[3]*x[4]  # 1,1
    J[2] = x[1]*x[3]*x[4]  # 1,2
    J[3] = x[1]*x[2]*x[4]  # 1,3
    J[4] = x[1]*x[2]*x[3]  # 1,4
    # Constraint (row) 2
    J[5] = 2*x[1]  # 2,1
    J[6] = 2*x[2]  # 2,2
    J[7] = 2*x[3]  # 2,3
    J[8] = 2*x[4]  # 2,4
end

####################################################################################################
# Similarly for the Hessian -  σ = weight on objective function, μ = multipliers on the constraints
####################################################################################################
function MathProgSolverInterface.eval_hesslag(d::HS071, H, x, σ, μ)
    # Again, only lower left triangle (since it is symmetric)
    # Objective
    H[1] = σ * (2*x[4])               # 1,1
    H[2] = σ * (  x[4])               # 2,1
    H[3] = 0                          # 2,2
    H[4] = σ * (  x[4])               # 3,1
    H[5] = 0                          # 3,2
    H[6] = 0                          # 3,3
    H[7] = σ* (2*x[1] + x[2] + x[3])  # 4,1
    H[8] = σ * (  x[1])               # 4,2
    H[9] = σ * (  x[1])               # 4,3
    H[10] = 0                         # 4,4

    # First constraint
    H[2] += μ[1] * (x[3] * x[4])  # 2,1
    H[4] += μ[1] * (x[2] * x[4])  # 3,1
    H[5] += μ[1] * (x[1] * x[4])  # 3,2
    H[7] += μ[1] * (x[2] * x[3])  # 4,1
    H[8] += μ[1] * (x[1] * x[3])  # 4,2
    H[9] += μ[1] * (x[1] * x[2])  # 4,3

    # Second constraint
    H[1]  += μ[2] * 2  # 1,1
    H[3]  += μ[2] * 2  # 2,2
    H[6]  += μ[2] * 2  # 3,3
    H[10] += μ[2] * 2  # 4,4

end

# Now put this all together to define and solve the model.
# First, define a model object and the solver to use
    m = MathProgSolverInterface.model(MathProgBase.defaultNLPsolver )
# The vector of lower bounds on the variables
    l = [1,1,1,1]
# The vector of upper bounds on the variables
    u = [5,5,5,5]
# Vector of lower bounds on the constraints
    lb = [25, 40]
# Vector of constraint upper bounds
    ub = [Inf, 40]
# Load the nonlinear programming problem into the model - syntax for loadnonlinearproblem!() is
# loadnonlinearproblem!(m::AbstractMathProgModel, numVar, numConstr, l, u, lb, ub, sense, d::AbstractNLPEvaluator)
    MathProgSolverInterface.loadnonlinearproblem!(m, 4, 2, l, u, lb, ub, :Min, HS071())
## Set the initial values for the optimisation
    MathProgSolverInterface.setwarmstart!(m,[1,5,5,1])

## Solve the optimisation problem
    MathProgSolverInterface.optimize!(m)
## Let's see the termination status after solving
    stat = MathProgSolverInterface.status(m)

# Now let's see the solution.
    x = MathProgSolverInterface.getsolution(m)

toc()

println(x)


using DataStructures,Distributions,StatsBase,Random,Plots, LaTeXStrings; pyplot()

function simMM1Wait(lambda,mu,T)
    tNextArr = rand(Exponential(1/(lambda)))
    tNextDep = Inf
    t = tNextArr

    waitingRoom = Queue{Float64}()
    serverBusy = false
    waitTimes = Array{Float64,1}()

    while t<T
        if t == tNextArr
            if !serverBusy
                tNextDep = t + rand(Exponential(1/mu))
                serverBusy = true
                push!(waitTimes,0.0)
            else
                enqueue!(waitingRoom,t)
            end
            tNextArr = t + rand(Exponential(1/(lambda)))
        else
            if length(waitingRoom) == 0
               tNextDep = Inf
               serverBusy = false
            else
               tArr = dequeue!(waitingRoom)
               waitTime = t - tArr
               push!(waitTimes, waitTime)
               tNextDep = t + rand(Exponential(1/mu))
            end
        end
        t = min(tNextArr,tNextDep)
    end

    return waitTimes
end

Random.seed!(1)
lambda, mu = 0.8, 1.0
T = 10^3

data = simMM1Wait(lambda,mu,T)
empiricalCDF = ecdf(data)

F(x) = 1-(lambda/mu)*MathConstants.e^(-(mu-lambda)x)
xGrid = 0:0.1:20

plot(xGrid, F.(xGrid),
	c=:blue,label="Analytic CDF of waiting time")
plot!(xGrid, empiricalCDF(xGrid),
	c=:red,label="ECDF of waiting times",
	xlabel=L"x", ylabel=L"\Prob(W \leq x)", xlims=(0,20),ylims=(0,1),
	legend=:bottomright)


# VWN parameterization of the exchange correlation energy
function excVWN( n::Array{Float64,1} )
    # Constants
    X1 = 0.75*(3.0/(2.0*pi))^(2.0/3.0)
    A = 0.0310907
    x0 = -0.10498
    b = 3.72744
    c = 12.9352
    Q = sqrt(4*c-b*b)
    X0 = x0*x0 + b*x0 + c
    Npoints = size(n)[1]
    out = zeros(Float64,Npoints)
    for ip = 1:Npoints
        rs = (4.0*pi/3.0*n[ip])^(-1.0/3.0) # Added internal conversion to rs
        x = sqrt(rs)
        X = x*x + b*x + c
        out[ip] = -X1/rs + A*( log(x^2/X) + 2*b/Q*atan(Q/(2*x + b))
                  -(b*x0)/X0*( log((x-x0)*(x-x0)/X) + 2*(2*x0+b)/Q*atan(Q/(2*x + b)) ))
    end
    return out
end

# d/dn deriv of VWN parameterization of the exchange correlation energy
function excpVWN( n::Array{Float64,1} )
    # Constants
    X1 = 0.75*(3.0/(2.0*pi))^(2.0/3.0)
    A = 0.0310907
    x0 = -0.10498
    b = 3.72744
    c = 12.9352
    Q = sqrt(4.0*c - b*b)
    X0 = x0*x0 + b*x0 + c
    #
    Npoints = size(n)[1]
    out = zeros(Float64,Npoints)
    for ip = 1:Npoints
        rs = (4.0*pi/3.0*n[ip])^(-1.0/3.0) # Added internal conversion to rs
        x = sqrt(rs)
        X = x*x + b*x + c
        dx = 0.5/x # Chain rule needs dx/drho!
        out[ip] = dx*( 2.0*X1 / (rs*x)
                  + A*( 2.0/x - (2*x+b)/X - 4*b/(Q*Q + (2*x+b)*(2*x+b))
                  - (b*x0)/X0*( 2.0/(x-x0) - (2*x+b)/X - 4*(2*x0+b) /
                    (Q*Q + (2*x+b) * (2*x+b)) ) ))
        out[ip] = (-rs/(3*n[ip]))*out[ip] # Added d(rs)/dn from chain rule from rs to n conv
    end
    return out
end


#spherecomponent.jl
#Algorithm 2.3 of Random Eigenvalues by Alan Edelman

#Experiment:    Random components of a vector on a sphere
#Plot:          Histogram of a single component
#Theory:        Beta distribution

## Parameters
t = 100000      # trials
n = 12          # dimensions of sphere
dx = 0.1        # bin size

## Experiment
v = randn(n, t)
v = (v[1,:] ./ sqrt(sum(v.^2,1)))'
grid, count = hist(v, -1:dx:1)

## Theory
x = grid
y = (gamma(n-1)/(2^(n-2)*gamma((n-1)/2)^2))*(1-x.^2).^((n-3)/2)

## Plot
using Winston
p = FramedPlot()
h = Histogram(count/(t*dx), step(grid))
h.x0 = first(grid)
add(p, h)
add(p, Curve(x, y, "color", "blue", "linewidth", 2))
if isinteractive()
    Winston.display(p)
else
    file(p, "spherecomponent.png")
end


function fft(a)
    y1 = Any[]; y2 = Any[]
    n = length(a)
    if n ==1 return a end
    wn(n) = exp(-2*π*im/n)
    y_even = fft(a[1:2:end])
    y_odd = fft(a[2:2:end])
    w = 1
    for k in 1:Int(n/2)
        push!(y1, y_even[k] + w*y_odd[k])
        push!(y2, y_even[k] - w*y_odd[k])
        w = w*wn(n)
    end
    return vcat(y1,y2)
end


struct Grid{FT<:Real}
	xmin :: FT
	xmax :: FT
	nx   :: Int
	δx   :: FT
	xc   :: Array{FT}
	xf   :: Array{FT}
	ymin :: FT
	ymax :: FT
	ny   :: Int
	δy   :: FT
	yc   :: Array{FT}
	yf   :: Array{FT}
end

function GenerateGrid(
	FT = Float64;
	size, x, y,
)

	nx,ny = size
	xmin,xmax = x
	ymin,ymax = y

	xv = collect(range(xmin,xmax,length=nx+1)); δx = (xv[end] - xv[1]) / nx
	xf = xv[2:end]
	xc = (xv[1:(end-1)] .+ xv[2:end]) / 2
    yf = collect(range(ymin,ymax,length=ny+1)); δy = (yf[end] - yf[1]) / ny
	yc = (yf[1:(end-1)] .+ yf[2:end]) / 2

	return Grid{FT}(
		xmin,xmax,nx,δx,xc,xf,
		ymin,ymax,ny,δy,yc,yf
	)

end

function show(io::IO, g::Grid{FT}) where FT <: Real
    print(
		io,
		"The Grid{$FT} is defined as follows:\n",
		"                  domain : x ∈ [$(g.xmin), $(g.xmax)], y ∈ [$(g.ymin), $(g.ymin)]\n",
		"      resolution (nx,ny) : ", (g.nx,g.ny), '\n',
		"    grid spacing (δx,δy) : ", (g.δx,g.δy), '\n'
	)
end


#overlap funcs {{{
## for binary states {{{
export overlap, overlapm

function overlap{T<:Real}(x::AbstractVector{T},y::AbstractVector{T})
    return dot(x, y)/length(x)
end

function overlap{T<:Real}(x::AbstractVector{Complex{T}},y::AbstractVector{Complex{T}})
    rsum = dot(real(x), real(y))
    isum = dot(imag(x), imag(y))
    return (rsum+isum)/length(x)*0.5
end

function overlap{T<:Real}(x::AbstractVector{Quaternion{T}},y::AbstractVector{Quaternion{T}})
    rsum = dot(real(x), real(y))
    isum = dot(imagi(x), imagi(y))
    jsum = dot(imagj(x), imagj(y))
    ksum = dot(imagk(x), imagk(y))
    return (rsum+isum+jsum+ksum)/length(x)*0.25
end
## }}}

## for multi states {{{
function overlapm{T<:Real}(x::AbstractVector{T},y::AbstractVector{T})
    return sum(x .== y)/length(x)
end

function overlapm{T<:Real}(x::AbstractVector{Complex{T}},y::AbstractVector{Complex{T}})
    rsum = sum(real(x).== real(y))
    isum = sum(imag(x).== imag(y))
    return (rsum+isum)/length(x)*0.5
end

function overlapm{T<:Real}(x::AbstractVector{Quaternion{T}},y::AbstractVector{Quaternion{T}})
    rsum = sum(real(x).== real(y))
    isum = sum(imagi(x).== imagi(y))
    jsum = sum(imagj(x).== imagj(y))
    ksum = sum(imagk(x).== imagk(y))
    return (rsum+isum+jsum+ksum)/length(x)*0.25
end
## }}}


"""
    eliminate!(g::SimpleGraph{Int64}, v::Int64, labels)::Nothing
Connect neighbors of v (in graph g) together if they are not already connected, delete v
and update the labels array. Labels is pop and element v updated.

# Arguments
- `g::SimpleGraph{Int64}` the graph to consider.
- `v::Int64` the vertex to eliminate.
- `labels::Vector{Int64}` labels to keep track of nodes id in original graph before elimination.

# Return
- Nothing.
"""
function eliminate!(g::SimpleGraph{Int64}, v::Int64, labels::Vector{Int64})::Nothing
    ns = neighbors(g, v)
    len = length(ns)
    for i = 1:(len-1)
        for j = (i+1):len
            add_edge!(g, ns[i], ns[j])
        end
    end
    rem_vertex!(g, v)
    labels[v] = labels[end]
    pop!(labels)
    nothing
end

"""
    treewidth_by_elimination!(g::SimpleGraph{Int64}, order::Vector{Int64})
Computes the treewidth by eliminating in order vertices of the graph and keep track of the max degree.

# Arguments
- `g::SimpleGraph{Int64}` the graph to consider.
- `order::Vector{Int64}` the vertex elimination order to perform.

# Return
- `treewidth::Int64` the max degree of vertices during elimination.
"""
function treewidth_by_elimination!(g::SimpleGraph{Int64}, order::Vector{Int64})::Int64
    treewidth = 0
    n = nv(g)
    labels = collect(1:n)
    for rm_origin_id in order
        toremove = findfirst(id -> id == rm_origin_id, labels)
        treewidth = max(treewidth, degree(g, toremove))
        eliminate!(g, toremove, labels)
    end
    return treewidth
end

"""
    tree_order!(graph, nodes)
Computes the optimal elimination order for nodes in a tree. Graph is modified, all its edges are deleted.

# Arguments
- `graph::SimpleGraph{Int64}` the subtree to consider.
- `nodes::Vector{Int64}` indices of nodes in the root greaph.

# Return
- `order::Vector{Int64}` vertex elimination order (indices in the root graph) to get treewidth of 1 in the subtree.
"""
function tree_order!(graph::SimpleGraph{Int64}, nodes::Vector{Int64})::Vector{Int64}
    n = length(nodes)
    eliminated = 1
    order = zeros(Int64, n)
    leafs = filter(node -> length(neighbors(graph, node)) == 1, 1:n)

    # trick to get the last node
    lastnode = sum(nodes)
    while eliminated < n
        parents = unique(map(it -> neighbors(graph, it)[1], leafs))
        for rm in leafs
            order[eliminated] = nodes[rm]
            lastnode -= nodes[rm]
            if ne(graph) != 0
                rem_edge!(graph, rm, neighbors(graph, rm)[1])
            end
            eliminated += 1
        end
        # new leafs are parents with 1 neighbors
        leafs = filter(
            node -> length(neighbors(graph, node)) == 1,
            parents
        )
    end
    # fix last node have no neighbor because all edges are deleted
    if order[n] == 0
        order[n] = lastnode
    end
    return order
end

"""
    graph_from_gr(filename::String)

Read a graph from the provided gr file.
"""
function graph_from_gr(filename::String)::SimpleGraph{Int64}
    lines = readlines(filename)

    # Create a Graph with the correct number of vertices.
    num_vertices, num_edges = parse.(Int, split(lines[1], ' ')[3:end])
    G = SimpleGraph(num_vertices)

    # Add an edge to the graph for every other line in the file.
    for line in lines[2:end]
        src, dst = parse.(Int, split(line, ' '))
        add_edge!(G, src, dst)
    end

    G
end

"""
    square_lattice_graph(n::Int64)
Return a square lattice graph of dimension n. The dimension means
a big square made of n x n squares so (n+1)² nodes in total.

# Example

n = 2 give a graph like this:
    1 ---- 2 ---- 3
    |      |      |
    4 ---- 5 ---- 6
    |      |      |
    7 ---- 8 ---- 9

"""
function square_lattice_graph(n::Int64)::SimpleGraph{Int64}
    nrow = (n + 1)
    nvertices =  nrow * nrow
    g = SimpleGraph{Int64}(nvertices)
    for i in 1:nvertices
        if i % nrow != 0
            add_edge!(g, i, i + 1)
        end
        if trunc(i / nrow) < nrow - 1
            add_edge!(g, i, i + nrow)
        end
    end
    return g
end


# These are the functions that need to be imported so that new methods do not overwrite old ones
import Base.show
import Base.one
import Base.∘
import Base.inv
import Base.^
import Base.:(==)

"""
    Permutation(image::Vector{Int64})
    Permutation(xs...::Int64)

A function ℕ → ℕ which fixes all but finitely many points.

Construct a `Permutation` from a vector of integers as the image of the integers `1:length(image)`.

Alternatively, construct from an arbitrary list of integers.

# Examples
```julia-repl
julia> Permutation([1,5,3,2,4])
( 2 4 5 )

julia> Permutation(4,5,2,3,1)
( 1 5 2 3 4 )
```
"""
struct Permutation
    image::Vector{Int64}
    # An inner constructor checks that the permutation is valid
    function Permutation(image::Vector{Int64})
        m = maximum([image[image .!= 1:length(image)]..., 1])
        sort(image) == 1:length(image) && return new(image[1:m])
        error("not a valid permutation")
    end
end

Permutation(imagevals::Int64...) = Permutation([x for x ∈ imagevals])

"""
    maxarg(σ::Permutation)

Find the largest integer which the `Permutation` `σ` does not fix.

For the identity `Permutation`, return `1`.

# Examples
```julia-repl
julia> maxarg(Permutation(1,3,2,4))
3

julia> maxarg(ι)
1
```
"""
maxarg(σ::Permutation) = length(σ.image)

"""
    (σ::Permutation)(x::Int64)

Evaluate the action of the `Permutation` `σ` on `x`.

# Examples
julia> σ = Permutation(4,5,2,1,3)
( 1 4 )( 2 5 3 )

julia> σ(5)
3
"""
(σ::Permutation)(x::Int64) = x ∈ 1:maxarg(σ) ? σ.image[x] : x

"""
    orbit(σ::Permutation,x::Int64)

Compute the orbit of `x` under the repeated action of the `Permutation` `σ`.

# Examples
```julia-repl
julia> orbit(Permutation(2,3,4,5,1),3)
5-element Vector{Int64}:
 3
 4
 5
 1
 2
```
"""
function orbit(σ::Permutation, x::Int64)
    orb = [x]
    y = σ(x)
    while y != x
        push!(orb,y)
        y = σ(y)
    end
    return orb
end

"""
    dcd(σ::Permutation)

Compute the disjoint cycle decomposition of the `Permutation` `σ`.

If an integer is fixed by `σ`, then it is not included in the decomposition.

# Examples
```julia-repl
julia> dcd(Permutation(4,3,5,1,2))
2-element Vector{Vector{Int64}}:
 [1, 4]
 [2, 3, 5]

julia> dcd(Permutation(1,3,2))
1-element Vector{Vector{Int64}}:
 [2, 3]
```
"""
function dcd(σ::Permutation)
    decomp = Vector{Int64}[]
    unaccounted = trues(maxarg(σ))
    while any(unaccounted)
        x = findfirst(unaccounted)
        xorbit = orbit(σ,x)
        unaccounted[xorbit] .= false
        length(xorbit) > 1 && push!(decomp,xorbit)
    end
    return decomp
end

function show(io::IO,σ::Permutation)
    toprint = *([ *( "( ", ["$y " for y ∈ x]... , ")" ) for x ∈ dcd(σ)]..., "")
    toprint == "" && (toprint = "ι")
    print(io, toprint)
end

"""
    ι

The identity `Permutation`.

# Examples
```julia-repl
julia> ι.(1:10) == 1:10
true
```
"""
const ι = Permutation(1)
one(::Permutation) = ι
one(::Type{Permutation}) = ι

"""
    ∘(σ::Permutation)
    ∘(σ::Permutation, τ::Permutation)
    ∘(σs...::Permutation)

Compose `Permutation`s.

Compose a single `Permutation` and recieve it back.

# Examples
```julia-repl
julia> ∘(Permutation(1,2,4,3))
( 3 4 )

julia> Permutation(5,2,3,1,4) ∘ Permutation(2,1,4,5,3)
( 1 2 5 3 )
```
"""
∘(σ::Permutation) = σ
∘(σ::Permutation, τ::Permutation) = Permutation([σ(τ(x)) for x ∈ 1:max(maxarg(σ),maxarg(τ))])
∘(σ::Permutation, τ::Permutation, υ::Permutation...) = ∘(σ ∘ τ, υ...)

inv(σ::Permutation) = Permutation([findfirst(σ.image .== x) for x ∈ 1:maxarg(σ)])

"""
    ^(σ::Permutation,n::Int)

Exponentiate a `Permutation` `σ` by an integer `n`.

If `n` is negative, compute the inverse and exponentiate by `-n`.
```julia-repl
julia> Permutation(5,4,2,3,1)^5
( 1 5 )( 2 3 4 )

julia> Permutation(5,4,2,3,1)^-5
( 1 5 )( 2 4 3 )
```
"""
function ^(σ::Permutation,n::Int64)
    n == 0 && return one(Permutation)
    n < 0 && ((n,σ) = (-n,inv(σ)))
    return ∘(fill(σ,n)...)
end

"""
    transposition(m::Int64,n::Int64)

Create the `Permutation` which is a transposition of `m` and `n`.

If `m == n`, the `Permutation` created is the identity.

# Examples
```julia-repl
julia> transposition(4,5)
( 4 5 )

julia> transposition(6,6)
ι
```
"""
function transposition(m::Int64, n::Int64)
    image = collect(1:max(m,n))
    image[m], image[n] = n, m
    return Permutation(image)
end

"""
    symmetricgroup(n::Int64)

Compute the `Permutation`s which make up the symmetric group of order `factorial(n)`.

# Examples
```julia-repl
julia> symmetricgroup(5)
120-element Vector{Permutation}:
[...]
```
"""
function symmetricgroup(n::Int64)
    n ≥ 1 || error("symmetric group must have a positive parameter")
    n == 1 && return [ι]

    permutations = vcat(fill(symmetricgroup(n-1),n)...)
    transpositions = vcat([fill(transposition(i,n),factorial(n-1)) for i ∈ 1:n]...)
    return permutations .∘ transpositions
end

"""
    parity(σ::Permutation)

Compute the parity of the `Permutation` `σ`.

# Examples
```julia-repl
julia> parity(Permutation(1,2,3,5,4))
-1

julia> parity(ι)
1
```
"""
parity(σ::Permutation) = (-1)^sum([iseven(length(x)) for x ∈ dcd(σ)])

"""
    alternatinggroup(n::Int64)

Compute the `Permutation`s which make up the `n`th alternating group.

# Examples
```julia-repl
julia> alternatinggroup(5)
60-element Vector{Permutation}:
[...]
```
"""
function alternatinggroup(n::Int64)
    Sₙ = symmetricgroup(n)
    return Sₙ[parity.(Sₙ) .== 1]
end

# Defines equality of permutations, to be used to check ∉
==(σ::Permutation, τ::Permutation) = (σ.image == τ.image)

"""
    generate(σs::Permutation...)

Compute the elements of the group generated by the elements listed.

# Examples
```julia-repl
julia> generate(Permutation(2,3,4,1))
4-element Vector{Permutation}:
 ( 1 2 3 4 )
 ( 1 3 )( 2 4 )
 ( 1 4 3 2 )
 ι

 julia> generate(Permutation(2,3,4,5,1),Permutation(5,4,3,2,1))
 10-element Vector{Permutation}:
  ( 1 2 3 4 5 )
  ( 1 5 )( 2 4 )
  ( 1 3 5 2 4 )
  ( 2 5 )( 3 4 )
  ( 1 5 4 3 2 )
  ( 1 4 )( 2 3 )
  ι
  ( 1 4 2 5 3 )
  ( 1 2 )( 3 5 )
  ( 1 3 )( 4 5 )
```
"""
function generate(σs::Permutation...)
    G = [σs...]
    while true
        newelements = Permutation[]

        for σ ∈ G, τ ∈ G
            υ = σ ∘ τ
            υ ∉ G && υ ∉ newelements && push!(newelements, υ)
        end

        if isempty(newelements)
            return G
        else
            G = vcat(G,newelements)
        end
    end
end


using DiffEqBayes, OrdinaryDiffEq, RecursiveArrayTools, Distributions, Plots, StatsPlots, BenchmarkTools, TransformVariables, CmdStan, DynamicHMC

function pendulum(du,u,p,t)
    ω,L = p
    x,y = u
    du[1] = y
    du[2] = - ω*y -(9.8/L)*sin(x)
end

u0 = [1.0,0.1]
tspan = (0.0,10.0)
prob1 = ODEProblem(pendulum,u0,tspan,[1.0,2.5])

sol = solve(prob1,Tsit5())
plot(sol)

t = collect(range(1,stop=10,length=10))
randomized = VectorOfArray([(sol(t[i]) + .01randn(2)) for i in 1:length(t)])
data = convert(Array,randomized)

scatter!(data')

priors = [Uniform(0.1,3.0), Normal(3.0,1.0)]

bayesian_result = turing_inference(prob1,Tsit5(),t,data,priors;num_samples=10_000,
                                   syms = [:omega,:L])

plot(bayesian_result)

plot(bayesian_result, colordim = :parameter)

@btime bayesian_result = turing_inference(prob1,Tsit5(),t,data,priors;syms = [:omega,:L],num_samples=10_000)

@btime bayesian_result = stan_inference(prob1,t,data,priors;num_samples=10_000,printsummary=false)

@btime bayesian_result = dynamichmc_inference(prob1,Tsit5(),t,data,priors;num_samples = 10_000)