module LogisticRegression2
using Revise
using Flux
using Flux: binarycrossentropy, throttle, @epochs
using Flux: Descent, TrackedArray, param, data
using Distributions
using LinearAlgebra
using Random
using Plots
using Base.Iterators: repeated

theme(:juno)

# Generate data
D = 2;   # Data dimension
n = 100; # Number of observations
x1 = rand(MvNormal([1., -5.], I), n);
x2 = rand(MvNormal([-2., -1.5], I), n);
y1 = repeat([1], n);
y2 = repeat([0], n);

classOneColor = :orange
classTwoColor = RGBA(0,.2,.7,1);
p = scatter(x1[1, :], x1[2, :], label="Class 1", color=classOneColor)
scatter!(p, x2[1, :], x2[2, :], label="Class 2", color=classTwoColor)
X = transpose([x1 x2]);
y = [y1; y2];

# Create model
struct LogisticRegression
    α :: TrackedArray # bias
    β :: TrackedArray # weights
    LogisticRegression(α :: Array, β :: Array) = new(param(α), param(β))
end

function (m :: LogisticRegression)(X)
    σ.(m.α .+ X*m.β)
end

function plot_decision_surface(m, xlim :: Tuple{Int, Int}, ylim :: Tuple{Int, Int})
    x = xlim[1]:0.5:xlim[2]
    y = ylim[1]:0.5:ylim[2]
    f = (x, y) -> (first ∘ data ∘ m)([x y])
    contour(x, y, f, fill = true, color = :pu_or)
end

function plot_fit(m)
    xlim = ylim = (-8, 8)
    p = plot_decision_surface(m, xlim, ylim)
    scatter!(p, x1[1, :], x1[2, :], label="Class 1", color=classOneColor)
    scatter!(p, x2[1, :], x2[2, :], label="Class 2", color=classTwoColor)
    p
end

function plot_loss(losses)
    plot(1:length(losses), losses)
end

function create_animation(αs :: Array, βs :: Array)
    anim = @animate for (α, β) in zip(αs, βs)
        m = LogisticRegression(α, β)
        display(plot_fit(m))
    end

    gif(anim, "plots/transition.gif", fps = 60)
end

m = LogisticRegression(rand(1), rand(D, 1)); display(plot_fit(m));
θ = Flux.params(m.α, m.β);
numEpochs = 100;
loss = (x, y) -> mean(binarycrossentropy.(m(x), y));

losses, αs, βs = [], [], []
cb = () -> begin
    l = Flux.data(loss(X, y));
    push!(losses, l);
    push!(αs, Flux.data(m.α));
    push!(βs, Flux.data(m.β));
end

dataset = repeated((X, y), numEpochs)
Flux.train!(loss, θ, dataset, Descent(0.1), cb = cb)

plot_loss(losses)
plot_fit(m)

create_animation(αs, βs)

end


### A Pluto.jl notebook ###
# v0.12.21

using Markdown
using InteractiveUtils

# ╔═╡ 17deace8-9071-11eb-3ff4-e70cf05a1bcb
begin
using Images, TestImages
using ImageQualityIndexes

img = testimage("cameraman") .|> float64
noisy_img = img .+ 0.1 .* randn(size(img))
assess_ssim(noisy_img, img) # 0.24112
assess_psnr(noisy_img, img) # 19.9697

kernel = ones(3, 3)./9 # mean filter
denoised_img = imfilter(noisy_img, kernel)
assess_psnr(denoised_img, img) # 28.4249
assess_ssim(denoised_img, img) # 0.6390
assess_msssim(denoised_img, img) # 0.8460

img = testimage("fabio");
colorfulness(img) # 68.5530
end

# ╔═╡ Cell order:
# ╠═17deace8-9071-11eb-3ff4-e70cf05a1bcb


@testset "Test equations for Rayleigh resolution" begin
    f = 100e-3
    D = 50e-3

    @test rayleigh_criterion(calc_NA(f, D)) ≈ rayleigh_criterion(f, D)
    @test rayleigh_criterion(calc_NA(f, D), λ=500e-9) ≈ rayleigh_criterion(f, D)
    @test ~(rayleigh_criterion(calc_NA(f, D)) ≈ rayleigh_criterion(f, D, λ=200e-9))

end

@testset "Calc NA" begin
    @test calc_NA(100, 100) ≈ 0.5
    @test calc_NA(10e-9, 40e-9, 2) ≈ 4
end


using ThreeBodyProblem
using LinearAlgebra
using Test

μ₁ = 398600     # Earth gravitational parameter
μ₂ = 4902       # Moon gravitational paramter
d = 384400      # Avg distance between Earth and Moon
p = [μ₁; μ₂; d]
μ = μ₂/(μ₁ + μ₂)
@test computeR1R2(μ) == (-μ, 1-μ)
@test computeL1(μ) == [0.8369247061700067;0;0]
@test computeL1(p) == [321713.8570517506;0;0]
@test computeL2(μ) == [1.1556746768583537;0;0]
@test computeL2(p) == [444241.3457843512;0;0]
@test computeL3(μ) == [-1.005061834632099;0;0]
@test computeL3(p) == [-386345.76923257887;0;0]
@test computeL4(μ) == [0.48785136133154233;0.8660254037844386;0]
@test computeL4(p) == [187530.06329584488;332900.16521473817;0]
@test computeL5(μ) == [0.48785136133154233;-0.8660254037844386;0]
@test computeL5(p) == [187530.06329584488;-332900.16521473817;0]

@test computeLpts(μ,tol=1e-3) != computeLpts(μ)
@test computeLpts(p,tol=1e-3) != computeLpts(p)

rv = [1,0,0,0,1,0]
@test computeUeff(rv,p) < 0
@test computeUeff([-μ;0;0],μ) == -Inf

@test_broken computeC(rv,p) > 0
@test_broken computeC([-μ;0;0],μ) == Inf

Φ = ones(6,6)
@test_broken stability_index(ϕ) > 1

sys = earth_moon()
Lpts = computeLpts(sys)
rv = vcat(Lpts[1],zeros(3))
C₁ = computeC(rv, sys)

A = [-1. 0.  0.;
      0. 1.  0.;
      0. 0. -1.]
@test det(roty(π) - A) == 0
@test det(rotyd(180) - A) == 0

mytime = [2000, 11, 17.625065277778]
mytime2 = [2000,11,17,15,0,5.63999999]
mytime3 = [1000, 1, 1]
mytime4 = [1582, 11, 1]
mytime5 = [1582, 10, 1]
mytime6 = [1582, 10, 11]
mytime7 = [1582, 10, 5]
mytime8 = [1993, 4, 19]
@test date2mjd(mytime2) == date2mjd(mytime)
@test date2mjd(mytime3) == -314577.0
@test date2mjd(mytime4) == -100823.0
@test date2mjd(mytime5) == -101728.0
@test date2mjd(mytime6) == -100844.0
@test isnan(date2mjd(mytime7))
@test date2mjd(mytime8) == 49096.0

@test_throws ErrorException("ut1_date should be [Y,M,D] or [Y,M,D,h,m,s]") date2mjd(1)

@test wrapto360(314) == 314
@test wrapto360(372) == 12
@test wrapto360(721) == 1
@test wrapto360(-1) == 359
@test wrapto360(-361) == 359

@test wrapto180(181) == -179
@test wrapto180(179) == 179
@test wrapto180(-181) == 179

@test wrapto2pi(2π) == 2π
@test wrapto2pi(3π) - π == 0.
@test wrapto2pi(-π) - π == 0.

@test wraptopi(π) == π
@test wraptopi(2π) == 0.
@test wraptopi(-2π) == 0.

@test rotlatlon(π/2,-π/2,ang_unit=:rad) == I
@test_throws ErrorException("ang_unit should be :deg or :rad") rotlatlon(0,0,ang_unit=:foo)

@test mjd2gmst(51544.5,ang_unit=:deg) == 280.4606
@test_throws ErrorException("ang_unit should be :deg or :rad") mjd2gmst(0,ang_unit=:foo)

xyz_sphere,N = deserno_hemisphere(100,[1,0,0])
@test abs(minimum([norm(xyz_sphere[:,i]) for i = 1:N]) - 1) < 1e-10
@test abs(maximum([norm(xyz_sphere[:,i]) for i = 1:N]) - 1) < 1e-10


# Polyharmonic functions used below.
function polyharmonic_phi1(r) return r; end
function polyharmonic_phi2(r) return r>1e-16 ? r^2 * log(r) : r; end
function polyharmonic_phi3(r) return r^3; end
function polyharmonic_phi4(r) return r>1e-16 ? r^4 * log(r) : r^3; end

# Uses polyharmonic interpolation to interpolate at x using values u at centers c.
# sizes are:
# x is d x nx
# c is d x nc
# u is nc
# (optional) ord determines the type of basis function. default is 3. 1,2,3,4 are available
function polyharmonic_interp(x, c, u, ord=3)
    dim = size(x,1);
    nx = size(x,2); # number of places to evaluate

    # If no centers are provided, this is meaningless, but return zero.
    if length(c)==0
        return zeros(nx);
    end

    nc = size(c,2); # number of centers

    # Constant if only one center is given
    if nc == 1
        return u[1] .* ones(nx);
    end

    if ord == 1
        phi = polyharmonic_phi1;
    elseif ord == 2
        phi = polyharmonic_phi2;
    elseif ord == 3
        phi = polyharmonic_phi3;
    else
        phi = polyharmonic_phi4;
    end

    # build A matrix
    A = zeros(nc + dim+1, nc + dim+1);
    for i=1:nc
        for j=1:nc
            A[i,j] = phi(norm(c[:,j] - c[:,i]));
        end
    end
    # build B matrix (within A matrix)
    A[1:nc, nc+1] .= 1;
    A[nc+1, 1:nc] .= 1;
    A[1:nc, (nc+2):(nc+dim+1)] = c';
    A[(nc+2):(nc+dim+1), 1:nc] = c;

    # build RHS vector
    b = vcat(u, zeros(dim+1));

    wv = A\b; # coefficients for the polyharmonic

    sol = zeros(nx);
    for i=1:nx
        # evaluate at x[i]
        sol[i] = wv[nc+1];
        for k=1:dim
            sol[i] += wv[nc+k+1] * x[k,i];
        end
        for j=1:nc
            sol[i] += wv[j]*phi(norm(c[:,j] - x[:,i]));
        end
    end

    return sol;
end


#
# Cubic fit
#

struct Cubic
  a :: Float64
  b :: Float64
  c :: Float64
  d :: Float64
  R :: Float64
  x :: Vector{Float64}
  y :: Vector{Float64}
  ypred :: Vector{Float64}
  residues :: Vector{Float64}
end

function fitcubic(X :: AbstractVector, Y :: AbstractVector, options :: Options)
  X, Y = checkdata(X,Y)
  @. model(x,p) = p[1]*x^3 + p[2]*x^2 + p[3]*x + p[4]
  fit = find_best_fit(model, X, Y, 4, options)
  R = pearson(X,Y,model,fit)
  x, y, ypred = finexy(X,options.fine,model,fit)
  return Cubic(fit.param...,R,x,y,ypred,fit.resid)
end
fitcubic(X :: AbstractVector, Y :: AbstractVector) = fitcubic(X,Y,Options())

function Base.show( io :: IO, fit :: Cubic )
  println("")
  println(" ------------------- Cubic Fit ----------------- ")
  println("")
  println(" Equation: y = ax^3 + bx^2 + cx + d ")
  println("")
  println(" With: a = ", fit.a)
  println("       b = ", fit.b)
  println("       c = ", fit.c)
  println("       d = ", fit.d)
  println("")
  println(" Pearson correlation coefficient, R = ", fit.R)
  println("")
  println(" Predicted Y: ypred = [",fit.ypred[1],", ",fit.ypred[2],"...")
  println(" residues = [", fit.residues[1],", ",fit.residues[2],"...")
  println("")
  println(" ----------------------------------------------- ")
end

export fitcubic


using Printf
import Flux: σ
using ModelingToolkit
using GalacticOptim
using Optim
using DiffEqFlux
using NeuralPDE
using Quadrature, Cubature, Cuba
using Plots

@parameters t,x,y
@variables c(..)
@derivatives Dt'~t
@derivatives Dxx''~x
@derivatives Dyy''~y
@derivatives Dx'~x
@derivatives Dy'~y

# Parameters

u = 1.0
v = 1.0
v_vector = (u, v)
R = 0
D = 0 # 0.1 # diffusion
t_max = 2.0
x_min = -1.0
x_max = 1.0
y_min = -1.0
y_max = 1.0

# exp(-(x^2+y^2)/0.1)
# div(v_vector * c) = dx(uc) + dy(vc)
# Equations, initial and boundary conditions
eqs = [ Dt(c(t,x,y)) ~ D * (Dxx(c(t,x,y)) + Dyy(c(t,x,y))) - (u*Dx(c(t,x,y)) + v*Dy(c(t,x,y))) + R]

bcs = [
        c(0, x, y) ~ exp(-(x^2+y^2)/0.1), #cos(π*x) * cos(π*y) + 1.0,
        c(t, x_min, y) ~ c(t, x_max, y),
        c(t, x, y_min) ~ c(t, x, y_max)
]

# Space and time domains
domains = [t ∈ IntervalDomain(0.0,t_max),
        x ∈ IntervalDomain(x_min,x_max),
        y ∈ IntervalDomain(y_min,y_max)
]

# Discretization
nx = 32
# ny = 32
dx = (x_max-x_min) / (nx - 1)
# dy = (y_max-y_min) / (ny -1)
dt = 0.01

# Neural network
dim = length(domains)
output = length(eqs)
hidden = 32

chain = FastChain( FastDense(dim, hidden, tanh),
                    FastDense(hidden, hidden, tanh),
                    FastDense(hidden, 1))

# strategy = GridTraining(dx=[dt,dx,dy])
strategy = StochasticTraining(900)

discretization = PhysicsInformedNN(chain, strategy=strategy)

pde_system = PDESystem(eqs, bcs, domains, [t,x,y], [c])
prob = discretize(pde_system,discretization)

cb = function (p,l)
    println("Current loss is: $l")
    return false
end

res = GalacticOptim.solve(prob,ADAM();cb=cb,maxiters=100)

# Plots

phi = discretization.phi

initθ = discretization.initθ

acum =  [0;accumulate(+, length.(initθ))]
sep = [acum[i]+1 : acum[i+1] for i in 1:length(acum)-1]
minimizers = [res.minimizer[s] for s in sep]
ts,xs,ys = [domain.domain.lower:dx:domain.domain.upper for domain in domains]

# Plot initial condition
c_predict = reshape([ phi([0, x, y], res.minimizer)[1] for x in xs for y in ys], length(xs), length(ys))
plot(xs, ys, c_predict)

# Animate
anim = @animate for (i, t) in enumerate(0:dt:t_max)
    @info "Animating frame $i..."
    c_predict = reshape([phi([t, x, y], res.minimizer)[1] for x in xs for y in ys], length(xs), length(ys))
    title = @sprintf("Advection-diffusion t = %.3f", t)
    heatmap(xs, ys, c_predict, label="", title=title , xlims=(-1, 1), ylims=(-1, 1), color=:thermal, clims=(0, 1))
end

gif(anim, "advection_diffusion_2d_pinn.gif", fps=15)

# Plot correct solution
using JLD2
file = jldopen("advection_diffusion/simulation/cosine_advection_diffusion.jld2")
iterations = parse.(Int, keys(file["timeseries/t"]))

anim = @animate for (i, iter) in enumerate(iterations)
    @info "Animating frame $i..."
    Hx = file["grid/Hx"]
    x = file["grid/xC"][1+Hx:end-Hx]
    t = file["timeseries/t/$iter"]
    c = file["timeseries/c/$iter"][:]

    title = @sprintf("Advection-diffusion t = %.3f", t)
    p = plot(x, c .+ 1, linewidth=2, title=title, label="Oceananigans",
             xlabel="x", ylabel="Tracer", xlims=(-1, 1), ylims=(0, 2))

    c_predict = reshape([phi([t, x], res.minimizer)[1] for x in xs], length(xs))

    plot!(p, x, c_predict, linewidth=2, label="Neural PDE")
end

gif(anim, "tanh_hidden32_it100_comparison.gif", fps=15)


# example 2
using Statistics
using Distributions
using DelimitedFiles
using VegaLite
using DataFrames

const INC_SIZE = 1_000
const activity_DIR="/home/graham_s/OU/DD226/docs/activities/"
incomes = zeros( INC_SIZE )
for i in 1:INC_SIZE
    incomes[i] = 1_000*exp(randn()) # kinda log-normal
end
sort!( incomes )
println("mean  ", mean(incomes))
println( "median ", median(incomes))
println( "kurtosis ", kurtosis(incomes))

writedlm( "$activity_DIR/activity_1.csv", incomes )

incframe = DataFrame( incomes=incomes )

plot_1 = incframe |>
 @vlplot(:bar,
    x={:incomes, bin={maxbins=40}},
    y="count()")

save( "$activity_DIR/activity_1.svg", plot_1 )

const GR_SIZE=100
incomes = zeros(GR_SIZE)
person = zeros(Int8,GR_SIZE)
for i in 1:GR_SIZE
    if rand() < (1/3) # (GR_SIZE ÷ 3)
        pt = 2
        pm = 500.0
    else
        pt = 1
        pm = 1000.0
    end
    incomes[i] = pm*exp(randn())
    person[i] = pt
end

writedlm(  "$activity_DIR/activity_2.csv", person )

non_pensioners = 5_000
pensioners = 5_000
populaton = non_pensioners + pensioners
wp = pensioners/ (sum( person.==2))
wnp = non_pensioners / (sum( person.==1))

gw = population/sum(person)

for i in 1:GR_SIZE
    global wp, wnp, pe;
    if person[i]==2
          pe += wp
    else
         pe += wnp
    end
end

wnp = 74.6268656716418

wp = 151.515151515151

gw = population/(size(person)[1])

print(sum( person.==1))

function mapep(p::Integer)::String

    if p == 0
        return "Not recorded"
    elseif p == 1
        return "Self-employed"
    elseif p in 2:3
        return "Employee"
    elseif p in 4:5
        return "Unemployed/Training Programme"
    elseif p in 6:7
        return "Retired/unoccupied"

    end
end

lcf = CSV.File( "/home/graham_s/OU/DD226/docs/activities/activity_3.csv", delim='\t' ) |> DataFrame

lcf[!,:economic_position] = mapep.(lcf.economic_pos)

plot_inc_vs_cons = lcf |> @vlplot(
    mark={:point,size=1,opacity=0.5},
    opactity=0.5,
    columns=2,
    y=:total_consumpt,
    x=:weekly_net_inc,
    wrap=:economic_position )

save( "$activity_DIR/activity_5.svg", plot_inc_vs_cons )

save( "$activity_DIR/activity_5.png", plot_inc_vs_cons )


# p11.jl - Chebyshev differentation of a smooth function

xx = -1:.01:1; uu = @. exp(xx)*sin(5*xx); clf();
for N = [10 20]
    (D,x) = cheb(N); u = @. exp(x)*sin(5*x);
    axes([.15,.66-.4*(N==20),.31,.28]);
    plot(x,u,".",markersize=6); grid(true);
    plot(xx,uu);
    title("u(x),  N=$N")
    error = D*u - @. exp(x)*(sin(5*x)+5*cos(5*x));
    axes([.55,.66-.4*(N==20),.31,.28]);
    plot(x,error,".-",markersize=6); grid(true);
    title("error in u'(x),  N=$N");
end


using LinearAlgebra

export OrthoMADS

"""
    OrthoMADS()

Return an empty OrthoMADS object.

OrthoMADS implements the poll set generation as described in Audet and Le Digabel 2015
Section 3.4.
"""
struct OrthoMADS <: AbstractPoll end

"""
    GenerateDirections(p::AbstractProblem, o::OrthoMADS)::Matrix

Generates the poll directions, using the poll set generation
as described in Audet and Le Digabel 2015 Section 3.4.
"""
(GenerateDirections(p::AbstractProblem, o::OrthoMADS)::Matrix) =
    GenerateDirections(p.N, o)
function GenerateDirections(N::Int64, ::OrthoMADS)::Matrix
    dirs_on_unit_sphere = GenerateDirectionsOnUnitSphere(N)
    H = HouseholderTransform(dirs_on_unit_sphere)
	return hcat(H, -H)
end

"""
    GenerateDirectionsOnUnitSphere(N::Int64)

Return an `N` length normalized direction on the unit sphere.
"""
function GenerateDirectionsOnUnitSphere(N::Int64)
    dir = randn(N)
    return dir ./ norm(dir)
end

"""
    HouseholderTransform(q)

Apply the Householder transformation to the vector `q`.
"""
function HouseholderTransform(q)
    nq = norm(q)
    v = q./nq
    return nq^2 .* (I - 2*v*v')
end


"""
    rotation_matrix_zyx(α1, α2, α3)

Calculate the rotation matrix ``R_{ab}`` that transforms a vector in b frame to a frame
(``v_{a} = R_{ab} v_{b}``) given the rotations (α1, α2, α3) (rad).

Frame b is obtained from frame a performing three intrinsic rotations of magnitude
α1, α2 and α3 in ZYX order.
"""
function rotation_matrix_zyx(α1, α2, α3)

    sα1, cα1 = sin(α1), cos(α1)
    sα2, cα2 = sin(α2), cos(α2)
    sα3, cα3 = sin(α3), cos(α3)

    R = @SMatrix [
        cα2 * cα1   (sα3 * sα2 * cα1 - cα3 * sα1)   (cα3 * sα2 * cα1 + sα3 * sα1)
        cα2 * sα1   (sα3 * sα2 * sα1 + cα3 * cα1)   (cα3 * sα2 * sα1 - sα3 * cα1)
        -sα2        sα3 * cα2                       cα3 * cα2
    ]

    return R
end

"""
    rotation_matrix_zyx(q0, q1, q2, q3)

Calculate the rotation matrix ``R_{ab}`` that transforms a vector in b frame to a frame
(``v_{a} = R_{ab} v_{b}``) given the quaternions ``q_0, q_1, q_2, q_3``.
"""
function rotation_matrix_zyx(q0, q1, q2, q3)

    q02, q12, q22, q32 = q0*q0, q1*q1, q2*q2, q3*q3

    R = @SMatrix [
        (q02+q12-q22-q32)    2*(q1*q2 - q0*q3)    2*(q1*q3 + q0*q2)
        2*(q1*q2 + q0*q3)    (q02-q12+q22-q32)    2*(q2*q3 - q0*q1)
        2*(q1*q3 - q0*q2)    2*(q2*q3 + q0*q1)    (q02-q12-q22+q32)
    ]

    return R
end

"""
    quaternions(ψ, θ, ϕ)

Calculate quaternion representation given the Euler angles (ψ, θ, ϕ) (rad).
"""
function quaternions(ψ, θ, ϕ)

    s_ψ2, c_ψ2 = sin(ψ/2), cos(ψ/2)
    s_θ2, c_θ2 = sin(θ/2), cos(θ/2)
    s_ϕ2, c_ϕ2 = sin(ϕ/2), cos(ϕ/2)

    q0 = c_ψ2*c_θ2*c_ϕ2 + s_ψ2*s_θ2*s_ϕ2
    q1 = c_ψ2*c_θ2*s_ϕ2 - s_ψ2*s_θ2*c_ϕ2
    q2 = c_ψ2*s_θ2*c_ϕ2 + s_ψ2*c_θ2*s_ϕ2
    q3 = s_ψ2*c_θ2*c_ϕ2 - c_ψ2*s_θ2*s_ϕ2

    return @SVector [q0, q1, q2, q3]
end

"""
    euler_angles(q0, q1, q2, q3)

Calculate Euler angles (ψ, θ, ϕ) (rad) given the quaternions ``q_0, q_1, q_2, q_3``.
"""
function euler_angles(q0, q1, q2, q3)

    ψ = atan(2 * (q1*q2 + q0*q3), q0*q0 + q1*q1 - q2*q2 - q3*q3)
    θ = asin(-2 * (q1*q3 - q0*q2))
    ϕ = atan(2 * (q2*q3 + q0*q1), q0*q0 - q1*q1 - q2*q2 + q3*q3)

    return @SVector [mod2pi(ψ), θ, ϕ]
end

"""
    body2horizon(x, y, z, ψ, θ, ϕ)

Transform the vector coordintes (x, y, z) given in body axis to local horizon given the
Euler angles (ψ, θ, ϕ) (rad).
"""
function body2horizon(x, y, z, ψ, θ, ϕ)
    v = @SVector [x, y, z]
    rv = rotation_matrix_zyx(ψ, θ, ϕ) * v
    return rv
end

"""
    horizon2body(x, y, z, ψ, θ, ϕ)

Transform the vector coordintes (x, y, z) given in local horizon axis to body given the
Euler angles (ψ, θ, ϕ) (rad).
"""
function horizon2body(x, y, z, ψ, θ, ϕ)
    v = @SVector [x, y, z]
    rv = transpose(rotation_matrix_zyx(ψ, θ, ϕ)) * v
    return rv
end

"""
    body2horizon(x, y, z, q0, q1, q2, q3)

Transform the vector coordintes (x, y, z) given in body axis to local horizon given the
quaternions ``q_0, q_1, q_2, q_3``.
"""
function body2horizon(x, y, z, q0, q1, q2, q3)
    v = @SVector [x, y, z]
    rv = rotation_matrix_zyx(q0, q1, q2, q3) * v
    return rv
end

"""
    horizon2body(x, y, z, q0, q1, q2, q3)

Transform the vector coordintes (x, y, z) given in local horizon axis to body given the
quaternions ``q_0, q_1, q_2, q_3``.
"""
function horizon2body(x, y, z, q0, q1, q2, q3)
    v = @SVector [x, y, z]
    rv = transpose(rotation_matrix_zyx(q0, q1, q2, q3)) * v
    return rv
end

"""
    wind2body(x, y, z, α, β)

Transform the vector coordintes (x, y, z) given in wind axis to body given the angle of
attack (α) and the angle of sideslip (β) (rad).
"""
function wind2body(x, y, z, α, β)
    v = @SVector [x, y, z]
    rv = transpose(rotation_matrix_zyx(-β, α, 0)) * v
    return rv
end

"""
    body2wind(x, y, z, α, β)

Transform the vector coordintes (x, y, z) given in body axis to wind given the angle of
attack (α) and the angle of sideslip (β) (rad).
"""
function body2wind(x, y, z, α, β)
    v = @SVector [x, y, z]
    rv = rotation_matrix_zyx(-β, α, 0) * v
    return rv
end

"""
    wind2horizon(x, y, z, χ, γ, μ)

Transform the vector coordintes (x, y, z) given in wind axis to local horizon given the
velocity angles (χ, γ, μ) (rad).
"""
function wind2horizon(x, y, z, χ, γ, μ)
    v = @SVector [x, y, z]
    rv = rotation_matrix_zyx(χ, γ, μ) * v
    return rv
end

"""
    horizon2wind(x, y, z, χ, γ, μ)

Transform the vector coordintes (x, y, z) given in local horizon axis to wind given the
velocity angles (χ, γ, μ) (rad).
"""
function horizon2wind(x, y, z, χ, γ, μ)
    v = @SVector [x, y, z]
    rv = transpose(rotation_matrix_zyx(χ, γ, μ)) * v
    return rv
end

"""
    ecef2horizon(x, y, z, lat, lon)

Transform the vector coordintes (x, y, z) given in ECEF (Earth Fixed Earth Centered)
coordinates to local horizon coordinates using geodetic latitude and longitude (rad).
"""
function ecef2horizon(x, y, z, lat, lon)
    v = @SVector [x, y, z]
    rv = transpose(rotation_matrix_zyx(lon, -lat - π/2, 0)) * v
end

"""
    horizon2ecef(x, y, z, lat, lon)

Transform the vector coordintes (x, y, z) given in local horizon axis to ECEF (Earth Fixed
Earth Centered) using geodetic latitude and longitude (rad).
"""
function horizon2ecef(x, y, z, lat, lon)
    v = @SVector [x, y, z]
    rv = rotation_matrix_zyx(lon, -lat - π/2, 0) * v
end


# chemical problems
# 22
function haverlys_pooling_problem(x)
    ## Haverly's Pooling Problem
    x1 = x[1]; x2 = x[2]; x3 = x[3];
    x4 = x[4]; x5 = x[5]; x6 = x[6];
    x7 = x[7]; x8 = x[8]; x9 = x[9];
    ## objective function
    f = zeros(2);
    f[1] = -9*x1-15*x2+6*x3+16*x4;
    f[2] = 10 .* (x5+x6);
    ## constraints
    g = zeros(2);
    h = zeros(4);
    g[1] = x9 .* x7+2*x5-2.5*x1;
    g[2] = x9 .* x8+2*x6-1.5*x2;
    h[1] = x7+x8-x4-x3;
    h[2] = x1-x5-x7;
    h[3] = x2-x6-x8;
    h[4] = x9 .* x7+x9 .* x8-3 .* x3-x4;

    return f, g, h
end

# 23
function reactor_network_design(x)
    ## Reactor Network Design
    k1 = 0.09755988; k2 = 0.99*k1;
    k3 = 0.0391908; k4 = 0.9*k3;
    x1 = x[1]; x2 = x[2]; x3 = x[3];
    x4 = x[4]; x5 = x[5]; x6 = x[6];
    ## objective function
    f = zeros(2);
    f[1] = -x4;
    f[2] = x5 .^ (0.5)+x6 .^ (0.5);
    ## constraints
    g = zeros(1);
    h = zeros(4);
    g[1] = f[2]-4;
    h[1] = k1 .* x5 .* x2 + x1 -1;
    h[2] = k3 .* x5 .* x3+x3+x1-1;
    h[3] = k2 .* x6 .* x2 - x1 + x2;
    h[4] = k4 .* x6 .* x4 + x2-x1+x4-x3;

    return f, g, h
end

# 24
function heat_exchanger_network_design(x)
    ## Heat Exchanger Network Design
    x1 = x[1]; x2 = x[2]; x3 = x[3]; x4 = x[4]; x5 = x[5];
    x6 = x[6]; x7 = x[7]; x8 = x[8]; x9 = x[9];
    ## objective function
    f = zeros(3);
    f[1] = 35 .* x1 .^ (0.6)+ 35 .* x2 .^ (0.6);
    f[2] = 200 .* x1 .* x4-x3;
    f[3] = 200 .* x2 .* x6-x5;
    ## constraints
    g = zeros(1);
    h = zeros(6);
    h[1] = x3 - 1e4 .* (x7-100);
    h[2] = x5 - 1e4 .* (300-x7);
    h[3] = x3 - 1e4 .* (600-x8);
    h[4] = x5 - 1e4 .* (900-x9);
    h[5] = x4 .* log(abs(x8-100)+1e-6)-x4 .* log(abs(600-x7)+1e-6)-x8+x7+500;
    h[6] = x6 .* log(abs(x9-x7)+1e-6)-x6 .* log(600)-x9+x7+600;

    return f, g, h
end


module FermiDiracIntegrals

using Polylogarithms

export F

"""
Complete Fermi-Dirac-integral

Arguments:
* j
* x

Formula:
``{F_j(x) = \\frac{1}{\\Gamma(j+1)} \\int_0^{\\infty}{\\frac{t^j}{\\exp(t-x)+1}dt}}``

Implementation:
Using the polylogarithm
"""
function F(j,x)
    -polylog(j+1,-exp(x))
end

"""
Approximation of the complete Fermi-Dirac-integral for j = 1/2

Checked for a relative tolerance of 3% in the range x = -100:0.1:100

Speed: 100 times faster than the polylog version

Source:
J. S. Blakemore: Approximations for Fermi-Dirac Integrals. Solid-State Electronics, 25(11):1067-1076, 1982.
"""
function F(::Val{1/2},x)
    if x < 1.3
        1/(exp(-x)+0.27)
    else
        4/3/sqrt(pi)*(x^2+pi^2/6)^(3/4)
    end
end

end


const DT = Integer

function find_amicable(limit::DT)
  T = typeof(limit)
  v = Vector{Tuple{Pair{T, Vector{T}}, Pair{T, Vector{T}}}}()
  t = Set{T}()

  for n ∈ one(T):limit
    d₁ = divisors(n)
    p = sum(d₁)
    d₂ = divisors(p)
    s = sum(d₂)

    if s == n && d₁ ≠ d₂ && s ∉ t
      push!(t, s)
      push!(t, p)
      push!(v, (n => d₁, p => d₂))
    end
  end

  (t, v)
end

# (Set([2924, 5564, 6232, 5020, 220, 284, 1184, 2620, 1210, 6368]),
#  [(220 => [1, 2, 4, 5, 10, 11, 20, 22, 44, 55, 110], 284 => [1, 2, 4, 71, 142]), (1184 => [1, 2, 4, 8, 16, 32, 37, 74, 148, 296, 592], 1210 => [1, 2, 5, 10, 11, 22, 55, 110, 121, 242, 605]), (2620 => [1, 2, 4, 5, 10, 20, 131, 262, 524, 655, 1310], 2924 => [1, 2, 4, 17, 34, 43, 68, 86, 172, 731, 1462]), (5020 => [1, 2, 4, 5, 10, 20, 251, 502, 1004, 1255, 2510], 5564 => [1, 2, 4, 13, 26, 52, 107, 214, 428, 1391, 2782]), (6232 => [1, 2, 4, 8, 19, 38, 41, 76, 82, 152, 164, 328, 779, 1558, 3116], 6368 => [1, 2, 4, 8, 16, 32, 199, 398, 796, 1592, 3184])])
#
# julia> s |> sum
# 31626

function is_amicable(n₁::Any, n₂::Any)
  throw(ArgumentError("n₁ and n₂ should be Integers"))
end

function is_amicable(n₁::DT, n₂::DT)::Bool
  d₁ = divisors(n₁)
  d₂ = divisors(n₂)
  n₂ == sum(d₁) && n₁ == sum(d₂)
end

function divisors(n::DT)::Vector{<: DT}
  T = typeof(n)
  ary::Vector{T} = [one(T), ]

  d = T(2)
  while d * d ≤ n
    q, r = divrem(n, d)
    if r == 0
      q == d ? push!(ary, d) : push!(ary, d, q)
    end
    d += 1
  end

  return ary |> sort
end


# # Singular Integrals

# This example shows how to use the `Integration` module for computing
# quadrature rules for functions with point singularities (such as those
# appearing in the numerical discretization of boundary integral equations).

# ## Change of variables

# The first set of *tricks* revolve a round a simple change of variables. We
# focus first on the one-dimensional case, where we wish to integrate
```math
    \int_0^1 f(x) dx,
```
# and where the function ``f`` can have an integrable singularity at

using WaveProp.Geometry
using WaveProp.Integration
using WaveProp.Integration
using QuadGK

f        = (x) -> x==0 ? 0.0 : log(abs(x))*cos(x)
I,_        = quadgk(f,0,1,rtol=1e-16)

rows = GaussLegendre.([5,10,20,40,80])
cols = [identity,IMT{1,2}(),Kress{8}(), Window{1,1,7}()]
ee   = []
for qstd in rows
    for shandler in cols
        q        = SingularQuadratureRule(qstd,shandler)
        Is       = integrate(f,q)
        er       = abs(I-Is)
        push!(ee,er)
    end
end

using NamedArrays
ee = reshape(ee,length(cols),length(rows)) |> transpose |> NamedArray
setnames!(ee,string.(rows),1)
setnames!(ee,string.(cols),2)
setdimnames!(ee,["Base quadrature","Singularity handler"])
show(ee)

# ##
# using Plots
# qstd = GaussLegendre(10)
# x̂,ŵ  = qstd()
# cols = [IMT{1,2}(),Kress{8}(), Window{0.5,1,7}()]
# fig = plot()
# for shandler in cols
#     phi       = shandler.(x̂)
#     phip      = [jacobian(shandler,x)[1] for x in x̂]
#     plot!(fig,x̂,f.(phi) .* phip,label=string(shandler),m=:x)
# end
# display(fig)


immutable HessenbergQR{T}
    cosθ::Vector{T}
    sinθ::Vector{T}
end

## QR decomposition on an upper Hessenberg matrix
## H will be overwritten by Q'HQ=RQ
function hessenqr!{T}(H::Matrix{T})
    ## Size of matrix
    n = size(H, 1)
    if n != size(H, 2)
        error("matrix must be square")
    end

    ## Rotation factors
    ## Each pair (cosθ[i], sinθ[i]) forms a rotation matrix
    ## Gi = [cosθ[i]  -sinθ[i]]
    ##      [sinθ[i]   cosθ[i]]
    cosθ = ones(T, n)
    sinθ = zeros(T, n)

    for i = 1:(n - 1)
        ## Make sure H is upper Hessenberg
        ## Zero the elements below H[i+1, i]
        H[(i + 2):end, i] = zero(T)
        ## Calculate cosθ and sinθ
        x = H[i, i]
        y = H[i + 1, i]
        r = hypot(x, y)
        ## If r is too small, (cosθ, sinθ) stores the original values (1, 0)
        if r < eps(T)
            r = zero(T)
        else
            cosθ[i] = x / r
            sinθ[i] = -y / r
        end
        ## Apply the rotation on the left H -> Gi * H
        ## H[i, :]     <- cosθ[i] * H[i, :] - sinθ[i] * H[i +1, :]
        ## H[i + 1, :] <- sinθ[i] * H[i, :] + cosθ[i] * H[i +1, :]
        H[i, i] = r
        H[i + 1, i] = zero(T)
        c = cosθ[i]
        s = sinθ[i]
        for j = (i + 1):n
            tmp = H[i, j]
            H[i, j]     = c * tmp - s * H[i + 1, j]
            H[i + 1, j] = s * tmp + c * H[i + 1, j]
        end
    end

    ## Apply the rotations on the right H -> H * Gi'
    ## H[:, i]     <- cosθ[i] * H[:, i] - sinθ[i] * H[:, i + 1]
    ## H[:, i + 1] <- sinθ[i] * H[:, i] + cosθ[i] * H[:, i + 1]
    for i = 1:(n - 1)
        for j = 1:(i + 1)
            tmp = H[j, i]
            H[j, i]     = cosθ[i] * tmp - sinθ[i] * H[j, i + 1]
            H[j, i + 1] = sinθ[i] * tmp + cosθ[i] * H[j, i + 1]
        end
    end

    return HessenbergQR(cosθ, sinθ)
end

## QR decomposition on a tridiagonal matrix
## H will be overwritten by Q'HQ=RQ
function tridiagqr!{T}(H::Matrix{T})
    ## Size of matrix
    n = size(H, 1)
    if n != size(H, 2)
        error("matrix must be square")
    end

    ## Force H to be tridiagonal
    maindiag = diag(H)
    subdiag = diag(H, -1)
    fill!(H, zero(T))
    for i = 1:(n - 1)
        H[i, i] = maindiag[i]
        H[i + 1, i] = subdiag[i]
        H[i, i + 1] = subdiag[i]
    end
    H[n, n] = maindiag[n]

    ## Rotation factors
    ## Each pair (cosθ[i], sinθ[i]) forms a rotation matrix
    ## Gi = [cosθ[i]  -sinθ[i]]
    ##      [sinθ[i]   cosθ[i]]
    cosθ = ones(T, n)
    sinθ = zeros(T, n)

    for i = 1:(n - 1)
        ## Calculate cosθ and sinθ
        x = H[i, i]
        y = H[i + 1, i]
        r = hypot(x, y)
        ## If r is too small, (cosθ, sinθ) stores the original values (1, 0)
        if r < eps(T)
            r = zero(T)
        else
            cosθ[i] = x / r
            sinθ[i] = -y / r
        end
        ## Apply the rotation on the left H -> Gi * H
        ## Update H[i, i] and H[i + 1, i]
        H[i, i] = r
        H[i + 1, i] = zero(T)
        ## Update H[i, i + 1] and H[i + 1, i + 1]
        c = cosθ[i]
        s = sinθ[i]
        tmp = H[i, i + 1]
        H[i,     i + 1] = c * tmp - s * H[i + 1, i + 1]
        H[i + 1, i + 1] = s * tmp + c * H[i + 1, i + 1]
        ## Update H[i, i + 2] and H[i + 1, i + 2]
        if i < n - 1
            H[i,     i + 2] = -s * H[i + 1, i + 2]
            H[i + 1, i + 2] *= c
        end
    end

    ## Apply the rotations on the right H -> H * Gi'
    ## H[:, i]     <- cosθ[i] * H[:, i] - sinθ[i] * H[:, i + 1]
    ## H[:, i + 1] <- sinθ[i] * H[:, i] + cosθ[i] * H[:, i + 1]
    for i = 1:(n - 1)
        c = cosθ[i]
        s = sinθ[i]

        tmp = H[i + 1, i]
        H[i, i]         = c * H[i, i] - s * H[i, i + 1]
        H[i + 1, i]     = c * tmp     - s * H[i + 1, i + 1]
        H[i + 1, i + 1] = s * tmp     + c * H[i + 1, i + 1]

        H[i, i + 1] = H[i + 1, i]
        if i < n - 1
            H[i, i + 2] = zero(T)
        end
    end

    return HessenbergQR(cosθ, sinθ)
end

## Apply the QR factorization to the right of a matrix A
## A -> A * Q'
function applyright!{T}(qr::HessenbergQR{T}, A::Matrix{T})
    cosθ = qr.cosθ
    sinθ = qr.sinθ
    n = length(cosθ)
    ## A[:, i]     <- cosθ[i] * A[:, i] - sinθ[i] * A[:, i + 1]
    ## A[:, i + 1] <- sinθ[i] * A[:, i] + cosθ[i] * A[:, i + 1]
    for i = 1:(n - 1)
        for j = 1:size(A, 1)
            tmp = A[j, i]
            A[j, i]     = cosθ[i] * tmp - sinθ[i] * A[j, i + 1]
            A[j, i + 1] = sinθ[i] * tmp + cosθ[i] * A[j, i + 1]
        end
    end
end


# Test get/set objective coefficients in LP

using Xpress, Compat.Test

@testset "Basics 3" begin
    # original model
    #
    #   maximize  2x + 2y
    #
    #   s.t. 0.2 <= x, y <= 1

    model = xpress_model(
    name="lp_03",
    sense=:maximize,
    f=[2.0, 2.0],
    lb=[0.2, 0.2],
    ub=[1.0, 1.0])

    lb_ = lowerbounds(model)
    ub_ = upperbounds(model)
    c_ = objcoeffs(model)

    @test lb_ == [0.2, 0.2]
    @test ub_ == [1.0, 1.0]
    @test c_ == [2.0, 2.0]

    optimize(model)

    ans = get_optiminfo(model)
    @test ans.status_lp == :optimal

    @test isapprox(get_solution(model), [1, 1]; atol = 1e-3)
    @test isapprox(get_objval(model), 4; atol = 1e-3)

    # change objective (warm start)
    #
    #    maximize x - y
    #
    #    s.t. 0.2 <= x, y <= 1
    #

    set_objcoeffs!(model, [1, -1])

    c_ = objcoeffs(model)
    @test c_ == [1.0, -1.0]

    optimize(model)

    ans = get_optiminfo(model)
    @test ans.status_lp == :optimal

    @test isapprox(get_solution(model), [1.0, 0.2]; atol = 1e-3)
    @test isapprox(get_objval(model), 0.8; atol = 1e-3)
end


"""
    FrankCopula{d,T}

Fields:
  - θ::Real - parameter

Constructor

    FrankCopula(d, θ)

The [Frank](https://en.wikipedia.org/wiki/Copula_(probability_theory)#Most_important_Archimedean_copulas) copula in dimension ``d`` is parameterized by ``\\theta \\in [0,\\infty)``. It is an Archimedean copula with generator :

```math
\\phi(t) = -\\frac{\\log\\left(1+e^{-t}(e^{-\\theta-1})\\right)}{\theta}
```
"""
struct FrankCopula{d,T} <: ArchimedeanCopula{d}
    θ::T
end
FrankCopula(d,θ) = θ >= 0 ? FrankCopula{d,typeof(θ)}(θ) : @error "Theta must be positive"
ϕ(  C::FrankCopula,       t) = -log(1+exp(-t)*(exp(-C.θ)-1))/C.θ
ϕ⁻¹(C::FrankCopula,       t) = -log((exp(-t*C.θ)-1)/(exp(-C.θ)-1))

D₁ = GSL.sf_debye_1 # sadly, this is C code.
# could be replaced by :
# using QuadGK
# D₁(x) = quadgk(t -> t/(exp(t)-1), 0, x)[1]/x
# to make it more general. but once gain, it requires changing the integrator at each evlauation,
# which is problematic.
# Better option is to try to include this function into SpecialFunctions.jl.

τ(C::FrankCopula) = 1+4(D₁(C.θ)-1)/C.θ
function τ⁻¹(::Type{FrankCopula},τ)
    if τ == zero(τ)
        return τ
    end
    x₀ = (1-τ)/4
    return Roots.fzero(x -> (1-D₁(x))/x - x₀, 10)
end

radial_dist(C::FrankCopula) = Logarithmic(1-exp(-C.θ))


# Test the finite differencing technique
@testset "finite_difference" begin

function f!(fvec, x)
    fvec[1] = (x[1]+3)*(x[2]^3-7)+18
    fvec[2] = sin(x[2]*exp(x[1])-1)
end

r = nlsolve(f!, [ 0.1; 1.2])

@test norm(r.zero - [ 0; 1]) < 1e-8

end


using EngEconomics

# Given
i = 0.1
xInit = 12500
yInit = 8900
xAnnualBenefit = 6800
yAnnualBenefit = 2000
xSalvage = 5000
ySalvage = 8900
xLife = 2
yLife = 3

# Find: Present Worth Analysis of Both

# Determine the least common multiple of xLife and yLife to figure out how many
# times we have to repeat the PW Analysis
years = 6
xTimes = years / xLife
yTimes = years / yLife

# Determine Product X's PW
xInitPW = -xInit * (1 + presentWorthFactor(i, xLife) + presentWorthFactor(i, 2 * xLife))
xAnnualBenefitPW = (xAnnualBenefit * seriesPresentAmountFactor(i, xLife)) * (1 + presentWorthFactor(i, xLife) + presentWorthFactor(i, 2 * xLife))
xSalvagePW = xSalvage * (presentWorthFactor(i, xLife) + presentWorthFactor(i, 2 * xLife) + presentWorthFactor(i, 3 * xLife))
xPW = xInitPW + xAnnualBenefitPW + xSalvagePW

# Determine Product Y's PW
yInitPW = -yInit * (1 + presentWorthFactor(i, yLife))
yAnnualBenefitPW = (yAnnualBenefit * seriesPresentAmountFactor(i, yLife)) * (1 + presentWorthFactor(i, yLife))
ySalvagePW = ySalvage * (presentWorthFactor(i, yLife) + presentWorthFactor(i, 2 * yLife))
yPW = yInitPW + yAnnualBenefitPW + ySalvagePW

# Determine which product is better by comparing they're present value
if xPW > yPW
	println("Pick Product X")
else
	println("Pick Product Y")
end


import Pkg; Pkg.add(Pkg.PackageSpec(url="https://github.com/JuliaComputing/JuliaAcademyData.jl"))
using JuliaAcademyData; activate("Deep learning with Flux")

#

# ## Going deep: Deep neural networks
#
# So far, we've learned that if we want to classify more than two fruits, we'll need to go beyond using a single neuron and use *multiple* neurons to get multiple outputs. We can think of stacking these multiple neurons together in a single neural layer.
#
# Even so, we found that using a single neural layer was not enough to fully distinguish between bananas, grapes, **and** apples. To do this properly, we'll need to add more complexity to our model. We need not just a neural network, but a *deep neural network*.
#
# There is one step remaining to build a deep neural network. We have been saying that a neural network takes in data and then spits out `0` or `1` predictions that together declare what kind of fruit the picture is. However, what if we instead put the output of one neural network layer into another neural network layer?
#
# This gets pictured like this below:
#
#
#
# On the left we have 3 data points in blue. Those 3 data points each get fed into 4 neurons in purple. Each of those 4 neurons produces a single output, but those output are each fed into three neurons (the second layer of purple). Each of those 3 neurons spits out a single value, and those values are fed as inputs into the last layer of 6 neurons. The 6 values that those final neurons produce are the output of the neural network. This is a deep neural network.

#-

# ### Why would a deep neural network be better?
#
# This is a little perplexing when you first see it. We used neurons to train the model before: why would sticking the output from neurons into other neurons help us fit the data better? The answer can be understood by drawing pictures. Geometrically, the matrix multiplication inside of a layer of neurons is streching and rotating the axis that we can vary:

## using Plots, Random; Random.seed!(0); xs = range(0, stop=10, length=200); ys = randn.() .+ xs .+ 0.5.*xs.^1.5 .+ 6.0.*sin.(clamp.(xs .- 5, 0, Inf)) .+ 4.5.*clamp.(.-abs.(xs .- 2), -2, 0); scatter(xs, ys, label="", ticks = false, title="Original data", ylim=extrema(ys).+[-.5,.5]); savefig("data/17-raw.png"); fit1 = [xs ones(size(xs))] \ ys; linear1(x) = fit1[1]*x + fit1[2]; plot!(linear1, label="Linear transform", linewidth=4, legend=:topleft); savefig("data/17-raw-withfit.png");
HTML("""""")

# A nonlinear transformation—such as the sigmoid function or a piecewise linear function—then adds a bump to this linearly-transformed data:

## scatter(xs, ys .- linear1.(xs), ticks = false, label = "", title="data with linear transform", ylim=extrema(ys).+[-.5,.5]); savefig("data/17-linear1.png"); nonlinearity(x) = clamp.(4.0.*x .- 4.5, -4, 4); plot!(nonlinearity, label = "nonlinearity", linewidth=4); savefig("data/17-linear1-withfit.png");
HTML("""""")

# Resulting in a bit more of the data accounted for:

## ys2 = ys .- linear1.(xs) .- nonlinearity.(xs); scatter(xs, ys2, ticks = false, label = "", title="data after first 'layer'", ylim=extrema(ys).+[-.5,.5]); savefig("data/17-nonlinear1.png"); fit2 = [xs ones(size(xs))] \ ys2; linear2(x) = fit2[1]*x + fit2[2]; plot!(linear2, label="second linear tranform", linewidth=4); savefig("data/17-nonlinear1-withfit.png");
HTML("""""")

# Now let's repeat this process. When we send the data through another layer of neurons, we get another rotation and another "bump":

## ys3 = ys2 .- linear2.(xs); scatter(xs .- 4.5, ys3, ticks = false, label = "", title="data after first layer + second linear transform", ylim=extrema(ys).+[-.5,.5]); savefig("data/17-linear2.png"); plot!(nonlinearity, label = "nonlinearity", linewidth=4); savefig("data/17-linear2-withfit.png");
HTML("""""")

#-

## ys4 = ys3 .- nonlinearity.(xs .- 4.5); scatter(xs, ys4, ticks = false, label = "", title="data after second 'layer'", ylim=extrema(ys).+[-.5,.5]); savefig("data/17-nonlinear2.png");
HTML("""""")

#
# Visually, we see that if we keep doing this process we can make the axis line up with any data. What this means is that **if we have enough layers, then our neural network can approximate any model**.
#
# The trade-off is that with more layers we have more parameters, so it may be harder (i.e. computationally intensive) to train the neural network. But we have the guarantee that the model has enough freedom such that there are parameters that will give the correct output.
#
# Because this model is so flexible, the problem is reduced to that of learning: do the same gradient descent method on this much larger model (but more efficiently!) and we can make it classify our data correctly. This is the power of deep learning.

#-

# **Important caveat**: The above is a _cartoon_ that loosely describes the concepts behind a deep neural network. In reality, remember that we're not successively fitting "the best" linear transformation and then subsequently fitting a nonlinearity; the two are happening simultaneously through gradient descent. This allows the linear transform to — in effect — rotate and stretch the data such that the nonlinearity "falls" in the appropriate place to best minimize the loss function.

#-

# # Deep learning with Flux

#-

# Let's load the same datasets from the previous lecture and pre-process them in the same way:

using CSV, DataFrames, Flux, Plots
apples1 = DataFrame(CSV.File("data/Apple_Golden_1.dat", delim='\t', allowmissing=:none, normalizenames=true))
apples2 = DataFrame(CSV.File("data/Apple_Golden_2.dat", delim='\t', allowmissing=:none, normalizenames=true))
apples3 = DataFrame(CSV.File("data/Apple_Golden_3.dat", delim='\t', allowmissing=:none, normalizenames=true))
apples = vcat(apples1, apples2, apples3)
bananas = DataFrame(CSV.File("data/Banana.dat", delim='\t', allowmissing=:none, normalizenames=true))
grapes1 = DataFrame(CSV.File("data/Grape_White.dat", delim='\t', allowmissing=:none, normalizenames=true))
grapes2 = DataFrame(CSV.File("data/Grape_White_2.dat", delim='\t', allowmissing=:none, normalizenames=true))
grapes = vcat(grapes1, grapes2)
## Extract out the features and construct the corresponding labels
x_apples  = [ [apples[i, :red], apples[i, :blue]] for i in 1:size(apples, 1) ]
x_bananas  = [ [bananas[i, :red], bananas[i, :blue]] for i in 1:size(bananas, 1) ]
x_grapes = [ [grapes[i, :red], grapes[i, :blue]] for i in 1:size(grapes, 1) ]
xs = vcat(x_apples, x_bananas, x_grapes)
ys = vcat(fill(Flux.onehot(1, 1:3), size(x_apples)),
          fill(Flux.onehot(2, 1:3), size(x_bananas)),
          fill(Flux.onehot(3, 1:3), size(x_grapes)));

# In the previous lecture, we used a `Dense(2, 3, σ)` as our model. Now we want to construct multiple layers and chain them together:

layer1 = Dense(2, 4, σ)
layer2 = Dense(4, 3, σ)

#-

layer2(layer1(xs[1]))

#-

#nb ?Chain
#jl @doc Chain

#-

m = Chain(layer1, layer2)
m(xs[1])

#-

xs[1] |> layer1 |> layer2

# ### The core algorithm from the last lecture

model = Chain(Dense(2, 3, σ)) # Update this!
L(x,y) = Flux.mse(model(x), y)
opt = SGD(params(model))
Flux.train!(L, zip(xs, ys), opt)

#-

## Recall his is how we repeatedly walked down our gradient previously...
for _ in 1:1000
    Flux.train!(L, zip(xs, ys), opt)
end
## But our model is now more complicated and this will take more time!

#-

data = zip(xs, ys)
@time Flux.train!(L, data, opt)
@time Flux.train!(L, data, opt)

# ### Improving efficiency by batching

length(data)

#-

first(data)

# Recall our matrix-vector multiplication from the previous lecture:

W = [10 1;
     20 2;
     30 3]
x = [3;
     2]
W*x

#-

Flux.batch(xs)

#-

model(Flux.batch(xs))

#-

databatch = (Flux.batch(xs), Flux.batch(ys))
@time Flux.train!(L, (databatch,), opt)
@time Flux.train!(L, (databatch,), opt)

#-

Flux.train!(L, Iterators.repeated(databatch, 10000), opt)

#-

L(databatch[1], databatch[2])

# ### Visualization

using Plots
function plot_decision_boundaries(model, x_apples, x_bananas, x_grapes)
    plot()

    contour!(0:0.01:1, 0:0.01:1, (x,y)->model([x,y]).data[1], levels=[0.5, 0.501], color = cgrad([:blue, :blue]), colorbar=:none)
    contour!(0:0.01:1, 0:0.01:1, (x,y)->model([x,y]).data[2], levels=[0.5,0.501], color = cgrad([:green, :green]), colorbar=:none)
    contour!(0:0.01:1, 0:0.01:1, (x,y)->model([x,y]).data[3], levels=[0.5,0.501], color = cgrad([:red, :red]), colorbar=:none)

    scatter!(first.(x_apples), last.(x_apples), m=:cross, label="apples", color = :blue)
    scatter!(first.(x_bananas), last.(x_bananas), m=:circle, label="bananas", color = :green)
    scatter!(first.(x_grapes), last.(x_grapes), m=:square, label="grapes", color = :red)
end
plot_decision_boundaries(model, x_apples, x_bananas, x_grapes)

# ### Further improvements with a better loss function and normalization of outputs

scatter([0],[0], label="correct answer", xlabel="model output: [1-x,x]", ylabel="loss against [1, 0]", legend=:topleft, title="Loss function behavior")
plot!(x->Flux.mse([1-x, x/2], [1,0]), -1.5, 1.5, label="mse")
## plot!(x->Flux.crossentropy([1-x, x/2], [1,0]), 0, 1, label="crossentropy")

#-

sum(model(xs[1]))

#-

Flux.mse([0.01,0.98,0.01], [1.0,0,0])

#-

softmax([1.0,-3,0])

# ### The new algorithm
#
# Use `softmax` as a final normalization and change the loss function to `crossentropy`:

model = Chain(Dense(2, 4, σ), Dense(4, 3, identity), softmax)
L(x,y) = Flux.crossentropy(model(x), y)
opt = SGD(params(model))

#-

Flux.train!(L, Iterators.repeated(databatch,5000), opt)

#-

plot_decision_boundaries(model, x_apples, x_bananas, x_grapes)

Courses/Deep learning with Flux/1700.Introduction-to-deep-learning.jl


# Faster than Polynomials.jl with less memory allocation + single-point type
# stability; adapted from Octave and
# https://github.com/JuliaMath/Polynomials.jl/blob/master/src/Polynomials.jl
function poly(x::Array{T,1}) where T <: Number
  n = length(x)
  y = zeros(T, n+1)
  y[1] = one(T)
  for j = 1:n
    y[2:j+1] .-= x[j].*y[1:j]
  end
  return y
end

function polyval(p::Array{T1,1}, x::T2) where {T1 <: Number, T2 <: Number}
  y = T2(p[1])
  for i = 2:lastindex(p)
    y = p[i] .+ x*y
  end
  return y
end

function polyval(p::Array{T1,1}, x::Array{T2,1}) where {T1 <: Number, T2 <: Number}
  y = ones(T2, length(x)) .* p[1]
  for i = 2:length(p)
    broadcast!(*, y, y, x)
    broadcast!(+, y, y, p[i])
  end
  return y
end

function polyfit(x::Array{T1,1}, y::Array{T2,1}, n::Integer=1) where {T1 <: Real, T2 <: Real}
  nx = length(x)
  nx == length(y) || error("SeisIO.polyfit requires length(t) == length(x)")
  -1 < n < nx || throw(DomainError)
  A = Array{T2, 2}(undef, length(x), n+1)
  A[:,n+1] .= one(T2)
  for i = n:-1:1
      A[:,i] .= A[:,i+1] .* x
  end
  return A \ y
end

# Convert to Float64 or use improved sum
function linreg(t::Array{Float64,1}, x::AbstractArray{T,1}) where T
  n = length(t)
  st = sum(t)
  sx = sum(x)
  stt = dot(t,t)
  stx = dot(t,x)
  sxx = dot(x,x)
  d = n*stt - st*st
  b = (stt*sx - st*stx)/d
  a = (n*stx - st*sx)/d
  return T[a,b]
end

# p = linreg(x, fs)
function linreg(x::AbstractArray{T,1}, dt::Float64) where T
  n = length(x)
  t = (1:n)*dt
  st = sum(t)
  sx = sum(x)
  stt = dot(t,t)
  stx = dot(t,x)
  sxx = dot(x,x)
  d = n*stt - st*st
  b = (stt*sx - st*stx)/d
  a = (n*stx - st*sx)/d
  return T[a,b]
end

#= CHANGELOG for this file
2019-08-08
* poly, polyval, and polyfit should now always output powers in descending
order, i.e., p^n ... p^0
* BUG fixed: polyval(p::Array{T,1}, x::T) used power ordering of Polynomials.jl;
corrected to be consistent with other routines
* polyfit now allows order n=0 and takes any Integer for n; n=0 returns the mean
2019-08-19
* added linreg for low-memory linear regression; identical to SAC detrend
2019-09-03
* bug fix for rare situation where float precision led to length(t) != length(x)
=#


"""
    $(SIGNATURES)

Test by regression that simulated data are consistent with "true" outcomes.

Regress simulated on true.
Check for 0 intercept, 1 slope.

Note: If data are very far from 0, then intercept can be quite a bit off.

# Arguments
- trueM
    true outcome by state
- simM
    simulated outcome by state
- wtM
    weights (e.g. to downweight rare states or states that cannot be solved precisely)
    scalar weights are expanded. Provide wtM = []
- doShow
    show scatter plot?

# Outputs
dev = max(abs(beta - [0;1]) / se(beta))
    how many std deviations are regression coefficient from 45 degree line?
betaV
    regression coefficients; should be [0;1]
seV
    std errors

Change:
    test hypothesis of 0 intercept and unit slope. Return prob of reject
    GLM cannot do that yet.
"""
function regression_test(trueM :: AbstractArray{F1}, simM :: AbstractArray{F1};
    weights = similar(trueM, 0)) where F1 <: AbstractFloat

    @assert size(trueM) == size(simM)
    n = length(trueM);
    @assert n > 2  "Too few observations: $n"

    m = fit(LinearModel, hcat(ones(F1, n),  vec(trueM)),  vec(simM);
        wts = weights);
    betaV = coef(m);
    seV = stderror(m);
    dev = maximum(abs.(betaV .- [zero(F1), one(F1)]) ./ max.(F1(0.01), seV));
    return dev, betaV, seV
end

# ---------------


#######################
## Prepare libraries ##
#######################

# export JULIA_NUM_THREADS=4

BASE_FOLDER = pwd()
(basename(BASE_FOLDER) == "examples") && (BASE_FOLDER = dirname(BASE_FOLDER))
using Pkg
Pkg.activate(BASE_FOLDER)
# Install all required packages if not present
isfile(joinpath(BASE_FOLDER,"Manifest.toml")) || Pkg.instantiate()

using Combinatorics, Distributions, LightGraphs, LinearAlgebra, QuantEcon
using Roots, SymPy, Manifolds
using Nash

## Create game from provided payoff matrices
# Player 1 payoff matrix is
    # 1  0
    # 0  1
# Player 2 payoff matrix is
    # 1  0
    # 1  1
generate_game([1 0; 0 1], [1 0; 1 1])

## Generate random game for 2 players.
# First player has 2 actions and second player has 3 actions
# Payoffs are sampled from standard normal Distributions

random_2players_game(Normal(0,1),2,3)

## Generate random game for 3 players.
# First player has 2 actions, second player has 3 actions, third player has 3 actions
# Payoffs are sampled from Binomial distribution with 5 trials and p = 0.5

random_nplayers_game(Binomial(5,0.5),[2,3,3])

## Get payoffs for each player from given mixed strategy profiles
# Player 1 plays first and second action with p = 0.5
# Player 2 plays first action with p = 0.75 and second action with p = 0.25

game = random_nplayers_game(Binomial(5,0.5),[2,2]);
s = [[0.5,0.5],[0.75,0.25]]
s[1]'*game["player2"]*s[2]
get_payoff(game,s)

## Return best reply of first (k = 1) player
# with perturbation epsil=0.5
best_reply(generate_game([1 0; 0 1], [1 0; 0 1]), [[1.0, 0.], [0, 1]], 1, 0.5)
# no perturbation, with convex hull
a2 = best_reply(generate_game(Matrix(I,3,3), Matrix(I,3,3)), [[1/2,1/2,0],[1/3,1/3,1/3]], 1, return_val = "chull")
# plot best responses
plot_br(a2)

# Check if passed strategy is best response for each player and is it a Nash equilibrium
is_nash_q(generate_game([1 0 ; 0 1], [1 0; 0 1]), [[1, 0], [1, 0]])

# Iteratively find best reply with perturbation (epsil=1/3) and limit of 10 iterations
game = generate_game(Matrix(I, 3, 3), Matrix(I, 3, 3))
s = [[1/2,1/2, 0], [1/3,1/3,1/3]]
game_history = iterate_best_reply(game, s, 1/3, 10)

# Transform game into Markov Chain
game = generate_game(Matrix(1I, 3, 3), Matrix(I, 3, 3))
s = [[1, 0, 0],[0, 1, 0]]
mc = game2markov(game, s)
simulate(mc, 9, init = 3)
simulate(mc, 9) # random initial condition

is_irreducible(mc) # can we reach any state from any other state?
communication_classes(mc)
period(mc)
is_aperiodic(mc)
stationary_distributions(mc)

plot_markov(10, mc)

# Generate random symmetric game
game = random_symmetric_2players_game(Binomial(10,1/2),2)

# Find necessary values to find symmetric nash equilibrium
x = symbols("x", real=true)
find_symmetric_nash_equilibrium_2players_game(game, [x, 1-x])

# Create equations for replicator
create_replicator_eqs(game, [x, 1-x])

# Generate graph of relations that need to hold to make a game symmetric
actions_no_vector = [2, 2, 2];
graph = create_symmetries_graph(actions_no_vector)

# Check single relation
check_equality_condition(graph,((2,2,1),1), ((2,1,2),1))

# Find all needed relations
find_all_equalities(graph)

# counting vFunction - three players coordination game
pay = reshape([1 0 0 0 0 0 0 1], (2,2,2))
g1 = generate_game(pay, pay, pay)
vFunction(g1, [[1, 0], [1, 0], [1, 0]])
vFunction(g1, [[1, 0], [1, 0], [0, 1]])

# Creates cartersian product of simplices
s1 = [[1, 0], [1, 0], [0, 1]]
v1 = cart_prod_simplices(s1)

s2 = [[1, 0], [1, 0], [1, 2, 3]]
v2 = cart_prod_simplices(s2)


@testset "Searches" begin
    @testset "Searches: Binary" begin
        @testset "binary_search - first method" begin
            sample = [0, 23, 52, 552, 555, 602, 1004]
            reversed_sample = reverse(sample)
            unsorted_sample = [124, 53, 21, 163]
            @test binary_search(sample, 52) == 3:3
            @test binary_search(sample, 602) == 6:6
            @test binary_search(reversed_sample, 52; rev=true) == 5:5
            @test binary_search(reversed_sample, 602; rev=true) == 2:2
            @test_throws ErrorException binary_search(unsorted_sample, 21)  # throws an error
        end
        @testset "binary_search - second method" begin
            # Second method used for exponential search
            arr = [1, 2, 3, 4, 13, 15, 20]
            # The next three values used are the result of the while-loop inside `exponential search`
            @test binary_search(arr, 4, 7, 4) == "Element present at index 4"
            @test binary_search(arr, 4, 7, 10) == "Element not present in array"
        end
    end

    @testset "Searches: Linear" begin
        array = [1, 3, 4, 7, 8, 11]
        linear_search(array, 3)
        linear_search(array, 8)
        linear_search(array, 12)
    end

    @testset "Searches: Exponential" begin
        arr = [1, 2, 3, 4, 13, 15, 20]
        x = 4
        n = size(arr)[1]
        l = 1
        r = n
        @test exponential_search(arr, 1) == "Element present at index 1"
        @test exponential_search(arr, x) == "Element present at index 4"
        @test exponential_search(arr, 100) == "Element not present in array"
    end

    @testset "Searches: Interpolation" begin
        arr = [1, 2, 3, 4, 13, 15, 20]
        x = 15
        n = size(arr)[1]
        l = 1
        r = n

        interpolation_search(arr, l, r, x)
    end

    @testset "Searches: Jump" begin
        arr = [1, 2, 3, 31, 4, 13]
        x = 31
        jump = 2 # optimum is sqroot(n)
        n = size(arr)[1]

        jump_search(arr, x, jump)
    end
end


# # Foundations of Bilevel Programming: Example 2.1
# This example is from the book _Foundations of Bilevel Programming_ by Stephan
# Dempe, Chapter 3.2, Page 25. [url](https://www.springer.com/gp/book/9781402006319)
# Moving the bound on x to lower level

# Model of the problem

# First level
# ```
# Min 3x + y
# s.t.
# 0 <= y <=8
# ```
# Second level
# ```
# Min -x
# s.t.
# x + y <= 8
# 4x + y >= 8
# 2x + y <= 13
# 2x - y <= 0
# x <= 5
# ```

using BilevelJuMP
using Ipopt
using JuMP
using Test

model = BilevelModel(Ipopt.Optimizer, mode = BilevelJuMP.ProductMode(1e-9))

# First we need to create all of the variables in the upper and lower problems:

@variable(Lower(model), x, start = 3.5 * 8 / 15)

#-

@variable(Upper(model), y, start = 8 / 15)

# Then we can add the objective and constraints of the upper problem:

@objective(Upper(model), Min, 3x + y)
#-

@constraint(Upper(model), y <= 8)
@constraint(Upper(model), y >= 0)

# Followed by the objective and constraints of the lower problem:

@objective(Lower(model), Min, -x)

#-
@constraint(Lower(model), x +  y <= 8)
@constraint(Lower(model), 4x +  y >= 8)
@constraint(Lower(model), 2x +  y <= 13)
@constraint(Lower(model), 2x - 7y <= 0)
@constraint(Lower(model), x <= 5)

# Now we can solve the problem and verify the solution again that reported by
# Dempe.

optimize!(model)

@test objective_value(model) ≈ 3 * (3.5 * 8 / 15) + (8 / 15) atol=1e-3
@test value(x) ≈ 3.5 * 8 / 15 atol=1e-6
@test value(y) ≈ 8 / 15 atol=1e-6

# TODO: why are these commented out?    #src
# @test dual(l2) #≈ [0] atol=atol       #src
# @test dual(l4) #≈ [0] atol=atol       #src


using Base.Profile;

function print_sudoku(m, n, sudoku)
    println(m, " ", n);
    println(sudoku);
end

function find_empty(m, n, sudoku)
    for i in 1:m*n, j in 1:m*n
        if sudoku[i, j] == 0
            return (i, j);
        end
    end
    return (m * n + 1, m * n + 1);
end

# Finds an empty cell using the most constrained variable heuristic
function smart_find_empty(m, n, sudoku, used_row, used_col, used_box)
    besti = m * n + 1;
    bestj = m * n + 1;
    best = m * n + 1;
    remaining = 0;
    for i in 1:m*n, j in 1:m*n
        if sudoku[i, j] == 0
            remaining += 1;
            box = div(i-1, m) * m + div(j-1, n) + 1;
            options = 0;
            for v in 1:m*n
                if used_row[i, v] == 0 && used_col[j, v] == 0 &&
                        used_box[box, v] == 0
                    options += 1;
                end
            end
            if options == 0
                return (0, 0);
            end
            if options < best
                besti = i;
                bestj = j;
                best = options;
            end
        end
    end
    if remaining == 0
        return (m * n + 1, m * n + 1);
    end

    return (besti, bestj)
end

# Basic backtracking using the arrays storing whether certain digits
# have already appeared to speed up checking.
function smart_solve(m, n, sudoku, used_row, used_col, used_box)

    i, j = smart_find_empty(m, n, sudoku, used_row, used_col, used_box);
    if i == 0
        return false;
    end
    if i > m * n
        return true;
    end

    bi = div(i-1, m);
    bj = div(j-1, n) + 1;
    box = bi * m + bj;

    for v in 1:m*n
        if used_row[i, v] == 0 && used_col[j, v] == 0 &&
                used_box[box, v] == 0
            sudoku[i, j] = v;
            used_row[i, v] = 1;
            used_col[j, v] = 1;
            used_box[box, v] = 1;
            if smart_solve(m, n, sudoku, used_row, used_col, used_box)
                return true;
            end
            sudoku[i, j] = 0;
            used_row[i, v] = 0;
            used_col[j, v] = 0;
            used_box[box, v] = 0;
        end
    end

    return false;
end

# Setup for the slightly smarter solve
# Initialises arrays storing whether each type of digit has occurred in
# each row, column and box and then passes this into the main function
function smart_solve(m, n, sudoku)
    used_row = zeros(Int64, m*n, m*n);
    used_col = zeros(Int64, m*n, m*n);
    used_box = zeros(Int64, m*n, m*n);

    for i in 1:m*n, j in 1:m*n
        if sudoku[i, j] > 0
            used_row[i, sudoku[i, j]] = 1;
            used_col[j, sudoku[i, j]] = 1;
        end
    end
    for i in 1:n, j in 1:m
        bi = (i-1) * m + 1;
        bj = (j-1) * n + 1;
        box = (i - 1) * m + j;
        for k in bi:bi+m-1, l in bj:bj+n-1
            if sudoku[k, l] > 0
                used_box[box, sudoku[k, l]] = 1;
            end
        end
    end

    if smart_solve(m, n, sudoku, used_row, used_col, used_box)
        return true;
    end
    return false;
end

# Dumbest backtracking solver
# Passes over all neighbours each time to check for a colliding value
function solve(m, n, sudoku)
    i, j = find_empty(m, n, sudoku);

    if i > m * n
        return true;
    end

    for v in 1:m*n

        # Determine if v is a valid option for this cell
        valid = true;
        for k in 1:m*n
            if (k != j && sudoku[i, k] == v) ||
                    (k != i && sudoku[k, j] == v)
                valid = false;
                break;
            end
        end
        # Check if v has been used in this box
        if valid
            bi = div(i-1, m) * m + 1;
            bj = div(j-1, n) * n + 1;
            for k in bi:bi+m-1, l in bj:bj+n-1
                if sudoku[k, l] == v
                    valid = false;
                    break;
                end
            end
        end

        if valid
            sudoku[i, j] = v
            if (solve(m, n, sudoku))
                return true;
            end
        end

    end

    sudoku[i, j] = 0;
    return false;
end

function try_solve(m, n, sudoku)
    s = copy(sudoku);
    smart_solve(m, n, s);
end

f = open("sudokus/2.sudoku");
m, n = [parse(Int64, s) for s = split(readline(f), " ")];

sudoku_raw = round(Int64, readdlm(f));
close(f);
sudoku = copy(sudoku_raw);

print_sudoku(m, n, sudoku);

x = smart_solve(m, n, sudoku);
println(x);

print_sudoku(m, n, sudoku);

# Actual sudoku solve
f = open("sudokus/4.sudoku");
m, n = [parse(Int64, s) for s = split(readline(f), " ")];

sudoku_raw = round(Int64, readdlm(f));
close(f);

print_sudoku(m, n, sudoku_raw);
sudoku = copy(sudoku_raw);

x = smart_solve(m, n, sudoku);
println(x);
print_sudoku(m, n, sudoku);


### A Pluto.jl notebook ###
# v0.12.20

using Markdown
using InteractiveUtils

# ╔═╡ 0b868b30-6420-11eb-18f9-1592f7542fe6
begin
    using PlutoUI
    using Plots
    using Printf
	using SpecialFunctions
end

# ╔═╡ 4c4fff40-64c8-11eb-2c45-57f4793ee997
md"_Parabolic Partial Differential Equation, version 1_"

# ╔═╡ 56d1f810-64c8-11eb-3e0d-81c0bbafdc62
md"""
## **Parabolic Partial Differential Equation**
#### **Description:**
This notebook computes the Parabolic Partial Differential Equation with 4 different discretization methods (2 explicit - 2 implicit)

		FTCS(Δt,Δy,ν)
		DuFortFrankel(Δt,Δy,ν)
		Laasonen(Δt,Δy,ν)
		CrankNicolson(Δt,Δy,ν)

If the arguments are missing, it will use the default values.

		FTCS(Δt=0.002,Δy=0.001,ν=0.000217)
		DuFortFrankel(Δt=0.002,Δy=0.001,ν=0.000217)
		Laasonen(Δt=0.002,Δy=0.001,ν=0.000217)
		CrankNicolson(Δt=0.002,Δy=0.001,ν=0.000217)

#### **Parabolic Partial Differential Equation**
In this problem, velocity profile between 2 infinite plates will be investigated. The bottom plates is moving with U₀ and the upper plate is stationary. The equation for the system can be represented by:

$$\frac{∂u}{∂t}=α\frac{∂²u}{∂x²}$$

Boundary conditions for the system:

$$u=U₀=1 \hspace{12pt} y=0$$
$$u=0    \hspace{36pt} y=h$$

where h is the distance between two plates. In this problem, it is taken as 0.04m. The partial system of equation is solved using following discritizations:

#### **Forward Time Central Space (FTCS) Method**

It is a first-order in time and second-order in space. The discritization for this partial equation:

$$u_{i}^{n+1}=u_{i}^{n}+d(u_{i+1}^{n}-2u_{i}^{n}+u_{i-1}^{n})$$

where

$$d=\frac{α(Δt)}{Δx^{2}}$$

For the stability condition:

$$\frac{α(Δt)}{Δx^{2}}=d\leqslant0.5$$

#### **DuFort-Frankel Method**

It is a second-order in time and second-order in space. The discritization for this partial equation:

$$(1+2d)u_{i}^{n+1}=(1-2d)u_{i}^{n-1}+2d(u_{i+1}^{n}+u_{i-1}^{n})$$

where

$$d=\frac{α(Δt)}{Δx^{2}}$$

For the stability condition:

$$\frac{α(Δt)}{Δx^{2}}\leqslant0.5$$

This method requires 2 sets of initial data so in this notebook, FTCS is used to obtain data set at t=Δt.

#### **Laasonen Method**

It is a first-order in time and second-order in space. The discritization for this partial equation:

$$du_{i-1}^{n+1}-(1+2d)u_{i}^{n+1}+du_{i+1}^{n+1}=-u_{i}^{n}$$

where

$$d=\frac{α(Δt)}{Δx^{2}}$$

This method is unconditionally stable.

#### **Crank Nicholson Method**

It is a second-order in time and second-order in space. The discritization for this partial equation:

$$-\frac{1}{2}du_{i-1}^{n+1}-(1+d)u_{i}^{n+1}-\frac{1}{2}du_{i+1}^{n+1}=\frac{1}{2}du_{i-1}^{n}+(1-d)u_{i}^{n}+\frac{1}{2}du_{i+1}^{n}$$

where

$$d=\frac{α(Δt)}{Δx^{2}}$$

This method is unconditionally stable.

Details of schemes:
Computational Fluid Dynamics, Volume I, Hoffmann - Chiang

Feel free to ask questions!

*Furkan Oz*
foz@okstate.edu
"""

# ╔═╡ 29683fb0-6419-11eb-05d0-2d7392bc8afa
"""
Computes the Forward Time Central Space for the Parabolic Partial Differential Equation
This notebook computes the Parabolic Partial Differential Equation
		FTCS(Δt,Δy,ν)
If the arguments are missing, it will use the default values.

		FTCS(Δt=0.002,Δy=0.001,ν=0.000217)
Furkan Oz,
foz@okstate.edu,

"""
function FTCS(Δt=0.002,Δy=0.001,ν=0.000217)
	#ν=0.000217 # viscosity [m²/s]
	h = 0.04 # Distance between 2 plate [m]
	T = 1.25 # Total time [s]
	#Δt = 0.002 # Time increment
	Nt = T/Δt+1 # Number of time step
	Nt = round(Int32,Nt)

	#Δy = 0.001 # Spacing between node points
	y = 0:Δy:h # y-coordinates
	Nx = h/Δy+1 # Number of Node points
	Nx = round(Int32,Nx)

	U₀ = 1 # Plate speed
	d = ν*Δt/(Δy)^2
	#Initializing
	u = zeros(Nx,Nt)

	# Boundary Conditions
	u[1,:] = [U₀ for i=1:Nt]
	u[Nx,:] = [0 for i=1:Nt]

	# Initial Conditions
	u[:,1] = [0 for i=1:Nx]
	u[1,1] = U₀
	for ii=1:Nt-1
		for i=2:Nx-1
			u[i,ii+1] = u[i,ii]+d*(u[i+1,ii]-2*u[i,ii]+u[i-1,ii])
		end
	end

	return u,y

end

# ╔═╡ 36be1c60-6442-11eb-204a-b37677c1acfd
"""
Computes the DuFort Frankel method for the Parabolic Partial Differential Equation
This notebook computes the Parabolic Partial Differential Equation
		DuFortFrankel(Δt,Δy,ν)
If the arguments are missing, it will use the default values.

		DuFortFrankel(Δt=0.002,Δy=0.001,ν=0.000217)
Furkan Oz,
foz@okstate.edu,

"""
function DuFortFrankel(Δt=0.002,Δy=0.001,ν=0.000217)
	#ν=0.000217 # viscosity [m²/s]
	h = 0.04 # Distance between 2 plate [m]
	T = 1.25 # Total time [s]
	#Δt = 0.002 # Time increment
	Nt = T/Δt+1 # Number of time step
	Nt = round(Int32,Nt)

	#Δy = 0.001 # Spacing between node points
	y = 0:Δy:h # y-coordinates
	Nx = h/Δy+1 # Number of Node points
	Nx = round(Int32,Nx)

	U₀ = 1 # Plate speed
	d = ν*Δt/(Δy)^2
	#Initializing
	u = zeros(Nx,Nt)

	# Boundary Conditions
	u[1,:] = [U₀ for i=1:Nt]
	u[Nx,:] = [0 for i=1:Nt]

	# Initial Conditions
	u[:,1] = [0 for i=1:Nx]
	u[1,1] = U₀
	for ii=1:1
		for i=2:Nx-1
			u[i,ii+1] = u[i,ii]+d*(u[i+1,ii]-2*u[i,ii]+u[i-1,ii])
		end
	end
	for ii=2:Nt-1
		for i=2:Nx-1
			u[i,ii+1] = ((1-2*d)*u[i,ii-1]+2*d*(u[i+1,ii]+u[i-1,ii]))/(1+2*d)
		end
	end
	return u,y

end

# ╔═╡ 899d6a70-6448-11eb-2870-692c6e93c421
"""
Computes the Laasonen method for the Parabolic Partial Differential Equation
This notebook computes the Parabolic Partial Differential Equation
		Laasonen(Δt,Δy,ν)
If the arguments are missing, it will use the default values.

		Laasonen(Δt=0.002,Δy=0.001,ν=0.000217)
Furkan Oz,
foz@okstate.edu,

"""
function Laasonen(Δt=0.002,Δy=0.001,ν=0.000217)
	#ν=0.000217 # viscosity [m²/s]
	h = 0.04 # Distance between 2 plate [m]
	T = 1.25 # Total time [s]
	#Δt = 0.002 # Time increment
	Nt = T/Δt+1 # Number of time step
	Nt = round(Int32,Nt)

	#Δy = 0.001 # Spacing between node points
	y = 0:Δy:h # y-coordinates
	Nx = h/Δy+1 # Number of Node points
	Nx = round(Int32,Nx)

	U₀ = 1 # Plate speed
	d = ν*Δt/(Δy)^2

	#Initializing
	A = zeros(Nx,Nx)
	b = zeros(Nx)
	u = zeros(Nx,Nt)

	# Boundary Conditions
	A[1,1] = 1
	b[1] = U₀
	A[Nx,Nx] = 1
	b[Nx] = 0

	# Initial Conditions
	u[:,1] = [0 for i=1:Nx]
	u[1,1] = U₀

	for ii=2:Nt-1
		# Boundary Conditions
		A[1,1] = 1
		b[1] = U₀
		A[Nx,Nx] = 1
		b[Nx] = 0
		for i=2:Nx-1
			A[i,i-1] = d
			A[i,i+0] =-1-2*d
			A[i,i+1] = d
			b[i] =-u[i,ii]
		end
		ulin = A\b
		u[1,ii+1] = b[1]
		u[Nx,ii+1] = b[Nx]
		u[2:Nx-1,ii+1] = [ulin[i] for i=2:Nx-1]
	end
	return u,y
end

# ╔═╡ 9c410110-644c-11eb-3bd9-93935242ac98
"""
Computes the Crank Nicolson method for the Parabolic Partial Differential Equation
This notebook computes the Parabolic Partial Differential Equation
		CrankNicolson(Δt,Δy,ν)
If the arguments are missing, it will use the default values.

		CrankNicolson(Δt=0.002,Δy=0.001,ν=0.000217)
Furkan Oz,
foz@okstate.edu,

"""
function CrankNicolson(Δt=0.002,Δy=0.001,ν=0.000217)
	#ν=0.000217 # viscosity [m²/s]
	h = 0.04 # Distance between 2 plate [m]
	T = 1.25 # Total time [s]
	#Δt = 0.002 # Time increment
	Nt = T/Δt+1 # Number of time step
	Nt = round(Int32,Nt)

	#Δy = 0.001 # Spacing between node points
	y = 0:Δy:h # y-coordinates
	Nx = h/Δy+1 # Number of Node points
	Nx = round(Int32,Nx)

	U₀ = 1 # Plate speed
	d = ν*Δt/(Δy)^2

	#Initializing
	A = zeros(Nx,Nx)
	b = zeros(Nx)
	u = zeros(Nx,Nt)

	# Boundary Conditions
	A[1,1] = 1
	b[1] = U₀
	A[Nx,Nx] = 1
	b[Nx] = 0

	# Initial Conditions
	u[:,1] = [0 for i=1:Nx]
	u[1,1] = U₀

	for ii=3:Nt-1
		# Boundary Conditions
		A[1,1] = 1
		b[1] = U₀
		A[Nx,Nx] = 1
		b[Nx] = 0
		for i=2:Nx-1
			A[i,i-1] =-0.5*d
			A[i,i+0] = 1+d
			A[i,i+1] =-0.5*d
			b[i] = 0.5*d*u[i+1,ii]+(1-d)*u[i,ii]+0.5*d*u[i-1,ii]
		end
		ulin = A\b
		u[1,ii+1] = b[1]
		u[Nx,ii+1] = b[Nx]
		u[2:Nx-1,ii+1] = [ulin[i] for i=2:Nx-1]
	end
	return u,y
end

# ╔═╡ 8939c23e-642a-11eb-2de6-ade5ae2f4b4d
function Analytical(t,Δy=0.001,ν=0.000217)
	#ν=0.000217 # viscosity [m²/s]
	h = 0.04 # Distance between 2 plate [m]
	y = 0:Δy:h
	Nx = h/Δy+1 # Number of Node points
	Nx = round(Int32,Nx)
	U₀ = 1 # Plate speed
	η = [y[i]/(2*sqrt(ν*t)) for i=1:Nx]
	η₁= h/(2*sqrt(ν*t))
	u = [U₀*(erfc(η[i])-erfc(2*η₁-η[i])+erfc(2*η₁+η[i])-erfc(4*η₁-η[i])+erfc(4*η₁+η[i])-erfc(8*η₁-η[i])+erfc(8*η₁+η[i])-erfc(16*η₁-η[i])) for i=1:Nx]
	return u,y
end

# ╔═╡ 70d8c3ee-641f-11eb-1d57-a943a361c397
begin
	uFTCS,yFTCS = FTCS()
	plot([uFTCS[:,1],uFTCS[:,90],uFTCS[:,180],uFTCS[:,270],uFTCS[:,360],uFTCS[:,450],uFTCS[:,540]],yFTCS,
			title = "FTCS - \\Deltat = 0.002 - \\Deltay = 0.001",
			label = ["t=0s" "t=0.18s" "t=0.36s" "t=0.54s" "t=0.72s" "t=0.90s" "t=1.08s"],
			legend = :topright,
			ylims = (0,0.04),
			xlabel = "U",
			ylabel = "y",
			linewidth = 2,
			linecolor = :black,
			markershape = :auto,
			markercolor = :red,
			)
end

# ╔═╡ 95697270-6427-11eb-02be-bf3b75fbbd33
begin
	u2,y2 = FTCS(0.005,0.001)

	plot([u2[:,1],u2[:,50],u2[:,100],u2[:,150],u2[:,200]],y2,
			title = "FTCS - \\Deltat = 0.005 - \\Deltay = 0.001",
			label = ["t=0s" "t=0.25s" "t=0.50s" "t=0.75s" "t=1.00s"],
			legend = :topright,
			ylims = (0,0.04),
			xlabel = "U",
			ylabel = "y",
			linewidth = 2,
			linecolor = :black,
			markershape = :auto,
			markercolor = :red,
			)
end

# ╔═╡ b1e5b270-643f-11eb-1a37-e3227c598177
begin
	ua1,ya1 = Analytical(0.18,0.001)
	uDF,yDF = DuFortFrankel()
	uLaasonen,yLaasonen = Laasonen()
	uCrankNicolson,yCrankNicolson = CrankNicolson()
	plot([ua1-uFTCS[:,90],ua1-uDF[:,90],ua1-uLaasonen[:,90],ua1-uCrankNicolson[:,90]],[ya1],
				title = "Error - t = 0.18 - \\Deltay = 0.001",
				label = ["FTCS" "D/F" "Laasonen" "CrankNicolson"],
				legend = :topright,
				xlims = (-0.005,0.01),
				ylims = (0,0.04),
				xlabel = "U",
				ylabel = "y",
				linewidth = 2,
				linecolor = :black,
				markershape = :auto,
				markercolor = :red,
				)
end

# ╔═╡ 8c11e680-6440-11eb-076a-0b1e169ef37a
begin
	ua2,ya2 = Analytical(1.08,0.001)
	plot([ua2-uFTCS[:,540],ua2-uDF[:,540],ua2-uLaasonen[:,540],ua2-uCrankNicolson[:,540]],[ya2],
				title = "Error - t = 1.08 -\\Deltay = 0.001",
				label = ["FTCS" "D/F" "Laasonen" "CrankNicolson"],
				legend = :topright,
				xlims = (-0.0005,0.0015),
				ylims = (0,0.04),
				xlabel = "U",
				ylabel = "y",
				linewidth = 2,
				linecolor = :black,
				markershape = :auto,
				markercolor = :red,
				)
end

# ╔═╡ 33a79720-644e-11eb-0eca-e794b9b87ccf
begin
	ua3,ya3 = Analytical(1.00,0.001)
	uLaasonen005,yLaasonen005 = Laasonen(0.005)
	uLaasonen01,yLaasonen01 = Laasonen(0.01)
	uLaasonen1,yLaasonen1 = Laasonen(0.1)
	uLaasonen2,yLaasonen2 = Laasonen(0.2)
	plot([ua3-uLaasonen005[:,200],ua3-uLaasonen01[:,100],ua3-uLaasonen1[:,10],ua3-uLaasonen2[:,5]],[ya2],
					title = "Error - Laasonen - t = 1.00 -\\Deltay = 0.001",
					label = ["\\Deltat=0.005" "\\Deltat=0.01" "\\Deltat=0.1" "\\Deltat=0.2"],
					legend = :topright,
					xlims = (0,0.2),
					ylims = (0,0.04),
					xlabel = "U",
					ylabel = "y",
					linewidth = 2,
					linecolor = :black,
					markershape = :auto,
					markercolor = :red,
					)
end

# ╔═╡ Cell order:
# ╟─0b868b30-6420-11eb-18f9-1592f7542fe6
# ╟─4c4fff40-64c8-11eb-2c45-57f4793ee997
# ╟─56d1f810-64c8-11eb-3e0d-81c0bbafdc62
# ╟─29683fb0-6419-11eb-05d0-2d7392bc8afa
# ╟─36be1c60-6442-11eb-204a-b37677c1acfd
# ╟─899d6a70-6448-11eb-2870-692c6e93c421
# ╟─9c410110-644c-11eb-3bd9-93935242ac98
# ╟─8939c23e-642a-11eb-2de6-ade5ae2f4b4d
# ╟─70d8c3ee-641f-11eb-1d57-a943a361c397
# ╟─95697270-6427-11eb-02be-bf3b75fbbd33
# ╟─b1e5b270-643f-11eb-1a37-e3227c598177
# ╟─8c11e680-6440-11eb-076a-0b1e169ef37a
# ╟─33a79720-644e-11eb-0eca-e794b9b87ccf


using Flux, Flux.Data.MNIST, Statistics, BSON, Random, StatsBase, Plots; pyplot()
using Flux: params, onehotbatch, crossentropy, update!
Random.seed!(0)

nTrain = 20000
miniBatchSize = 1000
imgs   = Flux.Data.MNIST.images()[1:nTrain]
labels = Flux.Data.MNIST.labels()[1:nTrain]

trainData = hcat([vcat(float.(imgs[i])...) for i in 1:nTrain]...)
trainLabels = labels

testImgs = Flux.Data.MNIST.images(:test)
testLabels = Flux.Data.MNIST.labels(:test)
nTest = length(testImgs)
testData = hcat([vcat(float.(testImgs[i])...) for i in 1:nTest]...)

W = randn(10,28*28)
b = randn(10)

logisticM(imgVec) = softmax(W*imgVec .+ b)
logisticMclassifier(imgVec) = argmax(logisticM(imgVec))-1
loss(x,y) = crossentropy(logisticM(x),onehotbatch(y,0:9))
opt = ADAM(0.01)

lossValue = 0.0
lossArray = []
epochNum = 0
while true
    global lossValue
    prevLossValue = lossValue
    for batch in Iterators.partition(1:nTrain,miniBatchSize)
      gs = gradient(()->loss(trainData[:,batch],trainLabels[batch]),params(W,b))
      for p in (W,b)
          update!(opt,p,gs[p])
      end
    end
    global epochNum += 1
    lossValue = loss(trainData,trainLabels)
    push!(lossArray,lossValue)
    print(".")
    abs(prevLossValue-lossValue) < 5e-4 && break
end

println("\nNumber of epochs: ", epochNum)
acccuracy = mean([logisticMclassifier(testData[:,k]) for k in 1:nTest]
                .== testLabels)
println("Accuracy: ", acccuracy)

plot(lossArray, xlabel = "Epoch", ylabel = "Cross Entropy Loss", legend = false)


using Test
using CartesianMesh
# using Revise
using CutCellDG
include("../useful_routines.jl")

cm = CutCellDG.CellMap([0.,2.],[3.,3.])
@test CutCellDG.dimension(cm) == 2
@test CutCellDG.jacobian(cm) == [1.5,0.5]
@test allapprox(cm([0.,0.]),[1.5,2.5])
@test allapprox(cm([1.,0.]),[3.,2.5])

mesh = UniformMesh([0.,0.],[1.,1.],[2,2])
@test allequal(CutCellDG.elements_per_mesh_side(mesh),[2,2])
cellmaps = CutCellDG.construct_cell_maps(mesh)
cm1 = CutCellDG.CellMap([0.,0.],[0.5,0.5])
cm2 = CutCellDG.CellMap([0.,0.5],[0.5,1.])
cm3 = CutCellDG.CellMap([0.5,0.],[1.,0.5])
cm4 = CutCellDG.CellMap([0.5,0.5],[1.,1.])
@test allequal(cellmaps,[cm1,cm2,cm3,cm4])

cm = CutCellDG.CellMap([0.,0.],[1.,1.])
@test CutCellDG.determinant_jacobian(cm) ≈ 0.25

cm = CutCellDG.CellMap([1.,1.],[5.,8.])
@test CutCellDG.determinant_jacobian(cm) ≈ 7.0

cm = CutCellDG.CellMap([2.,1.],[4.,2.])
@test allapprox(CutCellDG.inverse(cm,[3,1.5]),[0.,0.])
@test allapprox(CutCellDG.inverse(cm,[4,2.]),[1.,1.])
@test allapprox(CutCellDG.inverse(cm,[2,1]),[-1.,-1.])


#=
Root finding and ODE solver.
See Numerical Recipes 16.1 - 16.2
=#

function fdjac{T<:AbstractFloat}(f::Function, x::Array{T, 1}; epsilon=1e-6)
    #=
    Forward difference Jacobian
    =#
    n = length(x)
    y = f(x)
    jacobian = Array{Float64}(n, n)
    for i = 1:n
        temp = x[i]
        h = abs(x[i]) * epsilon
        if h == 0.0
            h = epsilon
        end # if
        x[i] = temp + h
        h = x[i] - temp
        ystep = f(x)
        x[i] = temp
        dy = (ystep - y) ./ h
        jacobian[1:end, i] = dy
    end # for
    return jacobian
end # fdjac

function newton(f, x0; epsilon=1e-2, max_count=1000)
    #=
    Multidimensional Newton-Raphson root finding.
    TODO: figure out scaled stopping critereon
    =#
    x = x0
    count = 1
    @info("newton: Starting Newton method with x0 = ", x0)
    damping = 0.001
    while true
        @info("newton: count = ", count)
        if count > max_count
            @warn("newton: Number of iterations has exceeded ", max_count)
            break
        end # if
        # approximate Jacobian
        J = fdjac(f, x)
        # invert for dx
        invJ = inv(J)
        dx = -invJ * f(x)
        frac = dx ./ x
        @info("newton: dx / x = ", frac)
        # adaptive damping
        if all(abs(frac) .< 0.1)
            damping = 1
        elseif any(abs(frac) .> 1)
            # restrict to 1% change
            damping = 0.01 / maximum(abs(frac))
        else
            # dampen to move no more than 10% change, and no more than 2
            damping = minimum([0.1 / maximum(abs(frac)), 2 * damping])
        end
        @info("newton: damping = ", damping)
        close_enough = all(abs(frac) .< epsilon)
        x = x + dx .* damping
        @info("newton: x = ", x)
        if close_enough
            @info("newton: Zero-point found at x = ", x)
            return x
        end # if
        count += 1
    end # while
end # newton

function rkstep{T<:AbstractFloat}(f::Function,
                                  y::Array{T, 1},
                                  x::T,
                                  h::T)
    k1 = f(x, y)
    k2 = f(x + h / 2, y + h / 2 .* k1)
    k3 = f(x + h / 2, y + h / 2 .* k2)
    k4 = f(x + h, y + h .* k3)
    dy = h / 6 * (k1 + 2 * k2 + 2 * k3 + k4)
    return y + dy
end # rkstep

function rk{T<:AbstractFloat}(f::Function,
                              y0::Array{T, 1},
                              xrange::Array{T, 1})
    #=
    Classic fourth-order Runge-Kutta method
    f(x, y) = dy / dx
    Integrate from x0 to xf, y0 = y(x0)
    =#
    # @debug("rk: starting Runge-Kutta, y0 = ", y0, ", h = ", h)
    x0 = xrange[1]
    xf = xrange[end]
    n = length(xrange)
    m = length(y0)
    x = Array{Float64}(n)
    y = Array{Float64}(n, m)
    x[1] = x0
    y[1, 1:end] = y0
    for (i, xstep) in enumerate(xrange[2:end])
        j = i + 1
        h = xstep - x[j - 1]
        yy = rkstep(f, y[j - 1, 1:end], x[j - 1], h)
        x[j] = xstep
        y[j, 1:end] = yy
    end # while
    return x, y
end # rk

# TODO: finish adaptive step size
function rkck{T<:AbstractFloat}(f::Array{Function, 1},
                                x0::AbstractFloat,
                                y0::Array{T, 1},
                                h0::AbstractFloat,
                                xf::AbstractFloat;
                                epsilon=1e-3)
    #=
    Cash-Karp embedded Runge-Kutta method with adaptive step size.
    f(x, y) = dy / dx
    Integrate from x0 to xf, y0 = y(x0)
    =#
    # Butcher tableau for Cash-Karp
    a = [0, 1 / 5, 3 / 10, 3 / 5, 1, 7 / 8]
    b = zeros(6, 5)
    b[2, 1] = 1/5
    b[3, 1:2] = [3 / 40, 9 / 40]
    b[4, 1:3] = [3 / 10, -9 / 10, 6 / 5]
    b[5, 1:4] = [-11 / 54, 5 / 2, -70 / 27, 35 / 27]
    b[6, :] = [1631 / 55296, 175 / 512, 575 / 13824, 44275 / 110592, 253 / 4096]
    c = [37 / 378, 0, 250 / 621, 125 / 594, 0, 512 / 1771]
    c_embedded = [2825 / 27648, 0, 18575 / 48384, 13525 / 55296, 277 / 14336, 1 / 4]

    # number of equations
    m = length(y0)

    x_list = []
    y_list = []
    h_list = []
    push!(x_list, x0)
    push!(y_list, y0)
    push!(h_list, h0)
    while x_list[end] < xf
        xx = x_list[end]
        yy = y_list[end]
        hh = h_list[end]
        k = zeros(6, m) # todo, speed up by putting column loop inside
        k[1, :] = map(g -> hh * g(xx, yy), f)
        for i = 2:6
            for j = 1:m
                k[i, j] = hh * f[j](xx + a[i] * hh, yy + dot(vec(b[i, 1:i - 1]), k[1:i - 1, j]))
            end # for
        end # for
        delta = abs(map(i -> dot(c - c_embedded, k[:, i]), 1:m))
        delta_target = epsilon * (abs(yy) + abs(vec(k[1, :])))
        if any(delta .> delta_target)
            # reject step, try a smaller step size
            h_list[end] = 0.9 * hh * minimum(delta_target ./ delta) ^ 0.25
            continue
        else
            # accept step
            push!(h_list, 0.9 * hh * minimum(delta_target ./ delta) ^ 0.2)
        end # if
        # println("accepted")
        x_next = xx + hh
        y_next = map(i -> yy[i] + dot(c, k[:, i]), 1:m)
        push!(x_list, x_next)
        push!(y_list, y_next)
    end # while
    x = Array{Float64}(x_list)
    h = Array{Float64}(h_list)
    y = Array{Float64}(length(y_list), length(y_list[1]))
    for i in 1:length(y_list)
        y[i, 1:end] = y_list[i]
    end #for
    return x, y, h
end # rkck


## This file implements Example H071 using the MathProgBase Interface.
## This is the example code from the MathProgBase test file from https://github.com/JuliaOpt/MathProgBase.jl/blob/master/test/nlp.jl , slightly modified to remove the tests.
## This interface is solver independent, but will use Ipopt as the solver here.

using MathProgBase
import MathProgBase.MathProgSolverInterface   ##In Julia, importing means that the prefix 'MathProgSolverInterface.' must be included before calling any functions within this module
using Ipopt

tic()  #Start the clock

# Here the type represents the complete instance, but it
# could also store instance data.
# The abstract type AbstractNLPEvaluator is used by solvers for accessing the objective function f and constraints g.
# Solvers may query the value, gradients, Hessian-vector products, and the Hessian of the Lagrangian.
type HS071 <: MathProgSolverInterface.AbstractNLPEvaluator
end

# hs071
# min x1 * x4 * (x1 + x2 + x3) + x3
# st  x1 * x2 * x3 * x4 >= 25
#     x1^2 + x2^2 + x3^2 + x4^2 = 40
#     1 <= x1, x2, x3, x4 <= 5
# Start at (1,5,5,1)
# End at (1.000..., 4.743..., 3.821..., 1.379...)

## Initialize must be called before any other methods. The vector requested_features lists features requested by the solver.
function MathProgSolverInterface.initialize(d::HS071, requested_features::Vector{Symbol})
    for feat in requested_features
        if !(feat in [:Grad, :Jac, :Hess])
            error("Unsupported feature $feat")
        end
    end
end

## Returns which features are available for the model HS071
MathProgSolverInterface.features_available(d::HS071) = [:Grad, :Jac, :Hess]

################################
# Define the Objective Function
################################
MathProgSolverInterface.eval_f(d::HS071, x) = x[1] * x[4] * (x[1] + x[2] + x[3]) + x[3]

###################
# Constraint set
##################
function MathProgSolverInterface.eval_g(d::HS071, g, x)
    g[1] = x[1]   * x[2]   * x[3]   * x[4]
    g[2] = x[1]^2 + x[2]^2 + x[3]^2 + x[4]^2
end

#####################################
# Gradient of the Objective Function
#####################################
function MathProgSolverInterface.eval_grad_f(d::HS071, grad_f, x)
    grad_f[1] = x[1] * x[4] + x[4] * (x[1] + x[2] + x[3])
    grad_f[2] = x[1] * x[4]
    grad_f[3] = x[1] * x[4] + 1
    grad_f[4] = x[1] * (x[1] + x[2] + x[3])
end

###################################################################################################################################
##These functions collect the sparsity structures of the Jacobian matrix and the Hessian of the Lagrangian (row first then columns)
MathProgSolverInterface.jac_structure(d::HS071) = [1,1,1,1,2,2,2,2],[1,2,3,4,1,2,3,4]
# lower triangle only
MathProgSolverInterface.hesslag_structure(d::HS071) = [1,2,2,3,3,3,4,4,4,4],[1,1,2,1,2,3,1,2,3,4]

#########################################################################################################################
##OR you could define the values of the non-zero elements. The function below does this for the Jacobian matrix and saves
## them to the matrix J. There is no need to create the matrix J.
####################################################################
function MathProgSolverInterface.eval_jac_g(d::HS071, J, x)
    # Constraint (row) 1
    J[1] = x[2]*x[3]*x[4]  # 1,1
    J[2] = x[1]*x[3]*x[4]  # 1,2
    J[3] = x[1]*x[2]*x[4]  # 1,3
    J[4] = x[1]*x[2]*x[3]  # 1,4
    # Constraint (row) 2
    J[5] = 2*x[1]  # 2,1
    J[6] = 2*x[2]  # 2,2
    J[7] = 2*x[3]  # 2,3
    J[8] = 2*x[4]  # 2,4
end

####################################################################################################
# Similarly for the Hessian -  σ = weight on objective function, μ = multipliers on the constraints
####################################################################################################
function MathProgSolverInterface.eval_hesslag(d::HS071, H, x, σ, μ)
    # Again, only lower left triangle (since it is symmetric)
    # Objective
    H[1] = σ * (2*x[4])               # 1,1
    H[2] = σ * (  x[4])               # 2,1
    H[3] = 0                          # 2,2
    H[4] = σ * (  x[4])               # 3,1
    H[5] = 0                          # 3,2
    H[6] = 0                          # 3,3
    H[7] = σ* (2*x[1] + x[2] + x[3])  # 4,1
    H[8] = σ * (  x[1])               # 4,2
    H[9] = σ * (  x[1])               # 4,3
    H[10] = 0                         # 4,4

    # First constraint
    H[2] += μ[1] * (x[3] * x[4])  # 2,1
    H[4] += μ[1] * (x[2] * x[4])  # 3,1
    H[5] += μ[1] * (x[1] * x[4])  # 3,2
    H[7] += μ[1] * (x[2] * x[3])  # 4,1
    H[8] += μ[1] * (x[1] * x[3])  # 4,2
    H[9] += μ[1] * (x[1] * x[2])  # 4,3

    # Second constraint
    H[1]  += μ[2] * 2  # 1,1
    H[3]  += μ[2] * 2  # 2,2
    H[6]  += μ[2] * 2  # 3,3
    H[10] += μ[2] * 2  # 4,4

end

# Now put this all together to define and solve the model.
# First, define a model object and the solver to use
    m = MathProgSolverInterface.model(MathProgBase.defaultNLPsolver )
# The vector of lower bounds on the variables
    l = [1,1,1,1]
# The vector of upper bounds on the variables
    u = [5,5,5,5]
# Vector of lower bounds on the constraints
    lb = [25, 40]
# Vector of constraint upper bounds
    ub = [Inf, 40]
# Load the nonlinear programming problem into the model - syntax for loadnonlinearproblem!() is
# loadnonlinearproblem!(m::AbstractMathProgModel, numVar, numConstr, l, u, lb, ub, sense, d::AbstractNLPEvaluator)
    MathProgSolverInterface.loadnonlinearproblem!(m, 4, 2, l, u, lb, ub, :Min, HS071())
## Set the initial values for the optimisation
    MathProgSolverInterface.setwarmstart!(m,[1,5,5,1])

## Solve the optimisation problem
    MathProgSolverInterface.optimize!(m)
## Let's see the termination status after solving
    stat = MathProgSolverInterface.status(m)

# Now let's see the solution.
    x = MathProgSolverInterface.getsolution(m)

toc()

println(x)


#!/usr/bin/env julia

function part1(elems)
    [(e ÷ 3 - 2) for e in elems] |> sum
end

function helper(elem)
    elem = (elem ÷ 3) - 2
    ans = elem
    while ans >= 0
        ans = (ans ÷ 3) - 2
        if ans > 0
            elem += ans;
        end # if
    end
    elem
end # function

function part2(elems)
    [helper(e) for e in elems] |> sum
end

function main()
    if isempty(ARGS)
        println(stderr, "Usage:\n\tjulia main.jl <INPUT_FILE>\nOR\n\tmain.jl <INPUT_FILE>")
        exit(1)
    else
        elems = [parse(Int64, e) for e in read(ARGS[1], String) |> split]
    end

    println("Part 1: $(part1(elems))")
    println("Part 2: $(part2(elems))")
end

main()


using DataStructures,Distributions,StatsBase,Random,Plots, LaTeXStrings; pyplot()

function simMM1Wait(lambda,mu,T)
    tNextArr = rand(Exponential(1/(lambda)))
    tNextDep = Inf
    t = tNextArr

    waitingRoom = Queue{Float64}()
    serverBusy = false
    waitTimes = Array{Float64,1}()

    while t<T
        if t == tNextArr
            if !serverBusy
                tNextDep = t + rand(Exponential(1/mu))
                serverBusy = true
                push!(waitTimes,0.0)
            else
                enqueue!(waitingRoom,t)
            end
            tNextArr = t + rand(Exponential(1/(lambda)))
        else
            if length(waitingRoom) == 0
               tNextDep = Inf
               serverBusy = false
            else
               tArr = dequeue!(waitingRoom)
               waitTime = t - tArr
               push!(waitTimes, waitTime)
               tNextDep = t + rand(Exponential(1/mu))
            end
        end
        t = min(tNextArr,tNextDep)
    end

    return waitTimes
end

Random.seed!(1)
lambda, mu = 0.8, 1.0
T = 10^3

data = simMM1Wait(lambda,mu,T)
empiricalCDF = ecdf(data)

F(x) = 1-(lambda/mu)*MathConstants.e^(-(mu-lambda)x)
xGrid = 0:0.1:20

plot(xGrid, F.(xGrid),
	c=:blue,label="Analytic CDF of waiting time")
plot!(xGrid, empiricalCDF(xGrid),
	c=:red,label="ECDF of waiting times",
	xlabel=L"x", ylabel=L"\Prob(W \leq x)", xlims=(0,20),ylims=(0,1),
	legend=:bottomright)


# VWN parameterization of the exchange correlation energy
function excVWN( n::Array{Float64,1} )
    # Constants
    X1 = 0.75*(3.0/(2.0*pi))^(2.0/3.0)
    A = 0.0310907
    x0 = -0.10498
    b = 3.72744
    c = 12.9352
    Q = sqrt(4*c-b*b)
    X0 = x0*x0 + b*x0 + c
    Npoints = size(n)[1]
    out = zeros(Float64,Npoints)
    for ip = 1:Npoints
        rs = (4.0*pi/3.0*n[ip])^(-1.0/3.0) # Added internal conversion to rs
        x = sqrt(rs)
        X = x*x + b*x + c
        out[ip] = -X1/rs + A*( log(x^2/X) + 2*b/Q*atan(Q/(2*x + b))
                  -(b*x0)/X0*( log((x-x0)*(x-x0)/X) + 2*(2*x0+b)/Q*atan(Q/(2*x + b)) ))
    end
    return out
end

# d/dn deriv of VWN parameterization of the exchange correlation energy
function excpVWN( n::Array{Float64,1} )
    # Constants
    X1 = 0.75*(3.0/(2.0*pi))^(2.0/3.0)
    A = 0.0310907
    x0 = -0.10498
    b = 3.72744
    c = 12.9352
    Q = sqrt(4.0*c - b*b)
    X0 = x0*x0 + b*x0 + c
    #
    Npoints = size(n)[1]
    out = zeros(Float64,Npoints)
    for ip = 1:Npoints
        rs = (4.0*pi/3.0*n[ip])^(-1.0/3.0) # Added internal conversion to rs
        x = sqrt(rs)
        X = x*x + b*x + c
        dx = 0.5/x # Chain rule needs dx/drho!
        out[ip] = dx*( 2.0*X1 / (rs*x)
                  + A*( 2.0/x - (2*x+b)/X - 4*b/(Q*Q + (2*x+b)*(2*x+b))
                  - (b*x0)/X0*( 2.0/(x-x0) - (2*x+b)/X - 4*(2*x0+b) /
                    (Q*Q + (2*x+b) * (2*x+b)) ) ))
        out[ip] = (-rs/(3*n[ip]))*out[ip] # Added d(rs)/dn from chain rule from rs to n conv
    end
    return out
end


#spherecomponent.jl
#Algorithm 2.3 of Random Eigenvalues by Alan Edelman

#Experiment:    Random components of a vector on a sphere
#Plot:          Histogram of a single component
#Theory:        Beta distribution

## Parameters
t = 100000      # trials
n = 12          # dimensions of sphere
dx = 0.1        # bin size

## Experiment
v = randn(n, t)
v = (v[1,:] ./ sqrt(sum(v.^2,1)))'
grid, count = hist(v, -1:dx:1)

## Theory
x = grid
y = (gamma(n-1)/(2^(n-2)*gamma((n-1)/2)^2))*(1-x.^2).^((n-3)/2)

## Plot
using Winston
p = FramedPlot()
h = Histogram(count/(t*dx), step(grid))
h.x0 = first(grid)
add(p, h)
add(p, Curve(x, y, "color", "blue", "linewidth", 2))
if isinteractive()
    Winston.display(p)
else
    file(p, "spherecomponent.png")
end


function fft(a)
    y1 = Any[]; y2 = Any[]
    n = length(a)
    if n ==1 return a end
    wn(n) = exp(-2*π*im/n)
    y_even = fft(a[1:2:end])
    y_odd = fft(a[2:2:end])
    w = 1
    for k in 1:Int(n/2)
        push!(y1, y_even[k] + w*y_odd[k])
        push!(y2, y_even[k] - w*y_odd[k])
        w = w*wn(n)
    end
    return vcat(y1,y2)
end


struct Grid{FT<:Real}
	xmin :: FT
	xmax :: FT
	nx   :: Int
	δx   :: FT
	xc   :: Array{FT}
	xf   :: Array{FT}
	ymin :: FT
	ymax :: FT
	ny   :: Int
	δy   :: FT
	yc   :: Array{FT}
	yf   :: Array{FT}
end

function GenerateGrid(
	FT = Float64;
	size, x, y,
)

	nx,ny = size
	xmin,xmax = x
	ymin,ymax = y

	xv = collect(range(xmin,xmax,length=nx+1)); δx = (xv[end] - xv[1]) / nx
	xf = xv[2:end]
	xc = (xv[1:(end-1)] .+ xv[2:end]) / 2
    yf = collect(range(ymin,ymax,length=ny+1)); δy = (yf[end] - yf[1]) / ny
	yc = (yf[1:(end-1)] .+ yf[2:end]) / 2

	return Grid{FT}(
		xmin,xmax,nx,δx,xc,xf,
		ymin,ymax,ny,δy,yc,yf
	)

end

function show(io::IO, g::Grid{FT}) where FT <: Real
    print(
		io,
		"The Grid{$FT} is defined as follows:\n",
		"                  domain : x ∈ [$(g.xmin), $(g.xmax)], y ∈ [$(g.ymin), $(g.ymin)]\n",
		"      resolution (nx,ny) : ", (g.nx,g.ny), '\n',
		"    grid spacing (δx,δy) : ", (g.δx,g.δy), '\n'
	)
end


using Graphs
using SimpleGraphs
using PyPlot

function plot_deg_hist(g::GenericGraph, out_fldr::ASCIIString,fname::ASCIIString,ac::Float64)
    # degree histogram
    deglst = zeros(num_vertices(g))
    for i in vertices(g)
        deglst[i] = out_degree(i,g)
    end
    plt_deg_hist(deglst,out_fldr,fname,ac)
end

function plot_deg_hist(g::SimpleGraphs.SimpleGraph, out_fldr::ASCIIString,fname::ASCIIString,ac::Float64)
    deglst = zeros(NV(g))
    v_lst = vlist(g)
    for i=1:NV(g)
        deglst[i] = deg(g,v_lst[i])
    end
    # deglst = deg_hist(g)
    plt_deg_hist(deglst,out_fldr,fname,ac)
end

function plot_deg_list(g::SimpleGraphs.SimpleGraph, out_fldr::ASCIIString,fname::ASCIIString,ac::Float64)
    # degree histogram
    deglst = deg(g)
    plt_deg_hist(deglst,out_fldr,fname,ac)
end

function plt_deg_hist(deglst::Array{},out_fldr::ASCIIString,fname::ASCIIString,ac::Float64)
    mu = mean(deglst)
    sig = std(deglst)
    PyPlot.clf()
    plt[:hist](deglst,maximum(deglst),normed=1,alpha=0.75,facecolor="green")
    PyPlot.axvline(mu,color="k",ls="dashed")
    PyPlot.axvline(mu-sig,color="r",ls="dotted")
    PyPlot.axvline(mu+sig,color="r",ls="dotted")
    title("Degree Distribution")
    xlabel("Degree")
    ylabel("Proportion")
    annotate(@sprintf("a(G)=%0.3f",ac),xy=(0.7,0.8),fontsize=20,xycoords="axes fraction")
    annotate(@sprintf("mu=%0.3f",mu),xy=(0.7,0.7),fontsize=20,xycoords="axes fraction")
    annotate(@sprintf("std=%0.3f",sig),xy=(0.7,0.6),fontsize=20,xycoords="axes fraction")
    fname = joinpath(out_fldr,fname)
    fname = string(fname,".pdf")
    savefig(fname)
end

function plot_c_hist(c_hist::Array{},out_fldr::ASCIIString,fname::ASCIIString)
    # connectivity history
    PyPlot.clf()
    PyPlot.plot(collect(1:length(c_hist)),c_hist)
    title("Connectivity History")
    xlabel("Iteration")
    ylabel("Algebraic Connectivity")
    fname = joinpath(out_fldr,fname)
    fname = string(fname,".pdf")
    savefig(fname)
    clf()
end

function plot_c_box(c_vals::Array{},n::Int,c_opt::Float64,out_fldr::ASCIIString,fname::ASCIIString)
    # connectivity distribution vs. optimal
    PyPlot.clf()
    boxplot(c_vals)
    PyPlot.boxplot(c_vals)
    PyPlot.axhline(c_opt,color="r",ls="dotted")
    title("Optimization Performance")
    # xlabel("n")
    xticks([1,2],["Opt","Random"])
    ylabel("Algebraic Connectivity")
    fname = joinpath(out_fldr,fname)
    fname = string(fname,".pdf")
    savefig(fname)
end


#overlap funcs {{{
## for binary states {{{
export overlap, overlapm

function overlap{T<:Real}(x::AbstractVector{T},y::AbstractVector{T})
    return dot(x, y)/length(x)
end

function overlap{T<:Real}(x::AbstractVector{Complex{T}},y::AbstractVector{Complex{T}})
    rsum = dot(real(x), real(y))
    isum = dot(imag(x), imag(y))
    return (rsum+isum)/length(x)*0.5
end

function overlap{T<:Real}(x::AbstractVector{Quaternion{T}},y::AbstractVector{Quaternion{T}})
    rsum = dot(real(x), real(y))
    isum = dot(imagi(x), imagi(y))
    jsum = dot(imagj(x), imagj(y))
    ksum = dot(imagk(x), imagk(y))
    return (rsum+isum+jsum+ksum)/length(x)*0.25
end
## }}}

## for multi states {{{
function overlapm{T<:Real}(x::AbstractVector{T},y::AbstractVector{T})
    return sum(x .== y)/length(x)
end

function overlapm{T<:Real}(x::AbstractVector{Complex{T}},y::AbstractVector{Complex{T}})
    rsum = sum(real(x).== real(y))
    isum = sum(imag(x).== imag(y))
    return (rsum+isum)/length(x)*0.5
end

function overlapm{T<:Real}(x::AbstractVector{Quaternion{T}},y::AbstractVector{Quaternion{T}})
    rsum = sum(real(x).== real(y))
    isum = sum(imagi(x).== imagi(y))
    jsum = sum(imagj(x).== imagj(y))
    ksum = sum(imagk(x).== imagk(y))
    return (rsum+isum+jsum+ksum)/length(x)*0.25
end
## }}}


# https://adventofcode.com/2020/day/1

# find the two entries that sum to 2020 and then multiply those two numbers together

function part1(list_of_numbers, total=2020)
    for (i, x) in enumerate(list_of_numbers)
        y = total - x
        if y in list_of_numbers[(i + 1):end]
            return x * y
        end
    end
end

example = [1721, 979, 366, 299, 675, 1456]
@assert part1(example) == 514579

# Using the above example again, the three entries that sum to 2020 are
# 979, 366, and 675. Multiplying them together produces the answer, 241861950.

function part2(list_of_numbers)
    for (i, x) in enumerate(list_of_numbers)
        partial = 2020 - x
        tmp = filter(y -> y <= partial, list_of_numbers[(i + 1):end])
        y = part1(tmp, partial)
        if !isnothing(y)
            return x * y
        end
    end
end

@assert part2(example) == 241861950

function read_array(string::AbstractString)::Vector{Int}
    return parse.(Int, split(string, '\n', keepempty=false))
end

test = read_array(read("data/day-01.txt", String))
println("Part 1: $(result1 = part1(test))")
println("Part 2: $(result2 = part2(test))")

@assert result1 == 326211
@assert result2 == 131347190