# utilitaries
using LinearAlgebra

include("quadrature.jl")
# include("connectique.jl")

@inline function mod_wrap(x, a)
    #= return the index x modulo a =#
    return mod(x-1, a) + 1
end

function polygonArea(vtx)
    area = 0.
    N = size(vtx, 2)
    for i=1:N
        j = mod_wrap(i+1, N)
        area += vtx[1, i]*vtx[2, j] - vtx[2, i]*vtx[1, j]
    end
    return 0.5*abs(area)
end

function polygonDiameter(vtx)
    diameter = 0.
    N = size(vtx, 2)
    for i=1:(N-1)
        for j=(i+1):N
            diameter = max(diameter, norm(vtx[:, i] - vtx[:, j]))
        end
    end
    return diameter
end

function polygonCentroid(vtx)
    N        = size(vtx, 2)
    centroid = zeros(Float64, 2)
    #
    den = 0.
    for i=1:N
        j = mod_wrap(i+1, N)
        w = (vtx[1, i]*vtx[2, j] - vtx[1, j]*vtx[2, i])
        centroid[1] += (vtx[1, i] + vtx[1, j])*w
        centroid[2] += (vtx[2, i] + vtx[2, j])*w
        den += w
    end
    #
    return centroid/(3 * den)
end

# INTEGRATION OF MULTIPLE FUNCTIONS OVER A POLYGON
function integratePolygon(pverts::Array{Float64,2}, xE::Array{Float64,1},
    ref::Quadrature2d, fs::Array{Function, 1})
    #=
    Integration of all functions in fs over a polygon by subdividing it
    into triangles
    =#
    n   = size(pverts, 2)
    nf  = length(fs)
    res = zeros(Float64, nf)
    for i=1:n
        ip = mod_wrap(i+1, n)
        @inbounds quadT = quadGL2d(ref, [pverts[1,i] pverts[1,ip] xE[1];
        pverts[2,i] pverts[2,ip] xE[2]])
        for f = 1:nf
            @inbounds res[f] += integrate(quadT, fs[f])
        end
    end
    return  res
end

function integratePolygon(pverts::Array{Float64,2}, xE::Array{Float64,1},
    ref::Quadrature2d, fs::Function)
    #= Same as before, but for a single function fs =#
    n   = size(pverts, 2)
    res = 0.
    for i=1:n
        ip = mod_wrap(i+1, n)
        @inbounds quadT = quadGL2d(ref, [pverts[1,i] pverts[1,ip] xE[1];
        pverts[2,i] pverts[2,ip] xE[2]])
        res += integrate(quadT, fs)
    end
    return  res
end


using Statistics
using BenchmarkTools
using LinearAlgebra

# Looking into pointers from http://fa.bianp.net/blog/2019/evaluate_logistic/
#

σ(x) = 1 / (1 + exp(-x))

## Order 0
function f_naive(x, A, b)
    z = A*x
    tmp = σ.(z)
    @. tmp = - b * log(tmp) - (1 - b) * log(1 - tmp)
    return sum(tmp)/length(tmp)
end

## blog logistic
function logsig_naive(t)
    return log(1 / (1 + exp(-t)))
end

function logsig_log1pexp(t)
    if t < -33.3
        return t
    elseif t <= -18
        return t - exp(t)
    elseif t <= 37
        return -log1p(exp(-t))
    else
        return -exp(-t)
    end
end

function f_logsumexp(x, A, b, logsig)
    m = size(A, 1)

    Ax = A * x
    fval = 0.0
    @inbounds @simd for i in 1:m
        fval += -logsig(b[i] * Ax[i])
    end
    fval /= m

    return fval
end

### Order 1
function ∇f_current(x, A, b)
    m = size(A, 1)

    σbAx = σ.(-b .* (A * x))
    res = transpose(A) * (σbAx .* -b)

    res /= m
    return res
end

function expit_b(x, b)
    out = zeros(size(x))

    @inbounds for (i, xi) in enumerate(x)
        if xi < 0
            out[i] = ((1 - b[i]) * exp(xi) - b[i]) / (1 + exp(xi))
        else
            out[i] = ((1 - b[i]) - b[i] * exp(-xi)) / (1 + exp(-xi))
        end
    end
    return out
end

function ∇f_prop(x, A, b)
    """Computes the gradient of the logistic loss.

    Parameters
    ----------
    x: array-like, shape (n_features,)
        Coefficients

    A: array-like, shape (n_samples, n_features)
        Data matrix

    b: array-like, shape (n_samples,)
        Labels

    Returns
    -------
    grad: array-like, shape (n_features,)
    """
    z = A*x
    s = expit_b(z, b)
    return A'*s / size(A, 1)
end

function main()
    A = [1 1;]
    b = [1]
    x = [20, 20]

    # mode = [:fvals]
    mode = [:∇f]

    if :fvals in mode
        @show f_naive(x, A, b)
        @show f_logsumexp(x, A, b, logsig_naive)
        @show f_logsumexp(x, A, b, logsig_log1pexp)
    elseif :∇f in mode
        @show ∇f_current(x, A, b)
        @show ∇f_prop(x, A, b)
    end

    println("")
    println("")

    println("Larger test")
    n, m = 500, 400
    A = rand(m, n)
    x = rand(n)
    b = rand(m)

    if :fvals in mode
        @show f_naive(x, A, b)
        @show f_logsumexp(x, A, b, logsig_naive)
        @show f_logsumexp(x, A, b, logsig_log1pexp)

        @btime f_naive($x, $A, $b)
        @btime f_logsumexp($x, $A, $b, $logsig_naive)
        @btime f_logsumexp($x, $A, $b, $logsig_log1pexp)
    elseif :∇f in mode
        ∇f_cu = ∇f_current(x, A, b)
        ∇f_pr = ∇f_prop(x, A, b)

        @show norm(∇f_cu - ∇f_pr)

        @btime ∇f_current($x, $A, $b)
        @btime ∇f_prop($x, $A, $b)
    end

    return
end

main()


using FastGaussQuadrature

get_nodes(::Val{:gauss_legendre}, n, T=Float64) = convert(Array{T}, gausslegendre(n)[1])
get_nodes(::Val{:lobatto_legendre}, n, T=Float64) = convert(Array{T}, gausslobatto(n)[1])

function get_nodes(::Val{:gauss_chebyshev}, n, T=Float64; k=1)
    x = zeros(T, n)
    if k == 1
        for i in eachindex(x)
            x[i] = sin( π*(n-2i+1) / (2n) )
        end
        return x[end:-1:1]
    elseif k == 2
        for i in eachindex(x)
            x[i] = cos( π*i / (n-1) )
        end
        return x
    end
end

function get_nodes(::Val{:lobatto_chebyshev}, n, T=Float64)
    x = zeros(T, n)
    for i in eachindex(x)
        x[i] = cos( (i-1)*π / (n-1) )
    end
    return x[end:-1:1]
end


#=
Accurate simple zeros of polynomials in floating point arithmetic
Stef Graillat
Computers and Mathematics with Applications 56 (2008) 1114–1120

   poly has N+1 coefficients a_0,a_1,..,a_N for x^0,x^1,..,x^N

   poly = polynomial(coeffs[a_0,a_1,..,a_N])
   poly(x) = sum[i=0:1:N]( a_i * x^i )          this the the classical Horner scheme

   function classicalHorner{R<:Real}(p::Poly,x::R)
      n = length(p.a)
      s = zeros(R, n)       # one-based indexing
      s[:] = p.a[:]
      for i in (n-1):(-1):1
          s[i] += s[i+1]*x
      end
      s[1]
    end

Algorithms for Accurate, Validated and Fast Polynomial Evaluation
Stef Graillat, Philippe Langlois and Nicolas Louvet
Japan J. Indust. Appl. Math., 26 (2009), 191–214

eftHorner(p_n,x) computes
(i)  the floating point evaluation Horner(p_n,x)
(ii) two polynomials pa, pb, of degree n-1
such that
  Horner(p_n,x), pa, pb = eftHorner(p_n,x)
  p_n(x) == Horner(p_n,x) + (pa + pb)(x)
so
  eftHorner is an errorfree transformation for polynomial evaluation with the Horner algorithm
     (unless there is underflow)
=#

function eftHorner{T<:AbstractFloat}(p::Poly{T},x::T)
   deg = degree(p); n=deg+1
   cofs = coeffs(p)
   pa = zeros(T,n)
   pb = zeros(T,n)
   r = cofs[n]
   for i in deg:(-1):1
      t, pa[i] = eftMul(r, x)
      r, pb[i] = eftAdd(t, cofs[i])
    end
    r, pa, pb
end

# expects length(a) == length(b), length(a)>0
function sumHorner{T<:AbstractFloat}(a::Vector{T}, b::Vector{T}, x::T)
   n = length(a)
   if ((n == 0)|(n != length(b)))
      throw(ErrorException("vectors must be of the same nonzero length"))
   end
   r = a[n]+b[n]
   for i in (n-1): -1: 1
     r = r * x + (a[i]+b[i])
   end
   r
end

function csdHornerA{T<:AbstractFloat}(p::Poly{T}, x::T)
    r, pa, pb = eftHorner(p,x)
    c = sumHorner(pa,pb,x)
    r + c
end

# as accurate as if computed at twice the working precision and then rounded to working precision
function csdHorner{T<:AbstractFloat}(p::Poly{T}, x::T)
    r, pa, pb = eftHorner(p,x)
    pab = Poly(pa + pb)
    c = polyval(pab, x)
    r + c
end


# A palindromic number reads the same both ways.
# The largest palindrome made from the product
# of two 2-digit numbers is 9009 = 91 × 99.
# Find the largest palindrome made from the product of two 3-digit numbers.

function is_palindrome(n)
    s = string(n)
    return s == reverse(s)
end

@assert is_palindrome(2)
@assert is_palindrome(11)
@assert is_palindrome(9009)
@assert is_palindrome(90109)
@assert is_palindrome(90209)

function find_next_palindrome_decreasing(n)
    while !is_palindrome(n)
        n -= 1
    end
    return n
end

@assert find_next_palindrome_decreasing(10000) == 9999
@assert find_next_palindrome_decreasing(9998) == 9889

function is_divisible_by_digits(n, digits)
    # Checks if a number is divisible by two x digits number
    max = (10 ^ digits) - 1
    min = minimum([max, trunc(Int, n / (10 ^ digits) + 1)])

    for i in range(max, -1, max-min)
        if n % i == 0 && (n / i) <= max
            return true
        end
    end
    return false
end

@assert is_divisible_by_digits(9999, 2) == false
@assert is_divisible_by_digits(9779, 2) == false
@assert is_divisible_by_digits(9009, 2) == true

function compute_palindrome_by_factors(n, digit)
    while !is_palindrome(n) || !is_divisible_by_digits(n, digit)
        n = find_next_palindrome_decreasing(n-1)
    end
    return n
end
@assert compute_palindrome_by_factors(99*99,2) == 9009
println(compute_palindrome_by_factors(999*999, 3))


# Make the Convex.jl module available
using Convex, SCS, ECOS
using LinearAlgebra

# Generate random problem data
m = 4;  n = 5
A = randn(m, n); b = randn(m, 1)

# Create a (column vector) variable of size n x 1.
x = Convex.Variable(n)

# The problem is to minimize ||Ax - b||^2 subject to x >= 0
# This can be done by: minimize(objective, constraints)
problem = minimize(sumsquares(A * x - b), [x >= 0])

# Solve the problem by calling solve!
solve!(problem, ECOS.Optimizer; silent_solver = true)

# Check the status of the problem
problem.status # :Optimal, :Infeasible, :Unbounded etc.

# Get the optimum value
problem.optval

# minimum distance spheres
pa = Variable(3)
pb = Variable(3)
xa = [0.0; 0.0; 0.0]
xb = [1.0; 0.0; 0.0]
ra = 0.1
rb = 0.1

problem = minimize(norm(pa - pb), [norm(pa - xa) <= ra, norm(pb - xb) <= rb])

# Solve the problem by calling solve!
solve!(problem, ECOS.Optimizer; silent_solver = true)

# Check the status of the problem
problem.status # :Optimal, :Infeasible, :Unbounded etc.

# Get the optimum value
problem.optval
pa
pb

# minimum distance ellipsoids
pa = Variable(3)
pb = Variable(3)
xa = [0.0; 0.0; 0.0]
xb = [1.0; 0.0; 0.0]
ax = 0.3
ay = 0.1
az = 0.1
bx = 0.1
by = 0.2
bz = 0.1

problem = minimize(norm(pa - pb),
    [
        quadform(pa - xa, Array(Diagonal([1.0 / ax^2, 1.0 / ay^2, 1.0 / az^2]))) <= 1.0,
        quadform(pb - xb, Array(Diagonal([1.0 / bx^2, 1.0 / by^2, 1.0 / bz^2]))) <= 1.0,
    ])

# Solve the problem by calling solve!
solve!(problem, ECOS.Optimizer; silent_solver = true)

# Check the status of the problem
problem.status # :Optimal, :Infeasible, :Unbounded etc.

# Get the optimum value
problem.optval
pa
pb

### minimum distance cylinders
pa = Variable(3)
pb = Variable(3)
sa = Variable(3)
sb = Variable(3)
xa = [-1.0; 0.0; 0.0]
xb = [1.0; 0.0; 0.0]
ra = 0.1
ha = 1.0
rb = 0.1
hb = 1.0

problem = minimize(norm(pa - pb),
    [
        sa == pa - xa,
        sb == pb - xb,
        norm(sa[1:2]) <= ra,
        sa[3] <= 0.5 * ha + sqrt(ra * ra - dot(sa[1:2], sa[1:2])),
        -0.5 * ha - sqrt(ra * ra - dot(sa[1:2], sa[1:2])) <= sa[3],
        norm(sb[1:2]) <= rb,
        sb[3] <= 0.5 * hb + sqrt(rb * rb - dot(sb[1:2], sb[1:2])),
        -0.5 * hb - sqrt(rb * rb - dot(sb[1:2], sb[1:2])) <= sb[3],
    ])

pa.value = [-1.0; 0.0; 0.0]
pb.value = [1.0; 0.0; 0.0]
# Solve the problem by calling solve!
solve!(problem, ECOS.Optimizer; silent_solver = false)

# Check the status of the problem
problem.status # :Optimal, :Infeasible, :Unbounded etc.

# Get the optimum value
problem.optval
pa
pb

## line segment
za = Convex.Variable(3)
zb = Convex.Variable(3)
ta = Convex.Variable(1)
tb = Convex.Variable(1)

xa = [-1.0; 0.0; 0.0]
qa = Quaternion(RotY(-0.25 * π) * RotX(0.0 * π))
qa = [qa.w; qa.x; qa.y; qa.z]

xb = [1.0; 0.0; 0.0]
qb = Quaternion(RotZ(0.0 * π) * RotY(0.0 * π) * RotX(0.0))
qb = [qb.w; qb.x; qb.y; qb.z]
ra = 0.1
ha = 1.0
rb = 0.1
hb = 1.0

p1a = xa + rotation_matrix(qa) * [0.0; 0.0; 0.5 * ha]
p2a = xa + rotation_matrix(qa) * [0.0; 0.0; -0.5 * ha]

p1b = xb + rotation_matrix(qb) * [0.0; 0.0;  0.5 * hb]
p2b = xb + rotation_matrix(qb) * [0.0; 0.0; -0.5 * hb]

problem = minimize(norm(za - zb),
    [
        za == ta * p1a + (1.0 - ta) * p2a,
        zb == tb * p1b + (1.0 - tb) * p2b,
        ta >= 0.0,
        ta <= 1.0,
        tb >= 0.0,
        tb <= 1.0,
    ])

# Solve the problem by calling solve!
solve!(problem, ECOS.Optimizer; silent_solver = false)

# Check the status of the problem
problem.status # :Optimal, :Infeasible, :Unbounded etc.

# Get the optimum value
problem.optval
za
zb


using NumericalIntegration
using LinearAlgebra
using FFTW

#=
This function computes the Fourier modes either integrating "by hand" or using FFTW. We split the comlex modes, associated to the complex exponentials into a and b, associated to the cos and sin functions by a(kx,ky) = 2*Re(f(kx,ky)) and b(kx,ky) = -2*Im(f(kx,ky)).
=#

function Fourier_modes_2D(e :: Array{T,2}, x :: Array{T,1}, y :: Array{T,1}, order_x :: Integer, order_y :: Integer, integration_method :: String) where {T<:Real}
    Lx, Ly = x[end]-x[1], y[end]-y[1]
    xmid, ymid = (x[end]+x[1])/2, (y[end]+y[1])/2
    nx, ny = length(x),length(y)

    if order_x > Integer(floor(nx)) || order_y > Integer(floor(ny))
        print("\n Error, requested order is higher than the number of grid points\n")
        nothing
    end

    if integration_method == "trapezoidal"

        a, b = zeros(order_x+1,order_y+1), zeros(order_x+1,order_y+1)
        xx, yy = zeros(nx,ny), zeros(nx,ny)
        #We create the 2D grid
        for i in 1:ny
            xx[:,i] = x
        end
        for i in 1:nx
            yy[i,:] = y
        end

        for my in 0:order_y
            for mx in 0:order_x
                integral_cos, integral_sin = zeros(nx), zeros(nx)
                integrand_cos = 2/(Lx*Ly).*e.*cos.(2*π*(mx/Lx.*(xx.-xmid).+my/Ly.*(yy.-ymid)))
                integrand_sin = 2/(Lx*Ly).*e.*sin.(2*π*(mx/Lx.*(xx.-xmid).+my/Ly.*(yy.-ymid)))
                for i in 1:nx
                    integral_cos[i] = integrate(y,integrand_cos[i,:], Trapezoidal())
                    integral_sin[i] = integrate(y,integrand_sin[i,:], Trapezoidal())
                end
                a[mx+1,my+1] = integrate(x,integral_cos, Trapezoidal())
                b[mx+1,my+1] = integrate(x,integral_sin, Trapezoidal())
            end
        end

    elseif integration_method == "FFTW"

        plan = plan_rfft(e)
        fft_z = (1/(nx*ny).*(plan*e)[1:order_x+1,1:order_y+1])
        a = 2 .*real(fft_z)
        b = -2 .*imag(fft_z)

    else
        print("\n No such integration method\n")
        nothing
    end
    return a, b
end


export gaussian, xavier, xavier_uniform, xavier_normal, bilinear
using Random: randn, rand

"""

    gaussian(a...; mean=0.0, std=0.01)

Return a Gaussian array with a given mean and standard deviation.  The
`a` arguments are passed to `randn`.

"""
function gaussian(a...; mean=0.0, std=0.01)
    r = randn(a...)
    T = eltype(r)
    r .* T(std) .+ T(mean)
end

"""

    xavier_uniform(a...; gain=1)
    xavier(a...; gain=1)

Return uniform random weights in the range `± gain * sqrt(6 / (fanin + fanout))`.  The `a`
arguments are passed to `rand` to specify type and dimensions.  See ([Glorot and Bengio
2010](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf)) or the [PyTorch
docs](https://pytorch.org/docs/stable/nn.init.html#torch.nn.init.xavier_uniform_) for a
description.  The function implements equation (16) of the referenced paper. Also known as
Glorot initialization. The function `xavier` is an alias for `xavier_uniform`. See also
`xavier_normal`.

"""
xavier, xavier_uniform

function xavier_uniform(a...; gain=1)
    w = rand(a...)
    if ndims(w) == 1
        fanout = 1
        fanin = length(w)
    elseif ndims(w) == 2
        fanout = size(w,1)
        fanin = size(w,2)
    else
        # if a is (3,3,16,8), then there are 16 input channels and 8 output channels
        # fanin = 3*3*16 = (3*3*16*8) ÷ 8
        # fanout = 3*3*8 = (3*3*16*8) ÷ 16
        fanin = div(length(w),  a[end])
        fanout = div(length(w), a[end-1])
    end
    s = convert(eltype(w), gain*sqrt(6 / (fanin + fanout)))
    return 2s .* w .- s
end

const xavier = xavier_uniform

"""

    xavier_normal(a...; gain=1)

Return normal distributed random weights with mean 0 and std `gain * sqrt(2 / (fanin +
fanout))`.  The `a` arguments are passed to `rand`.  See ([Glorot and Bengio
2010](http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf)) and [PyTorch
docs](https://pytorch.org/docs/stable/nn.init.html#torch.nn.init.xavier_normal_) for a
description. Also known as Glorot initialization. See also `xavier_uniform`.

"""
function xavier_normal(a...; gain=1)
    w = randn(a...)
    if ndims(w) == 1
        fanout = 1
        fanin = length(w)
    elseif ndims(w) == 2
        fanout = size(w,1)
        fanin = size(w,2)
    else
        # if a is (3,3,16,8), then there are 16 input channels and 8 output channels
        # fanin = 3*3*16 = (3*3*16*8) ÷ 8
        # fanout = 3*3*8 = (3*3*16*8) ÷ 16
        fanin = div(length(w),  a[end])
        fanout = div(length(w), a[end-1])
    end
    s = convert(eltype(w), gain*sqrt(2 / (fanin + fanout)))
    return s .* w
end

"""

Bilinear interpolation filter weights; used for initializing deconvolution layers.

Adapted from https://github.com/shelhamer/fcn.berkeleyvision.org/blob/master/surgery.py#L33

Arguments:

`T` : Data Type

`fw`: Width upscale factor

`fh`: Height upscale factor

`IN`: Number of input filters

`ON`: Number of output filters

Example usage:

w = bilinear(Float32,2,2,128,128)

"""
function bilinear(T,fw,fh,IN,ON)
    @assert fw == fh "Filter must be square"
    @assert IN == ON "Number of input and output filters must be equal"
    f=fw; N=IN;

    sz = 2*f-f%2
    center = f-0.5
    if sz%2 == 1
        center = f-1
    end
    r = range(0,stop=sz-1,length=sz)
    c = range(0,stop=sz-1,length=sz)'

    kernel = (1 .- abs.(r .- center) ./ f) .* (1 .- abs.(c .- center) ./ f)
    w = zeros(T,sz,sz,N,N);
    for i=1:N
        w[:,:,i,i] = kernel
    end
    return w
end


using SimpleGraphs, SimpleGraphAlgorithms

const DEFAULT_FILE_NAME = "tree_codes.jl"

# initialize the table of unlabeled trees

"""
    init_trees_table()
Create a new table of distinct trees on 1 and 2 vertices.
"""
function init_trees_table()
    TT = Dict{Int,Vector{SimpleGraph{Int}}}()

    TT[1] = [IntGraph(1)]
    T = IntGraph(2)
    add!(T, 1, 2)
    TT[2] = [T]

    return TT
end

"""
    check_in(G,S)
See if the set `S` contains a graph isomorphic to `G`. Return `true` if so.
"""
function check_in(G::SimpleGraph{Int}, S::Set{SimpleGraph{Int}})
    if isempty(S)
        return false
    end
    for H ∈ S
        if uhash(G) == uhash(H) #is_iso(G, H)
            return true
        end
    end
    return false
end

"""
    extend_trees_table!(TT)
Given a table of distinct trees up to size `n`, extend that table to include
all distinct trees of size `n+1`.
"""
function extend_trees_table!(TT::Dict{Int,Vector{SimpleGraph{Int}}})::Nothing
    n = maximum(keys(TT))
    outset = Set{SimpleGraph{Int}}()  # set of trees with n+1 vertices
    for T ∈ TT[n]
        for w = 1:n
            X = deepcopy(T)
            add!(X, w, n + 1)
            if check_in(X, outset)
                continue
            end
            push!(outset, X)
        end
    end

    TT[n+1] = collect(outset)

    @info "Added $(length(TT[n+1])) new trees with $(n+1) vertices"
    nothing
end

function build_trees_table(nmax::Int)
    TT = init_trees_table()
    while maximum(keys(TT)) < nmax
        @info "Adding trees of size $(maximum(keys(TT))+1)"
        extend_trees_table!(TT)
    end
    return TT
end

"""
    create_codes_table(TT)
Given a table of distinct trees, convert that into a table of Prufer codes.
This is used by `save_trees_table` and not useful to be called directly.
"""
function create_codes_table(TT::Dict{Int64,Vector{SimpleGraph{Int}}})
    codes = Dict{Int64,Vector{Vector{Int}}}()
    codes[2] = [Int[]]
    for n = 3:maximum(keys(TT))
        codes[n] = prufer_code.(TT[n])
    end
    return codes
end

"""
    save_tree_table(TT, filename)
Save a trees table into a file specified by `filename`.
If the file name is omitted, use `codes.jl`.
"""
function save_trees_table(
    TT::Dict{Int64,Vector{SimpleGraph{Int}}},
    filename::String = DEFAULT_FILE_NAME,
)
    outfile = open(filename, "w")
    codes = create_codes_table(TT)
    println(outfile, "codes = Dict{Int,Vector{Vector{Int}}}()")
    for n = 2:maximum(keys(TT))
        print(outfile, "codes[$n] = ")
        println(outfile, codes[n])
    end
    close(outfile)
    nothing
end

"""
    load_trees_table(filename)
Create a table of distinct trees by reading in a file that has been
precomputed (and presumably saved using `save_tree_table`). If `filename`
is omitted, use `codes.jl`.
"""
function load_trees_table(filename::String = DEFAULT_FILE_NAME)
    include(filename)
    TT = init_trees_table()
    nmax = maximum(keys(codes))

    for n = 3:nmax
        TT[n] = prufer_restore.(codes[n])
    end
    @info "Read in table of distinct trees up to $nmax vertices"
    return TT

end


## PCA vs LDA
## Height Weight data
# Grab data from https://www.kaggle.com/mustafaali96/weight-height
# https://opendata.stackexchange.com/questions/7793/age-weight-and-height-dataset
# https://www.reddit.com/r/datasets/comments/1dii3t/data_sets_for_heights_and_weights/

# https://vincentarelbundock.github.io/Rdatasets/datasets.html

# a related tutorial
# http://goelhardik.github.io/2016/10/04/fishers-lda/

##
using Plots
##
# https://vincentarelbundock.github.io/Rdatasets/csv/carData/Davis.csv
using CSV
df = CSV.read(download("https://vincentarelbundock.github.io/Rdatasets/csv/carData/Davis.csv"))
X = Float64.([df[:weight] df[:height]])

##
g = df[:sex]

##
scatter(X[:,1],X[:,2])
xlabel!("Weight")
ylabel!("Height")

## Filter the data
goodpts = X[:,1] .<= 115
Xf = X[goodpts,:]
gf = g[goodpts]

##
theme(:dark)
scatter(Xf[:,1],Xf[:,2],color=2)
xlabel!("Weight")
ylabel!("Height")

## Save a picture
pyplot()
scatter(Xf[:,1],Xf[:,2],color=2,size=(300,300), legend=false)
xlabel!("Weight")
ylabel!("Height")
savefig("weight-height-1.pdf")

## Partition by classes
s1 = gf .== "F"
s2 = gf .== "M"
X1 = Xf[s1,:]
X2 = Xf[s2,:]
gr()
scatter(X1[:,1],X1[:,2])
scatter!(X2[:,1],X2[:,2])

## Save a picture
pyplot()
scatter(X1[:,1],X1[:,2],size=(300,300), legend=false)
scatter!(X2[:,1],X2[:,2])
savefig("weight-height-2.pdf")
##
using Statistics
m = mean(Xf,dims=1)
m1 = mean(X1,dims=1)
m2 = mean(X2,dims=1)
n1 = size(X1,1)
n2 = size(X2,1)

B = (m1-m)'*n1*(m1-m) + (m2-m)'*n2*(m2-m)
W = (X1.-m1)'*(X1.-m1) + (X2.-m2)'*(X2.-m2)

## The matrices B and W add up to the total covariance
using LinearAlgebra
C = (Xf.-m)'*(Xf.-m)
norm(B+W - C)

##
lams,V = eigen(B,W)

##
v = V[:,2]
p1 = histogram((X1)*v,nbins=10)
histogram!((X2)*v,nbins=10)
## Save
plot!(size=(300,300),legend=false)
savefig("weight-height-lda.pdf")
##

## Compare to PCA
lams,V = eigen(C)
v2 = V[:,2]
p2 = histogram(X1*v2,nbins=10)
histogram!(X2*v2,nbins=10)
plot(p1,p2)
##
## Save
plot(p2)
plot!(size=(300,300),legend=false)
savefig("weight-height-pca.pdf")
##
X1c = X1.-m
X2c = X2.-m
scatter(X1c[:,1],X1c[:,2], label="")
scatter!(X2c[:,1],X2c[:,2], label="")
Plots.abline!(v[2]/v[1],0,label="LDA")
Plots.abline!(v2[2]/v2[1],0,label="PCA")
##
plot!(size=(300,300),legend=false)
savefig("weight-height-lda-pca.pdf")


"""
    _setdiff(x::Interval{T}, y::Interval{T})

Computes the set difference x\\y and always returns a tuple of two intervals.
If the set difference is only one interval or is empty, then the returned tuple contains 1
or 2 empty intervals.
"""
function _setdiff(x::Interval{T}, y::Interval{T}) where T
    intersection = x ∩ y

    isempty(intersection) && return (x, emptyinterval(T))
    intersection == x && return (emptyinterval(T), emptyinterval(T))  # x is subset of y; setdiff is empty

    x.lo == intersection.lo && return (Interval(intersection.hi, x.hi), emptyinterval(T))
    x.hi == intersection.hi && return (Interval(x.lo, intersection.lo), emptyinterval(T))

    return (Interval(x.lo, y.lo), Interval(y.hi, x.hi))

end

"""
    setdiff(A::IntervalBox{N,T}, B::IntervalBox{N,T})

Returns a vector of `IntervalBox`es that are in the set difference `A ∖ B`,
i.e. the set of `x` that are in `A` but not in `B`.

Algorithm: Start from the total overlap (in all directions);
expand each direction in turn.
"""
function setdiff(A::IntervalBox{N,T}, B::IntervalBox{N,T}) where {N,T}

    intersection = A ∩ B
    isempty(intersection) && return [A]

    result_list = fill(IntervalBox(emptyinterval(T), N), 2*N)
    offset = 0
    x = A.v
    @inbounds for i = 1:N
        tmp = _setdiff(A[i], B[i])
        @inbounds for j = 1:2
            x = setindex(x, tmp[j], i)
            result_list[offset+j] = IntervalBox{N, T}(x)
        end
        offset += 2
        x = setindex(x, intersection[i], i)
    end
    filter!(!isempty, result_list)
end


### A Pluto.jl notebook ###
# v0.14.1

using Markdown
using InteractiveUtils

# ╔═╡ c96551a0-881e-11eb-0d92-61edbb6c86e6
begin
	using Pkg; Pkg.activate("MLJ_env", shared=true)
  using PlutoUI
	using Test

end

# ╔═╡ accc6f84-881d-11eb-3b49-e9cd25a3b6fd
md"""
## Linear Algebra

ref. from book **"Data Science from Scratch"**, Chap 4

$(html"
© Pascal, March 2021
")
"""

# ╔═╡ 136cdd48-8835-11eb-0482-737cf7bd5a3e
PlutoUI.TableOfContents(indent=true, depth=4, aside=true)

# ╔═╡ b22cf860-8834-11eb-02de-d3ad803d1db6
html"""

"""

# ╔═╡ b1e26cfa-8834-11eb-16f5-85efb64c6a4c
md"""
### Vectors
Points in some finite dimensional space. A vector has a direction and a magnitude.$(html"
")
Vectors are useful to represent data (which lie in some multi-finite-dimensional space).
We will want to perform operations on those vectors, so let defines and implements those operations. $(html"
")
We will be using the `Vector` type from `Julia`.
"""

# ╔═╡ 9b2a7694-881e-11eb-325c-f3edd9178641
begin
	import Base: +, -

	function +(v₁::Vector{T}, v₂::Vector{T})::Vector{T} where T <: Number
		"""
		Vector Additon is component-wise
		"""
		@assert length(v₁) == length(v₂)
		v₁ .+ v₂
	end

	function +(vc::Vector{Vector{T}})::Vector{T} where T <: Number
		"""
		Sum all corresponding components of a collection of vector to return
		one vector
		"""
		@assert length(vc) > 0 "expect the collection to have at leat one vector element"
		n = length(vc[1])
		@assert all(v -> length(v) == n, vc) "Expect all vectors to be of he same length"
		v = zeros(T, n)
		for v₀ ∈ vc
			v .+= v₀
		end
		v
	end

	function -(v₁::Vector{T}, v₂::Vector{T})::Vector{T} where T <: Number
		"""
		Vector Additon is component-wise
		"""
		@assert length(v₁) == length(v₂)
		v₁ .- v₂
	end
end

# ╔═╡ 21a74512-881f-11eb-31e6-974bcaf59141
begin
	@test +([1, 2, 3], [4, 5, 6]) == [5, 7, 9]
	@test [1, 2, 3] + [4, 5, 6] == [5, 7, 9]

	vᵣ = [1.0, -1.0, π, ℯ]
	@test vᵣ + [-1.0, 1.0, -π, -ℯ] == zeros(eltype(vᵣ), length(vᵣ))
end

# ╔═╡ 11ff42a0-8828-11eb-3ba5-dbd12b5ffdda
md"""
Sometimes we want to sum a collection of vector component wise. that is create a new vector whose first component is the sum of evry first comp[onent of thew collection and  so for for the other components.

Here the collection is itsef a vector in `Julia`'s term.
cf. celle above for implementation
"""

# ╔═╡ 7dc3365c-8828-11eb-3c57-69dae41619cd
begin
	@test +([[1, 2], [3, 4], [5, 6], [7, 8]]) == [16, 20]
	@test +([[1, 3, 2] for _ ∈ 1:3]) == [3, 9, 6]
end

# ╔═╡ e9b7d526-8829-11eb-34d1-e317fdc7a1d0
md"""
`Multiplication` by a scalar, `Mean` and `dot` product.
"""

# ╔═╡ 3a7741c2-882a-11eb-2ffd-65b97e75a0b2
begin
	import Base.*

	*(x::T, v::Vector{T}) where T <: Number = v .* x
	*(v::Vector{T}, x::T) where T <: Number = v .* x

	function μ(vc::Vector{Vector{T}})::Vector{AbstractFloat} where T <: Number
		"""Element-wise mean"""
		@assert length(vc) > 0
		n = length(vc[1])
		@assert all(v -> length(v) == n, vc)
		reduce(+, vc) / n
 	end

	function dot(v₁::Vector{T}, v₂::Vector{T})::T where T <: Number
		@assert length(v₁) == length(v₂)
		sum(v₁ .* v₂)
	end
end

# ╔═╡ b3f61d4e-881f-11eb-2dcb-4dd0fd7fba06
begin
	@test -([1, 2, 3], [4, 5, 6]) == [-3, -3, -3]
	@test [1, 2, 3] - [4, 5, 6] == [-3, -3, -3]

	# vᵣ = [1.0, -1.0, π, ℯ]
	@test vᵣ - [-1.0, 1.0, -π, -ℯ] == [2.0, -2.0, 2π, 2ℯ]
end

# ╔═╡ 19f5b176-882b-11eb-3713-73c85f599be1
begin
	@test *(2, [1, 2, 3]) == [2, 4, 6]
	@test *(2, [1, 2, 3]) == *([1, 2, 3], 2)
	@test 2 * [1, 2, 3] == [1, 2, 3] * 2  ==  [2, 4, 6]
end

# ╔═╡ 50444c92-882b-11eb-2893-bf66b63ee595
begin
	μ([[1, 2, 1, 2], [3, 4, 3, 4], [5, 6, 5, 6]]) == Float64[3., 4., 3., 4.]
end

# ╔═╡ d358476c-882d-11eb-26bd-c167cabd7b28
begin
	@test dot([1, 2, 3], [4, 5, 6]) == 32
end

# ╔═╡ d33d417e-882d-11eb-0aae-a1d10512920f
function sum_of_square(v::Vector{T})::T where T <: Number
	"""
	return vᵢ * vᵢ ∀ i ∈ 1:length(v)
	"""
	dot(v, v)
end

# ╔═╡ d3202a62-882d-11eb-2e24-6d874dce773d
begin
	@test sum_of_square([1, 2, 3]) == 14
	@test sum_of_square([4, 5, 6]) == 77
end

# ╔═╡ 9db8e630-882e-11eb-07e5-0d5036f7e8e2
md"""
Let's now implement L₂ norm and euclidean distance.
"""

# ╔═╡ ba1d63da-882e-11eb-1736-37a246585172
function norm(v::Vector{T})::AbstractFloat where T <: Number
	√ sum(v .* v)
end

# ╔═╡ 9da1fcfc-882e-11eb-021f-1f89149817a9
begin
	@test norm([3, 4]) == 5
	@test norm([1, 0, 2]) ≈ √5
end

# ╔═╡ 9d8634ae-882e-11eb-15ad-75b06c847379
function distance(v₁::Vector{T}, v₂::Vector{T})::AbstractFloat where T <: Number
	"""
	distance(v₁, v₂) ≡ √(Σ (v₁ᵢ - v₂ᵢ)²)
	"""
	@assert length(v₁) == length(v₂)
	√ sum((v₁ .- v₂).^2)  # == norm(v₁ .- v₂)
end

# ╔═╡ 9d69ec22-882e-11eb-0637-d7e29f7b9f37
begin
	@test distance([2, 6, 7, 7, 5, 13, 14, 17, 11, 8],
		[3, 5, 5, 3, 7, 12, 13, 19, 22, 7]) ≈ 12.409673645990857
end

# ╔═╡ 0539916a-8835-11eb-3407-41f17efc2ee4
html"""

"""

# ╔═╡ 73e7de7a-8830-11eb-05ef-b93f6c774587
md"""
### Matrices

A two-dimensional collection of numbers.
"""

# ╔═╡ 7f051430-8830-11eb-1e43-797df6b195ed
function shape(m::Matrix{T})::Tuple{Integer, Integer} where T <: Number
	size(m)
end

# ╔═╡ 7ee4e642-8830-11eb-3d15-813c4dbb6420
@test shape([1 2 3; 4 5 6]) == (2, 3) # 2 rows, 3 columns

# ╔═╡ 7ec91688-8830-11eb-18a0-d5a67e267f9b
function get_row(m::Matrix{T}, r::Integer)::Vector{T}  where T <: Number
	@assert 1 ≤ r ≤ size(m)[1]
	view(m, r, :)  ## no copy!
end

# ╔═╡ 7e9afe92-8830-11eb-3772-150255e27923
begin
	@test get_row([1 2 3; 4 5 6], 1) == [1, 2, 3]
	@test get_row([1 2 3; 4 5 6], 2) == [4, 5, 6]
	@test_throws AssertionError get_row([1 2 3; 4 5 6], 0)
end

# ╔═╡ 648af952-8831-11eb-340b-5bc7ef7aaa73
function get_column(m::Matrix{T}, c::Integer)::Vector{T}  where T <: Number
	@assert 1 ≤ c ≤ size(m)[2]
	view(m, :, c)  ## no copy!
end

# ╔═╡ 6470ad90-8831-11eb-0413-5d016bf66536
begin
	@test get_column([1 2 3; 4 5 6], 1) == [1, 4]
	@test get_column([1 2 3; 4 5 6], 2) == [2, 5]
	@test get_column([1 2 3; 4 5 6], 3) == [3, 6]
	@test_throws AssertionError get_column([1 2 3; 4 5 6], 0)
end

# ╔═╡ 6458987c-8831-11eb-2b89-efac8fabf803
md"""
We will want to be able to create a new matrix given its shape and a function generator.
"""

# ╔═╡ c832331c-8831-11eb-0b7c-9f978c82f890
function make_matrix(nrows::Integer, ncols::Integer, fn::Function;
		DT::DataType=Float64)::Matrix
	@assert 1 ≤ nrows && 1 ≤ ncols

	m = zeros(DT, (nrows, ncols))
	for c ∈ 1:ncols, r ∈ 1:nrows
		m[r, c] = fn(r, c)
	end
	m
end

# ╔═╡ be10ce56-8832-11eb-0742-d90fa7f6f669
function identity_matrix(n::Integer; DT::DataType=Float64)::Matrix
	"""Returns the n × n identity matrix"""
	make_matrix(n, n, (i, j) -> i == j ? one(DT) : zero(DT); DT)
end

# ╔═╡ c81a12dc-8831-11eb-27e9-07132c6310a1
@test identity_matrix(5) == [1. 0. 0. 0. 0.; 0. 1. 0. 0. 0.; 0. 0. 1. 0. 0.; 0. 0. 0. 1. 0.; 0. 0. 0. 0. 1.]

# ╔═╡ 3c69e83a-8835-11eb-218e-0bb108ad7b9c
html"""

Mar 2021, Corto Inc
"""

# ╔═╡ Cell order:
# ╟─accc6f84-881d-11eb-3b49-e9cd25a3b6fd
# ╟─136cdd48-8835-11eb-0482-737cf7bd5a3e
# ╟─b22cf860-8834-11eb-02de-d3ad803d1db6
# ╟─b1e26cfa-8834-11eb-16f5-85efb64c6a4c
# ╠═c96551a0-881e-11eb-0d92-61edbb6c86e6
# ╠═9b2a7694-881e-11eb-325c-f3edd9178641
# ╠═21a74512-881f-11eb-31e6-974bcaf59141
# ╠═b3f61d4e-881f-11eb-2dcb-4dd0fd7fba06
# ╟─11ff42a0-8828-11eb-3ba5-dbd12b5ffdda
# ╠═7dc3365c-8828-11eb-3c57-69dae41619cd
# ╟─e9b7d526-8829-11eb-34d1-e317fdc7a1d0
# ╠═3a7741c2-882a-11eb-2ffd-65b97e75a0b2
# ╠═19f5b176-882b-11eb-3713-73c85f599be1
# ╠═50444c92-882b-11eb-2893-bf66b63ee595
# ╠═d358476c-882d-11eb-26bd-c167cabd7b28
# ╠═d33d417e-882d-11eb-0aae-a1d10512920f
# ╠═d3202a62-882d-11eb-2e24-6d874dce773d
# ╟─9db8e630-882e-11eb-07e5-0d5036f7e8e2
# ╠═ba1d63da-882e-11eb-1736-37a246585172
# ╠═9da1fcfc-882e-11eb-021f-1f89149817a9
# ╠═9d8634ae-882e-11eb-15ad-75b06c847379
# ╠═9d69ec22-882e-11eb-0637-d7e29f7b9f37
# ╟─0539916a-8835-11eb-3407-41f17efc2ee4
# ╟─73e7de7a-8830-11eb-05ef-b93f6c774587
# ╠═7f051430-8830-11eb-1e43-797df6b195ed
# ╠═7ee4e642-8830-11eb-3d15-813c4dbb6420
# ╠═7ec91688-8830-11eb-18a0-d5a67e267f9b
# ╠═7e9afe92-8830-11eb-3772-150255e27923
# ╠═648af952-8831-11eb-340b-5bc7ef7aaa73
# ╠═6470ad90-8831-11eb-0413-5d016bf66536
# ╟─6458987c-8831-11eb-2b89-efac8fabf803
# ╠═c832331c-8831-11eb-0b7c-9f978c82f890
# ╠═be10ce56-8832-11eb-0742-d90fa7f6f669
# ╠═c81a12dc-8831-11eb-27e9-07132c6310a1
# ╟─3c69e83a-8835-11eb-218e-0bb108ad7b9c

Data Science From Scratch/04_Linear_Algebra.jl


##################################################
## Classical derivative-based, iterative, root-finding algorithms
##
## If ri = f^(i-1)/f^(i), then these have an update step `xn - delta` where:
##
## * Newton: delta = r1  # order 2 if simple root (multiplicity 1)
## * Halley: delta = 2*r2/(2r2 - r1) * r1 # order 3 if simple root
## * Schroder: delta = r2  / (r2 - r1) * r1  # order 2
## * Thukral(3): delta =  (-2*r3)*(r2 - r1)/(r1^2 - 3*r1*r3 + 2*r2*r3) * r1 # order 3
## * Thukral(4): delta =  3*r1*r2*r4*(r1^2 - 3*r1*r3 + 2*r2*r3)/(-r1^3*r2 + 4*r1^2*r2*r4 + 3*r1^2*r3*r4 - 12*r1*r2*r3*r4 + 6*r2^2*r3*r4) # order 4
##
## The latter two come from
## [Thukral](http://article.sapub.org/10.5923.j.ajcam.20170702.05.html). They are not implemented.

## Newton
abstract type AbstractNewtonLikeMethod <: AbstractUnivariateZeroMethod end
fn_argout(::AbstractNewtonLikeMethod) = 2
struct Newton <: AbstractNewtonLikeMethod end
"""

    Roots.Newton()

Implements Newton's [method](http://tinyurl.com/b4d7vls):
`xᵢ₊₁ =  xᵢ - f(xᵢ)/f'(xᵢ)`.  This is a quadratically convergent method requiring
one derivative. Two function calls per step.

Example
```
find_zero((sin,cos), 3.0, Roots.Newton())
```

If function evaluations are expensive one can pass in a function which returns (f, f/f') as follows

```
find_zero(x -> (sin(x), sin(x)/cos(x)), 3.0, Roots.Newton())
```

This can be advantageous if the derivative is easily computed from the
value of f, but otherwise would be expensive to compute.

The error, `eᵢ = xᵢ - α`, can be expressed as `eᵢ₊₁ = f[xᵢ,xᵢ,α]/(2f[xᵢ,xᵢ])eᵢ²` (Sidi, Unified treatment of regula falsi, Newton-Raphson, secant, and Steffensen methods for nonlinear equations).

"""
Newton

# we store x0,x1,fx0,fx1 **and** Δ = fx1/f'(x1)
struct NewtonState{T,S} <: AbstractUnivariateZeroState{T,S}
    xn1::T
    xn0::T
    Δ::T
    fxn1::S
    fxn0::S
end

function init_state(M::Newton, F::Callable_Function, x)
    x₀ = float(first(x))
    fx₀, Δ = F(x₀)
    x₁ = x₀ - Δ
    state = init_state(M, F, x₀, x₁, fx₀, fx₀)
end

# compute fx₁, Δ
function init_state(::Newton, F, x₀, x₁, fx₀, fx₁)
    fx₁, Δ = F(x₁)
    NewtonState(x₁, x₀, Δ, fx₁, fx₀)
end

initial_fncalls(M::Newton) = 2

function update_state(method::Newton, F, o::NewtonState{T,S}, options, l=NullTracks()) where {T, S}

    xn0, xn1 = o.xn0, o.xn1
    fxn0, fxn1 = o.fxn0, o.fxn1
    Δ = o.Δ

    if isissue(Δ)
        log_message(l, "Issue with `f/f′'")
        return o, true
    end

    xn0, xn1 = xn1, xn1-Δ
    fxn0 = fxn1
    fxn1, Δ = F(xn1)
    incfn(l,2)

    @set! o.xn0 = xn0
    @set! o.xn1 = xn1
    @set! o.Δ = Δ
    @set! o.fxn0 = fxn0
    @set! o.fxn1 = fxn1

    return o, false

end

"""
    Roots.newton(f, fp, x0; kwargs...)

Implementation of Newton's method: `xᵢ₊₁ =  xᵢ - f(xᵢ)/f'(xᵢ)`.

Arguments:

* `f::Function` -- function to find zero of

* `fp::Function` -- the derivative of `f`.

* `x0::Number` -- initial guess. For Newton's method this may be complex.

With the `FowardDiff` package derivatives may be computed automatically. For example,  defining
`D(f) = x -> ForwardDiff.derivative(f, float(x))` allows `D(f)` to be used for the first derivative.

Keyword arguments are passed to `find_zero` using the `Roots.Newton()` method.

See also `Roots.newton((f,fp), x0)` and `Roots.newton(fΔf, x0)` for simpler implementations.

"""
newton(f, fp, x0; kwargs...) = find_zero((f, fp), x0, Newton(); kwargs...)

## Halley
abstract type AbstractHalleyLikeMethod <: AbstractUnivariateZeroMethod end
fn_argout(::AbstractHalleyLikeMethod) = 3

"""
    Roots.Halley()

Implements Halley's [method](http://tinyurl.com/yd83eytb),
`xᵢ₊₁ = xᵢ - (f/f')(xᵢ) * (1 - (f/f')(xᵢ) * (f''/f')(xᵢ) * 1/2)^(-1)`
This method is cubically converging, but requires more function calls per step (3) than
other methods.

Example
```
find_zero((sin, cos, x->-sin(x)), 3.0, Roots.Halley())
```

If function evaluations are expensive one can pass in a function which
returns `(f, f/f',f'/f'')` as follows

```
find_zero(x -> (sin(x), sin(x)/cos(x), -cos(x)/sin(x)), 3.0, Roots.Halley())
```

This can be advantageous if the derivatives are easily computed from
the computation for f, but otherwise would be expensive to compute separately.

The error, `eᵢ = xᵢ - α`, satisfies
`eᵢ₊₁ ≈ -(2f'⋅f''' -3⋅(f'')²)/(12⋅(f'')²) ⋅ eᵢ³` (all evaluated at `α`).

"""
struct Halley <: AbstractHalleyLikeMethod
end

struct HalleyState{T,S} <: AbstractUnivariateZeroState{T,S}
    xn1::T
    xn0::T
    Δ::T
    ΔΔ::T
    fxn1::S
    fxn0::S
end

# we compute one step here to get x₁
function init_state(M::AbstractHalleyLikeMethod, F::Callable_Function, x)
    x₀ = float(first(x))
    fx₀, (Δ, ΔΔ) = F(x₀)
    x₁ = x₀ - 2ΔΔ/(2ΔΔ - Δ) * Δ
    state = init_state(M, F, x₀, x₁, fx₀, fx₀)
end

function init_state(::AbstractHalleyLikeMethod, F, x₀, x₁, fx₀, fx₁)
    fx₁, (Δ, ΔΔ) = F(x₁)
    HalleyState(x₁, x₀, Δ, ΔΔ, fx₁, fx₀)
end

initial_fncalls(M::AbstractHalleyLikeMethod) = 2*3

function update_state(method::Halley, F, o::HalleyState{T,S}, options::UnivariateZeroOptions, l=NullTracks()) where {T,S}
    xn = o.xn1
    fxn = o.fxn1
    r1, r2 = o.Δ, o.ΔΔ

    Δ =  2*r2/(2r2 - r1) * r1
    if isissue(Δ)
        log_message(l, "Issue with computing `Δ`")
        return (o, true)
    end

    xn1::T = xn - Δ
    fxn1::S, (r1::T, r2::T) = F(xn1)
    incfn(l,3)

    @set! o.xn0 = xn
    @set! o.xn1 = xn1
    @set! o.fxn0 = fxn
    @set! o.fxn1 = fxn1
    @set! o.Δ = r1
    @set! o.ΔΔ = r2

    return o, false

end

"""
    Roots.halley(f, fp, fpp, x0; kwargs...)

Implementation of Halley's method (cf `?Roots.Halley()`).

Arguments:

* `f::Function` -- function to find zero of

* `fp::Function` -- derivative of `f`.

* `fpp:Function` -- second derivative of `f`.

* `x0::Number` -- initial guess

With the `FowardDiff` package derivatives may be computed automatically. For example,  defining
`D(f) = x -> ForwardDiff.derivative(f, float(x))` allows `D(f)` and `D(D(f))` to be used for the first and second
derivatives, respectively.

Keyword arguments are passed to `find_zero` using the `Roots.Halley()` method.

"""
halley(f, fp, fpp, x0; kwargs...) = find_zero((f, fp, fpp), x0, Halley(); kwargs...)


using DifferentialEquations
using IterableTables, DataFrames
using NamedTuples
using Gadfly

function F(du,u,p,t)
    S_H, E_H, I_H, R_H, S_V, E_V, I_V = u

    # host dynamics
    host_infection = (p.β*S_H*I_V)/p.N_H
    host_mortality = p.μ_H .* u[1:4] # include S_H, so easier to remove mortality
    host_births = sum(host_mortality)
    host_progression = p.σ_H*E_H
    recovery = p.λ*I_H

    du[1] = -host_infection + host_births
    du[2] = host_infection - host_progression
    du[3] = host_progression - recovery
    du[4] = recovery
    du[1:4] -= host_mortality

    # vector dynamics
    vec_infection = (p.β*S_V*I_H)/p.N_H
    vec_mortality = p.μ_V .* u[5:7] # include S_V, so easier to remove mortality
    vec_births = sum(vec_mortality)
    vec_progression = p.σ_V*E_V

    du[5] = -vec_infection + vec_births
    du[6] = vec_infection - vec_progression
    du[7] = vec_progression
    du[5:7] -= vec_mortality

end

# nb: in >= Julia v0.7, can eliminate this import
#  and the @NT syntax
u0 = [
    S_H=100.0,   E_H=0.0, I_H=1.0, R_H=0.0,
    S_V=10000.0, E_V=0.0, I_V=0.0
]
p = @NT(
  μ_H=1/365, μ_V=1/30, σ_H=1/3, σ_V=1/7, λ=1/14,
  β=0.05, N_H = sum(u0[1:4])
)
tspan = (0.0, 365.0)
prob = ODEProblem(F, u0, tspan, p)
sol = @time solve(prob,Tsit5(),reltol=1e-8,abstol=1e-8,saveat=linspace(0,365,365*10+1))

df = DataFrame(sol)
rename!(df, Dict(:timestamp => :t,
  :value1 => :S_H, :value2 => :E_H, :value3 => :I_H, :value4 => :R_H,
  :value5 => :S_V, :value6 => :E_V, :value7 => :I_V
))
mlt = melt(df,:t) # convert results into long format for plotting
mlt[:host] = contains.(string.(mlt[:variable]),"H"); # tag which entries are host vs vector
df

# Plot
fig1a = plot(mlt[mlt[:host] .== true,:], x=:t, y=:value, color=:variable, Geom.line)
fig1b = plot(mlt[mlt[:host] .!= true,:], x=:t, y=:value, color=:variable, Geom.line)
vstack(fig1a,fig1b)


# f(x)=abs(x)

function Fabs(x)
    return (abs(x),nothing)
end

function Fabs_inplace(value::Array,auxvalue,x) # inplace
    copy!(value,abs(x))
end

function Dabs(derivativeIDX,f_c,faux_c,grad_c,grad_n,x)
    axpy!(1.0,sign(x).*grad_c,grad_n)
end

if PROC=="GPU"
    function Fabs_inplace(value::CudaArray,auxvalue,x::CudaArray) # inplace
        fill!(value,0.0)
        abs!(x,value)
    end

    function Dabs(derivativeIDX,f_c,faux_c,grad_c,grad_n,x::CudaArray)
        xsigny_update!(grad_c,x,grad_n)
    end
end

Derivative[Fabs]=Dabs # Define dictionary lookup
Inplace[Fabs]=Fabs_inplace

import Base.abs

abs(n::ADnode)=ADnode(Fabs,n)

###abs(A::ADtrans)=transpose(abs(node[A.parent])) # abs(A')=(abs(A))' TODO:check

export abs


export poisson, poisson!

 # get the maximum intensity of an image
mymax(X::AbstractArray{<:RGB}) = max_rgb(X)
mymax(X::AbstractArray{<:Gray}) = gray(maximum(X))
mymax(X::AbstractArray) = maximum(X)

 # noise function for poisson
function noise_f(x, scaling, max_intens)
    # scale image to max_intensity and apply poisson noise
    # after Poisson noise scale it back
    return pois_rand(x * scaling / max_intens) * max_intens / scaling
end

f_pois(scaling, max_intens) = x -> scaling == nothing ? pois_rand(x) : noise_f(x, scaling, max_intens)

comb_pois(x, n) = n

poisson(X::AbstractArray, scaling=nothing; clip=false) = poisson!(copy(X), scaling, clip=clip)
function poisson!(X::AbstractArray, scaling=nothing; clip=false)
    max_intens = convert(Float64, mymax(X))
    return apply_noise!(comb_pois, f_pois(scaling, max_intens), X, clip)
end

"""
    poisson(X; scaling=nothing, clip=false)

Returns the array `X` affected by Poisson noise.
At every position the Poisson noise affects the intensity individually
and the values at the positions represent the expected value of the Poisson
Distribution.
Since Poisson Noise due to discrete events you should
provide the optional argument `scaling`. This `scaling` connects
the highest value of the array with the discrete number of events.
The highest value will be then scaled and the poisson noise is applied
Afterwards we scale the whole array back so that the initial intensity
is preserved but with applied Poisson noise.
`clip` is a keyword argument. If given, it clips the values to [0, 1]
"""
poisson


### PARALLELISATION:
# start julia with "julia -p auto" or "julia -p 4" to use multiple processors
using Revise
using Random
using SpecialFunctions
using PyCall
using PyPlot
using Plots
using StatsPlots
using CmdStan

#%%

# If run from command line externally:
#projDir = dirname(@__FILE__)

# If run from Jupyter/Hydrogen, maybe change to suit you:
projDir= "/home/johhub/Desktop/ABDA/A4"
#projDir= "/lhome/johhub/Desktop/ABDA/A4"
tmpDir = projDir*"/tmp"

################################################################

function runStanMC(y,a,b,burnIn,N,projDir,tmpDir,keepchains=false,noOfChains=4)
    # REQUIRES MODULES:
    # CmdStan
    #
    # INPUT:
    # y --> observed data
    # burnIn --> number of bur in samples
    # N --> total number of sample points
    # projDir --> directory where the analysis should be conducted, and plots should be saved
    # tmpDir --> directory where the temp files can be stored
    # keepchains (optional) --> keep all vectors of the various chains, false
    #                           per defaul, i.e. all chain are thrown in one vector
    #
    # OUTPUT:
    # x_s --> sampled x values for all chains (Nx4 array)

    ### 1) Define the Stan model, note that variable names must
    # correspond to observed data later
    # dichtonomous model
    dichtModel = "
    data {
    int<lower=0> J; // number of flips in the observed data, setting the minimum amount of data under lower
    int<lower=0,upper=1> y[J];  // coin flips
    }
    parameters {
    real<lower=0,upper=1> theta; // prob of getting a head
    }
    transformed parameters { // no transformed variables to use
    }
    model {
    theta ~ beta($a, $b);         // prior distribution for theta
    y ~ bernoulli(theta);       // likelihood, note that stan will create the posterior automatically.
    }
    ";

    ### 2) Data for the Stan model, note that variable names must
    # correspond to defined model in stan-code
    observedData = Dict("J" => length(y), "y" => y);

    ### 3) Chain specs
    myModel = Stanmodel(
                    Sample(save_warmup=true,
                           num_warmup=burnIn,
                           num_samples=N,
                           thin=1),   # thin: Period between saved samples
                    name = "flipping",
                    model = dichtModel,
                    printsummary = false,
                    tmpdir = tmpDir,
                    nchains = noOfChains);    # number of chains (4 default)

    ### 4) Run CmdStan:
    # rc:     return code (0 if all is fine)
    # chn:    chain results
    # cnames: vector of variable names
    rc, chn, cnames = stan(myModel,
                           observedData,
                           projDir,
                           diagnostics = false,
                           CmdStanDir = CMDSTAN_HOME);

    ### Results without burn-in
    #chns = chn[(n_burnin+1):end, :, :]
    # the 7th (out of 8) coulmn in the .value method are the actual sampled points
    # the 3rd dimension are the different chains: 1,2,3 --> chain 1,2,3

    x_s = 1* chn.value[(burnIn+1):end,7,:]  # the 1* converts the "Axis Array" in the chain into an Array{Float64,2}

    if keepchains == false
        x_s = reshape(x_s,:);   # Throw all the chain samples in a single vetor
    end

    return x_s
end

#%%

# Chain inputs:
N = 10^6
N_burnin = 1000

#%%

# First Column of Figure 6.4:
a = 100 # prior specs
b = 100 # prior specs
y = [ones(Int64,17); zeros(Int64,3)];    # make an Array{Intt64,1} array

# to receive the prior curve, just set the data to empty array []:
chain = runStanMC([],a,b,N_burnin,N,projDir,tmpDir);
histogram(chain, bins=100, normalize=:pdf, label="prior")
density!(chain,linewidth=3,label="prior")

# to receive the likelihood curve, just set the prior to a=1, b=1 (flat):
chain = runStanMC(y,1,1,N_burnin,N,projDir,tmpDir);
histogram!(chain, bins=100, normalize=:pdf, label="likelihood")
density!(chain,linewidth=3,label="likelihood")

posterior1 = runStanMC(y,a,b,N_burnin,N,projDir,tmpDir);
histogram!(posterior1, bins=100, normalize=:pdf, label="posterior")
density!(posterior1,linewidth=3,label="posterior")

Plots.savefig("A4-A-C1_Stan.pdf")

#%%
# Second Column
a = 18.25
b = 6.75
y = [ones(Int64,17); zeros(Int64,3)];    # make an Array{Intt64,1} array

# to receive the prior curve, just set the data to empty array []:
chain = runStanMC([],a,b,N_burnin,N,projDir,tmpDir);
histogram(chain, bins=100, normalize=:pdf, label="prior")
density!(chain,linewidth=3,label="prior")

# to receive the likelihood curve, just set the prior to a=1, b=1 (flat):
chain = runStanMC(y,1,1,N_burnin,N,projDir,tmpDir);
histogram!(chain, bins=100, normalize=:pdf, label="likelihood")
density!(chain,linewidth=3,label="likelihood")

posterior2 = runStanMC(y,a,b,N_burnin,N,projDir,tmpDir);
histogram!(posterior2, bins=100, normalize=:pdf, label="posterior")
density!(posterior2,linewidth=3,label="posterior")

Plots.savefig("A4-A-C2_Stan.pdf")

#%%
# Third Column
a = 1
b = 1
y = [ones(Int64,17); zeros(Int64,3)];    # make an Array{Intt64,1} array

# to receive the prior curve, just set the data to empty array []:
chain = runStanMC([],a,b,N_burnin,N,projDir,tmpDir);
histogram(chain, bins=100, normalize=:pdf, label="prior")
density!(chain,linewidth=3,label="prior")

# to receive the likelihood curve, just set the prior to a=1, b=1 (flat):
chain = runStanMC(y,1,1,N_burnin,N,projDir,tmpDir);
histogram!(chain, bins=100, normalize=:pdf, label="likelihood")
density!(chain,linewidth=3,label="likelihood")

posterior3 = runStanMC(y,a,b,N_burnin,N,projDir,tmpDir);
histogram!(posterior3, bins=100, normalize=:pdf, label="posterior")
density!(posterior3,linewidth=3,label="posterior")

Plots.savefig("A4-A-C3_Stan.pdf")

#%%

density(posterior1, label="a=100, b=100")
density!(posterior2, label="a=18.25, b=6.75")
density!(posterior3, label="a=1, b=1")
Plots.savefig("A4-A-all_Stan.pdf")


#=
Pentagonal numbers are generated by the formula, Pn=n(3n−1)/2. The first ten pentagonal numbers are:

1, 5, 12, 22, 35, 51, 70, 92, 117, 145, ...

It can be seen that P4 + P7 = 22 + 70 = 92 = P8. However, their difference, 70 − 22 = 48, is not pentagonal.

Find the pair of pentagonal numbers, Pj and Pk, for which their sum and difference are pentagonal and D = |Pk − Pj| is minimised; what is the value of D?
=#
function pent(x)
  div(x*(3x-1),2)
end

function ispent(x)
  n = (sqrt(24x+1)+1)/6
  return n == round(Int,n) && n > 0
end

function calc()
  i = 0
  pentagonals = Int64[]
  while true
    i += 1
    p = pent(i)
    for j in pentagonals
      diff = abs(p-j)
      if ispent(diff)
        intsum = p + j
        if ispent(intsum)
          return diff
        end
      end
    end
    push!(pentagonals, p)
  end
end
@time println(calc())


#----------------- Get the A matrix
function getDivGrad(n1,n2,n3)
    # the Divergence
    D1 = kron(speye(n3),kron(speye(n2),ddx(n1)))
    D2 = kron(speye(n3),kron(ddx(n2),speye(n1)))
    D3 = kron(ddx(n3),kron(speye(n2),speye(n1)))
    # DIV from faces to cell-centers

    Div = [D1 D2 D3]

    return Div*Div'
end
#----------------- 1D finite difference on staggered grid

# generate 1D derivatives
ddx(n)=spdiags(ones(n)*[-1 1],[0,1],n,n+1)

#------------- Build a diagonal matrix
function spdiags(B,d,m,n)
    # spdiags(B,d,m,n)
    # creates a sparse matrix from its diagonals
    d = d[:]
    p = length(d)

    len = zeros(p+1,1)
    for k = 1:p
        len[k+1] = Int(len[k]+length(max(1,1-d[k]):min(m,n-d[k])))
    end
    a = zeros(round(Int, len[p+1]), 3)
    for k = 1:p
        # Append new d[k]-th diagonal to compact form
        i = max(1,1-d[k]):min(m,n-d[k])
        a[(round(Int, len[k])+1):round(Int, len[k+1]),:] = [i i+d[k] B[i+(m >= n)*d[k], k]]
    end

    sparse(round(Int, a[:,1]), round(Int, a[:,2]), a[:,3], m, n)
end


# using LinearAlgebra
# include("sphere.jl")
##

"""
This function compute the associated Legendre polynomial P(l,m,x)
    for m >= 0, l >= m, and abs(x) <= 1
"""
function Plm(l::Int, m::Int, x::AbstractFloat)
    @assert 0 <= m <= l
    pmm = 1.0
    if m > 0
        somx2 = sqrt((1.0 - x) * (1.0 + x))
        fact = 1.0
        for i = 1:m
            pmm *= (-fact) * somx2
            fact += 2.0
        end
    end

    if l == m
        return pmm
    end
    pmmp1 = x * (2.0 * m + 1.0) * pmm
    if (l == (m + 1))
        return pmmp1
    end

    pll = 0.0

    for ll = (m+2):l
        pll = ((2.0 * ll - 1.0) * x * pmmp1 - (ll + m - 1.0) * pmm) / (ll - m)
        pmm = pmmp1
        pmmp1 = pll
    end

    return pll
end
"""
This function compute the normalized associated Legendre polynomial P(l,m,x)
    for m >= 0, l >= m, and abs(x) <= 1. It is alias  of GSL sf_legendre_Plm
"""
function sph_Plm(l::Int, m::Int, x::AbstractFloat)
    @assert 0 <= m <= l
    return sf_legendre_sphPlm(l, m, x)
end
"""
This function compute all of the normalized associated Legendre polynomial P(l,m)
    for m >= 0, l >= m, and abs(x) <= 1. It is alias  of GSL sf_legendre_array.
    This implementation include the Condon-Shortley phase factor.
"""
function Plm_array(l_max::Int, x::AbstractFloat; normal = :SPHARM)
    normalization = Dict(
        :SPHARM => GSL_SF_LEGENDRE_SPHARM,
        :NONE => GSL_SF_LEGENDRE_NONE,
        :SCHMIDT => GSL_SF_LEGENDRE_SCHMIDT,
        :FULL => GSL_SF_LEGENDRE_FULL,
    )
    return sf_legendre_array_e(normalization[normal], l_max, x,1)
end
"""
This function returns the index of Plm_array corresponding to P(l,m)
"""
function Plm_array_index(l::Int, m::Int)
    @assert 0 <= m <= l
    return sf_legendre_array_index(l,m) + 1
end
"""
This function returns the the general real spherical harmonics function Y(l,m) at
    pol angle and az_angle. l ∈ (0,..,n) and ∈ (-l,..l) m are the
    order and degree of spherical harmonics respectively.
    Pol_angle ∈ (0,..pi) and az_angle ∈ (0,2pi) are the polar angle and azimuthal angle
    coordinate.
"""
function real_spharm(l::Int, m::Int, pol_angle, az_angle)
    sqrt2 = sqrt(2.0)
    if m == 0
        return sph_Plm(l, m, cos(pol_angle))
    elseif m > 0
        return sqrt2 * cos(m * az_angle) * sph_Plm(l, m, cos(pol_angle))
    elseif m < 0
        return sqrt2 * sin(-m * az_angle) * sph_Plm(l, -m, cos(pol_angle))
    end
end
"""
This function returns all of general real spherical harmonics function Y(l,m)
    with order l.
    Pol_angle ∈ (0,..pi) and az_angle ∈ (0,2pi) are the polar angle and azimuthal angle
    coordinate.
"""
function real_spharm_array(l_max::Int, pol_angle, az_angle)
    sqrt2 = sqrt(2.0)
    len_array = (l_max + 1)^2
    plm_array = Plm_array(l_max, cos(pol_angle); normal = :SPHARM)
    spharm_array = Vector{Float64}(undef, len_array)
    for j = 1:len_array
        l, m = j2lm(j)
        idx = Plm_array_index(l, abs(m))
        if m == 0
            spharm_array[j] = plm_array[idx]
        elseif m > 0
            spharm_array[j] = sqrt2 * cos(m * az_angle) * plm_array[idx]
        elseif m < 0
            spharm_array[j] = sqrt2 * sin(-m * az_angle) * plm_array[idx]
        end
    end
    return spharm_array
end

"""
This function returns the order l and degree m of Y(l,m) from linear
    index j of spharm_array
"""
function j2lm(j)
    j = j - 1
    l = floor(Int, sqrt(j))
    m = j - l * (l + 1)
    return l, m
end
"""
This function returns the linear index j of spharm_array
    from the order l and degree m of Y(l,m)
"""
function lm2j(l, m)
    return l^2 + l + m + 1
end

"""
This function returns the The matrix of general real spherical
    harmonics function Y(l,m). The row index i  corresponds to points x_i and
    column index j obtained from j =  l^2 + l + m +1 where l and m are the order
    and degree of spherical harmonics respectively.
"""
function spharm_mat(l_max, pol_angle::T, az_angle::T) where {T<:AbstractVector}
    n_row = length(pol_angle)
    n_column = (l_max + 1)^2
    mat = Matrix{eltype(pol_angle)}(undef, n_row, n_column)
    for i = 1:n_row
        mat[i, :] .= real_spharm_array(l_max, pol_angle[i], az_angle[i])
    end
    return mat
end

"""
This function returns the The matrix of coefficient of SH function Y(l,m). The
    row index i  corresponds to points x_i and column index j obtained from j =  l^2 + l + m +1 where l and m are the order
    and degree of spherical harmonics respectively.
"""
function spharm_coefs(l_max::Int, x, pol_angle,az_angle)
    mat = spharm_mat(l_max, pol_angle, az_angle)
    coefs = mat \ x
    return coefs
end


# Problem 1
# If we list all the natural numbers below 10 that are multiples of 3 or 5,
# we get 3, 5, 6 and 9. The sum of these multiples is 23.
# Find the sum of all the multiples of 3 or 5 below 1000.

multiple_3_5(x) = ((x % 3) == 0) || ((x % 5) == 0)

∑ = 0
for i in 1:1000-1
    global ∑ += multiple_3_5(i) ? i : 0
end
print(∑)


using SpecialFunctions

@doc raw"""
Evaluates the Boys function ``Fₘ(x) = ∫₀¹ t²ᵐ \exp(-xt²) dt``.
`IND==0` gives around 14 digits, `IND==1` around 6 and `IND==3` around 3.
"""
function boys(m::Int, x::T, IND=0) where T <: Real
    x < 0 && throw(DomainError(x, "`x` must be nonnegative."))
    if x > 0
        # gamma_inc returns tuple of incomplete Γ function ratios,
        # we only care about the first returned result
        Px, _ = gamma_inc(m + 1 / T(2), x, IND)
        1 / 2x^(m + 1 / T(2)) * gamma(m + 1 / T(2)) * Px
    else
        1 / T(2m + 1)
    end
end

function doublefactorial(n::Integer)
    n < 0 && throw(DomainError(n, "`n` must be nonnegative."))
    start = iseven(n) ? 2 : 1
    f::typeof(n*n) = 1
    for i::typeof(n*n) = start:2:n
        f *= i
    end
    return f
end


#! /usr/bin/julia

# Rosetta Code, Arithmetic-geometric mean/Calculate Pi

function agm_step{T<:FloatingPoint}(x::T, y::T)
    (0.5*(x + y), sqrt(x*y))
end

function approx_pi_step{T<:FloatingPoint,U<:Integer}(x::T, y::T, z::T, n::U)
    (a, g) = agm_step(x, y)
    k = n + 1
    s = z + 2^(k+1)*(a^2 - g^2)
    return (a, g, s, k)
end

function approx_pi{T<:FloatingPoint}(a::T, g::T, s::T)
    4a^2/(1 - s)
end

prec = 512
set_bigfloat_precision(prec)
println("Approximating pi using ", prec, "-bit floats.")
println("   k     Error  Result")
a = big(1.0)
g = a/sqrt(big(2.0))
s = big(0.0)
k = 0
oldagpi = big(0.0)
for i in 1:100
    (a, g, s, k) = approx_pi_step(a, g, s, k)
    agpi = approx_pi(a, g, s)
    2eps(agpi) < abs(agpi-oldagpi) || break
    oldagpi = agpi
    err = pi - agpi
    print(@sprintf("  %2d ", i))
    print(@sprintf(" %9.1e", err))
    println(@sprintf(" %.60e", agpi))
end


include("../optimizers/linesearch.jl")
include("../utils/functions.jl")
using Gadfly

function optimize_linear_constraints(f, A, f_g, f_h, yinit=0)
    """
    Convert constrained optimization problem min f(x) s.t. Ax = b
        to unconstrained problem min g(y) = f(B*y + x0), where x0 is a solution to Ax = b
        and B is a basis for the kernel of A.

    Defaults to using Newton's line search algorithm.

    f: constrained objective
    A: constraint matrix
    f_g: gradient of f
    f_h: hessian of f
    yinit: init y with a vector of all yinit values
    """

    m, n = size(A)
    b = zeros(m)
    x0 = A \ b
    B = nullspace(A)

    y0 = ones(size(B)[2]) .* yinit

    g(y) = f(B*y + x0)
    g_g(y) = B' * f_g(B*y + x0)
    g_h(y) = B' * f_h(B*y + x0) * B

    yvals = line_search(g, y0, g_g, g_h, "newton", 5000)
    xvals = [B*y + x0 for y in yvals]
    lambdas = [A' \ f_g(x) for x in xvals]

    return yvals, xvals, lambdas
end

function generate_constraint_matrix(n)
    A = ones(2,n)
    for i in 1:n
        if i % 2 == 1
            A[2,i] = -1
        end
    end
    return A
end

A = generate_constraint_matrix(30)
b = zeros(2)
x0 = A \ b; B = nullspace(A)
f = cute
f_g = cute_g
f_h = cute_h
g(y) = f(B*y + x0)
g_g(y) = B' * f_g(B*y + x0)
g_h(y) = B' * f_h(B*y + x0) * B

yvals, xvals, lambdas = optimize_linear_constraints(f, A, f_g, f_h, 0);

niters = length(xvals)
fx = [f(x) for x in xvals]
Gadfly.plot(x=1:niters, y=fx, Geom.line,
    Guide.xlabel("iteration"), Guide.ylabel("f(x)"), Guide.title("Value of function"),
    Scale.x_log10, Scale.y_log10)

grads = [norm(g_g(y),2) for y in yvals]
ratios = grads[2:niters,:]./grads[1:niters-1,:]
Gadfly.plot(x=1:niters-1, y=ratios, Geom.line,
Guide.xlabel("iteration"), Guide.ylabel("f_g(x)"), Guide.title("Gradient norm ratios"),
    Scale.x_log10, Scale.y_log10)

Gadfly.plot(x=1:niters, y=grads, Geom.line,
Guide.xlabel("iteration"), Guide.ylabel("f_g(x)"), Guide.title("Gradient norm"),
    Scale.x_log10, Scale.y_log10)

# Lagrange multipliers
lambdas[end]

A = generate_constraint_matrix(100)
x0 = A \ b; B = nullspace(A)
f = cute
f_g = cute_g
f_h = cute_h
g(y) = f(B*y + x0)
g_g(y) = B' * f_g(B*y + x0)
g_h(y) = B' * f_h(B*y + x0) * B

yvals, xvals, lambdas = optimize_linear_constraints(f, A, f_g, f_h, 0);
# Lagrange multipliers
println(lambdas[end])
# Final value
println(f(xvals[end]))

niters = length(xvals)
fx = [f(x) for x in xvals]
#gy = [g(y) for y in yvals]

Gadfly.plot(x=1:niters, y=fx, Geom.line,
    Guide.xlabel("iteration"), Guide.ylabel("f(x)"), Guide.title("Value of function"),
    Scale.x_log10, Scale.y_log10)

grads = [norm(g_g(y),2) for y in yvals]
ratios = grads[2:niters,:]./grads[1:niters-1,:]
Gadfly.plot(x=1:niters-1, y=ratios, Geom.line,
Guide.xlabel("iteration"), Guide.ylabel("f_g(x)"), Guide.title("Gradient norm ratios"),
    Scale.x_log10, Scale.y_log10)

Gadfly.plot(x=1:niters, y=grads, Geom.line,
Guide.xlabel("iteration"), Guide.ylabel("f_g(x)"), Guide.title("Gradient norm"),
    Scale.x_log10, Scale.y_log10)


abstract type AbstractUtility end

#
# Separable utility
#

# Consumption utility

@doc doc"""
Type used to evaluate log utility. Log utility takes the form

u(c) = \log(c)

Additionally, this code assumes that if c < 1e-10 then

u(c) = log(1e-10) + 1e10*(c - 1e-10)

"""
struct LogUtility <: AbstractUtility
    ξ::Float64
end

LogUtility() = LogUtility(1.0)

(u::LogUtility)(c::Float64) =
    c > 1e-10 ? u.ξ*log(c) : u.ξ*(log(1e-10) + 1e10*(c - 1e-10))
derivative(u::LogUtility, c::Float64) =
    c > 1e-10 ? u.ξ / c : u.ξ*1e10

"""
Type used to evaluate CRRA utility. CRRA utility takes the form

u(c) = ξ c^(1 - γ) / (1 - γ)

Additionally, this code assumes that if c < 1e-10 then

u(c) = ξ (1e-10^(1 - γ) / (1 - γ) + 1e-10^(-γ) * (c - 1e-10))
"""
struct CRRAUtility <: AbstractUtility
    γ::Float64
    ξ::Float64

    function CRRAUtility(γ, ξ=1.0)
        if abs(γ - 1.0) < 1e-8
            error("Your value for γ is very close to 1... Consider using LogUtility")
        end

        return new(γ, ξ)
    end
end

(u::CRRAUtility)(c::Float64) =
    c > 1e-10 ?
           u.ξ * (c^(1.0 - u.γ) - 1.0) / (1.0 - u.γ) :
           u.ξ * ((1e-10^(1.0 - u.γ) - 1.0) / (1.0 - u.γ) + 1e-10^(-u.γ)*(c - 1e-10))
derivative(u::CRRAUtility, c::Float64) =
    c > 1e-10 ? u.ξ * c^(-u.γ) : u.ξ*1e-10^(-u.γ)

# Labor Utility

"""
Type used to evaluate constant Frisch elasticity (CFE) utility. CFE
utility takes the form

v(l) = ξ l^(1 + 1/ϕ) / (1 + 1/ϕ)

Additionally, this code assumes that if l < 1e-10 then

v(l) = ξ (1e-10^(1 + 1/ϕ) / (1 + 1/ϕ) - 1e-10^(1/ϕ) * (1e-10 - l))
"""
struct CFEUtility <: AbstractUtility
    ϕ::Float64
    ξ::Float64

    function CFEUtility(ϕ, ξ=1.0)
        if abs(ϕ - 1.0) < 1e-8
            error("Your value for ϕ is very close to 1... Consider using LogUtility")
        end

        return new(ϕ, ξ)
    end
end

(u::CFEUtility)(l::Float64) =
    l > 1e-10 ?
           -u.ξ * l^(1.0 + 1.0/u.ϕ)/(1.0 + 1.0/u.ϕ) :
           -u.ξ * (1e-10^(1.0 + 1.0/u.ϕ)/(1.0 + 1.0/u.ϕ) + 1e-10^(1.0/u.ϕ) * (l - 1e-10))
derivative(u::CFEUtility, l::Float64) =
    l > 1e-10 ? -u.ξ * l^(1.0/u.ϕ) : -u.ξ * 1e-10^(1.0/u.ϕ)

"""
Type used to evaluate elliptical utility function. Elliptical utility takes form

v(l) = b (1 - l^μ)^(1 / μ)
"""
struct EllipticalUtility <: AbstractUtility
    b::Float64
    μ::Float64
end

# These defaults are pulled straight from Evans Phillips 2017
EllipticalUtility(;b=0.5223, μ=2.2926) = EllipticalUtility(b, μ)

(u::EllipticalUtility)(l::Float64) =
    u.b * (1.0 - l^u.μ)^(1.0 / u.μ)
derivative(u::EllipticalUtility, l::Float64) =
    -u.b * (1.0 - l^u.μ)^(1.0/u.μ - 1.0) * l^(u.μ - 1.0)


# Julia translation of http://nbviewer.jupyter.org/github/barbagroup/AeroPython/blob/master/lessons/04_Lesson04_vortex.ipynb
# Lesson 3 doublet

using PyPlot
using Distributions

close("all")
meshgrid(x,y) = (repmat(x',length(y),1),repmat(y,1,length(x)))

N = 50                                # number of points in each direction
x_start, x_end = -2.0, 2.0            # boundaries in the x-direction
y_start, y_end = -1.0, 1.0            # boundaries in the y-direction
x = linspace(x_start, x_end, N)    # creates a 1D-array with the x-coordinates
y = linspace(y_start, y_end, N)    # creates a 1D-array with the y-coordinates

X,Y=meshgrid(x,y)

gamma = 5.0

x_vortex, y_vortex = 0.0, 0.0   # location of the source

function get_velocity_vortex(strength, xv, yv, X, Y):
      """Returns the velocity field generated by a vortex.

      Arguments
      ---------
      strength -- strength of the vortex.
      xv, yv -- coordinates of the vortex.
      X, Y -- mesh grid.
      """
      u = + strength/(2*pi).*(Y-yv)./((X-xv).^2+(Y-yv).^2)
      v = - strength/(2*pi).*(X-xv)./((X-xv).^2+(Y-yv).^2)

      return u, v
    end

function get_stream_function_vortex(strength, xv, yv, X, Y):
    """Returns the stream-function generated by a vortex.

    Arguments
    ---------
    strength -- strength of the vortex.
    xv, yv -- coordinates of the vortex.
    X, Y -- mesh grid.
    """
    psi = strength/(4*pi).*log((X-xv).^2+(Y-yv).^2)

    return psi
  end

  # computes the velocity field on the mesh grid
  u_vortex, v_vortex = get_velocity_vortex(gamma, x_vortex, y_vortex, X, Y)

  # computes the stream-function on the mesh grid
  psi_vortex = get_stream_function_vortex(gamma, x_vortex, y_vortex, X, Y)

  # plots the streamlines
  size = 10
  PyPlot.figure(figsize=(size, (y_end-y_start)/(x_end-x_start)*size))
  PyPlot.grid(true)
  PyPlot.xlabel("x", fontsize=16)
  PyPlot.ylabel("y", fontsize=16)
  PyPlot.xlim(x_start, x_end)
  PyPlot.ylim(y_start, y_end)
  PyPlot.streamplot(X, Y, u_vortex, v_vortex, density=2, linewidth=1, arrowsize=1, arrowstyle="->")
  PyPlot.scatter(x_vortex, y_vortex, color="#CD2305", s=80, marker="o")

strength_sink = -1.0            # strength of the sink
x_sink, y_sink = 0.0, 0.0       # location of the sink

function get_velocity_sink(strength, xs, ys, X, Y):
    """Returns the velocity field generated by a sink.

    Arguments
    ---------
    strength -- strength of the sink.
    xs, ys -- coordinates of the sink.
    X, Y -- mesh grid.
    """
    u = strength/(2*pi).*(X-xs)./((X-xs).^2+(Y-ys).^2)
    v = strength/(2*pi).*(Y-ys)./((X-xs).^2+(Y-ys).^2)

    return u, v
  end

function get_stream_function_sink(strength, xs, ys, X, Y):
      """Returns the stream-function generated by a sink.

      Arguments
      ---------
      strength -- strength of the sink.
      xs, ys -- coordinates of the sink.
      X, Y -- mesh grid.
      """
      psi = strength/(2*pi)*atan2((Y-ys), (X-xs))

      return psi
    end

    # computes the velocity field on the mesh grid
u_sink, v_sink = get_velocity_sink(strength_sink, x_sink, y_sink, X, Y)

# computes the stream-function on the mesh grid
psi_sink = get_stream_function_sink(strength_sink, x_sink, y_sink, X, Y)

# superposition of the sink and the vortex
u = u_vortex + u_sink
v = v_vortex + v_sink
psi = psi_vortex + psi_sink

# plots the streamlines
size = 10
PyPlot.figure(figsize=(size, (y_end-y_start)/(x_end-x_start)*size))
PyPlot.xlabel("x", fontsize=16)
PyPlot.ylabel("y", fontsize=16)
PyPlot.xlim(x_start, x_end)
PyPlot.ylim(y_start, y_end)
PyPlot.streamplot(X, Y, u, v, density=2, linewidth=1, arrowsize=1, arrowstyle="->")
PyPlot.scatter(x_vortex, y_vortex, color="#CD2305", s=80, marker="o");


using JuMP
using Ipopt
using CSV, DataFrames, DataStructures, MathOptFormat
using Statistics,Distributions, StatsFuns, StatsBase
using PyPlot

function critical_fractile(dist_type)
    #closed form of solution for newsvendor for normal distribution
    # dist_type - str, "norm" or "exp" for normal or exponential distribution, respectively

    b = 1 #underage cost
    h = 1 #overage cost

    if dist_type == "norm"
        mean = 50
        var = 50

        beta = b/(b+h)

        b_inv = norminvcdf(beta) #inverse of the cdf for normal distribution

        return mean + b_inv * var

    elseif dist_type == "exp"
        # plugged and chugged with cdf of exponential function to get warm start value (solver kept converging to point of infeasibility)
        cdf_func(x) = cdf(Exponential(50),x)

        # print(cdf_func(34.65))

        model3 = Model(with_optimizer(Ipopt.Optimizer,print_level=0))

        @variable(model3, x_var, start=34) #use warm start to avoid converge to infeasible point (see above)

        JuMP.register(model3, :cdf_func, 1, cdf_func; autodiff = true)  #to use cdf function in constraint

        @NLconstraint(model3,con1, cdf_func(x_var) >= b/(b+h))

        @objective(model3,Min,x_var)
        optimize!(model3)

        return value(x_var)

    else
        return nothing
    end
end

function scarf(df)
    #closed form scarf solution
    # df - dataframe, data of observed demand

    b = 1 #underage cost
    h = 1 #overage cost

    mu_hat = mean(df[:,"Column1"]) #sample mean
    sigma_sq_hat = var(df[:,"Column1"]) #sample variance

    return mu_hat + sigma_sq_hat/2 * (sqrt(b / h) - sqrt(h / b))
end

function SAA(df)
    # compute SAA solution (empirical risk minimization)
    # df - dataframe, data of observed demand

    b = 1 #underage cost
    h = 1 #overage cost

    model = Model(with_optimizer(Ipopt.Optimizer,print_level=0))

    data_size = length(df[:,"Column1"])

    @variable(model, x_var) #stocking quantity
    @variable(model, y_vars[1:data_size]) #variable to linearize max(d_i - x, x - d_i) in objective function

    @constraint(model,con1[i = 1:data_size], y_vars[i] >= x_var - df[:,"Column1"][i])
    @constraint(model, con2[i = 1:data_size], y_vars[i] >= df[:,"Column1"][i] - x_var )

    @NLobjective(model,Min, 1 / data_size * sum(y_vars[i] for i = 1:data_size))

    optimize!(model)

    return value(x_var)
end


using DifferentialEquations
using ModelingToolkit
using OrdinaryDiffEq
using DataFrames
using DataFrames
using StatsPlots
using BenchmarkTools

@parameters t β c γ
@variables S(t) I(t) R(t)
@derivatives D'~t
N=S+I+R # This is recognized as a derived variable
eqs = [D(S) ~ -β*c*I/N*S,
       D(I) ~ β*c*I/N*S-γ*I,
       D(R) ~ γ*I];

sys = ODESystem(eqs)
sys = ode_order_lowering(sys);

δt = 0.1
tmax = 40.0
tspan = (0.0,tmax)
t = 0.0:δt:tmax;

u0 = [S => 990.0,
      I => 10.0,
      R => 0.0];

p = [β=>0.05,
     c=>10.0,
     γ=>0.25];

prob_ode = ODEProblem(sys,u0,tspan,p;jac=true)

sol_ode = solve(prob_ode);

df_ode = DataFrame(sol_ode(t)')
df_ode[!,:t] = t;

@df df_ode plot(:t,
    [:x1 :x2 :x3],
    label=["S" "I" "R"],
    xlabel="Time",
    ylabel="Number")

@benchmark solve(prob_ode)

include(joinpath(@__DIR__,"tutorials","appendix.jl"))
appendix()


function generate_primes(target)
    primes = [2]
    i = 3

    while length(primes) != target
        square_root = sqrt(i)
        for prime in primes

            if prime > square_root
                push!(primes, i)
                break
            end

            if i % prime == 0
                break
            end

        end
        i += 2
    end

    return primes[end]
end

println(generate_primes(10001))


### A Pluto.jl notebook ###
# v0.15.1

using Markdown
using InteractiveUtils

# ╔═╡ 9af04825-842b-4794-a317-6f1b78a64fb5
begin
	using Statistics, CSV, DataFrames
	using ForwardDiff
end

# ╔═╡ be7baa4f-6871-4d74-b4d2-1c2534cdfa17
# Row Orientation
begin
	using Random
	Random.seed!(0)

	struct Layer_dense
		w::Matrix{Float64}
		b::Matrix{Float64}
		σ::Function

		function Layer_dense(n_in::Int, n_out::Int; σ::Function = identity)
			w = 0.01 .* randn(n_out, n_in)
			b = reshape(zeros(n_out), (1, n_out))
			return new(w, b, σ)
		end
	end

	function (L::Layer_dense)(input)
		W, b, σ = L.w, L.b, L.σ
		σ( (input * W') .+ b )
	end
end

# ╔═╡ c4ea1432-abb4-11eb-1cf5-edac0735d67d
md"# LayerDense"

# ╔═╡ 45d31083-883b-4e84-913e-c040c6fdfd5f
md"""
>No futuro, podemos pensar em novas formas de fazer esse objeto. Por exemplo, passando a matriz W como input construtor do layer.
"""

# ╔═╡ dc76be1b-292e-4545-887e-1e62faf8e1b6
md"""
### Struct by Column Orientation
"""

# ╔═╡ fe237f5f-8f90-429a-8ddc-e5933b8fd808
begin
	Random.seed!(0)

	struct LayerDense{T<:Real, F<:Function}
		W::Matrix{T}
		b::Vector{T}
		σ::F

		function LayerDense(W::Matrix{T},
							b::Vector{T},
							σ::Function = identity) where {T<:Real}
			@assert size(W, 1) == length(b)

			new{T, typeof(σ)}(W, b, σ)
		end
	end

	function LayerDense(W::Matrix{T},
						b::Vector{S},
						σ::Function = identity) where {T<:Real, S<:Real}
		R = promote_type(T, S)
		return LayerDense(Matrix{R}(W), Vector{R}(b), σ)
	end

	function LayerDense(W::Matrix{T}, σ::Function = identity) where {T<:Real}
		b = zeros(T, size(W, 1))
		return LayerDense(W, b, σ)
	end

	function LayerDense(n_in::Int, n_out::Int, σ::Function = identity)
		W = 0.01 * randn(n_out, n_in)
		b = zeros(n_out)
		return LayerDense(W, b, σ)
	end

	function (L::LayerDense)(input::AbstractVector{<:Real})
		W, b, σ = L.W, L.b, L.σ
		return σ(W * input + b)
	end

	function (L::LayerDense)(input::AbstractMatrix{<:Real})
		W, b, σ = L.W, L.b, L.σ
		return σ(W * input .+ b)
	end
end

# ╔═╡ 4e6d165a-9707-4029-af4a-2356b0fe9c41
md"#### Example using the struct LayerDense"

# ╔═╡ 5d3ae0e6-7c5d-48b7-9845-d204a1df9934
W = rand(3, 4)

# ╔═╡ e85bc0f7-65ab-4a41-a4ba-e63c8b7c8184
B = LayerDense(W) # Layer de 4 inputs e 3 outputs

# ╔═╡ 142bb8f9-4188-46d2-94a2-8407d633c496
begin
	obs01 = [2, 3, 4, 5]
	obs02 = [2, 3, 4, 5]
	input = [obs01 obs02] # Each observation must be on a colunm
end

# ╔═╡ 2377f014-0bd0-457b-b9e9-3ace1ad7c331
B(input) # Each column is an output

# ╔═╡ c99c7f4f-7e62-4036-ba08-4478e5480d19
md"#### Adding sigmoid functions"

# ╔═╡ 96ce3aa7-9261-4e60-93ce-4d9321657d85
md"""
Função **Logística**: \
$$\;\;\;\;\;$$ $$\sigma(x) = {1 \over 1 + e^{-x}}$$

Na implementação via código é necessário fazer uma adaptação que evita problemas numéricos de *Overflow* resultando na seguinte função: \
$$\sigma(x) = \left\{
  \begin{array}{lr}
    {1 \over 1 + e^{-|x|}}, & x \ge 0\\
    {e^{-|x|} \over 1 + e^{-|x|}}, & x < 0
  \end{array}
\right.$$ \

Para isso, utilizamos o módulo de $$x$$ na expressão $$e^{-|x|}$$ que possui a seguinte propriedade: \
$$e^{-|x|} = \left\{
  \begin{array}{lr}
    e^{-x}, & x \ge 0\\
    e^x, & x < 0
  \end{array}
\right.$$

Com o modulo, quando x é positivo ou zero ($$x ≥ 0$$) a adaptação resulta na própria função logística, mas quando x é negativo são necessários os seguintes passos: \
$$Se \space x \ge 0:$$  $$\;\;\;\;\;$$  $$\sigma(x) = {1 \over 1 + e^{-|x|}}$$ \
$$Se \space x < 0:$$ $$\;\;\;\;\;$$ $$1 = e^{-x} ⋅ e^x → {1 \over 1 + e^{-x}} = {e^{-x} ⋅ e^x \over e^{-x} ⋅ e^x + e^{-x}} = {e^{-x} ⋅ e^x \over e^{-x}(e^x + 1)} = {e^x \over 1 + e^x} = {e^{-|x|} \over 1 + e^{-|x|}}$$
"""

# ╔═╡ 58265283-a36a-4c83-9101-3387165da021
begin
	function σ(x::Real)
		t = exp(-abs(x))
		return ifelse(x ≥ 0, inv(1 + t), t / (1 + t))
	end

	σ(x::AbstractVecOrMat{T}) where {T<:Real} = σ.(x)

	const sigmoid = σ
end

# ╔═╡ fc1ddc84-62dc-4af1-9b9e-9d37f581a252
md"""
#### ReLU function
$$ReLU(x) = \left\{
  \begin{array}{lr}
    x, & x \ge 0\\
    0, & x < 0
  \end{array}
\right.$$
$$Or$$
$$ReLU(x) = max(0, x)$$
"""

# ╔═╡ 45bafada-34ec-4c41-b0a2-000f35ebffef
begin
	relu(x::Real) = max(zero(x), x) # zero function keep the type of x
	relu(x::AbstractVecOrMat{<:Real}) = relu.(x)
end

# ╔═╡ c85cd202-7223-4c46-80ef-afd750f28ead
md"""
$$softplus(x) = ln(1 + e^x)$$
$$For \space x > 0:$$
$$1 = e^{-x} ⋅ e^x → ln(1 + e^x) = ln(e^{-x} ⋅ e^x + e^x) = ln(e^x(e^{-x} + 1)) = ln(e^x) + ln(1 + e^{-x})$$
$$softplus(x) = x + ln(1 + e^{-x})$$
"""

# ╔═╡ d6fce490-1498-469d-8ecc-55d0a71d1565
begin
	#log1p(x) is the seme as log(1 + x)
	softplus(x::Real) = ifelse(x > 0, x + log(1 + exp(-x)), log(1 + exp(x)))
	softplus(x::AbstractVecOrMat{<:Real}) = softplus.(x)
end

# ╔═╡ 04cadf6a-4fd1-4b5f-aa4d-140ea6e5cff2
function softmax(x::Vector{T}) where {T<:Real}
	m = maximum(x)
	exp_val = exp.(x .- m)
	s = sum(exp_val)
	return exp_val ./ s
end

# ╔═╡ a7898ffd-1acb-4220-bbc8-3b428b95db85
function softmax(x::Matrix{T}) where {T<:Real}
	m = maximum(x, dims = 1)
	exp_val = exp.(x .- m)
	s = sum(exp_val, dims = 1)
	return exp_val ./ s
end

# ╔═╡ c116a1e4-fd4d-446f-bdea-02301f43b2c0
md"##### Example with sigmoid functions"

# ╔═╡ d0c91a1c-8798-4985-978b-4c47d1abc212
C = LayerDense(2, 3, σ)

# ╔═╡ 3de0841e-f0f3-436c-a891-dd4af32d2af6
# C(randn(2, 3))
C([[1, 4] [2, 3] [3, 5]])

# ╔═╡ 963f6276-2440-4bfe-ae0f-539d8bfae0a2
D = LayerDense(2, 3, softmax)

# ╔═╡ 56191b0c-fe49-4ed4-9ca7-2e22e7b028ba
md"""
#### Cost Function
"""

# ╔═╡ 7b5a6859-a9e1-48b1-a0fd-1cde06249d03
begin
	function xlogy(x::Real, y::Real)
		result = x * log(y)
		ifelse(iszero(x) && !isnan(y), zero(result), result)
	end

	function crossentropy(ŷ::AbstractVecOrMat{<:Real},
						  y::AbstractVecOrMat{<:Real};
						  dims::Int = 1,
						  agg::Function = mean)
    	agg(.-sum(xlogy.(y, ŷ), dims = dims))
	end
end

# ╔═╡ 4b070348-fdbc-46cd-85f0-29b64432ae21
begin
	outp2 = D([1 4; 2 3; 3 5]')
	target2 = [0, 1, 0]
	crossentropy(outp2, target2)
end

# ╔═╡ e3ec0333-14ad-42a8-9cd3-4e3937dc5ff0
begin
	outp1 = [0.7 0.1 0.2
			 0.1 0.5 0.4
	   		 0.02 0.9 0.08]'
	target1 = [1 0 0
			   0 1 0
		  	   0 1 0]'
	crossentropy(outp1, target1)
end

# ╔═╡ 8f3cbc01-53ba-4ff0-8245-564f8fee53f2
md"##### Importing data from nnfs python package"

# ╔═╡ 099f5fa8-6687-4c77-9f61-119d75377bab
data = CSV.read("spiral_data.csv", DataFrame)

# ╔═╡ b5981a68-16e7-451f-9e13-d74b2b78a958
md"#### Chain type, params(), gradient()"

# ╔═╡ d2fa0422-7f39-4365-8f3b-bfa6997ae0ae
begin
	Random.seed!(1998)

	L1 = LayerDense(10, 5, σ)
	L2 = LayerDense(5, 2, softmax)

	output = L2(L1(rand(10)))
end

# ╔═╡ 72f68243-8ae1-465c-854d-a76c07c5e346
function params(layers)
	p = []
	for layer in layers
		push!(p, layer.W[:], layer.b )
	end
	return vcat(p...)
end

# ╔═╡ e84cff98-1bea-42cb-8e8d-6f5ef08226f4
begin
	Θ = params([L1, L2])

	U = (
		size(L1.W, 1)*size(L1.W, 2), length(L1.b),
		size(L2.W, 1)*size(L2.W, 2), length(L2.b),
	)
	U = cumsum(U)

	L1w = reshape( Θ[ 1:U[1] ], size(L1.W, 1), size(L1.W, 2))

	L1b = Θ[ U[1]+1:U[2] ]

	L2w = reshape( Θ[ U[2]+1:U[3] ], size(L2.W, 1), size(L2.W, 2))

	L2b = Θ[ U[3]+1:U[4] ]

	( length(Θ), U, (L1w, L1b, L2w, L2b) )
end

# ╔═╡ a5a511f7-38f0-4074-9192-7f11d611bf42
begin

	struct Chain2
		Layers::Vector{LayerDense{Float64}}
		Wl::Vector{Int}
		Wc::Vector{Int}
		b::Vector{Int}

		function Chain(L...)
			Wl = Int[]
			Wc = Int[]
			b = Int[]
			for (i, l) in enumerate(L)
				Wl[i], Wc[i] = size(l.W)
				b[i] = length(l.b)
			end
			return new{}(L, Wl, Wc, b)
		end

	end

end

# ╔═╡ 154e4560-0769-429e-89bd-3b0392209d6d
typeof( (1.3,2.12) )

# ╔═╡ 74ccd8c6-f513-4178-a41f-7b70835516e7
begin
	struct Chain{T<:Tuple}
		layers::T
		Chain(layers...) = new{typeof(layers)}(layers)
	end

	function (L::Chain)(input::AbstractVecOrMat{<:Real})
		return ∘(reverse(L.layers)...)(input)
	end
end

# ╔═╡ 4504eaff-0c86-4606-a410-c94a8e8a653e
begin
	#L1 = LayerDense(10, 5, σ)
	#L2 = LayerDense(5, 2, softmax)

	C1 = Chain(L1, L2)
	typeof(C1.layers)
end

# ╔═╡ ac678c79-a804-44e0-80d8-15a40dff327d
begin
	p = []
	for layer in C1.layers
		push!(p, layer.W[:], layer.b )
	end
	vcat(p...)
end

# ╔═╡ 4b019add-7eb4-496a-b0ef-3c3eee9d85bd
begin
	C1(rand(10,12))
end

# ╔═╡ 00000000-0000-0000-0000-000000000001
PLUTO_PROJECT_TOML_CONTENTS = """
[deps]
CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

[compat]
CSV = "~0.8.5"
DataFrames = "~1.2.2"
ForwardDiff = "~0.10.19"
"""

# ╔═╡ 00000000-0000-0000-0000-000000000002
PLUTO_MANIFEST_TOML_CONTENTS = """
# This file is machine-generated - editing it directly is not advised

[[ArgTools]]
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"

[[Artifacts]]
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"

[[Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[CSV]]
deps = ["Dates", "Mmap", "Parsers", "PooledArrays", "SentinelArrays", "Tables", "Unicode"]
git-tree-sha1 = "b83aa3f513be680454437a0eee21001607e5d983"
uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
version = "0.8.5"

[[ChainRulesCore]]
deps = ["Compat", "LinearAlgebra", "SparseArrays"]
git-tree-sha1 = "bdc0937269321858ab2a4f288486cb258b9a0af7"
uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
version = "1.3.0"

[[CommonSubexpressions]]
deps = ["MacroTools", "Test"]
git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7"
uuid = "bbf7d656-a473-5ed7-a52c-81e309532950"
version = "0.3.0"

[[Compat]]
deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
git-tree-sha1 = "727e463cfebd0c7b999bbf3e9e7e16f254b94193"
uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
version = "3.34.0"

[[CompilerSupportLibraries_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"

[[Crayons]]
git-tree-sha1 = "3f71217b538d7aaee0b69ab47d9b7724ca8afa0d"
uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
version = "4.0.4"

[[DataAPI]]
git-tree-sha1 = "ee400abb2298bd13bfc3df1c412ed228061a2385"
uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
version = "1.7.0"

[[DataFrames]]
deps = ["Compat", "DataAPI", "Future", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrettyTables", "Printf", "REPL", "Reexport", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"]
git-tree-sha1 = "d785f42445b63fc86caa08bb9a9351008be9b765"
uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
version = "1.2.2"

[[DataStructures]]
deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
git-tree-sha1 = "7d9d316f04214f7efdbb6398d545446e246eff02"
uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
version = "0.18.10"

[[DataValueInterfaces]]
git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464"
version = "1.0.0"

[[Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"

[[DelimitedFiles]]
deps = ["Mmap"]
uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"

[[DiffResults]]
deps = ["StaticArrays"]
git-tree-sha1 = "c18e98cba888c6c25d1c3b048e4b3380ca956805"
uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5"
version = "1.0.3"

[[DiffRules]]
deps = ["NaNMath", "Random", "SpecialFunctions"]
git-tree-sha1 = "3ed8fa7178a10d1cd0f1ca524f249ba6937490c0"
uuid = "b552c78f-8df3-52c6-915a-8e097449b14b"
version = "1.3.0"

[[Distributed]]
deps = ["Random", "Serialization", "Sockets"]
uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"

[[DocStringExtensions]]
deps = ["LibGit2"]
git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f"
uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
version = "0.8.5"

[[Downloads]]
deps = ["ArgTools", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"

[[Formatting]]
deps = ["Printf"]
git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8"
uuid = "59287772-0a20-5a39-b81b-1366585eb4c0"
version = "0.4.2"

[[ForwardDiff]]
deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "NaNMath", "Printf", "Random", "SpecialFunctions", "StaticArrays"]
git-tree-sha1 = "b5e930ac60b613ef3406da6d4f42c35d8dc51419"
uuid = "f6369f11-7733-5829-9624-2563aa707210"
version = "0.10.19"

[[Future]]
deps = ["Random"]
uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"

[[InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

[[InvertedIndices]]
deps = ["Test"]
git-tree-sha1 = "15732c475062348b0165684ffe28e85ea8396afc"
uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f"
version = "1.0.0"

[[IrrationalConstants]]
git-tree-sha1 = "f76424439413893a832026ca355fe273e93bce94"
uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
version = "0.1.0"

[[IteratorInterfaceExtensions]]
git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
uuid = "82899510-4779-5014-852e-03e436cf321d"
version = "1.0.0"

[[JLLWrappers]]
deps = ["Preferences"]
git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e"
uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
version = "1.3.0"

[[LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"

[[LibCURL_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"

[[LibGit2]]
deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"

[[LibSSH2_jll]]
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"

[[Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

[[LinearAlgebra]]
deps = ["Libdl"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[LogExpFunctions]]
deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"]
git-tree-sha1 = "3d682c07e6dd250ed082f883dc88aee7996bf2cc"
uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
version = "0.3.0"

[[Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"

[[MacroTools]]
deps = ["Markdown", "Random"]
git-tree-sha1 = "0fb723cd8c45858c22169b2e42269e53271a6df7"
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
version = "0.5.7"

[[Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"

[[MbedTLS_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"

[[Missings]]
deps = ["DataAPI"]
git-tree-sha1 = "2ca267b08821e86c5ef4376cffed98a46c2cb205"
uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
version = "1.0.1"

[[Mmap]]
uuid = "a63ad114-7e13-5084-954f-fe012c677804"

[[MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"

[[NaNMath]]
git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb"
uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
version = "0.3.5"

[[NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"

[[OpenSpecFun_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1"
uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
version = "0.5.5+0"

[[OrderedCollections]]
git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c"
uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
version = "1.4.1"

[[Parsers]]
deps = ["Dates"]
git-tree-sha1 = "bfd7d8c7fd87f04543810d9cbd3995972236ba1b"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "1.1.2"

[[Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

[[PooledArrays]]
deps = ["DataAPI", "Future"]
git-tree-sha1 = "a193d6ad9c45ada72c14b731a318bedd3c2f00cf"
uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720"
version = "1.3.0"

[[Preferences]]
deps = ["TOML"]
git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a"
uuid = "21216c6a-2e73-6563-6e65-726566657250"
version = "1.2.2"

[[PrettyTables]]
deps = ["Crayons", "Formatting", "Markdown", "Reexport", "Tables"]
git-tree-sha1 = "0d1245a357cc61c8cd61934c07447aa569ff22e6"
uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
version = "1.1.0"

[[Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"

[[REPL]]
deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[Random]]
deps = ["Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[Reexport]]
git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
version = "1.2.2"

[[SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"

[[SentinelArrays]]
deps = ["Dates", "Random"]
git-tree-sha1 = "54f37736d8934a12a200edea2f9206b03bdf3159"
uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c"
version = "1.3.7"

[[Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

[[SharedArrays]]
deps = ["Distributed", "Mmap", "Random", "Serialization"]
uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"

[[Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

[[SortingAlgorithms]]
deps = ["DataStructures"]
git-tree-sha1 = "b3363d7460f7d098ca0912c69b082f75625d7508"
uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
version = "1.0.1"

[[SparseArrays]]
deps = ["LinearAlgebra", "Random"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

[[SpecialFunctions]]
deps = ["ChainRulesCore", "LogExpFunctions", "OpenSpecFun_jll"]
git-tree-sha1 = "a322a9493e49c5f3a10b50df3aedaf1cdb3244b7"
uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
version = "1.6.1"

[[StaticArrays]]
deps = ["LinearAlgebra", "Random", "Statistics"]
git-tree-sha1 = "3240808c6d463ac46f1c1cd7638375cd22abbccb"
uuid = "90137ffa-7385-5640-81b9-e52037218182"
version = "1.2.12"

[[Statistics]]
deps = ["LinearAlgebra", "SparseArrays"]
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

[[TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"

[[TableTraits]]
deps = ["IteratorInterfaceExtensions"]
git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39"
uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
version = "1.0.1"

[[Tables]]
deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"]
git-tree-sha1 = "d0c690d37c73aeb5ca063056283fde5585a41710"
uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
version = "1.5.0"

[[Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"

[[Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[[Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

[[Zlib_jll]]
deps = ["Libdl"]
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"

[[nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"

[[p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
"""

# ╔═╡ Cell order:
# ╟─c4ea1432-abb4-11eb-1cf5-edac0735d67d
# ╟─45d31083-883b-4e84-913e-c040c6fdfd5f
# ╟─dc76be1b-292e-4545-887e-1e62faf8e1b6
# ╠═fe237f5f-8f90-429a-8ddc-e5933b8fd808
# ╟─4e6d165a-9707-4029-af4a-2356b0fe9c41
# ╠═5d3ae0e6-7c5d-48b7-9845-d204a1df9934
# ╠═e85bc0f7-65ab-4a41-a4ba-e63c8b7c8184
# ╠═142bb8f9-4188-46d2-94a2-8407d633c496
# ╠═2377f014-0bd0-457b-b9e9-3ace1ad7c331
# ╟─c99c7f4f-7e62-4036-ba08-4478e5480d19
# ╟─96ce3aa7-9261-4e60-93ce-4d9321657d85
# ╠═58265283-a36a-4c83-9101-3387165da021
# ╟─fc1ddc84-62dc-4af1-9b9e-9d37f581a252
# ╠═45bafada-34ec-4c41-b0a2-000f35ebffef
# ╟─c85cd202-7223-4c46-80ef-afd750f28ead
# ╠═d6fce490-1498-469d-8ecc-55d0a71d1565
# ╠═04cadf6a-4fd1-4b5f-aa4d-140ea6e5cff2
# ╠═a7898ffd-1acb-4220-bbc8-3b428b95db85
# ╟─c116a1e4-fd4d-446f-bdea-02301f43b2c0
# ╠═d0c91a1c-8798-4985-978b-4c47d1abc212
# ╠═3de0841e-f0f3-436c-a891-dd4af32d2af6
# ╠═963f6276-2440-4bfe-ae0f-539d8bfae0a2
# ╟─56191b0c-fe49-4ed4-9ca7-2e22e7b028ba
# ╠═7b5a6859-a9e1-48b1-a0fd-1cde06249d03
# ╠═4b070348-fdbc-46cd-85f0-29b64432ae21
# ╠═e3ec0333-14ad-42a8-9cd3-4e3937dc5ff0
# ╟─8f3cbc01-53ba-4ff0-8245-564f8fee53f2
# ╟─099f5fa8-6687-4c77-9f61-119d75377bab
# ╠═9af04825-842b-4794-a317-6f1b78a64fb5
# ╟─b5981a68-16e7-451f-9e13-d74b2b78a958
# ╠═d2fa0422-7f39-4365-8f3b-bfa6997ae0ae
# ╠═ac678c79-a804-44e0-80d8-15a40dff327d
# ╠═72f68243-8ae1-465c-854d-a76c07c5e346
# ╠═e84cff98-1bea-42cb-8e8d-6f5ef08226f4
# ╠═a5a511f7-38f0-4074-9192-7f11d611bf42
# ╠═154e4560-0769-429e-89bd-3b0392209d6d
# ╠═74ccd8c6-f513-4178-a41f-7b70835516e7
# ╠═4504eaff-0c86-4606-a410-c94a8e8a653e
# ╠═4b019add-7eb4-496a-b0ef-3c3eee9d85bd
# ╟─be7baa4f-6871-4d74-b4d2-1c2534cdfa17
# ╟─00000000-0000-0000-0000-000000000001
# ╟─00000000-0000-0000-0000-000000000002


export tanhActivation, tanhActivation!

"""
 hyperbolic tan activation A = tanh(Y)

 Input:

   Y - array of features

 Optional Input:

   doDerivative - flag for computing derivative, set via varargin
                  Ex: tanhActivation(Y,true);

 Output:

  A  - activation
  dA - derivatives
"""
function tanhActivation(Y::Array{T,2},doDerivative::Bool=false) where {T <: Number}

    A = tanh.(Y)
    dA = zeros(A)
    if doDerivative
         dA .= one(T) .- A.^2
    end
    return A, dA
end

function tanhActivation!(A::Array{T,2},dA=[],doDerivative::Bool=false) where {T <: Number}

    A .= tanh.(A)
    if doDerivative
        if isempty(dA)
            dA = one(T) .- A.^2
        else
            dA .= one(T) .- A.^2
        end
    end
    return A, dA
end


# # Minimizing the sum of the k-largest λ
#
# We show how to find the sum of absolute value of the `k` largest eigenvalues of a symmetric matrix $A \in \mathbb{S}^n$. This problem can be solved as a semidefinite program. The primal and dual forms are stated in *Alizadeh* \[1\]:
# $$
# \begin{array}{llll} \text{maximize} &   \text{Tr}(AY) - \text{Tr}(AW) &    \text{minimize} &  kz + Tr(U) + Tr(V)            \\
# \text{subject to} &  \text{Tr}(Y + W) = k                             &   \text{subject to} &   zI + V - A \succeq 0            \\
#                   &  0 \preceq  Y \preceq I                           &  &                    zI + U + A \succeq 0             \\
#                   &  0 \preceq  W \preceq I                           & &                     U, V \succeq 0,
# \end{array}
# $$
# where $Y, W$ are the variables of the primal and $U, V$ are the variables of the dual problem.
#-
using LinearAlgebra, JuMP, COSMO, Random
rng = Random.MersenneTwister(212)

n = 10
A = 5 .* randn(rng, 10, 10)
A = Symmetric(A, :U)

# We are interested in minimizing the sum of absolute values of the `k=3` largest eigenvalues. Let's formulate the problem in `JuMP` with `COSMO` as the backend solver:

k = 3
model = JuMP.Model(optimizer_with_attributes(COSMO.Optimizer, "verbose" => true));
@variable(model, Y[1:n, 1:n], PSD);
@variable(model, W[1:n, 1:n], PSD);

@objective(model, Max, tr(A * Y) - tr(A * W));
@constraint(model, tr(Y + W) == k);
@constraint(model, Symmetric(I - Y) in PSDCone());
@constraint(model, Symmetric(I - W) in PSDCone());
status = JuMP.optimize!(model)

#-
opt_objective = JuMP.objective_value(model)

# Now, we can check the solution by computing the sum of the absolute value of the 3-largest eigenvalues:
k_λ_abs = sum(sort(abs.(eigen(A).values), rev = true)[1:k])

# ### Solve the dual
#
# Alternatively, we can solve the dual problem:
#-
model = JuMP.Model(with_optimizer(COSMO.Optimizer, verbose=true));
@variable(model, V[1:n, 1:n], PSD);
@variable(model, U[1:n, 1:n], PSD);
@variable(model, z);

@objective(model, Min, k * z + tr(V) + tr(U));
@constraint(model, Symmetric(z .* diagm(0 => ones(n)) + V - A) in PSDCone());
@constraint(model, Symmetric(z .* diagm(0 => ones(n)) + U + A) in PSDCone());
status = JuMP.optimize!(model)

#-
opt_objective = JuMP.objective_value(model)

# This gives the same result.
# ## Problem with A as variable
#
# Above problems are mostly helpful for illustrative purpose. It is obviously easier to find the sum of the k-largest eigenvalues by simply computing the eigenvalues of $A$. However, above results become useful if finding $A$ itself is part of the problem. For example, assume we want to find a valid matrix $A$ under the constraints: $C\, \text{vec}(A) = b$ with the minimum sum of absolute values of the k-largest eigenvalues. We can then solve the equivalent problem:
# $$
# \begin{array}{ll} \text{minimize} &  kz + Tr(U) + Tr(V)     \\
#  \text{subject to} &   C \text{vec}(A) = b \\
#                    & zI + V - A \succeq 0            \\
#                    &    zI + U + A \succeq 0             \\
#                    &   U, V \succeq 0.
# \end{array}
# $$
#
# ## References
# [1] Alizadeh - Interior point methods in semidefinite programming with applications to combinatorial optimization (1995)


using CompScienceMeshes
using SauterSchwabQuadrature

pI = point(1,5,3)
pII = point(2,5,3)
pIII = point(7,1,0)

Sourcechart = simplex(pI, pII, pIII)
Testchart = simplex(pII, pIII, pI)

Accuracy = 12

function integrand(x,y)
			return(((x-pI)'*(y-pII))*exp(-im*1*norm(x-y))/(4pi*norm(x-y)))
end

result = sauterschwabintegral(Sourcechart, Testchart, integrand, Accuracy, Accuracy)
println(result)

#=For those who want to test the sauterschwab_nonparameterized() function,
may uncomment the following five lines=#

#sourcechart = simplex(pI,pII,pIII)
#testchart = simplex(pI,pII,pIII)
#cf = CommonFace(Accuracy)
#result2 = sauterschwab_nonparameterized(sourcechart, testchart, integrand, cf)
#println(result2)

#=sourcechart = testchart, hence the required condition is fulfilled.
The user may also compare the two results and see that both are equal=#


function normalize(array, target_range)
    target_min, target_max = target_range
    orig_min, orig_max = minimum(array), maximum(array)
    normalized_array = target_min .+ ((target_max-target_min).*(array.-orig_min))./(orig_max-orig_min)
    return normalized_array
end
a = [1,2,3,4]
print(normalize(a, [-1,1]))


## Nerlove model by OLS (fminunc)
# and fmincon, imposing that factor shares are in (0,1) and sum to 1
using DelimitedFiles, Econometrics
## prepare the data
cd(@__DIR__)
data = readdlm("../Data/nerlove.data")
data = log.(data[:,2:end])
n = size(data,1)
y = data[:,1]
x = [ones(n,1) data[:,2:5]]

## bounds and restriction
lb = [-1e6, -1e6, 0., 0., 0.0]
ub = [1e6, 1e6, 1., 1., 1.]
R = [0. 0. 1. 1. 1.]
r = [1.0]

## define the objective function and start value
obj = theta -> (y-x*theta)'*(y-x*theta)
startval = (ub+lb)/2.0

## OLS
thetahat, objvalue = fminunc(obj, startval)
println("the OLS estimates: obj. value: ", round(objvalue,digits=5))
prettyprint(thetahat)

## restricted LS
thetahat, objvalue_r, flag = fmincon(obj, startval, R, r, lb, ub) # both lower and upper bounds
println("the restricted LS estimates: obj. value: ", round(objvalue_r,digits=5))
prettyprint(thetahat)

println("Exercise: you should construct a qF test using the unrestricted and")
println("restricted objective function values")


function hailstonelength(n::Integer)
    len = 1
    while n > 1
        n = ifelse(iseven(n), n ÷ 2, 3n + 1)
        len += 1
    end
    return len
end

@show hailstonelength(27); nothing
@show findmax([hailstonelength(i) for i in 1:100_000]); nothing