#=
Polynomial operations made over unsigned integers in extension fields GF(2^m).

Author: Ivan A. Moreno Soto
Last updated: 2019/July/08
=#

module Polynomials

export
    PolynomialConstants,
    monomial,
    evaluate!,
    addRing!,
    multiplyRing!,
    divRing!,
    modRing!,
    makePolynomialMonic!,
    inverseRing!,
    isPolynomialIrreducible!,
    randomPoly,
    splitPolynomial!

include("./ExtensionField.jl")
using .ExtensionField

PolynomialConstants = Dict(
    '0' => Array{UInt32, 1}([0]),
    '1' => Array{UInt32, 1}([1]),
    'x' => Array{UInt32, 1}([0, 1])
)

"""
    monomial(degree)

Returns a monomial of the given degree.
"""
function monomial(degree)
    monomial = Array{UInt32, 1}([0 for _ in 0:degree])
    monomial[end] = 1
    return monomial
end

"""
    evaluate!(x, p, g)

Evaluates p(x) modulo g(x).
"""
function evaluate!(x, p, g)
    y = UInt32(0)

    for i in length(p):-1:1
        y = add(modulo(multiply(x, y), g), p[i])
    end

    return y
end

"""
    reducePolynomialArray!(p)

Takes a polynomial p and returns an array without unnecessary zero
coefficients.
"""
function reducePolynomialArray!(p)
    if length(p) == 1
        return p
    elseif !any(x -> x > 0, p)
        return [p[1]]
    else
        return p[1:findlast(x -> x > 0, p)]
    end
end

"""
    addRing!(a, b)

Adds two polynomials with coefficients in GF(2^m).
"""
function addRing!(a, b)
    if length(a) >= length(b)
        longPolynomial = a
        shortPolynomial = b
    else
        longPolynomial = b
        shortPolynomial = a
    end

    total = Array{UInt32, 1}([coeff for coeff in longPolynomial])

    for i in 1:length(shortPolynomial)
        total[i] = add(shortPolynomial[i], longPolynomial[i])
    end

    return reducePolynomialArray!(total)
end

"""
    multiplyRing!(a, b, m)

Multiplies two polynomials with coefficients in GF(2^m).
"""
function multiplyRing!(a, b, m)
    product = Array{UInt32, 1}([0 for _ in 1:(length(a) + length(b) - 1)])

    for i in 1:length(a)
        for j in 1:length(b)
            term = modulo(multiply(a[i], b[j]), primitivePolynomials[m])
            product[i+j - 1] = add(product[i+j - 1], term)
        end
    end

    return reducePolynomialArray!(product)
end

"""
    polynomialLongDivision!(a, b, m)

Returns the quotient and the modulo of a/b with coefficients
in GF(2^m).
"""
function polynomialLongDivision!(a, b, m)
    if a == b
        return Array{UInt32, 1}([1]), Array{UInt32, 1}([0])
    elseif length(a) < length(b) #a < b
        return Array{UInt32, 1}([0]), a
    end

    quotient = Array{UInt32, 1}([])
    remainder = a
    for i in (length(a) - length(b)):-1:0
        if length(remainder) >= length(b) + i
            step = Array{UInt32, 1}([0 for _ in 0:i])
            step[end] = modulo(multiply(inverse(b[end], primitivePolynomials[m]), remainder[end]), primitivePolynomials[m])

            partial = multiplyRing!(b, step, m)

            remainder = addRing!(remainder, partial)
            push!(quotient, step[end])
        else
            push!(quotient, 0)
        end
    end

    reverse!(quotient) # push! adds the coefficients in reverse order.

    return reducePolynomialArray!(quotient), reducePolynomialArray!(remainder)
end

"""
    divRing!(a, b, m)

Returns the quotient of a/b with coefficients in GF(2^m).
"""
function divRing!(a, b, m)
    quotient, _ = polynomialLongDivision!(a, b, m)
    return quotient
end

"""
    modRing!(a, b, m)

Returns the remainder of a/b with coefficients in GF(2^m).
"""
function modRing!(a, b, m)
    _, remainder = polynomialLongDivision!(a, b, m)
    return remainder
end

"""
    makePolynomialMonic!(p, m)

Returns p as a monic polynomial by dividing p by the inverse of its leading
coefficient.
"""
function makePolynomialMonic!(p, m)
    return multiplyRing!([inverse(p[end], primitivePolynomials[m])], p, m)
end

"""
    ringgcs!(a, b, m)

Computes the GCD of a and b with coefficients in GF(2^m).
"""
function ringgcd!(a, b, m)
    while b != PolynomialConstants['0']
        t = copy(b)
        b = modRing!(a, b, m)
        a = t
    end

    return makePolynomialMonic!(a, m)
end

"""
    inverseRing!(a, g, m)

Computes the inverse of a mod g with coefficients in GF(2^m). Throws
an exception if the inverse doesn't exist.
"""
function inverseRing!(a, g, m)
    # Extended euclidean algorithm.
    r = copy(a)
    R = copy(g)
    s = Array{UInt32, 1}([1])
    S = Array{UInt32, 1}([0])
    t = Array{UInt32, 1}([0])
    T = Array{UInt32, 1}([1])

    while R != Array{UInt32, 1}([0])
        q = divRing!(r, R, m)

        x = R; R = addRing!(r, multiplyRing!(q, R, m)); r = x
        x = S; S = addRing!(s, multiplyRing!(q, S, m)); s = x
        x = T; T = addRing!(t, multiplyRing!(q, T, m)); t = x
    end

    c = inverse(r[end], primitivePolynomials[m])

    if makePolynomialMonic!(r, m) != PolynomialConstants['1']
        error("The element doesn't have an inverse.")
    end

    return multiplyRing!([c], s, m)
end

"""
    isPolynomialIrreducible!(polynomial, n, m)

Checks if the given polynomial has at least one root in GF(2^m). Returns
true if it doesn't have roots, false otherwise.
"""
function isPolynomialIrreducible!(polynomial, n, m)
    for i in 0:(2^m - 1)
        if evaluate!(UInt32(i), polynomial, primitivePolynomials[m]) == 0
            return false
        end
    end

    return true
end

"""
    getRandomCoefficients(t, m)

Returns a random sequence of elements from GF(2^m)
to serve as the coefficients of polynomial of degree t.
"""
function getRandomCoefficients(t, m)
    polynomial = [UInt32(0) for _ in 1:(t + 1)]
    polynomial[end] = UInt32(1) # Making the polynomial monic.
    # Adding random coefficients to the polynomial.
    for i in 1:t
        if rand() >= 0.5
            polynomial[i] = UInt32(rand(1:(2^m - 1)))
        end
    end

    return polynomial
end

"""
    randomPoly(t::Int, m::Int)

Returns a random monic irreducible polynomial of degree t
over the extension field GF(2^m).
"""
function randomPoly(t::Int, m::Int)
    polynomial = getRandomCoefficients(t, m)

    while !isPolynomialIrreducible!(polynomial, t, m)
        polynomial = getRandomCoefficients(t, m)
    end

    return polynomial
end

"""
    splitPolynomial!(p, m)

Splits the polynomial p into even and odd terms in GF(2^m).
"""
function splitPolynomial!(p, m)
    even = [squareroot(coeff, m) for coeff in p[1:2:end]]
    odd = [squareroot(coeff, m) for coeff in p[2:2:end]]
    return even, odd
end

end # Module Polynomials


#== # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# Description
#
#   Functions to convert anomalies related to the orbit.
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# References
#
#   [1] Vallado, D. A (2013). Fundamentals of Astrodynamics and Applications.
#       Microcosm Press, Hawthorn, CA, USA.
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # ==#

export M_to_E, M_to_f
export E_to_f, E_to_M
export f_to_E, f_to_M

################################################################################
#                              From Mean Anomaly
################################################################################

"""
    M_to_E(e::Number, M::Number, tol::Number = 1e-10)

Compute the eccentric anomaly (0,2π) \\[rad] given the eccentricity `e` and the
mean anomaly `M` [rad]. This function uses the Newton-Raphson algorithm and the
tolerance to accept the solution is `tol`.

"""
@inline function M_to_E(e::Number, M::Number, tol::Number = 1e-10)
    # Compute the eccentric anomaly using the Newton-Raphson method.
    # ==============================================================

    # Make sure that M is in the interval [0,2π].
    M = mod(M,2π)

    # Initial guess.
    #
    # See [1, p. 75].
    E = (M > π) ? M - e : M + e

    sin_E, cos_E = sincos(E)

    # Newton-Raphson iterations.
    while ( abs(E - e*sin_E - M) > tol )
        E = E - (E - e*sin_E - M)/(1-e*cos_E)

        sin_E, cos_E = sincos(E)
    end

    # Return the eccentric anomaly in the interval [0, 2π].
    mod(E, 2π)
end

"""
    M_to_f(e::Number, M::Number, tol::Number = 1e-10)

Compute the true anomaly (0,2π) \\[rad] given the eccentricity `e` and the mean
anomaly `M` [rad]. This function uses the Newton-Raphson algorithm and the
tolerance to accept the solution is `tol`.

"""
@inline function M_to_f(e::Number, M::Number, tol::Number = 1e-10)
    # Compute the eccentric anomaly.
    E = M_to_E(e, M, tol)

    # Compute the true anomaly in the interval [0,2π].
    E_to_f(e,E)
end

################################################################################
#                            From Eccentric Anomaly
################################################################################

"""
    E_to_f(e::Number, E::Number)

Compute the true anomaly (0,2π) \\[rad] given the eccentricity `e` and the
eccentric anomaly `E` [rad].

"""
@inline function E_to_f(e::Number, E::Number)
    sin_Eo2, cos_Eo2 = sincos(E/2)

    # Compute the true anomaly in the interval [0, 2*π].
    mod( 2atan(sqrt(1+e)*sin_Eo2, sqrt(1-e)*cos_Eo2) , 2π )
end

"""
    E_to_M(e::Number, E::Number)

Compute the mean anomaly (0,2π) \\[rad] given the eccentricity `e` and the
eccentric anomaly `E` [rad].

"""
E_to_M(e::Number, E::Number) = mod(E - e*sin(E), 2π)

################################################################################
#                              From True Anomaly
################################################################################

"""
    f_to_E(e::Number,f::Number)

Compute the eccentric anomaly (0,2π) \\[rad] given the eccentricity `e` and
the true anomaly `f` [rad].

"""
@inline function f_to_E(e::Number, f::Number)
    sin_fo2, cos_fo2 = sincos(f/2)

    mod( 2atan(sqrt(1-e)*sin_fo2, sqrt(1+e)*cos_fo2), 2π )
end

"""
    f_to_E(orb::Orbit)

Compute the eccentric anomaly (0,2π) \\[rad] given the orbit `orb` (see
`Orbit`).

"""
f_to_E(orb::Orbit) = f_to_E(orb.e, orb.f)

"""
    f_to_M(e::Number, f::Number)

Compute the mean anomaly (0,2π) \\[rad] given the eccentricity `e` and the
true anomaly `f` [rad].

"""
@inline function f_to_M(e::Number, f::Number)
    # Compute the eccentric anomaly.
    E = f_to_E(e, f)

    # Compute the true anomaly in the interval [0, 2π].
    E_to_M(e,E)
end

"""
    f_to_M(orb::Orbit)

Compute the mean anomaly (0,2π) \\[rad] given the orbit `orb` (see `Orbit`).

"""
f_to_M(orb::Orbit) = f_to_M(orb.e, orb.f)


export
    scale2log,
    normlog,
    sph2cart,
    cart2sph,
    mod_angle,
    mod_angle_deg,
    mod_angle_rad

# operation
scale2log(x) = 20log10(x)
normlog(vec) = vec .- maximum(vec)
sph2cart(θ, ϕ, r) = begin
    (
        x=r * sin(θ)cos(ϕ),
        y=r * sin(θ)sin(ϕ),
        z=r * cos(θ)
    )
end
cart2sph(x, y, z) = begin
    # XXX r are not calcuated, may cause problem
    r = sqrt(x^2 + y^2 + z^2)
    θ = acos(z / r)
    ϕ = atan(y, x)
    (
        θ=θ,
        ϕ=ϕ,
        r=r
    )
end
# convert to domain θ ∈[0°, 180°], ϕ∈ [-180°, 180°]
# bad implementation
function mod_angle(θ, ϕ)
    sign = x -> x >= 0 ? 1 : -1
    item1 = -sign.(mod2pi.(θ) .- π)
    θ_ = mod1.(θ .* item1, π)
    θ_[θ.==0] .= 0
    ϕ_ = item1 .* ϕ .|> mod2pi
    (θ_, ϕ_)
end
function mod_angle(θ::Number, ϕ)
    sign = x -> x >= 0 ? 1 : -1
    item1 = -sign(mod2pi(θ) - π)
    θ_ = (θ==0 ?  0.0 : mod1(θ * item1, π))
    ϕ_ = (item1 .* ϕ) .|> mod2pi
    (θ_, ϕ_)
end

# convert to domain θ ∈[0°, 180°], ϕ∈ [-180°, 180°]
function mod_angle_deg(θ::Number, ϕ)
    θ_res = 0.
    ϕ_res = 0.
    invert_ϕ = false
    # in domain
    if θ >= 0 && θ <= 180
        θ_res = θ
    elseif θ < 0 && θ >= -180
        invert_ϕ = true
        θ_res = -θ
    else
        throw("θ = $(θ) is out of range [-180,180]")
    end

    if ϕ >=-180 && ϕ <= 180
        ϕ_res = ϕ
    elseif ϕ > 180 && ϕ <=360
        ϕ_res = ϕ - 360
    else
        throw("ϕ = $(ϕ) is out of range [-180,360]")
    end

    if invert_ϕ
        if ϕ_res <= 0
            ϕ_res += 180
        else
            ϕ_res -= 180
        end
    end
    θ_res, ϕ_res
end

# not test
function mod_angle_rad(θ::Number, ϕ)
    θ_res = 0.
    ϕ_res = 0.
    invert_ϕ = false
    # in domain
    if θ >= 0 && θ <= pi
        θ_res = θ
    elseif θ < 0 && θ >= -pi
        invert_ϕ = true
        θ_res = -θ
    else
        throw("θ = $(θ) is out of range [-pi, pi]")
    end

    if ϕ >=-pi && ϕ <= pi
        ϕ_res = ϕ
    elseif ϕ > pi && ϕ <=2pi
        ϕ_res = ϕ - 2pi
    else
        throw("ϕ = $(ϕ) is out of range [-pi, 2pi]")
    end

    if invert_ϕ
        if ϕ_res <= 0
            ϕ_res += pi
        else
            ϕ_res -= pi
        end
    end
    θ_res, ϕ_res
end


module HWunconstrained

	# imports: which packages are we going to use in this module?
	using Distributions, Optim, Plots, DataFrames
	using Random
	using Statistics
	using LinearAlgebra

    export maximize_like_grad, makeData, loglik, grad!, maximize_like, plotLike, plotGrad

	# methods/functions
	# -----------------

	# data creator
	# should/could return a dict with beta,numobs,X,y,norm)
	# true coeff vector, number of obs, data matrix X, response vector y, and a type of parametric distribution for G.
	function makeData(n=10_000)
	           beta = [ 1; 1.5; -0.5 ]
	           Random.seed!(54321)
	           numobs = n
	           X = randn(numobs,3)    # n,k
	           epsilon = randn(numobs)
	           Y = X * beta + epsilon
	           y = 1.0 * (Y .> 0)
	           norm = Normal(0,1)    # create a normal distribution object with mean 0 and variance 1
	           return Dict("beta"=>beta,"n"=>numobs,"X"=>X,"y"=>y,"dist"=>norm)
	       end

	function loglik(betas::Vector, d::Dict)
		lh = log.(cdf.(d["dist"],d["X"]*betas) .^ d["y"] .* (1 .- cdf.(d["dist"],d["X"]*betas)) .^ (1 .- d["y"]))
		return (-1) * sum(lh) / d["n"]
	end

	# gradient of the likelihood at x
	function grad!(storage::Vector,betas::Vector,d)

		for j=1:3
			temp = zeros(length(d["y"]))
 			for i=1:d["n"]
				temp[i] = (d["y"][i] - cdf.(d["dist"], d["X"][i,:]' * betas)) * d["X"][i,j]
			end
			storage[j] = sum(temp)
		end
	end

	# hessian of the likelihood at x
	function hessian!(storage::Matrix,betas::Vector,d)

		#call the gradient we just made

		# Compute the hessian
		.*(storage, storage')
		#change it to matrix

	end

	function info_mat(betas::Vector,d)

	end

	function inv_Info(betas::Vector,d)

	end

	"""
	inverse of observed information matrix
	"""
	function inv_observedInfo(betas::Vector,d)

	end

	"""
	standard errors
	"""
	function se(betas::Vector,d::Dict)

	end

	# function that maximizes the log likelihood without the gradient
	# with a call to `optimize` and returns the result

# Check i( is intentional that the maximization is not so good)
	function maximize_like(x0=[0.8, 1.0, -1.0],meth=NelderMead())
		maximum_like(betas) = loglik(betas, makeData())
		#maximum_like(betas) = (-1) * sum(log.(cdf.(makeData()["dist"], (makeData()["X"] * betas)) ^makeData()["y"]) .+
		#		(log.(1 .- cdf.(makeData()["dist"], (makeData()["X"] * betas)) ^(1 .- makeData()["y"]))))
		nm = optimize(maximum_like, x0, meth)
		return(nm)
	end

	function maximize_like_helpNM(x0=[ 1; 1.5; -0.5 ],meth=NelderMead())

	end

	# function that maximizes the log likelihood with the gradient
	# with a call to `optimize` and returns the result
	function maximize_like_grad(x0=[0.8,1.0,-0.1],meth=BFGS())

	end

	function maximize_like_grad_hess(x0=[0.8,1.0,-0.1],meth=Newton())

	end

	function maximize_like_grad_se(x0=[0.8,1.0,-0.1],meth=BFGS())

	end

	# visual diagnostics
	# ------------------

	# function that plots the likelihood
	# we are looking for a figure with 3 subplots, where each subplot
	# varies one of the parameters, holding the others fixed at the true value
	# we want to see whether there is a global minimum of the likelihood at the the true value
	function plotLike()

		# You want to call the loglik function for different vectors of beta and keep loglik in mind
		# increments = 0.01
		totest_values = collect(-3.0:0.01:3.0)
		#for the first element
		j = zeros(length(totest_values))

		d = makeData()
		b1 = d["beta"][1]
		b2 = d["beta"][2]
		b3 = d["beta"][3]
		for i=1:length(totest_values)
			betas_calc = [totest_values[i], b2, b3]
			j[i] = loglik(betas_calc, d)
		end
		p1 = plot(totest_values, j,
			lab="B2 = 1.5 ; B3 = -0.5",
			xlabel = "B1")

		#for the second element
		j = zeros(length(totest_values))
		for i=1:length(totest_values)
			j[i] = loglik([makeData()["beta"][1], totest_values[i], makeData()["beta"][3]],
					makeData())
		end
		p2 = plot(totest_values, j,
			lab="B1 = 1.0 ; B3 = -0.5",
			xlabel = "B2", ylabel = "Log Likelihood function")

		#for the second element
		j = zeros(length(totest_values))
		for i=1:length(totest_values)
			j[i] = loglik([makeData()["beta"][1], makeData()["beta"][2], totest_values[i]],
					makeData())
		end
		p3 = plot(totest_values, j,
			lab="B1 = 1.0 ; B2 = 1.5",
			xlabel = "B3")

		# Plot final graph
		l = @layout [a; b; c]
		return(plot(p1, p2, p3, layout = l))
	end

	function plotGrad()
			# You want to call the loglik function for different vectors of beta and keep loglik in mind
			# increments = 0.01
			totest_values = collect(-2.0:0.01:2.0)

# Element 1
		# reinitialize storage to true values
			betas = makeData()["beta"]
			storage = zeros(3)
			j = zeros(length(totest_values))
		# run the function grad!() to get the gradient & extract the coef of interest
			for i=1:length(totest_values)
				grad!(storage, [totest_values[i], betas[2], betas[3]], makeData())
				j[i] = storage[1]
			end
			p1 = plot(totest_values, j,
				lab="B2 = 1.5 ; B3 = -0.5",
				xlabel = "B1")

# Element 2
			betas = makeData()["beta"]
			storage = zeros(3)
			j = zeros(length(totest_values))
			for i=1:length(totest_values)
				grad!(storage, [betas[1], totest_values[i], betas[3]], makeData())
				j[i] = storage[2]
			end
			p2 = plot(totest_values, j,
				lab="B1 = 1.0 ; B3 = -0.5",
				xlabel = "B2", ylabel = "Log likelihood gradient")

# Element 3
			betas = makeData()["beta"]
			storage = zeros(3)
			j = zeros(length(totest_values))
			for i=1:length(totest_values)
				grad!(storage, [betas[1], betas[2], totest_values[i]], makeData())
				j[i] = storage[3]
			end
			p3 = plot(totest_values, j,
				lab="B1 = 1.0 ; B2 = 1.5",
				xlabel = "B3")
				l = @layout [a; b; c]
			return(plot(p1, p2, p3, layout = l))
		end

	function saveplots()
		p1 = plotLike()
		savefig(p1,joinpath(dirname(@__FILE__),"..","likelihood.png"))
		p2 = plotGrad()
		savefig(p2,joinpath(dirname(@__FILE__),"..","gradient.png"))
		@info("saved both plots.")
	end

end


line(x0, x1, x) = x0 + (x1-x0)*x

function array2dist(xs :: AbstractArray{<:AbstractFloat})
    len = length(xs)
    @assert mod(len, 3) |> iszero

    nmodes = len÷3
    μs = xs[begin:nmodes]
    σs = xs[nmodes+1:2nmodes]
    ws = xs[2nmodes+1:end]
    ws = ws ./ sum(ws)

    return truncated(MixtureModel(Normal, zip(μs, σs) |> collect, ws), 0, Inf)
end

function target_function(image  :: AbstractArray{Bool},
                         ncloud :: Integer,
                         cutoff :: Integer)
    cf = corrfn(image, cutoff)
    p  = porosity(image)
    s  = size(image)

    return xs -> -likelihood(cf, s, p, array2dist(xs); ncloud = ncloud)
end

"""
~~~~
sandcorn_parameters(image; nmodes     = 1,
                           ncloud     = 100,
                           cutoff     = 100,
                           iterations = 15,
                           μ_bounds   = (0.0,  40.0),
                           σ_bounds   = (1e-3, 10.0),
                           w_bounds   = (0.05, 1.0))
~~~~

Find a distribution of radii in an image which consists of partially
overlapping grains (like an slice of a sandstone). `nmodes` is an
assumed number of modes in the distribution. The resulting
distribution can be constructed as
`Truncated(MixtureModel(Normal, sandcorn_parameters(…)), 0, Inf)`.
"""
function sandcorn_parameters(image      :: AbstractArray{Bool};
                             nmodes     :: Integer = 1,
                             ncloud     :: Integer = 100,
                             cutoff     :: Integer = 100,
                             iterations :: Integer = 15,
                             μ_bounds = (0.0,  40.0),
                             σ_bounds = (1e-3, 10.0),
                             w_bounds = (0.05, 1.0))
    μ_lo, μ_hi = μ_bounds
    σ_lo, σ_hi = σ_bounds
    w_lo, w_hi = w_bounds

    μ_lower = fill(μ_lo, nmodes)
    σ_lower = fill(σ_lo, nmodes)
    w_lower = fill(w_lo, nmodes)
    μ_upper = fill(μ_hi, nmodes)
    σ_upper = fill(σ_hi, nmodes)
    w_upper = fill(w_hi, nmodes)

    lower = vcat(μ_lower, σ_lower, w_lower)
    upper = vcat(μ_upper, σ_upper, w_upper)
    start = line.(lower, upper, rand(Float64, 3nmodes))

    opt = optimize(target_function(image, ncloud, cutoff), start,
                   ParticleSwarm(lower = lower, upper = upper),
                   Optim.Options(show_trace = true,
                                 show_every = 1,
                                 iterations = iterations))
    return opt |> Optim.minimizer |> array2dist
end


import Pkg
Pkg.add("Plots")
Pkg.add("PyPlot")
import Plots as plt
plt.pyplot()

include("./solver.jl")
import .solver as sol
import LinearAlgebra as lina

# function f(t::Float64, y::AbstractVector{Float64})::AbstractVector{Float64}
#     return y
# end

# a = 0.0
# b = 10.0
# h = 0.1
# y_euler = sol.euler([1.0], f, a, b, h)
# y_midpoint = sol.midpoint([1.0], f, a, b, h)

# k = ceil(Integer, (b - a) / h)
# t_range = range(a, stop=b, length=k)
# y_true = exp.(t_range)
# p = plt.plot(t_range, y_true, label="true")
# plt.plot!(t_range, lina.transpose(y_euler), label="Euler")
# plt.plot!(t_range, lina.transpose(y_midpoint), label="midpoint")
# plt.png(p, "solver/exp_plot.png")

function f_lorenz(t::Float64, y::AbstractVector{Float64})::AbstractVector{Float64}
    return [
   -y[2] - y[3]
   y[1] + 0.2 * y[2]
   0.2 + y[1] * y[3] - 5.7 * y[3]
 ]
end

a = 0.0
b = 200.0
h = 0.01
println("starting")
y_midpoint = sol.midpoint([-1.0; 0; 0], f_lorenz, a, b, h)
println("finished")
y_trans = lina.transpose(y_midpoint)
p = plt.plot(y_trans[:,1], y_trans[:,2], y_trans[:,3], label="")
plt.png(p, "solver/lorenz.png")


# Numerical_Integration.jl
# Date: 28.06.2019. Created by Joshua Simon.

# Define some functions you want to integrate

function com_simpson(f, a, b, n)
    # Composite Simpson's rule
    # Integration of function f over interval [a,b] where n is
    # the number of subintervals. Accuracy only depends on the
    # number of subintervals.

    sum = 0

    if(a > b)
        println("ERROR: Lower interval bound a has to be smaller than upper bound b.")
        return 0
    end

    # Step length h
    h = (b - a) / n

    # Calculate integral value with composite simpson's rule
    for k = 1:n

        x_k = a + k * h
		x_k1 = a + (k - 1) * h

		simpson = h/6 * ( f(x_k1) + f(x_k) + 4* f((x_k1 + x_k)/2) )

        sum += simpson
    end

    return sum

end

integral_k_1 = com_simpson(sin, 0.0, pi*2.0, 100000)

println(integral_k_1)


using PyPlot

# debug
function say(sth)
  println(sth)
end

XI100 = 124.342
XI80 = 101.879

function test_a(;verbose=true)
  n = 10000
  bins = 101

  U = rand(n)

  (range, counts) = hist(U, 0:1/bins:1)

  D = sum((counts - n/bins).^2/(n/bins))

  verbose ? say("n=$n, bins=$bins, D=$(round(D*1000)/1000) $(D>XI100?"> $XI100, Reject H0, bad random":"< $XI100, Fail to reject H0, good random")") : nothing

  D
end

# dist
function chi_square(x, nu)
  x.^(nu/2-1).*exp(-x/2)./(2^(nu/2)*gamma(nu/2))
end

function test_c(;verbose=true)
  n = 10000
  k0 = 9
  bins = k0^2
  ind(x) = floor(x*9)

  U = rand(n,2)

  Data = [Float64(k0 * ind(U[i,1]) + ind(U[i,2])) for i=1:n]

  (range, counts) = hist(Data[:], -1:80)

  D = sum((counts - n/bins).^2/(n/bins))

  verbose ? say("n=$n, k0=$k0, bins=$bins, D=$(round(D*1000)/1000) $(D>XI80?"> $XI80, Reject H0, bad random":"< $XI80, Fail to reject H0, good random")") : nothing

  D
end


using ClassicalOrthogonalPolynomials, Test
import ClassicalOrthogonalPolynomials: OrthogonalPolynomialRatio, recurrencecoefficients

@testset "OrthogonalPolynomialRatio" begin
    P = Legendre()
    R = OrthogonalPolynomialRatio(P,0.1)
    @test P[0.1,1:10] ./ P[0.1,2:11] ≈ R[1:10]
    R = OrthogonalPolynomialRatio(P,-1)
    @test R[1:10] ≈ fill(-1,10)
end


using IntervalArithmetic, StaticArrays, IntervalRootFinding
using Test

function rand_vec(n::Int)
    a = randn(n)
    A = Interval.(a)
    mA = MVector{n}(A)
    sA = SVector{n}(A)
    return A, mA, sA
end

function rand_mat(n::Int)
    a = randn(n, n)
    A = Interval.(a)
    mA = MMatrix{n, n}(A)
    sA = SMatrix{n, n}(A)
    return A, mA, sA
end

@testset "Linear Equations" begin

    As = [[2..3 0..1; 1..2 2..3], ]
    bs = [[0..120, 60..240], ]
    xs = [[-120..90, -60..240], ]

    for i in 1:10
        rand_A = rand_mat(i)[1]
        rand_x = rand_vec(i)[1]
        rand_b = rand_A * rand_x
        push!(As, rand_A)
        push!(bs, rand_b)
        push!(xs, rand_x)
    end

    n = length(As)

    for solver in (gauss_seidel_interval, gauss_seidel_contractor, gauss_elimination_interval, \)
        @testset "Solver $solver" begin
            #for precondition in (false, true)

                for i in 1:n
                    soln = solver(As[i], bs[i])
                    @test all(xs[i] .⊆ soln)

                end
            #end
        end
    end
end


using DataStructures

function check(x, as)
    N = length(as)
    tmps = [(a < x) ? -1 : 1 for a in as]
    ss = vcat(0, cumsum(tmps))
    ps = sortperm(ss)
    bs = FenwickTree{Int}(length(ps))

    q_inv = 0
    for p in ps
        q_inv += prefixsum(bs, p)
        inc!(bs, p, 1)
    end

    return 2q_inv ≥ N * (N +1) ÷ 2
end

function main()
    N = parse(Int, readline())
    as = parse.(Int, split(readline()))

    x̲, x̄ = minimum(as), maximum(as)
    while x̄ - x̲ > 1
        x = (x̲ + x̄) ÷ 2

        if check(x, as)
            x̲ = x
        else
            x̄ = x
        end
    end

    if check(x̄, as)
        println(x̄)
    else
        println(x̲)
    end

end

main()


# Simulate a Thomas cluster point process on a rectangle
# Author: H. Paul Keeler, 2019.
# Website: hpaulkeeler.com
# Repository: github.com/hpaulkeeler/posts
# For more details, see the post:
# hpaulkeeler.com/testing-the-julia-language-with-point-process-simulations/

#Note: Need the .+ for adding a scalar to an array
#Also need . for sqrt, exp, cos, sin etc and assinging scalars to arrays
#Best to use vectors instead of 1-D matrices eg x=rand(n),  NOT x=rand(n,1).

using Distributions #for random simulations
using Plots #for plotting

#Simulation window parameters
xMin=-.5;
xMax=.5;
yMin=-.5;
yMax=.5;

#Parameters for the parent and daughter point processes
lambdaParent=10;#density of parent Poisson point process
lambdaDaughter=10;#mean number of points in each cluster
sigma=0.05; #sigma for normal variables (ie random locations) of daughters

#Extended simulation windows parameters
rExt=7*sigma; #extension parameter
#for rExt, use factor of deviation sigma eg 6 or 7
xMinExt=xMin-rExt;
xMaxExt=xMax+rExt;
yMinExt=yMin-rExt;
yMaxExt=yMax+rExt;
#rectangle dimensions
xDeltaExt=xMaxExt-xMinExt;
yDeltaExt=yMaxExt-yMinExt;
areaTotalExt=xDeltaExt*yDeltaExt; #area of extended rectangle

#Simulate Poisson point process
numbPointsParent=rand(Poisson(areaTotalExt*lambdaParent)); #Poisson number of points

#x and y coordinates of Poisson points for the parent
xxParent=xMinExt.+xDeltaExt*rand(numbPointsParent);
yyParent=yMinExt.+yDeltaExt*rand(numbPointsParent);

#Simulate Poisson point process for the daughters (ie final poiint process)
numbPointsDaughter=rand(Poisson(lambdaDaughter),numbPointsParent);
numbPoints=sum(numbPointsDaughter); #total number of points

#Generate the (relative) locations in Cartesian coordinates by
#simulating independent normal variables
xx0=rand(Normal(0,sigma),numbPoints);
yy0=rand(Normal(0,sigma),numbPoints);

#replicate parent points (ie centres of disks/clusters)
xx=vcat(fill.(xxParent, numbPointsDaughter)...);
yy=vcat(fill.(yyParent, numbPointsDaughter)...);

#Shift centre of disk to (xx0,yy0)
xx=xx.+xx0;
yy=yy.+yy0;

#thin points if outside the simulation window
booleInside=((xx.>=xMin).&(xx.<=xMax).&(yy.>=yMin).&(yy.<=yMax));
#retain points inside simulation window
xx=xx[booleInside];
yy=yy[booleInside];

#Plotting
plot1=scatter(xx,yy,xlabel ="x",ylabel ="y", leg=false);
display(plot1);


# Cross-correlation and convolution
function FFTConvolution( image::Array{<:Real,N}, kernel::Array{<:Real,N}; typ=Float32 ) where {N}
	return FFTConvolution!( image, kernel, zeros( typ, size(image).+size(kernel).-1) );
end

function FFTConvolution!( image::Array{<:Real,N}, kernel::Array{<:Real,N}, conv::Array{T,N} ) where {T,N}

	@assert all( size(conv) .== size(image) .+ size(kernel) .- 1 );

	paddedkernel = zeros( Complex{T}, size(image) .+ size(kernel) .- 1 );
	paddedkernel[ 1:size(kernel,1), 1:size(kernel,2), 1:size(kernel,3) ] .= convert.( Complex{T}, kernel )

	paddedimage  = zeros( Complex{T}, size(image) .+ size(kernel) .- 1 );
	paddedimage[ 1:size(image,1), 1:size(image,2), 1:size(image,3) ] .= convert.( Complex{T}, image )

	FFTW.fft!( paddedimage  );
	FFTW.fft!( paddedkernel );

	@inbounds @simd for idx in 1:length(paddedimage)
		paddedimage[idx] = paddedimage[idx] * paddedkernel[idx]
	end

	FFTW.ifft!( paddedimage );

	@inbounds @simd for idx in 1:length(paddedimage)
		conv[idx] = real( paddedimage[idx] )
	end

	return conv
end

function FFTConvolution_crop( image::Array{<:Real,N}, kernel::Array{<:Real,N}; typ=Float32 ) where {N}
	return FFTConvolution_crop!( image, kernel, zeros( typ, size(image) ) );
end

function FFTConvolution_crop!( image::Array{<:Real,N}, kernel::Array{<:Real,N}, conv::Array{T,N} ) where {T,N}

	@assert all( size(conv) .== size(image) );

	paddedkernel = zeros( Complex{T}, size(image) .+ size(kernel) .- 1 );
	paddedkernel[ 1:size(kernel,1), 1:size(kernel,2), 1:size(kernel,3) ] .= convert.( Complex{T}, kernel )

	paddedimage  = zeros( Complex{T}, size(image) .+ size(kernel) .- 1 );
	paddedimage[ 1:size(image,1), 1:size(image,2), 1:size(image,3) ] .= convert.( Complex{T}, image )

	FFTW.fft!( paddedimage  );
	FFTW.fft!( paddedkernel );

	@inbounds @simd for idx in 1:length(paddedimage)
		paddedimage[idx] = paddedimage[idx] * paddedkernel[idx]
	end

	FFTW.ifft!( paddedimage );

	offs = div.( size( kernel ), 2 );
	zoff = ( N == 2 ) ? 0 : offs[3];

	for zet in 1+zoff:size(image,3)+zoff, col in 1+offs[2]:size(image,2)+offs[2]
		@simd for row in 1+offs[1]:size(image,1)+offs[1]
			conv[row-offs[1],col-offs[2],zet-zoff] = real( paddedimage[row,col,zet] )
	end end

	return conv
end

function FFTConvolution_b( image::Array{T1,2}, kernel::Array{T2,2}; typ=Float32 ) where {T1<:Real,T2<:Real}

	paddedkernel = zeros( typ, size(image) .+ size(kernel) .- 1 );
	paddedkernel[ 1:size(kernel,1), 1:size(kernel,2) ] .= convert.( typ, kernel )

	paddedimage  = zeros( typ, size(image) .+ size(kernel) .- 1 );
	paddedimage[ 1:size(image,1), 1:size(image,2) ] .= convert.( typ, image )

	Ff = FFTW.fft( paddedimage  );
	Fg = FFTW.fft( paddedkernel );

	Fcorr =  Ff .* Fg;

	return real.( FFTW.bfft( Fcorr ) )
end

function paddedConvolution( image::Array{T1,2}, kernel::Array{T2,2}; typ=Float32 ) where {T1<:Real,T2<:Real}

	N1, N2 = size( image  );
	M1, M2 = size( kernel );

	paddedImage = similar(image, N1+2*M1-2, N2+2*M2-2);
	paddedImage[ M1:(M1+N1-1), M2:(M2+N2-1) ] .= image;

	corrmatrix = zeros( typ, N1 + M1 - 1, N2 + M2 - 1 )

	for col in 1:( N2 + M2 - 1)
		for row in 1:( N1 + M1 - 1 )
			corrmatrix[ row, col ] = sum( paddedImage[ row:row+M1-1, col:col+M2-1 ] .* kernel )
		end
	end
	return corrmatrix
end


# Naive Bayes Module

module NB
export NaiveBayes, train, predict

"""
Data structure for Naive Bayes\\
`pY` is probability for each target\\
`mean` is the corresponding mean of Xi for each taregt\\
`var` is the corresponding variance of Xi for each target
"""
mutable struct NaiveBayes
    n_features::Integer
    pY::Dict{Number,AbstractFloat}
    mean::Dict{Number,Array{AbstractFloat}}
    var::Dict{Number,Array{AbstractFloat}}
end

"""
Training function for Naive Bayes\\
`X_data` has shape (M,N)\\
`Y_data` has shape (M,)\\
Returns the computed data structure `NaiveBayes`
"""
function train(X_data::Array{T} where T<:Number, Y_data::Array{K} where K<:Number)::NaiveBayes
    @assert ndims(X_data) == ndims(Y_data) + 1
    @assert size(X_data)[1] == size(Y_data)[1]
    n_features = size(X_data)[2]
    # compute probability of Y
    pY = Dict{Number,AbstractFloat}()
    for Y_val in Y_data
        if haskey(pY, Y_val)
            pY[Y_val] += 1.0
        else
            pY[Y_val] = 1.0
        end
    end
    for Y_val in keys(pY)
        pY[Y_val] /= size(Y_data)[1]
    end
    # compute Xi mean and var for each Y
    mean = Dict{Number,Array{AbstractFloat}}()
    var = Dict{Number,Array{AbstractFloat}}()
    for Y_val in keys(pY)
        X_part = X_data[Y_data .== Y_val, :]
        mean[Y_val] = vec(sum(X_part, dims=1) ./ size(X_part)[1])
        var[Y_val] = vec(sum((X_part .- reshape(mean[Y_val], 1, :)).^2, dims=1) ./ size(X_part)[1])
    end
    return NaiveBayes(n_features, pY, mean, var)
end

"""
Vectorized gaussian function\\
Returns the product of P(xi|y)
"""
function gaussian(X_vec::Array, mean::Array, var::Array)::AbstractFloat
    @assert ndims(X_vec) == ndims(mean) == ndims(var) == 1
    @assert length(X_vec) == length(mean) == length(var)
    left = 1.0 ./ sqrt.((2.0 * pi) .* var)
    right = exp.(-(X_vec .- mean).^2 ./ (2.0 .* var))
    p = left .* right
    return prod(p)
end

"""
Predict function for Naive Bayes\\
`X_data` has ndims of 1 or 2
"""
function predict(X_data::Array{T} where T<:Number, data::NaiveBayes)::Array
    if ndims(X_data) == 1
        X_data = reshape(X_data, 1, :)
    end
    @assert ndims(X_data) == 2
    @assert size(X_data)[2] == data.n_features
    prediction = Array{Number}(undef, size(X_data)[1])
    for i in 1:size(X_data)[1]
        X_vec = X_data[i, :]
        Y_dict = Dict{Number,AbstractFloat}()
        for Y_val in keys(data.pY)
            Y_dict[Y_val] = data.pY[Y_val] * gaussian(X_vec, data.mean[Y_val], data.var[Y_val])
        end
        prediction[i] = sort(collect(Y_dict), by=m->m[2], rev=true)[1][1]
    end
    return prediction
end

end


function initial_velocity(data, np)
    constante = -2*data.R*data.T/data.m
    p=[rand() for i in 1:np]
    vmod =  [sqrt(constante * log(1-p[i])) for i in 1:np]
    angle = [2*pi*rand() for i in 1:np]
return [ [vmod[i]*cos(angle[i]), vmod[i]*sin(angle[i])] for i in 1:np ], vmod
end


# p17.jl - Helmholtz eq. u_xx + u_yy + (k^2)u = f
#         on [-1,1]x[-1,1]    (compare p16.jl)

# Set up spectral grid and tensor product Helmholtz operator:
N = 24; (D,x) = cheb(N); y = x;
xx = x[2:N]; yy = y[2:N];
f = @. exp(-10*((yy-1)^2+(xx'-.5)^2));
D2 = D^2; D2 = D2[2:N,2:N]; I = eye(N-1);
k = 9;
L = kron(I,D2) + kron(D2,I) + k^2*eye((N-1)^2);

# Solve for u, reshape to 2D grid, and plot:
u = L\f[:];
uu = zeros(N+1,N+1); uu[N:-1:2,N:-1:2] = reshape(u,N-1,N-1);
xxx = yyy = -1:.0333:1;
s = Spline2D(x[end:-1:1],y[end:-1:1],uu);
uuu = evalgrid(s,xxx,yyy);
figure(1); clf(); surf(xxx,yyy,uuu,rstride=1,cstride=1);
xlabel("x"); ylabel("y"); zlabel("u"); view(-37.5,30);
value = signif(uu[Int(N/2+1),Int(N/2+1)],10);
text3D(.2,1,.022,"u(0,0) = $value");
figure(2); clf(); contour(xxx,yyy,uuu,10); axis("square");


### A Pluto.jl notebook ###
# v0.12.7

using Markdown
using InteractiveUtils

# ╔═╡ 6fd616f6-1ea1-11eb-3814-8bfb4a096c49
using Pkg, DrWatson

# ╔═╡ 6ffe2628-1ea1-11eb-24ea-57f985146a72
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using Distributions, Random, StatsBase, DataFrames, Plots, LaTeXStrings
	Random.seed!(1)
end;

# ╔═╡ ed174bc4-1ea0-11eb-1e2f-a32874cec549
md"## Listing 3.21"

# ╔═╡ 70179b62-1ea1-11eb-13ce-7fb4d54bad4b
begin
	cUnif = Uniform(0,2π)
	xGrid, N = 0:0.1:2π, 10^6
end

# ╔═╡ 287be85a-2165-11eb-2609-eba593feae95
begin
	stephist( rand(N)*2π, bins=xGrid,
		normed=:true, c=:blue,
		label="MC Estimate")
	plot!( xGrid, pdf.(cUnif,xGrid),
		c=:red,ylims=(0,0.2),label="PDF", ylabel="Density",xticks=([0:π/2:2π;],
		["0", L"\dfrac{\pi}{2}", L"\pi", L"\dfrac{3\pi}{2}", L"2\pi"]))
end

# ╔═╡ 70182cd8-1ea1-11eb-094d-8d0d49cf15f3
md"## End of listing 3.21"

# ╔═╡ Cell order:
# ╟─ed174bc4-1ea0-11eb-1e2f-a32874cec549
# ╠═6fd616f6-1ea1-11eb-3814-8bfb4a096c49
# ╠═6ffe2628-1ea1-11eb-24ea-57f985146a72
# ╠═70179b62-1ea1-11eb-13ce-7fb4d54bad4b
# ╠═287be85a-2165-11eb-2609-eba593feae95
# ╟─70182cd8-1ea1-11eb-094d-8d0d49cf15f3


using ModelingToolkit, DiffEqOperators, LinearAlgebra, OrdinaryDiffEq
using ModelingToolkit: operation, istree, arguments

# Define some variables
@parameters t x
@variables u(..) v(..)
Dt = Differential(t)
Dx = Differential(x)
Dxx = Differential(x)^2
eqs  = [Dt(u(t,x)) ~ Dxx(u(t,x)),
        Dt(v(t,x)) ~ Dxx(v(t,x))]
bcs = [u(0,x) ~ - x * (x-1) * sin(x),
       v(0,x) ~ - x * (x-1) * sin(x),
       u(t,0) ~ 0.0, u(t,1) ~ 0.0,
       v(t,0) ~ 0.0, v(t,1) ~ 0.0]

domains = [t ∈ IntervalDomain(0.0,1.0),
           x ∈ IntervalDomain(0.0,1.0)]

pdesys = PDESystem(eqs,bcs,domains,[t,x],[u(t,x),v(t,x)])
discretization = MOLFiniteDifference([x=>0.1],t;grid_align=edge_align)
prob = discretize(pdesys,discretization) # This gives an ODEProblem since it's time-dependent
sol = solve(prob,Tsit5())

@parameters t x y
@variables u(..)
Dxx = Differential(x)^2
Dyy = Differential(y)^2
Dt = Differential(t)
t_min= 0.
t_max = 2.0
x_min = 0.
x_max = 2.
y_min = 0.
y_max = 2.

# 3D PDE
eq  = Dt(u(t,x,y)) ~ Dxx(u(t,x,y)) + Dyy(u(t,x,y))

analytic_sol_func(t,x,y) = exp(x+y)*cos(x+y+4t)
# Initial and boundary conditions
bcs = [u(t_min,x,y) ~ analytic_sol_func(t_min,x,y),
       u(t,x_min,y) ~ analytic_sol_func(t,x_min,y),
       u(t,x_max,y) ~ analytic_sol_func(t,x_max,y),
       u(t,x,y_min) ~ analytic_sol_func(t,x,y_min),
       u(t,x,y_max) ~ analytic_sol_func(t,x,y_max)]

# Space and time domains
domains = [t ∈ IntervalDomain(t_min,t_max),
           x ∈ IntervalDomain(x_min,x_max),
           y ∈ IntervalDomain(y_min,y_max)]
pdesys = PDESystem([eq],bcs,domains,[t,x,y],[u(t,x,y)])

# Method of lines discretization
dx = 0.1; dy = 0.2
discretization = MOLFiniteDifference([x=>dx,y=>dy],t)
prob = ModelingToolkit.discretize(pdesys,discretization)
sol = solve(prob,Tsit5())

# Diffusion in a sphere
@parameters t r
@variables u(..)
Dt = Differential(t)
Dr = Differential(r)
Drr = Dr^2
eq  = Dt(u(t,r)) ~ (1/r^2 * Dr(r^2 * Dr(u(t,r))))
bcs = [u(0,r) ~ - r * (r-1) * sin(r),
       Dr(u(t,0)) ~ 0.0, u(t,1) ~ sin(1)]

domains = [t ∈ IntervalDomain(0.0,1.0),
           r ∈ IntervalDomain(0.0,1.0)]

pdesys = PDESystem(eq,bcs,domains,[t,r],[u(t,r)])
discretization = MOLFiniteDifference([r=>0.1],t)
prob = discretize(pdesys,discretization) # This gives an ODEProblem since it's time-dependent
sol = solve(prob,Tsit5())


function GenAmericanBinomialDownOutCall(K, T, S, sig, r, H, N)

        ## as discussed in clewlow and strickland chapter 2, figure 2.12
        ## deviates as different stepping through the matrix requires sometimes
        ## to make use of * instead of /

  dt = T/N ## Time to Maturity/ Number of Steps
  nu = r - 0.5*sig^2
  dxu = sqrt(sig^2*dt + (nu*dt)^2)
  dxd = -dxu
  pu = 1/2 + 1/2*(nu*dt/dxu)
  pd = 1 - pu

  disc = exp(-r*dt)
  dpu = disc*pu
  dpd = disc * pd
  edxud = exp(dxu - dxd)
  edxd = exp(dxd)

  St = zeros(Float64, (N+1, N+1))

        # Initialising asset prices at maturity

  St[1 , N+1] = S / exp(N * dxd)
  for j in 2:(N+1)
    St[j, N+1] = St[j-1, N+1] / edxud
  end

  # create Array for options value tree

  P = zeros(Float64, (N+1, N+1))#linspace(1, N+1, N+1)

  # Initialise Option Values at maturity

for j in 1:(N+1)

  if St[j, N+1] > H ## check if market price is above knock out

    C[j, N+1] = max(0.0, St[j, N+1] - K)

  else

    C[j, N+1] = 0.0 ## if below, assign 0 to the call value

  end

end

  # Stepping back through the tree

for i = linrange(N, 1, N)

  for j in 1:i

    St[j,i] = St[j,i+1] * edxd  ## generating the asset price at node i, j first

    if St[j,i] > H ## checking if asset above knock out

      C[j, i] = dpu * C[j, i+1] + dpd * C[j+1, i+1]
      C[j, i] = max(C[j, i], St[j,i] - K)

    else

      C[j,i] = 0.0  ## apply 0 if not

    end

  end

end

C, St
end


# Setting up Bayesian inference, plotting distributions and
# Bayesian linear regression
using Plots
using Distributions

μ = 5.0
σ = 4.0
n = 1000
lim = 10
x = range(0, stop = lim, length=1000)

f(μ, σ, x) = μ .+ (σ * rand(Normal(0,1),lim))

y = f(μ, σ, x)

function likelihood!(y, n, sd)
    yy = repeat(y, outer=(1,n))
    ran = range(0, stop = length(y), length = n)
    xx = repeat(ran, outer=(1, length(y)))'
    likelihood = exp.(-0.5 * sum((yy .- xx ).^2,dims=1) / σ^2)
    norm_likelihood = likelihood / sum(likelihood)
    return norm_likelihood
end

likelihood = likelihood!(y, n, σ)

ρ = exp.((-0.5 .* (x .- 0 ).^2) / 5^2)
ρ = ρ/sum(ρ)

plot(x, likelihood[1,:])
plot!(x, ρ)
scatter!(y, zeros(n))

#= Linear regression
Use linear regression to estimate the posterior mean and std
y = B*x + e
e = N(0,s^2)

y = N(B^T * x, s^2 * I)
=#

beta1 = 1 / σ^2
beta2 = 1 / 5^2

post_σ = 1 / √(lim * beta1 + beta2)
post_μ = (lim * beta1 * mean(y) + beta2 * 0) / (lim * beta1 + beta2)

posterior = pdf(Normal(post_μ, post_σ), x)
posterior = posterior/sum(posterior)

plot!(x, posterior)


using KrylovMethods
using Base.Test

println("=== Testing bicgstb for real matrix === ")
A  = sprandn(100,100,.1) + 10*speye(100)
D  = diag(A)
Af(x) = A*x
M(x)  = D.\x
rhs = randn(100)

# test flag for early stopping
xt = bicgstb(A,rhs,tol=1e-6,maxIter=3,out=2,storeInterm=true)
@test xt[2]==-1

# test handling of zero rhs
xt = bicgstb(A,zeros(size(A,2)),tol=1e-6,maxIter=3,out=2)
@test  all(xt[1].==0)
@test xt[2]==-9

x0 = bicgstb(full(A),rhs,tol=1e-6)
x1 = bicgstb(A,rhs,tol=1e-6,out=1)
x2 = bicgstb(Af,rhs,tol=1e-6)
x3 = bicgstb(Af,rhs,tol=1e-6,maxIter=100,x=randn(size(rhs)))
x5 = bicgstb(Af,rhs,tol=1e-6,maxIter=100,M1=M)

@test norm(A*x0[1]-rhs)/norm(rhs) < 1e-6
@test norm(A*x1[1]-rhs)/norm(rhs) < 1e-6
@test norm(A*x2[1]-rhs)/norm(rhs) < 1e-6
@test norm(A*x3[1]-rhs)/norm(rhs) < 1e-6
@test norm(A*x5[1]-rhs)/norm(rhs) < 1e-6
@test norm(x2[1]-x1[1])/norm(x1[1]) < 1e-12
@test norm(x3[1]-x1[1])/norm(x1[1]) < 1e-5
@test norm(x0[1]-x1[1])/norm(x1[1]) < 1e-5

println("Testing bicgstb for complex matrix")
A  = sprandn(100,100,.1) + 10*speye(100) + im*(sprandn(100,100,.1) + 10*speye(100) )
D  = diag(A)
Af(x) = A*x
M(x)  = D.\x
rhs = randn(100) + 1im * randn(100)

x1 = bicgstb(A,rhs,tol=1e-6)
x2 = bicgstb(Af,rhs,tol=1e-6)
x3 = bicgstb(Af,rhs,tol=1e-6,maxIter=200,x=randn(size(rhs))+im*randn(size(rhs)))
x4 = bicgstb(Af,rhs,tol=1e-6,maxIter=200,M1=M)

@test norm(A*x1[1]-rhs)/norm(rhs) < 1e-6
@test norm(A*x2[1]-rhs)/norm(rhs) < 1e-6
@test norm(A*x3[1]-rhs)/norm(rhs) < 1e-6
@test norm(A*x4[1]-rhs)/norm(rhs) < 1e-6
@test norm(x2[1]-x1[1])/norm(x1[1]) < 1e-12
@test norm(x3[1]-x1[1])/norm(x1[1]) < 1e-5

println("=== BICGSTB: All tests passed ===")


precision(proposal, goldstandard) =
    num_common_cons(proposal, goldstandard) / num_cons(proposal)
recall(proposal, goldstandard) =
    num_common_cons(proposal, goldstandard) / num_cons(goldstandard)
f1_measure(proposal, goldstandard) =
    2 * precision(proposal, goldstandard) * recall(proposal, goldstandard) /
       (precision(proposal, goldstandard) + recall(proposal, goldstandard))

"number of constituents of tree"
function num_cons(tree::Tree)
    M = boolean_dependency_matrix(tree)
    count(x->x, M)
end

"number of common constituents"
function num_common_cons(tree1::Tree, tree2::Tree)
    A = boolean_dependency_matrix(tree1)
    B = boolean_dependency_matrix(tree2)
    sum(a && b for (a,b) in zip(A,B))
end

"number of crossed bracket pairs"
function num_crossed_brackets(tree1, tree2)
    num_cons(tree1) - num_common_cons(tree1, tree2)
end

# t1 = tree("[0maj[0maj[0maj][1maj]][0maj[7maj][0maj]]]")
# t2 = tree("[0maj[1maj[2min][1maj]][0maj[7maj][0maj]]]")
# t3 = tree("[0maj[0maj][0maj[0maj][0maj[7maj][0maj]]]]")
#
# num_crossed_brackets(t1, t3)


@testset "reconstruction_matrix.jl" begin
    a = 0.0
    b = 2π
    N = 100
    M = N
    Δx = (b - a) / N

    # Domains
    closed_interval = ClosedIntervalDomain(a, b)
    periodic_interval = PeriodicIntervalDomain(a, b)
    unknown_domain = UnknownDomain()

    ## Identity filter
    f = IdentityFilter()

    # ClosedIntervalDomain
    R = reconstruction_matrix(f, closed_interval, M, N)
    @test R isa SparseMatrixCSC
    @test size(R) == (N + 1, M + 1)
    @test all(sum(R, dims = 2) .≈ 1)

    # PeriodicIntervalDomain
    R = reconstruction_matrix(f, periodic_interval, M, N)
    @test R isa SparseMatrixCSC
    @test size(R) == (N, M)
    @test all(sum(R, dims = 2) .≈ 1)

    # Unknown domain
    @test_throws Exception reconstruction_matrix(f, unknown_domain, M, N)

    ## Top hat filter
    h₀ = 0.05
    h = x -> h₀ * (1 - 1 / 2 * cos(x))
    f = TopHatFilter(h)

    # ClosedIntervalDomain
    R = reconstruction_matrix(f, closed_interval, M, N)
    @test R isa SparseMatrixCSC
    @test size(R) == (N + 1, M + 1)
    @test all(sum(R, dims = 2) .≈ 1)

    # PeriodicIntervalDomain
    R = reconstruction_matrix(f, periodic_interval, M, N)
    @test R isa SparseMatrixCSC
    @test size(R) == (N, M)
    @test all(sum(R, dims = 2) .≈ 1)

    # Unknown domain
    @test_throws Exception reconstruction_matrix(f, unknown_domain, M, N)

    ## Convolutional filter
    σ = Δx
    g = GaussianFilter(σ)

    # ClosedIntervalDomain
    R = reconstruction_matrix(g, closed_interval, M, N)
    @test R isa SparseMatrixCSC
    @test size(R) == (N + 1, M + 1)
    @test all(sum(R, dims = 2) .≈ 1)

    # PeriodicIntervalDomain
    R = reconstruction_matrix(g, periodic_interval, M, N)
    @test R isa SparseMatrixCSC
    @test size(R) == (N, M)
    @test all(sum(R, dims = 2) .≈ 1)

    # Unknown domain
    @test_throws Exception reconstruction_matrix(f, unknown_domain, M, N)

    ## Constant mesh-wide top hat filter
    h = x -> Δx / 2
    f = TopHatFilter(h)

    # ClosedIntervalDomain
    R = reconstruction_matrix_meshwidth(f, closed_interval, N)
    @test R isa SparseMatrixCSC
    @test size(R) == (N + 1, N + 1)
    @test all(sum(R, dims = 2) .≈ 1)

    # PeriodicIntervalDomain
    R = reconstruction_matrix_meshwidth(f, periodic_interval, N)
    @test R isa SparseMatrixCSC
    @test size(R) == (N, N)
    @test all(sum(R, dims = 2) .≈ 1)

    # Unknown domain
    @test_throws Exception reconstruction_matrix_meshwidth(f, unknown_domain, N)
end


using Distributions
using SDE
using Base.Test
srand(8)
# sum([ norm(mean([ sample(MvWiener(d),  linspace(0.,2.,5)).X[1,end] for i in 1:n])) for j in 1:1000] .< r*sqrt(2/n))
#tests with alpha 0.01

n = 1000
d= 2

#0.99 quantile vor mean of sum of squares of d-dimensional normals
#r = sqrt(quantile(Distributions.Chisq(d), 0.99))
r = sqrt(9.211) # for d = 2

#  call 'sample(t,MvWiener(d)).X[:,i]' n times
function Wn(d, t, n, i)
 wn = zeros(d, n)
 for j in 1:n
   wn[:, j] = sample(t, Wiener(d)).yy[:,i]
 end
 wn
end

@test size(sample(linspace(3.,4.,2), Wiener(d)).yy) == (d,2)

# the mean and variance of a Brownian motion at t=2 is 0 and 2
@test norm(mean([ sample(linspace(0.,2.,5),Wiener(d)).yy[:,end] for i in 1:n])) < r*sqrt(2/n) #scale with std(W_2)/sqrt(n)
#chiupper =  quantile(Distributions.Chisq(n), 0.995) #upper 0.005 percentile
#chilower = quantile(Distributions.Chisq(n), 0.005) #lower 0.005 percentile

chiupper = 1119 #n = 1000, 0.995
chilower = 888.5

@test chiupper > n*var( Wn(1, linspace(0.,2.,5), n, 5))/2 > chilower

# check that W(2) has the right covariance matrix
@test (d-> norm(cov( Wn(d, linspace(0.,2.,5), n, 5)') - diagm(2ones(d)))*sqrt(n)/sqrt(d))(50) < 5. # did not figure that out exactly, should fail less then 99 %
@test (d-> norm(cov( Wn(d,[0.,0.1, 0.3, 0.5, 2.], n, 5)') - diagm(2ones(d)))*sqrt(n)/sqrt(d))(50) < 5. # did not figure that out exactly, should fail less then 99 %

function Bn(d, u, v, t, n, i)
 wn = zeros(d, n)
 for j in 1:n
   wn[:, j] = samplebridge(t, u, t[end], v, Wiener(d)).yy[:,i]
 end
 wn
end

# a "deterministic" bridge with only start end endpoint
@test (global t1 = norm(samplebridge( linspace(1.,4.,2), [1.,2.], 4., [5.,7.], Wiener(2)).yy - [1. 5.; 2. 7.])) < eps(10.)

# Check that the bridge has the right mean and Variance

# Covariance of a Brownian bridge from t_1 to t_2 (t_2-t)(s-t_1)/(t_2-t_1)
# here (3-0.5)*(0.5-0)/(3) = 0.4166666666666667

#cq = quantile(Distributions.Chisq(2), 0.99)
qchisq = 9.21034037197618
@test norm(mean(Bn(2,[1.,2.], [5.,7.],[1.,1.1, 1.3, 1.5, 3.], n, 4),2) -  [1.,2.] - ([5.,7.] - [1.,2.])*.5/2) < sqrt(qchisq)*sqrt(0.416/n)

 # (3-0.5)*(0.5-0.1)/(2.9) = 0.3448275862068966
@test  chilower < n*var(Bn(1,[1.], [5.],[0.1, 0.3, 0.5, 3.], n, 3),2 )[1]/0.345  <  chiupper


using DiscreteChoice
using JLD2
using ForwardDiff
using Optim
using StatsBase

@load "example/data/homo.jld2" Y X β ξ

P = zeros(10)
[P[i] = mean(Y.==i) for i in 1:10]
sum(P)

function y_prob(X, β, ξ)
    ξ = vcat(zero(eltype(ξ)), ξ)
    index = exp.(X*β .+ ξ)
    y_prob = index ./ sum(index)
    return y_prob
end

model_1 = DCModel(Y, X; hetero_preference=false)
model_2 = DCModel(Y, X; hetero_preference=false)

# Start from true value
estDCModel!(model_1; init_guess = vcat(β, ξ))
abs.(Optim.minimizer(model_1.optResults) .- vcat(β, ξ)) |> maximum
updateCoef!(model_1)

P .- y_prob(model_1.X, model_1.β, model_1.ξ)

# Start from random value
estDCModel!(model_2; init_guess = randn(length(β)+length(ξ)))
updateCoef!(model_2)

y_emp = y_prob(model_2.X, model_2.β, model_2.ξ)
P .- y_emp
# Notice that this can perfectly match the data. Why?
# simple reason is that the number of products is too small !!
# Because there are only 10 products, essentially only 10 data points, too many dimensions
# This will get changed if we include individual demographics into the data

abs.(Optim.minimizer(model_2.optResults) .- vcat(β, ξ)) |> maximum
Optim.minimizer(model_2.optResults) .- vcat(β, ξ)

β
ξ
# Why does the random starting value performs poorly? Is this some non-identification issue?

res_homo = opt_homo(Y, X, vcat(β, ξ))


# p15.jl - solve eigenvalue BVP u_xx = lambda*u, u(-1)=u(1)=0

N = 36; (D,x) = cheb(N); D2 = D^2; D2 = D2[2:N,2:N];
(lam,V) = eig(D2);
ii = sortperm(-lam);          # sort eigenvalues and -vectors
lam = lam[ii]; V = V[:,ii]; clf();
for j = 5:5:30                  # plot 6 eigenvectors
    u = [0;V[:,j];0]; subplot(7,1,j/5)
    plot(x,u,".",markersize=6); grid(true);
    xx = -1:.01:1; uu = polyval(polyfit(x,u),xx);
    plot(xx,uu); axis("off");
    text(-.4,.1,"eig $j = $(lam[j]*4/pi^2) π^2/4");
    text(.7,.1,"$(signif(4*N/(pi*j),2))  ppw");
end


# Part 1
data = readlines("day13_input.txt")
earliest = parse(Int64, first(data))
ids = [parse(Int64, id) for id in split(last(data), ",") if id != "x"]
waitingtimes = [ceil(Int, earliest / id) * id - earliest for id in ids]
println("Part 1: answer = $(ids[argmin(waitingtimes)] * minimum(waitingtimes))")

# Part 2 - each id, delta pair fulfils equation t + delta_i = N * id_i
# i.e. (t + delta_i) (mod id_i) = 0
function findt(deltas)
    t = first(deltas)[2]
    modulo = first(deltas)[1]
    for (id, delta) in deltas[2:end]
        while (t + delta) % id != 0 t += modulo end
        modulo = lcm(modulo, id)
    end
    t
end

deltas = [(parse(Int64, id), i-1) for (i, id) in enumerate(split(last(data), ",")) if id != "x"]
println("Part 2: $(findt(deltas))")


using PyPlot
x=logspace(-2,2,1000)
clf()
plot(x,log.(2x),label=L"$f(x)=\log{(2x)}$")
loglog(x,asinh.(x),label=L"$f(x)=\sinh^{-1}{x}$")
loglog(x,x,label=L"$f(x)=x$")
legend(loc="upper left")
ylabel(L"$f(x)$")
xlabel(L"$x$")


N=10000000
cnt=0
for i = 1:N
    x=rand()
    y=rand()
    if x^2+y^2<1.0
        cnt+=1
    end
end

@printf "Estimate of PI for %d trials is %8.5f\n" N 4.0*(cnt / N)


       # centroid
        @test centroid(tri) ≈ Point2D{T}(1//3, 1//3)

                # centroid
        @test centroid(tri) ≈ Point3D{T}(0, 1//3, 1//3)

   # centroid
        @test centroid(quad) ≈ Point2D{T}(1//2, 1//2)

                    # centroid
            p₁ = Point2D{T}(0, 0)
            p₂ = Point2D{T}(1, 0)
            p₃ = Point2D{T}(0, 1)
            p₄ = Point2D{T}(1//2, 0)
            p₅ = Point2D{T}(1//2, 1//2)
            p₆ = Point2D{T}(0, 1//2)
            tri6 = QuadraticTriangle(p₁, p₂, p₃, p₄, p₅, p₆)
            @test centroid(tri6) ≈ tri6(1//3, 1//3)

        # centroid
        p₁ = Point2D{T}(0, 0)
        p₂ = Point2D{T}(1, 0)
        p₃ = Point2D{T}(1, 1)
        p₄ = Point2D{T}(0, 1)
        p₅ = Point2D{T}(1//2,    0)
        p₆ = Point2D{T}(   1, 1//2)
        p₇ = Point2D{T}(1//2,    1)
        p₈ = Point2D{T}(   0, 1//2)
        quad8 = QuadraticQuadrilateral(p₁, p₂, p₃, p₄, p₅, p₆, p₇, p₈)
        @test centroid(quad8) ≈ quad8(1//2, 1//2)


using Plots
using Formatting

function logistic(r,x)
    r*x*(1.0-x)
end

function logisticmap(;x0=0.2,r=4,n=10)
    results = [(x0,logistic(r,x0))];
    for i in 1:(n-1)
      xold = results[end][2]
      xnew = logistic(r,xold)
      append!(results,[(xold,xold)])
      append!(results,[(xold,xnew)])
  end
  results
end

function vary(dname;x0_vals=[],r_vals=[], nvals=[])
 if !isdir(dname)
  mkdir(dname)
 end
 fno = 1
 for x0 in x0_vals
  for r in r_vals
      for n in nvals
          fname = format(dname*"/frame{:05d}.png",fno)
          fno = fno + 1
          simres = logisticmap(x0=x0, r=r, n=n)
          xvals = [x for (x, y) in simres]
          yvals = [y for (x, y) in simres]
          plot(xvals,yvals,legend=false,xaxis=false,yaxis=false,xlim=(0,1),ylim=(0,1));
          savefig(fname)
        end
    end
end
end

# name/number each iteration then plot 1, 1+2, 1+2+3 ...

# To make frames varying r, specify x0_vals as an array with a single value
# along with whatever values of r you want to loop through
vary("frames_r";x0_vals=[0.5],r_vals=0:0.1:5, n=[100])

# To make frames varying x0, specify r_vals as an array with a single value
# along with whatever values of x0 you want to loop through
vary("frames_x0";x0_vals=0:0.05:1,r_vals=[4.0], n=[100])

vary("frames_n";x0_vals=[0.2],r_vals=[4], nvals = 1:500)

r_vals = 4.1
vary("loop3";x0_vals=[(r_vals+sqrt(r_vals-4)*sqrt(r_vals))/(2*r_vals)],r_vals=[4], nvals=1:10)

vary("loop2";x0_vals=[(r_vals-sqrt(r_vals-4)*sqrt(r_vals))/(2*r_vals)],r_vals=[4], nvals=1:10)

ffmpeg -i frame%05d.png -vf "fps=12,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse" -loop 0 output.gif


# # Control

#-

# A simple control problem on a system usually involves a variable $x(t)$
# that denotes the state of the system over time, and a variable $u(t)$ that
# denotes the input into the system over time. Linear constraints are used to
# capture the evolution of the system over time:
#
# $$
# x(t) = Ax(t - 1) + Bu(t), \ \text{for} \ t = 1,\ldots, T,
# $$
#
# where the numerical matrices $A$ and $B$ are called the dynamics and input matrices,
# respectively.
#
# The goal of the control problem is to find a sequence of inputs
# $u(t)$ that will allow the state $x(t)$ to achieve specified values
# at certain times. For example, we can specify initial and final states of the system:
#
# $$
# \begin{aligned}
#   x(0) &= x_i \\
#   x(T) &= x_f
# \end{aligned}
# $$
#
# Additional states between the initial and final states can also be specified. These
# are known as waypoint constraints. Often, the input and state of the system will
# have physical meaning, so we often want to find a sequence inputs that also
# minimizes a least squares objective like the following:
#
# $$
#   \sum_{t = 0}^T \|Fx(t)\|^2_2 + \sum_{t = 1}^T\|Gu(t)\|^2_2,
# $$
#
# where $F$ and $G$ are numerical matrices.
#
# We'll now apply the basic format of the control problem to an example of controlling
# the motion of an object in a fluid over $T$ intervals, each of $h$ seconds.
# The state of the system at time interval $t$ will be given by the position and the velocity of the
# object, denoted $p(t)$ and $v(t)$, while the input will be forces
# applied to the object, denoted by $f(t)$.
# By the basic laws of physics, the relationship between force, velocity, and position
# must satisfy:
#
# $$
#   \begin{aligned}
#     p(t+1) &= p(t) + h v(t) \\
#     v(t+1) &= v(t) + h a(t)
#   \end{aligned}.
# $$
#
# Here, $a(t)$ denotes the acceleration at time $t$, for which we we use
# $a(t) = f(t) / m + g - d v(t)$,
# where $m$, $d$, $g$ are constants for the mass of the object, the drag
# coefficient of the fluid, and the acceleration from gravity, respectively.
#
# Additionally, we have our initial/final position/velocity conditions:
#
# $$
#   \begin{aligned}
#     p(1) &= p_i\\
#     v(1) &= v_i\\
#     p(T+1) &= p_f\\
#     v(T+1) &= 0
#   \end{aligned}
# $$
#
# One reasonable objective to minimize would be
#
# $$
#   \text{objective} = \mu \sum_{t = 1}^{T+1} (v(t))^2 + \sum_{t = 1}^T (f(t))^2
# $$
#
# We would like to keep both the forces small to perhaps save fuel, and keep
# the velocities small for safety concerns.
# Here $\mu$ serves as a parameter to control which part of the objective we
# deem more important, keeping the velocity small or keeping the force small.
#
# The following code builds and solves our control example:

using Convex, SCS, Plots

## Some constraints on our motion
## The object should start from the origin, and end at rest
initial_velocity = [-20; 100]
final_position = [100; 100]

T = 100 # The number of timesteps
h = 0.1 # The time between time intervals
mass = 1 # Mass of object
drag = 0.1 # Drag on object
g = [0, -9.8] # Gravity on object

## Declare the variables we need
position = Variable(2, T)
velocity = Variable(2, T)
force = Variable(2, T - 1)

## Create a problem instance
mu = 1

## Add constraints on our variables
constraints = Constraint[
    position[:, i+1] == position[:, i] + h * velocity[:, i] for i in 1:T-1
]

for i in 1:T-1
    acceleration = force[:, i] / mass + g - drag * velocity[:, i]
    push!(constraints, velocity[:, i+1] == velocity[:, i] + h * acceleration)
end

## Add position constraints
push!(constraints, position[:, 1] == 0)
push!(constraints, position[:, T] == final_position)

## Add velocity constraints
push!(constraints, velocity[:, 1] == initial_velocity)
push!(constraints, velocity[:, T] == 0)

## Solve the problem
problem = minimize(sumsquares(force), constraints)
solve!(problem, SCS.Optimizer; silent_solver = true)

# We can plot the trajectory taken by the object.

pos = evaluate(position)
plot([pos[1, 1]], [pos[2, 1]], st = :scatter, label = "initial point")
plot!([pos[1, T]], [pos[2, T]], st = :scatter, label = "final point")
plot!(pos[1, :], pos[2, :], label = "trajectory")

# We can also see how the magnitude of the force changes over time.

plot(vec(sum(evaluate(force) .^ 2, dims = 1)), label = "force (magnitude)")


using DynamicHMCModels, LinearAlgebra, StatsFuns

ProjDir = @__DIR__
cd(ProjDir)

delim = ';'
df = CSV.read(joinpath("..", "..", "data", "chimpanzees.csv"), DataFrame; delim)
df.pulled_left = convert(Array{Int64}, df.pulled_left)
df.prosoc_left = convert(Array{Int64}, df.prosoc_left)
first(df, 5)

Base.@kwdef mutable struct Chimpanzees{Ty <: AbstractVector,
  Tx <: AbstractMatrix}
    "Observations."
    y::Ty
    "Covariates"
    x::Tx
    "Number of observations"
    N::Int
end

# Write a function to return properly dimensioned transformation.

function make_transformation(model::Chimpanzees)
  as( (β = as(Array, size(model.x, 2)), ) )
end

# Instantiate the model with data and inits.

N = size(df, 1)
x = hcat(ones(Int64, N), df[:, :prosoc_left]);
y = df[:, :pulled_left]
model = Chimpanzees(;y=y, x=x, N=N);

# Make the model callable with a single argument.

function (model::Chimpanzees)(θ)
    @unpack y, x, N = model   # extract the data
    @unpack β = θ  # works on the named tuple too
    ll = 0.0
    ll += sum(logpdf.(Normal(0, 10), β)) # a & bp
    ll += sum([loglikelihood(Binomial(1, logistic(dot(x[i, :], β))), [y[i]]) for i in 1:N])
    ll
end

println()
θ = (β = [1.0, 2.0],)
model(θ)
println()

# Wrap the problem with a transformation, then use Flux for the gradient.

P = TransformedLogDensity(make_transformation(model), model)
∇P = ADgradient(:ForwardDiff, P)
results = mcmc_with_warmup(Random.GLOBAL_RNG, ∇P, 1000)
posterior = P.transformation.(results.chain)

p = as_particles(posterior)
display(p)

stan_result = "
Iterations = 1:1000
Thinning interval = 1
Chains = 1,2,3,4
Samples per chain = 1000

Empirical Posterior Estimates:
      Mean        SD       Naive SE       MCSE      ESS
 a 0.05103234 0.12579086 0.0019889282 0.0035186307 1000
bp 0.55711212 0.18074275 0.0028577937 0.0040160451 1000

Quantiles:
       2.5%        25.0%       50.0%      75.0%      97.5%
 a -0.19755400 -0.029431425 0.05024655 0.12978825 0.30087758
bp  0.20803447  0.433720250 0.55340400 0.67960975 0.91466915
";

# End of `10/m10.2d.jl`


using NLSolvers, StaticArrays, Test
@testset "mixed optimization problems" begin
function theta(x)
   if x[1] > 0
       return atan(x[2] / x[1]) / (2.0 * pi)
   else
       return (pi + atan(x[2] / x[1])) / (2.0 * pi)
   end
end
f(x) = 100.0 * ((x[3] - 10.0 * theta(x))^2 + (sqrt(x[1]^2 + x[2]^2) - 1.0)^2) + x[3]^2

function f∇f!(∇f, x)
    if !(∇f==nothing)
        if ( x[1]^2 + x[2]^2 == 0 )
            dtdx1 = 0;
            dtdx2 = 0;
        else
            dtdx1 = - x[2] / ( 2 * pi * ( x[1]^2 + x[2]^2 ) );
            dtdx2 =   x[1] / ( 2 * pi * ( x[1]^2 + x[2]^2 ) );
        end
        ∇f[1] = -2000.0*(x[3]-10.0*theta(x))*dtdx1 +
            200.0*(sqrt(x[1]^2+x[2]^2)-1)*x[1]/sqrt( x[1]^2+x[2]^2 );
        ∇f[2] = -2000.0*(x[3]-10.0*theta(x))*dtdx2 +
            200.0*(sqrt(x[1]^2+x[2]^2)-1)*x[2]/sqrt( x[1]^2+x[2]^2 );
        ∇f[3] =  200.0*(x[3]-10.0*theta(x)) + 2.0*x[3];
    end

    fx = f(x)
    return ∇f==nothing ? fx : (fx, ∇f)
end

function f∇f(∇f, x)
    if !(∇f == nothing)
        gx = similar(x)
        return f∇f!(gx, x)
    else
        return f∇f!(∇f, x)
    end
end
function f∇fs(∇f, x)
    if !(∇f == nothing)
        if ( x[1]^2 + x[2]^2 == 0 )
            dtdx1 = 0;
            dtdx2 = 0;
        else
            dtdx1 = - x[2] / ( 2 * pi * ( x[1]^2 + x[2]^2 ) )
            dtdx2 =   x[1] / ( 2 * pi * ( x[1]^2 + x[2]^2 ) )
        end

        s1 = -2000.0*(x[3]-10.0*theta(x))*dtdx1 +
            200.0*(sqrt(x[1]^2+x[2]^2)-1)*x[1]/sqrt( x[1]^2+x[2]^2 )
        s2 = -2000.0*(x[3]-10.0*theta(x))*dtdx2 +
            200.0*(sqrt(x[1]^2+x[2]^2)-1)*x[2]/sqrt( x[1]^2+x[2]^2 )
        s3 = 200.0*(x[3]-10.0*theta(x)) + 2.0*x[3]
        ∇f = @SVector [s1, s2, s3]
        return f(x), ∇f
    else
        return f(x)
    end
end
obj_inplace = OnceDiffed(f∇f!)
obj_outofplace = OnceDiffed(f∇f)
obj_static = OnceDiffed(f∇fs)

x0 = [-1.0, 0.0, 0.0]
xopt = [1.0, 0.0, 0.0]
x0s = @SVector [-1.0, 0.0, 0.0]

println("Starting  from: ", x0)
println("Targeting     : ", xopt)

shortname(::GradientDescent) = "GD  "
shortname(::Inverse) = "(inverse)"
shortname(::Direct) = " (direct)"

function printed_minimize(f∇f, x0, method)
    res = minimize(f∇f, x0, method, OptOptions())
    print("NN  $(shortname(method)) $(shortname(method.approx)): ")
    @printf("%2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
end
function printed_minimize!(obj_inplace, x0, method)
    res = minimize!(obj_inplace, x0, method, OptOptions())
    print("NN! $(shortname(method)) $(shortname(method.approx)): ")
    @printf("%2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
end

printed_minimize(obj_inplace, x0, GradientDescent(Inverse()))
printed_minimize!(obj_inplace, copy(x0), GradientDescent(Inverse()))
res = minimize(obj_static, x0s, GradientDescent(Inverse()))
@printf("NN  GD(S) (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])

printed_minimize(obj_inplace, x0, GradientDescent(Direct()))
printed_minimize!(obj_inplace, copy(x0), GradientDescent(Direct()))
res = minimize(obj_static, x0s, GradientDescent(Direct()))
@printf("NN  GD(S)  (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
println()

res = minimize(obj_inplace, x0, BFGS(Inverse()), OptOptions())
@test res[4] == 30
@printf("NN  BFGS    (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(x0), (BFGS(Inverse()), Backtracking()), OptOptions())
@test res[4] == 30
@printf("NN! BFGS    (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(x0), (BFGS(Inverse()), Backtracking(interp=FFQuadInterp())), OptOptions())
@test res[4] == 32
@printf("NN! BFGS    (inverse, quad): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_static, x0s, BFGS(Inverse()))
@test res[4] == 30
@printf("NN  BFGS(S) (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_static, x0s, (BFGS(Inverse()), Backtracking(interp=FFQuadInterp())))
@test res[4] == 32
@printf("NN  BFGS(S) (inverse, quad): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])

res = minimize(obj_inplace, x0, BFGS(Direct()), OptOptions())
@printf("NN  BFGS    (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(x0), BFGS(Direct()), OptOptions())
@printf("NN! BFGS    (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_static, x0s, BFGS(Direct()))
@printf("NN  BFGS(S) (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
println()
# res = optimize(obj_inplace, copy(x0), Optim.BFGS(linesearch=Backtracking()))
# @time optimize(obj_inplace, copy(x0), Optim.BFGS(linesearch=Backtracking()))
# @printf("OT! BFGS (inverse): %2.2e  %2.2e\n", norm(Optim.minimizer(res)-xopt,Inf), Optim.g_residual(res))

res = minimize(obj_inplace, x0, DFP(Inverse()))
@printf("NN  DFP    (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(x0), DFP(Inverse()))
@printf("NN! DFP    (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_static, x0s, DFP(Inverse()))
@printf("NN  DFP(S) (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_inplace, x0, DFP(Direct()))
@printf("NN  DFP    (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(x0), DFP(Direct()))
@printf("NN! DFP    (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_static, x0s, DFP(Direct()))
@printf("NN  DFP(S) (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
println()

res = minimize(obj_inplace, x0, SR1(Inverse()))
@printf("NN  SR1     (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(x0), SR1(Inverse()))
@printf("NN! SR1     (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_static, x0s, SR1(Inverse()))
@printf("NN  SR1(S)  (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_inplace, x0, SR1(Direct()))
@printf("NN  SR1     (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(x0), SR1(Direct()))
@printf("NN! SR1     (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_static, x0s, SR1(Direct()))
@printf("NN  SR1(S)  (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(x0), (SR1(Direct()), NWI()))
@printf("NN! SR1   (direct, NW): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])

xrand = rand(3)
xrands = SVector{3}(xrand)
println("\nFrom a random point: ", xrand)
res = minimize(obj_inplace, xrand, GradientDescent(Inverse()))
@printf("NN  GD   (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(xrand), GradientDescent(Inverse()))
@printf("NN! GD   (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_inplace, xrand, GradientDescent(Direct()))
@printf("NN  GD    (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(xrand), GradientDescent(Direct()))
@printf("NN! GD    (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
println()
res = minimize(obj_inplace, xrand, BFGS(Inverse()))
@printf("NN  BFGS (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(xrand), (BFGS(Inverse()), Backtracking()))
@printf("NN! BFGS (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(xrand), (BFGS(Inverse()), HZAW()))
@printf("NN! BFGS (inverse) HZ: %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_inplace, xrand, BFGS(Direct()))
@printf("NN  BFGS  (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(xrand), BFGS(Direct()))
@printf("NN! BFGS  (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_inplace, xrand, DBFGS(Direct()))
@printf("NN  DBFGS  (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(xrand), DBFGS(Direct()))
@printf("NN! DBFGS  (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(xrand), (DBFGS(Direct()), NWI()))
@printf("NN! DBFGS  (direct, NW): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(xrand), (DBFGS(Direct()), Dogleg()))
@printf("NN! DBFGS  (direct, DL): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
println()

res = minimize(obj_inplace, xrand, DFP(Inverse()))
@printf("NN  DFP  (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(xrand), DFP(Inverse()))
@printf("NN! DFP  (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_inplace, xrand, DFP(Direct()))
@printf("NN  DFP   (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(xrand), DFP(Direct()))
@printf("NN! DFP   (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
println()

res = minimize(obj_inplace, xrand, SR1(Inverse()))
@printf("NN  SR1  (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(xrand), SR1(Inverse()))
@printf("NN! SR1  (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_static, xrand, SR1(Inverse()))
@printf("NN  SR1(S)   (inverse): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_inplace, xrand, SR1(Direct()))
@printf("NN  SR1   (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(xrand), SR1(Direct()))
@printf("NN! SR1   (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize(obj_static, xrand, SR1(Direct()))
@printf("NN  SR1(S)   (direct): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])

# res = minimize(obj_inplace, xrand, (SR1(Direct()), NWI()))
# @printf("NN  SR1   (direct, tr): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
res = minimize!(obj_inplace, copy(xrand), (SR1(Direct()), NWI()))
@printf("NN! SR1   (direct, NW): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])
# res = minimize(obj_static, xrand, (SR1(Direct()), NWI()))
# @printf("NN  SR1(S)   (direct, tr): %2.2e  %2.2e  %d\n", norm(res[1]-xopt,Inf), norm(res[3], Inf), res[4])

println()

function himmelblau!(∇f, x)
    if !(∇f == nothing)
        ∇f[1] = 4.0 * x[1]^3 + 4.0 * x[1] * x[2] -
            44.0 * x[1] + 2.0 * x[1] + 2.0 * x[2]^2 - 14.0
        ∇f[2] = 2.0 * x[1]^2 + 2.0 * x[2] - 22.0 +
            4.0 * x[1] * x[2] + 4.0 * x[2]^3 - 28.0 * x[2]
    end

    fx = (x[1]^2 + x[2] - 11)^2 + (x[1] + x[2]^2 - 7)^2
    return ∇f == nothing ? fx : (fx, ∇f)
end

function himmelblaus(∇f, x)
    f = (x[1]^2 + x[2] - 11)^2 + (x[1] + x[2]^2 - 7)^2
    if !(∇f == nothing)
        ∇f1 = 4.0 * x[1]^3 + 4.0 * x[1] * x[2] -
            44.0 * x[1] + 2.0 * x[1] + 2.0 * x[2]^2 - 14.0
        ∇f2 = 2.0 * x[1]^2 + 2.0 * x[2] - 22.0 +
            4.0 * x[1] * x[2] + 4.0 * x[2]^3 - 28.0 * x[2]
        return f, @SVector([∇f1, ∇f2])
    else
        return f
    end
end

function himmelblau(∇f, x)
    g = ∇f == nothing ? ∇f : similar(x)

    return himmelblau!(g, x)
end

him_inplace = OnceDiffed(himmelblau!)
him_static = OnceDiffed(himmelblaus)
him_outofplace = OnceDiffed(himmelblau)

println("\nHimmelblau function")
x0 = [3.0, 1.0]
x0s = SVector{2}(x0)
res = minimize(him_outofplace, x0, GradientDescent(Inverse()))
@printf("NN  GD    (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace, copy(x0), GradientDescent(Inverse()))
@printf("NN! GD    (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize(him_static, x0s, GradientDescent(Inverse()))
@printf("NN  GD(S) (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
println()
res = minimize(him_outofplace, x0, GradientDescent(Direct()))
@printf("NN  GD    (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace, copy(x0), GradientDescent(Direct()))
@printf("NN! GD    (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize(him_static, x0s, GradientDescent(Direct()))
@printf("NN  GD(S) (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
println()

res = minimize(him_outofplace, x0, BFGS(Inverse()))
@printf("NN  BFGS    (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace, copy(x0), BFGS(Inverse()))
@printf("NN! BFGS    (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize(him_outofplace, x0s, BFGS(Inverse()))
@printf("NN  BFGS(S) (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
println()
res = minimize(him_outofplace, x0, BFGS(Direct()))
@printf("NN  BFGS    (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace, copy(x0), BFGS(Direct()))
@printf("NN! BFGS    (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize(him_outofplace, x0s, BFGS(Direct()))
@printf("NN  BFGS(S) (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace, copy(x0), (DBFGS(Direct()), NWI()))
@printf("NN! DBFGS  (direct, NW): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace, copy(x0), (DBFGS(Direct()), Dogleg()))
@printf("NN! DBFGS  (direct, DL): %2.2e  %d\n", norm(res[3], Inf), res[4])

println()

res = minimize(him_outofplace, x0, DFP(Inverse()))
@printf("NN  DFP  (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace, copy(x0), DFP(Inverse()))
@printf("NN! DFP  (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize(him_outofplace, x0s, DFP(Inverse()))
@printf("NN  DFP(S)  (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize(him_outofplace, x0, DFP(Direct()))
println()
@printf("NN  DFP   (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace, copy(x0), DFP(Direct()))
@printf("NN! DFP   (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize(him_outofplace, x0s, DFP(Direct()))
@printf("NN  DFP(S)   (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
println()

res = minimize(him_outofplace, x0, SR1(Inverse()))
@printf("NN  SR1  (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace, copy(x0), SR1(Inverse()))
@printf("NN! SR1  (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize(him_outofplace, x0s, SR1(Inverse()))
@printf("NN  SR1(S)  (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize(him_outofplace, x0, SR1(Direct()))
@printf("NN  SR1   (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace, copy(x0), SR1(Direct()))
@printf("NN! SR1   (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize(him_outofplace, x0s, SR1(Direct()))
@printf("NN  SR1(S)   (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
println()
xrand = rand(2)
xrands = SVector{2}(xrand)
println("\nFrom a random point: ", xrand)

res = minimize(him_outofplace, xrand, GradientDescent(Inverse()))
@printf("NN  GD   (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace, copy(xrand), GradientDescent(Inverse()))
@printf("NN! GD   (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize(him_outofplace, xrand, GradientDescent(Direct()))
@printf("NN  GD    (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace, copy(xrand), GradientDescent(Direct()))
@printf("NN! GD    (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])

res = minimize(him_outofplace, xrand, BFGS(Inverse()))
@printf("NN   BFGS (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace, copy(xrand), BFGS(Inverse()))
@printf("NN!  BFGS (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize(him_static, xrand, BFGS(Direct()))
@printf("NN(S) BFGS (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize(him_outofplace, xrand, BFGS(Direct()))
@printf("NN  BFGS  (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace, copy(xrand), BFGS(Direct()))
@printf("NN! BFGS  (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])

res = minimize(him_outofplace, xrand, DFP(Inverse()))
@printf("NN  DFP  (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace,  copy(xrand), DFP(Inverse()))
@printf("NN! DFP  (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize(him_outofplace, xrand, DFP(Direct()))
@printf("NN  DFP   (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace,  copy(xrand), DFP(Direct()))
@printf("NN! DFP   (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])

res = minimize(him_outofplace, xrand, SR1(Inverse()))
@printf("NN  SR1  (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace,  copy(xrand), SR1(Inverse()))
@printf("NN! SR1  (inverse): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize(him_outofplace, xrand, SR1(Direct()))
@printf("NN  SR1   (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])
res = minimize!(him_inplace,  copy(xrand), SR1(Direct()))
@printf("NN! SR1   (direct): %2.2e  %d\n", norm(res[3], Inf), res[4])

end