# # [Lotka-Volterra Model](@id predation_example)
#
#md # [![](https://img.shields.io/badge/show-nbviewer-579ACA.svg)](@__NBVIEWER_ROOT_URL__/examples/predation/lotka-volterra.ipynb)

using AlgebraicPetri

using Petri
using OrdinaryDiffEq
using Plots

using Catlab
using Catlab.Theories
using Catlab.Programs
using Catlab.CategoricalAlgebra.ShapeDiagrams
using Catlab.WiringDiagrams
using Catlab.Graphics

display_wd(ex) = to_graphviz(ex, orientation=LeftToRight, labels=true);

# #### Step 1: Define the building block Petri nets needed to construct the model

birth_petri = PetriCospan([1], Petri.Model([1], [(Dict(1=>1), Dict(1=>2))]), [1]);
Graph(decoration(birth_petri))
#-
predation_petri = PetriCospan([1,2], Petri.Model(1:2, [(Dict(1=>1,2=>1), Dict(2=>1.15))]), [2]);
Graph(decoration(predation_petri))
#-
death_petri = PetriCospan([1], Petri.Model([1], [(Dict(1=>1), Dict{Int,Int}())]), [1]);
Graph(decoration(death_petri))

# #### Step 2: Define a presentation of the free biproduct category
# that encodes the domain specific information

@present Predation(FreeBiproductCategory) begin
    prey::Ob
    predator::Ob
    birth::Hom(prey,prey)
    predation::Hom(prey⊗predator,predator)
    death::Hom(predator,predator)
end;

rabbits,wolves,birth,predation,death = generators(Predation);

F(ex) = functor((PetriCospanOb, PetriCospan), ex, generators=Dict(
                 rabbits=>PetriCospanOb(1),wolves=>PetriCospanOb(1),
                 birth=>birth_petri, predation=>predation_petri, death=>death_petri));

# #### Step 3: Generate models using the hom expression or program notations

lotka_volterra = (birth ⊗ id(wolves)) ⋅ predation ⋅ death
lotka_petri = decoration(F(lotka_volterra))
display_wd(lotka_volterra)
#-
Graph(lotka_petri)

# Generate appropriate vector fields, define parameters, and visualize solution

u0 = [40, 2];
p = [4.0, 1.0, 2.0];
prob = ODEProblem(lotka_petri,u0,(0.0,8.0),p);
sol = solve(prob,Tsit5(),abstol=1e-6);
plot(sol)

# There is also a second syntax that is easier to write for programmers
# than the hom expression syntax. Here is an example of the same model
# as before along with a test of equivalency

lotka_volterra2 = @program Predation (r::prey, w::predator) begin
  r_2 = birth(r)
  w_2 = predation(r_2, w)
  return death(w_2)
end
lotka_petri2 = decoration(F(to_hom_expr(FreeBiproductCategory, lotka_volterra2)))
lotka_petri == lotka_petri2

# #### Step 4: Extend your presentation to handle more complex phenomena
# such as a small food chain

@present DualPredation <: Predation begin
    Predator::Ob
    Predation::Hom(predator⊗Predator,Predator)
    Death::Hom(Predator,Predator)
end;

fish,Fish,Shark,birth,predation,death,Predation,Death = generators(DualPredation);

F(ex) = functor((PetriCospanOb, PetriCospan), ex, generators=Dict(
                 fish=>PetriCospanOb(1),Fish=>PetriCospanOb(1),
                 birth=>birth_petri, predation=>predation_petri, death=>death_petri,
                 Shark=>PetriCospanOb(1),Predation=>predation_petri, Death=>death_petri));

# Define a new model where fish are eaten by Fish which are then eaten by Sharks

dual_lv = @program DualPredation (fish::prey, Fish::predator, Shark::Predator) begin
  f_2 = birth(fish)
  F_2 = predation(f_2, Fish)
  F_3 = death(F_2)
  S_2 = Predation(F_3, Shark)
  S_3 = Death(S_2)
end
display_wd(dual_lv)
#-
dual_lv_petri = decoration(F(to_hom_expr(FreeBiproductCategory, dual_lv)))
Graph(dual_lv_petri)

# Generate a new solver, provide parameters, and analyze results

u0 = [40, 2, 1];
p = [4.0, 1.0, 2.0, 2.5, 1.5];
prob = ODEProblem(dual_lv_petri,u0,(0.0,8.0),p);
sol = solve(prob,Tsit5(),abstol=1e-6);
plot(sol)


"""
    Ormsby(; <keyword arguments>)

Create a Ormsby wavelet sampled every dt seconds with corner frequencies
defined by the vector f = [f1, f2, f3, f4]. The final wavelet is multiplied by
a Hamming window.

# Arguments

**Keyword arguments**

* `dt::Real=0.002`: sampling interval in secs.
* `f::Vector{Real}=[2.0, 10.0, 40.0, 60.0]`: corner frequencies in Hz.

      ^
    1 |     ***************
      |    *               *
      |   *                 *
      |  *                   *
      | *                     *
      -----------------------------> f
        f1  f2           f3  f4

# Example
```julia
julia> w = Ormsby(); plot(w);
```
"""

function Ormsby{T<:Real}(; dt::Real=0.002, f::Vector{T}=[2.0, 10.0, 40.0, 60.0])

    f1 = f[1]
    f2 = f[2]
    f3 = f[3]
    f4 = f[4]

    fc = (f2+f3)/2.0
    nw = 2.2/(fc*dt)
    nc = floor(Int, nw/2)
    t = dt*collect(-nc:1:nc)
    nw = 2*nc + 1
    a4 = (pi*f4)^2/(pi*(f4-f3))
    a3 = (pi*f3)^2/(pi*(f4-f3))
    a2 = (pi*f2)^2/(pi*(f2-f1))
    a1 = (pi*f1)^2/(pi*(f2-f1))

    u = a4*(sinc(f4*t)).^2 - a3*(sinc(f3*t)).^2
    v = a2*(sinc(f2*t)).^2 - a1*(sinc(f1*t)).^2

    w = u - v
    w = w.*Hamming(nw)/maximum(w)

end


using XDiff
import ReverseDiffSource

function ann(w1, w2, w3, x1)
    x2 = w1 * x1
    # x2 = log.(1. + exp.(_x2))   # soft RELU unit
    x3 = w2 * x2
    # x3 = log.(1. + exp.(_x3))   # soft RELU unit
    x4 = sum.(w3 * x3)
    # return 1. ./ (1. + exp.(-x4))  # sigmoid output
    return x4
end

# function ann(w1, w2, w3, x1)
#     _x2 = w1 * x1
#     x2 = log.(1. + exp.(_x2))   # soft RELU unit
#     _x3 = w2 * x2
#     x3 = log.(1. + exp.(_x3))   # soft RELU unit
#     x4 = sum.(w3 * x3)
#     return 1. ./ (1. + exp.(-x4))  # sigmoid output
# end

# ANN input parameter types
types = (Matrix{Float64}, Matrix{Float64}, Matrix{Float64}, Vector{Float64})

# generate example input
w1, w2, w3 = randn(10,10), randn(10,10), randn(1,10)
x1 = randn(10)

dann = ReverseDiffSource.rdiff(ann, types)
_, rds_dw1_val, _, _, _ = dann(w1, w2, w3, x1)

dw1, _, _, _ = fdiff(ann, types)
dw1_val = dw1(w1, w2, w3, x1)

if isapprox(dw1_val, rds_dw1_val)
    println("Equal!")
else
    error("Not equal!")
end


using QuasiGeostrophy, LinearAlgebra, Test
# Polynomial Order
n = 16

# Domain [a b]
Ω = [-1.0 1.0]
b = Ω[end]
a = Ω[1]
L = b - a

D, cx = cheb(n)

x =  (b-a) .* (cx .+ 1) ./ 2 .+ a

∂ˣ = 2 / L .* D

# Equation Dy = y, y[x=-1] = 1
A = ∂ˣ - I

# Boundary Conditions
A[end, :] .= 0.0
A[end, end] = 1.0

# Solve
rhs = 0.0 * x
rhs[end] = 1.0
y = A \ rhs

# Exact solution
solution = exp.( x .+ 1)

@testset "ODE Test 1" begin
    bool = norm(y - solution) < eps(1.0) * 100
    @test bool
end

# Polynomial Order
n = 32

# Domain [a b]
Ω = [-1.0 0.0]
b = Ω[end]
a = Ω[1]
L = b - a

D, cx = cheb(n)

x =  (b-a) .* (cx .+ 1) ./ 2 .+ a

∂ˣ = 2 / L .* D

# Equation Dy = y, y[x=-1] = 1
A = ∂ˣ - I

# Boundary Conditions
A[end, :] .= 0.0
A[end, end] = 1.0

# Solve
rhs = 0.0 * x
rhs[end] = 1.0
y = A \ rhs

# Exact solution
solution = exp.( x .+ 1)

@testset "ODE Test 2" begin
    bool = norm(y - solution) < eps(1.0) * 100
    @test bool
end


include("utils.jl")

using Plots
using LaTeXStrings
using Statistics

n = 100
maxk = 50
repeats = 20

A = hilbert(n)
ks = 1:maxk
λ = 1e-4

errors = zeros(maxk, repeats, 3)

methods = [
    ridge_weights(A, λ),
    uniform_weights(A),
    squared_norms_weights(A)
]

for k in ks
    for rep = 1:repeats
        for (i, w) in enumerate(methods)
            C = samplemat(A, w, k)
            C = unique(C, dims=2)
            errors[k, rep, i] = approxerror(A, C)
        end
    end
end

best_approx_errors = zeros(maxk)

for k in ks
    C = best_approx(A, k)
    best_approx_errors[k] = approxerror(A, C)
end

means = mean(errors, dims=2)[:, 1, :]
stds = std(errors, dims=2)[:, 1, :]

plot(xlabel=L"k", ylabel=L"\textrm{error}")
plot!(ks, means[:, 2], yaxis=:log, label="uniform", w=2)
plot!(ks, means[:, 3], yaxis=:log, label="squared norm", w=2)
plot!(ks, means[:, 1], yaxis=:log, label="ridge", w=2)
p = plot!(ks, best_approx_errors, yaxis=:log, label="best", c=:gray, w=2, ls=:dashdot)
savefig(p, "./plot.pdf")
display(p)  # sometimes the display doesn't work


## quadrature.jl : nystrom integration methods

abstract type QuadratureRule end

struct GaussLegendre <: QuadratureRule end
struct EOLE <: QuadratureRule end
struct Simpson <: QuadratureRule end
struct Midpoint <: QuadratureRule end
struct Trapezoidal <: QuadratureRule end

# get nodes and weights of Gauss-Legendre quadrature on [a,b]
function get_nodes_and_weights(n,a,b,q::GaussLegendre)
    nodes, weights = gausslegendre(n)
    weights = (b-a)/2*weights
    nodes = (b-a)/2*nodes .+ (a+b)/2
    return nodes, weights
end

# get nodes and weights of structured grid on [a,b]
function get_nodes_and_weights(n,a,b,q::EOLE)
    nodes = range(a,stop = b,length = n)
    weights = (b-a)/n*fill(1.0, size(nodes))
    return nodes, weights
end

# get nodes and weights of Simpson's rule on [a,b]
function get_nodes_and_weights(n,a,b,q::Simpson)
    n % 2 == 0 || begin
        warn("to use Simpson's rule, n must be even (received $(n)). I will continue with n = $(n+1)")
        n += 1
    end
    Δx = (b-a)/n
    nodes = a:Δx:b
    weights = repeat(2:2:4, outer=Int(n/2))
    weights[1] = 1
    push!(weights,1)
    weights *= Δx/3
    return nodes, weights
end

# get nodes and weights of Midpoint rule on [a,b]
function get_nodes_and_weights(n,a,b,q::Midpoint)
    Δx = (b-a)/n
    nodes = a+Δx/2:Δx:b-Δx/2
    weights = Δx*fill(1.0, size(nodes))
    return nodes, weights
end

# get nodes and weights of Trapezoidal rule on [a,b]
function get_nodes_and_weights(n,a,b,q::Trapezoidal)
    Δx = (b-a)/n
    nodes = a:Δx:b
    weights = Δx*fill(1.0, size(nodes))
    weights[1] /= 2
    weights[end] /= 2
    return nodes, weights
end


#!/usr/bin/env julia

"""
Tests for the [`Primes`](@ref) common module.
"""
module TestPrimes

include(joinpath("..", "src", "Primes.jl"))

using Test

@testset "Primes" begin
    @testset "prime_factorization" begin
        @test Primes.prime_factorization(2) == [(2, 1)]
        @test Primes.prime_factorization(3) == [(3, 1)]
        @test Primes.prime_factorization(4) == [(2, 2)]
        @test Primes.prime_factorization(5) == [(5, 1)]
        @test Primes.prime_factorization(6) == [(2, 1), (3, 1)]
        @test Primes.prime_factorization(12) == [(2, 2), (3, 1)]
        @test Primes.prime_factorization(600) == [(2, 3), (3, 1), (5, 2)]
        @test Primes.prime_factorization(2903) == [(2903, 1)]
        @test(
            Primes.prime_factorization(61740)
                == [(2, 2), (3, 2), (5, 1), (7, 3)]
        )
        @test(
            Primes.prime_factorization(4928693)
                == [(7, 1), (11, 3), (23, 2)]
        )
        @test(
            Primes.prime_factorization(169165232)
                == [(2, 4), (17, 1), (313, 1), (1987, 1)]
        )
        @test(
            Primes.prime_factorization(74435892358158)
                == [(2, 1), (3, 3), (29, 1), (3049, 2), (5113, 1)]
        )
    end
end

end


# to create vector, put comma
V = [1, 2, 3, 4, 5, 6]

# to create matrix just add space
V1 = [1 2 3 4 5 6]

# to declare an array just put semicolon
M1 = [1 2 3; 4 5 6; 7 8 9]

# to resahepe vector or matrix just use resahepe()

V3 = [1, 2, 3, 4, 5, 6, 7, 8, 9]
reshape(V3, 3,3) # for vector

M2 = [1 2 3; 4 5 6]
reshape(M2, 3, 2) # for matrix 2x3 to 3x2

# to create identitiy matirx
using LinearAlgebra
Matrix{Int}(I, 4, 4)

# for dimensions and size of matrix use ndims() and size(), and to see type of elements in matrix using eltype()
ndims(M2)
size(M2)
eltype(M1)

# to concatenate two vector use vcat (similar to cbind() in R) and use hcat similar to (rbind() in R)

d1 = [1 2 3; 4 5 6]
d2 = [6 7 8; 9 10 11]

hcat(d1, d2)
vcat(d1, d2)

# functions in julia

function f(x, y, z)
    x + y + z
end

# or just assign

g(a, b, c) = a + b + c

# function with return

function β(a, b, c)
    result = a + b + c
    return result
end

β(2,π,3)

# we can use opetator as function

2 * 3 * 4
*(2,3,4)

# function with default parameters

z(x, y, a = 1, b = 2) = a*x + b*y
z(3,5)

 # Anonymous function

x -> x + 1 # or

function (x)
    x +1
end

# Anonymous function generally useing with other first class function
map(x -> x + 1, [1, 2])

# multiple return values

function ff( a, b, c)
    a * b * c, a + b + c
end;

ff(1, 2, 3)

x, y  = ff(1, 2, 3);

display(x)
display(y)

function fff(a, b, c)
    multi = a * b * c
    add = a + b + c

    return multi, add
end

fff(1, 2, 3)

# using Anonymous function with multiple piece of code, do-blocks will help

map(x -> x + 1, [1, 2, 3, 4]) # basic way

map([1, 2, 3, 4]) do x # other way
     x + 1
end

# control flow
# compound expression

x = (a = 5; b = 6; a*b)

# other way

x = begin
    a = 5
    b = 6
    a * b
end

# conditional evaluation

function ineq(x, y)
    if x > y
        x = 2
    elseif x < y
        x = 1
    else
        x = 0
    end
end

ineq(1, 2)

# repeated evaluatins: loops

j = 3;

while j > 0
    println(j^2)
    global  j -= 1
end

# for loops

for j in [1, 2, 3]
    println(j)
end

# we can add ∈ to loops

for j ∈ [1, 2, 3]
    println(j)
end

# warning and informational messages

function f(x)
    if x<1
        @warn "x must be positive!"
        else
            return x^2
    end
end

f(2)

function g(x)
    if x < 1
        @info "X must be positive!"
    else
        return x^2
    end
end

g(-2)

# Plotting

using PyPlot
x = 1:100
y = rand(100)
display(gcf())
plot(x,y, color = "tomato")
title("basic graphs in PyPlot")


using LowRankApprox

@testset "Truncation with standard SVD works correctly" begin
    D = 100
    Dcut = D - 1
    tol = 1E-8

    a = rand(D, D)

    U1, Σ1, V1 = svd(a)

    δ = min(Dcut, size(Σ1)...)
    U1 = U1[:, 1:δ]
    Σ1 = Σ1[1:δ]
    V1 = V1[:, 1:δ]

    U2, Σ2, V2 = svd(a)

    δ = min(Dcut, size(Σ2)...)
    U2 = U2[:, 1:δ]
    Σ2 = Σ2[1:δ]
    V2 = V2[:, 1:δ]

    r1 = U1 * Diagonal(Σ1) * V1'
    r2 = U2 * Diagonal(Σ2) * V2'

    @test norm(r1-r2) < tol
end

@testset "Truncation with with random SVD works correctly" begin

    D = 100
    Dcut = D - 1
    tol = 1E-8

    a = rand(D, D)

    U1, Σ1, V1 = psvd(a, rank=Dcut, atol=1E-16, rtol=1E-16)
    U2, Σ2, V2 = psvd(a, rank=Dcut, atol=1E-16, rtol=1E-16)

    r1 = U1 * Diagonal(Σ1) * V1'
    r2 = U2 * Diagonal(Σ2) * V2'

    println(norm(a-r2))
    println(norm(a-r1))

    @test norm(r1-r2) < tol
end


"""
    logit10(x::Real)
Function transforming interval [0,1] to [-Inf, Inf] using logit transformation.
"""
function logit10(x::Real)
    log10(x / (1.0 - x))
end

"""
    logistic10(x::Real)
Function transforming interval [-Inf, Inf] to [0,1] using logistic transformation.
Inverse function for [`logit10`](@ref).
"""
function logistic10(x::Real)
    exp10(x) / (exp10(x) + 1.0)
end

"""
    scaling(x::Real, scale::Symbol = :direct)
Transforms values from specific scale to range [-Inf, Inf] based on option.

## Return
Transformed value.

## Arguments
* `x`: input value.
* `scale`: transformation type: `:direct` (`:lin`), `:log`, `:logit`.
"""
function scaling(x::Real, scale::Symbol = :direct)
    if scale == :direct || scale == :lin
        return x
    elseif scale == :log
        return log10(x)
    elseif scale == :logit
        return logit10(x)
    else
        throw(DomainError(scale, "scale type is not supported"))
    end
end

"""
    unscaling(x::Real, scale::Symbol = :direct)
Transforms values from [-Inf, Inf] to specific scale based on option. Inverse function
for [`scaling`](@ref).

## Return
Transformed value.

## Arguments
* `x`: input value.
* `scale`: transformation type: `:direct` (`:lin`), `:log`, `:logit`.
"""
function unscaling(x::Real, scale::Symbol = :direct)
    if scale == :direct || scale == :lin
        return x
    elseif scale == :log
        return exp10(x)
    elseif scale == :logit
        return logistic10(x)
    else
        throw(DomainError(scale, "scale type is not supported"))
    end
end

scaling(x_tup::Tuple{Float64,Float64}, scale::Symbol = :direct) = scaling.(x_tup, scale)

unscaling(x_tup::Tuple{Float64,Float64}, scale::Symbol = :direct) = unscaling.(x_tup, scale)

scaling(::Nothing, ::Symbol) = nothing

unscaling(::Nothing, ::Symbol) = nothing

"""
    function get_endpoint(
        theta_init::Vector{Float64},
        theta_num::Int,
        loss_func::Function,
        method::Symbol,
        direction::Symbol = :right;

        loss_crit::Float64 = 0.0,
        scale::Vector{Symbol} = fill(:direct, length(theta_init)),
        theta_bounds::Vector{Tuple{Float64,Float64}} = unscaling.(
            fill((-Inf, Inf), length(theta_init)),
            scale
            ),
        scan_bound::Float64 = unscaling(
            (direction==:left) ? -9.0 : 9.0,
            scale[theta_num]
            ),
        scan_tol::Float64 = 1e-3,
        loss_tol::Float64 = 0.,
        local_alg::Symbol = :LN_NELDERMEAD,
        max_iter::Int = 10^5,
        loss_grad::Union{Function, Symbol} = :EMPTY,
        silent::Bool = false
    )

Calculates confidence interval's right or left endpoints for a given parameter `theta_num`.

## Return
[`EndPoint`](@ref) object storing confidence interval's endpoint and intermediate profile points.

## Arguments
- `theta_init`: starting values of parameter vector ``\\theta``. The starting values should not necessary be the optimum values of `loss_func` but `loss_func(theta_init)` should be lower than `loss_crit`.
- `theta_num`: index of vector component for identification: `theta_init(theta_num)`.
- `loss_func`: loss function ``\\Lambda\\left(\\theta\\right)`` for profile likelihood-based (PL) identification. Usually we use log-likelihood for PL analysis: ``\\Lambda( \\theta ) = - 2 ln\\left( L(\\theta) \\right)``.
- `method`: computational method to estimate confidence interval's endpoint. Currently the following methods are implemented: `:CICO_ONE_PASS`, `:LIN_EXTRAPOL`, `:QUADR_EXTRAPOL`.
- `direction`: `:right` or `:left` endpoint to estimate.

## Keyword arguments
- `loss_crit` : critical level of loss function. Confidence interval's endpoint value is the intersection point of profile likelihood and `loss_crit` level.
- `scale` : vector of scale transformations for each parameters' component. Possible values: `:direct` (`:lin`), `:log`, `:logit`. This option can speed up the optimization, especially for wide `theta_bounds`. The default value is `:direct` (no transformation) for all parameters.
- `theta_bounds` : vector of tuple `(lower_bound, upper_bound)` for each parameter. Bounds define the ranges for possible parameter values. Default bounds are `(-Inf,Inf)`.
- `scan_bound` : value which states the area of confidence point analysis.
- `scan_tol` : Absolute tolerance for `theta_num` parameter used as termination criterion.
- `loss_tol` : Absolute tolerance controlling `loss_func` closenes to `loss_crit` (termination criterion). Currently doesn't work for `:CICO_ONE_PASS` method because of limitation in `LN_AUGLAG` interface.
- `local_alg` : algorithm of optimization. Derivative-free and gradient-based algorithms form NLopt package.
- `max_iter` : maximal number of fitter iterations. If reaches the result status will be `:MAX_ITER_STOP`.
- `loss_grad` : For gradient optimization methods it is necessary to set how the gradient of `loss_func` should be calculated.
    There are options:
    - `:EMPTY` (default) no gradient is set. It works only for gradient-free methods.
    - `:AUTODIFF` means autodifferentiation from `ForwardDiff` package is used.
    - `:FINITE` means finite difference method from `Calculus` is used.
    - It is also possible to set gradient function here `function(x::Vector{Float64})` which returns gradient vector.
- `silent` : Boolean argument declaring whether we display the optimization progress. Default is `false`

See also [`get_interval`](@ref)
"""
function get_endpoint(
    theta_init::Vector{Float64},
    theta_num::Int,
    loss_func::Function,
    method::Symbol,
    direction::Symbol = :right;

    loss_crit::Float64 = 0.0,
    scale::Vector{Symbol} = fill(:direct, length(theta_init)), # :direct, :lin, :log, :logit
    theta_bounds::Vector{Tuple{Float64,Float64}} = unscaling.(
        fill((-Inf, Inf), length(theta_init)),
        scale
        ),
    scan_bound::Float64 = unscaling(
        (direction==:left) ? -9.0 : 9.0,
        scale[theta_num]
        ),
    scan_tol::Float64 = 1e-3,
    loss_tol::Float64 = 0.,
    local_alg::Symbol = :LN_NELDERMEAD,
    max_iter::Int = 10^5,
    loss_grad::Union{Function, Symbol} = :EMPTY,
    silent::Bool = false,
    #kwargs... # other options for get_right_endpoint
)
    isLeft = direction == :left
    n_theta = length(theta_init)

    # checking arguments
    # theta_bound[1] < theta_init < theta_bound[2]
    theta_init_outside_theta_bounds = .! [theta_bounds[i][1] < theta_init[i] < theta_bounds[i][2] for i in 1:length(theta_init)]
    if any(theta_init_outside_theta_bounds)
        throw(ArgumentError("theta_init is outside theta_bound: $(findall(theta_init_outside_theta_bounds))"))
    end
    # scan_bound should be within theta_bounds
    !(theta_bounds[theta_num][1] < scan_bound < theta_bounds[theta_num][2]) &&
        throw(ArgumentError("scan_bound are outside of the theta_bounds $(theta_bounds[theta_num])"))
    # theta_init should be within scan_bound
    if (theta_init[theta_num] >= scan_bound && !isLeft) || (theta_init[theta_num] <= scan_bound && isLeft)
        throw(ArgumentError("init values are outside of the scan_bound $scan_bound"))
    end
    # 0 <= theta_bound[1] for :log
    less_than_zero_theta_bounds = (scale .== :log) .& [theta_bounds[i][1] < 0 for i in 1:length(theta_init)]
    if any(less_than_zero_theta_bounds)
        throw(ArgumentError(":log scaled theta_bound min is negative: $(findall(less_than_zero_theta_bounds))"))
    end
    # 0 <= theta_bounds <= 1 for :logit
    less_than_zero_theta_bounds = (scale .== :logit) .& [theta_bounds[i][1] < 0 || theta_bounds[i][2] > 1 for i in 1:length(theta_init)]
    if any(less_than_zero_theta_bounds)
        throw(ArgumentError(":logit scaled theta_bound min is outside range [0,1]: $(findall(less_than_zero_theta_bounds))"))
    end
    # loss_func(theta_init) < loss_crit
    !(loss_func(theta_init) < loss_crit) &&
        throw(ArgumentError("Check theta_init and loss_crit: loss_func(theta_init) should be < loss_crit"))

    # set counter in the scope
    counter::Int = 0
    # set supreme, maximal or minimal value of scanned parameter inside critical
    supreme_gd = nothing

    # progress info
    prog = ProgressUnknown("$direction CP counter:"; spinner=false, enabled=!silent, showspeed=true)

    # transforming loss
    theta_init_gd = scaling.(theta_init, scale)
    if isLeft theta_init_gd[theta_num] *= -1 end # change direction

    function loss_func_gd(theta_gd)
        theta_g = copy(theta_gd)
        if isLeft theta_g[theta_num] *= -1 end # change direction
        theta = unscaling.(theta_g, scale)
        # calculate function
        loss_norm = loss_func(theta) - loss_crit

        # update counter
        counter += 1
        # update supreme ?
        update_supreme = (loss_norm < 0.) &&
            (typeof(supreme_gd)==Nothing || (theta_gd[theta_num] > supreme_gd))
        if update_supreme
            supreme_gd = theta_gd[theta_num]
        end
        # display current
        supreme = if (isLeft && typeof(supreme_gd)!==Nothing)
            unscaling(-supreme_gd, scale[theta_num])
        else
            unscaling(supreme_gd, scale[theta_num])
        end
        ProgressMeter.update!(prog, counter, spinner="⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"; showvalues = [(:supreme,round(supreme; sigdigits=4))])

        return loss_norm
    end
    theta_bounds_gd = scaling.(theta_bounds, scale)
    if isLeft theta_bounds_gd[theta_num] = (-1*theta_bounds_gd[theta_num][2], -1*theta_bounds_gd[theta_num][1]) end # change direction

    scan_bound_gd = scaling(scan_bound, scale[theta_num])
    if isLeft scan_bound_gd *= -1 end # change direction

    # transform gradient
    loss_grad_gd = loss_grad
    loss_grad_gd = if isa(loss_grad, Function)
        function(theta_gd)
            theta_g = copy(theta_gd)
            if isLeft theta_g[theta_num] *= -1 end # change direction
            theta = unscaling.(theta_g, scale)
            loss_grad_value = loss_grad(theta)
            loss_grad_value_gd = zeros(n_theta)

            for i in 1:n_theta
                if scale[i] == :log
                    loss_grad_value_gd[i] = loss_grad_value[i] * theta[i] * log(10.)
                elseif scale[i] == :logit
                    loss_grad_value_gd[i] = loss_grad_value[i] * theta[i] * (1. - theta[i]) * log(10.)
                else # for :direct and :lin
                    loss_grad_value_gd[i] = loss_grad_value[i]
                end
            end

            if isLeft loss_grad_value_gd[theta_num] *= -1 end # change direction

            return loss_grad_value_gd
        end
    else
        loss_grad
    end

    # calculate endpoint using base method
    (optf_gd, pp_gd, status) = get_right_endpoint(
        theta_init_gd,
        theta_num,
        loss_func_gd,
        Val(method);
        theta_bounds = theta_bounds_gd,
        scan_bound = scan_bound_gd,
        scan_tol,
        loss_tol,
        local_alg,
        max_iter,
        loss_grad = loss_grad_gd
        #kwargs...
    )

    # transforming back
    if (isLeft && typeof(optf_gd)!==Nothing) optf_gd *= -1 end # change direction
    optf = unscaling(optf_gd, scale[theta_num])
    temp_fun = (pp::ProfilePoint) -> begin
        if isLeft pp.params[theta_num] *= -1 end # change direction
        ProfilePoint(
            unscaling(pp.params[theta_num], scale[theta_num]),
            pp.loss + loss_crit,
            unscaling.(pp.params, scale),
            pp.ret,
            pp.counter
        )
    end
    pps = [ temp_fun(pp_gd[i]) for i in 1:length(pp_gd) ]
    # transforming supreme back
    supreme = if (isLeft && typeof(supreme_gd)!==Nothing)
        unscaling(-supreme_gd, scale[theta_num])
    else
        unscaling(supreme_gd, scale[theta_num])
    end
    ProgressMeter.finish!(prog)

    EndPoint(optf, pps, status, direction, counter, supreme)
end

"""
    function get_endpoint(
        theta_init::Vector{Float64},
        scan_func::Function,
        loss_func::Function,
        method::Symbol,
        direction::Symbol = :right;

        loss_crit::Float64 = 0.0,
        scale::Vector{Symbol} = fill(:direct, length(theta_init)),
        theta_bounds::Vector{Tuple{Float64,Float64}} = unscaling.(
            fill((-Inf, Inf), length(theta_init)),
            scale
            ),
        scan_bound::Float64 = unscaling(
            (direction==:left) ? -9.0 : 9.0,
            scale[theta_num]
            ),
        scan_tol::Float64 = 1e-3,
        loss_tol::Float64 = 0.,
        local_alg::Symbol = :LN_NELDERMEAD,
        kwargs...
        )

Calculates confidence interval's right or left endpoints for a function of parameters `scan_func`.

## Return
[`EndPoint`](@ref) object storing confidence interval's endpoint and intermediate profile points.

## Arguments
- `theta_init`: starting values of parameter vector ``\\theta``. The starting values should not necessary be the optimum values of `loss_func` but `loss_func(theta_init)` should be lower than `loss_crit`.
- `scan_func`: function of parameters.
- `loss_func`: loss function ``\\Lambda\\left(\\theta\\right)`` for profile likelihood-based (PL) identification. Usually we use log-likelihood for PL analysis: ``\\Lambda( \\theta ) = - 2 ln\\left( L(\\theta) \\right)``.
- `method`: computational method to estimate confidence interval's endpoint. Currently the only supported method is: `:CICO_ONE_PASS`.
- `direction`: `:right` or `:left` endpoint to estimate.

## Keyword arguments
- `loss_crit`: critical level of loss function. Confidence interval's endpoint value is the intersection point of profile likelihood and `loss_crit` level.
- `scale`: vector of scale transformations for each parameters' component. Possible values: `:direct` (`:lin`), `:log`, `:logit`. This option can speed up the optimization, especially for wide `theta_bounds`. The default value is `:direct` (no transformation) for all parameters.
- `theta_bounds`: vector of tuple `(lower_bound, upper_bound)` for each parameter. Bounds define the ranges for possible parameter values. Default bounds are `(-Inf,Inf)`.
- `scan_bound`: value which states the area of confidence point analysis.
- `scan_tol`: Absolute tolerance for `theta_num` parameter used as termination criterion.
- `loss_tol`: Absolute tolerance controlling `loss_func` closenes to `loss_crit` (termination criterion). Currently doesn't work for `:CICO_ONE_PASS` method because of limitation in `LN_AUGLAG` interface.
- `local_alg`: algorithm of optimization. Derivative-free and gradient-based algorithms form NLopt package.
- `max_iter` : maximal number of fitter iterations. If reaches the result status will be `:MAX_ITER_STOP`.
- `scan_grad` : For gradient optimization methods it is necessary to set how the gradient of `scan_func` should be calculated.
    - `:EMPTY` (default) no gradient is set. It works only for gradient-free methods.
    - `:AUTODIFF` means autodifferentiation from `ForwardDiff` package is used.
    - `:FINITE` means finite difference method from `Calculus` is used.
    - `function(x::Vector{Float64})` which returns gradient vector.
- `loss_grad` : For gradient optimization methods it is necessary to set how the gradient of `loss_func` should be calculated.
    - `:EMPTY` (default) no gradient is set. It works only for gradient-free methods.
    - `:AUTODIFF` means autodifferentiation from `ForwardDiff` package is used.
    - `:FINITE` means finite difference method from `Calculus` is used.
    - `function(x::Vector{Float64})` which returns gradient vector.
- `silent` : Boolean argument declaring whether we display the optimization progress. Default is `false`

See also [`get_interval`](@ref)
"""
function get_endpoint(
    theta_init::Vector{Float64},
    scan_func::Function,
    loss_func::Function,
    method::Symbol,
    direction::Symbol = :right;

    loss_crit::Float64 = 0.0,
    # :direct, :lin, :log, :logit
    scale::Vector{Symbol} = fill(:direct, length(theta_init)),
    theta_bounds::Vector{Tuple{Float64,Float64}} = unscaling.(
        fill((-Inf, Inf), length(theta_init)),
        scale
        ),
    scan_bound::Float64 = (direction==:left) ? -1e9 : 1e9, # log scan bound is not implemented
    scan_tol::Float64 = 1e-3,
    loss_tol::Float64 = 0.,
    local_alg::Symbol = :LN_NELDERMEAD,
    max_iter::Int = 10^5,
    scan_grad::Union{Function, Symbol} = :EMPTY,
    loss_grad::Union{Function, Symbol} = :EMPTY,
    silent::Bool = false,
    #kwargs... # other options for get_right_endpoint
)
    isLeft = direction == :left
    n_theta = length(theta_init)
    loss_init = loss_func(theta_init)

    # checking arguments
    # theta_bound[1] < theta_init < theta_bound[2]
    theta_init_outside_theta_bounds = .! [theta_bounds[i][1] < theta_init[i] < theta_bounds[i][2] for i in 1:length(theta_init)]
    if any(theta_init_outside_theta_bounds)
        throw(ArgumentError("theta_init is outside theta_bound: $(findall(theta_init_outside_theta_bounds))"))
    end

    # 0 <= theta_bound[1] for :log
    less_than_zero_theta_bounds = (scale .== :log) .& [theta_bounds[i][1] < 0 for i in 1:length(theta_init)]
    if any(less_than_zero_theta_bounds)
        throw(ArgumentError(":log scaled theta_bound min is negative: $(findall(less_than_zero_theta_bounds))"))
    end
    # 0 <= theta_bounds <= 1 for :logit
    less_than_zero_theta_bounds = (scale .== :logit) .& [theta_bounds[i][1] < 0 || theta_bounds[i][2] > 1 for i in 1:length(theta_init)]
    if any(less_than_zero_theta_bounds)
        throw(ArgumentError(":logit scaled theta_bound min is outside range [0,1]: $(findall(less_than_zero_theta_bounds))"))
    end
    # loss_func(theta_init) < loss_crit
    !(loss_init < loss_crit) &&
        throw(ArgumentError("Check theta_init and loss_crit: loss_func(theta_init) should be < loss_crit"))

    # set counter in the scope
    prog = ProgressUnknown("Fitter counter:"; spinner=false, enabled=!silent, showspeed=true)
    counter::Int = 0
    # set supreme, maximal or minimal value of loss_func inside critical
    supreme_gd = nothing
    scan_val_gd = nothing

    # transforming
    theta_init_gd = scaling.(theta_init, scale)

    function scan_func_gd(theta_gd)
        theta = unscaling.(theta_gd, scale)
        scan_val = scan_func(theta)
        scan_val_gd = isLeft ? (-1)*scan_val : scan_val

        return scan_val_gd
    end

    function loss_func_gd(theta_gd)
        #theta_g = copy(theta_gd) # why copy?
        theta = unscaling.(theta_gd, scale)
        loss_value = loss_func(theta) - loss_crit

        counter += 1
        #println("$loss_value => $supreme_gd")
        should_update = (loss_value < 0.) &&
            !isa(scan_val_gd, Nothing) &&
            (isa(supreme_gd, Nothing) || (scan_val_gd > supreme_gd)) &&
            !isa(scan_val_gd, ForwardDiff.Dual)
        if should_update
            supreme_gd = scan_val_gd
        end
        supreme = if isa(supreme_gd, Nothing)
            "-"
        elseif isLeft
            round(-supreme_gd; sigdigits=4)
        else
            round(supreme_gd; sigdigits=4)
        end
        ProgressMeter.update!(prog, counter; showvalues = [(:supreme,supreme)])

        return loss_value
    end
    theta_bounds_gd = scaling.(theta_bounds, scale)

    # TODO: transformed by scan_scale: scaling(scan_bound, scan_scale)
    scan_bound_gd = scan_bound
    if isLeft scan_bound_gd *= -1 end # change direction

    # transform gradients
    scan_grad_gd = if isa(scan_grad, Function)
        function(theta_gd)
            theta = unscaling.(theta_gd, scale)
            scan_grad_value = scan_grad(theta)
            scan_grad_value_gd = zeros(n_theta)

            for i in 1:n_theta
                if scale[i] == :log
                    scan_grad_value_gd[i] = scan_grad_value[i] * theta[i] * log(10.)
                elseif scale[i] == :logit
                    scan_grad_value_gd[i] = scan_grad_value[i] * theta[i] * (1. - theta[i]) * log(10.)
                else # for :direct and :lin
                    scan_grad_value_gd[i] = scan_grad_value[i]
                end
            end

            return scan_grad_value_gd
        end
    else
        scan_grad
    end

    loss_grad_gd = if isa(loss_grad, Function)
        function(theta_gd)
            theta = unscaling.(theta_gd, scale)
            loss_grad_value = loss_grad(theta)
            loss_grad_value_gd = zeros(n_theta)

            for i in 1:n_theta
                if scale[i] == :log
                    loss_grad_value_gd[i] = loss_grad_value[i] * theta[i] * log(10.)
                elseif scale[i] == :logit
                    loss_grad_value_gd[i] = loss_grad_value[i] * theta[i] * (1. - theta[i]) * log(10.)
                else # for :direct and :lin
                    loss_grad_value_gd[i] = loss_grad_value[i]
                end
            end

            return loss_grad_value_gd
        end
    else
        loss_grad
    end

    # calculate endpoint using base method
    (optf_gd, pp_gd, status) = get_right_endpoint(
        theta_init_gd,
        scan_func_gd,
        loss_func_gd,
        Val(method);

        theta_bounds = theta_bounds_gd,
        scan_bound = scan_bound_gd,
        scan_tol,
        loss_tol,
        local_alg,
        max_iter,
        scan_grad = scan_grad_gd,
        loss_grad = loss_grad_gd,
        #kwargs...
    )

    # transforming back
    temp_fun = (pp::ProfilePoint) -> begin
        value = isLeft ? (-1)*pp.value : pp.value # TODO: scan_scale
        ProfilePoint(
            value,
            pp.loss + loss_crit,
            unscaling.(pp.params, scale),
            pp.ret,
            pp.counter
        )
    end
    pps = [ temp_fun(pp_gd[i]) for i in 1:length(pp_gd) ]
    # transforming supreme back
    if (isLeft && typeof(supreme_gd)!==Nothing) supreme_gd *= -1 end # change direction

    if (isLeft && typeof(optf_gd)!==Nothing) optf_gd *= -1 end # change direction
    # optf = unscaling(optf_gd, scan_scale)
    optf = optf_gd

    EndPoint(optf, pps, status, direction, counter, supreme_gd)
end


module IntervalAnalysis
using Plots
using Base
using Random
using Symbolics
using Documenter

export Interval,*,+,-,subdivide

"""
    Interval

Defines a single interval between two real numbers.

# Examples
```jldoctest
julia> Interval(-1,1)
-1.0 ←-→ 1.0
```
"""
struct Interval
	"Lower interval value"
	l::Real
	"Upper interval value"
	u::Real
	Interval(l,u) = l > u ? new(u,l) : new(l,u)

end

Base.show(io::IO, x::Interval) = print(io,round(x.l,digits=6)," ←-→ ",round(x.u,digits=6))

"""

	+(x₁::Interval, x₂::Interval)

Adds two intervals

# Examples
```jldoctest
julia> Interval(-1,1) + Interval(-2,4)
-3.0 ←-→ 5.0
```
"""
function Base.:+(x₁::Interval, x₂::Interval)
	Interval(x₁.l + x₂.l,x₁.u + x₂.u)
    end

function Base.:+(x₁::Real, x₂::Interval)
	Interval(x₁ + x₂.l,x₁ + x₂.u)
    end

function Base.:+(x₁::Interval, x₂::Real)
	Interval(x₁.l + x₂,x₁.u + x₂)
    end

 """

	-(x₁::Interval, x₂::Interval)

Subtracts one interval from another

# Examples
```jldoctest
julia> Interval(-1,1) - Interval(-2,4)
-5.0 ←-→ 3.0
```
"""
function Base.:-(x₁::Interval, x₂::Interval)
	Interval(x₁.l - x₂.u,x₁.u - x₂.l)
    end

function Base.:-(x::Interval)
	Interval(-x.l,-x.u)
    end

function Base.:-(x₁::Interval, x₂::Real)
	Interval(x₁.l - x₂,x₁.u - x₂)
    end

    function Base.:-(x₁::Real, x₂::Interval)
	Interval(x₁ - x₂.l,x₁ - x₂.u)
    end

"""

	*(x₁::Interval, x₂::Interval)

Multiplies two intervals

# Examples
```jldoctest
julia> Interval(-1,1) * Interval(-2,4)
-4.0 ←-→ 4.0
```
"""
function Base.:*(x₁::Interval, x₂::Interval)
	if 0 ≤ x₁.l
		if 0 ≤ x₂.l
			return Interval(x₁.l*x₂.l,x₁.u*x₂.u)
		elseif x₂.l < 0 < x₂.u
			return Interval(x₁.u*x₂.l,x₁.u*x₂.u)
		elseif x₂.u ≤ 0
			return Interval(x₁.u*x₂.l,x₁.l*x₂.u)
		end
	elseif x₁.l < 0 < x₁.u
		if 0 ≤ x₂.l
			return Interval(x₁.l*x₂.u,x₁.u*x₂.u)
		elseif x₂.u ≤ 0
			return Interval(x₁.u*x₂.l,x₁.l*x₂.l)
		elseif x₂.l < 0 < x₂.u
			return Interval(min(x₁.l*x₂.u,x₁.u*x₂.l),max(x₁.l*x₂.l,x₁.u*x₂.u))
		end
	elseif x₁.u ≤ 0
		if 0 ≤ x₂.l
			return Interval(x₁.l*x₂.u,x₁.u*x₂.l)
		elseif x₂.l < 0 < x₂.u
			return Interval(x₁.l*x₂.u,x₁.l*x₂.l)
		elseif x₂.u ≤ 0
			return Interval(x₁.u*x₂.u,x₁.l*x₂.l)
		end
	end
end

function Base.:*(x₁::Interval,x₂::Real)
	Interval(x₁.l*x₂,x₁.u*x₂)
end

function Base.:*(x₁::Real,x₂::Interval)
	Interval(x₁*x₂.l,x₁*x₂.u)
end

"""

	/(x₁::Interval, x₂::Interval)

Divides two intervals

# Examples
```jldoctest
julia> Interval(-1,1) / Interval(-2,4)
-0.5 ←-→ 0.5
```
"""
function Base.:/(x₁::Interval, x₂::Interval)
	x₁ * Interval(1/x₂.u,1/x₂.l)
    end

function Base.:/(x₁::Real, x₂::Interval)
	Interval(x₁/x₂.u,x₁/x₂.l)
    end

function Base.:/(x₁::Interval, x₂::Real)
	Interval(x₁.l/x₂,x₁.u/x₂)
    end

function w(x::Interval)
	x.u-x.l
end

function w(X::Vector{Interval})
	widths = w.(X)
	d,i = findmax(widths)
end

function m(x::Interval)
	(x.u+x.l)/2
end

function m(X::Vector{Interval})
	m.(X)
end

function Base.sqrt(x::Interval)
	Interval(sqrt(x.l),sqrt(x.u))
end

function Base.:^(x::Interval,p::Integer)
	if x.l > 0 || p%2 != 0
		return Interval(x.l^p,x.u^p)
	elseif x.u < 0 && p%2 == 0
		return Interval(x.u^p,x.l^p)
	else
		return Interval(0,abs(x)^p)
	end
end

function Base.exp(x::Interval)
	Interval(exp(x.l),exp(x.u))
end

function Base.log(x::Interval)
	Interval(log(x.l),log(x.u))
end

function Base.log10(x::Interval)
	Interval(log10(x.l),log10(x.u))
end

function Base.log2(x::Interval)
	Interval(log2(x.l),log2(x.u))
end

function Base.sin(x::Interval)
	if x.l ≥ -(π/2) && x.u ≤ π/2
			return Interval(sin(x.l),sin(x.u))
	end
	return Interval(-1,1)
end

function Base.cos(x::Interval)
	if x.l ≥ 0 && x.u ≤ π
			return Interval(cos(x.l),cos(x.u))
	end
	return Interval(-1,1)
end

function Base.abs(x::Interval)
	max(abs(x.l),abs(x.u))
end

function qdist(x₁::Interval,x₂::Interval)
	max(abs(x₁.l-x₂.l),abs(x₁.u-x₂.u))
end

"""

	subdivide(x::Interval,N::Integer)

Subdivides an interval into N intervals.

# Examples
```jldoctest
julia> subdivide(Interval(0,1),5)
[0.0 ←-→ 0.2, 0.2 ←-→ 0.4, 0.4 ←-→ 0.6, 0.6 ←-→ 0.8, 0.8 ←-→ 1.0]
```
"""
function subdivide(x::Interval,N::Integer)
	A = Array{Interval}(undef,N)
	w_x = w(x)/N
	for j ∈ 1:N
		A[j] = Interval(x.l+(j-1)*w_x,x.l+j*w_x)
	end
	return A
end

function subdivide_vector(X::Vector{Interval},N::Integer)
	A = Array{Vector{Interval}}(undef,length(X))
	for i ∈ 1:length(X)
		A[i] = subdivide(X[i],N)
	end
	return A
end

function bisect_box(X::Vector{Interval})
	# bisects a box over the largest axis
	d,i = w(X)
	X_s = subdivide(X[i],2)
	X₁ = copy(X)
	X₁[i] = X_s[1]
	X₂ = copy(X)
	X₂[i] = X_s[2]
	return X₁,X₂
end

function hull(X::Vector{Interval})
	l = [x.l for x ∈ X]
	u = [x.u for x ∈ X]
	Interval(minimum(l),maximum(u))
end

function refinement(x,f,N)
	X = subdivide(x,N)
	Y = f.(X)
	hull(Y)
end

function plot_refinement(x,f,∇f,N)
	plot(f,x.l,x.u,c="black",linewidth=3,label="")
	X = subdivide(x,N)
	Y = f.(X)
	Yₘᵥ = mean_value_form.(f,∇f,X)
	for i ∈ 1:length(X)
		if i == 1
			plot!([X[i].l,X[i].u],[Y[i].l,Y[i].l],fillrange=[Y[i].u,Y[i].u],linewidth=0,c="black",label="Interval Extension",fillalpha=0.15)
			plot!([X[i].l,X[i].u],[Yₘᵥ[i].l,Yₘᵥ[i].l],fillrange=[Yₘᵥ[i].u,Yₘᵥ[i].u],linewidth=0,c="red",label="Mean Value Form",fillalpha=0.25)
		else
			plot!([X[i].l,X[i].u],[Y[i].l,Y[i].l],fillrange=[Y[i].u,Y[i].u],linewidth=0,c="black",label="",fillalpha=0.15)
			plot!([X[i].l,X[i].u],[Yₘᵥ[i].l,Yₘᵥ[i].l],fillrange=[Yₘᵥ[i].u,Yₘᵥ[i].u],linewidth=0,c="red",label="",fillalpha=0.25)
		end

	end
	display(plot!(dpi=600))
end

function skelboe_moore(f,X)

	fᵤ = f(X).u # smallest upper bound (AKA the only upper bound at this stage)

	# creating a store of intervals, mean values, and interval values
	L = [Dict("X"=>X,"f_m"=>f(m(X)),"F"=>f(X))]
	Lᵣ = []
	it_count = 0

	# until tolerance...
	for it ∈ 1:30

		# initialise list of intervals to sack off
		del_index =[]
		for i in 1:length(L)
			# if the lowest interval value is above the smallest upper bound..
			if L[i]["F"].l > fᵤ
				# remove interval, we know global minimum is not here...
				push!(del_index,i)
			end
		end

		# delete all the ones we said we would
		for d_i ∈ del_index
			push!(Lᵣ,L[d_i])
		end
		deleteat!(L,del_index)

		# pick the remaining interval with smallest mean value
		s = argmin([l["f_m"] for l ∈ L])

		# divide the box into two
		X_s = bisect_box(L[s]["X"])

		# get rid of the containing interval
		deleteat!(L,s)

		for x ∈ X_s
			# evaluate interval values
			F = f(x)

			# check if we find a smallest upper bound
			if F.u < fᵤ
				fᵤ = F.u
			end
			# add new interval information to store
			push!(L,Dict("X"=>x,"f_m"=>f(m(x)),"F"=>F))

		end

	end
	return L,Lᵣ
end

function plot_boxes(L,Lᵣ)
	for l ∈ L
		x1 = l["X"][1]
		x2 = l["X"][2]
		plot!(Shape([x1.l,x1.l,x1.u,x1.u] , [x2.u,x2.l,x2.l,x2.u]),label="",linewidth=1,alpha=0,linealpha=1)
	end
	for l ∈ Lᵣ
		x1 = l["X"][1]
		x2 = l["X"][2]
		plot!(Shape([x1.l,x1.l,x1.u,x1.u] , [x2.u,x2.l,x2.l,x2.u]),c="black",alpha=0.55,label="",linewidth=1)
	end
end

function mean_value_form(f,∇f,X)
	if typeof(X) == Interval
		D = ∇f(X)[1]
	else
		D = ∇f(X)
	end
	mean = m(X)
	fₘ = f(mean)
	if typeof(X) == Interval
		return fₘ + D*(X-mean)
	else
		return fₘ + sum(D[i]*(X[i]-mean[i]) for i ∈ 1:length(X))
	end

end

function build_gradient(f,len_x)
	if len_x > 1
		@variables x[1:len_x]
		∇f = Symbolics.gradient(f(x),[x[i] for i ∈ 1:len_x])
		∇f_expr = build_function(∇f, x)
		∇f = eval(∇f_expr[1])
		return ∇f
	else
		@variables x
		∇f = Symbolics.gradient(f(x),[x])
		∇f_expr = build_function(∇f, x)
		∇f = eval(∇f_expr[1])
		return ∇f
	end

end

function g(x1,x2)
	return x1*exp(x1+x2^2)-x2^2
end

function g_vec(X)
	return g(X[1],X[2])
end

# f(x) = x^3 - x^2
# ∇f = build_gradient(f,1)
# plot_refinement(Interval(0,1),f,∇f,20)
# print(Interval(-1,1) / Interval(-2,4))

end


using DiffEqFlux, OrdinaryDiffEq, Flux, Optim, Test

# General loss function to compare single shooting and multiple shooting predictions
function general_loss_function(result_neuralode)
	return sum(abs2, (ode_data[:,:] .- Array(prob_neuralode(u0, result_neuralode.minimizer)) ))
end

# Define initial conditions and timesteps
datasize = 30
u0 = Float32[2.0, 0.0]
tspan = (0.0f0, 5.0f0)
tsteps = range(tspan[1], tspan[2], length = datasize)

# Get the data
function trueODEfunc(du, u, p, t)
    true_A = [-0.1 2.0; -2.0 -0.1]
    du .= ((u.^3)'true_A)'
end
prob_trueode = ODEProblem(trueODEfunc, u0, tspan)
ode_data = Array(solve(prob_trueode, Tsit5(), saveat = tsteps))

# Define the Neural Network
dudt2 = FastChain((x, p) -> x.^3,
                  FastDense(2, 16, tanh),
                  FastDense(16, 2))
prob_neuralode = NeuralODE(dudt2, (0.0,5.0), Tsit5(), saveat = tsteps)

function loss_neuralode(p)
    pred = Array(prob_neuralode(u0, p))
    loss = sum(abs2, (ode_data[:,1:size(pred,2)] .- pred))
    return loss, pred
end

result_neuralode = DiffEqFlux.sciml_train(loss_neuralode, prob_neuralode.p,
                                          ADAM(0.05),
										  maxiters = 300)

single_shoot_loss = general_loss_function(result_neuralode)
println("single_shoot_loss: ",single_shoot_loss)

# Define parameters for Multiple Shooting
grp_size_param = 1
loss_multiplier_param = 100

neural_ode_f(u,p,t) = dudt2(u,p)
prob_param = ODEProblem(neural_ode_f, u0, tspan, initial_params(dudt2))

function loss_function_param(ode_data, pred):: Float32
	return sum(abs2, (ode_data .- pred))^2
end

function loss_neuralode_param(p)
	return multiple_shoot(p, ode_data, tsteps, prob_param, loss_function_param, grp_size_param, loss_multiplier_param)
end

multiple_shoot_result_neuralode_1 = DiffEqFlux.sciml_train(loss_neuralode_param, prob_neuralode.p,
                                          ADAM(0.05),
                                          maxiters = 300)

multiple_shoot_loss_1 = general_loss_function(multiple_shoot_result_neuralode_1)
println("multiple_shoot_loss_1: ",multiple_shoot_loss_1)

# test for grp_size = 1
@test multiple_shoot_loss_1 < single_shoot_loss

# test for grp_size = 5
grp_size_param = 5
multiple_shoot_result_neuralode_2 = DiffEqFlux.sciml_train(loss_neuralode_param, prob_neuralode.p,
                                          ADAM(0.05),
                                          maxiters = 300)

multiple_shoot_loss_2 = general_loss_function(multiple_shoot_result_neuralode_2)
println("multiple_shoot_loss_2: ",multiple_shoot_loss_2)

@test multiple_shoot_loss_2 < single_shoot_loss


using LinearAlgebra
using ForwardDiff
using Test

export newton_raphson
export test_newton_raphson

∇(f, x) = ForwardDiff.jacobian(f, x)

function newton_raphson(f::Function, x0::Vector{Float64}, ϵ::Float64)
    xs = [x0]
    d = inv(-∇(f, x0)) * f(x0)
    while norm(d) > ϵ
        x = xs[end] + d
        d = inv(-∇(f, x)) * f(x)
        push!(xs, x)
    end
    return xs
end

function test_newton_raphson()
    f(x) = [x[1]^2 + x[2]^2 - 4*x[3] - 0;
            x[1]^2 + x[3]^2 + 0*x[3] - (1/4);
            x[1]^2 + x[2]^2 + x[3]^2 - 1]
    x0 = [1.0, 1.0, 1.0]
    ϵ = 0.001
    x = newton_raphson(f, x0, ϵ)
    @info "Newton-Raphson iterations:"
    @info x
    @test isapprox(norm(f(x[end])), 0; atol=ϵ)
end


import Base: exp, log, sqrt, ^
import Base: cos, sin, tan, cosh, sinh, tanh, acos, asin, atan, acosh, asinh, atanh

function exp(q::Quaternion)
    z = sqrt(q.q1*q.q1 + q.q2*q.q2 + q.q3*q.q3)
    return exp(q.q0)*(cos(z) + imag(q)*(z != zero(q.q0) ? sin(z)/z : one(q.q0)))
end

function log(q::Quaternion)
    z = abs(imag(q))
    return log(abs(q)) + ((imag(q)!=zero(q)) ? imag(q)/z*atan(z,q.q0) :
    q.q0 < zero(q.q0) ? quaternion(pi*im) : zero(q))
end

function cos(q::Quaternion)
    z = abs(imag(q))
    return cos(q.q0)*cosh(z)-imag(q)*sin(q.q0)*(z != zero(q.q0) ? sinh(z)/z : one(q.q0))
end

function sin(q::Quaternion)
    z = abs(imag(q))
    return sin(q.q0)*cosh(z)+imag(q)*cos(q.q0)*(z != zero(q.q0) ? sinh(z)/z : one(q.q0))
end

function tan(q::Quaternion)
    return sin(q)/cos(q)
end

function cosh(q::Quaternion)
    return (exp(q)+exp(-q))/2
end

function sinh(q::Quaternion)
    return (exp(q)-exp(-q))/2
end

function tanh(q::Quaternion)
    return sinh(q)/cosh(q)
end

function sqrt(q::Quaternion)
    return exp(log(q)/2)
end

function (^)(q::Quaternion, p::AbstractFloat)
    return exp(p*log(q))
end

function (^)(q::Quaternion, p::Quaternion)
    return exp(p*log(q))
end

function acos(q::Quaternion)
    x = imag(q)
    y = abs(x)
    z = y!=zero(q.q0) ? x/y : quaternion(one(q.q0)*im)

    return -z*log(q + z*sqrt(one(q.q0) - q*q))
    # return -z*log(q - z*sqrt(one(q.q0) - q*q))
end

function asin(q::Quaternion)
    x = imag(q)
    y = abs(x)
    z = y!=zero(q.q0) ? x/y : quaternion(one(q.q0)*im)

    return -z*log(z*q + sqrt(one(q.q0) - q*q))
    #= return -z*log(z*q - sqrt(one(q.q0) - q*q)) =#
end

function atan(q::Quaternion)
    x = imag(q)
    y = abs(x)
    z = y!=zero(q.q0) ? x/y : quaternion(one(q.q0)*im)

    return -z*log((one(q.q0)+z*q)/(one(q.q0)-z*q))/2
end


"""
    erlang_rng(k, λ, shape=1; seed=nothing)

Generate a `shape` element array of random variables from a Erlang_{`k`}(`λ`) distribution. Optionally you can set a specific seed.

# Notes

The pdf of an Erlang_{k}(λ) distribution is given as:

``f(x, k, λ) = \\frac{λ^k e^{-λx} x^{k-1}}{(k-1)!} \\quad x ≥ 0``

# Examples

```julia-repl
julia> erlang_rng(5, .5)
1-element Vector{Float64}:
 10.803989701023117

julia> erlang_rng(3, 1, (2,2))
2×2×1 Array{Float64, 3}:
[:, :, 1] =
 2.19956  4.18505
 5.46892  2.5633
```

# References

D. Goldsman, P. Goldsman. A first course in probability and statistics. 2021.

L. Martino, D. Luengo. Extremely efficient generation of Gamma random variables for α ≥ 1. 2013.
"""
function erlang_rng(k::Int, λ::Real, shape::Union{Int, Tuple{Vararg{Int}}}=1; seed::Union{Int, Nothing}=nothing)
    U = get_std_uniform((shape..., k), seed=seed)
    k_dim = ndims(U)  # k is final dimension
    X = (-1/λ) .* log.(prod(U, dims=k_dim))  # Here (-1/λ) represents mean
    return X
end


## Spectral clustering on artists
using DelimitedFiles
using SparseArrays
names = readlines("../2-unit-1-demos/artistsim.names")
data = readdlm("../2-unit-1-demos/artistsim.smat")
xy = readdlm("artistsim.xy")
A = sparse(Int.(data[2:end,1]).+1,
           Int.(data[2:end,2]).+1,
           Int.(data[2:end,3]),
           Int(data[1,1]),Int(data[1,1]))
n = size(A,1)
##
function simplepagerank(A,α,v)
  @assert(0 ≤ α < 1, "needs probably α")
  @assert(all(vi -> vi ≥ 0, v), "needs non-negative v")
  v = v ./ sum(v) # we can normalize for them.
  d = vec(sum(A,dims=2)) # compute the degrees
  x = copy(v) # start of with v
  nsteps = 2*ceil(Int,log(eps(1.0))/log(α)) # upper bound on steps
  for i=1:nsteps
    x = α*(A'*(x./d)) .+ (1-α).*v
  end
  return x./sum(x) # renormalize to probability
end
##
using MatrixNetworks
pr = MatrixNetworks.pagerank(A,0.85)
##
x = simplepagerank(A,0.85,ones(n)./n)
##
using LinearAlgebra
@show norm(x-pr)

##
@time pr = MatrixNetworks.pagerank(A,0.85)
@time x = simplepagerank(A,0.85,ones(n)./n)
@show norm(x-pr)
## Let's watch PageRank converge
using Plots
function pagerank_converence_animation(A,xy,α,nsteps)
  n = size(A,1)
  v = ones(n)/n
  @assert(0 ≤ α < 1, "needs probably α")
  @assert(all(vi -> vi ≥ 0, v), "needs non-negative v")
  v = v ./ sum(v) # we can normalize for them.
  d = vec(sum(A,dims=2)) # compute the degrees
  x = copy(v) # start of with v

  @assert(nsteps <= size(A,1))
  anim = @animate for i=1:nsteps
    x = α*(A'*(x./d)) .+ (1-α).*v
    f2 = copy(x)
    f2 = f2/sum(f2)
    p = sortperm(f2) # this gets things to draw in the right z-order
    scatter(xy[p,1],xy[p,2], marker_z=-abs.(log10.(f2[p])).^0.5,colorbar=false,alpha=0.5,
      color=:magma,
      size=(1200,1200),
      markerstrokecolor=nothing,markerstrokewidth=0,label="",framestyle=:none,background=:black)
  end
  return anim
end
anim = pagerank_converence_animation(A,xy,0.85,25)
gif(anim,"pagerank-convergence-artistsim.gif",fps=5)

## Change between iterates
function pagerank_converence_plot(A,xy,α,nsteps)
  n = size(A,1)
  v = ones(n)/n
  @assert(0 ≤ α < 1, "needs probably α")
  @assert(all(vi -> vi ≥ 0, v), "needs non-negative v")
  v = v ./ sum(v) # we can normalize for them.
  d = vec(sum(A,dims=2)) # compute the degrees
  x = copy(v) # start of with v
  y = copy(v)
  hist = zeros(0)
  for i=1:nsteps
    x = α*(A'*(x./d)) .+ (1-α).*v
    push!(hist, norm(y - x/sum(x),1))
    y = x/sum(x)
  end
  return hist
end
hist = pagerank_converence_plot(A,xy,0.85,100)
##
pyplot()
theme(:dark)
plot(hist, yscale=:log10, legend=false, size=(300,300))
xlabel!("Iteration")
ylabel!("1-norm change in normalized vector")
savefig("pagerank-convergence-artistsim-norm.pdf")
##
x85 = simplepagerank(A,0.85,ones(n)./n)
x99 = simplepagerank(A,0.99,ones(n)./n)
x5 = simplepagerank(A,0.5,ones(n)./n)
##

## Show a few different vectors
gr()
p=sortperm(x99)
scatter(xy[p,1],xy[p,2],marker_z=-abs.(log10.(x99[p])).^0.5,markerstrokewidth=0,
color=:magma)
##
gr()
p=sortperm(x5)
scatter(xy[p,1],xy[p,2],marker_z=-abs.(log10.(x5[p])).^0.5,markerstrokewidth=0,
color=:magma)

## Now show the entire graph
using GraphRecipes
using LinearAlgebra
using Plots

ei, ej = findnz(A)
graphplot(ei, ej, x =xy[:,1], y=xy[:,2],
  markercolor=:black, markerstrokecolor=:white,
  size=(1200,1200),dpi=300,
  markersize=0, linecolor=1, linealpha=0.01, linewidth=0.5,
  markeralpha=0.2,colorbar=false,
  axis_buffer=0.02, background=nothing)
p=sortperm(x5)
scatter!(xy[p,1],xy[p,2],marker_z=-abs.(log10.(x5[p])).^0.5,markerstrokewidth=0)
savefig("artistsim-pagerank-50.png")

##
ei, ej = findnz(A)
graphplot(ei, ej, x =xy[:,1], y=xy[:,2],
  markercolor=:black, markerstrokecolor=:white,
  size=(1200,1200),dpi=300,
  markersize=0, linecolor=1, linealpha=0.01, linewidth=0.5,
  markeralpha=0.2,colorbar=false,
  axis_buffer=0.02, background=nothing)
p=sortperm(x99)
scatter!(xy[p,1],xy[p,2],marker_z=-abs.(log10.(x99[p])).^0.5,markerstrokewidth=0)
savefig("artistsim-pagerank-99.png")


### A Pluto.jl notebook ###
# v0.12.16

using Markdown
using InteractiveUtils

# ╔═╡ e077ab16-22e5-11eb-15ed-5f0654bf7328
using Pkg, DrWatson

# ╔═╡ e75a9a06-22e5-11eb-2e36-8d4f62b830ed
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using Random, StatsBase, Distributions, Plots, LaTeXStrings
	Random.seed!(0)
end

# ╔═╡ ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
md"## Listing5.16"

# ╔═╡ fa68607e-22e5-11eb-0558-c9a4d9f77426
begin
	mu0, mu1a, mu1b, mu1c, sd = 15, 16, 18, 20, 2
	tauGrid = 5:0.1:25

	dist0 = Normal(mu0,sd)
	dist1a, dist1b, dist1c  = Normal(mu1a,sd), Normal(mu1b,sd), Normal(mu1c,sd)

	falsePositive = ccdf.(dist0,tauGrid)
	truePositiveA, truePositiveB, truePositiveC =
	    ccdf.(dist1a,tauGrid), ccdf.(dist1b,tauGrid), ccdf.(dist1c,tauGrid)
end

# ╔═╡ a80f7e58-3af9-11eb-04b3-57fcb9ee857c
begin
	plot(falsePositive, [truePositiveA truePositiveB truePositiveC],
		c=[:blue :red :green],
		label=[L"H1a: \mu_1 = 16" L"H1b: \mu_1 = 18" L"H1c: \mu_1 = 20"])
	plot!([0,1], [0,1], c=:black, ls=:dash, label="H0 = H1 = 15",
		xlims=(0,1), ylims=(0,1), xlabel=L"\alpha", ylabel="Power",
		ratio=:equal, legend=:bottomright)
end

# ╔═╡ 475ff888-22e6-11eb-2354-09f8f40a8e12
md"## End of listing5.16"

# ╔═╡ Cell order:
# ╟─ca5fbdc8-22e5-11eb-0a60-cb0a127a3ca5
# ╠═e077ab16-22e5-11eb-15ed-5f0654bf7328
# ╠═e75a9a06-22e5-11eb-2e36-8d4f62b830ed
# ╠═fa68607e-22e5-11eb-0558-c9a4d9f77426
# ╠═a80f7e58-3af9-11eb-04b3-57fcb9ee857c
# ╟─475ff888-22e6-11eb-2354-09f8f40a8e12


using Concorde
using Test

@testset "Concorde.jl" begin
    @testset "Symmetric TSP" begin
        M = [
             0  16   7  14
            16   0   3   5
             7   3   0  16
            14   5  16   0
        ]
        opt_tour, opt_len = solve_tsp(M)
        @show opt_tour, opt_len
        @test opt_len == 29
    end

    @testset "Asymmetric TSP" begin
        M = [
             0   1   7  14
            16   0   1   5
             7   5   0   1
             1   3  16   0
        ]
        @test_throws ErrorException solve_tsp(M)
    end

    @testset "Coordinates" begin
        n_nodes = 10
        x = rand(n_nodes) .* 10000
        y = rand(n_nodes) .* 10000
        opt_tour, opt_len = solve_tsp(x, y; dist="EUC_2D")
        opt_tour, opt_len = solve_tsp(x, y; dist="MAN_2D")
        opt_tour, opt_len = solve_tsp(x, y; dist="MAX_2D")
        opt_tour, opt_len = solve_tsp(x, y; dist="GEO")
    end

    @testset "Input File" begin
        opt_tour, opt_len = solve_tsp("gr17.tsp")
        # @test opt_len == -1
        @test opt_len == 2085
    end
end


#"Read" data for parametric sinusoidal "simulation" & parameter extraction
#-------------------------------------------------------------------------------

using CMDimData
using CMDimData.MDDatasets

const π = MathConstants.π

#==Emulate reading in simulated data file
===============================================================================#

#=COMMENT
The code below emulates a parametric "simulation" of a sinusoidal response where
the 𝜙, A, and 𝑓 parameters of `signal = A * sin(𝜔*t + 𝜙); 𝜔 = 2π*𝑓` are varied.

The parametric signal can therefore be fully represented as:
	signal(𝜙, A, 𝑓, t)
=#

#But really construct multidimensional DataRS dataset manually:
signal = fill(DataRS, PSweep("phi", [0, 0.5, 1] .* (π/4))) do 𝜙
	fill(DataRS, PSweep("A", [1, 2, 4] .* 1e-3)) do A
	#Inner-most sweep: need to specify element type (DataF1):
	#(Other (scalar) element types: DataInt/DataFloat/DataComplex)
	fill(DataRS{DataF1}, PSweep("freq", [1, 4, 16] .* 1e3)) do 𝑓
		𝜔 = 2π*𝑓
		T = 1/𝑓
		Δt = T/100 #Define resolution from # of samples per period
		Tsim = 4T #Simulated time

		t = DataF1(0:Δt:Tsim) #DataF1 creates a t:{y, x} container with y == x
		sig = A * sin(𝜔*t + 𝜙) #Still a DataF1 sig:{y, x=t} container
		return sig
end; end; end

return signal


#######################################################################################################################################################################################################
#
# Changes made to this function
# General
#     2020-Mar-30: migrate the function from SCOPE
#     2021-Oct-21: rename the function to average_transmittance
#     2022-Feb-02: fix documentation
# To do
#     TODO: figure out where does this equation comes from
#
#######################################################################################################################################################################################################
"""

    average_transmittance(α::FT, nr::FT) where {FT<:AbstractFloat}

Return the average transmittance of isotropic radiation across an interface between two dielectrics, given
- `α` angle of incidence
- `nr` Index of refraction

# References
- Stern (1964) Transmission of isotropic radiation across an interface between two dielectrics. Applied Optics 3(1): 111-113.
- Allen (1973) Transmission of isotropic light across a dielectric surface in two and three dimensions. Journal of the Optical Society of America 63(6): 664-666.
"""
function average_transmittance(α::FT, nr::FT) where {FT<:AbstractFloat}
    @assert 0 < α <= 90;

    # some shortcuts to avoid overly comlicated equation
    _a     = (nr + 1) ^ 2 / 2;
    _a³    = _a ^ 3;
    _n²    = nr ^ 2;
    _n⁴    = nr ^ 4;
    _n⁶    = nr ^ 6;
    _n²p   = _n² + 1;
    _n²p²  = _n²p ^ 2;
    _n²p³  = _n²p ^ 3;
    _n²m²  = (_n² - 1) ^ 2;
    _k     = -1 * _n²m² / 4;
    _k²    = _k ^ 2;
    _sin²α = sind(α) ^ 2;
    _b₂    = _sin²α - _n²p/2;
    _b₁    = (α==90 ? 0 : sqrt(_b₂ ^ 2 + _k));
    _b     = _b₁ - _b₂;
    _b³    = _b ^ 3;
    _npanm = 2 * _n²p * _a - _n²m²;
    _npbnm = 2 * _n²p * _b - _n²m²;

    # S polarization
    _ts  = ( _k² / (6*_b³) + _k/_b - _b/2 ) - ( _k² / (6*_a³) + _k/_a - _a/2 );

    # P polarization
    _tp₁ = -2 * _n² * (_b - _a) / _n²p²;
    _tp₂ = -2 * _n² * _n²p * log(_b / _a) / _n²m²;
    _tp₃ = _n² * (1/_b - 1/_a) / 2;
    _tp₄ = 16 * _n⁴ * (_n⁴ + 1) * log(_npbnm / _npanm) / (_n²p³ * _n²m²);
    _tp₅ = 16 * _n⁶ * (1/_npbnm - 1/_npanm) / _n²p³;
    _tp  = _tp₁ + _tp₂ + _tp₃ + _tp₄ + _tp₅;

    return  (_ts + _tp) / (2 * _sin²α)
end


export logadd, logsub, log1mexp, log1pexp, inf

function inf(::Type{T})::T where T
    one(T) / zero(T)
end

function inf(x::T)::T where T
    inf(T)
end

"log(x+y) from log(x) and log(y)"
function logadd(lx, ly)
    if lx > ly
        return logadd(ly, lx)
    elseif -Inf < lx ≤ ly < Inf
        return ly + log1p(exp(lx - ly))
    else
        return ly
    end
end

logadd(lx, ly, lz, ls...) = foldl(logadd, (logadd(logadd(lx,ly),lz), ls...))

"log(|x-y|) from log(x) and log(y)"
function logsub(lx, ly)
    if min(lx, ly) > -Inf
        return max(lx, ly) + log1mexp(abs(lx - ly))
    else
        return max(lx, ly)
    end
end

"log(1 - exp(-x)), x ≥ 0"
function log1mexp(x)
    # https://cran.r-project.org/web/packages/Rmpfr/vignettes/log1mexp-note.pdf, Eq. 7
    x ≥ 0 || throw(DomainError())
    if x ≤ log(2)
        log(-expm1(-x))
    else
        log1p(-exp(-x))
    end
end

"log(1 + exp(x))"
function log1pexp(x)
    # https://cran.r-project.org/web/packages/Rmpfr/vignettes/log1mexp-note.pdf, Eq. 10
    if x ≤ -37
        exp(x)
    elseif -37 < x ≤ 18
        log1p(exp(x))
    elseif 18 < x ≤ 33.3
        x + exp(-x)
    else
        x
    end
end


using Test
using Flux: onehotbatch, mse, crossentropy, logitcrossentropy,
            σ, binarycrossentropy, logitbinarycrossentropy, flatten,
            xlogx, xlogy

const ϵ = 1e-7

@testset "xlogx & xlogy" begin
  @test iszero(xlogx(0))
  @test isnan(xlogx(NaN))
  @test xlogx(2) ≈ 2.0 * log(2.0)
  @inferred xlogx(2)
  @inferred xlogx(0)

  @test iszero(xlogy(0, 1))
  @test isnan(xlogy(NaN, 1))
  @test isnan(xlogy(1, NaN))
  @test isnan(xlogy(NaN, NaN))
  @test xlogy(2, 3) ≈ 2.0 * log(3.0)
  @inferred xlogy(2, 3)
  @inferred xlogy(0, 1)
end

@testset "losses" begin
  # First, regression-style y's
  y = [1, 1, 0, 0]
  ŷ = [.9, .1, .1, .9]

  @testset "mse" begin
    @test mse(ŷ, y) ≈ (.1^2 + .9^2)/2
  end

  @testset "mae" begin
    @test Flux.mae(ŷ, y) ≈ 1/2
  end

  @testset "huber_loss" begin
    @test Flux.huber_loss(ŷ, y) ≈ 0.20500000000000002
  end

  y = [123.0,456.0,789.0]
  ŷ = [345.0,332.0,789.0]
  @testset "msle" begin
    @test Flux.msle(ŷ, y) ≈ 0.38813985859136585
  end

  # Now onehot y's
  y = onehotbatch([1, 1, 0, 0], 0:1)
  ŷ = [.1 .9; .9 .1; .9 .1; .1 .9]'
  v = log(.1 / .9)
  logŷ = [v 0.0; 0.0 v; 0.0 v; v 0.0]'
  lossvalue = 1.203972804325936

  @testset "crossentropy" begin
    @test crossentropy([0.1,0.0,0.9], [0.1,0.0,0.9]) ≈ crossentropy([0.1,0.9], [0.1,0.9])
    @test crossentropy(ŷ, y) ≈ lossvalue
  end

  @testset "logitcrossentropy" begin
    @test logitcrossentropy(logŷ, y) ≈ lossvalue
  end

  @testset "weighted_crossentropy" begin
    @test crossentropy(ŷ, y, weight = ones(2)) ≈ lossvalue
    @test crossentropy(ŷ, y, weight = [.5, .5]) ≈ lossvalue/2
    @test crossentropy(ŷ, y, weight = [2, .5]) ≈ 1.5049660054074199
  end

  @testset "weighted_logitcrossentropy" begin
    @test logitcrossentropy(logŷ, y, weight = ones(2)) ≈ lossvalue
    @test logitcrossentropy(logŷ, y, weight = [.5, .5]) ≈ lossvalue/2
    @test logitcrossentropy(logŷ, y, weight = [2, .5]) ≈ 1.5049660054074199
  end

  logŷ, y = randn(3), rand(3)
  @testset "binarycrossentropy" begin
    @test binarycrossentropy.(σ.(logŷ), y; ϵ=0) ≈ -y.*log.(σ.(logŷ)) - (1 .- y).*log.(1 .- σ.(logŷ))
    @test binarycrossentropy.(σ.(logŷ), y) ≈ -y.*log.(σ.(logŷ) .+ eps.(σ.(logŷ))) - (1 .- y).*log.(1 .- σ.(logŷ) .+ eps.(σ.(logŷ)))
  end

  @testset "logitbinarycrossentropy" begin
    @test logitbinarycrossentropy.(logŷ, y) ≈ binarycrossentropy.(σ.(logŷ), y; ϵ=0)
  end

  y = [1 2 3]
  ŷ = [4.0 5.0 6.0]
  @testset "kldivergence" begin
    @test Flux.kldivergence([0.1,0.0,0.9], [0.1,0.0,0.9]) ≈ Flux.kldivergence([0.1,0.9], [0.1,0.9])
    @test Flux.kldivergence(ŷ, y) ≈ -1.7661057888493457
    @test Flux.kldivergence(y, y) ≈ 0
  end

  y = [1 2 3 4]
  ŷ = [5.0 6.0 7.0 8.0]
  @testset "hinge" begin
    @test Flux.hinge(ŷ, y) ≈ 0
    @test Flux.hinge(y, 0.5 .* y) ≈ 0.125
  end

  @testset "squared_hinge" begin
    @test Flux.squared_hinge(ŷ, y) ≈ 0
    @test Flux.squared_hinge(y, 0.5 .* y) ≈ 0.0625
  end

  y = [0.1 0.2 0.3]
  ŷ = [0.4 0.5 0.6]
  @testset "poisson" begin
    @test Flux.poisson(ŷ, y) ≈ 0.6278353988097339
    @test Flux.poisson(y, y) ≈ 0.5044459776946685
  end

  y = [1.0 0.5 0.3 2.4]
  ŷ = [0 1.4 0.5 1.2]
  @testset "dice_coeff_loss" begin
    @test Flux.dice_coeff_loss(ŷ, y) ≈ 0.2799999999999999
    @test Flux.dice_coeff_loss(y, y) ≈ 0.0
  end

  @testset "tversky_loss" begin
    @test Flux.tversky_loss(ŷ, y) ≈ -0.06772009029345383
    @test Flux.tversky_loss(ŷ, y, β = 0.8) ≈ -0.09490740740740744
    @test Flux.tversky_loss(y, y) ≈ -0.5576923076923075
  end

  @testset "no spurious promotions" begin
    for T in (Float32, Float64)
      y = rand(T, 2)
      ŷ = rand(T, 2)
      for f in (mse, crossentropy, logitcrossentropy, Flux.kldivergence, Flux.hinge, Flux.poisson,
              Flux.mae, Flux.huber_loss, Flux.msle, Flux.squared_hinge, Flux.dice_coeff_loss, Flux.tversky_loss)
        fwd, back = Flux.pullback(f, ŷ, y)
        @test fwd isa T
        @test eltype(back(one(T))[1]) == T
      end
    end
  end
end

@testset "helpers" begin
  @testset "flatten" begin
    x = randn(Float32, 10, 10, 3, 2)
    @test size(flatten(x)) == (300, 2)
  end
end


module SDM

using LinearAlgebra
using Printf

include("utility.jl")
include("graphics.jl")

export sdm

function sdm(fun, fun_derivative, starting_point, step_size_params, epsilon; display_g=true)
    type = get(step_size_params, "type", "constant")
    method = get(SDM_step_size_methods, type, constant_step_size_sdm)
    return method(fun, fun_derivative, starting_point, step_size_params, epsilon, display_g)
end

function constant_step_size_sdm(fun, fun_derivative, starting_point, step_size_params, epsilon, display_g)
    @printf("Press enter to continue with iterations\n")
    @printf("step\t\tposition\t\tgradient_norm\n")

    gradient = fun_derivative(starting_point[1], starting_point[2])
    x = [starting_point[1], starting_point[2]]

    step_counter = 1
    while(true)

        #we move in the opposite direction of the gradient
        d = -gradient
        #step size is constant
        alpha = get(step_size_params, "step_size", 0.1)
        #move to the next point
        movement = alpha*d
        if display_g
            display(add_vector(x, movement))
        end
        x = x + movement

        gradient_norm = norm(gradient, 2)
        @printf("%d\t\t[%g,%g]\t\t%g", step_counter, x[1], x[2], gradient_norm)

        #we stop whenever the norm of the gradient is lower than a certain epsilon
        if (gradient_norm < epsilon)
            break
        end

        wait_for_key("")

        step_counter += 1

        #calculate the gradient for the new iteration
        gradient = fun_derivative(x[1], x[2])
    end

    @printf("\n")
    return x
end

function quadratic_step_size_sdm(fun, fun_derivative, starting_point, step_size_params, epsilon, display_g)
    @printf("Press enter to continue with iterations\n")
    @printf("step\t\tposition\t\tgradient_norm\n")

    Q = get(step_size_params, "Q", false)
    if Q==false error("Q not set in step_size_params"); return; end;

    gradient = fun_derivative(starting_point[1], starting_point[2])
    x = [starting_point[1], starting_point[2]]

    step_counter = 1
    while(true)

        #we move in the opposite direction of the gradient
        d = -gradient
        #best step size, if the function is a quadratic function
        alpha = (norm(d,2)^2)/(d'*Q*d)
        #move to the next point
        movement = alpha*d
        if display_g
            display(add_vector(x, movement))
        end
        x = x + movement

        gradient_norm = norm(gradient, 2)
        @printf("%d\t\t[%f,%f]\t\t%g", step_counter, x[1], x[2], gradient_norm)

        #we stop whenever the norm of the gradient is lower than a certain epsilon
        if (gradient_norm < epsilon)
            break
        end

        wait_for_key("")

        step_counter += 1

        #calculate the gradient for the new iteration
        gradient = fun_derivative(x[1], x[2])
    end

    @printf("\n")
    return x
end

SDM_step_size_methods = Dict([
    ("constant", constant_step_size_sdm),
    ("quadratic", quadratic_step_size_sdm)
])

end


""" Predator prey model with logistic function from http://www.math.lsa.umich.edu/~rauch/256/F2Lab5.pdf """
function foxes_and_rabbits(oct::Bool = true)
    m = Model(SOLVER_SILENT)
    t = 20
    r = 0.2
    x1 = 0.6
    y1 = 0.5
    @variable(m, x[1:t] >= 0.01) # Note: Ipopt solution does not converge with an upper bound!!
    @variable(m, dx[1:t-1])
    @variable(m, y[1:t] >= 0.01)
    @variable(m, dy[1:t-1])
    @constraint(m, x[1] == x1)
    @constraint(m, y[1] == y1)
    @constraint(m, [i=2:t], x[i] == x[i-1] + dx[i-1])
    @constraint(m, [i=2:t], y[i] == y[i-1] + dy[i-1])

    # GlobalModel representation
    set_upper_bound.(x, 1)
    set_upper_bound.(dx, 1)
    set_lower_bound.(dx, -1)
    set_upper_bound.(y, 1)
    set_upper_bound.(dy, 1)
    set_lower_bound.(dy, -1)

    if !oct
        @NLconstraint(m, [i=1:t-1], dx[i] == x[i]*(1-x[i]) - x[i]*y[i]/(x[i]+1/5))
        @NLconstraint(m, [i=1:t-1], dy[i] == r*y[i]*(1-y[i]/x[i]))
        return m
    else
        gm = GlobalModel(model = m, name = "foxes_rabbits")
        add_nonlinear_constraint(gm, :((x, y, dx) -> dx[1] - (x[1]*(1-x[1]) -x[1]*y[1]/(x[1]+$(r)))),
                                vars = [x[1], y[1], dx[1]], equality=true)
        add_nonlinear_constraint(gm, :((x, y, dy) -> dy[1] - $(r)*y[1]*(1-y[1]/x[1])),
                                vars = [x[1], y[1], dy[1]], equality=true)
        for i = 2:t-1
            add_linked_constraint(gm, gm.bbls[1], [x[i], y[i], dx[i]])
            add_linked_constraint(gm, gm.bbls[2], [x[i], y[i], dy[i]])
        end
        return gm
    end
end


function swap(array, i, j)
  tmp = array[i]
  array[i] = array[j]
  array[j] = tmp
end

function partition(array, left, right)
  pivotIndex = rand(left:right)
  pivot = array[pivotIndex]
  swap(array, pivotIndex, right)

  pos = left
  for i = left:right-1
    if array[i] < pivot
      swap(array, i, pos)
      pos = pos+1
    end
  end

  swap(array, pos, right)

  return pos
end

function quick_sort(array, left, right)
  if left < right
    pivotIndex = partition(array, left, right)
    quick_sort(array, left, pivotIndex-1)
    quick_sort(array, pivotIndex+1, right)
  end
end

function is_sorted(array)
  max = 0
  i = 1
  while i < length(array)
    if array[i] < max
      return false
    end
    max = array[i]
    i = i+1
  end
  return true
end

function generate_random_array(length)
  rand(0:length, length)
end

for i = 1:100
  array = generate_random_array(1000)
  quick_sort(array, 1, length(array))
  if !is_sorted(array)
    println("Error! Array is not sorted!")
    exit()
  end
end

println("Quick sort works fine!")


## @lc code=start
using LeetCode

function count_operations(num1::Int, num2::Int)::Int
    if num2 > num1
        num1, num2 = num2, num1
    end
    res = 0
    while num1 > 0
        res += num2 ÷ num1
        num1, num2 = num2 % num1, num1
    end
    res
end
## @lc code=end

## @lc test=start
@testset "2169.count-operations-to-obtain-zero.jl" begin
    @test count_operations(2, 3) == 3
    @test count_operations(10, 10) == 1
    @test count_operations(2022, 1077) == 20
end
## @lc test=end

## @lc info=start
# link: [solution to 2169](https://leetcode-cn.com/problems/count-operations-to-obtain-zero/solution/6004-zhan-zhuan-xiang-chu-de-ci-shu-juli-23oc/)
## @lc info=end


#' ## What is learning?
#'
#' Computers read data, as we saw in notebooks 1 and 2. We can then build
#' functions that model that data to make decisions, as we saw in notebooks
#' 3 and 5.
#'
#' But how do you make sure that the model actually fits the data well?
#' In the last notebook, we saw that we can fiddle with the parameters of our
#' function defining the model to reduce the loss function. However,
#' we don't want to have to pick the model parameters ourselves.
#' Choosing parameters ourselves works *well enough* when we have a
#' simple model and only a few data points, but can quickly become
#' extremely complex for more detailed models and larger data sets.
#'
#' Instead, we want our machine to *learn* the parameters that fit the model
#' to our data, without needing us to fiddle with the parameters ourselves.
#' In this notebook, we'll talk about the "learning" in machine learning.

#' ### Motivation: Fitting parameters by hand
#'
#' Let's go back to our example of fitting parameters from notebook 3.
#' Recall that we looked at whether the amount of green in the pictures
#' could distinguish between an apple and a banana, and used a sigmoid
#' function to model our choice of "apple or banana" using the amount of
#' green in an image.

using Plots
gr()
using Images, Statistics

sigma(x,w,b) = 1/(1 + exp(-w*x + b))

apple = load("data/10_100.jpg")
banana = load("data/104_100.jpg")
apple_g = mean(Float64.(green.(apple)))
banana_g = mean(Float64.(green.(banana)))

w = 10.0 # Try manipulating w between 0 and 30 to see how the plot changes
b = 15.0 # Try manipulating b bewteen 0 and 30

pl = plot(x->sigma(x,w,b), xlims=(0, 1))
scatter!([apple_g], [0.0], label="Apple")
scatter!([banana_g], [1.0], label="Banana")

#' Intuitively, how did you tweak the sliders so that way the model sends
#' apples to 0 and bananas to 1? Most likely, you did the following:

#' #### Move the sliders a bit, see whether the curve moves in the right
#' direction, and if it did, keep doing it.

#' For a machine, "learning" is that same process, translated into math!

#' ## "Learning by nudging": The process of descent
#'
#' Let's start to formalize this idea. In order to push the curve in the
#' "right direction", we need some measurement of "how right" and "how wrong"
#' the model is. When we translate the idea of a "right direction" into math,
#' we end up with a **loss function**, `L(w, b)`, as we saw in notebook 5.
#' We say that the loss function is lowest when the model `σ(x, w, b)`
#' performs the best.
#'
#' Now we want to create a loss function that is the lowest when the apple is
#' at `0` and the banana is at `1`. If the data (the amount of green) for
#' our apple is $x_1$, then our model will output $σ(x_1,w, b)$ for our apple.
#' So, we want the difference $0 - σ(x_1, w, b)$ to be small. Similarly, if our
#' data for our banana (the banana's amount of green) is $x_2$, we want the
#' difference $1 - σ(x_2, w, b)$ to be small.
#'
#' To create our loss function, let's add together the squares of the
#' difference of the model's output from the desired output for the apple
#' and the banana. We get
#'
#' $$ L(w,b) = (0 - σ(x_1, w, b))^2 + (1 - σ(x_2, w, b))^2. $$
#'
#' $L(w, b)$ is lowest when it outputs `0` for the apple and `1` for the
#' banana, and thus the cost is lowest when the model "is correct".
#'
#' We can visualize this function by plotting it in 3D with the
# `surface` function or in 2D with contour lines

# plotly() # The plotly backend is nice for 3d surface plots
# gr() # The GR backend is good for faster interactive plots

L(w,b) = (0 - sigma(apple_g,w,b))^2 + (1 - sigma(banana_g,w,b))^2

w_range = 10:0.1:13
b_range = 0:1:20
L_values = [L(w,b) for b in b_range, w in w_range]

w = 11.5 # Try manipulating w with values from w_range (between 10 and 13)
b = 7 # Try manipulating b with values from b_range (between 0 and 20)
##     p1 = surface(w_range, b_range, L_values, xlabel="w", ylabel="b", cam=(70,40), cbar=false, leg=false)
##     scatter!(p1, [w], [b], [L(w,b)+1e-2], markersize=5, color = :blue)
p1 = contour(w_range, b_range, L_values, levels = 0.05:0.1:1,
    xlabel="w", ylabel="b", cam=(70,40), cbar=false, legend=false)
scatter!(p1, [w], [b], markersize=5, color=RGB(0.05, 0.05, 0.90))

p2 = plot(x->sigma(x,w,b), 0, 1, label="Model", legend=(0.1,0.9), lw=3)
scatter!(p2, [apple_g], [0.0], label="Apple", markersize=8)
scatter!(p2, [banana_g], [1.0], label="Banana", markersize=8, xlim=(0,1),
    ylim=(-0.1,1.1))
plot(p1, p2, layout=(2,1))

#' The blue ball on the 3D plot shows the current parameter choices,
#' plotted as `(w,b)`. Shown below the 3D plot is a 2D plot of the
#' corresponding model with those parameters. Notice that as the blue ball
#' rolls down the hill, the model becomes a better fit. Our loss function
#' gives us a mathematical notion of a "hill", and the process of
#' "learning by nudging" is simply rolling the ball down that hill.

#' To do this mathematically, we need to know which direction is "downhill".
#' Recall from calculus that the derivative of `L` with respect to `b` tells
#' you how `L` changes when `b` changes. Thus to roll downhill, we should go
#' in the direction where the derivative is negative (the function goes down)
#' for each parameter. This direction is the negative of what's called the
#' **gradient**, $\nabla L$. This means that the "learn by nudging method"
#' can be rephrased in mathematical terms as:
#'
#' 1. Calculate the gradient
#' 2. Move a little bit in the direction of the negative gradient
#' 3. Repeat

#' This process of rolling the ball in the direction of the negative gradient
#' is called **gradient descent**; written mathematically, it is
#'
#' $$p_{n+1} = p_n - \eta \nabla L(p_n).$$
#'
#' Here, $p_n$ represents the vector of current parameters
#' $(w, b)$; $\nabla L(p_n)$ is the gradient of the loss function, given
#' those parameters. We start from $p_n$ and change it by $\eta \nabla L(p_n)$,
#' where $\eta$ is a small step size that determines how far we move the
#' parameters in the direction of the negative gradient; notice that if you
#' step too far, you'll overshoot the minimum!. The result is
#' $p_{n+1}$, the new vector of parameters.
#'
#' [Picture of Gradient Descent Vectors]
#'
#' If we repeat this process, then we will end up at parameters where the
#' model correctly labels apples as `0` and bananas as `1`. When this happens,
#' the model has learned from the data and can then read pictures and tell
#' you whether they are apples or bananas!

#' #### Exercise 1
#'
#' Use the following terms to fill in the sentences below.
#' Terms may be used more than once or not at all:
#' > gradient, loss function, derivative, gradient descent, learning.
#'
#' * We can think of a _(A)_ as a 1D version of a _(B)_.
#' * We can visualize a _(C)_ as a hill.
#' * In the explanation above, rolling downhill is called _(D)_ and means traveling along the _(E)_.
#' * To quantify the correctness of a model we use a _(F)_.
#' * When our program can minimize a _(G)_ on its own, we say it is _(H)_.
#'
#'

#'
#' A)

#' B)

#' C)

#' D)

#' E)

#' F)

#' G)

#' H)


# Linear ODE
linear = (u,p,t) -> (p*u)
linear_analytic = (u0,p,t) -> u0*exp(p*t)
"""
Linear ODE

```math
\\frac{du}{dt} = αu
```

with initial condition ``u0=1/2``, ``α=1.01``, and solution

```math
u(t) = u0e^{αt}
```

with Float64s. The parameter is ``α``
"""
prob_ode_linear = ODEProblem(
                  ODEFunction(linear,analytic=linear_analytic),
                  1/2,(0.0,1.0),1.01)

"""
Linear ODE

```math
\\frac{du}{dt} = αu
```

with initial condition ``u0=1/2``, ``α=1.01``, and solution

```math
u(t) = u0e^{αt}
```

with BigFloats
"""
prob_ode_bigfloatlinear = ODEProblem(
                          ODEFunction(linear,analytic=linear_analytic),
                          big(0.5),(0.0,1.0),big(1.01))

f_2dlinear = (du,u,p,t) -> (@. du = p*u)
f_2dlinear_analytic = (u0,p,t) -> @. u0*exp(p*t)
"""
4x2 version of the Linear ODE

```math
\\frac{du}{dt} = αu
```

with initial condition ``u0=1/2``, ``α=1.01``, and solution

```math
u(t) = u0e^{αt}
```

with Float64s
"""
prob_ode_2Dlinear = ODEProblem(
                    ODEFunction(f_2dlinear,analytic=f_2dlinear_analytic),
                    rand(4,2),(0.0,1.0),1.01)

"""
100x100 version of the Linear ODE

```math
\\frac{du}{dt} = αu
```

with initial condition ``u0=1/2``, ``α=1.01``, and solution

```math
u(t) = u0e^{αt}
```

with Float64s
"""
prob_ode_large2Dlinear = ODEProblem(
                    ODEFunction(f_2dlinear,analytic=f_2dlinear_analytic),
                    rand(100,100),(0.0,1.0),1.01)

"""
4x2 version of the Linear ODE

```math
\\frac{du}{dt} = αu
```

with initial condition ``u0=1/2``, ``α=1.01``, and solution

```math
u(t) = u0e^{αt}
```

with BigFloats
"""
prob_ode_bigfloat2Dlinear = ODEProblem(
                    ODEFunction(f_2dlinear,analytic=f_2dlinear_analytic),
                    BigFloat.(rand(4,2)).*ones(4,2)/2,(0.0,1.0),big(1.01))

f_2dlinear_notinplace = (u,p,t) -> p*u
"""
4x2 version of the Linear ODE

```math
\\frac{du}{dt} = αu
```

with initial condition ``u0=1/2``, ``α=1.01``, and solution

```math
u(t) = u0e^{αt}
```

on Float64. Purposefully not in-place as a test.
"""
prob_ode_2Dlinear_notinplace = ODEProblem(
                    ODEFunction(f_2dlinear_notinplace,analytic=f_2dlinear_analytic),
                    rand(4,2),(0.0,1.0),1.01)


@testset "Testing sampler.jl" begin

    n = 4
    mu = collect(range(0.2, stop=0.6, length=n))
    @testset "check positive definite" begin
        Sigma = [3.0 1.0 1.0 1.0;
                 1.0 2.0 1.0 1.0;
                 1.0 1.0 2.0 1.0;
                 1.0 1.0 1.0 1.0]
        mvns = MVNSampler(mu, Sigma)
        @test isapprox(mvns.Q * mvns.Q', mvns.Sigma)
    end

    @testset "check positive semi-definite zeros" begin
        mvns = MVNSampler(mu, zeros(n, n))
        @test rand(mvns) == mu
    end

    @testset "check positive semi-definite ones" begin
        mvns = MVNSampler(mu, ones(n, n))
        c = rand(mvns)-mvns.mu
        @test all(broadcast(isapprox,c[1],c))
    end

    @testset "check positive semi-definite 1 and -1/(n-1)" begin
        Sigma = -1/(n-1)*ones(n, n) + n/(n-1)*Matrix(Matrix(I, n, n))
        mvns = MVNSampler(mu, Sigma)
        @test isapprox(sum(rand(mvns)) , sum(mu), atol=1e-4, rtol=1e-4)
    end

    @testset "check non-positive definite" begin
        Sigma = [2.0 1.0 3.0 1.0;
                 1.0 2.0 1.0 1.0;
                 3.0 1.0 2.0 1.0;
                 1.0 1.0 1.0 1.0]
        @test_throws ArgumentError MVNSampler(mu, Sigma)
    end

    @testset "check availability of rank deficient matrix" begin
        A = randn(n,n)
        for r=1:n-2
            r=2
            for i = 1:r
                A[:, end+1-i] = sum(A[:, 1:end-r], dims = 2)
            end
            Sigma = A * A'
            @test typeof(MVNSampler(mu,Sigma)) <: MVNSampler
        end
    end

    @testset "test covariance matrices of Int and Rational" begin
        n = 2
        mu = zeros(2)
        for T in [Int, Rational{Int}]
            Sigma = Matrix{T}(I, n, n)
            @test typeof(MVNSampler(mu, Sigma)) <: MVNSampler
        end
    end

end  # @testset


using BenchmarkTools
BenchmarkTools.DEFAULT_PARAMETERS.samples = 100

function compute(path::String)::Int
    base_exp = map(x -> parse.(Int, split(x, ",")), split(read(path, String), "\n"))
    max_exponent = 0
    result = 0
    for i ∈ 1:length(base_exp)
        base, exponent = base_exp[i]
        exponent = log10(base) * exponent
        if exponent > max_exponent
            max_exponent = exponent
            result = i
        end
    end
    return result
end

compute("problems/0099/p099_base_exp.txt")

@benchmark compute("problems/0099/p099_base_exp.txt")


using DataFrames

A = readdlm("u1.base",'\t';header=false)
file = "u1.base"

function pearsonCorrelation(x,y)
        @assert(length(x) == length(y))
        n = length(x)
        @assert(n>0)
        avg_x = mean(x)
        avg_y = mean(y)
        prod_diff = 0
        xdiff2 = 0
        ydiff2 = 0
        for id = 1:n
                xdiff = x[id] - avg_x
                ydiff = y[id] - avg_y
                prod_diff += xdiff*ydiff
                xdiff2 += xdiff^2
                ydiff2 += ydiff^2
        end
        return prod_diff/sqrt(xdiff2*ydiff2)
end

function PearsonCorrelationHelper(file)
     df = readtable(file,separator = '\t',header = false);
     rename!(df,:x1,:userId)
     rename!(df,:x2,:itemId)
     rename!(df,:x3,:rating)
     #rename!(df,:x4,:timestamp)
     M1 =Set(df[df[:userId] .== 1, :itemId])
     CorrelationArray = Float64[]
     (N,ind) = findmax(df[:,1])
     for(i = 2:N)
            M_i = Set(df[df[:userId] .== i, :itemId])
            if(length(intersect(M1,M_i)) == 0) push!( CorrelationArray , NA)
            else
                  X = Float64[]
                  Y = Float64[]
                  for(movie in intersect(M1,M_i))
                      push!(X , (df[(df[:userId]   .== 1)  & (df[:itemId] .== movie), :rating])[1])
                      push!(Y , (df[(df[:userId]   .== i)  & (df[:itemId] .== movie), :rating])[1])
                  end
                  push!( CorrelationArray , pearsonCorrelation(X,Y))
            end
     end
end


######### Stan program example  ###########

using Mamba, Stan

ProjDir = dirname(@__FILE__)
cd(ProjDir) do

  bernoullimodel = "
  data {
    int<lower=1> N;
    int<lower=0,upper=1> y[N];
  }
  parameters {
    real<lower=0,upper=1> theta;
  }
  model {
    theta ~ beta(1,1);
    y ~ bernoulli(theta);
  }
  "

  observeddata = [
    Dict("N" => 10, "y" => [0, 1, 0, 1, 0, 0, 0, 0, 0, 1]),
    Dict("N" => 10, "y" => [0, 1, 0, 0, 0, 0, 1, 0, 0, 1]),
    Dict("N" => 10, "y" => [0, 0, 0, 0, 0, 0, 1, 0, 1, 1]),
    Dict("N" => 10, "y" => [0, 0, 0, 1, 0, 0, 0, 1, 0, 1])
  ]

  initparms = Dict{String, Any}[
    Dict("theta" => 0.1),
    Dict("theta" => 0.4),
    Dict("theta" => 0.5),
    Dict("theta" => 0.9),
  ]

  monitor = ["theta", "lp__", "accept_stat__"]

  global stanmodel, rc, sim1
  stanmodel = Stanmodel(name="bernoulli", model=bernoullimodel, num_warmup=1);
  rc, sim1 = stan(stanmodel, observeddata, init=initparms, CmdStanDir=CMDSTAN_HOME)

  rc == 0 && sim1.value[1, end, :] |> display
end # cd


# sample from a harmonic potential in 1D using GJF
using Plots
using Printf
using Random
using LinearAlgebra
using ForwardDiff
using BasicMD

include("potentials.jl")

β = 5.0;
γ = 1.5;
# M = Float64[2.1];
M = 2.1;
q₀ = [0.0];
p₀ = [0.0];
Δt = 1e-1;
n_iters = 10^4;

V = x->Harmonic(x);
cfg = ForwardDiff.GradientConfig(V, q₀);
gradV! = (gradV, x)-> ForwardDiff.gradient!(gradV, V, x, cfg);

sampler = GJF(gradV!, β, γ, M, Δt)

Random.seed!(100);
Q₀ = copy(q₀);
P₀ = copy(p₀);
sample_trajectory!([Q₀, P₀], sampler, options=MDOptions(n_iters=n_iters));
@printf("In Place (Q,P) after %d iterations: (%g,%g)\n",n_iters, Q₀[1], P₀[1]);
#
Random.seed!(100);
Xvals = sample_trajectory([q₀, p₀], sampler, options=MDOptions(n_iters=n_iters,n_save_iters=n_iters));
Q = Xvals[end][1][1]
P = Xvals[end][2][1]
@printf("(Q,P) after %d iterations: (%g,%g)\n",n_iters, Q[1], P[1]);
#
Random.seed!(100);
Xvals = sample_trajectory([q₀, p₀], sampler, options=MDOptions(n_iters=n_iters));
histogram([X[1][1] for X in Xvals],label="Samples",normalize=true)
qq=LinRange(-2,2,200)
plot!(qq, sqrt((β)/(2*π))*exp.(-0.5 * β * qq.^2),label="Density")
xlabel!("q")
ylabel!("Frequency")


# basisfunctions.jl

"""
    fourier(k::Int)

fourier implements the Fourier basis of functions ϕ_k, defined by ϕ_0≡1, and if k≥1 and k is odd, by
ϕ_k(x)=sqrt(2)sin((k+1)*π*x) and if k≥1 and even, by ϕ_k(x)=sqrt(2)cos(π*k*x).
This function is defined for k≥0.

# Examples

```julia
x = -1.0:0.01:2.0
y = fourier(3).(x)
```
"""
function fourier(k::Int)
  k ≥ 0 || throw(AssertionError("k should be non-negative"))
  if k == 0
    return x -> 1.0
  elseif isodd(k)
    kplusonefloat = float(k+1)
    return x -> 1.4142135623730951 * sinpi(kplusonefloat*x)
  else
    kfloat = float(k)
    return x -> 1.4142135623730951 * cospi(kfloat*x)
  end
end

"""
    faberschauderone(x)

Implements the first Faber-Schauder function defined by 1-2x for 0≤x≤1/2 and
-1+2x for 0.5≤x≤1, and is 1-periodically extended to all x∈R.

#See also: faberschauder

#Warning

Note the difference between faberschauderone and faberschauder. The first is a
function that takes a Float64 and returns a Float64, the second takes (j,k) and
returns an anonymous function that takes a Float64 and returns a Float64.

#Examples

```julia
x=-2.0:0.001:2.0
y=faberschauderone.(x)
```
"""
function faberschauderone(x)
    y = mod(x, 1.0)
    if 0≤y≤0.5
        return 1.0 - 2*y
    else
        return -1.0 + 2*y
    end
end

"""
    faberschauder(j::Int, k::Int)

faberschauder implements the k-th Faber-Schauder function of level j. Here, j≥0 and 1≤k≤2^j.
It is a one-periodic function and defined on [0,1] by 2^(j+1)(x-(k-1)2^(-j)) on
(k-1)2^(-j)≤x≤(k-1/2)2^(-j) and 1 - 2^(j+1)(x-(k-1/2)2^(-j)) on
[(k-1/2)2^(-j), k2^(-j)] and zero outside these intervals.

# See also: faberschauderone.

#Warning

Note the difference between faberschauderone and faberschauder. The first is a
function that takes a Float64 and returns a Float64, the second takes (j,k) and
returns an anonymous function that takes a Float64 and returns a Float64.

#Example with Plots
```julia
using Plots
J=2
x=0.0:0.001:1.0
p=plot()
for j in 0:J
  for k in 1:2^j
      y = faberschauder(j,k).(x)
      plot!(p,x,y)
  end
end
display(p)
```
"""
function faberschauder(j::Int,k::Int)
  j ≥ 0 || throw(AssertionError("j should be a nonnegative integer."))
  1≤k≤2^j || throw(AssertionError("k should be an integer between 1 and 2^j."))
  return function(x::Float64)
    y = mod(x,1.0)
    if y ≤ ldexp(float(k-1), -j) || y ≥ ldexp(float(k), -j)
      return 0.0
    elseif ldexp(float(k-1), -j) < y ≤ ldexp(k-0.5, -j)
      return ldexp(y, j+1)-2*(k-1)
    else
      return 2*k - ldexp(y, j+1)
    end
  end
end


# 1D diffusion problem

# Packages and inclusions
using ModelingToolkit,DiffEqOperators,LinearAlgebra,Test,OrdinaryDiffEq
using ModelingToolkit: Differential

# Beam Equation
@test_broken begin
    @parameters x, t
    @variables u(..)
    Dt = Differential(t)
    Dtt = Differential(t)^2
    Dx = Differential(x)
    Dxx = Differential(x)^2
    Dx3 = Differential(x)^3
    Dx4 = Differential(x)^4

    g = -9.81
    EI = 1
    mu = 1
    L = 10.0
    dx = 0.4

    eq = Dtt(u(t,x)) ~ -mu*EI*Dx4(u(t,x)) + mu*g

    bcs = [u(0, x) ~ 0,
           u(t,0) ~ 0,
           Dx(u(t,0)) ~ 0,
           Dxx(u(t, L)) ~ 0,
           Dx3(u(t, L)) ~ 0]

    # Space and time domains
    domains = [t ∈ IntervalDomain(0.0,1.0),
               x ∈ IntervalDomain(0.0,L)]

    pdesys = PDESystem(eq,bcs,domains,[t,x],[u(t,x)])
    discretization = MOLFiniteDifference([x=>dx],t, centered_order=4)
    prob = discretize(pdesys,discretization)
end

# Beam Equation with Velocity
@test_broken begin
    @parameters x, t
    @variables u(..), v(..)
    Dt = Differential(t)
    Dtt = Differential(t)^2
    Dx = Differential(x)
    Dxx = Differential(x)^2
    Dx3 = Differential(x)^3
    Dx4 = Differential(x)^4

    g = -9.81
    EI = 1
    mu = 1
    L = 10.0
    dx = 0.4

    eqs = [v(t, x) ~ Dt(u(t,x)),
           Dt(v(t,x)) ~ -mu*EI*Dx4(u(t,x)) + mu*g]

    bcs = [u(0, x) ~ 0,
           v(0, x) ~ 0,
           u(t,0) ~ 0,
           v(t,0) ~ 0,
           Dxx(u(t, L)) ~ 0,
           Dx3(u(t, L)) ~ 0]

    # Space and time domains
    domains = [t ∈ IntervalDomain(0.0,1.0),
               x ∈ IntervalDomain(0.0,L)]

    pdesys = PDESystem(eqs,bcs,domains,[t,x],[u(t,x),v(t,x)])
    discretization = MOLFiniteDifference([x=>dx],t, centered_order=4)
    prob = discretize(pdesys,discretization)
end

@testset "Kuramoto–Sivashinsky equation" begin
    @parameters x, t
    @variables u(..)
    Dt = Differential(t)
    Dx = Differential(x)
    Dx2 = Differential(x)^2
    Dx3 = Differential(x)^3
    Dx4 = Differential(x)^4

    α = 1
    β = 4
    γ = 1
    eq = Dt(u(x,t)) ~ -u(x,t)*Dx(u(x,t)) - α*Dx2(u(x,t)) - β*Dx3(u(x,t)) - γ*Dx4(u(x,t))

    u_analytic(x,t;z = -x/2+t) = 11 + 15*tanh(z) -15*tanh(z)^2 - 15*tanh(z)^3
    du(x,t;z = -x/2+t) = 15/2*(tanh(z) + 1)*(3*tanh(z) - 1)*sech(z)^2

    bcs = [u(x,0) ~ u_analytic(x,0),
           u(-10,t) ~ u_analytic(-10,t),
           u(10,t) ~ u_analytic(10,t),
           Dx(u(-10,t)) ~ du(-10,t),
           Dx(u(10,t)) ~ du(10,t)]

    # Space and time domains
    domains = [x ∈ IntervalDomain(-10.0,10.0),
               t ∈ IntervalDomain(0.0,1.0)]
    # Discretization
    dx = 0.4; dt = 0.2

    discretization = MOLFiniteDifference([x=>dx],t;centered_order=4,grid_align=center_align)
    pdesys = PDESystem(eq,bcs,domains,[x,t],[u(x,t)])
    prob = discretize(pdesys,discretization)

    sol = solve(prob,Tsit5(),saveat=0.1,dt=dt)

    @test sol.retcode == :Success

    xs = domains[1].domain.lower+dx+dx:dx:domains[1].domain.upper-dx-dx
    ts = sol.t

    u_predict = sol.u
    u_real = [[u_analytic(x, t) for x in xs] for t in ts]
    u_diff = u_real - u_predict
    @test_broken u_diff[:] ≈ zeros(length(u_diff)) atol=0.01;
    #plot(xs, u_diff)
end


export cellvolume

"""
    cellvolume(a, b, c, α, β, γ)

Calculates the cell volume from 6 cell parameters.
"""
cellvolume(a, b, c, α, β, γ) =
    a * b * c * sqrt(sind(α)^2 - cosd(β)^2 - cosd(γ)^2 + 2 * cosd(α) * cosd(β) * cosd(γ))
"""
    cellvolume(l::Lattice)
    cellvolume(c::Cell)

Calculates the cell volume from a `Lattice` or a `Cell`.
"""
cellvolume(lattice::AbstractLattice) = abs(det(lattice.data))
cellvolume(cell::Cell) = cellvolume(cell.lattice)
"""
    cellvolume(g::MetricTensor)

Calculates the cell volume from a `MetricTensor`.
"""
cellvolume(g::MetricTensor) = sqrt(det(g.data))  # `sqrt` is always positive!


# # Example 2 on MibS

# Model of the problem
# First level
# ```math
# \min_{x} 2x -4y + 10z,\\
# \notag s.t.\\
# -3x + 2y + 2z \leq 12,\\
# x + 2y \leq 20,\\
# x \leq 10,\\
# x \in \mathbb{Z}, z \in \mathbb{B},\\
# ```
# Second level
# ```math
# \min_{y} y,\\
# \notag s.t.\\
# 2x - y + 3z<= 7,\\
# -2x + 4y <= 16,\\
# y <= 5\\
# y \in \mathbb{Z}\\
# ```

using BilevelJuMP
using JuMP
using Test

model = BilevelModel()

# First we need to create all of the variables in the upper and lower problems:

# Upper level variables
@variable(Upper(model), x, Int)
@variable(Upper(model), z, Bin)

#Lower level variables
@variable(Lower(model), y, Int)

# Then we can add the objective and constraints of the upper problem:

# Upper level objecive function
@objective(Upper(model), Min, 2x - 4y + 10z)

# Upper constraints
@constraints(Upper(model), begin
    u1, -3x + 2y + 5z  <= 12
    u2, x + 2y <= 20
    u3, x <= 10
end)

# Followed by the objective and constraints of the lower problem:

# Lower objective function
@objective(Lower(model), Min, y)

# Lower constraints
@constraint(Lower(model), l1,  2x -  y + 3z <= 7)
@constraint(Lower(model), l2, -2x +  4y <= 16)
@constraint(Lower(model), l3, y <= 5)

# Using MibS Solver
solution = BilevelJuMP.solve_with_MibS(model, silent = true)

# Auto testing
@test solution.status == true
@test solution.objective ≈ -8.0
@test solution.nonzero_upper == Dict(0 => 6.0)
@test solution.nonzero_lower == Dict(0 => 5.0)
@test solution.all_upper["x"] == 6.0
@test solution.all_upper["z"] == 0
@test solution.all_lower["y"] == 5.0


using LinearAlgebra

const companion = CommutativeRings.companion

@testset "vector spaces" begin

    Z = ZZ/7

    xnull = VectorSpace(Matrix{Z}(undef, 5, 0))
    xall = VectorSpace(diagm(ones(Z, 5)))
    @test intersect(xnull, xnull) == xnull
    @test intersect(xnull, xall) == xnull
    @test intersect(xall, xall) == xall
    @test sum(xnull, xnull) == xnull
    @test sum(xall, xnull) == xall
    @test sum(xall, xall) == xall

    va = VectorSpace(Z[1 0; 0 1; 0 0; 0 0; 0 0])
    vb = VectorSpace(Z[0 0; 0 0; 0 0; 0 1; 1 0])

    vc = sum(va, vb)
    @test Matrix(vc) == [1 0 0 0; 0 1 0 0; 0 0 0 0; 0 0 0 1; 0 0 1 0]

    vc = intersect(va, vb)
    @test Matrix(vc) == Matrix{Z}(undef,5,0)

    vc = complement(vb)
    @test Matrix(vc) == Z[0 0 1; 0 1 0; 1 0 0; 0 0 0; 0 0 0]

    wa = VectorSpace(Z[1, 2, 3, 4, 5])
    wb = VectorSpace(Z[1, 2, 4, 4, 0])
    wc = VectorSpace(Z[0, 0, 2, 1, 0])
    xa = sum(wa, wc)
    xb = sum(wb, wc)

    @test intersect(wa, xa) == wa
    @test intersect(xa, wc) == wc
    @test intersect(wc, xb) == wc
    @test intersect(xb, wb) == wb

    xint = intersect(xa, xb)
    @test xint == wc

    xsum = sum(xa, xb)
    @test issubset(wa, xsum)
    @test issubset(wb, xsum)
    @test issubset(wc, xsum)
    @test issubset(xa, xsum)
    @test issubset(xb, xsum)

    @test intersect(xnull, xsum) == xnull
    @test intersect(xall, xsum) == xsum
    @test sum(xnull, xsum) == xsum
    @test sum(xall, xsum) == xall
end

@testset "determinant" begin
    G = ZZ/89
    A = G.([1 2 3; 4 5 6; 7 8 9])
    B = G.([1 2 3; 4 5 6; 7 8 8])
    @test iszero(det(A))
    @test iszero(adjugate(A) * A)
    @test !iszero(det(B))
    @test adjugate(B) ./ det(B) == inv(B)

    p = characteristic_polynomial(B)
    @test iszero(p(B))
    @test characteristic_polynomial(companion(p)) == p
end


using Pkg
Pkg.activate(".")
Pkg.instantiate()

using BenchmarkTools
using Distributions
using GLM
using HypothesisTests
using LinearAlgebra
using Plots
using Random
using RDatasets
using SpecialFunctions
using Statistics

Random.seed!(666)

plot(0:0.1:10, gamma;
    xlabel="x",
    ylabel="gamma(x): log scale",
    label="",
    yscale=:log10,
)

xs_aux = rand(Uniform(-1, 1), 100)

histogram(xs_aux)

d_aux = Normal()
pdf(d_aux, 1)
cdf(TDist(10), 1)
fit(Normal, xs_aux)

OneSampleTTest(xs_aux)

wages_aux = dataset("plm", "Snmesp")
model_aux = lm(@formula(W ~ 1 + N + Y + I + K + F), wages_aux)
model_aux = glm(@formula(W ~ 1 + N + Y + I + K + F), wages_aux, Normal(), IdentityLink())

X_aux = randn(10, 4)
eigen_aux = eigen(X_aux'*X_aux)

@time eigen(X_aux'*X_aux);


using TensorOperations
using KrylovKit
using LinearAlgebra
using Random

function init_random_MPS(d,m,N)

    MPS=Dict()
    MPS[1] = im*rand(1,d,m)
    for i in 2:N-1
        MPS[i]= im*rand(m,d,m)
    end
    MPS[N] = im*rand(m,d,1)
    return MPS
end

function Construct_Ising_MPO(h,J,N)
    d=2
    D=3
    id = [ 1 0 ; 0 1 ]
    sp = [ 0 1 ; 0 0 ]
    sm = [ 0 0 ; 1 0 ]
    sz = [ 1 0 ; 0 -1 ]
    sx = [ 0 1 ; 1 0 ]
    id = [ 1 0 ; 0 1 ]
    W = im *  zeros(D,D,d,d)
    W1 = im *  zeros(1,D,d,d)
    W2 = im *  zeros(D,1,d,d)
    W[1,1,:,:]=id
    W[2,1,:,:]=sz
    W[3,1,:,:]=-h*sx
    W[3,2,:,:]=-J*sz
    W[3,3,:,:]=id;

    W1[1,1,:,:]=-h*sx
    W1[1,2,:,:]=-J*sz
    W1[1,3,:,:]=id

    W2[1,1,:,:]=id
    W2[2,1,:,:]=sz
    W2[3,1,:,:]=-h*sx

    MPO = Dict()
    MPO[1] = W1
    for i in 2:(N-1)
        MPO[i] = W
    end
    MPO[N] = W2

    return MPO
end

function Normalize(MPS)
    norm = MPS_dot(MPS,MPS)
    for i in 1:length(MPS)
        MPS[i] = MPS[i]/norm^(1/(2*length(MPS)))
    end
    return MPS
end

function MPS_dot(A,B)
    E=ones(1,1)
    for i in 1:length(A)
        E = contract_from_left_MPS( A[i] , E , B[i] )
    end
    return E[1]
end

## initial E and F matrices for the left and right vacuum states
function initial_L(W,N)
    sW=size(W)
    L = ones(sW[1],1,1)#im*zeros(sW[1],1,1)
    #L[1] = 1
    #println(size(L))
    return L
end

function initial_R(W,N)
    sW=size(W)
    R = ones(sW[2],1,1)#im*zeros(sW[2],1,1)
    #R[end] = 1
    #println(size(R))
    return R
end

function construct_R(MPS, MPO, Blist, N)
    R = Dict()
    R[N] = initial_R(MPO[N],N)
    for i in N:-1:2
        R[i-1] = contract_from_right(R[i], MPS[i], MPO[i])
    end
    return R
end

function construct_L(Alist, MPO, Blist,N)
    L = Dict()
    L[1] = initial_L(MPO[1],N)
    return L
end