include("testfunctions.jl")

module opti
using LinearAlgebra

# include("line_search.jl")
include("opti_1d.jl")

function bracketing_line_search(fcn, x0, step_direction; max_iter = 100, alpha = 1.0, x_tol = 1e-2)

    obj_fcn(alpha) = fcn( x0 + step_direction * alpha )
    x_lo = 0.0
    x_hi = alpha
    bracketing_interval = bracket_min(obj_fcn, x_lo, x_hi, max_iter = max_iter)

    stat = golden_section_search(
        obj_fcn,
        bracketing_interval.x_lo,
        bracketing_interval.x_hi,
        max_iter = max_iter,
        x_tol = x_tol)

    x_sol = x0 + stat.x_sol * step_direction
    return (x_sol, stat.y_sol)
end

function basis(n_dim::Int, ii::Int)
    e_i = zeros(n_dim)
    e_i[ii] = 1.0
    return e_i
end

struct DirectMethodToleranceOptions
    f_abs_tol::Float64
    f_rel_tol::Float64
end
function DirectMethodToleranceOptions()
    f_abs_tol = 1e-10
    f_rel_tol = 1e-10
    return DirectMethodToleranceOptions(f_abs_tol, f_rel_tol)
end

function powell(
    fcn,
    x0;
    max_iter = 100,
    step_size_tol = 1e-10,
    line_search_x_tol = 1e-2,
    reset_iter = 5 * length(x0),
    tol_options = DirectMethodToleranceOptions()
    )

    x = copy(x0)
    mach_eps = tol_options.f_abs_tol * 1e-5

    n_dim = length(x0)
    U = Matrix( one(eltype(x0)) * I , n_dim, n_dim )

    break_flag = false

    f_prev = fcn(x0)
    iter = 0
    for outer iter = 1:max_iter

        for ii = 1:n_dim
            (x, f_act) = bracketing_line_search(
                fcn,
                x,
                U[:, ii];
                max_iter = max_iter,
                x_tol = line_search_x_tol)

                # stopping criteria for function value
            if abs(f_act - f_prev) <= tol_options.f_abs_tol || abs(f_act - f_prev) / ( abs(f_prev) == 0 ? mach_eps : abs(f_prev)) < tol_options.f_rel_tol
                break_flag = true
                break
            end

            println(" x = $(x) ")
        end

        if break_flag
            break
        end

        for ii = 2:n_dim
            U[:, ii-1] = U[:, ii]
        end
        d = x - U[:,n_dim]
        U[:,n_dim] = d
        (x_hat, f_act) = bracketing_line_search(
                fcn,
                x,
                U[:, n_dim];
                max_iter = max_iter,
                x_tol = line_search_x_tol)
        println(" x_hat = $(x_hat) ")
        step = x_hat - x
        if norm(step) <= step_size_tol
            break
        end

        if iter % reset_iter == 0
            U = Matrix( one(eltype(x0)) * I , n_dim, n_dim )
        end

    end

    return (x, iter)

end

function exponential_annealing_schedule(iter, start_value = 1.0, gamma = 1.0)
    return gamma * start_value^iter
end

function fast_annealing_schedule(iter, start_value = 1.0)
    return start_value / iter
end

function logarithmic_annealing_schedule(iter, gamma = 1.0)
    return log( gamma * iter )
end

function simulated_annealing(
    fcn,
    x0,
    rand_fcn;
    max_iter = 100,
    annealing_schedule = exponential_annealing_schedule
    )

    # gamma = 10
    # annealing_schedule(iter) = gamma * iter

    n_dim = length(x0)
    x_path = zeros(n_dim, max_iter)

    f_old = fcn(x0)
    x_best = x0
    f_best = f_old

    x_old = copy(x0)
    iter = 0
    for outer iter = 1:max_iter

        x_new = x_old + rand_fcn(x_old, iter)
        x_path[:, iter] = x_new

        f_new = fcn(x_new)
        df = f_new - f_old

        if df <= 0
            x_old = x_new
            f_old = f_new
        elseif rand() < exp( -df / annealing_schedule(iter) )
            x_old = x_new
            f_old = f_new
        end

        if f_new <= f_best
            x_best = x_new
            f_best = f_new
        end
    end

    return (x_best, f_best, iter, x_path)

end

function corana_update!(v, a, c, n_crit_cycles)
    for ii = 1:length(v)

        if a[ii] > 0.6 * n_crit_cycles
            v[ii] *= (1 + c[ii] / 0.4 * ( a[ii]/n_crit_cycles - 0.6 )  )
        elseif a[ii] < 0.4 * n_crit_cycles
            v[ii] /= (1 + c[ii] / 0.4 * (0.4 - a[ii]/n_crit_cycles))
        end

    end
    return v
end

function adaptive_simulated_annealing(
    fcn,
    x0,
    v,
    temperature,
    eps;
    n_crit_cycles = 20,
    neps = 4,
    n_crit_resets = max( 100, 5 * length(x0) ),
    gamma = 0.85,
    c = fill(2, length(x0)),
    max_iter = 100
    )

    x_best = copy(x0)
    x_old  = copy(x0)

    f_old  = fcn(x0)
    f_best = f_old

    y_arr = []
    n_dim = length(x0)
    uniform_random() = 2 * (rand() - 0.5)

    a = zeros(n_dim)
    counts_cycles = 0
    counts_resets = 0

    basis_vectors = Matrix{eltype(x0)}( one(eltype(x0)) * I, n_dim, n_dim )

    iter = 0
    for outer iter = 1:max_iter

        for ii = 1:n_dim
            x_new = x_old + basis_vectors[:,ii] * uniform_random() * v[ii]
            f_new = fcn(x_new)

            df = f_new - f_old
            if df < 0 || rand() < exp( df / temperature )
                x_old = x_new
                f_old = f_new
                a[ii] += 1
                if f_new < f_best
                    x_best = x_new
                    f_best = f_new
                end
            end
        end

        counts_cycles += 1
        # counts_cycles >= n_crit_cycles || continue
        if counts_cycles < n_crit_cycles
            continue
        end

        counts_cycles = 0
        corana_update!(v, a, c, n_crit_cycles)
        fill!(a, 0)
        counts_resets += 1
        if counts_resets < n_crit_resets
            continue
        end

        temperature *= gamma
        counts_resets = 0
        push!(y_arr, f_old)

        Ly = length(y_arr)
        cond = Ly > neps && y_arr[end] - y_best <= eps &&
         (all(abs(y_arr[end] - y_arr[end-u])) <= eps for u in 1:neps)

        if cond
            x_old = x_best
            f_old = f_best
        else
            break
        end

    end

    return (x_best, f_best)
end

end # opti

module otest
using ..optitestfun
using ..opti
using PyPlot
PyPlot.pygui(true)

fcn = optitestfun.rosenbrock # optitestfun.quadratic
x0 = [-1.8, -2.0] # [10, 8.0]

rand_fcn(x, iter) = 1 / log(iter+1) * randn( length(x) )

# (x_best, y_best, iter, x_path) = opti.simulated_annealing(
#     fcn,
#     x0,
#     rand_fcn;
#     max_iter = 300,
#     annealing_schedule = opti.fast_annealing_schedule) # opti.exponential_annealing_schedule)

v = [1.0, 1.0]
temp = 1.0
eps = 1e-1
(x_best, f_best) = opti.adaptive_simulated_annealing(
    fcn,
    x0,
    v,
    temp,
    eps;
    max_iter = 1000)

# PyPlot.figure()
# PyPlot.grid()
# PyPlot.plot(
#     x_path[1,:], x_path[2,:], linestyle = :none, marker = :.)
# PyPlot.plot( x_best[1], x_best[2], linestyle = :none, marker = :., color = :r, markersize = 15)

function test_powell()
    # fcn = optitestfun.quadratic
    # x0 = [10, 8.0]

    fcn(x) = optitestfun.rosenbrock(x, 2)
    x0 = [-1.8, -2.0]

    (x_sol, iter) = opti.powell( fcn, x0, step_size_tol = 1e-2, line_search_x_tol = 1e-3 )

    # xs = collect( range(-1.0, 10, 30) )
    # ys = collect( range(-1.0, 10, 30) )

    # zs = zeros( length(xs), length(ys) )
    # ii = 0
    # for xx in xs
    #     for yy in ys
    #         ii += 1
    #         zs[ii] = fcn([xx, yy])
    #     end
    # end

    # PyPlot.figure()
    # PyPlot.grid()
    # PyPlot.contour(xs, ys, zs, [1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 0.2, 0.5, 1, 2, 5, 10, 20, 30, 50, 100])
end

end


using Revise
using DrWatson
@quickactivate "SICMModelingToolkit"
DrWatson.greet()
using ModelingToolkit, Unitful, DifferentialEquations, Optim
using Latexify
using BenchmarkTools, LinearAlgebra, ForwardDiff, Zygote
using Quadrature, StaticArrays
using SICMModelingToolkit
using Plots
using Measures

#### section 1.4

@parameters t m k
D = Differential(t)

locals1 = generate_generic_localtuple(1)
locals2 = generate_generic_localtuple(2)
locals3 = generate_generic_localtuple(3)

velocity(locals3)

L_free_mass = L_free_particle(m)

Lagrange_equations(L_free_mass, locals3)

L_harmonic_spring_apply = L_harmonic_spring(m, k)
Lagrange_equations(L_harmonic_spring_apply, locals1)
Lagrange_equations(L_harmonic_spring_apply, locals3)

ode_generator = ode_problem_generator(L_free_particle, locals3, [m])
#Lagrangian_pre_param, locals, params = L_free_particle, locals3, [m]

#ode_func = ODEFunction(ode_generator[1])

#du = [0., 0., 0., 0., 0., 0.]
#u0 = @SVector [1., 0.5, 0.25, 0., 0., 0.]
#p0 = @SVector [1.0]
#ode_func(du, u0, p0, 0.0)

#@show du

#dyn_ode_func(du, u, p, t) = DynamicalODEFunction{true}(ode_func)

du = [0., 0., 0.]
q0 = [0., 0., 0.]
qv0 = [1.0, 0.5, 0.25]
#u0 = [1., 0.5, 0.25, 0., 0., 0.]
u0 = ArrayPartition(qv0, q0)
p0 = [1.0]
t0 = 0.0

tspan = (0.0, 2.0)

prob = ode_generator(qv0, q0, tspan, p0; saveat=0.05)

sol = solve(prob, McAte5(); dt=0.05)

plot(sol, vars=(4,5,6); dpi=350)

### kepler

kepler_ode_generator = ode_problem_generator(L_kepler, locals2, [m, k])

begin
    q0_kep = [0.0, 1.0]
    qv0_kep = [-1.0, 0.0]
	tspan_kep = (0.0, 5.0)
    p_kep = [3.0, 10.0]
    dt = 0.01

    kepler_prob = kepler_ode_generator(qv0_kep, q0_kep, tspan_kep, p_kep)
end

kepler_sol = solve(kepler_prob, McAte5(); dt=dt)
kepler_sol_nosymp = solve(kepler_prob, Tsit5(); dt=dt)

curxlims = (-1.1, 1.1)
curylims = (9.0/16) .* curxlims
cur_left_margin = Measures.Length{:mm, Float64}(12)
cur_bottom_margin = Measures.Length{:mm, Float64}(7)
gr()
ellipseplot = plot(kepler_sol, vars=(4,3); dpi=500, ylims=curylims, xlims=curxlims,
    label="orbiter", xlabel="x", ylabel="y", title="Kepler Orbital Problem Symplectic",
    left_margin=cur_left_margin, bottom_margin=cur_bottom_margin)
scatter!(ellipseplot, [0.0], [0.0]; label="planet")
save("kepler_symplectic.png", ellipseplot)
ellipseplot_nosymp = plot(kepler_sol_nosymp, vars=(4,3); dpi=500, ylims=curylims, xlims=curxlims,
    label="orbiter", xlabel="x", ylabel="y", title="Kepler Orbital Problem Nonsymplectic",
    left_margin=cur_left_margin, bottom_margin=cur_bottom_margin)
scatter!(ellipseplot_nosymp, [0.0], [0.0]; label="planet")
save("kepler_nosymplectic.png", ellipseplot_nosymp)
plot(kepler_sol, vars=(0,3,4); dpi=350)

#### harmonic

harmonic_ode_generator = ode_problem_generator(L_harmonic_spring, locals1, [m, k])

begin
    q0_harm = [1.0]
    qv0_harm = [0.0]
	tspan_harm = (0.0, 5.0)
    p_harm = [1.0, 1.0]
    dt = 0.01

    harmonic_prob = harmonic_ode_generator(qv0_harm, q0_harm, tspan_harm, p_harm)
end

harmonic_sol = solve(harmonic_prob, McAte5(); dt=dt)
plot(harmonic_sol; dpi=500, left_margin=cur_left_margin, bottom_margin=cur_bottom_margin)


# A Julia-native gamma function. Pretty darn accurate and fast, and using this
# instead of SpecialFunctions.gamma would mean that the dependency can be
# removed. That doesn't seem worth it to me at this point in time, but putting
# this here in case.

const _G  = 607/128
const _C1 = 0.99999999999999709182
const _C  = (57.156235665862923517,
            -59.597960355475491248,
             14.136097974741747174,
            -0.49191381609762019978,
            .33994649984811888699e-4,
            .46523628927048575665e-4,
           -.98374475304879564677e-4,
            .15808870322491248884e-3,
           -.21026444172410488319e-3,
            .21743961811521264320e-3,
           -.16431810653676389022e-3,
            .84418223983852743293e-4,
           -.26190838401581408670e-4,
            .36899182659531622704e-5)
const _Cl = length(_C)
const SQRT_2_PI = sqrt(2*pi)

# Lanczos approximation.
@inline function _gamma_pos(_z::T) where{T}
  T_one    = one(T)
  T_half   = T_one/(T_one+T_one)
  z_half   = _z-T_half
  z_g_half = z_half+_G
  out = 0.99999999999999709182
  zpj = _z
  @inbounds for c in _C
    out += c/zpj
    zpj += T_one
  end
  out*SQRT_2_PI*exp(z_half*log(z_g_half) - z_g_half)
end

function _gamma(x)
  iszero(x) && return Inf
  (isinteger(x) && x < zero(x)) && return NaN
  x > zero(x) && return _gamma_pos(x)
  flx = floor(x)
  xf  = x - flx
  g   = _gamma_pos(xf)
  _x  = xf-one(x)
  for _ in 1:Int(-flx)
    g  /= _x
    _x -= one(x)
  end
  g
end


# Hello!
# This is the second Project Euler question where I find the sum of the
# even-valued terms of the Fibonacci Sequence under 4 million!

# I am using Binet's Formula to calculate the terms in the Fibonacci Sequence!

# The Golden Ratio!
φ = (1 + sqrt(5))/2

# The Binet's Formula is :
# the golden ratio (~ 1.618) raised to the nth power and divide it by sqrt of 5
# and rounding the answer to the nearest integer. The error approaches 0 as you
# approach the infinitely big terms!

fibonacci_term = 0
a = (φ ^ fibonacci_term)/sqrt(5)
b = round(Int, a)

# I can calculate many Fibonacci Numbers at once!
# Binet's Formula in ACTION!!!

c = 100
d = 1 : c
e = φ .^ d
f = e/sqrt(5)
fibonacci_numbers = round.(f)

# I can just find the even-valued Fibonacci terms!
# Every third Fibonacci term is even!

g = 100
h = 1 : g
# 3 * h is too skip 2 terms and go to the next even-valued Fibonacci Number.
i = φ .^ (3 * h)
j = i / sqrt(5)
even_valued_fibonacci_numbers = round.(j)

# I control the size of the Fibonacci Numbers.
# I am just setting the domain for the Fibonacci Numbers because I do not want
# them to be too big (The even-valued Fibonacci terms cannot be greater than
# 4 million).

k = 1
sum = 0
while even_valued_fibonacci_numbers[k] < 4000000
	k = k + 1
	sum = sum + even_valued_fibonacci_numbers[k - 1]
end
println(sum)


using LinearAlgebraicRepresentation
Lar = LinearAlgebraicRepresentation
using Plasm

VOID = [[]], [[1]]

# example 1

V = [[0,0] [1,0] [2,0] [0,1] [1,1] [2,1] [0,2] [1,2] [2,2]];
FV = [[1,2,4],[2,3,5],[3,5,6],[4,5,7],[5,7,8],[6,8,9]];
pattern = repeat([1,2,-3],outer=4);
model = (V,FV);
W,FW = Lar.extrudeSimplicial(model, pattern);
Plasm.view(W,FW)

# example 2

model = Lar.extrudeSimplicial( VOID, ones(10) )
Plasm.view(model)
model = Lar.extrudeSimplicial( model, ones(10) )
Plasm.view(model)
model = Lar.extrudeSimplicial( model, ones(10) )
Plasm.view(model)

# example 3

model = Lar.extrudeSimplicial( VOID, repeat([1,-1],outer=10) )
Plasm.view(model)
model = Lar.extrudeSimplicial( model, repeat([1,-1],outer=10) )
Plasm.view(model)

# example 4

grid_2d = Lar.simplexGrid([3,3])
Plasm.view(grid_2d)
grid_3d = Lar.simplexGrid([2,3,4])
Plasm.view(grid_3d)
V,CV = Lar.simplexGrid([1,1,1])
Plasm.view(V,CV)

# example 5

SK2 = Lar.simplexFacets(CV)
Plasm.view(V, SK2)
SK1 = Lar.simplexFacets(SK2)
Plasm.view(V, SK1)


#*******************************************************/
#* Copyright(c) 2018 by Artelys                        */
#* This source code is subject to the terms of the     */
#* MIT Expat License (see LICENSE.md)                  */
#*******************************************************/

#++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
# This example demonstrates how to use Knitro to solve the following
# mixed-integer nonlinear optimization problem(MINLP).  This model
# is test problem 1(Synthesis of processing system) in
# M. Duran & I.E. Grossmann,  "An outer approximation algorithm for
# a class of mixed integer nonlinear programs", Mathematical
# Programming 36, pp. 307-339, 1986.  The problem also appears as
# problem synthes1 in the MacMINLP test set.
#
# min   5 x3 + 6 x4 + 8 x5 + 10 x0 - 7 x2 -18 log(x1 + 1)
#      - 19.2 log(x0 - x1 + 1) + 10
# s.t.  0.8 log(x1 + 1) + 0.96 log(x0 - x1 + 1) - 0.8 x2 >= 0
#       log(x1 + 1) + 1.2 log(x0 - x1 + 1) - x2 - 2 x5 >= -2
#       x1 - x0 <= 0
#       x1 - 2 x3 <= 0
#       x0 - x1 - 2 x4 <= 0
#       x3 + x4 <= 1
#       0 <= x0 <= 2
#       0 <= x1 <= 2
#       0 <= x2 <= 1
#       x0, x1, x2 continuous
#       x3, x4, x5 binary
#
# The solution is(1.30098, 0, 1, 0, 1, 0).
#++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

using KNITRO
using Test

function example_minlp1(; verbose=true)
    #*------------------------------------------------------------------*
    #*     FUNCTION callbackEvalFC                                      *
    #*------------------------------------------------------------------*
    # The signature of this function matches KNITRO.KN_eval_callback in knitro.h.
    # Only "obj" and "c" are set in the KNITRO.KN_eval_result structure.
    function callbackEvalFC(kc, cb, evalRequest, evalResult, userParams)
        x = evalRequest.x

        # Evaluate nonlinear objective structure
        dTmp1 = x[1] - x[2] + 1.0
        dTmp2 = x[2] + 1.0
        evalResult.obj[1] = -18.0 * log(dTmp2) - 19.2 * log(dTmp1)

        # Evaluate nonlinear constraint structure
        evalResult.c[1] = 0.8 * log(dTmp2) + 0.96 * log(dTmp1)
        evalResult.c[2] = log(dTmp2) + 1.2 * log(dTmp1)

        return 0
    end

    #*------------------------------------------------------------------*
    #*     FUNCTION callbackEvalGA                                      *
    #*------------------------------------------------------------------*
    # The signature of this function matches KNITRO.KN_eval_callback in knitro.h.
    # Only "objGrad" and "jac" are set in the KNITRO.KN_eval_result structure.
    function callbackEvalGA(kc, cb, evalRequest, evalResult, userParams)
        x = evalRequest.x

        # Evaluate gradient of nonlinear objective structure
        dTmp1 = x[1] - x[2] + 1.0
        dTmp2 = x[2] + 1.0
        evalResult.objGrad[1] = -(19.2 / dTmp1)
        evalResult.objGrad[2] = (-18.0 / dTmp2) + (19.2 / dTmp1)

        # Gradient of nonlinear structure in constraint 0.
        evalResult.jac[1] = 0.96 / dTmp1                      # wrt x0
        evalResult.jac[2] = (-0.96 / dTmp1) + (0.8 / dTmp2)   # wrt x1
        # Gradient of nonlinear structure in constraint 1.
        evalResult.jac[3] = 1.2 / dTmp1                       # wrt x0
        evalResult.jac[4] = (-1.2 / dTmp1) + (1.0 / dTmp2)    # wrt x1

        return 0
    end

    #*------------------------------------------------------------------*
    #*     FUNCTION callbackEvalH                                       *
    #*------------------------------------------------------------------*
    # The signature of this function matches KNITRO.KN_eval_callback in knitro.h.
    # Only "hess" or "hessVec" are set in the KNITRO.KN_eval_result structure.
    function callbackEvalH(kc, cb, evalRequest, evalResult, userParams)
        x = evalRequest.x
        lambda_ = evalRequest.lambda
        # Scale objective component of hessian by sigma
        sigma = evalRequest.sigma

        # Evaluate the non-zero components in the Hessian of the Lagrangian.
        # Note: Since the Hessian is symmetric, we only provide the
        #       nonzero elements in the upper triangle(plus diagonal).
        dTmp1 = x[1] - x[2] + 1.0
        dTmp2 = x[2] + 1.0
        evalResult.hess[1] = sigma * (19.2 /(dTmp1 * dTmp1)) +
                                lambda_[1] * (-0.96 / (dTmp1 * dTmp1)) + lambda_[2] * (-1.2 / (dTmp1 * dTmp1))
        evalResult.hess[2] = sigma * (-19.2 /(dTmp1 * dTmp1)) +
                                lambda_[1] * (0.96 / (dTmp1 * dTmp1)) + lambda_[2] * (1.2 / (dTmp1 * dTmp1))
        evalResult.hess[3] = sigma * ((19.2 / (dTmp1 * dTmp1)) +(18.0 / (dTmp2 * dTmp2))) +
                                lambda_[1] * ((-0.96 / (dTmp1 * dTmp1)) -(0.8 / (dTmp2 * dTmp2))) +
                                lambda_[2] * ((-1.2 / (dTmp1 * dTmp1)) -(1.0 / (dTmp2 * dTmp2)))

        return 0
    end

    #*------------------------------------------------------------------*
    #*     FUNCTION callbackProcessNode                                 *
    #*------------------------------------------------------------------*
    # The signature of this function matches KNITRO.KN_user_callback in knitro.h.
    # Argument "kcSub" is the context pointer for the last node
    # subproblem solved inside Knitro.  The application level context
    # pointer is passed in through "userParams".
    function callbackProcessNode(kc, x, lambda_, userParams)
        # The Knitro context pointer was passed in through "userParams".

        # Print info about the status of the MIP solution.
        numNodes = KNITRO.KN_get_mip_number_nodes(kc)
        relaxBound = KNITRO.KN_get_mip_relaxation_bnd(kc)
        # Note: To retrieve solution information about the node subproblem
        # we need to pass in "kcSub" here.
        nodeObj = KNITRO.KN_get_obj_value(kc)
        mip_io = KNITRO.KN_get_mip_incumbent_obj(kc)
        mip_ag = KNITRO.KN_get_mip_abs_gap(kc)
        mip_rg = KNITRO.KN_get_mip_rel_gap(kc)

        if verbose
            println("callbackProcessNode:")
            println("    Node number    = ", numNodes)
            println("    Node objective = ", nodeObj)
            println("    Current relaxation bound = ", relaxBound)
            println("    Current incumbent bound  = ", mip_io)
            println("    Absolute integrality gap = ", mip_ag)
            println("    Relative integrality gap = ", mip_rg)
        end

        # User defined termination example.
        # Uncomment below to force termination after 3 nodes.
        #if(numNodes == 3)
        #    return KNITRO.KN_RC_USER_TERMINATION

        return 0
    end

    #*------------------------------------------------------------------*
    #*     main                                                         *
    #*------------------------------------------------------------------*

    # Create a new Knitro solver instance.
    kc = KNITRO.KN_new()

    # Illustrate how to override default options.
    KNITRO.KN_set_param(kc, "mip_method", KNITRO.KN_MIP_METHOD_BB)
    KNITRO.KN_set_param(kc, "algorithm", KNITRO.KN_ALG_ACT_CG)
    KNITRO.KN_set_param(kc, "outmode", KNITRO.KN_OUTMODE_SCREEN)
    KNITRO.KN_set_param(kc, KNITRO.KN_PARAM_OUTLEV, KNITRO.KN_OUTLEV_ALL)
    KNITRO.KN_set_param(kc, KNITRO.KN_PARAM_MIP_OUTINTERVAL, 1)
    KNITRO.KN_set_param(kc, KNITRO.KN_PARAM_MIP_MAXNODES, 10000)

    # Initialize Knitro with the problem definition.

    # Add the variables and set their bounds and types.
    # Note: any unset lower bounds are assumed to be
    # unbounded below and any unset upper bounds are
    # assumed to be unbounded above.
    n = 6
    KNITRO.KN_add_vars(kc, n)
    KNITRO.KN_set_var_lobnds(kc, zeros(Float64, n))
    KNITRO.KN_set_var_upbnds(kc, [2., 2., 1., 1., 1., 1.])
    KNITRO.KN_set_var_types(kc, [KNITRO.KN_VARTYPE_CONTINUOUS,
                                KNITRO.KN_VARTYPE_CONTINUOUS,
                                KNITRO.KN_VARTYPE_CONTINUOUS,
                                KNITRO.KN_VARTYPE_BINARY,
                                KNITRO.KN_VARTYPE_BINARY,
                                KNITRO.KN_VARTYPE_BINARY])

    # Note that variables x2..x5 only appear linearly in the
    # problem.  We mark them as linear variables, which may
    # help Knitro do more extensive presolving resulting in
    # faster solves.
    for i in 2:(n-1)
        KNITRO.KN_set_var_property(kc, i, KNITRO.KN_VAR_LINEAR)
    end

    # Add the constraints and set their bounds
    KNITRO.KN_add_cons(kc, 6)
    KNITRO.KN_set_con_lobnds(kc,  [0, -2,
                                -KNITRO.KN_INFINITY,
                                -KNITRO.KN_INFINITY,
                                -KNITRO.KN_INFINITY,
                                -KNITRO.KN_INFINITY])
    KNITRO.KN_set_con_upbnds(kc, [KNITRO.KN_INFINITY, KNITRO.KN_INFINITY, 0, 0, 0, 1])

    # Add the linear structure in the objective function.
    objGradIndexVars = Int32[3, 4, 5, 0, 2]
    objGradCoefs = [5.0, 6.0, 8.0, 10.0, -7.0]
    KNITRO.KN_add_obj_linear_struct(kc, objGradIndexVars, objGradCoefs)

    # Add the constant in the objective function.
    KNITRO.KN_add_obj_constant(kc, 10.0)

    # Load the linear structure for all constraints at once.
    jacIndexCons = Int32[0, 1, 1, 2, 2, 3, 3, 4, 4, 4, 5, 5]
    jacIndexVars = Int32[2, 2, 5, 1, 0, 1, 3, 0, 1, 4, 3, 4]
    jacCoefs = [-0.8, -1.0, -2.0, 1.0, -1.0, 1.0, -2.0, 1.0, -1.0, -2.0, 1.0, 1.0]
    KNITRO.KN_add_con_linear_struct(kc, jacIndexCons, jacIndexVars, jacCoefs)

    # Add a callback function "callbackEvalFC" to evaluate the nonlinear
    # structure in the objective and first two constraints.  Note that
    # the linear terms in the objective and first two constraints were
    # added above in "KNITRO.KN_add_obj_linear_struct()" and
    # "KNITRO.KN_add_con_linear_struct()" and will not be specified in the
    # callback.
    cIndices = Int32[0, 1] # Constraint indices for callback
    cb = KNITRO.KN_add_eval_callback(kc, true, cIndices, callbackEvalFC)

    # Also add a callback function "callbackEvalGA" to evaluate the
    # gradients of all nonlinear terms specified in the callback.  If
    # not provided, Knitro will approximate the gradients using finite-
    # differencing.  However, we recommend providing callbacks to
    # evaluate the exact gradients whenever possible as this can
    # drastically improve the performance of Knitro.
    # Objective gradient non-zero structure for callback
    objGradIndexVarsCB = Int32[0, 1]
    # Constraint Jacobian non-zero structure for callback
    jacIndexConsCB = Int32[0, 0, 1, 1]
    jacIndexVarsCB = Int32[0, 1, 0, 1]
    KNITRO.KN_set_cb_grad(kc, cb, callbackEvalGA,
                        nV=length(objGradIndexVarsCB),
                        objGradIndexVars=objGradIndexVarsCB,
                        jacIndexCons=jacIndexConsCB,
                        jacIndexVars=jacIndexVarsCB)

    hessIndexVars1CB = Int32[0, 0, 1]
    hessIndexVars2CB = Int32[0, 1, 1]
    # Add a callback function "callbackEvalH" to evaluate the Hessian
    #(i.e. second derivative matrix) of the objective.  If not specified,
    # Knitro will approximate the Hessian. However, providing a callback
    # for the exact Hessian(as well as the non-zero sparsity structure)
    # can greatly improve Knitro performance and is recommended if possible.
    # Since the Hessian is symmetric, only the upper triangle is provided.
    KNITRO.KN_set_cb_hess(kc, cb, length(hessIndexVars1CB), callbackEvalH,
                        hessIndexVars1=hessIndexVars1CB,
                        hessIndexVars2=hessIndexVars2CB)

    # Specify that the user is able to provide evaluations
    #  of the Hessian matrix without the objective component.
    #  turned off by default but should be enabled if possible.
    KNITRO.KN_set_param(kc, KNITRO.KN_PARAM_HESSIAN_NO_F, KNITRO.KN_HESSIAN_NO_F_ALLOW)

    # Set minimize or maximize(if not set, assumed minimize)
    KNITRO.KN_set_obj_goal(kc, KNITRO.KN_OBJGOAL_MINIMIZE)

    # Set a callback function that performs some user-defined task
    # after completion of each node in the branch-and-bound tree.
    KNITRO.KN_set_mip_node_callback(kc, callbackProcessNode)

    kn_outlev = verbose ? KNITRO.KN_OUTLEV_ALL : KNITRO.KN_OUTLEV_NONE
    KNITRO.KN_set_param(kc, KNITRO.KN_PARAM_OUTLEV, kn_outlev)

    # Solve the problem.
    #
    # Return status codes are defined in "knitro.h" and described
    # in the Knitro manual.

    nStatus = KNITRO.KN_solve(kc)
    # An example of obtaining solution information.
    nSTatus, objSol, x, lambda_ = KNITRO.KN_get_solution(kc)

    if verbose
        println("Optimal objective value  = ", objSol)
        println("Optimal x")
        for i in 1:n
            println("  x[$i] = ", x[i])
        end
    end

    # Delete the Knitro solver instance.
    KNITRO.KN_free(kc)

    @testset "Example minlp1" begin
        @test nStatus == 0
        @test objSol ≈ 6.0097589
        @test x ≈ [1.30097589, 0., 1., 0., 1., 0.] atol=1e-5
    end
end

example_minlp1(; verbose=isdefined(Main, :KN_VERBOSE) ? KN_VERBOSE : true)


# ---
# title: 1137. N-th Tribonacci Number
# id: problem1137
# author: Tian Jun
# date: 2020-10-31
# difficulty: Easy
# categories: Recursion
# link: <https://leetcode.com/problems/n-th-tribonacci-number/description/>
# hidden: true
# ---
#
# The Tribonacci sequence Tn is defined as follows:
#
# T0 = 0, T1 = 1, T2 = 1, and Tn+3 = Tn \+ Tn+1 \+ Tn+2 for n >= 0.
#
# Given `n`, return the value of Tn.
#
#
#
# **Example 1:**
#
#
#
#     Input: n = 4
#     Output: 4
#     Explanation:
#     T_3 = 0 + 1 + 1 = 2
#     T_4 = 1 + 1 + 2 = 4
#
#
# **Example 2:**
#
#
#
#     Input: n = 25
#     Output: 1389537
#
#
#
#
# **Constraints:**
#
#   * `0 <= n <= 37`
#   * The answer is guaranteed to fit within a 32-bit integer, ie. `answer <= 2^31 - 1`.
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


using Pkg
Pkg.activate(pwd())

# # Constrained Optimization
# ## Numerical method

using Plots, Random

function create_anim(
    f,
    path,
    xlims,
    ylims,
    file_name = joinpath(pwd(), randstring(12) * ".gif");
    xbounds = xlims,
    ybounds = ylims,
    fps = 15,
)
    xs = range(xlims...; length = 100)
    ys = range(ylims...; length = 100)
    plt = contourf(xs, ys, f; color = :jet)

    # add constraints if provided
    if !(xbounds == xlims && ybounds == ylims)
        x_rect = [xbounds[1]; xbounds[2]; xbounds[2]; xbounds[1]; xbounds[1]]
        y_rect = [ybounds[1]; ybounds[1]; ybounds[2]; ybounds[2]; ybounds[1]]

        plot!(x_rect, y_rect; line = (2, :dash, :red), label="")
    end

    # add an empty plot
    plot!(Float64[], Float64[]; line = (4, :arrow, :black), label = "")

    # extract the last plot series
    plt_path = plt.series_list[end]

    # create the animation and save it
    anim = Animation()
    for x in eachcol(path)
        push!(plt_path, x[1], x[2]) # add a new point
        frame(anim)
    end
    gif(anim, file_name; fps = fps, show_msg = false)
    return nothing
end

f(x) = sin(x[1] + x[2]) + cos(x[1])^2
f(x1,x2) = f([x1;x2])
g(x) = [cos(x[1] + x[2]) - 2*cos(x[1])*sin(x[1]); cos(x[1] + x[2])]

#+

function optim(f, g, P, x, α; max_iter=100)
    xs = zeros(length(x), max_iter+1)
    ys = zeros(length(x), max_iter)
    xs[:,1] = x
    for i in 1:max_iter
        ys[:,i] = xs[:,i] - α*g(xs[:,i])
        xs[:,i+1] = P(ys[:,i])
    end
    return xs, ys
end

#+

P(x, x_min, x_max) = min.(max.(x, x_min), x_max)

x_min = [-1; -1]
x_max = [0; 0]

xs, ys = optim(f, g, x -> P(x,x_min,x_max), [0;-1], 0.1)

xlims = (-3, 1)
ylims = (-2, 1)

#+

create_anim(f, xs, xlims, ylims, joinpath(pwd(), "lecture_08", "anim6.gif");
    xbounds=(x_min[1], x_max[1]),
    ybounds=(x_min[2], x_max[2]),
)

#+

xys = hcat(reshape([xs[:,1:end-1]; ys][:], 2, :), xs[:,end])

create_anim(f, xys, xlims, ylims, joinpath(pwd(), "lecture_08", "anim7.gif");
    xbounds=(x_min[1], x_max[1]),
    ybounds=(x_min[2], x_max[2]),
)


module NumericError

import Base: convert, promote_rule, +, -, *, /, ^

export Measure

type Measure <: Number
  x::Float64
  σ::Float64
  Measure(x::Real) = new(Float64(x), 0)
  Measure(x::Real, σ::Real) = new(Float64(x), σ)
end

Base.show(io::IO, x::Measure) = print(io, string(x.x, " ± ", x.σ))

Base.convert(::Type{Measure}, x::Real) = Measure(Float64(x), 0.0)
Base.promote_rule(::Type{Float64}, ::Type{Measure}) = Measure
Base.promote_rule(::Type{Int64}, ::Type{Measure}) = Measure

+(a::Measure, b::Measure) = Measure(a.x + b.x, sqrt(a.σ ^ 2 + b.σ ^ 2))
-(a::Measure, b::Measure) = Measure(a.x - b.x, sqrt(a.σ ^ 2 + b.σ ^ 2))
-(a::Measure) = Measure(-a.x, a.σ)

*(a::Measure, b::Measure) = begin
  x = a.x * b.x
  σ = sqrt(x ^ 2 * ((a.σ / a.x) ^ 2 + (b.σ / b.x) ^ 2))
  Measure(x, σ)
end
/(a::Measure, b::Measure) = begin
  x = a.x / b.x
  σ = sqrt(x ^ 2 * ((a.σ / a.x) ^ 2 + (b.σ / b.x) ^ 2))
  Measure(x, σ)
end

^(a::Measure, b::Float64) = begin
  x = a.x ^ b
  σ = abs(x * b * a.σ / a.x)
  Measure(x, σ)
end

Base.sqrt(a::Measure) = a ^ .5

end  # module NumericError

using NumericError

# x1 = 100 ± 1.1
# y1 = 50 ± 1.2
# x2 = 200 ± 2.2
# y2 = 100 ± 2.3

x1 = Measure(100, 1.1)
y1 = Measure(50,  1.2)
x2 = Measure(200, 2.2)
y2 = Measure(100, 2.3)

d = sqrt((x1 - x2) ^ 2 + (y1 - y2) ^ 2)

@show x1 y1 x2 y2 sqrt((x1 - x2) ^ 2 + (y1 - y2) ^ 2)


using QuantEcon.compute_fixed_point, QuantEcon.DiscreteRV, QuantEcon.draw, QuantEcon.LinInterp

"""
For a given probability return expected loss of choosing model 0
"""
expect_loss_choose_0(p::Real, L0::Real) = (1 - p) * L0

"""
For a given probability return expected loss of choosing model 1
"""
expect_loss_choose_1(p::Real, L1::Real) = p * L1

"""
We will need to be able to evaluate the expectation of our Bellman
equation J. In order to do this, we need the current probability
that model 0 is correct (p), the distributions (f0, f1), and a
function that can evaluate the Bellman equation
"""
function EJ(p::Real, f0::AbstractVector, f1::AbstractVector, J::LinInterp)
    # Get the current distribution we believe (p * f0 + (1 - p) * f1)
    curr_dist = p * f0 + (1 - p) * f1

    # Get tomorrow's expected distribution through Bayes law
    tp1_dist = clamp.((p * f0) ./ (p * f0 + (1 - p) * f1), 0, 1)

    # Evaluate the expectation
    EJ = dot(curr_dist, J.(tp1_dist))

    return EJ
end

expect_loss_cont(p::Real, c::Real,
                 f0::AbstractVector, f1::AbstractVector, J::LinInterp) =
    c + EJ(p, f0, f1, J)

"""
Evaluates the value function for a given continuation value
function; that is, evaluates

    J(p) = min(pL0, (1-p)L1, c + E[J(p')])

Uses linear interpolation between points
"""
function bellman_operator(pgrid::AbstractVector,
                          c::Real,
                          f0::AbstractVector,
                          f1::AbstractVector,
                          L0::Real,
                          L1::Real,
                          J::AbstractVector)
    m = length(pgrid)
    @assert m == length(J)

    J_out = zeros(m)
    J_interp = LinInterp(pgrid, J)

    for (p_ind, p) in enumerate(pgrid)
        # Payoff of choosing model 0
        p_c_0 = expect_loss_choose_0(p, L0)
        p_c_1 = expect_loss_choose_1(p, L1)
        p_con = expect_loss_cont(p, c, f0, f1, J_interp)

        J_out[p_ind] = min(p_c_0, p_c_1, p_con)
    end

    return J_out
end

# Create two distributions over 50 values for k
# We are using a discretized beta distribution

p_m1 = linspace(0, 1, 50)
f0 = clamp.(pdf.(Beta(1, 1), p_m1), 1e-8, Inf)
f0 = f0 / sum(f0)
f1 = clamp.(pdf.(Beta(9, 9), p_m1), 1e-8, Inf)
f1 = f1 / sum(f1)

# To solve
pg = linspace(0, 1, 251)
J1 = compute_fixed_point(x -> bellman_operator(pg, 0.5, f0, f1, 5.0, 5.0, x),
    zeros(length(pg)), err_tol=1e-6, print_skip=5);


function log_taylor(x::Number)
    if x <= 0
        return undef
    end

    e = float(Base.MathConstants.e)
    lb = 0.546
    ub = lb * e

    res = 0

    while x < lb
        x *= e
        res -= 1
    end

    while x > ub
        x /= e
        res += 1
    end

    s = 1

    for i in 1 : 40
        res += s * (x - 1)^i / i
        s *= -1
    end

    return res
end


using DrWatson
@quickactivate "StatReth"

# %%
using CSV
using DataFrames
using Turing
using Plots

include(srcdir("quap.jl"))
include(srcdir("tools.jl"))

# %%
# This is from the chapter
d = DataFrame(CSV.File(datadir("exp_raw/Howell_1.csv")))
sort!(d, :weight)  # to make the plots look nice

z_weight, unz_weight = zscore_transform(d.weight)
d.weight_s = z_weight(d.weight)

f_parabola(weight_s, a, b1, b2) = a + b1 * weight_s + b2 * weight_s^2

@model function parabola(weight_s, height)
    a ~ Normal(178, 20)
    b1 ~ LogNormal(0, 1)
    b2 ~ Normal(0, 1)
    σ ~ Uniform(0, 50)
    μ = f_parabola.(weight_s, a, b1, b2)
    height ~ MvNormal(μ, σ)
end

mparab = parabola(d.weight_s, d.height)
prior = sample(mparab, Prior(), 1_000) |> DataFrame

# %%
plot(legend = false)
for c in eachrow(prior[1:100, :])
    p = f_parabola.(d.weight_s, c.a, c.b1, c.b2)
    plot!(d.weight_s, p, color = :black, alpha = 0.2)
end
plot!()

# %%
"""
Thinking about the cbrt explanation, b2 should be negative so I get a right turn here as
well. Since I don't want the extremum to be in the range of my data I need the maximum to
be at like 3 or 4 or something like that. Over the range of ~4 stddev I want the height to
change 100 cm or so.
f(x) = b₂ * x² + b₁ * x + a
f'(x) = 2b₂ * x + b₁

f(2) - f(-2) = 100
f'(3) = 0
b₂ < 0

f(2) - f(-2) = (b₂ * 4 + b₁ * 2 + a) - (b₂ * 4 + b₁ * -2 + a)
             = 4b₁ = 100
            =>  b₁ =  25
f'(3) = 2b₂ * 3 + b₁ = 0 => -b₁ = 6b₂
     => b₂ = -25 / 6 ≈ -4

I'll move a by hand until it seams reasonable.
"""

@model function parabola(weight_s, height)
    a ~ Normal(130, 20)
    b1 ~ Normal(25, 5)
    b2 ~ truncated(Normal(-4, 2), -Inf, 0)
    σ ~ Exponential(5)
    μ = f_parabola.(weight_s, a, b1, b2)
    height ~ MvNormal(μ, σ)
end

mparab = parabola(d.weight_s, d.height)
prior = sample(mparab, Prior(), 100) |> DataFrame

plot(legend = false)
for c in eachrow(prior)
    p = f_parabola.(d.weight_s, c.a, c.b1, c.b2)
    plot!(d.weight_s, p, color = :black, alpha = 0.2)
end
plot!(ylims = (0, 200))


using FourierTools, FFTW, PGFPlotsX

N_low = 64
x_min = 0.0
x_max = 8*2π

xs_low = range(x_min, x_max, length=N_low+1)[1:N_low]
xs_high = range(x_min, x_max, length=5000)[1:end-1]
f(x) = sin(0.5*x) + cos(x) + cos(2 * x) + sin(0.25*x)
arr_low = f.(xs_low)
arr_high = f.(xs_high)

N = 1000
xs_interp = range(x_min, x_max, length=N+1)[1:N]
arr_interp = resample(arr_low, N)

plt1 = @pgf PGFPlotsX.Axis(
    {
        xmin=0, xmax=50, ymin=-3, ymax=4.5,
        legend_pos="north east", legend_entries={"low sampling", "sinc interpolated", "high sampling"},
        width = "10cm",
        height= "6cm",
    },

    Plot({mark="x", style="only marks", color="green",mark_options = {scale=1.3},
        },
        Table([xs_low, arr_low])
    ),

    Plot({"sharp plot", color="red"
        },
        Table([xs_interp, arr_interp])
    ),

    Plot({"sharp plot", style="dashed", color="blue"
        },
        Table([xs_high, arr_high])
    ),
)
pgfsave("../../paper/figures/resampling.pdf", plt1)


"""
    generate_conditioned_dist(covar::CovarianceAtDate, conditioning_draws::Array{Symbol,T}) where T<:Real
Given some subset of known stochastic integral values this generates a conditional multivariate normal distribution.
"""
function generate_conditioned_distribution(covar::CovarianceAtDate, conditioning_draws::Dict{Symbol,T}) where T<:Real
    # https://stats.stackexchange.com/questions/30588/deriving-the-conditional-distributions-of-a-multivariate-normal-distribution
    what_conditioned_on = collect(keys(conditioning_draws))
    len = length(what_conditioned_on)
    if len < 1 error("Nothing that was input was in the covariance matrix. So nothing to condition on.") end
    conditioning_indices = Array{typeof(Integer(1)),1}()
    for i in 1:len
        append!(conditioning_indices, findall(what_conditioned_on[i] .==  covar.covariance_labels_))
    end
    other_indices = setdiff(1:length(covar.covariance_labels_), conditioning_indices)
    labels = covar.covariance_labels_[other_indices]
    # Segmenting the covariance matrix.
    sigma11 = covar.covariance_[other_indices,other_indices]
    sigma12 = covar.covariance_[other_indices,conditioning_indices]
    sigma21 = covar.covariance_[conditioning_indices,other_indices]
    sigma22 = covar.covariance_[conditioning_indices,conditioning_indices]
    mu1     = zeros(length(other_indices))
    mu2     = zeros(length(conditioning_indices))
    conditioned_values = map( x -> conditioning_draws[x], what_conditioned_on)
    sigma12_invsigma22 = sigma12 * inv(sigma22)
    conditional_mu = mu1 + sigma12_invsigma22 * (conditioned_values - mu2)
    conditional_sigma = sigma11 -  sigma12_invsigma22 * sigma21
    return conditional_mu, conditional_sigma, labels
end


# Pkg.add("ForwardDiff")
using ForwardDiff

"""
```
f(x) = x^2 - 2;
x0 = 1;
x = newton(f, x0);
```
"""
function newton(f, x0)
  # 求目标函数导数
  Δf = x -> ForwardDiff.derivative(f, x);

  xk = Base.float(x0);
  xk_1 = Base.float(x0);

  # 比机器精度少一个数量级
  epsilon = Base.eps(xk) * 10;

  while true
    xk_1 = xk - f(xk)/Δf(xk);
    if Base.abs(xk_1 - xk) < epsilon || Base.abs(f(xk_1)) < epsilon
      break;
    end
    xk = xk_1;
  end

  return xk_1;
end


"""
    bernoulli(n)

 Calculates the first 35 Bernoulli numbers ``B_n``  (of the first-kind or NIST type)
 e.g., see

 + [http://mathworld.wolfram.com/BernoulliNumber.html](http://mathworld.wolfram.com/BernoulliNumber.html)
 + [https://en.wikipedia.org/wiki/Bernoulli_number](https://en.wikipedia.org/wiki/Bernoulli_number)
 + [http://dlmf.nist.gov/24](http://dlmf.nist.gov/24)

 N.B. Bernoulli numbers of second kind only seem to differ in that ``B_1 = + 1/2`` (instead of -1/2)

## Arguments
* ``n`` `::Integer`: the index into the series, ``n=0,1,2,3,...,35`` (for larger ``n`` use `bernoulli(n,0.0)` )

 We only provide the 1st 36 values as beyond this, we can't
 return Int64 rationals, so best to compute the real approximation
 using `bernoulli(n,0.0)`.

 Odd values for ``n>1`` are all zero.

## Examples
```jldoctest; setup = :(using Polylogarithms)
julia> bernoulli(6)
1//42
```
"""
function bernoulli(n::Integer)
    # this just does a lookup -- seemed like it would be easier to code and faster
    # for the size of numbers I am working with
    if n<0
        throw(DomainError(n))
    elseif n > 1 && isodd(n)
        return 0 // 1
    elseif n > 35
        throw(DomainError(n, "If n > 35, then the numerator needs Int128 at least, and worse, so this code is not the code you want. Try using bernoulli(n, 0.0) to get a floating point approximation to the result."))
    end

    # Denominator of Bernoulli number B_n
    #   http://oeis.org/A027642
    D = [2, 6, 1, 30, 1, 42, 1, 30, 1, 66, 1, 2730, 1, 6, 1, 510, 1, 798, 1, 330, 1, 138, 1, 2730, 1, 6, 1, 870, 1, 14322, 1, 510, 1, 6, 1, 1919190, 1, 6, 1, 13530, 1, 1806, 1, 690, 1, 282, 1, 46410, 1, 66, 1, 1590, 1, 798, 1, 870, 1, 354, 1, 56786730]

    # Numerator of Bernoulli number B_n (storing 62 of these because they are easy)
    #   http://oeis.org/A027641
    N = [-1, 1, 0, -1, 0, 1, 0, -1, 0, 5, 0, -691, 0, 7, 0, -3617, 0, 43867, 0, -174611, 0, 854513, 0, -236364091, 0, 8553103, 0, -23749461029, 0, 8615841276005, 0, -7709321041217, 0, 2577687858367, 1]

    if n==0
        return 1
    else
        return N[n] // D[n]
    end
end


module DataGenerator

import Distributions; D = Distributions
using LinearAlgebra

export linregdata, logregdata, poissonregdata

eye(p) = Matrix(1.0 * I, p, p)

defaultβ(p) = collect(range(-1, stop=1, length=p))

function linregdata(n, p; β = defaultβ(p), V = eye(p), μ = zeros(p), σ = 1.0)
	x = rand(D.MvNormal(μ, V), n)'
	y = x * β + σ * randn(n)
	return x, y, β
end

function logregdata(n, p, sgn = true; β = defaultβ(p), V = eye(p), μ = zeros(p))
	x = rand(D.MvNormal(μ, V), n)'
	y = Float64[rand(D.Bernoulli(1 / (1 + exp(-η)))) for η in x * β]
	if sgn
		y = 2y .- 1
	end
	return x, y, β
end

function poissonregdata(n, p, β = defaultβ(p), V = eye(p), μ = zeros(p))
	x = rand(D.MvNormal(μ, V), n)'
	y = Float64[rand(D.Poisson(exp(η))) for η in x * β]
	return x, y, β
end

end


"""
    randSphere(K::Int)

Returns a random K-vector from the unit sphere
"""
randSphere(K::Int) = normalize(randn(K))

"Computes `γ` from `ρ`"
gammasoc(ρ::Float64) = sqrt(1 - ρ^2) / ρ

"Computes `ρ` from `γ`"
rhosoc(γ::Float64)   = 1/ sqrt(1 + γ^2)

"Computes `ϵ` from `μ` and `s^2`"
epssoc(μ::Float64, s2::Float64) = phi(μ/sqrt(1+s2))

"The cumulative function of the Normal distribution"
phi(x::Real) = cdf(Normal(), x)

"The density function of the Normal distribution"
G(x::Real) = pdf(Normal(), x)

"Returns the two modulation functions: Fw and Fϵ"
function modfunc(hσ1γ::Float64, μ1sqs2::Float64)
    phiϵ = phi(μ1sqs2)
    phiw = phi(hσ1γ)
    Z    = phiϵ + phiw - 2*phiϵ*phiw

    Fw = (1 - 2phiϵ) * G(hσ1γ) / Z
    Fϵ = (1 - 2phiw) * G(μ1sqs2) / Z

    return (Fw, Fϵ)
end


# Find a stationary solution to the Gross-Pitaevskii equation by
# minimizing the energy of a normalised wave function in a rotating
# frame

@testset "Gross-Pitaveskii vortex" begin
    C = 10
    μ = 10
    Ω = 2*0.575
    h = 0.2
    N = 40

    # Finite difference matrices.  ∂ on left is ∂y, ∂' on right is ∂x

    function op(stencil)
        mid = (length(stencil)+1)÷2
        diags = [i-mid=>fill(stencil[i],N-abs(i-mid)) for i = keys(stencil)]
        BandedMatrix(Tuple(diags), (N,N))
    end

    ∂ = (1/h).*op([-1/60, 3/20, -3/4, 0, 3/4, -3/20, 1/60])
    ∂² = (1/h^2).*op([1/90, -3/20, 3/2, -49/18, 3/2, -3/20, 1/90])

    y = h/2*(1-N:2:N-1)
    x = y'
    z = Complex.(x,y)
    V = r² = abs2.(z)

    init = Complex.(exp.(-r²/2)/√π)
    init = conj.(z).*init./sqrt(1 .+ r²)
    init ./= norm(init)

    # The energy that the steady state minimizes
    function E(ψ)
        Lψ = -∂²*ψ-ψ*∂²+V.*ψ+C*abs2.(ψ).*ψ-1im*Ω*(y.*(ψ*∂')-x.*(∂*ψ))
        sum(conj.(ψ).*Lψ)/norm(ψ)^2 |> real
    end

    # The residual for the GPE, zero in a steady state
    function rdl(ψ)
        Lψ = -∂²*ψ-ψ*∂²+V.*ψ+C*abs2.(ψ).*ψ-1im*Ω*(y.*(ψ*∂')-x.*(∂*ψ))
        E = sum(conj.(ψ).*Lψ)/norm(ψ)^2 |> real
        norm(Lψ-E*ψ)/norm(ψ)
    end

    # break out real and imaginary parts, constraint is unit sphere
    function reconstruct(xy)
        n = length(xy) ÷ 2
        ψ = xy[1:n] + 1im*xy[n+1:end]
        m = sqrt(n) |> Int
        ψ = reshape(ψ,m,m)
    end

    # norm taken from SOR solution
    cost(xy) = E(20.31*reconstruct(xy))

    result = optimize(cost, [real.(init[:]); imag.(init[:])], NelderMead(manifold=Sphere()))
    ψ₂ = 20.31*reconstruct(result.minimizer)

    # 3000 steps of successive over-relaxation gets a residual of 1e-3.
    # The default settings in Optim should beat that.

    @test_broken rdl(ψ₂) < 1e-3

end


using Dataframes, StatsBase
aapl = readtable("AAPL-Short,csv");

naapl = size(aapl)[1]
m1 = int(mean((aapl[:Volume]))); # => 6306547

using Match
wts = zeros(naapl);
for i in 1:naapl
    dt = aapl[:Date][i]
    wts[i] = @match dt begin
         r"^2000" => 1.0
         r"^2001" => 2.0
         r"^2002" => 4.0
         _        => 0.0
    end
end;

wv = WeightVec(wts);
m2 = int(mean(aapl[:Volume], wv))
mean(aapl[:Close])
mean(aapl[:Close], wv)
trimmean(aapl[:Close], 0.1)

std(aapl[:Close])
skewness(aapl[:Close])
kurtosis(aapl[:Close])


# L0 pseudo-norm (times a constant)

export NormL0

"""
**``L_0`` pseudo-norm**

    NormL0(λ=1)

Returns the function
```math
f(x) = λ\\cdot\\mathrm{nnz}(x)
```
for a nonnegative parameter `λ`.
"""
struct NormL0{R <: Real} <: ProximableFunction
    lambda::R
    function NormL0{R}(lambda::R) where {R <: Real}
        if lambda < 0
            error("parameter λ must be nonnegative")
        else
            new(lambda)
        end
    end
end

NormL0(lambda::R=1) where {R <: Real} = NormL0{R}(lambda)

function (f::NormL0)(x::AbstractArray{T}) where {R, T <: RealOrComplex{R}}
    return f.lambda * R(count(v -> v != 0, x))
end

function prox!(y::AbstractArray{T}, f::NormL0, x::AbstractArray{T}, gamma::Real=1) where {R, T <: RealOrComplex{R}}
    countnzy = R(0)
    gl = gamma * f.lambda
    for i in eachindex(x)
        over = abs(x[i]) > sqrt(2 * gl)
        y[i] = over * x[i]
        countnzy += over
    end
    return f.lambda * countnzy
end

fun_name(f::NormL0) = "weighted L0 pseudo-norm"
fun_dom(f::NormL0) = "AbstractArray{Real}, AbstractArray{Complex}"
fun_expr(f::NormL0) = "x ↦ λ countnz(x)"
fun_params(f::NormL0) = "λ = $(f.lambda)"

function prox_naive(f::NormL0, x::AbstractArray{T}, gamma::Real=1) where {R, T <: RealOrComplex{R}}
    over = abs.(x) .> sqrt(2 * gamma * f.lambda)
    y = x.*over
    return y, f.lambda * R(count(v -> v != 0, y))
end


#################################################################
#
#    1st order derivation testing (loops)
#
#################################################################

### vector setting
    ex = quote
        a = zeros(2)
        for i in 1:2
            a[i] = x
        end
        sum(a)
    end
    compare(ex, 1.)

    b = [1,2,4,5]
    ex = quote
        a=zeros(1+4)
        for i in 1:4
            t = 4+3+2
            a[i] = b[i]+t-x
        end
        sum(a)
    end
    compare(ex, 1.)

    ex = quote
        a=zeros(1+3)
        for i in 1:4
            t = 4+3+2
            a[i] = b[i]*x+t
        end
        sum(a)
    end
    compare(ex, 1.)

### vector accumulation
    ex = quote
        a = zeros(2)
        for i in 1:2
            a[i] += x
        end
        sum(a)
    end
    compare(ex, 1.)

    ex = quote
        a=zeros(1+3)
        for i in 1:4
            t = 4+3+2
            a[i] += b[i]*x+t
        end
        sum(a)
    end
    compare(ex, 1.)

### same var setting (vector)
    ex = quote
        a = zeros(2)
        for i in 1:2
            a[1] = x
        end
        sum(a)
    end
    compare(ex, 2.)

    m.@deriv_rule %(x,y)      x     0
    m.@deriv_rule %(x,y)      y     0
    ex = quote
        a = zeros(2)
        for i in 1:4
            a[1 + i % 2] = x
        end
        sum(a)
    end
    compare(ex, 1.)

### same var accumulator (vector)
    ex = quote
        a = zeros(2)
        for i in 1:2
            a[1] += x
        end
        sum(a)
    end
    compare(ex, 2.)

### same var setting (scalar)
    ex = quote
        a = 0.
        for i in 1:4
            a = 4*x
        end
        a
    end
    compare(ex, 3.)

    ex = quote
        a = 0.
        for i in 1:4
            a = x
        end
        a
    end
    compare(ex, 3.)

    ex = quote
        a = 0.
        for i in 1:length(x)
            a = x[i]
        end
        a
    end
    compare(ex, [3., 2.])
    compare(ex, [1.])
    compare(ex, ones(10))

### same var accumulator (scalar)
    ex = quote
        a = 0.
        for i in 1:4
            a += x
        end
        a
    end
    compare(ex, 2.)

    ex = quote
        a = 0.
        for i in 1:4
            a += 2+x
        end
        a
    end
    compare(ex, 2.)

    ex = quote
        a = 0.
        for i in 1:3
            a += x^i
        end
        a
    end
    compare(ex, 2.)

    ex = quote
        a = 0.
        for i in 1:length(x)
            a += x[i]
        end
        a
    end
    compare(ex, [3., 2.])
    compare(ex, [1.])
    compare(ex, ones(10))

    ex = quote
        a = 0.
        for i in 1:length(x)
            a += x[1]^i
        end
        a+1
    end
    compare(ex, [3., 2.])
    compare(ex, [1.])
    compare(ex, ones(10))

### nested loops
    ex = quote
        a=0
        for i in 1:10
            for j in 1:10
                a += x
            end
        end
        a
    end
    compare(ex, 0.1)

    ex = quote
        a=0
        for i in 1:10
            for j in 1:10
                a += log(x) * sin(j)
            end
        end
        a
    end
    compare(ex, 0.1)

    ex = quote
        a=0
        for i in 1:10
            for j in 1:10
                a += (j < 4) * log(x) * sin(j)
            end
        end
        a
    end
    compare(ex, 0.1)


#! /usr/bin/julia

# Rosetta Code, Fibonacci n-step number sequences

type NFib{T<:Integer}
    n::T
    klim::T
    seeder::Function
end

type FState
    a::Array{BigInt,1}
    adex::Integer
    k::Integer
end

function Base.start{T<:Integer}(nf::NFib{T})
    a = nf.seeder(nf.n)
    adex = 1
    k = 1
    return FState(a, adex, k)
end

function Base.done{T<:Integer}(nf::NFib{T}, fs::FState)
    fs.k > nf.klim
end

function Base.next{T<:Integer}(nf::NFib{T}, fs::FState)
    f = sum(fs.a)
    fs.a[fs.adex] = f
    fs.adex = rem1(fs.adex+1, nf.n)
    fs.k += 1
    return (f, fs)
end

function fib_seeder{T<:Integer}(n::T)
    a = zeros(BigInt, n)
    a[1] = one(BigInt)
    return a
end

function fib{T<:Integer}(n::T, k::T)
    NFib(n, k, fib_seeder)
end

function luc_rc_seeder{T<:Integer}(n::T)
    a = zeros(BigInt, n)
    a[1] = 3
    a[2] = -1
    return a
end

function luc_rc{T<:Integer}(n::T, k::T)
    NFib(n, k, luc_rc_seeder)
end

function luc_seeder{T<:Integer}(n::T)
    a = -ones(BigInt, n)
    a[end] = big(n)
    return a
end

function luc{T<:Integer}(n::T, k::T)
    NFib(n, k, luc_seeder)
end

lo = 2
hi = 10
klim = 16

print("n-step Fibonacci for n = (", lo, ",", hi)
println(") up to k = ", klim, ":")
for i in 2:10
    print(@sprintf("%5d => ", i))
    for j in fib(i, klim)
        print(j, " ")
    end
    println()
end

println()
print("n-step Rosetta Code Lucas for n = (", lo, ",", hi)
println(") up to k = ", klim, ":")
for i in 2:10
    print(@sprintf("%5d => ", i))
    for j in luc_rc(i, klim)
        print(j, " ")
    end
    println()
end

println()
print("n-step MathWorld Lucas for n = (", lo, ",", hi)
println(") up to k = ", klim, ":")
for i in 2:10
    print(@sprintf("%5d => ", i))
    for j in luc(i, klim)
        print(j, " ")
    end
    println()
end


using NumericalMethodsforEngineers, Test

#=
@sym begin
  LagrangePolynomial(xi_, yi_) := Module([N, sum, num, den],
    begin
      N = Length(xi)
      sum = 0
      For( i=1, i <= N, Increment(i),
        begin
          num = 1
          den = 1
          For( j=1, j <= N, Increment(j),
            begin
              If(j != i, num *= (x-xi[j]))
              If(j != i, den *= (xi[i]-xi[j]))
            end
          )
          sum += yi[i] * num/den
        end
      ),
      Return(Simplify(sum))
    end
  )
  xi = [0, 1//2, 1]
  yi = [0, a, 1]
  Y(x_) := LagrangePolynomial(xi, yi)
  Println(Y(x))
end
=#

@sym begin
  ClearAll(xi, yi, Y, R, sol, ytilde)
  xi = [0, 1//2, 1]
  yi = [0, a, 1]
  Y(x_) := LagrangePolynomial(xi, yi)
  #
  # Can be formulated as ytile(x) = F(x) + C1(a) * Ψ(x)
  #
  # F(x_) := 2*x^2 - x
  # C(a_) := -4a
  # Ψ(x_) := x^2 - x
  #
  #Y(x_) := F(x) + C(a)*Ψ(x)
  #
  R(x_) = Simplify(D(Y(x), x, 2) - 3*x - 4*Y(x))
  R(x_) = Simplify(R(x) ./ (a => -C1/4))
  SetJ(r, ToString(Simplify(R(x))))
end

println("\n\nExample 7.13: y'' = 3x + 4y, y(0)=0, y(1)=1")
println("Residual for Weighted Residual Method using 1 point Lagragian Polynomial")
@sym Println("\nY(x) = ", Y(x), "\n")
@sym Println("R(x) = ", R(x), "\n")
println("( Example 7.13 gives: R = -4x^2*(2 + C1) + x*(1 + 4C1) + 2*(2 + C1) )", "\n")

println(r)

@eval f713(x, C1) = $(Meta.parse(r))

@test f713(1, 2) == 4.0 + 2.0*2 - 3 - 4*(-1.0 + 2.0*1 + 2*(-1 + 1))


# # Comparison with MathOptInterface on a Probability Simplex

# In this example, we project a random point onto a probability simplex with the Frank-Wolfe algorithm using
# either the specialized LMO defined in the package or a generic LP formulation using `MathOptInterface.jl` (MOI) and
# `GLPK` as underlying LP solver.
# It can be found as Example 4.4 [in the paper](https://arxiv.org/abs/2104.06675).

using FrankWolfe

using LinearAlgebra
using LaTeXStrings

using Plots

using JuMP
const MOI = JuMP.MOI

import GLPK

n = Int(1e3)
k = 10000

xpi = rand(n);
total = sum(xpi);
const xp = xpi ./ total;

f(x) = norm(x - xp)^2
function grad!(storage, x)
    @. storage = 2 * (x - xp)
    return nothing
end

lmo_radius = 2.5
lmo = FrankWolfe.FrankWolfe.ProbabilitySimplexOracle(lmo_radius)

x00 = FrankWolfe.compute_extreme_point(lmo, zeros(n))
gradient = collect(x00)

x_lmo, v, primal, dual_gap, trajectory_lmo = FrankWolfe.frank_wolfe(
    f,
    grad!,
    lmo,
    collect(copy(x00)),
    max_iteration=k,
    line_search=FrankWolfe.Shortstep(),
    L=2,
    print_iter=k / 10,
    emphasis=FrankWolfe.memory,
    verbose=false,
    trajectory=true,
);

# Create a MathOptInterface Optimizer and build the same linear constraints:

o = GLPK.Optimizer()
x = MOI.add_variables(o, n)

for xi in x
    MOI.add_constraint(o, xi, MOI.GreaterThan(0.0))
end

MOI.add_constraint(
    o,
    MOI.ScalarAffineFunction(MOI.ScalarAffineTerm.(1.0, x), 0.0),
    MOI.EqualTo(lmo_radius),
)

lmo_moi = FrankWolfe.MathOptLMO(o)

x, v, primal, dual_gap, trajectory_moi = FrankWolfe.frank_wolfe(
    f,
    grad!,
    lmo_moi,
    collect(copy(x00)),
    max_iteration=k,
    line_search=FrankWolfe.Shortstep(),
    L=2,
    print_iter=k / 10,
    emphasis=FrankWolfe.memory,
    verbose=false,
    trajectory=true,
);

# Alternatively, we can use one of the modelling interfaces based on `MOI` to formulate the LP.
# The following example builds the same set of constraints using `JuMP`:

m = JuMP.Model(GLPK.Optimizer)
@variable(m, y[1:n] ≥ 0)

@constraint(m, sum(y) == lmo_radius)

lmo_jump = FrankWolfe.MathOptLMO(m.moi_backend)

x, v, primal, dual_gap, trajectory_jump = FrankWolfe.frank_wolfe(
    f,
    grad!,
    lmo_jump,
    collect(copy(x00)),
    max_iteration=k,
    line_search=FrankWolfe.Shortstep(),
    L=2,
    print_iter=k / 10,
    emphasis=FrankWolfe.memory,
    verbose=false,
    trajectory=true,
);

x_lmo, v, primal, dual_gap, trajectory_lmo_blas = FrankWolfe.frank_wolfe(
    f,
    grad!,
    lmo,
    x00,
    max_iteration=k,
    line_search=FrankWolfe.Shortstep(),
    L=2,
    print_iter=k / 10,
    emphasis=FrankWolfe.blas,
    verbose=false,
    trajectory=true,
);

x, v, primal, dual_gap, trajectory_jump_blas = FrankWolfe.frank_wolfe(
    f,
    grad!,
    lmo_jump,
    x00,
    max_iteration=k,
    line_search=FrankWolfe.Shortstep(),
    L=2,
    print_iter=k / 10,
    emphasis=FrankWolfe.blas,
    verbose=false,
    trajectory=true,
);

iteration_list = [[x[1] + 1 for x in trajectory_lmo], [x[1] + 1 for x in trajectory_moi]]
time_list = [[x[5] for x in trajectory_lmo], [x[5] for x in trajectory_moi]]
primal_gap_list = [[x[2] for x in trajectory_lmo], [x[2] for x in trajectory_moi]]
dual_gap_list = [[x[4] for x in trajectory_lmo], [x[4] for x in trajectory_moi]]

label = [L"\textrm{Closed-form LMO}", L"\textrm{MOI LMO}"]

FrankWolfe.plot_results(
    [primal_gap_list, primal_gap_list, dual_gap_list, dual_gap_list],
    [iteration_list, time_list, iteration_list, time_list],
    label,
    ["", "", L"\textrm{Iteration}", L"\textrm{Time}"],
    [L"\textrm{Primal Gap}", "", L"\textrm{Dual Gap}", ""],
    xscalelog=[:log, :identity, :log, :identity],
    yscalelog=[:log, :log, :log, :log],
    legend_position=[:bottomleft, nothing, nothing, nothing]
)


# mathematical constants related to statistics

@irrational twoπ   6.2831853071795864769 big(π) * 2.0
@irrational fourπ  12.566370614359172954 big(π) * 4.0
@irrational halfπ  1.5707963267948966192 big(π) * 0.5
@irrational quartπ 0.7853981633974483096 big(π) * 0.25

@irrational invπ     0.31830988618379067154 inv(big(π))
@irrational twoinvπ  0.63661977236758134308 big(invπ) * 2.0
@irrational fourinvπ 1.27323954473516268615 big(invπ) * 4.0
@irrational inv2π    0.159154943091895335769 big(invπ) * 0.5
@irrational inv4π    0.079577471545947667884 big(invπ) * 0.25

@irrational sqrt2     1.4142135623730950488 sqrt(big(2.0))
@irrational sqrt3     1.7320508075688772935 sqrt(big(3.0))
@irrational sqrtπ     1.7724538509055160273 sqrt(big(π))
@irrational sqrt2π    2.5066282746310005024 sqrt(big(π) * 2.0)
@irrational sqrt4π    3.5449077018110320546 sqrt(big(π) * 4.0)
@irrational sqrthalfπ 1.2533141373155002512 sqrt(big(0.5)*π)

@irrational invsqrt2  0.7071067811865475244 inv(big(sqrt2))
@irrational invsqrt2π 0.3989422804014326779 inv(big(sqrt2π))


module GetSpectrum

using DSP
using FFTW

export getSpectrum
export getWaterfall
#greet() = print("Hello World!")

function getSpectrum(fs,sig;N=nothing)
	if isnothing(N)
		N = length(sig);
	end
    freqAx = collect(((0:N-1)./N .- 0.5)*fs);
	ss	   = @view sig[1:N];
	y	   = 10*log10.(abs2.(fftshift(fft(ss))));
	return (freqAx,y);
end
getSpectrum(sig) = getSpectrum(1,sig);

function getWaterfall(fe,sig;sizeFFT=1024)
    nbSeg   = length(sig) ÷ sizeFFT;
    ss      =  @views sig[1:nbSeg*sizeFFT];
    ss      = reshape(ss,sizeFFT,nbSeg)
    sMatrix = zeros(Float64,sizeFFT,nbSeg);
    for iN = 1 : 1 : nbSeg
        sMatrix[:,iN] = abs2.(fftshift(fft(ss[:,iN])));
    end
    fAx = collect(((0:1:sizeFFT-1)./sizeFFT .- 0.5) .* fe);
    tAx = (0:nbSeg-1) * (sizeFFT/fe);
    return tAx,fAx,sMatrix;
end
getWaterfall(sig;sizeFFT=1024) = getWaterfall(1,sig;sizeFFT=sizeFFT);
end # module


# 1D diffusion problem

# TODO: Add more complex tests.

# Packages and inclusions
using ModelingToolkit,DiffEqOperators,DiffEqBase,LinearAlgebra,Test

# Tests
@testset "Test 00: Dt(u(t,x)) ~ Dxx(u(t,x))" begin
    # Parameters, variables, and derivatives
    @parameters t x
    @variables u(..)
    @derivatives Dt'~t
    @derivatives Dxx''~x

    # 1D PDE and boundary conditions
    eq  = Dt(u(t,x)) ~ Dxx(u(t,x))
    bcs = [u(0,x) ~ -x*(x-1)*sin(x),
           u(t,0) ~ 0.0,
           u(t,1) ~ 0.0]

    # Space and time domains
    domains = [t ∈ IntervalDomain(0.0,1.0),
               x ∈ IntervalDomain(0.0,1.0)]

    # PDE system
    pdesys = PDESystem(eq,bcs,domains,[t,x],[u])

    # Method of lines discretization
    dx = 0.1
    order = 2
    discretization = MOLFiniteDifference(dx,order)

    # Convert the PDE problem into an ODE problem
    prob = discretize(pdesys,discretization)

    # Solve ODE problem
    using OrdinaryDiffEq
    sol = solve(prob,Tsit5(),saveat=0.1)

    #Plot and save results
    # using Plots
    # plot(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,1]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,2]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,3]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,4]))
    # savefig("MOL_1D_Linear_Diffusion_Test00.png")

    # Test
    n = size(sol,1)
    t_f = size(sol,3)

    @test sol[:,1,t_f] ≈ zeros(n) atol = 0.001;
end

@testset "Test 01: Dt(u(t,x)) ~ D*Dxx(u(t,x))" begin
    # Parameters, variables, and derivatives
    @parameters t x D
    @variables u(..)
    @derivatives Dt'~t
    @derivatives Dxx''~x

    D = 1.1

    # 1D PDE and boundary conditions
    eq  = Dt(u(t,x)) ~ D*Dxx(u(t,x))
    bcs = [u(0,x) ~ -x*(x-1)*sin(x),
           u(t,0) ~ 0.0,
           u(t,1) ~ 0.0]

    # Space and time domains
    domains = [t ∈ IntervalDomain(0.0,1.0),
               x ∈ IntervalDomain(0.0,1.0)]

    # PDE system
    pdesys = PDESystem(eq,bcs,domains,[t,x,D],[u])

    # Method of lines discretization
    dx = 0.1
    order = 2
    discretization = MOLFiniteDifference(dx,order)

    # Convert the PDE problem into an ODE problem
    prob = discretize(pdesys,discretization)

    # Solve ODE problem
    using OrdinaryDiffEq
    sol = solve(prob,Tsit5(),saveat=0.1)

    # Plot and save results
    # using Plots
    # plot(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,1]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,2]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,3]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,4]))
    # savefig("MOL_1D_Linear_Diffusion_Test01.png")

    # Test
    n = size(sol,1)
    t_f = size(sol,3)
    @test sol[:,1,t_f] ≈ zeros(n) atol = 0.001;
end

@testset "Test 02: Dt(u(t,x)) ~ Dx(D(t,x))*Dx(u(t,x))+D(t,x)*Dxx(u(t,x))" begin
    # Parameters, variables, and derivatives
    @parameters t x
    @variables u(..) D(..)
    @derivatives Dt'~t
    @derivatives Dx'~x
    @derivatives Dxx''~x

    # 1D PDE and boundary conditions

    eq  = [ Dt(u(t,x)) ~ Dx(D(t,x))*Dx(u(t,x))+D(t,x)*Dxx(u(t,x)),
            D(t,x) ~ 0.999 + 0.001 * t * x  ]

    bcs = [u(0,x) ~ -x*(x-1)*sin(x),
           u(t,0) ~ 0.0,
           u(t,1) ~ 0.0,
           D(0,x) ~ 0.999,
           D(t,0) ~ 0.999,
           D(t,1) ~ 0.999 + 0.001 * t ]

    # Space and time domains
    domains = [t ∈ IntervalDomain(0.0,1.0),
               x ∈ IntervalDomain(0.0,1.0)]

    # PDE system
    pdesys = PDESystem(eq,bcs,domains,[t,x],[u,D])

    # Method of lines discretization
    dx = 0.1
    order = 2
    discretization = MOLFiniteDifference(dx,order)

    # Convert the PDE problem into an ODE problem
    prob = discretize(pdesys,discretization)

    # Solve ODE problem
    using OrdinaryDiffEq
    sol = solve(prob,Tsit5(),saveat=0.1)

    # Plot and save results
    # using Plots
    # plot(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,1]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,2]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,3]))
    # plot!(prob.space[2],Array(prob.extrapolation[1]*sol[:,1,4]))
    # savefig("MOL_1D_Linear_Diffusion_Test02.png")

    # Test
    n = size(sol,1)
    t_f = size(sol,3)
    @test sol[:,1,t_f] ≈ zeros(n) atol = 0.1;
end


#############################################################################
#############################################################################
#
# This file implements polynomial division
#
#############################################################################
#############################################################################

"""  Modular algorithm.
f divide by g

f = q*g + r

p is a prime
"""

function divide(num::Polynomial, den::Polynomial)
    function division_function(p::Int)
        f, g = mod(num,p), mod(den,p)
        degree(f) < degree(num) && return nothing
        iszero(g) && throw(DivideError())
        q = Polynomial()
        prev_degree = degree(f)
        while degree(f) ≥ degree(g)
            h = Polynomial( (leading(f) ÷ leading(g))(p) )  #syzergy
            f = mod((f - h*g), p)
            q = mod((q + h), p)
            prev_degree == degree(f) && break
            prev_degree = degree(f)
        end

        @assert iszero( mod((num  - (q*g + f)),p))
        return q, f
    end
    return division_function
end

function divide(num::PolynomialModP, den::PolynomialModP)
    p = num.mod
    @assert num.mod == den.mod
    f, g = num, den
    if degree(g) > degree(f)
        output =  [zero(PolynomialModP,p) , f]
        return output
    end

    iszero(g) && throw(DivideError())
    iszero(f) && return zero(PolynomialModP,p), zero(PolynomialModP,p)

    q = PolynomialModP(f.mod)
    prev_degree = degree(f)
    while degree(f) ≥ degree(g)
        h = (leading(f) ÷ leading(g))(p) #syzergy
        hg = - h*g
        f = f+hg
        q = q+h
        prev_degree == degree(f) && break
        prev_degree = degree(f)
    end

    @assert iszero(num - (q*g + f))

    return q,f
end

"""
The quotient from polynomial division. Returns a function of an integer.
"""
÷(num::Polynomial, den::Polynomial)  = (p::Int) -> first(divide(num,den)(p))
÷(num::PolynomialModP, den::PolynomialModP) = first(divide(num,den))

"""
The remainder from polynomial division. Returns a function of an integer.
"""
rem(num::Polynomial, den::Polynomial)  = (p::Int) -> last(divide(num,den)(p))
rem(num::PolynomialModP, den::PolynomialModP)  = last(divide(num,den))


using Symata

@sym begin
  ClearAll(f, y0, a, l, x)
  y0(x_) := (e*x)*(l^3 - 2*l*x^2 + x^3)

  # Stodola-Vianello method, e.g. see Den Hartog, Advanced Strength of Materials, pg 268
  f(maxiters_, y0_) := Module([i, yder, res=ConstantArray(0.0,[maxiters,6])],
    (
      # Initialize the first row of the res matrix
      res[1,2] = y0(x),
      For(i=1, i<=4, Increment(i),
        [
          yder(x_) = Simplify(D(y0, x, i))
          res[1, 2+i] = yder(x)
        ]
      ),
      Println("N[res[1, 1]]: ", res[1,1]),
      # Start the loop to improve accuracy stored in subsequent rows,
      # res[i, 1] contains the Euler ratio.
      For(i=2, i<=maxiters, Increment(i),
        begin
          res[i, 6] = Simplify(w/(EI)*((x-l)*res[i-1,4] + res[i-1,3]))
          # Above the basic Euler equation for a pinned-pinned flagpole
          ClearAll(K1, K11, tmpy5, tmpy4, tmp)
          tmpy5(x_) = Simplify(Integrate(res[i,6], x) + K1)
          tmpy4(x_) = Simplify(Integrate(tmpy5(x), x))
          # Use first pinned constraint at x=l to resolve K1
          K11 = Solve(-tmpy4(l), K1)[1]
          res[i,5] = Simplify( tmpy5(x) ./ K11 )
          res[i,4] = Simplify( tmpy4(x) ./ K11 )
          ClearAll(K2, K22, tmpy3, tmpy2, tmp)
          tmpy3(x_) = Simplify(Integrate(res[i,4], x) + K2)
          tmpy2(x_) = Simplify(Integrate(tmpy3(x), x))
          # Use second pinned constraint at x=l to resolve K2
          K22 = Solve(-tmpy2(l), K2)[1]
          res[i,3] = Simplify( tmpy3(x) ./ K22 )
          res[i,2] = Simplify( tmpy2(x) ./ K22 )
          tmp = Simplify( (((l^3)*w)/EI)*(res[i-1,3]/res[i, 3]) )
          res[i,1] = N(Simplify(ReplaceAll(tmp, x => l)))
          Println("N[res[$i, 1]]: ", N(res[i,1], 4))
        end
      ),
      Println(),
      Return(res)
    )
  )

  r = f(8, y0(x))
  Println(N(Transpose(r)[1], 4))
end


module FirstOrder

using LinearAlgebra

import ..LocalDescent:
search,
LineSearch, StrongBacktracking

struct Termination
    max_iter
    ϵ_abs
    ϵ_rel
    ϵ_grad
    Termination(;
        max_iter = 10_000,
        ϵ_abs = eps(),
        ϵ_rel = eps(),
        ϵ_grad = eps()) =
        new(max_iter, ϵ_abs, ϵ_rel, ϵ_grad)
    Termination(ϵ;
        max_iter = 10_000) =
        new(max_iter, ϵ, ϵ, ϵ)
end

mutable struct TerminationConditions
    abs
    rel
    grad
    TerminationConditions(term::Termination) =
    new((fx, fx_next)->fx - fx_next < term.ϵ_abs,
        (fx, fx_next)->fx - fx_next < term.ϵ_abs * abs(fx),
        ∇fx_next->norm(∇fx_next) < term.ϵ_grad)
end

function descent_until(term::Termination, descent_method, f, ∇f, X, trace)
    term_cond = TerminationConditions(term)
    for _ = 1:term.max_iter
        x = X[:, end]
        x_next = descent_method(f, ∇f, x)
        if trace
            X = hcat(X, x_next)
        else
            X = x_next
        end
        fx, fx_next, ∇fx_next = f(x), f(x_next), ∇f(x_next)
        if term_cond.abs(fx, fx_next) ||
            term_cond.rel(fx, fx_next) ||
            term_cond.grad(∇fx_next)
            return X
        end
    end
    @warn "descent_until: max number of iterations reached"
    X
end

abstract type FirstOrderMethods end

# Maximum Gradient Descent

struct MaximumGradientDescent <: FirstOrderMethods end

function search(::MaximumGradientDescent, f, ∇f, x_0; term = Termination(), trace = false)
    descent_until(term, descent_step_maxgd, f, ∇f, x_0, trace)
end

function descent_step_maxgd(f, ∇f, x)
    """
    Maximum Gradient Descent

    A step of maximum gradient descent produces the value of x(k+1)
    where f(x) in the direction of maximum descent is at its local
    minimum.
    """
    g = ∇f(x)
    d = direction_maxgd(g)
    search(LineSearch(), f, x, d)
end

function direction_maxgd(g)
    -g / norm(g)
end

# Gradient Descent

struct GradientDescent <: FirstOrderMethods
    α
    GradientDescent(; α = 0.001) = new(α)
end

function search(params::GradientDescent, f, ∇f, x_0; term = Termination(), trace = false)
    descent_step = (f, ∇f, x)->descent_step_gd(params.α, f, ∇f, x)
    descent_until(term, descent_step, f, ∇f, x_0, trace)
end

function descent_step_gd(α, f, ∇f, x)
    """
    Gradient Descent
    Algorithm 5.1

    A step of a gradient descent algorithm with fixed learning rate α.
    """
    g = ∇f(x)
    x - α * g
end

# Conjugate Gradient Descent

struct ConjugateGradientDescent <: FirstOrderMethods end

mutable struct CGDProblemState
    g
    d
    CGDProblemState(dim) = new(ones(dim), zeros(dim))
end

function search(params::ConjugateGradientDescent, f, ∇f, x_0; term = Termination(), trace = false)
    state = CGDProblemState(size(x_0))
    descent_step = (f, ∇f, x)->descent_step_cgd!(state, f, ∇f, x)
    descent_until(term, descent_step, f, ∇f, x_0, trace)
end

function descent_step_cgd!(state, f, ∇f, x)
    """
    Conjugate Gradient Descent
    Algorithm 5.2

    https://en.wikipedia.org/wiki/Nonlinear_conjugate_gradient_method
    """
    g = ∇f(x)
    d = direction_cgd(g, state.g, state.d)
    state.g = g
    state.d = d
    search(StrongBacktracking(), f, ∇f, x, d)
end

function direction_cgd(g, gm1, dm1)
    """
    Polak-Ribiere update
    Notation:
        g is for g(k)
        dm1 is for d(k-1)
        gm1 indicates g(k-1)
    """
    β_PR = (g' * (g - gm1)) / (gm1'gm1)
    β = max(β_PR, 0)
    -g + β * dm1
end

end # module


export read_resp_from_sacpz

# Set the response to acceleration
const flag = 2

function read_resp_from_sacpz(input_sacpz::String, sampling_rate::Float64, N::Int)
    read_zeros = false
    read_poles = false
    zeros = Array{Complex{Float64}}(undef, 1)
    poles = Array{Complex{Float64}}(undef, 1)
    zeros_cnt, poles_cnt = 1, 1
    constant = .0

    lines = readlines(input_sacpz)
    for line in lines
        contents = split(line, " ")
        if contents[1] == "CONSTANT"
            constant = parse(Float64, contents[2])
        elseif contents[1] == "ZEROS"
            zeros_size = parse(Int64, contents[2])
            zeros = Base.zeros(Complex{Float64}, zeros_size)
            read_zeros = true
            read_poles = false
        elseif contents[1] == "POLES"
            poles_size = parse(Int64, contents[2])
            poles = Base.zeros(Complex{Float64}, poles_size)
            read_zeros = false
            read_poles = true
        else
            # Seems that some SACPZ file will let Julia pase as follows:
            # "<real>", "", "<imag>"
            num = parse(Float64, contents[1]) + parse(Float64, contents[3]) * im
            if read_zeros == true
                zeros[zeros_cnt] = num
                zeros_cnt += 1
            elseif read_poles == true
                poles[poles_cnt] = num
                poles_cnt += 1
            end
        end
    end

    return create_resp(poles, zeros, constant, sampling_rate, N)
end

function create_resp(poles::Vector{Complex{Float64}},
                     zeros::Vector{Complex{Float64}},
                     constant::Float64,
                     sampling_rate::Float64,
                     N::Int)
    freqs = Vector{Float64}(undef, N)
    delta = 1. / sampling_rate
    total_duration = delta * N
    for i = 1:(div(N, 2) + 1)
        freqs[i] = i / total_duration
    end
    for i = (div(N, 2) + 1):N
        freqs[i] = -(N - i) / total_duration
    end

    instrument_response = Vector{Complex{Float64}}(undef, N)
    for i = 1:N
        numerator = 1. + 0. * im
        denominator = 1. + 0. * im
        omega = 2 * π * freqs[i]
        i_omega = 0. + omega * im

        for j = 1:(size(zeros, 1) - flag)
            numerator *= (i_omega - zeros[j])
        end
        for j = 1:size(poles, 1)
            denominator *= (i_omega - poles[j])
        end

        instrument_response[i] = constant * numerator / denominator
    end

    return instrument_response
end


include("kruskal.jl")

"""
    count_spanning_trees(g::AGraph)

Returns the number of spanning trees of `g`, computed through
[Kirchhoff's theorem](https://en.wikipedia.org/wiki/Kirchhoff%27s_theorem).
The return type is a float, since the number can be very large.
"""
function count_spanning_trees(g::AGraph)
    @assert is_connected(g) "The graph has to be connected"
    nv(g) <= 1 && return 1.
    Δ = laplacian_matrix(g)[2:nv(g),2:nv(g)]
    return round(det(Δ))
end


using PlainLinearAlgebra
import LinearAlgebra
using Test

@testset "diagonal" begin
    for d in (0, 1, 2, 3, 10)
        m = rand(d, d)
        mdiagonal = diagonal(m)
        mdiag = LinearAlgebra.diag(m)
        @test mdiagonal == mdiag
        @test sum(mdiagonal) ≈ sum(mdiag)
        @test Base.mightalias(mdiagonal, m)
        d0 = d - 1
        for k in -d0:d0
            @test diagonal(m, k) == LinearAlgebra.diag(m, k)
        end
    end
end

@testset "properties" begin
    for d in (1, 2, 3, 10)
        for T in (Int, Float64, ComplexF64)
            m = rand(T, d, d)
            dm = diagonal(m)
            @test eltype(dm) == T
            @test Base.elsize(dm) == sizeof(T)
            @test size(dm) == (d,)
            @test size(dm, 1) == d
            @test size(dm, 2) == 1
        end
    end
end

@testset "diagonal setindex!" begin
    for d in (1, 2, 3, 10)
        m = rand(d, d)
        md = diagonal(m)
        md[1] = md[1] * 2
        @test md[1] == m[1,1]
    end
end

@testset "identity matrix" begin
    for d in (0, 1, 2, 3, 10)
        @test idmat(d) == idmat(Float64, d)
        for T in (Int, Rational{Int})
            @test isa(idmat(T, d), Matrix{T})
        end
    end
end


using AlphaShapes
include("../src/utils.jl")
using Plots
using Main.AlphaShapes
using LinearAlgebra
using Random
Random.seed!(31415926535897)

N = 1000
r1 = 0.75
r2 = 2.0
X = randn(N,2)
ind = [((norm(X[i,:])>r1) + (norm(X[i,:])<r2)) == 2 for i in 1:size(X,1)]
X = X[ind.==true,:]
# normally distributed points with a whole in the middle
p1 = scatter(X[:,1],X[:,2],label="",aspect_ratio=:equal)

T = AlphaShapes.GetDelaunayTriangulation(X);
p1 = scatter(X[:,1],X[:,2],label="",aspect_ratio=:equal)
[DrawTri!(T[i,:,:]) for i in 1:size(T,1)]
plot!(title="Delaunay Triangulation")

A = AlphaShape(X);
p2 = scatter(X[:,1],X[:,2],label="",aspect_ratio=:equal)
[DrawTri!(A[i,:,:]) for i in 1:size(A,1)]
plot!(title="Optimum Alpha Shape Triangulation")

p = plot(p1,p2,size=(900,300))
savefig(p,"Example.png")


"""
    gpw(ζ, φ; k=1)

Closure that defines a generalized plane wave in polar coordinates.

The closure captures the values of the evanescence parameter ``ζ``, the
angle ``φ`` and the wavenumber ``k``.
It does not include any normalization.
The function can then be evaluated at any ``(r,θ)`` point.

The generalized plane wave in polar coordinates is defined by
```math
ϕ := (r,θ) ↦ e^{ı k \\mathbf{d} ⋅ \\mathbf{x}},
```
where
```math
\\mathbf{d} = (\\cos[φ+ıζ], \\sin[φ+ıζ]),
\\qquad\\text{and}\\qquad
\\mathbf{x} = (r \\cos θ, r\\sin θ).
```
"""
function gpw(ζ,φ;k=1)
    d = [cos(φ+im*ζ), sin(φ+im*ζ)] # Direction
    return (r,θ) -> exp(im * k * [r*cos(θ), r*sin(θ)] ⋅ d)
end

"""
    approximation_set(Y, W; k=1)

Generate a set of generalized plane waves.

The parameters `Y` and `W` are respectively the sets of complex angles
``(ζ_j,φ_j)`` and associated weights ``ω_j`` corresponding to the generalized
plane waves, see [`gpw`](@ref)
```math
ϕ_j := (r,θ) ↦ e^{ı k \\mathbf{d}_j ⋅ \\mathbf{x}},
```
where
```math
\\mathbf{d}_j = (\\cos[φ_j+ıζ_j], \\sin[φ_j+ıζ_j]),
\\qquad\\text{and}\\qquad
\\mathbf{x} = (r \\cos θ, r\\sin θ).
```
Approximations are constructed in the form of
```math
(r, θ) ↦ \\sum_j ξ_j ω_j ϕ_j(r, θ),
```
where ``(ξ_j)_j`` is the set of unknown coefficients.

The value of the wavenumber `k` defaults to ``1`` and can be provided as an
optional parameter.
"""
function approximation_set(Y, W; k=1)
    @assert length(Y) == length(W)
    return [(r, θ) -> w * gpw(y...;k=k)(r, θ) for (y, w) in zip(Y, W)]
end

"""
    approximation_set(N; k=1)

Generate a set of ``N`` propagative plane waves with equispaced angles.

The equispaced angles are defined by
```math
θ_n := 2π s / R, \\qquad n = 0,…,N-1.
```

The value of the wavenumber `k` defaults to ``1`` and can be provided as an
optional parameter.
"""
function approximation_set(N; k=1)
    Y = [(0, 2π * n / N) for n in 0:N-1]
    W = ones(Int64, N) ./ sqrt(N)
    return approximation_set(Y, W; k=k)
end

"""
    approximation_set(N, qs, smpl_type; k=1)
    approximation_set(N, Q::Integer, smpl_type; k=1)

Generate a set of ``N`` evanescent plane waves according to some sampling type.

The second parameter can be an integer ``Q`` which controls the truncation
parameter of the [`TruncKernel`](@ref), or a set of integers `qs` to specify
only some specific terms in the expansion.
The parameter `smpl_type` should be a function name: see
[`uniform_sampling`](@ref), [`sobol_sampling`](@ref) and
[`random_sampling`](@ref).

The value of the wavenumber `k` defaults to ``1`` and can be provided as an
optional parameter.
"""
function approximation_set(N, qs, smpl_type; k=1)
    K = TruncKernel(Tuple(qs), logwz(;k=k));
    smpl = Sampler(K)
    Y, W = smpl_type(smpl, N)
    W ./= sqrt(N)
    return approximation_set(Y, W; k=k)
end
function approximation_set(N, Q::Int64, smpl_type; k=1)
    return approximation_set(N, -Q:Q, smpl_type; k=k)
end


# code for Video 4.3:
fname = "iris.csv"
using DataFrames
data = readtable(fname, separator = ',')

### scatter plots:
using Gadfly
set_default_plot_size(15cm, 15cm/golden)
p = plot(data, x="sepal_width", y="sepal_length", Geom.point,
         Guide.xlabel("Sepal Width"),
         Guide.ylabel("Sepal Length"),
         Guide.title("Scatterplot of Sepal Length versus Width") )
draw(PDF("Length_versus_Width.pdf", 14cm, 10cm), p)

cor(data[:sepal_width], data[:sepal_length]) # -0.10936924995064931

# add colour dimension:
p = plot(data, x="sepal_width", y="sepal_length", color="species",
         Guide.xlabel("Sepal Width"),
         Guide.ylabel("Sepal Length"),
         Guide.title("Scatterplot of Sepal Length versus Width") )
draw(PDF("Length_versus_Width_Color.pdf", 14cm, 10cm), p)

cor(data[1:50, :sepal_width], data[1:50, :sepal_length]) # 0.7467803732639265
cor(data[51:100, :sepal_width], data[51:100, :sepal_length]) # 0.5259107172828241
cor(data[101:150, :sepal_width], data[101:150, :sepal_length]) # 0.4572278163941129

### linear regression:
x = convert(Array, data[1:50, :sepal_width])
y = convert(Array, data[1:50, :sepal_length])
a, b = linreg(x, y)
# 2-element Array{Float64,1}:
#  2.64466
#  0.690854
# sepal_length = 2.64466 + 0.690854 * sepal_width

# adding a linear regression fit to the plot:
using Winston
pl = plot(data[1:50, :sepal_width], data[1:50, :sepal_length], "g^") # use the symbol for green triangles
s = Slope(0.690854, (0, 2.64466), colorant="blue", linekind="dashed") # this is the linear fit
add(pl, s)
display(pl) # not necessary in the REPL
savefig(pl, "Winston_Linreg.png")

using PyPlot
x = convert(Array, data[1:50, :petal_width])
y = convert(Array, data[1:50, :petal_length])
a, b = linreg(x, y)
# 2-element Array{Float64,1}:
#  1.34304
#  0.495739
cor(x, y) # correlation = 0.30630821115803575
plot(x, y, "o")
plot(x, [a+b*i for i in x])

### histogram:
Gadfly.plot(data, x="petal_width", Geom.histogram(bincount=30))

### boxplot with Gadfly:
Gadfly.plot(data, x = "species", y = "sepal_length", Geom.boxplot)


# # Geometry

# The `WaveProp.Geometry` module handles everything related to the geometrical
# descriptions of curves/surfaces. Its main interface is exposed through
# `AbstractElement`, which represents an `M`-dimensional geometical object
# embedded in ℝᴺ. In abstract terms, an *element* τ is a subset of ℝᴺ described
# by
# * a reference element ``\tau \subset \mathbb{R}^N``
# * a *push-forward map* χ : τ̂ → τ ⊂ ℝᴺ.

# Concrete implementations of `AbstractElement`s are responsible for
# implementing a few crucial methods used to derive almost all functionality of
# an `el::AbstractElement` used throughout this package. In the [next section](#interface) we
# describe this interface.

# ## Interface

# The interface is composed of the following methods:
# - `domain(el)` : return the reference element `R::AbstractReferenceElement`
# - `el(u)` : evaluate the parametrization of the element at (parametric).
#   coordinate `u ∈ domain(el)`.
# - `jacobian(u)` : evaluate the jacobian of the parametrization at (parametric)
#   coordinate `u ∈ domain(el)`.

# ## Lagrangian elements

# An concrete implementation of `AbstractElement` is the parametric type
# `LagrangeElement{D,Np,N,T}`. The type parameters represent:
# - `D` : the domain of the element. Thus, `domain(el)` return an instance of
#   the singleton type `R`.
# - `Np` : the number of points/DOF which describe the element. For example, for
#   a flat triangle, `Np=3`, while for a flat quadrilateral element `Np=4`.
# - `N` : the dimension of the ambient/embedding space.
# - `T` : the data type used in the presentation of `el`. Defaults to `Float64`.


# Fourier demo with gtk-server
# #
# Note: requires package Gaston for plotting, or adjust the code to work with your
# plotting front-end.

using GtkServer, Gaston, FFTW

# plot
function doplots(t, A, f₀, ϕ, fs)
    s = A.*sin.(2π*f₀*t .+ ϕ)
    S = fftshift(fft(s))./length(s)
    F = fftshift(fftfreq(length(s), fs))

    p1 = plot(t, s, handle = 1,
              Axes(title = "'Sinusoidal in time domain'", xlabel = "'Time (s)'",
                   ylabel = "'Amplitude'", yrange = (-5.5, 5.5), ytics = -6:2:6))
    p2 = plot(F, abs.(S), handle = 2,
              Axes(title = "'Magnitude spectrum'", xlabel = "'Frequency (Hz)'",
                   yrange = (0, 3)))
    p3 = plot(F, angle.(S), handle = 3,
              Axes(title = "'Phase spectrum'", xlabel = "'Frequency (Hz)'",
                   yrange = (-3, 3)))
    P = plot([p1, p2, p3], handle = 4)
    display(P)
end

function demo()
    # Set initial values for amplitude, frequency and phase, and display
    # the sine function.
    fs = 100
    t = 0.0:1.0/fs:1.0
    A = 1.0
    f₀ = 1.0
    ϕ = 0.0

    # Do initial plot with default values
    doplots(t, A, f₀, ϕ, fs)

    # Main program
    # This program interactively plots a sine function in time, and its
    # magnitude and phase spectra. The amplitude, frequency and phase of
    # sine function are selected through three sliders.
    gtk_init()
    # 'win' is a handle (ID number) to the main GUI window.
    gtk_send("gtk_init NULL NULL")
    win = gtk_send("gtk_window_new 0")
    # Set window's title, size and position.
    # Note that the message sent to gtk-server is always a string.
    gtk_send("gtk_window_set_title $win 'Fourier Series Demo'")
    gtk_send("gtk_window_set_default_size $win 300 200")
    gtk_send("gtk_window_set_position $win 1")
    # We'll set our widgets in a table, one widget per row-column combination.
    tbl = gtk_send("gtk_table_new 8 10 0")
    gtk_send("gtk_container_add $win $tbl")
    # Create three labels...
    lblA = gtk_send("gtk_label_new Amplitude:")
    lblF = gtk_send("gtk_label_new Frequency:")
    lblP = gtk_send("gtk_label_new Phase:")
    # and put them in specific position on our table.
    gtk_send("gtk_table_attach_defaults $tbl $lblA 1 3 1 2")
    gtk_send("gtk_table_attach_defaults $tbl $lblF 1 3 3 4")
    gtk_send("gtk_table_attach_defaults $tbl $lblP 1 3 5 6")
    # horizontal separator
    hsep = gtk_send("gtk_hseparator_new")
    gtk_send("gtk_table_attach_defaults $tbl $hsep 1 9 6 7")
    # Some Julia advertisement
    adv = gtk_send("gtk_label_new 'Created with Julia!'")
    gtk_send("gtk_table_attach_defaults $tbl $adv 1 6 7 8")
    # Create (and put on the table) the quit button.
    but = gtk_send("gtk_button_new_from_stock 'gtk-quit'")
    gtk_send("gtk_table_attach_defaults $tbl $but 7 9 7 8")
    # Create three sliders...
    sliA = gtk_send("gtk_hscale_new_with_range 1 5 0.5")
    sliF = gtk_send("gtk_hscale_new_with_range 1 40 1")
    sliP = gtk_send("gtk_hscale_new_with_range -1.6 1.6 0.1")
    # and put them on the table.
    gtk_send("gtk_table_attach_defaults $tbl $sliA 4 9 1 2")
    gtk_send("gtk_table_attach_defaults $tbl $sliF 4 9 3 4")
    gtk_send("gtk_table_attach_defaults $tbl $sliP 4 9 5 6")
    # Now we need to actually display all the widgets we've created.
    gtk_send("gtk_widget_show_all $win")

    # endless loop; we only break out of it when the quit button is clicked
    while true
        # Up to this point, we have sent commands to gtk-server in order
        # to set up the GUI and display it. However, once the GUI is
        # created, gtk-server will send messages (events) to us whenever
        # the user clicks something on the GUI. In order to read these
        # events, we send gtk-server the message "gtk_server_callback wait".
        # So, if the user has clicked on a widget on the GUI, then
        # 'event' contains the widget's handle (ID number).
        event = gtk_send("gtk_server_callback wait")
        # The user clicked on the windows 'close' button.
        if event == win
            break
        end
        # The user clicked on the quit button.
        if event == but
            break
        end
        # The user clicked on the 'amplitude' slider. Now we need
        # to go to gtk-server again to ask for the new slider's value.
        if event == sliA
            # here we get the new value
            A_g = gtk_send("gtk_range_get_value $sliA")
            A = parse(Float32, A_g)
            # plot again with the new sine values
            P = doplots(t, A, f₀, ϕ, fs)
        end
        # Frequency slider.
        if event == sliF
            f₀_g = gtk_send("gtk_range_get_value $sliF")
            f₀ = parse(Float32, f₀_g)
            P = doplots(t, A, f₀, ϕ, fs)
        end
        # ϕ slider.
        if event == sliP
            ϕ_g = gtk_send("gtk_range_get_value $sliP")
            ϕ = parse(Float32, ϕ_g)
            P = doplots(t, A, f₀, ϕ, fs)
        end
    end

    # Clean up -- close all figures and tell gtk-server to quit.
    closeall()
    gtk_exit()
end


# Practical Machine learning
# Regression Analysis - Linear Regression example
# Chapter 10

using Gadfly

# Normalizes the features in x
# The mean value of each feature is 0 and the standard deviation is 1
# Returns normalized x, μ and σ
function featureNormalize(x)
rows = size(x,1)
cols = size(x,2)

μ = mean(x,1)
σ = std(x,1)
xNorm = zeros(x)

# normalize
for i in 1:cols
	for j in 1:rows
		xNorm[j,i] = (x[j,i] - μ[i]) / σ[i];
	end
end

(xNorm, μ, σ)
end

println("Loading data ... ")
data = readdlm("data.txt",',')
x = data[:,1:2]
y = data[:, 3]
m = length(y)

@printf("First 10 examples from the dataset: \n");
t = [x[1:10,:] y[1:10,:]]'
for i in 1:10
  @printf(" x = [%.0f %.0f], y = %.0f \n", t[1,i], t[2,i], t[3,i]);
end

# Scale features and set them to zero mean
(x, μ, σ) = featureNormalize(x);

# Add intercept term to x
x = [ones(m,1) x]

#### Run Gradient Descent
α = 0.001
numIter = 4000
θ = zeros(3,1)
jHist = zeros(numIter, 1)

for i in 1:numIter
  # next theta
  θ = θ - (α/m) * (x' * ((x*θ)-y))
  # compute cost
  jHist[i] = sum((x*θ-y).^2)/(2m)
end

# plot convergence graph
pl = plot(
  x=collect(1:numIter),
  y=jHist,
  Guide.xlabel("Iterations"),
  Guide.ylabel("Error"),
  Guide.title("Convergence Graph"),
  Geom.line
  )
draw(SVGJS("jHist.js.svg", 6inch, 6inch), pl)

# Estimate the price of a 1650 sq-ft, 3 br house
price = [1, (1650-μ[1])/σ[1], (3-μ[2])/σ[2]]' * θ
println("Estimated price for a 1650 sq-ft, 3 br house: $price")

println("done!")


using LinearAlgebra, SparseArrays, Random, Statistics
using Plots, Plots.PlotMeasures

function gomp(y0::AbstractVector, Ψ::AbstractMatrix,
    K::Int = 2, S::Int = 1;
    max_iter::Int = 100, ϵ::Real = eps())

	# Get the dimensions
	m = length(y0)
	m_psi, n = size(Ψ)
	# Assert the dimensionality
	@assert m == m_psi "Please provide consistent input sizes"
	# Assert the selector
	@assert S <=  min(K, m/K) "S <= min(K, m/K)"

	# Normalize
	ψ = deepcopy(Ψ)
    # Inplace
	normalize!.(eachcol(ψ), 2)

	# Preconditioning
	P = ψ'pinv(ψ*ψ')
	# New matrix
	ψ = P*ψ
	y = P*y0
	# Iteration
	iters = 0

	# Support
	Λ = zeros(Bool, n)
	u = zeros(eltype(y), n)
	r = y
	amps = zeros(eltype(y), n)

	# Find the magnitudes
	for i in 1:max_iter
		# Compute the similarities via magnitude
		amps .= ψ'r
		# Get the largest entry
		idx = sortperm(abs.(amps), rev = true)[1:S]
		# Update the support
		Λ[idx] .= true
		# Update the coefficients
		u[Λ] .= (y' / ψ')[1, Λ]
		# Update r
		r .= y - ψ*u
		# Convergence
		(norm(r,2) < ϵ || sum(Λ) >= K) && break
	end

    # Last time to get the right coefficients
	u[Λ] .= (y0' / Ψ')[1, Λ]

	return u
end

## Test data
Random.seed!(1111) # Lucky
x̂ = sprandn(Float64, 100, 0.1)
k_opt = Int(norm(x̂, 0))
A = randn(Float64, 300, 100)
y = A*x̂

# Evaluate
x = gomp(y, A, k_opt)

# Check for different sparsties levels
sp_n = map(0.01:0.01:0.2) do i
	x̃ = sprandn(Float64, 100, i)
	ỹ = A*x̃
	norm(gomp(ỹ, A, 50, 1, ϵ = 0.01), 0), norm(x̃, 0)
end

gr()
pl_1 = scatter(x̂, label = "Ground Truth", markeralpha = 0.5, xlabel = "Index", ylabel = "x", legend = :bottomright)
scatter!(x, label = "Estimated", markershape = :cross, markersize = 12, markerstrokewidth = 5)
savefig(joinpath(pwd(), "figures", "gomp.pdf"))

xs, ys = first.(sp_n), last.(sp_n)
pl_2 = scatter(ys, xlabel = "Run", ylabel = "Sparsity", label = "Ground Truth", legend = :topleft)
scatter!(xs, label = "Estimated", markershape = :cross, markersize = 12, markerstrokewidth = 5)
savefig(joinpath(pwd(), "figures", "gomp_vary.pdf"))

plot(pl_1, pl_2, layout = (1,2), size = (900, 300), margins = 5mm)
savefig(joinpath(pwd(), "figures", "merged.pdf"))


using ReachabilityAnalysis: logarithmic_norm

@testset "Logarithmic norm" begin

    # Example 1
    A = [1 2; 3 -4]

    @test logarithmic_norm(A, 1) == 4
    @test logarithmic_norm(A, Inf) == 3

    # Example 2 (see Example 2 in "Explicit Error Bounds for Carleman Linearization.
    # Marcelo Forets and Amaury Pouly (2018)").
    A = [0 1; -1 -2]
    Aop2 = kron(A, A)

    @test logarithmic_norm(A, Inf) == 1
    @test logarithmic_norm(Aop2, Inf) == 9;

    @test logarithmic_norm(A, 1) == 1
    @test logarithmic_norm(Aop2, 1) == 9

    @test logarithmic_norm(A, 2) == 0
    @test logarithmic_norm(Aop2, 2) ≈ 4.23606797749979
end


using ApproxFun

#The following solves the Airy ODE with dirichlet boundary conditions

x=Fun(identity,-1000..15)   # Fun corresponding to multiplication by x, on [-100,15]
d=domain(x)
D=Derivative()             # The derivative operator
B=Dirichlet()              # Dirichlet boundary conditions, [u(-100),u(15)]

#Construct operator

A=[B;D^2-x]                # This is dirichlet conditions and u'' - x u
b=[airyai(leftendpoint(d)),0]     # We want it to equal airyai(-100) at -100, and 0 at
                           # 10, with 0 rhs
#Solve ODE

u=A\[b,0]                  # u satisfies A*u = b, or in other words,
                           # B*u = [airyai(-100.),0.] and (D2 - x)*u = 0.

# Check the accuracy
norm(u - Fun(airyai,d))

## We now solve with Neumann conditions

B=Neumann()
A=[B;D^2-x]
b=[[airyaiprime(leftendpoint(d)),0.],0.]

u=A\b

# Check the accuracy
norm(u - Fun(airyai,d))


# Test for GaussRadau()

n = 1
x, w = gaussradau(n)
@test x[1] ≈ -1.
@test w[1] ≈ 2.

n = 2
x, w = gaussradau(n)
@test x[1] ≈ -1.
@test x[2] ≈ 1/3
@test w[1] ≈ .5
@test w[2] ≈ 1.5

tol = 1e-14
n = 42
ntests = 4
x, w = gaussradau(n)
@test length(x) == n && length(w) == n
@test abs(x[37] - 0.908847278001044) < tol
@test abs(w[37] - 0.031190846817016) < tol
@test x[1] == -1

@test dot( w,exp.(x)) ≈ exp(1)-exp(-1)


#
# Denoise an SPD Example with Douglas Rachford
#
# L2-TV functional with anisotropic TV
#
#
#
using Manopt, Manifolds
using Images, CSV, DataFrames, LinearAlgebra, JLD2, Dates
#
# Settings
ExportOrig = false
ExportResult = true
ExportTable = true
asy_render_detail = 2
resultsFolder = "examples/Total_Variation/SPD_TV/"
comparisonData = "ImageCPPA-CostValue.jld2"
experimantName = "ImageDR"
if !isdir(resultsFolder)
    mkdir(resultsFolder)
end
#
# Manifold & Data
f = artificial_SPD_image2(32)
if ExportOrig
    asymptote_export_SPD(resultsFolder*experimantName*"orig.asy"; data=f, scaleAxes=(7.5,7.5,7.5))
    render_asymptote(resultsFolder * experimentName * "-orig.asy", render=asy_render_detail)
end
#
# Parameters
η  = 0.58
λ = 0.93
α = 6.
#
# Build Problem for L2-TV
pixelM = SymmetricPositiveDefinite(3);
M = PowerManifold(pixelM, NestedPowerRepresentation(), size(f)...)
d = length(size(f))
rep(d) = (d>1) ? [ones(Int,d)...,d] : d
fidelity(x) = 1/2*distance(M,x,f)^2
Λ(x) = forward_logs(M,x) # on T_xN
prior(x) = norm(norm.(Ref(pixelM), repeat(x,rep(d)...), Λ(x)),1)
#
# Setup & Optimize
print("--- Douglas–Rachford with η: $(η) and λ: $(λ) ---\n")
cost(x) = fidelity(x) + α*prior(x)
N = PowerManifold(pixelM, NestedPowerRepresentation(),5)
prox1 = (η,x) -> cat( prox_distance(M,η,f,get_component(N,x,1)), prox_parallel_TV(M,α*η,get_component(N,x,2:5)), dims=1)
prox2 = (η,x) -> fill(mean(M,x;stopping_criterion=StopAfterIteration(20)),5)
sC = StopAfterIteration(400)
try
    cost_threshold = load(resultsFolder*comparisonData)["compareCostFunctionValue"]
    global sC = StopWhenCostLess(cost_threshold)
    @info "Comparison to CPPA (`SPDImage_CPPA.jl`) and its cost of $cost_threshold."
catch y
    if isa(y, SystemError)
        @info "Comparison to CPPA only possible after runninng `SPDImage_CPPA.jl` its cost was stored."
    end
end
x0 = f
@time o = DouglasRachford(M, cost, [prox1,prox2], f;
    λ = i -> η, α = i -> λ, # map from Paper notation of BPS16 to toolbox notation
    debug = [:Iteration," | ", :Change, " | ", :Cost,"\n",10,:Stop],
    record = [:Iteration, :Cost ],
    stopping_criterion = sC,
    parallel=5,
    return_options = true
)
y = get_solver_result(o)
r = get_record(o)
#
# Result
numIter = length(r)
if ExportResult
    asymptote_export_SPD(resultsFolder*experimantName*"img-result-$(numIter)-α$(replace(string(α), "." => "-")).asy"; data=y, render=4, scaleAxes=(7.5,7.5,7.5) )
    render_asymptote(resultsFolder * experimentName * "img-result-$(numIter)-α$(replace(string(α), "." => "-")).asy", render=asy_render_detail)
end
if ExportTable
    A = cat( [ri[1] for ri in r], [ri[2] for ri in r]; dims=2 )
    CSV.write(resultsFolder*experimantName*"-Cost.csv",  DataFrame(A), writeheader=false);
end


# Arvind Prasadan
# 2019 January
# DMF Package Code
# Optshrink Function

"""
	optshrink(Y, r)

Perform a rank-r denoising of a data matrix Y via the OptShrink algorithm, described in http://doi.org/10.1109/TIT.2014.2311661 (R R Nadakuditi, 2014)

# Arguments
- `Y`: 2D Array (Data Matrix), where Y = X + Noise, and we want to find X
- `r`: An estimate of the rank of X (Integral)

# Output
- `Xh`: An estimate of X
- `w`: The shrunken singular values; if r = rank(Y), w is all zero
"""
function optshrink(Y, r)

    r = round(abs(r[1])) # Sanity check rank
    r = (1 <= r) ? r : 1

    (m, n) = size(Y)
    r = minimum([r, m, n]) # ensure r <= min(m, n)
    if r == m || r == n
	return Y, zeros(r)
    end

    # U is [m, min(m, n)], s is min(m, n), V is [n, min(n, m)]
    (U, s, V) = svd(Y, full = false)

    sv = s[(r + 1):end] # tail singular values for noise estimation

    w = zeros(r)
    for k = 1:1:r
        (D, Dder) = D_transform_from_vector(s[k], sv, max(m, n) - r, min(m, n) - r)
        w[k] = -2.0 * D / Dder
    end

    w[isnan.(w)] .= 0.0

    Xh = U[:, 1:r] * Diagonal(w) * V[:, 1:r]'

    return Xh, w
end

"""
	 D_transform_from_vector(z, sn, m, n)

Find the D Transform of the singular value spectrum sn at a point z, for an m x n matrix.

# Arguments
- `z`: Point to evaluate transform
- `sn`: Vector of singular values
- `m`: Matrix dimension
- `n`: Matrix dimension

# Outputs
- `D`: D Transform
- `D_der`: D Transform Derivative
"""
function D_transform_from_vector(z, sn, m, n)
    # sn is of length n <= m

    sm = [sn; zeros(m - n)] # m x 1

    inv_n = 1.0 ./ (z^2.0 .- sn.^2.0) # vector corresponding to diagonal
    inv_m = 1.0 ./ (z^2.0 .- sm.^2.0)

    D1 = (1.0 / n) * sum(z * inv_n)
    D2 = (1.0 / m) * sum(z * inv_m)

    D = D1 * D2 # eq (16a) in paper

    # derivative of D transform
    D1_der = (1.0 / n) * sum(-2.0 * z^2.0 .* inv_n.^2.0 + inv_n)
    D2_der = (1.0 / m) * sum(-2.0 * z^2.0 .* inv_m.^2.0 + inv_m)

    D_der = D1 * D2_der + D2 * D1_der # eq (16b) in paper

    return (D, D_der)
end


using JuMP

##### Simple LP
# Example taken directly from https://jump.readthedocs.org/en/latest/quickstart.html

println("***** Problem 1 *****")

m = Model()
@variable(m, 0 <= x <= 2 )
@variable(m, 0 <= y <= 30 )

@objective(m, Max, 5x + 3*y )
@constraint(m, 1x + 5y <= 3.0 )

print(m)

status = solve(m)

println("Objective value: ", getobjectivevalue(m))
println("x = ", getvalue(x))
println("y = ", getvalue(y))

##### Another simple LP with logical conditions in constraint definition
# Example from https://groups.google.com/forum/#!topic/julia-opt/pMHz-9YHN2o
# Note: This example only works with Julia-v0.4 and later

println(" ")
println("***** Problem 2 *****")

m = Model()

I = 1:5

@variable(m, 0 <= x[I] <= 10)
@variable(m, 0 <= y[I] <= 10)

# This constraint is only considered when i + j are less than or equal to 3
# Note: Only supported by Julia-v0.4 and later
@constraint(m, constr[i=I,j=I; i+j <= 3], x[i] - y[j] == 1)

@objective(m, Min, sum{x[i] + y[i], i=I})

print(m)

status = solve(m)

println(" ")
println("Objective value: ", getobjectivevalue(m))
println("x = ", getvalue(x))
println("y = ", getvalue(y))