"""
Solve the constrained least-squares problem

    minimize  ||Ax-b||₂  st  0 ≤ x ≤ u.

Rephrases the problem as linear SOC program:

    minimize  [0]'[x]  st  [A 0 I][x] = [b] m
              [0] [r]             [t]
                                  [r]
                            n 1 m

                               x, (t,r) ∈  Rn₊ x (Qn₊)

with dimensions  len(x) = n  len(b) = m
                 len(r) = m

Solve using ECOS, which formulates linear SOC programs as

    minimize  c'x  st  A*x=b, G*x + s = h, s in cone
"""
function lsbox(Ain::Matrix{Float64}, b::Vector{Float64},
               u::Vector{Float64}=zeros(0); kwargs...)
    m, n = size(Ain)
    nu = length(u)       # number of vars x with upper bound
    n_ecos = n + 1 + m   # number of vars == len(x)
    p = m                # number of equality constraints == len(b)
    l = n + nu           # number of vars in positive orthant
    ncones = 1           # number of SOC cones
    q = [1+m]            # number of vars in the SOC
    e = 0                # number of exponential cones

    GG = [ sparse(-1.0I,n ,n)  spzeros(n ,1)       spzeros(n ,m)
           sparse(+1.0I,nu,n)  spzeros(nu,1)       spzeros(nu,m)
                spzeros(1 ,n)      -1.0            spzeros( 1,m)
                spzeros(m ,n)  spzeros(m ,1)  sparse(-1.0I,m ,m)]
    G = ECOS.ECOSMatrix(GG)
    h = vcat(zeros(n), u,   0,  zeros(m))
    m_ecos =       n + nu + 1 + m
    A = ECOS.ECOSMatrix([Ain spzeros(m,1) sparse(1.0I,m,m)])
    c = zeros(n_ecos); c[n+1] = 1.0

    ecos_ptr = ECOS.setup(n_ecos, m_ecos, p, l, ncones, q, e,
                          G, A, c, h, b; kwargs...)
    ECOS.solve(ecos_ptr)
    # finalizer(cleanup!, problem)

    # Extract the primal and dual solutions of the ECOS problem
    ecos_prob = unsafe_wrap(Array, ecos_ptr, 1)[1]
    xtr = unsafe_wrap(Array, ecos_prob.x, n_ecos)[:]

    ECOS.cleanup(ecos_ptr, 0)
    x = xtr[1:n]
    t = xtr[n+1] # t = norm(r)
    r = xtr[n+2:end]
    return x, r, t
end


using Manopt, Manifolds, Plots
#
# Settings
resultsFolder = "examples/Total_Variation/S1_TV/"
experimentName = "S1_TV12"
plotAndExportResult = true

n = 500
σ = 0.2
α = 0.5
β = 1.0

dataColor = RGBA{Float64}(colorant"#BBBBBB")
s2dColor = RGBA{Float64}(colorant"#EE7733") # data Color: Tol Vibrant Orange
s1Color = RGBA{Float64}(colorant"#0077BB") # control point data color: Tol Virbant Blue
nColor = RGBA{Float64}(colorant"#33BBEE") # tangent vector: Tol Vibrant Teal

if !isdir(resultsFolder)
    mkdir(resultsFolder)
end
#
# Manifolds and Data
M = Circle()
N = PowerManifold(M, n)
f = artificial_S1_signal(n)
xCompare = f
fn = exp.(Ref(M),f, random_tangent.(Ref(M),f, Val(:Gaussian), σ))
data = fn
t = range(0.0, 1.0, length = n)

if plotAndExportResult
    scene = scatter(
        t,
        f,
        markersize = 2,
        markercolor = dataColor,
        markerstrokecolor = dataColor,
        lab = "original",
    )
    scatter!(
        scene,
        t,
        fn,
        markersize = 2,
        markercolor = nColor,
        markerstrokecolor = nColor,
        lab = "noisy",
    )
    yticks!(
        [-π, -π/2, 0, π/2, π],
        [raw"$-\pi$",raw"$-\frac{\pi}{2}$",raw"$0$",raw"$\frac{\pi}{2}$",raw"$\pi$"]
    )
    png(scene, "$(resultsFolder)$(experimentName)-original.png")
end
#
# Setup and Optimize
F = x -> costL2TVTV2(N, data, α, β, x)
proxes = [
    (λ, x) -> prox_distance(N, λ, data, x,2),
    (λ, x) -> prox_TV(N, α * λ, x),
    (λ, x) -> prox_TV2(N, β * λ, x),
]

o = cyclic_proximal_point(
    N,
    F,
    proxes,
    data;
    λ = i -> π / (2*i),
    debug = Dict(
        :Stop => DebugStoppingCriterion(),
        :Step => DebugEvery(
            DebugGroup([
                DebugIteration(),
                DebugDivider(),
                DebugProximalParameter(),
                DebugDivider(),
                DebugCost(),
                DebugDivider(),
                DebugChange(),
                DebugDivider("\n"),
            ]),
            1000,
        ),
        :Start => DebugDivider("Starting the solver\n"),
    ),
    record = [:Iteration, :Cost, :Change, :Iterate],
    return_options = true,
)
fR = get_solver_result(o)
r = get_record(o)
#
# Result
if plotAndExportResult
    scene = scatter(
        t,
        f,
        markersize = 2,
        markercolor = dataColor,
        markerstrokecolor = dataColor,
        lab = "original",
    )
    scatter!(
        scene,
        t,
        fR,
        markersize = 2,
        markercolor = nColor,
        markerstrokecolor = nColor,
        lab = "reconstruction",
    )
    yticks!(
        [-π, -π/2, 0, π/2, π],
        [raw"$-\pi$",raw"$-\frac{\pi}{2}$",raw"$0$",raw"$\frac{\pi}{2}$",raw"$\pi$"]
    )
    png(scene, "$(resultsFolder)$(experimentName)-result.png")
end

print("MSE (input):  ", 1 / n * distance(N, xCompare, data)^2, "\n")
print("MSE (result): ", 1 / n * distance(N, xCompare, fR)^2, "\n")


# Integrates the following set of equations outwards:
# dy1/dx =                y2
# dy2/dx = C1 * y1 + C2 * y2
function sch_rk4_step!(
    rmesh::Array{Float64,1}, # Grid
    y0::Array{Float64,1}, # Initial condition
    C1::Array{Float64,1},
    C2::Array{Float64,1},
    C1mid::Array{Float64,1},
    C2mid::Array{Float64,1},
    y1::Array{Float64,1},
    y2::Array{Float64,1}, # Solution y1 and y2
    maxval_stop::Float64,  # Maximum value (if y1 > max_val, the integration stops)
)

    Neqn = size(y0,1)  # should be hardcoded at 2 ?
    @assert Neqn == 2

    dym = zeros(Float64,2)
    dyt = zeros(Float64,2)
    yt = zeros(Float64,2)
    dydx = zeros(Float64,2)
    y = zeros(Float64, Neqn)

    Nr = size(rmesh,1)
    y = copy(y0)
    y1[1] = y[1]
    y2[1] = y[2]

    for i in 2:Nr
        # rk4 step size
        h = rmesh[i] - rmesh[i-1]

        # k1, evaluate F
        dydx[1] = y[2]
        dydx[2] = C1[i-1] * y[1] + C2[i-1] * y[2]
        #
        yt = y + h/2 * dydx  # yi + h/2*k1
        # k2, evaluate F at x_i + h/2 and yi + k1/2
        dyt[1] = yt[2]
        dyt[2] = C1mid[i-1] * yt[1] + C2mid[i-1] * yt[2]
        #
        yt = y + h/2 * dyt # yi + h/2*k2
        #
        dym[1] = yt[2]
        dym[2] = C1mid[i-1] * yt[1] + C2mid[i-1] * yt[2]
        #
        yt = y + h * dym
        dym = dyt + dym
        dyt[1] = yt[2]
        dyt[2] = C1[i] * yt[1] + C2[i] * yt[2]
        #
        y = y + h/6 * (dydx + dyt + 2*dym)
        #
        y1[i] = y[1]
        y2[i] = y[2]

        # The integration stops at R(imax)
        if abs(y[1]) >= maxval_stop
            imax = i
            return imax
        end
    end
    imax = Nr
    return imax
end


using LinearAlgebra
using JLD2
import StatisticalNonlocality: ou_transition_matrix
import StatisticalNonlocality: fourier

filename = "three_state.jld2"

n = 2
M = ou_transition_matrix(n)

Λ, V = eigen(M)
Λ, W = eigen(M')

# Right eigenvectors
V³ = round.(V[:, 1] ./ V[1, 1])
V² = round.(V[:, 2] ./ V[1, 2])
V¹ = round.(V[:, 3] ./ V[1, 3])

# Left eigenvectors
W³ = round.(W[:, 1] ./ W[1, 1])
W² = round.(W[:, 2] ./ W[1, 2])
W¹ = round.(W[:, 3] ./ W[1, 3])

W[:, 1] .= W¹
W[:, 2] .= -W²
W[:, 3] .= W³

U = [-1 0 0; 0 0 0; 0 0 1] # advection operator structure
W * U * inv(W)

N = 128

D, x = fourier(N, a = 0, b = 2π)

γ = 1.0 # 1/ γ is the eddy timescale
U = 1.0
κ = 1.0

fluxkernel =
    -U * inv(κ .* D^2 - γ * I - U * D * inv(κ .* D^2 - 2 * γ * I) * U * D) * U
λ = eigvals(fluxkernel)
σ = svdvals(fluxkernel)

data_directory = "data"
mkpath(data_directory)
filepath = data_directory * "/" * filename
if isfile(filepath)
    rm(filepath)
end

file = jldopen(filepath, "a+")

file["kernel"] = fluxkernel
file["eigenvalues"] = λ
file["singularvalues"] = σ

close(file)


using Plots
using Zygote
using LaTeXStrings

vecindario(;x₀, ϵ=2.0, Δ=0.05) = x₀ - ϵ:Δ:x₀ + ϵ

f(x) = sin(5x) - cos(3x)

x₀ = 10
x = vecindario(x₀=x₀);

f₁ = f'(x₀)
f₂ = f''(x₀)

y = f.(x);
y = hcat(y, ones(length(y)) .* f(x₀))
y = hcat(y, @. y[:, end] + (x - x₀) * f₁)
y = hcat(y, @. y[:, end] + (x - x₀)^2 * f₂)

gr(size=(800, 800))
p = plot(x, y[:,1], label=L"f(x)")
plot!(p, x, y[:,2], label="Orden cero")
plot!(p, x, y[:,3], label="Primer orden")
plot!(p, x, y[:,4], label="Segundo orden")
plot!(p, [x₀], [f(x₀)], mark=:cirle, label=L"x_0")
title!("Aproximación por series de Taylor")
xlabel!(p, L"x")
ylabel!(p, L"f(x)")
xlims!(p, (x[1], x[end]))

savefig(p, "Taylor_borrar.png")


"""
    Module Basis2D
Includes DG core functions for evaluating 2D bases on quads
"""
module Basis2D

export vandermonde2D, grad_vandermonde2D
export chebyshev2D, equidistant_nodes2D, gauss_quad2D
export dPhi2D

import Basis1D
using LinearAlgebra, SparseArrays

"""
    vandermonde2D(p, x, y)
Return the 2D vandermonde matrix - the order 0:p Legendre polynomials
evaluated at 2D nodes (x,y)
"""
function vandermonde2D(p, x, y)
    V = Array{Float64,4}(undef,length(x),length(y),p+1,p+1)
    lx = Basis1D.vandermonde1D(p, x)
    ly = Basis1D.vandermonde1D(p, y)
    for ipy = 1:p+1
        for ipx = 1:p+1
            V[:,:,ipx,ipy] = lx[:,ipx]*(ly[:,ipy]')
        end
    end
    return V
end

"""
    grad_vandermonde2D(p, x, y)
Return the gradient of the order 0:p Legendre polynomials
evaluated at 2D nodes (x,y)
"""
function grad_vandermonde2D(p, x, y)
    dV = Array{Float64,5}(undef, length(x),length(y), p+1,p+1, 2)
    lx = Basis1D.vandermonde1D(p, x)
    ly = Basis1D.vandermonde1D(p, y)
    dlx = Basis1D.grad_vandermonde1D(p, x)
    dly = Basis1D.grad_vandermonde1D(p, y)
    # Derivative wrt x
    for ipy = 1:p+1
        for ipx = 1:p+1
            dV[:,:,ipx,ipy,1] = dlx[:,ipx]*(ly[:,ipy]')
        end
    end
    # Derivative wrt y
    for ipy = 1:p+1
        for ipx = 1:p+1
            dV[:,:,ipx,ipy,2] = lx[:,ipx]*(dly[:,ipy]')
        end
    end
    return dV
end

"""
    gauss_quad2D(p)
Return the 2D Gaussian quadrature points and weights (x,w) of order p on the domain [-1,1]^2
"""
function gauss_quad2D(p)
    (x1D,w1D) = Basis1D.gauss_quad(p)
    n = length(x1D)
    x2D = Array{Float64,3}(undef, 2, n,n)
    w2D = Array{Float64,2}(undef, n,n)
    for iy = 1:n
        for ix = 1:n
            x2D[1,ix,iy] = x1D[ix]
            x2D[2,ix,iy] = x1D[iy]
            w2D[ix,iy] = w1D[ix]*w1D[iy]
        end
    end
    return (x2D, w2D)
end

"""
    chebyshev2D(p)
Return the 2D chebyshev points of order p on the domain [-1,1]^2
"""
function chebyshev2D(p)
    x1D = Basis1D.chebyshev(p)
    n = length(x1D)
    x2D = Array{Float64,3}(undef, 2, n, n)
    for iy = 1:n
        for ix = 1:n
            x2D[1,ix,iy] = x1D[ix]
            x2D[2,ix,iy] = x1D[iy]
        end
    end
    return x2D
end

"""
    equidistant_nodes2D(p)
Return the "order p" 2D points on the domain [-1,1]^2
"""
function equidistant_nodes2D(p)
    x1D = -1:2/p:1
    n = length(x1D)
    x2D = Array{Float64,3}(undef, 2, n, n)
    for iy = 1:n
        for ix = 1:n
            x2D[1,ix,iy] = x1D[ix]
            x2D[2,ix,iy] = x1D[iy]
        end
    end
    return x2D
end

"""
    interpolation_matrix2D(xy_from, xy_to)
Compute an interpolation matrix from a set of 2D points to another set of 2D points.
Assumes that points interpolating from provide enough accuracy (aka - they are
well spaced out and of high enough order), and define a square. Points
interpolating onto can be of any size, but must be defined on this same square.
Interpolation matrix ∈ ℜ^(2D size of xy_to x 2D size of xy_from)
"""
function interpolation_matrix2D(xy_from, xy_to)
    # Create nodal representation of reference bases
    (x_from, y_from) = xy_from
    order = size(x_from,1) - 1 # Assumes order = (size of x_from) - 1
    @assert size(x_from,1) == size(y_from,1)
    n_from = size(x_from,1)*size(y_from,1)
    l_from = vandermonde2D(order, x_from, y_from)

    eye = diagm(0=>ones(n_from))
    V = reshape(l_from, n_from,n_from)
    coeffs_phi = V \ eye

    # Compute reference bases on the output points
    l_to = vandermonde2D(order, xy_to...)
    n_to = prod(size.(xy_to,1))
    V_to = reshape(l_to, n_to,n_from)

    # Construct interpolation matrix
    Interp2D = V_to*coeffs_phi
end

"""
    dPhi2D(xy_from, xy_to)
Compute the gradient of the basis functions defined by 2D points on another
set of 2D points.

Assumes that points interpolating from provide enough accuracy (aka - they are
well spaced out and of high enough order), and define a square. Points
interpolating onto can be of any size, but must be defined on this same square.
dPhi ∈ ℜ^(size of xy_to × size of xy_from × 2)
"""
function dPhi2D(xy_from, xy_to)
    dim = 2
    # Create nodal representation of reference bases
    (x_from, y_from) = xy_from
    order = size(x_from,1) - 1 # Assumes order = (size of x_from) - 1
    @assert size(x_from,1) == size(y_from,1)
    n_from = size(x_from,1)*size(y_from,1)
    l_from = vandermonde2D(order, x_from, y_from)

    eye = diagm(0=>ones(n_from))
    V = reshape(l_from, n_from,n_from)
    coeffs_phi = V \ eye

    # Compute derivative of reference bases on the output points
    dl_to = grad_vandermonde2D(order, xy_to...)
    n_to = prod(size.(xy_to,1))
    dV = reshape(dl_to, n_to,n_from,dim)
    # Construct gradient of phi = dV*coeffs_phi
    dPhi_to = Array{Float64,3}(undef, n_to,n_from,dim)
    for l = 1:dim
        dPhi_to[:,:,l] = dV[:,:,l]*coeffs_phi
    end
    return dPhi_to
end

end


## Pendulum
# https://github.com/HarvardAgileRoboticsLab/unscented-dynamic-programming/blob/master/pendulum_dynamics.m
function pendulum_dynamics!(ẋ::AbstractVector{T},x::AbstractVector{T},u::AbstractVector{T}) where T
    m = 1.
    l = 0.5
    b = 0.1
    lc = 0.5
    I = 0.25
    g = 9.81
    ẋ[1] = x[2]
    ẋ[2] = (u[1] - m*g*lc*sin(x[1]) - b*x[2])/I
end

n,m = 2,1
pendulum = Model(pendulum_dynamics!,n,m) # inplace model

function pendulum_dynamics2!(ẋ::AbstractVector{T},x::AbstractVector{T},u::AbstractVector{T}) where T
    ẋ[1] = x[2];
    ẋ[2] = sin(x[1]) + u[1];

end

pendulum2 = Model(pendulum_dynamics2!,n,m) # inplace model

# unknown mass
function pendulum_dynamics_uncertain!(ẋ,x,u,w)
    m = 1. + w[1]
    l = 0.5
    b = 0.1
    lc = 0.5
    I = 0.25
    g = 9.81

    ẋ[1] = x[2]
    ẋ[2] = u[1]/(m*lc*lc) - g*sin(x[1])/lc - b*x[2]/(m*lc*lc)
    return nothing
end

n = 2; m = 1; r = 1
pendulum_uncertain = UncertainModel(pendulum_dynamics_uncertain!,n,m,r)


# ---
# title: 9. Palindrome Number
# id: problem9
# author: Tian Jun
# date: 2020-10-31
# difficulty: Easy
# categories: Math
# link: <https://leetcode.com/problems/palindrome-number/description/>
# hidden: true
# ---
#
# Determine whether an integer is a palindrome. An integer is a palindrome when
# it reads the same backward as forward.
#
# **Follow up:** Could you solve it without converting the integer to a string?
#
#
#
# **Example 1:**
#
#
#
#     Input: x = 121
#     Output: true
#
#
# **Example 2:**
#
#
#
#     Input: x = -121
#     Output: false
#     Explanation: From left to right, it reads -121. From right to left, it becomes 121-. Therefore it is not a palindrome.
#
#
# **Example 3:**
#
#
#
#     Input: x = 10
#     Output: false
#     Explanation: Reads 01 from right to left. Therefore it is not a palindrome.
#
#
# **Example 4:**
#
#
#
#     Input: x = -101
#     Output: false
#
#
#
#
# **Constraints:**
#
#   * `-231 <= x <= 231 - 1`
#
#
## @lc code=start
using LeetCode

function is_palindrome(x::Int)::Bool
    if x < 0
        false
    else
        y = x
        y_reverse = 0
        while y > 0
            y, n = divrem(y, 10)
            y_reverse = y_reverse * 10 + n
        end
        x == y_reverse
    end
end
## @lc code=end


# This file is a part of SimilaritySearch.jl

export JaccardDistance, DiceDistance, IntersectionDissimilarity
import Distances: evaluate

"""
    JaccardDistance()

The Jaccard distance is defined as

```math
J(u, v) = \\frac{|u \\cap v|}{|u \\cup v|}
```
"""
struct JaccardDistance <: PreMetric end

"""
    DiceDistance()

The Dice distance is defined as

```math
D(u, v) = \\frac{2 |u \\cap v|}{|u| + |v|}
```
"""
struct DiceDistance <: PreMetric end

"""
    IntersectionDissimilarity()

The intersection dissimilarity uses the size of the intersection as a mesuare of similarity as follows:

```math
I(u, v) = 1 - \\frac{|u \\cap v|}{\\max \\{|u|, |v|\\}}
```
"""
struct IntersectionDissimilarity <: PreMetric end

"""
    union_intersection(a::T, b::T)

Computes both the size of the unions an the size the intersections of `a` and `b`;
specified as ordered sequences.
"""
function union_intersection(a::T, b::T) where {T <: AbstractVector}
    len_a::Int = length(a)
    len_b::Int = length(b)
    ia::Int = 1
    ib::Int = 1
    intersection_size::Int = 0
    c::Int = 0
    @inbounds while ia <= len_a && ib <= len_b
        c = cmp(a[ia], b[ib])
        if c == 0
            ia += 1
            ib += 1
            intersection_size += 1
        elseif c < 0
            ia += 1
        else
            ib += 1
        end
    end

    len_a + len_b - intersection_size, intersection_size
end

"""
    evaluate(::JaccardDistance, a, b)

Computes the Jaccard's distance of `a` and `b` both sets specified as
sorted vectors.
"""
function evaluate(::JaccardDistance, a, b)
    u, i = union_intersection(a, b)
    1.0 - i / u
end

"""
    evaluate(::DiceDistance, a, b)

Computes the Dice's distance of `a` and `b` both sets specified as
sorted vectors.
"""
function evaluate(::DiceDistance, a, b)
    u, i = union_intersection(a, b)
    1.0 - 2 * i / (length(a) + length(b))
end

"""
    evaluate(::IntersectionDissimilarity, a, b)

Uses the intersection as a distance function (non-metric)
"""
function evaluate(::IntersectionDissimilarity, a, b)
    u, i = union_intersection(a, b)

    return 1.0 - i / max(length(a), length(b))
end


using SciPyDiffEq
using Test

function lorenz(u,p,t)
 du1 = 10.0(u[2]-u[1])
 du2 = u[1]*(28.0-u[3]) - u[2]
 du3 = u[1]*u[2] - (8/3)*u[3]
 [du1, du2, du3]
end
u0 = [1.0;0.0;0.0]
tspan = (0.0,100.0)
prob = ODEProblem(lorenz,u0,tspan)
sol = solve(prob,SciPyDiffEq.RK45())
sol = solve(prob,SciPyDiffEq.RK23())
sol = solve(prob,SciPyDiffEq.Radau())
sol = solve(prob,SciPyDiffEq.BDF())
sol = solve(prob,SciPyDiffEq.LSODA())
sol = solve(prob,SciPyDiffEq.odeint())

function lorenz(du,u,p,t)
 du[1] = 10.0(u[2]-u[1])
 du[2] = u[1]*(28.0-u[3]) - u[2]
 du[3] = u[1]*u[2] - (8/3)*u[3]
end
u0 = [1.0;0.0;0.0]
tspan = (0.0,100.0)
prob = ODEProblem(lorenz,u0,tspan)
sol = solve(prob,SciPyDiffEq.RK45())
sol(4.0)
sol = solve(prob,SciPyDiffEq.RK23())
sol(4.0)
sol = solve(prob,SciPyDiffEq.Radau())
sol(4.0)
sol = solve(prob,SciPyDiffEq.BDF())
sol(4.0)
sol = solve(prob,SciPyDiffEq.LSODA())
sol(4.0)
sol = solve(prob,SciPyDiffEq.odeint())
sol(4.0)

#using Plots; plot(sol,vars=(1,2,3))


# Autonomous van der Pol
using DrWatson
@quickactivate "NonlinearDynamicsTextbook"
include(srcdir("style.jl"))
using DynamicalSystems, PyPlot

# parameters and initial values
dvec = [0.1, 5]    # damping parameters
y_0 = [0.1, 0.1]   # initial condition
ttrans = 100   # transient time
tattr = 16     # time on attractor
Δt = 0.01   # sampling time
tvec = 0:Δt:tattr

fig, axs = subplots(1,2; figsize=(0.6figx,0.8figy))

for id = 1:2
    d = dvec[id]
    y_init = y_0
    ds = Systems.vanderpol(; F = 0, μ = d)
    tr = trajectory(ds, tattr, y_0; Δt, Ttr = ttrans)
    x1vec, x2vec = columns(tr)
    axs[1].plot(tvec,x1vec; c = COLORS[id] )
    axs[2].plot(x1vec,x2vec; c = COLORS[id] )
end
axs[1].set_xlabel(L"$t$"; labelpad = -20)
axs[1].set_ylabel(L"$x$", rotation = 0)
axs[2].set_xlabel(L"$x$"; labelpad = -20)
axs[2].set_ylabel(L"$\dot x$", rotation = 0)
axs[1].set_xticks([0,15])
axs[2].set_xticks([-2,0,2])
axs[2].set_xticklabels(["-2", "", "2"])
add_identifiers!(fig)
fig.tight_layout(pad=0.33)

wsave(plotsdir("9", "vanderPol_autonomous"), fig)


### A Pluto.jl notebook ###
# v0.12.21

using Markdown
using InteractiveUtils

# ╔═╡ 4fb1c47c-8d70-11eb-3d5c-01c5bb7190fe
using Pkg, DrWatson

# ╔═╡ 5838d806-8d70-11eb-2a78-fb586f056012
begin
	using Distributions
	using StatsPlots
	using StatsBase
	using LaTeXStrings
	using CSV
	using DataFrames
	using LinearAlgebra
	using Random
	using StatisticalRethinking
end

# ╔═╡ 3562c0e2-8d70-11eb-0f89-9d039fe996b9
md" ## Clip-11-01-05s.jl"

# ╔═╡ 07c85e9e-8d95-11eb-295d-63a99f7aa921
md" ##### Fig 10.7."

# ╔═╡ 9bfd33fc-8d70-11eb-3a9c-bb9a990cbcf9
x = collect(-1.0:0.01:1.0);

# ╔═╡ ee25cbe6-8d8c-11eb-3323-c521a67b10f0
begin
	α = 0.0
	β = 2.0

	# p = exp.(α .+ β .* x) ./ (1 .+ exp.(α .+ β .* x))

	p = logistic.(α .+ β .* x)
end

# ╔═╡ 1cba8970-8d91-11eb-01d8-09d8d076dc73
begin
	odds = p ./ (1 .- p)
	logodds = log.(odds)
end

# ╔═╡ 6b96d95e-8d91-11eb-25ac-75f21ee9c98e
logistic.(x)

# ╔═╡ 15dcde1e-8d8c-11eb-0964-d9c5f616e59c
begin
	fig1 = plot(x, logodds, leg=false)
	fig2 = plot(x, p, ylims=(0,1), leg=false)
	plot(fig1, fig2, layout=(1, 2))
end

# ╔═╡ 1404a820-8d95-11eb-27d2-531cea578fda
md" ##### Fig 10.8."

# ╔═╡ c859986c-8d93-11eb-1211-db9c435de034
logsig = exp.(α .+ β .* x)

# ╔═╡ f3bac6a4-8d93-11eb-2614-1572443e3415
begin
	fig3 = plot(x, log.(logsig), leg=false)
	fig4 = plot(x, logsig, ylims=(0,10), leg=false)
	plot(fig3, fig4, layout=(1, 2))
end

# ╔═╡ 7e069492-8d70-11eb-2093-8375b07f4099
md" ## End of clip-11-01-05s.jl"

# ╔═╡ Cell order:
# ╠═3562c0e2-8d70-11eb-0f89-9d039fe996b9
# ╠═4fb1c47c-8d70-11eb-3d5c-01c5bb7190fe
# ╠═5838d806-8d70-11eb-2a78-fb586f056012
# ╟─07c85e9e-8d95-11eb-295d-63a99f7aa921
# ╠═9bfd33fc-8d70-11eb-3a9c-bb9a990cbcf9
# ╠═ee25cbe6-8d8c-11eb-3323-c521a67b10f0
# ╠═1cba8970-8d91-11eb-01d8-09d8d076dc73
# ╠═6b96d95e-8d91-11eb-25ac-75f21ee9c98e
# ╠═15dcde1e-8d8c-11eb-0964-d9c5f616e59c
# ╟─1404a820-8d95-11eb-27d2-531cea578fda
# ╠═c859986c-8d93-11eb-1211-db9c435de034
# ╠═f3bac6a4-8d93-11eb-2614-1572443e3415
# ╟─7e069492-8d70-11eb-2093-8375b07f4099


### A Pluto.jl notebook ###
# v0.14.1

using Markdown
using InteractiveUtils

# ╔═╡ e03dd22c-884c-11eb-2432-3dc04b0cf286
begin
  using Pkg; Pkg.activate("MLJ_env", shared=true)
  using Test
  using Plots
  using PlutoUI
  using Random
  using Distributions

  push!(LOAD_PATH, "./src")
  using YaLinearAlgebra
end

# ╔═╡ 380c26d0-881d-11eb-0461-f339488ea81e
md"""
## Gradient Descent

ref. from book **"Data Science from Scratch"**, Chap 8

$(html"
© Pascal, April 2021
")
"""

# ╔═╡ 83e01f80-884a-11eb-1ef7-e9cb5ab3ed20
PlutoUI.TableOfContents(indent=true, depth=4, aside=true)

# ╔═╡ 853ab02c-884a-11eb-01bf-1712dc177ffb
html"""

"""

# ╔═╡ 4c951444-8869-11eb-0b88-55a9fb9e51e6
html"""

"""

# ╔═╡ 8c87c3b8-884a-11eb-10c8-73e2dab560ea
md"""
### Context
"""

# ╔═╡ 8c6ba818-884a-11eb-23e0-2f1c297d88f5
md"""
The gradient (vector of partial derivatives) gives the direction in which the function most quickly increase.

In ML we are looking at a best model (relative to some criteria) to fit the data. In this context "best" is obtained by minimizing a quantity (ex. minimize MSE between prediction and ground truth in a supervised setting) or maximizing a quantity (ex. likelihood of the data). This means that the problem is cast as an optimization problem. One technique to solve those optimization problems, which scale rather weel is **Gradient Descent** (or **Gradient Ascent**).

In a nutshell, Gradient Descent consists in picking a random starting point, computing the gradient (derivatives) of a given function w.r.t its parameters and take a small step in the opposite direction of the gradient (minimization setup, alternatively in the direction, for gradient ascent and a maximization setup) and repeat the process with this updated point until some convergence criteria.
"""

# ╔═╡ d2d18b22-884c-11eb-1f93-037453c5395b
begin
  f(x, y) = x^2 + y^2
  x = -10:10
  y = x
  plot(x, y, f, linetype=:surface)

  xₛ = 0:0.5:10
  yₛ = 0:-.45:-9
  zₛ = f.(xₛ, yₛ)
  scatter!(xₛ, yₛ, zₛ, marker=:v, markersize=4, color=:steelblue, legend=false)
end

# ╔═╡ 221734fe-8869-11eb-331a-b5810d0b784c
html"""

"""

# ╔═╡ d297fa06-884c-11eb-294e-85992f901be1
md"""
### Estimating the gradient

If f is a function of one variable, its derivative at point x measures how f(x) changes as we make a very small change of x. The derivative is the limit of the difference of the quotients
"""

# ╔═╡ 1859c082-8853-11eb-18f6-2bc7a180b4a2
begin
  const AF =  AbstractFloat
  const VT = Vector{T} where T <: AbstractFloat
end

# ╔═╡ d27feefa-884c-11eb-3b84-1f529d7f6410
begin
  function diff_quotient(f::Function, x::AF, Δ::AF)::AF
    (f(x + Δ) - f(x)) / Δ
  end

  diff_quotient(f::Function, v::Vector{AF}, Δ::AF) = diff_quotient.(f, v, Δ)
end

# ╔═╡ 5618fb40-8853-11eb-3d2f-032437d22a09
md"""
The derivative is the slope of the tangent line at (x, f (x)) , while the difference quotient is the slope of the almost-tangent line that runs through (x + Δ, f (x + Δ)) . As Δ gets tends to 0,, the almost-tangent line gets closer and closer to the tangent line.
"""

# ╔═╡ 55fab426-8853-11eb-01f8-cbde45c7de9c
# TODO figure

# ╔═╡ 55e3efea-8853-11eb-0821-c5c580bc8fc5
md"""
For many function it is easy to compute the exact derivatives. For example for the square function we have:
"""

# ╔═╡ 55c56cc8-8853-11eb-04f4-3be84a352808
begin
  square(x::AF) = x * x
  square(v::Vector{AF}) = square.(v)

  deriv_square(x::AF) = 2. * x
  deriv_square(v::VT) = deriv_square.(v)
  end

# ╔═╡ d25d8eac-884c-11eb-2063-27413f8aba04
md"""
What if we cannot? Weel we can estimate the derivative by evaluating the difference quotient for very small Δ. For example:
"""

# ╔═╡ 480f7b18-8854-11eb-2898-8d876317ebe2
begin
  xᵣ = -10.:10.
  Δ = 1e-3

  yᵣ = square.(collect(xᵣ));
  dyᵣ = deriv_square.(xᵣ);
  dŷᵣ = diff_quotient.(square, xᵣ, Δ);
end

# ╔═╡ 47ee8cb4-8854-11eb-252b-852fd56aa40f
begin
  plot(xᵣ, yᵣ, label="Square", marker=:o, color=:green)
  scatter!(xᵣ, dyᵣ, label="Actual", marker=:o, color=:salmon)
  scatter!(xᵣ, dŷᵣ, label="Estimate", marker=:x, color=:darkslateblue,
    markersize=5, title="Actual derivatives vs Estimates")
end

# ╔═╡ 47d2c2cc-8854-11eb-35f9-952876481d02
md"""
When a function has sevral variables, it has multiple *partila derivatives* each indicating how f change when a small change is made in one of the input variable (while keeping the rest constant).
"""

# ╔═╡ f5980b54-8856-11eb-3e30-89c3f188ca8d
begin
  function ∂_diff_quotient(f::Function, v::VT, i::Integer, Δ::AF)::AF
    """
    Returns the i-th partial difference quotient of f at v
    """
    w = copy(v)
    w[i] += Δ
    (f.(w...) - f.(v...)) / Δ
  end

  ∂_diff_quotient(f::Function, v::Vector{AF}, Δ::AF)::AF =
    ∂_diff_quotient.(f, Tuple(v), 1:length(v), Δ)
end

# ╔═╡ f57b16b6-8856-11eb-3409-47f7b438f491
md"""
With this definition we can then estimate the gradient.
"""

# ╔═╡ f55dd09c-8856-11eb-1823-290fcb200662
function estimate_∇(f::Function, v::VT, Δ::AF)::VT
  [∂_diff_quotient(f, v, i, Δ) for i in 1:length(v)]
end

# ╔═╡ 33df0458-885c-11eb-2635-371a9590d833
begin
  fn(u::AF, v::AF) = u^2 + v

  ∂fnᵤ(u::AF, _v::AF) = 2. * u  # ∂fn w.r.t u
  ∂fnᵥ(_u::AF, v::AF) = 1       # ∂fn w.r.t v
end

# ╔═╡ e76a5b58-885c-11eb-0067-010ba1d4c9b5
begin
  Base.:≈(x::AF, y::AF; ϵ=1e-4) = abs(x - y) ≤ ϵ

  Base.:≈(u::Vector{T}, v::Vector{T}; ϵ=1e-4) where T <: AF = abs.(u - v) .≤ ϵ
end

# ╔═╡ e783d618-8858-11eb-0c0b-b57e54bd3757
begin
  Δ₁ = 1e-6
  vm = Float64[1., 2.]
  ix = 1
  @test ∂fnᵤ(vm...) ≈ ∂_diff_quotient(fn, vm, ix, Δ₁)
end

# ╔═╡ e767c1bc-8858-11eb-1b29-596a25a4fea4
html"""

"""

# ╔═╡ e747b052-8858-11eb-3c50-5355fef542c9
md"""
### Using the Gradient
"""

# ╔═╡ cc17413a-885d-11eb-1629-47fc5220529e
md"""
It is easy to see that the function `f` defines above is smallest when its input is a vector of zeros. But imagine let's say we do not know. In this case we can use  the gradients to find the minimum (this function is bivariate convexe) among all three-dimensional vectors.

We will start by picking a random starting point and then take tiny steps in the opposite direction of the gradient until we reach a point where the gradient is very small (which is guarantee to happen)
"""

# ╔═╡ cbfa193c-885d-11eb-3a48-97b26377b4fb
function ∇_step(v::VT, ∇::VT, η::AF)::VT
  @assert length(v) == length(∇)
  v + η * ∇
end

# ╔═╡ cbe180fe-885d-11eb-1307-7f3ac4897441
∇_sum_of_square(v::VT) = deriv_square(v)

# ╔═╡ d21bb0fe-884c-11eb-224f-216a811365a8
function gradient_descent(fn;range=1000, rng=MersenneTwister(42), η=1e-2)
  v = rand(rng, 3)
  for epoch ∈ 1:range
    ∇ = fn(v)
    v = ∇_step(v, ∇, -η) ## take a step in negative direction of the gradient
    epoch % 100 == 0 && println("$(epoch) => v: $(v)")
  end
  v
end

# ╔═╡ de2e6604-8863-11eb-3a27-8f8e8b55e5d8
with_terminal() do
  gradient_descent(∇_sum_of_square)
end

# ╔═╡ 3082d12e-8864-11eb-06bc-cdf9202cd20a
begin
  @test all(Base.:≈(gradient_descent(∇_sum_of_square), zeros(Float64, 3); ϵ=1.e-7))
  @test all(gradient_descent(∇_sum_of_square) ≈ zeros(Float64, 3))
end9.93698

-1.0823

# ╔═╡ 306b8d52-8864-11eb-1cbd-1341afcd52ff
html"""

"""

# ╔═╡ 304ff894-8864-11eb-1449-b71da9112247
md"""
### Choosing the right step

It is clear why we choose to move in the opposite direction of the gradient while minimizing an (objective) function. Determining the right step however is something like an art and its value depends on the context.
  - Too large a step size and we are likely to overshoot and not converge to the optimum.
  - Too small a step and it will take ages to converge...

Not easy. There are mutliple options like fixed step size, gradual shrinking of the step size over time and other fancier options... To select the right step (or rnage of steps) one can try different options on a validation set...
"""

# ╔═╡ 8c4d28fc-884a-11eb-0de2-b58988dc7422
html"""

"""

# ╔═╡ 8a07d906-8869-11eb-0bae-2582b352c05c
md"""
### Using Gradient Descent (GD) to fit models

In this collection of notebooks, we will be using gradient descent to fit parameterized models to data.

The usual case involves some dataset which we try to fit with a model (our hypothesis) that is characterized by some differentiable parameters. Also to measure the quality of our progress we will have a loss function.

Taking the data as being fixed, the loss function tells us how good (or bad) our model parameters are. We can use gradient descent to optimize this loss.

Let us try on a toy example.
"""

# ╔═╡ 89ebdac6-8869-11eb-25ac-2dac6a4b20aa
begin
  rng = MersenneTwister(42)
  data = [[x, 10x + 5. + 40. * randn(rng, 1)[1]] for x ∈ -50.:50.]
  # linear pattern with some (Normally distributed) noise
end

# ╔═╡ 89d1728a-8869-11eb-0001-875c2a452cf8
scatter(map(x -> x[1], data), map(y -> y[2], data), legend=false)

# ╔═╡ c0cc67c8-888b-11eb-272c-05a727c634ea
typeof(data)

# ╔═╡ 4bf3d986-886d-11eb-3ea8-7926f5babce0
function linear_∇(x::AF, y::AF, θ::VT)::VT
  slope, intercept = θ
  ŷ = slope * x + intercept   # prediction model
  Δerr = (ŷ - y)
  # squared_Δerr = Δerr^2
  [2. * Δerr * x, 2. * Δerr]
end

# ╔═╡ d8f972a0-8881-11eb-2be8-db89fb95a9f1
md"""
We are going to:
  1. Start with random initialisation of the parameters (θ)
  1. Compute the mean of the gradient
  1. Adjust θ
  1. Repeat
"""

# ╔═╡ 1b26337a-8882-11eb-2439-e106cdb884f5
function ∇_descent(inputs::Vector{Vector{T}};
    η=1e-5, nepochs=5_000) where T <: AF
  u = Uniform(-1., 1.)
  θ = [rand(u) for _ ∈ 1:2]

  for epoch ∈ 1:nepochs
    ∇ = μ([linear_∇(x, y, θ) for (x, y) ∈ inputs])
    θ = ∇_step(θ, ∇, -η)
    epoch % 100 == 0 && println("$(epoch) => θ: $(θ)")
  end

  (slope, intercept) = θ
end

# ╔═╡ 1b09cc9e-8882-11eb-0efc-630028d87217
begin
  slope, intercept = ∇_descent(data; nepochs=1_000)
  @test 9.90 ≤ slope ≤ 10.1
  @test abs(intercept) ≤ 1.0
end

# ╔═╡ 28b44b3c-88f8-11eb-23dd-bbfe325d70d3
md"""
Let's plot our result:
"""

# ╔═╡ 1c89f188-88f6-11eb-026c-e59bb08ac7ac
begin
  x_ = [-50.0, 50.0]
  y_ = slope .* x_ .+ intercept
  #
  scatter(map(x -> x[1], data), map(y -> y[2], data), legend=false)
  plot!(x_, y_, lw=3, color=:darkred)
end

# ╔═╡ 1ad358e4-8882-11eb-0559-bd43867dacfd
html"""

"""

# ╔═╡ d8d3bb0a-8881-11eb-35b2-8f486d6cc094
md"""
### Mini-batch and Stochastic Gradient Descent

In the preceding approach we evaluated the gradient on the whole dataset. This was fine because our dataset was small. In practice datasets can be big/large and this technique  becomes computationally expensive. This is why other techniques were devised, namely stochastic gradient descent and mini-batch gradient descent (a good trade-off between full gradient descent and stochastic gradient descent).

In mini-batch gradient descent, we compute the gradient and take a  gradient step based on a mini-batch sampled from the dataset:
"""

# ╔═╡ cd7c444a-8893-11eb-2409-f161ebc287b5
begin
	"""
	Generates batch-size sized min-batches from given dataset
	"""

	struct MiniBatch{T <: Any}
		ds::Vector{Vector{T}}
		bsize::Integer  # batch-size

		function MiniBatch{T}(ds::Vector{Vector{T}}, bs::Integer) where T <: Any
			@assert 0 < bs ≤ length(ds)
			new(ds, bs)
    	end
  	end

  	Base.collect(iter::MiniBatch) = iter.ds

	function Base.iterate(iter::MiniBatch,
			state=(view(iter.ds, 1:iter.bsize), iter.bsize + 1))
		batch, ix = state
		##
		isnothing(batch) && return nothing
		ix ≥ length(iter.ds) && return (batch, (nothing, ix))
		## otherwise
		jx = ix
		ix += iter.bsize
		ix = ix ≥ length(iter.ds) ? length(iter.ds) : ix
		(batch, (view(iter.ds, jx:ix - 1), ix))
	end

	Base.length(iter::MiniBatch) = length(iter.ds)
	Base.eltype(iter::MiniBatch) = eltype(iter.ds)
end

# ╔═╡ 714137c2-8900-11eb-3b4b-57abc5409bd2
let _ix = 0, bsz = 15
	for batch ∈ MiniBatch{Float64}(data, bsz)
		@assert length(batch) == bsz || 0 ≤ length(batch) ≤ bsz
		_ix += 1
	end
	@test _ix == 7
end

# ╔═╡ 7125e906-8900-11eb-3cd5-69755931c32d
let _ix = 0, bsz = 12
	for batch ∈ MiniBatch{Float64}(data, bsz)
		@assert length(batch) == bsz || 0 ≤ length(batch) ≤ bsz
		_ix += 1
	end
	@test _ix == 9
end

# ╔═╡ 401f3d8e-8904-11eb-1065-1de6c9bf32be
md"""
let's rewrite our gradient(∇) descent with min-batch:
"""

# ╔═╡ 70eef976-8900-11eb-3a03-af04f8dea9cc
function minibatch_∇_descent(inputs::Vector{Vector{T}};
    η=1e-5, nepochs=5_000, bsize=10) where T <: AF
  u = Uniform(-1., 1.)
  θ = [rand(u) for _ ∈ 1:2]

  for epoch ∈ 1:nepochs
		for batch ∈ MiniBatch{eltype(inputs[1][1])}(inputs, bsize)
			∇ = μ([linear_∇(x, y, θ) for (x, y) ∈ inputs])
    		θ = ∇_step(θ, ∇, -η)
		end
    epoch % 100 == 0 && println("$(epoch) => θ: $(θ)")
  end

  (slope, intercept) = θ
end

# ╔═╡ 70d67f74-8900-11eb-0174-f5c05571848e
begin
  slope₂, intercept₂ = minibatch_∇_descent(data; nepochs=100) ## only 100 epochs
  @test 9.90 ≤ slope₂ ≤ 10.1
  @test abs(intercept₂) ≤ 1.0
end

# ╔═╡ 8d33a0f4-8906-11eb-3690-ddd49fc6cee1
begin
  # x_₂ = [-50.0, 50.0]
  y_₂ = slope .* x_ .+ intercept
  #
  scatter(map(x -> x[1], data), map(y -> y[2], data), legend=false)
  plot!(x_, y_₂, lw=3, color=:darkred)
end

# ╔═╡ 0a1acf66-8907-11eb-3eac-59a3e18d9c43
md"""
In this instance our mini-batch gradient descent worked pretty well, finding the optimal parameters in just 100 iteration (and actually less)...

Basing gradient step on a small min-batch or just on one sampel (in the case of stochastic gradient descent) allows to take more steps, but the gradient can fluctuate a lot...
"""

# ╔═╡ Cell order:
# ╟─380c26d0-881d-11eb-0461-f339488ea81e
# ╠═e03dd22c-884c-11eb-2432-3dc04b0cf286
# ╟─83e01f80-884a-11eb-1ef7-e9cb5ab3ed20
# ╟─853ab02c-884a-11eb-01bf-1712dc177ffb
# ╟─4c951444-8869-11eb-0b88-55a9fb9e51e6
# ╟─8c87c3b8-884a-11eb-10c8-73e2dab560ea
# ╟─8c6ba818-884a-11eb-23e0-2f1c297d88f5
# ╠═d2d18b22-884c-11eb-1f93-037453c5395b
# ╟─221734fe-8869-11eb-331a-b5810d0b784c
# ╟─d297fa06-884c-11eb-294e-85992f901be1
# ╠═1859c082-8853-11eb-18f6-2bc7a180b4a2
# ╠═d27feefa-884c-11eb-3b84-1f529d7f6410
# ╟─5618fb40-8853-11eb-3d2f-032437d22a09
# ╠═55fab426-8853-11eb-01f8-cbde45c7de9c
# ╟─55e3efea-8853-11eb-0821-c5c580bc8fc5
# ╠═55c56cc8-8853-11eb-04f4-3be84a352808
# ╟─d25d8eac-884c-11eb-2063-27413f8aba04
# ╠═480f7b18-8854-11eb-2898-8d876317ebe2
# ╠═47ee8cb4-8854-11eb-252b-852fd56aa40f
# ╟─47d2c2cc-8854-11eb-35f9-952876481d02
# ╠═f5980b54-8856-11eb-3e30-89c3f188ca8d
# ╟─f57b16b6-8856-11eb-3409-47f7b438f491
# ╠═f55dd09c-8856-11eb-1823-290fcb200662
# ╠═33df0458-885c-11eb-2635-371a9590d833
# ╠═e76a5b58-885c-11eb-0067-010ba1d4c9b5
# ╠═e783d618-8858-11eb-0c0b-b57e54bd3757
# ╟─e767c1bc-8858-11eb-1b29-596a25a4fea4
# ╟─e747b052-8858-11eb-3c50-5355fef542c9
# ╟─cc17413a-885d-11eb-1629-47fc5220529e
# ╠═cbfa193c-885d-11eb-3a48-97b26377b4fb
# ╠═cbe180fe-885d-11eb-1307-7f3ac4897441
# ╠═d21bb0fe-884c-11eb-224f-216a811365a8
# ╠═de2e6604-8863-11eb-3a27-8f8e8b55e5d8
# ╠═3082d12e-8864-11eb-06bc-cdf9202cd20a
# ╟─306b8d52-8864-11eb-1cbd-1341afcd52ff
# ╟─304ff894-8864-11eb-1449-b71da9112247
# ╟─8c4d28fc-884a-11eb-0de2-b58988dc7422
# ╟─8a07d906-8869-11eb-0bae-2582b352c05c
# ╠═89ebdac6-8869-11eb-25ac-2dac6a4b20aa
# ╠═89d1728a-8869-11eb-0001-875c2a452cf8
# ╠═c0cc67c8-888b-11eb-272c-05a727c634ea
# ╠═4bf3d986-886d-11eb-3ea8-7926f5babce0
# ╟─d8f972a0-8881-11eb-2be8-db89fb95a9f1
# ╠═1b26337a-8882-11eb-2439-e106cdb884f5
# ╠═1b09cc9e-8882-11eb-0efc-630028d87217
# ╟─28b44b3c-88f8-11eb-23dd-bbfe325d70d3
# ╠═1c89f188-88f6-11eb-026c-e59bb08ac7ac
# ╟─1ad358e4-8882-11eb-0559-bd43867dacfd
# ╟─d8d3bb0a-8881-11eb-35b2-8f486d6cc094
# ╠═cd7c444a-8893-11eb-2409-f161ebc287b5
# ╠═714137c2-8900-11eb-3b4b-57abc5409bd2
# ╠═7125e906-8900-11eb-3cd5-69755931c32d
# ╟─401f3d8e-8904-11eb-1065-1de6c9bf32be
# ╠═70eef976-8900-11eb-3a03-af04f8dea9cc
# ╠═70d67f74-8900-11eb-0174-f5c05571848e
# ╠═8d33a0f4-8906-11eb-3690-ddd49fc6cee1
# ╟─0a1acf66-8907-11eb-3eac-59a3e18d9c43

Data Science From Scratch/08_Gradient_Descent.jl


#=
# 3. Applying pulse forcing to a flow
Here, we will show how to apply a transient force (a 'pulse') to a flow.
=#

using ViscousFlow
#-
using Plots

#=
For this case, we first seek to introduce a pulse to an otherwise quiescent fluid,
with no boundaries. The pulse shape will be a smooth dipole, directed upward.

We will start with the usual steps: specify the problem parameters and the discretization.
=#

Re = 200
xlim, ylim = (-2,2), (-2,4)
Δx,Δt = setstepsizes(Re,gridRe=4)

#=
### Construct the pulse
Now create the pulse. For the spatial shape of this pulse, we make use of a Gaussian
field, differentiated once in the `x` direction. We will center it at (0,0), and
give it a strength of 10.
=#
σx = 0.5
σy = 0.1
gauss = SpatialGaussian(σx,σy,0,0,10,deriv=1);

#=
Now we now 'shape' the pulse in time, also with a Gaussian. We center it at `t = 0.1`
with a half-width in time equal to 0.1.
=#
t0 = 0.1
σt = 0.1
pparams = PulseParams(gauss,t0,σt);

#=
### Construct the system structure
We supply these pulse characteristics with the keyword argument `pulses` as we
set up the usual system structure:
=#
sys = NavierStokes(Re,Δx,xlim,ylim,Δt,pulses=pparams)

#=
The remaining steps go just as they did for the previous examples. We will simulate
the pulse for 5 time units and make an animation of the results:
=#
u0 = newstate(sys)
tspan = (0.0,10.0) # longer than we need, but just being safe
integrator = init(u0,tspan,sys)
#-
step!(integrator,5.0)

#=
### Examine
=#
sol = integrator.sol
@gif for t in sol.t
    plot(vorticity(sol,sys,t),sys)
end every 5
#-
plot(streamfunction(integrator),sys,title="Streamfunction at t = $(round(integrator.t,digits=2))")

#=
We can also supply more than one pulse. Let's give 3, each separated by 1 time unit.
=#
pparams = [PulseParams(gauss,0.1,0.1), PulseParams(gauss,1.1,0.1), PulseParams(gauss,2.1,0.1)];
#-
sys = NavierStokes(Re,Δx,xlim,ylim,Δt,pulses=pparams)
#-
u0 = newstate(sys)
tspan = (0.0,10.0)
integrator = init(u0,tspan,sys)
#-
step!(integrator,4.0)
#=
### Examine
=#
sol = integrator.sol
@gif for t in sol.t
    plot(vorticity(sol,sys,t),sys)
end every 5


# cubic L2 norm (times a constant

export CubeNormL2

"""
**Cubic Euclidean norm (weighted)**

    CubeNormL2(λ=1.0)

With a nonnegative scalar `λ`, returns the function
```math
f(x) = λ\\|x\\|^3.
```
"""
struct CubeNormL2{R <: Real} <: ProximableFunction
    lambda::R
    function CubeNormL2{R}(lambda::R) where R
        if lambda < 0
            error("coefficient λ must be nonnegative")
        else
            new(lambda)
        end
    end
end

is_convex(f::CubeNormL2) = true
is_smooth(f::CubeNormL2) = true

CubeNormL2(lambda::R=1.0) where {R <: Real} = CubeNormL2{R}(lambda)

function (f::CubeNormL2{R})(x::AbstractArray{T}) where {R, T <: RealOrComplex{R}}
    return f.lambda * norm(x)^3
end

function gradient!(y::AbstractArray{T}, f::CubeNormL2{R}, x::AbstractArray{T}) where {R, T <: RealOrComplex{R}}
    norm_x = norm(x)
    y .= (3 * f.lambda * norm_x) .* x
    return f.lambda * norm_x^3
end

function prox!(y::AbstractArray{T}, f::CubeNormL2{R}, x::AbstractArray{T}, gamma::R=R(1)) where {R, T <: RealOrComplex{R}}
    norm_x = norm(x)
    scale = 2 / (1 + sqrt(1 + 12 * gamma * f.lambda * norm_x))
    y .= scale .* x
    return f.lambda * (scale * norm_x)^3
end

function prox_naive(f::CubeNormL2{R}, x::AbstractArray{T}, gamma=R(1)) where {R, T <: RealOrComplex{R}}
    y = 2 / (1 + sqrt(1 + 12 * gamma * f.lambda * norm(x))) * x
    return y, f.lambda * norm(y)^3
end


# Triangulations of point sets
# ===========================
#
#
# These examples can be loaded into Julia (Revise.jl recommended)
# and run by calling one of the methods with the optional arguments "Plotter=PyPlot".
# Alternatively, you can download a [jupyter notebook](example_pointsets.ipynb) created
# from this source.
#
#
#---
#
# Set  up environment

using Triangulate
using Test

injupyter()=(isdefined(Main, :IJulia) && Main.IJulia.inited)

if injupyter()
    import PyPlot
end

# ### Delaunay triangulation of point set
#
# Create a set of random points in the plane and calculate
# the Delaunay triangulation of this set of points. It is a triangulation
# where for each triangle, the interior of its circumcircle  does not contain any points
# of the trianglation.
#
# The Delaunay triangulation of a set of points in general position
# (no 4 of them on a circle) is unique. At the same time, it is a
# triangulation of the convex hull of these points.
#
# Given an input list of points, without any further flags, Triangle creates
# just this triangulation (the "Q" flag suppresses the text output of Triangle).
# For this and the next examples, the input list of points is created randomly,
# but on a raster, preventing the appearance of too close points.
#
function example_convex_hull(;Plotter=nothing,n=10,raster=10)
    triin=Triangulate.TriangulateIO()
    triin.pointlist=hcat(unique([ Cdouble[rand(1:raster)/raster, rand(1:raster)/raster] for i in 1:n])...)
    display(triin)
    (triout, vorout)=triangulate("Q", triin)
    display(triout)
    plot_in_out(Plotter,triin,triout,title="Convex hull")
    @test numberofpoints(triin)==numberofpoints(triout)
    @test numberoftriangles(triout)>0
end
#
injupyter()&& example_convex_hull(Plotter=PyPlot,n=10,raster=10);

# ### Delaunay triangulation of point set with boundary
#
# Same as the previous example, but in addition specify the "c" flag
# In this case, Triangle outputs an additional list of segments
# describing the boundary of the convex hull. In fact this is a constrained
# Delaunay triangulation (CDT) where the boundary segments
# are the seen constraining edges which must appear in the output.
function example_convex_hull_with_boundary(;Plotter=nothing,n=10,raster=10)
    triin=Triangulate.TriangulateIO()
    triin.pointlist=hcat(unique([ Cdouble[rand(1:raster)/raster, rand(1:raster)/raster] for i in 1:n])...)
    display(triin)
    (triout, vorout)=triangulate("cQ", triin)
    display(triout)
    @test numberofpoints(triin)==numberofpoints(triout)
    @test numberoftriangles(triout)>0
    @test numberofsegments(triout)>0
    plot_in_out(Plotter,triin,triout,title="Convex hull with boundary")
end
#
injupyter()&&  example_convex_hull_with_boundary(Plotter=PyPlot,n=10,raster=10);

# ### Delaunay triangulation of point set with Voronoi diagram
#
# Same as the previous example, but instead of "c" specify the "v" flag
# In this case, Triangle outputs information about the Voronoi diagram
# of the point set which is a structure dual to  the Delaunay triangulation.
#
# The Voronoi cell around a point $p$ a point set $S$ is defined
# as the set of points $x$ such that $|x-p|<|x-q|$ for all $q\in S$
# such that $p\neq q$.
# The Voronoi cells of boundary points
# of the convex hull are of infinite size. The corners of the Voronoi
# cells are the circumcenters of the triangles. They can be far
# outside of the triangulated domain.
function example_convex_hull_voronoi(;Plotter=nothing,n=10,raster=10)
    triin=Triangulate.TriangulateIO()
    triin.pointlist=hcat(unique([ Cdouble[rand(1:raster)/raster, rand(1:raster)/raster] for i in 1:n])...)
    display(triin)
    (triout, vorout)=triangulate("vQ", triin)
    display(triout)
    display(vorout)
    plot_in_out(Plotter,triin,triout,voronoi=vorout,title="Convex hull with Voronoi diagram")
    @test numberofpoints(triin)==numberofpoints(triout)
    @test numberoftriangles(triout)>0
    @test numberofpoints(vorout)>0
end
#
injupyter()&&  example_convex_hull_voronoi(Plotter=PyPlot,n=10,raster=10);

# ### Boundary conforming Delaunay triangulation of point set
# Specify "c" flag for convex hull segments, "v" flag for Voronoi
# and "D" flag for creating a boundary conforming Delaunay triangulation of
# the point set. In this case additional points are created which split
# the boundary segments and ensure that all triangle circumcenters
# lie within the convex hull.
# Due to random input, there may be situations where Triangle fails with this task,
# so we check for the corresponding exception.
function example_convex_hull_voronoi_delaunay(;Plotter=nothing,n=10,raster=10)
    triin=Triangulate.TriangulateIO()
    triin.pointlist=hcat(unique([ Cdouble[rand(1:raster)/raster, rand(1:raster)/raster] for i in 1:n])...)
    display(triin)
    try
        (triout, vorout)=triangulate("vcDQ", triin)
        plot_in_out(Plotter,triin,triout,voronoi=vorout,title="Convex hull with Voronoi diagram")
        display(triout)
        display(vorout)
        @test numberofpoints(triin)<=numberofpoints(triout)
        @test numberoftriangles(triout)>0
        @test numberofpoints(vorout)>0
    catch err
        if typeof(err)==TriangulateError
            println("Triangle had some problem.")
            return true
        end
    end
end
#
injupyter()&&  example_convex_hull_voronoi_delaunay(Plotter=PyPlot,n=10,raster=10);

# ### Constrained Delaunay triangulation (CDT) of a point set with edges
# Constrained Delaunay triangulation (CDT) of a point set with
# additional constraints given a priori. This is obtained when
# specifying the "p" flag and an additional list of segments each described
# by two points
# which should become edges of the triangulation. Note that
# the resulting triangulation is not Delaunay in the sense
# given above.
function example_cdt(;Plotter=nothing,n=10,raster=10)
    triin=Triangulate.TriangulateIO()
    triin.pointlist=hcat(unique([ Cdouble[rand(1:raster)/raster, rand(1:raster)/raster] for i in 1:n])...)
    npt=size(triin.pointlist,2)
    triin.segmentlist=Matrix{Cint}([1 2; npt-1 npt-2;  1 npt;]')
    triin.segmentmarkerlist=Vector{Cint}([2,3,4])
    display(triin)
    (triout, vorout)=triangulate("pcQ", triin)
    display(triout)
    plot_in_out(Plotter,triin,triout,title="CDT")
    @test numberofpoints(triin)<=numberofpoints(triout)
    @test numberofsegments(triout)>=numberofsegments(triin)
    @test numberoftriangles(triout)>0
end
#
injupyter()&&  example_cdt(Plotter=PyPlot,n=10,raster=10);


# ---
# title: 713. Subarray Product Less Than K
# id: problem713
# author: Indigo
# date: 2021-06-29
# difficulty: Medium
# categories: Array, Two Pointers
# link: <https://leetcode.com/problems/subarray-product-less-than-k/description/>
# hidden: true
# ---
#
# Your are given an array of positive integers `nums`.
#
# Count and print the number of (contiguous) subarrays where the product of all
# the elements in the subarray is less than `k`.
#
# **Example 1:**
#
#
#
#     Input: nums = [10, 5, 2, 6], k = 100
#     Output: 8
#     Explanation: The 8 subarrays that have product less than 100 are: [10], [5], [2], [6], [10, 5], [5, 2], [2, 6], [5, 2, 6].
#     Note that [10, 5, 2] is not included as the product of 100 is not strictly less than k.
#
#
# **Note:**
#
# * `0 < nums.length <= 50000`.
# * `0 < nums[i] < 1000`.
# * `0 <= k < 10^6`.
#
#
## @lc code=start
using LeetCode

function num_subarray_productLess_than_k(nums::Vector{Int}, k::Int)
    k <= 1 && return 0
    res, left = 0, 1
    p = 1
    for (right, val) in enumerate(nums)
        p *= val
        while p >= k
            p ÷= nums[left]
            left += 1
        end
        res += right - left + 1
    end
    return res
end
## @lc code=end


export Sum

"""
**Sum of functions**

    Sum(f₁,…,fₖ)

Given functions `f₁` to `fₖ`, returns their sum

```math
g(x) = ∑_{i=1}^k f_i(x).
```

The functions `fᵢ` should all have the same domain, which will be the domain of
the result. If all `fᵢ` have gradients then this function will also have a
gradient.

"""
struct Sum{T <: Tuple} <: ProximableFunction fs::T end

Sum(fs::Vararg{ProximableFunction}) = Sum((fs...,))

# note: is_prox_accurate false because prox in general doesn't exist?
is_prox_accurate(f::Sum) = false
is_convex(f::Sum) = all(is_convex.(f.fs))
is_set(f::Sum) = all(is_set.(f.fs))
is_cone(f::Sum) = all(is_cone.(f.fs))
is_affine(f::Sum) = all(is_affine.(f.fs))
is_smooth(f::Sum) = all(is_smooth.(f.fs))
is_quadratic(f::Sum) = all(is_quadratic.(f.fs))
is_generalized_quadratic(f::Sum) = all(is_generalized_quadratic.(f.fs))
is_strongly_convex(f::Sum) = all(is_convex.(f.fs)) && any(is_strongly_convex.(f.fs))

function (sumobj::Sum)(x::AbstractArray{T}) where {R <: Real, T <: Union{R, Complex{R}}}
    sum = zero(R)
    for f in sumobj.fs
        sum += f(x)
    end
    sum
end

function gradient!(grad::AbstractArray{T}, sumobj::Sum, x::AbstractArray{T}) where {R <: Real, T <: Union{R, Complex{R}}}
    # gradient of sum is sum of gradients
    val = zero(R)
    # to keep track of this sum, i may not be able to
    # avoid allocating an array
    grad .= zero(T)
    temp = similar(grad)
    for f in sumobj.fs
        val += gradient!(temp, f, x)
        grad .+= temp
    end
    return val
end

fun_name(f::Sum) = "sum"
fun_dom(f::Sum) = fun_dom(f.fs[1]) # check to make sure same??
fun_expr(f::Sum) = "x ↦ f₁(x) + … + fₖ(x)"
fun_params(f::Sum) = "n/a"


### A Pluto.jl notebook ###
# v0.12.7

using Markdown
using InteractiveUtils

# ╔═╡ 6fd616f6-1ea1-11eb-3814-8bfb4a096c49
using Pkg, DrWatson

# ╔═╡ 6ffe2628-1ea1-11eb-24ea-57f985146a72
begin
	@quickactivate "StatisticsWithJuliaPlutoNotebooks"
	using Distributions, Statistics, Random, StatsBase, DataFrames, Plots
	Random.seed!(1)
end;

# ╔═╡ ed174bc4-1ea0-11eb-1e2f-a32874cec549
md"## Listing 3.7"

# ╔═╡ 70179b62-1ea1-11eb-13ce-7fb4d54bad4b
begin
	dist1 = TriangularDist(0,1,1)
	dist2 = TriangularDist(0,1,0)
	N=10^6

	data1, data2 = rand(dist1,N), rand(dist2,N)
	dataSum = data1 + data2
end

# ╔═╡ 8f60b2f6-20f9-11eb-0a66-771e655035a3
mgf(s) = 4(1+(s-1)*MathConstants.e^s)*(MathConstants.e^s-1-s)/s^4

# ╔═╡ 8f61044a-20f9-11eb-1485-d145145c4c57
mgfPointEst(s) = mean([MathConstants.e^(s*z) for z in
					rand(dist1,20) + rand(dist2,20)])

# ╔═╡ 8f6181a4-20f9-11eb-3eae-6dc331b9a88a
begin
	p1 = histogram(dataSum, bins=80, normed=:true,
		ylims=(0,1.4), xlabel="z", ylabel="PDF")

	sGrid = -1:0.01:1
	p2 = plot(sGrid, mgfPointEst.(sGrid), c=:blue, ylims=(0,3.5))
	p2 = plot!(sGrid, mgf.(sGrid), c=:red)
	p2 = plot!( [minimum(sGrid),maximum(sGrid)],
		[minimum(sGrid),maximum(sGrid)].+1,
		c=:black, xlabel="s", ylabel="MGF")

	plot(p1, p2, legend=:none, size=(800, 400))
end

# ╔═╡ 70182cd8-1ea1-11eb-094d-8d0d49cf15f3
md"## End of listing 3.7"

# ╔═╡ Cell order:
# ╟─ed174bc4-1ea0-11eb-1e2f-a32874cec549
# ╠═6fd616f6-1ea1-11eb-3814-8bfb4a096c49
# ╠═6ffe2628-1ea1-11eb-24ea-57f985146a72
# ╠═70179b62-1ea1-11eb-13ce-7fb4d54bad4b
# ╠═8f60b2f6-20f9-11eb-0a66-771e655035a3
# ╠═8f61044a-20f9-11eb-1485-d145145c4c57
# ╠═8f6181a4-20f9-11eb-3eae-6dc331b9a88a
# ╟─70182cd8-1ea1-11eb-094d-8d0d49cf15f3


@doc raw"""
	quantiledep(x,y,q)

Quantile Dependence at the `q`-th quantile of vector-like data `x` and `y`.

Let ``q_x`` and ``q_y`` be the `q`-th quantiles of `x` and `y`.

Returns ``E[xy | x \leq q_x, y \leq q_y] / q`` if `q<=0.5` and
``(1 -2q + E[xy | x \leq q_x, y \leq q_y]) / (1-q)`` otherwise.
"""
function quantiledep(x,y,q)
    qx = quantile(x,q)
    qy = quantile(y,q)
    tmp = mean((x.<=qx).*(y.<=qy))
    if q <= 0.5
        return tmp/q
    else
        return (1-2*q + tmp)/(1-q)
    end
end

"""
	exceedancecor(x,y,c;kind=:both,side=:lower)

Computes exceedance correlations between `x` and `y`, using exceedance threshold `c`.
Keyword `kind` refers to which type of conditioning: `:both` conditions on `x` and `y` simultaneously,
while `:onx` conditions only on `x`. Keyword `side` refers to look below (`:lower`) or above (`:upper`)
the threshold `c`. Inputs `x` and `y` are standardized before conditioning,
so `c` should be in standard deviation units.
"""
function exceedancecor(x,y,c;kind=:both,side=:lower)
    xs = standardize(x)
	ys = standardize(y)
    if side==:lower
        if kind==:both
            loc = (xs .<= c) .& (ys .<= c)
        elseif kind==:onx
            loc = (xs .<= c)
        end
    elseif side==:upper
        if kind==:both
            loc = (xs .>= c) .& (ys .>= c)
        elseif kind==:onx
            loc = (xs .>= c)
        end
    end
	return cor(xs[loc],ys[loc])
end

"""
	downsidebeta(x,y;kind=:both)

Computes the downside beta of `y` on `x` as defined in Ang & Chen.

`kind=:onx` allows to condition only on `x`.
"""
function downsidebeta(x,y;kind=:both)
	mux = mean(x)
	muy = mean(y)
	if kind==:both
		loc = (x .<= mux) .& (y .<= muy)
	elseif kind==:onx
		loc = (x .<= mux)
	end
	xtmp = x[loc]
	ytmp = y[loc]
	varx = var(xtmp)
	covxy = cov(xtmp,ytmp)
	return covxy/varx
end

"""
	upsidebeta(x,y;kind=:both)

Computes the upside beta of `y` on `x` as defined in Ang & Chen.

`kind=:onx` allows to condition only on `x`.
"""
function upsidebeta(x,y;kind=:both)
	mux = mean(x)
	muy = mean(y)
	if kind==:both
		loc = (x .>= mux) .& (y .>= muy)
	elseif kind==:onx
		loc = (x .>= mux)
	end
	xtmp = x[loc]
	ytmp = y[loc]
	varx = var(xtmp)
	covxy = cov(xtmp,ytmp)
	return covxy/varx
end

# x here is "the market return"

@doc raw"""
	coskewness(x,y)

Computes the coskewness between vectors `x` and `y`. The vector `x` receives the
powers in the expression.

``\frac{E[(x - E[x])^2 (y-E[y])]}{\sigma_x^2 \sigma_y}``

"""
function coskewness(x,y)
    dmx = x .- mean(x)
    dmy = y .- mean(y)
    top = mean( dmy .* (dmx.^2))
    bot = std(dmy) * var(dmx)
    return top/bot
end

@doc raw"""
	cokurtosis(x,y;kind::Symbol=:asymmetric)

Computes the cokurtosis between vectors `x` and `y`. The default is asymmetric cokurtosis.

`kind=:asymmetric`:
``\frac{E[(x - E[x])^3 (y-E[y])]}{\sigma_x^3 \sigma_y}``

`kind=:symmetric`:
``\frac{E[(x - E[x])^2 (y-E[y])^2]}{\sigma_x^2 \sigma_y^2}``
"""
function cokurtosis(x,y;kind::Symbol=:asymmetric)
    dmx = x .- mean(x)
    dmy = y .- mean(y)
    if kind == :symmetric
        top = mean( (dmy.^2) .* (dmx.^2))
        bot = var(dmy) * var(dmx)
    else
        top = mean( dmy .* (dmx.^3))
        bot = std(dmy)  * std(dmx)^3
    end
    return top/bot
end

#### TODO
# - better API

#### Correlation-based (Ang & Chen 2001 + statistical test of Hong, Tu, & Zhou 2007 + one-sided as in Schreindorfer 2019)
function downsidecor(x,y)
    qs = 0.15:0.025:1
    out = zeros(length(qs))
    for (i,qi) in enumerate(qs)
        qxi = quantile(x,qi)
        loc = x .<= qxi
        tmpxi = x[loc]
        tmpyi = y[loc]
        out[i] = cor(tmpxi,tmpyi)
        #println([length(tmpxi) length(tmpyi) length(tmpxj) length(tmpyj)])
    end
    return out
end

function upsidecor(x,y)
    qs = 0:0.025:0.85
    out = zeros(length(qs))
    for (i,qi) in enumerate(qs)
        qxi = quantile(x,qi)
        loc = x .>= qxi
        tmpxi = x[loc]
        tmpyi = y[loc]
        out[i] = cor(tmpxi,tmpyi)
        #println([length(tmpxi) length(tmpyi) length(tmpxj) length(tmpyj)])
    end
    return out
end

##### Asymmetry test from Guofu Zhou et al.
#standardize(x) = (x .- mean(x)) ./ std(x)
bartlett(z) = abs(z) < 1 ? (1-abs(z)) : zero(eltype(z))
# this only does 1 value of c
function buildEC(x,y,c)
	T = length(x)
	xs = standardize(x)
	ys = standardize(y)
	iL = (xs .< -c) #.& (ys .< -c)
	iU = (xs .> c) #.& (ys .> c)
	lower_xs = xs[iL]
	upper_xs = xs[iU]
	lower_ys = ys[iL]
	upper_ys = ys[iU]
	Tl = length(lower_xs)
	Tu = length(upper_xs)
	lower_cor = cor(lower_xs,lower_ys)
	upper_cor = cor(upper_xs,upper_ys)

	# standardize again
	xs[iL] = standardize(lower_xs)
	ys[iL] = standardize(lower_ys)
	xs[iU] = standardize(upper_xs)
	ys[iU] = standardize(upper_ys)

	EC = (((T-Tu)/T) .* (xs.*ys .- upper_cor) .* iU) .- (((T-Tl)/T) .* (xs.*ys .- lower_cor) .* iL)
	return (EC=EC,corL=lower_cor,corU=upper_cor)
end

# this wraps it together
# conditioning is standardized x .< -c[i] vs standardized x .> c[i]
# p represents some sort of bandwith? -- lag order?
function asymmetrictest(x,y,c,p)
	T = length(x)
	Nc = length(c)

	if Nc == 1
		outs=buildEC(x,y,c)
		ECmat=outs.EC
		uppercors = outs.corU
		lowercors = outs.corL
		S = 0.0
		for L=-p:p
			w = bartlett(L/p)
			g_l = 0.0
			for t=(abs(L)+1):T
				g_l += ECmat[t]*ECmat[t-abs(L)]
			end
			S += w*(1/T)*g_l
		end
	else
		ECmat = zeros(eltype(x),T,Nc)
		uppercors = zeros(eltype(x),Nc)
		lowercors = zeros(eltype(x),Nc)
		for i =1:Nc
			outs=buildEC(x,y,c[i])
			ECmat[:,i]=outs.EC
			uppercors[i] = outs.corU
			lowercors[i] = outs.corL
		end
		S = zeros(eltype(x),Nc,Nc)
		for L=-p:p
			w = bartlett(L/p)
			g_l = zeros(eltype(x),Nc,Nc)
			for ic=1:Nc,jc=1:Nc
				for t=(abs(L)+1):T
					g_l[ic,jc] += ECmat[t,ic]*ECmat[t-abs(L),jc]
				end
			end
			S += w*(1/T).*g_l
		end
	end

	J = T * (uppercors - lowercors)' * inv(S) * (uppercors - lowercors)
	pval = pdf(Chisq(Nc),J)
	return (pval=pval,J=J,S=S,corL=lowercors,corU=uppercors,T=T)
end

#### Entropy-based (Jiang, Wu, & Zhou 2018)
quadrantprobabilities(f,c;kwargs...) = (LQP=hcubature(f, [-10;-10], [-c;-c];kwargs...), UQP=hcubature(f, [c;c], [10;10];kwargs...))
function entropy_S(f,c;kwargs...)
    newf(x) = (sqrt(abs(f(x))) - sqrt(abs(f(-x))))^2
    int,se = hcubature(x->newf(x),[c;c],[20;20];kwargs...)
    return (int/2,se)
end
function DOWN_ASY_JWZ(f,c=0;kwargs...)
    lqp,uqp = quadrantprobabilities(f,c;kwargs...)
    S = entropy_S(f,c;kwargs...)
    val = sign(first(lqp)-first(uqp))*first(S)
    return val
end
# FF = kde((standardize(dfnew[!,:GDPg]),standardize(dfnew[!,:NDSg])))
# PF = InterpKDE(FF)
# quadrantprobabilities.(x->pdf(PF,x[1],x[2]),0:0.5:1.5)


### A Pluto.jl notebook ###
# v0.16.0

using Markdown
using InteractiveUtils

# ╔═╡ 1c7e914c-03c9-11eb-11cf-0d9346305a53
begin
	using Pkg
	cd(joinpath(dirname(@__FILE__),".."))
    Pkg.activate(pwd())
	using NLopt
end

# ╔═╡ 1866ec44-03c9-11eb-0a21-416587e4e0e0
# Solve the problem with NLopt
let

	"""
	Function to be minimized (with its gradient)
	"""
	function myfun(x::Vector, grad::Vector)
		if length(grad) > 0
			grad[1] = -1
			grad[2] = -1
		end
		return -x[1] - x[2]
	end

	"""
	Constraints in form c_i(x)  <= 0 (with gradient)
	"""
	function c1(x::Vector, grad::Vector)
		if length(grad) > 0
			grad[1] = 2*x[1]
			grad[2] = -1
		end
		x[1]^2 - x[2]
	end

	function c2(x::Vector, grad::Vector)
		if length(grad) > 0
			grad[1] = 2*x[1]
			grad[2] = 2*x[2]
		end
		x[1]^2 + x[2]^2 - 1
	end

	# optimisation setup
	opt = Opt(:LD_SLSQP, 2)
	opt.lower_bounds = [-Inf, 0.] # only x_2 > 0
	opt.xtol_rel = 1e-4
	opt.min_objective = myfun
	inequality_constraint!(opt, c1, 1e-8)
	inequality_constraint!(opt, c2, 1e-8)

	# solving it.
	(minf,minx,ret) = NLopt.optimize(opt, [0, 0])
	numevals = opt.numevals # the number of function evaluations
	println("got $minf at $minx after $numevals iterations (returned $ret)")
end

# ╔═╡ 2121e5be-03c9-11eb-2b8b-a10cb146216d
ret

# ╔═╡ Cell order:
# ╠═1c7e914c-03c9-11eb-11cf-0d9346305a53
# ╠═1866ec44-03c9-11eb-0a21-416587e4e0e0
# ╠═2121e5be-03c9-11eb-2b8b-a10cb146216d

Exercises/PS04 - Optimisation techniques II.jl


"""
    LogitNormal

* Parametrization
parametrized in \\mu and \\sigma^2

* Score

* Fisher Information

* `time_varying_params` map.

* Default link
"""
LogitNormal

logit(x) = log(x) - log(1-x)

function score!(score_til::Matrix{T}, y::T, ::Type{LogitNormal}, param::Matrix{T}, t::Int) where T
    score_til[t, 1] = (logit(y) - param[t, 1])/param[t, 2]
    score_til[t, 2] = -(0.5/param[t, 2]) * (1 - ((logit(y) - param[t, 1])^2)/param[t, 2])
    return
end

function fisher_information!(aux::AuxiliaryLinAlg{T}, ::Type{LogitNormal}, param::Matrix{T}, t::Int) where T
    aux.fisher[1, 1] = 1/(param[t, 2])
    aux.fisher[2, 2] = 1/(2*(param[t, 2]^2))
    aux.fisher[2, 1] = 0
    aux.fisher[1, 2] = 0
    return
end

function log_likelihood(::Type{LogitNormal}, y::Vector{T}, param::Matrix{T}, n::Int) where T
    loglik = -0.5*log(2*pi)*n
    for t in 1:n
        loglik -= log((y[t] * (1 - y[t])) * sqrt(param[t, 2])) + 0.5*(logit(y[t]) - param[t, 1])^2/param[t, 2]
    end
    return -loglik
end

# Links
function link!(param_tilde::Matrix{T}, ::Type{LogitNormal}, param::Matrix{T}, t::Int) where T
    param_tilde[t, 1] = link(IdentityLink, param[t, 1])
    param_tilde[t, 2] = link(LogLink, param[t, 2], zero(T))
    return
end
function unlink!(param::Matrix{T}, ::Type{LogitNormal}, param_tilde::Matrix{T}, t::Int) where T
    param[t, 1] = unlink(IdentityLink, param_tilde[t, 1])
    param[t, 2] = unlink(LogLink, param_tilde[t, 2], zero(T))
    return
end
function jacobian_link!(aux::AuxiliaryLinAlg{T}, ::Type{LogitNormal}, param::Matrix{T}, t::Int) where T
    aux.jac[1] = jacobian_link(IdentityLink, param[t, 1])
    aux.jac[2] = jacobian_link(LogLink, param[t, 2], zero(T))
    return
end

# utils
function update_dist(::Type{LogitNormal}, param::Matrix{T}, t::Int) where T
    # LogitNormal here is parametrized as sigma^2
    return LogitNormal(param[t, 1], sqrt(param[t, 2]))
end

function params_sdm(d::LogitNormal)
    pars = Distributions.params(d)
    return (pars[1], pars[2]^2)
end

function num_params(::Type{LogitNormal})
    return 2
end


# Algorithm 3.1
# Neville polynomial interpolation.

function neville(x, y, x0)
	n = length(x)
	q = zeros(n, n)
	for i ∈ 1:n
		q[i, 1] = y[i];
	end
	for i ∈ 2:n
		for j ∈ i:n
			q[j,i] = ((x0 - x[j-i+1]) * q[j,i-1] - (x0 - x[j]) * q[j-1,i-1]) / (x[j] - x[j-i+1])
		end
	end
	return q[n,n]
end

println(neville([1, 56, 73], [67, 5, 35], 3))


@doc_str "Calculates x for `0 = a*x^2+b*x+c`, arguments types can be defined
in function definitions"
function quadratic2(a::Float64, b::Float64, c::Float64)
    quadratic(a, sqr_term, b) = (-b + sqr_term) / 2a
    sqr_term = sqrt(b^2-4a*c)
    r1 = quadratic(a, sqr_term, b)
    r2 = quadratic(a, -sqr_term, b)
    r1, r2
end